refactor: Split py-polars crate (#18204)

Cargo.toml

@@ -55,6 +55,7 @@ hex = "0.4.3"
 indexmap = { version = "2", features = ["std"] }
 itoa = "1.0.6"
 itoap = { version = "1", features = ["simd"] }
+libc = "0.2"
 memchr = "2.6"
 memmap = { package = "memmap2", version = "0.7" }
 multiversion = "0.7"
@@ -109,6 +110,7 @@ polars-ops = { version = "0.42.0", path = "crates/polars-ops", default-features
 polars-parquet = { version = "0.42.0", path = "crates/polars-parquet", default-features = false }
 polars-pipe = { version = "0.42.0", path = "crates/polars-pipe", default-features = false }
 polars-plan = { version = "0.42.0", path = "crates/polars-plan", default-features = false }
+polars-python = { version = "0.42.0", path = "crates/polars-python", default-features = false }
 polars-row = { version = "0.42.0", path = "crates/polars-row", default-features = false }
 polars-sql = { version = "0.42.0", path = "crates/polars-sql", default-features = false }
 polars-stream = { version = "0.42.0", path = "crates/polars-stream", default-features = false }

crates/polars-plan/README.md

@@ -1,4 +1,4 @@
-# polars-plan-
+# polars-plan
 
 `polars-plan` is an **internal sub-crate** of the [Polars](https://crates.io/crates/polars) library, that provides source code responsible for Polars logical planning.
 

crates/polars-python/Cargo.toml

@@ -0,0 +1,254 @@
+[package]
+name = "polars-python"
+version = { workspace = true }
+authors = { workspace = true }
+edition = { workspace = true }
+homepage = { workspace = true }
+license = { workspace = true }
+repository = { workspace = true }
+description = "Enable running Polars workloads in Python"
+
+[dependencies]
+polars-core = { workspace = true, features = ["python"] }
+polars-error = { workspace = true }
+polars-io = { workspace = true }
+polars-lazy = { workspace = true, features = ["python"] }
+polars-ops = { workspace = true }
+polars-parquet = { workspace = true, optional = true }
+polars-plan = { workspace = true }
+polars-time = { workspace = true }
+polars-utils = { workspace = true }
+
+# TODO! remove this once truly activated. This is required to make sdist building work
+polars-stream = { workspace = true }
+
+ahash = { workspace = true }
+arboard = { workspace = true, optional = true }
+bytemuck = { workspace = true }
+ciborium = { workspace = true }
+either = { workspace = true }
+itoa = { workspace = true }
+libc = { workspace = true }
+ndarray = { workspace = true }
+num-traits = { workspace = true }
+# TODO: Pin to released version once NumPy 2.0 support is merged
+# https://github.com/PyO3/rust-numpy/issues/409
+numpy = { git = "https://github.com/stinodego/rust-numpy.git", rev = "9ba9962ae57ba26e35babdce6f179edf5fe5b9c8", default-features = false }
+once_cell = { workspace = true }
+pyo3 = { workspace = true, features = ["abi3-py38", "chrono", "extension-module", "multiple-pymethods"] }
+recursive = { workspace = true }
+serde_json = { workspace = true, optional = true }
+smartstring = { workspace = true }
+thiserror = { workspace = true }
+
+[dependencies.polars]
+workspace = true
+features = [
+  "abs",
+  "approx_unique",
+  "array_any_all",
+  "arg_where",
+  "business",
+  "concat_str",
+  "cum_agg",
+  "cumulative_eval",
+  "dataframe_arithmetic",
+  "month_start",
+  "month_end",
+  "offset_by",
+  "diagonal_concat",
+  "diff",
+  "dot_diagram",
+  "dot_product",
+  "dtype-categorical",
+  "dtype-full",
+  "dynamic_group_by",
+  "ewma",
+  "ewma_by",
+  "fmt",
+  "fused",
+  "interpolate",
+  "interpolate_by",
+  "is_first_distinct",
+  "is_last_distinct",
+  "is_unique",
+  "is_between",
+  "lazy",
+  "list_eval",
+  "list_to_struct",
+  "array_to_struct",
+  "log",
+  "mode",
+  "moment",
+  "ndarray",
+  "partition_by",
+  "product",
+  "random",
+  "range",
+  "rank",
+  "reinterpret",
+  "replace",
+  "rolling_window",
+  "rolling_window_by",
+  "round_series",
+  "row_hash",
+  "rows",
+  "semi_anti_join",
+  "serde-lazy",
+  "string_encoding",
+  "string_reverse",
+  "string_to_integer",
+  "string_pad",
+  "strings",
+  "temporal",
+  "to_dummies",
+  "true_div",
+  "unique_counts",
+  "zip_with",
+  "cov",
+]
+
+[build-dependencies]
+version_check = { workspace = true }
+
+[features]
+# Features below are only there to enable building a slim binary during development.
+avro = ["polars/avro"]
+parquet = ["polars/parquet", "polars-parquet"]
+ipc = ["polars/ipc"]
+ipc_streaming = ["polars/ipc_streaming"]
+is_in = ["polars/is_in"]
+json = ["polars/serde", "serde_json", "polars/json"]
+trigonometry = ["polars/trigonometry"]
+sign = ["polars/sign"]
+asof_join = ["polars/asof_join"]
+cross_join = ["polars/cross_join"]
+pct_change = ["polars/pct_change"]
+repeat_by = ["polars/repeat_by"]
+
+streaming = ["polars/streaming"]
+meta = ["polars/meta"]
+search_sorted = ["polars/search_sorted"]
+decompress = ["polars/decompress-fast"]
+regex = ["polars/regex"]
+csv = ["polars/csv"]
+clipboard = ["arboard"]
+extract_jsonpath = ["polars/extract_jsonpath"]
+pivot = ["polars/pivot"]
+top_k = ["polars/top_k"]
+propagate_nans = ["polars/propagate_nans"]
+sql = ["polars/sql"]
+performant = ["polars/performant"]
+timezones = ["polars/timezones"]
+cse = ["polars/cse"]
+merge_sorted = ["polars/merge_sorted"]
+list_gather = ["polars/list_gather"]
+list_count = ["polars/list_count"]
+array_count = ["polars/array_count", "polars/dtype-array"]
+binary_encoding = ["polars/binary_encoding"]
+list_sets = ["polars-lazy/list_sets"]
+list_any_all = ["polars/list_any_all"]
+array_any_all = ["polars/array_any_all", "polars/dtype-array"]
+list_drop_nulls = ["polars/list_drop_nulls"]
+list_sample = ["polars/list_sample"]
+cutqcut = ["polars/cutqcut"]
+rle = ["polars/rle"]
+extract_groups = ["polars/extract_groups"]
+ffi_plugin = ["polars-plan/ffi_plugin"]
+cloud = ["polars/cloud", "polars/aws", "polars/gcp", "polars/azure", "polars/http"]
+peaks = ["polars/peaks"]
+hist = ["polars/hist"]
+find_many = ["polars/find_many"]
+new_streaming = ["polars-lazy/new_streaming"]
+
+dtype-i8 = []
+dtype-i16 = []
+dtype-u8 = []
+dtype-u16 = []
+dtype-array = []
+object = ["polars/object"]
+
+dtypes = [
+  "dtype-array",
+  "dtype-i16",
+  "dtype-i8",
+  "dtype-u16",
+  "dtype-u8",
+  "object",
+]
+
+operations = [
+  "array_any_all",
+  "array_count",
+  "is_in",
+  "repeat_by",
+  "trigonometry",
+  "sign",
+  "performant",
+  "list_gather",
+  "list_count",
+  "list_sets",
+  "list_any_all",
+  "list_drop_nulls",
+  "list_sample",
+  "cutqcut",
+  "rle",
+  "extract_groups",
+  "pivot",
+  "extract_jsonpath",
+  "asof_join",
+  "cross_join",
+  "pct_change",
+  "search_sorted",
+  "merge_sorted",
+  "top_k",
+  "propagate_nans",
+  "timezones",
+  "peaks",
+  "hist",
+  "find_many",
+]
+
+io = [
+  "json",
+  "parquet",
+  "ipc",
+  "ipc_streaming",
+  "avro",
+  "csv",
+  "cloud",
+  "clipboard",
+]
+
+optimizations = [
+  "cse",
+  "polars/fused",
+  "streaming",
+]
+
+polars_cloud = ["polars/polars_cloud"]
+# also includes simd
+nightly = ["polars/nightly"]
+
+all = [
+  "optimizations",
+  "io",
+  "operations",
+  "dtypes",
+  "meta",
+  "decompress",
+  "regex",
+  "sql",
+  "binary_encoding",
+  "ffi_plugin",
+  "polars_cloud",
+  # "new_streaming",
+]
+
+# we cannot conditionally activate simd
+# https://github.com/rust-lang/cargo/issues/1197
+# so we have an indirection and compile
+# with --no-default-features --features=all for targets without simd
+default = [
+  "all",
+]

crates/polars-python/LICENSE

@@ -0,0 +1 @@
+../../LICENSE

crates/polars-python/README.md

@@ -0,0 +1,6 @@
+# polars-python
+
+`polars-python` is an **internal sub-crate** of the [Polars](https://crates.io/crates/polars) library.
+It enables running Polars workloads in Python.
+
+**Important Note**: This crate is **not intended for external usage**. Please refer to the main [Polars crate](https://crates.io/crates/polars) for intended usage.

crates/polars-python/build.rs

@@ -0,0 +1,7 @@
+fn main() {
+    println!("cargo:rerun-if-changed=build.rs");
+    let channel = version_check::Channel::read().unwrap();
+    if channel.is_nightly() {
+        println!("cargo:rustc-cfg=feature=\"nightly\"");
+    }
+}

py-polars/src/dataframe/general.rs → crates/polars-python/src/dataframe/general.rs

@@ -368,6 +368,7 @@ impl PyDataFrame {
         Ok(df.into())
     }
 
+    #[allow(clippy::should_implement_trait)]
     pub fn clone(&self) -> Self {
         PyDataFrame::new(self.df.clone())
     }

py-polars/src/gil_once_cell.rs → crates/polars-python/src/gil_once_cell.rs

@@ -14,6 +14,7 @@ unsafe impl<T: Send> Send for GILOnceCell<T> {}
 
 impl<T> GILOnceCell<T> {
     /// Create a `GILOnceCell` which does not yet contain a value.
+    #[allow(clippy::new_without_default)]
     pub const fn new() -> Self {
         Self(UnsafeCell::new(None))
     }