Upgrading to rs-0.42

.github/workflows/docs.yaml

@@ -14,7 +14,7 @@ jobs:
       - name: Install latest Rust nightly
         uses: dtolnay/rust-toolchain@stable
         with:
-          toolchain: nightly-2024-06-23
+          toolchain: nightly-2024-07-26
           components: rustfmt, clippy
       - name: Install ghp-import
         uses: actions/setup-python@v5
@@ -27,7 +27,7 @@ jobs:
       - name: Set up Node
         uses: actions/setup-node@v4
         with: 
-          node-version: 20
+          node-version: 22
       - name: Build node reference
         run: | 
           npm install

.github/workflows/test-js.yaml

@@ -7,7 +7,7 @@ jobs:
     strategy:
       matrix:
         os: ["ubuntu-latest", "macos-latest", "windows-latest"]
-        node: ["18", "20"]
+        node: ["18", "20", "22"]
     steps:
       - uses: actions/checkout@v4
       - name: Enable Corepack
@@ -20,7 +20,7 @@ jobs:
       - name: Install latest Rust nightly
         uses: dtolnay/rust-toolchain@stable
         with:
-          toolchain: nightly-2024-06-23
+          toolchain: nightly-2024-07-26
           components: rustfmt, clippy
       - name: Check yarn version
         run: yarn --version
@@ -46,7 +46,7 @@ jobs:
       - name: Install latest Rust nightly
         uses: dtolnay/rust-toolchain@stable
         with:
-          toolchain: nightly-2024-06-23
+          toolchain: nightly-2024-07-26
           components: rustfmt, clippy
       - name: Bun version
         uses: oven-sh/setup-bun@v1

.yarnrc.yml

@@ -4,4 +4,4 @@ enableGlobalCache: false
 
 nodeLinker: node-modules
 
-yarnPath: .yarn/releases/yarn-4.3.1.cjs
+yarnPath: .yarn/releases/yarn-4.4.0.cjs

Cargo.toml

@@ -15,14 +15,14 @@ crate-type = ["cdylib", "lib"]
 [dependencies]
 ahash = "0.8.11"
 bincode = "1.3.3"
-napi = { version = "2.16.8", default-features = false, features = [
+napi = { version = "2.16.9", default-features = false, features = [
     "napi8",
     "serde-json",
 ] }
-napi-derive = { version = "2.16.8", default-features = false }
-polars-core = { git = "https://github.com/pola-rs/polars.git", rev = "91a423fea2dc067837db65c3608e3cbc1112a6fc", default-features = false }
-polars-io = { git = "https://github.com/pola-rs/polars.git", rev = "91a423fea2dc067837db65c3608e3cbc1112a6fc", default-features = false }
-polars-lazy = { git = "https://github.com/pola-rs/polars.git", rev = "91a423fea2dc067837db65c3608e3cbc1112a6fc", default-features = false }
+napi-derive = { version = "2.16.11", default-features = false }
+polars-core = { git = "https://github.com/pola-rs/polars.git", rev = "7686025ac7738607f2d4f6887e9a1313b7c8b1e2", default-features = false }
+polars-io = { git = "https://github.com/pola-rs/polars.git", rev = "7686025ac7738607f2d4f6887e9a1313b7c8b1e2", default-features = false }
+polars-lazy = { git = "https://github.com/pola-rs/polars.git", rev = "7686025ac7738607f2d4f6887e9a1313b7c8b1e2", default-features = false }
 thiserror = "1"
 smartstring = { version = "1" }
 serde_json = { version = "1" }
@@ -154,14 +154,10 @@ features = [
   "string_pad",
   "replace",
   "cov",
-  "http",
-  "cloud",
-  "aws",
-  "gcp",
-  "azure"
+  "http"
 ]
 git = "https://github.com/pola-rs/polars.git"
-rev = "91a423fea2dc067837db65c3608e3cbc1112a6fc"
+rev = "7686025ac7738607f2d4f6887e9a1313b7c8b1e2"
 
 [build-dependencies]
 napi-build = "2.1.3"

__tests__/expr.test.ts

@@ -1069,15 +1069,15 @@ describe("expr.str", () => {
   });
   test("jsonDecode", () => {
     const df = pl.DataFrame({
-      json: ['{"a":1, "b": true}', null, '{"a":2, "b": false}'],
+      json: [
+        '{"a":1, "b": true}',
+        '{"a": null, "b": null }',
+        '{"a":2, "b": false}',
+      ],
     });
     const actual = df.select(pl.col("json").str.jsonDecode());
     const expected = pl.DataFrame({
-      json: [
-        { a: 1, b: true },
-        { a: null, b: null },
-        { a: 2, b: false },
-      ],
+      json: [{ a: 1, b: true }, "null", { a: 2, b: false }],
     });
     expect(actual).toFrameEqual(expected);
     let s = pl.Series(["[1, 2, 3]", null, "[4, 5, 6]"]);
@@ -1089,7 +1089,11 @@ describe("expr.str", () => {
       new pl.Field("a", pl.Int64),
       new pl.Field("b", pl.Bool),
     ]);
-    s = pl.Series("json", ['{"a":1, "b": true}', null, '{"a":2, "b": false}']);
+    s = pl.Series("json", [
+      '{"a":1, "b": true}',
+      '{"a": null, "b": null }',
+      '{"a":2, "b": false}',
+    ]);
     expect(s.str.jsonDecode().as("json")).toSeriesEqual(
       expected.getColumn("json"),
     );

__tests__/lazyframe.test.ts

@@ -1260,7 +1260,11 @@ describe("lazyframe", () => {
     });
     const actual = pl
       .DataFrame({
-        json: ['{"a": 1, "b": true}', null, '{"a": 2, "b": false}'],
+        json: [
+          '{"a": 1, "b": true}',
+          '{"a": null, "b": null }',
+          '{"a": 2, "b": false}',
+        ],
       })
       .lazy()
       .select(pl.col("json").str.jsonDecode())

package.json

@@ -58,16 +58,16 @@
     "@napi-rs/cli": "^2.18.4",
     "@types/chance": "^1.1.6",
     "@types/jest": "^29.5.12",
-    "@types/node": "^20.14.10",
+    "@types/node": "^22.3.0",
     "chance": "^1.1.12",
     "jest": "^29.7.0",
     "source-map-support": "^0.5.21",
-    "ts-jest": "^29.2.0",
+    "ts-jest": "^29.2.4",
     "ts-node": "^10.9.2",
-    "typedoc": "^0.26.3",
-    "typescript": "5.5.3"
+    "typedoc": "^0.26.5",
+    "typescript": "5.5.4"
   },
-  "packageManager": "yarn@4.3.1",
+  "packageManager": "yarn@4.4.0",
   "workspaces": [
     "benches"
   ]

rust-toolchain

@@ -1 +1 @@
-nightly-2024-06-23
+nightly-2024-07-26

src/conversion.rs

@@ -78,7 +78,7 @@ impl ToNapiValue for Wrap<&Series> {
         match dtype {
             DataType::Struct(_) => {
                 let ca = s.struct_().map_err(JsPolarsErr::from)?;
-                let df: DataFrame = ca.clone().into();
+                let df: DataFrame = ca.clone().unnest();
 
                 let (height, _) = df.shape();
                 let mut rows = env.create_array(height as u32)?;
@@ -760,6 +760,7 @@ impl ToNapiValue for Wrap<ParallelStrategy> {
             ParallelStrategy::Columns => "columns",
             ParallelStrategy::RowGroups => "row_groups",
             ParallelStrategy::None => "none",
+            ParallelStrategy::Prefiltered => "prefiltered",
         };
         let _ = strategy.set("strategy", unit);
         Object::to_napi_value(napi_env, strategy)
@@ -1248,6 +1249,14 @@ where
     container.into_iter().map(|s| s.as_ref().into()).collect()
 }
 
+pub(crate) fn strings_to_selector<I, S>(container: I) -> Vec<Selector>
+where
+    I: IntoIterator<Item = S>,
+    S: AsRef<str>,
+{
+    container.into_iter().map(|s| s.as_ref().into()).collect()
+}
+
 pub(crate) fn parse_parquet_compression(
     compression: String,
     compression_level: Option<i32>,

src/dataframe.rs

@@ -285,7 +285,7 @@ pub fn read_parquet(
                 .with_projection(projection)
                 .with_columns(columns)
                 .read_parallel(parallel.0)
-                .with_n_rows(n_rows)
+                .with_slice(n_rows.map(|x| (0, x)))
                 .with_row_index(row_count)
                 .finish()
         }
@@ -295,7 +295,7 @@ pub fn read_parquet(
                 .with_projection(projection)
                 .with_columns(columns)
                 .read_parallel(parallel.0)
-                .with_n_rows(n_rows)
+                .with_slice(n_rows.map(|x| (0, x)))
                 .with_row_index(row_count)
                 .finish()
         }
@@ -927,16 +927,14 @@ impl JsDataFrame {
         &self,
         id_vars: Vec<String>,
         value_vars: Vec<String>,
-        value_name: Option<String>,
         variable_name: Option<String>,
-        streamable: Option<bool>,
+        value_name: Option<String>
     ) -> napi::Result<JsDataFrame> {
-        let args = UnpivotArgs {
-            index: strings_to_smartstrings(id_vars),
+        let args = UnpivotArgsIR {
             on: strings_to_smartstrings(value_vars),
-            value_name: value_name.map(|s| s.into()),
+            index: strings_to_smartstrings(id_vars),
             variable_name: variable_name.map(|s| s.into()),
-            streamable: streamable.unwrap_or(false),
+            value_name: value_name.map(|s| s.into())
         };
 
         let df = self.df.unpivot2(args).map_err(JsPolarsErr::from)?;
@@ -1049,7 +1047,7 @@ impl JsDataFrame {
         k2: Wrap<u64>,
         k3: Wrap<u64>,
     ) -> napi::Result<JsSeries> {
-        let hb = polars::export::ahash::RandomState::with_seeds(k0.0, k1.0, k2.0, k3.0);
+        let hb = PlRandomState::with_seeds(k0.0, k1.0, k2.0, k3.0);
         let hash = self.df.hash_rows(Some(hb)).map_err(JsPolarsErr::from)?;
         Ok(hash.into_series().into())
     }

src/functions.rs

@@ -9,7 +9,7 @@ use polars_core::functions as pl_functions;
 #[napi(catch_unwind)]
 pub fn horizontal_concat(dfs: Vec<&JsDataFrame>) -> napi::Result<JsDataFrame> {
     let dfs: Vec<DataFrame> = dfs.iter().map(|df| df.df.clone()).collect();
-    let df = pl_functions::concat_df_horizontal(&dfs).map_err(crate::error::JsPolarsErr::from)?;
+    let df = pl_functions::concat_df_horizontal(&dfs, true).map_err(crate::error::JsPolarsErr::from)?;
     Ok(df.into())
 }
 

src/lazy/dataframe.rs

@@ -504,16 +504,14 @@ impl JsLazyFrame {
         &self,
         id_vars: Vec<&str>,
         value_vars: Vec<&str>,
-        value_name: Option<&str>,
         variable_name: Option<&str>,
-        streamable: Option<bool>,
+        value_name: Option<&str>,
     ) -> JsLazyFrame {
-        let args = UnpivotArgs {
-            index: strings_to_smartstrings(id_vars),
-            on: strings_to_smartstrings(value_vars),
-            value_name: value_name.map(|s| s.into()),
+        let args = UnpivotArgsDSL {
+            on: strings_to_selector(value_vars),
+            index: strings_to_selector(id_vars),
             variable_name: variable_name.map(|s| s.into()),
-            streamable: streamable.unwrap_or(false),
+            value_name: value_name.map(|s| s.into())
         };
         let ldf = self.ldf.clone();
         ldf.unpivot(args).into()
@@ -606,14 +604,14 @@ impl JsLazyFrame {
             StatisticsOptions::empty()
         };
         let row_group_size = options.row_group_size.map(|i| i as usize);
-        let data_pagesize_limit = options.data_pagesize_limit.map(|i| i as usize);
+        let data_page_size = options.data_pagesize_limit.map(|i| i as usize);
         let maintain_order = options.maintain_order.unwrap_or(true);
 
         let options = ParquetWriteOptions {
             compression,
             statistics,
             row_group_size,
-            data_pagesize_limit,
+            data_page_size,
             maintain_order,
         };
 
@@ -768,6 +766,7 @@ pub fn scan_parquet(path: String, options: ScanParquetOptions) -> napi::Result<J
             ..Default::default()
         },
         glob: true,
+        include_file_paths: None
     };
     let lf = LazyFrame::scan_parquet(path, args).map_err(JsPolarsErr::from)?;
     Ok(lf.into())
@@ -796,6 +795,8 @@ pub fn scan_ipc(path: String, options: ScanIPCOptions) -> napi::Result<JsLazyFra
         row_index,
         memory_map,
         cloud_options: Default::default(),
+        hive_options: Default::default(),
+        include_file_paths: None,
     };
     let lf = LazyFrame::scan_ipc(path, args).map_err(JsPolarsErr::from)?;
     Ok(lf.into())

src/lazy/dsl.rs

@@ -1109,7 +1109,7 @@ impl JsExpr {
     #[napi(catch_unwind)]
     pub fn hash(&self, k0: Wrap<u64>, k1: Wrap<u64>, k2: Wrap<u64>, k3: Wrap<u64>) -> JsExpr {
         let function = move |s: Series| {
-            let hb = polars::export::ahash::RandomState::with_seeds(k0.0, k1.0, k2.0, k3.0);
+            let hb = PlRandomState::with_seeds(k0.0, k1.0, k2.0, k3.0);
             Ok(Some(s.hash(hb).into_series()))
         };
         self.clone()

src/series.rs

@@ -599,7 +599,7 @@ impl JsSeries {
 
     #[napi(catch_unwind)]
     pub fn has_validity(&self) -> bool {
-        self.series.has_validity()
+        self.series.has_nulls()
     }
 
     #[napi(catch_unwind)]
@@ -868,14 +868,14 @@ impl JsSeries {
     #[napi(catch_unwind)]
     pub fn struct_to_frame(&self) -> napi::Result<crate::dataframe::JsDataFrame> {
         let ca = self.series.struct_().map_err(JsPolarsErr::from)?;
-        let df: DataFrame = ca.clone().into();
+        let df: DataFrame = ca.clone().unnest();
         Ok(df.into())
     }
 
     #[napi(catch_unwind)]
     pub fn struct_fields(&self) -> napi::Result<Vec<&str>> {
         let ca = self.series.struct_().map_err(JsPolarsErr::from)?;
-        Ok(ca.fields().iter().map(|s| s.name()).collect())
+        Ok(ca.struct_fields().iter().map(|s| s.name().as_str()).collect())
     }
     // String Namespace
 
@@ -1141,7 +1141,7 @@ impl JsSeries {
 
     #[napi(catch_unwind)]
     pub fn hash(&self, k0: Wrap<u64>, k1: Wrap<u64>, k2: Wrap<u64>, k3: Wrap<u64>) -> JsSeries {
-        let hb = polars::export::ahash::RandomState::with_seeds(k0.0, k1.0, k2.0, k3.0);
+        let hb = PlRandomState::with_seeds(k0.0, k1.0, k2.0, k3.0);
         self.series.hash(hb).into_series().into()
     }
     #[napi(catch_unwind)]