diff --git a/package-lock.json b/package-lock.json index 874331b8..79e0a3a9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,7 +14,7 @@ "@geoarrow/deck.gl-layers": "^0.3.0-beta.16", "apache-arrow": "^15.0.2", "maplibre-gl": "^3.6.2", - "parquet-wasm": "0.5.0", + "parquet-wasm": "0.6.0", "react": "^18.2.0", "react-dom": "^18.2.0", "react-map-gl": "^7.1.7", @@ -3273,9 +3273,9 @@ } }, "node_modules/parquet-wasm": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/parquet-wasm/-/parquet-wasm-0.5.0.tgz", - "integrity": "sha512-XN3EW+3xf915QgpxCvfCdVYsxDLXz+BckKLDlMo41cbipSYD8x+F/3lGcrWtxdY6uJgAmtb+SX3tS+9PPsplQA==" + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/parquet-wasm/-/parquet-wasm-0.6.0.tgz", + "integrity": "sha512-2qIG14DOOGdvsdmZ8WNgBc8pAhSM2Cv/ArxkmLLt1G3Z9+EAw27+SSRAUz/bET+6gD6mm7CM4kN1eKEAnTCwCA==" }, "node_modules/path-browserify": { "version": "1.0.1", diff --git a/package.json b/package.json index dd44bf68..dc1ad742 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,7 @@ "@geoarrow/deck.gl-layers": "^0.3.0-beta.16", "apache-arrow": "^15.0.2", "maplibre-gl": "^3.6.2", - "parquet-wasm": "0.5.0", + "parquet-wasm": "0.6.0", "react": "^18.2.0", "react-dom": "^18.2.0", "react-map-gl": "^7.1.7", diff --git a/src/parquet.ts b/src/parquet.ts index 719a005d..e57baff0 100644 --- a/src/parquet.ts +++ b/src/parquet.ts @@ -1,11 +1,10 @@ -import { useEffect, useState } from "react"; -import _initParquetWasm, { readParquet } from "parquet-wasm/esm/arrow2"; +import _initParquetWasm, { readParquet } from "parquet-wasm"; import * as arrow from "apache-arrow"; // NOTE: this version must be synced exactly with the parquet-wasm version in // use. -const PARQUET_WASM_VERSION = "0.5.0"; -const PARQUET_WASM_CDN_URL = `https://cdn.jsdelivr.net/npm/parquet-wasm@${PARQUET_WASM_VERSION}/esm/arrow2_bg.wasm`; +const PARQUET_WASM_VERSION = "0.6.0"; +const PARQUET_WASM_CDN_URL = `https://cdn.jsdelivr.net/npm/parquet-wasm@${PARQUET_WASM_VERSION}/esm/parquet_wasm_bg.wasm`; let WASM_READY: boolean = false; export async function initParquetWasm() { @@ -27,10 +26,10 @@ export function parseParquet(dataView: DataView): arrow.Table { console.time("readParquet"); - // TODO: use arrow-js-ffi for more memory-efficient wasm --> js transfer - const arrowIPCBuffer = readParquet( - new Uint8Array(dataView.buffer), - ).intoIPCStream(); + // TODO: use arrow-js-ffi for more memory-efficient wasm --> js transfer? + const arrowIPCBuffer = readParquet(new Uint8Array(dataView.buffer), { + batchSize: Math.pow(2, 31), + }).intoIPCStream(); const arrowTable = arrow.tableFromIPC(arrowIPCBuffer); console.timeEnd("readParquet");