-
Notifications
You must be signed in to change notification settings - Fork 33
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
react helpers for parquet and accessors
- Loading branch information
1 parent
a72c935
commit 20df93a
Showing
5 changed files
with
175 additions
and
124 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,64 @@ | ||
| import * as arrow from "apache-arrow"; | ||
| import { parseParquet } from "./parquet"; | ||
| import { useState, useEffect } from "react"; | ||
|
|
||
| // /** | ||
| // * @template T | ||
| // * | ||
| // * @param {string} key | ||
| // * @returns {[T, (value: T) => void]} | ||
| // */ | ||
| // export function useModelState(key) { | ||
| // let model = useModel(); | ||
| // let [value, setValue] = React.useState(model.get(key)); | ||
| // React.useEffect(() => { | ||
| // let callback = () => setValue(model.get(key)); | ||
| // model.on(`change:${key}`, callback); | ||
| // return () => model.off(`change:${key}`, callback); | ||
| // }, [model, key]); | ||
| // return [ | ||
| // value, | ||
| // (value) => { | ||
| // model.set(key, value); | ||
| // model.save_changes(); | ||
| // }, | ||
| // ]; | ||
| // } | ||
|
|
||
| export function useTableBufferState( | ||
| wasmReady: boolean, | ||
| dataRaw: DataView | ||
| ): [arrow.Table | null] { | ||
| const [dataTable, setDataTable] = useState<arrow.Table | null>(null); | ||
| // Only parse the parquet buffer when the data itself or wasmReady has changed | ||
| useEffect(() => { | ||
| const callback = () => { | ||
| if (wasmReady && dataRaw && dataRaw.byteLength > 0) { | ||
| setDataTable(parseParquet(dataRaw)); | ||
| } | ||
| }; | ||
| callback(); | ||
| }, [wasmReady, dataRaw]); | ||
|
|
||
| return [dataTable]; | ||
| } | ||
|
|
||
| export function useAccessorState(wasmReady: boolean, accessorRaw: any): any { | ||
| const [accessorValue, setAccessorValue] = useState(null); | ||
|
|
||
| // Only parse the parquet buffer when the data itself or wasmReady has changed | ||
| useEffect(() => { | ||
| const callback = () => { | ||
| setAccessorValue( | ||
| accessorRaw instanceof DataView | ||
| ? wasmReady && accessorRaw.byteLength > 0 | ||
| ? parseParquet(accessorRaw).getChildAt(0) | ||
| : null | ||
| : accessorRaw | ||
| ); | ||
| }; | ||
| callback(); | ||
| }, [wasmReady, accessorRaw]); | ||
|
|
||
| return [accessorValue]; | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,89 @@ | ||
| import { useEffect, useState } from "react"; | ||
| import _initParquetWasm, { readParquet } from "parquet-wasm/esm/arrow2"; | ||
| import * as arrow from "apache-arrow"; | ||
|
|
||
| // NOTE: this version must be synced exactly with the parquet-wasm version in | ||
| // use. | ||
| const PARQUET_WASM_VERSION = "0.5.0-alpha.1"; | ||
| const PARQUET_WASM_CDN_URL = `https://cdn.jsdelivr.net/npm/parquet-wasm@${PARQUET_WASM_VERSION}/esm/arrow2_bg.wasm`; | ||
| let WASM_READY: boolean = false; | ||
|
|
||
| export async function initParquetWasm() { | ||
| if (WASM_READY) { | ||
| return; | ||
| } | ||
|
|
||
| await _initParquetWasm(PARQUET_WASM_CDN_URL); | ||
| WASM_READY = true; | ||
| } | ||
|
|
||
| /** | ||
| * Parse a Parquet buffer to an Arrow JS table | ||
| */ | ||
| export function parseParquet(dataView: DataView): arrow.Table { | ||
| if (!WASM_READY) { | ||
| throw new Error("wasm not ready"); | ||
| } | ||
|
|
||
| console.time("readParquet"); | ||
|
|
||
| // TODO: use arrow-js-ffi for more memory-efficient wasm --> js transfer | ||
| const arrowIPCBuffer = readParquet(new Uint8Array(dataView.buffer)).intoIPC(); | ||
| const arrowTable = arrow.tableFromIPC(arrowIPCBuffer); | ||
|
|
||
| console.timeEnd("readParquet"); | ||
|
|
||
| return arrowTable; | ||
| } | ||
|
|
||
| export function useParquetWasm(): [boolean] { | ||
| const [wasmReady, setWasmReady] = useState<boolean>(false); | ||
|
|
||
| // Init parquet wasm | ||
| useEffect(() => { | ||
| const callback = async () => { | ||
| await initParquetWasm(); | ||
| setWasmReady(true); | ||
| }; | ||
|
|
||
| callback(); | ||
| }, []); | ||
|
|
||
| return [wasmReady]; | ||
| } | ||
|
|
||
| // // NOTE: this was an attempt to only parse Parquet for the initial data and | ||
| // // whenever the data buffer changed. But I had issues where the wasm wasn't | ||
| // // ready yet when the original data needed to be instantiated | ||
| // // | ||
| // // NOTE2: I worked around this by adding a useEffect in the main App().. so this | ||
| // // function can probably be deleted | ||
| // function useModelParquetState( | ||
| // key: string, | ||
| // ...deps | ||
| // ): [arrow.Table | undefined, (value: DataView) => void] { | ||
| // let model = useModel(); | ||
|
|
||
| // console.log("WASM_READY", WASM_READY); | ||
| // let [table, setTable] = useState<arrow.Table | undefined>( | ||
| // WASM_READY ? parseParquet(model.get(key)) : undefined | ||
| // ); | ||
| // console.log(deps); | ||
| // useEffect(() => { | ||
| // let parquetCallback = () => { | ||
| // console.log("inside parquetCallback"); | ||
| // setTable(WASM_READY ? parseParquet(model.get(key)) : undefined); | ||
| // }; | ||
| // model.on(`change:${key}`, parquetCallback); | ||
| // return () => model.off(`change:${key}`, parquetCallback); | ||
| // }, [model, key, deps]); | ||
|
|
||
| // console.log("useModelParquetState table", table); | ||
| // return [ | ||
| // table, | ||
| // (value) => { | ||
| // model.set(key, value); | ||
| // model.save_changes(); | ||
| // }, | ||
| // ]; | ||
| // } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters