diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 54c5fda8bf..2cf6cf0b09 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -168,8 +168,6 @@ jobs: packages/perspective-viewer-d3fc/dist packages/perspective-viewer-datagrid/dist packages/perspective-viewer-openlayers/dist - packages/perspective-esbuild-plugin/dist - packages/perspective-webpack-plugin/dist packages/perspective-cli/dist packages/perspective-workspace/dist @@ -947,12 +945,6 @@ jobs: - run: pnpm pack --pack-destination=../.. working-directory: ./packages/perspective-cli - - run: pnpm pack --pack-destination=../.. - working-directory: ./packages/perspective-webpack-plugin - - - run: pnpm pack --pack-destination=../.. - working-directory: ./packages/perspective-esbuild-plugin - - run: pnpm pack --pack-destination=../.. working-directory: ./packages/perspective-jupyterlab diff --git a/.gitignore b/.gitignore index 93974bab79..55c2cb6cdf 100644 --- a/.gitignore +++ b/.gitignore @@ -112,7 +112,7 @@ packages/*/cjs cppbuild docsbuild tools/perspective-build/lib -packages/perspective-esbuild-plugin/lib +tools/perspective-esbuild-plugin/lib rust/perspective-python/perspective/nbextension/static rust/perspective-python/perspective/labextension boost_*.tar.gz @@ -215,7 +215,6 @@ results.debug.json tools/perspective-build/lib docs/.docusaurus -packages/perspective-esbuild-plugin/lib docs/static/blocks test-results/ playwright-report/ @@ -252,3 +251,4 @@ rust/perspective-python/LICENSE_* rust/perspective-viewer/docs/exprtk.md rust/perspective-server/docs/lib_gen.md rust/perspective-client/docs/expression_gen.md +docs/book diff --git a/docs/book.toml b/docs/book.toml new file mode 100644 index 0000000000..9d226dea7e --- /dev/null +++ b/docs/book.toml @@ -0,0 +1,39 @@ +# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +# ┃ Copyright (c) 2017, the Perspective Authors. ┃ +# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +# ┃ This file is part of the Perspective library, distributed under the terms ┃ +# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +[book] +authors = ["Andrew Stein"] +language = "en" +multilingual = false +src = "md" +title = "Perspective" + + +[output.html] +# theme = "my-theme" +# default-theme = "light" +# preferred-dark-theme = "navy" +# smart-punctuation = true +# mathjax-support = false +# copy-fonts = true +additional-css = [ + "md/perspective.css", + "node_modules/@finos/perspective-viewer/dist/css/themes.css", +] +# additional-js = [] +# no-section-label = false +# git-repository-url = "https://github.com/rust-lang/mdBook" +# git-repository-icon = "fa-github" +# edit-url-template = "https://github.com/rust-lang/mdBook/edit/master/guide/{path}" +# site-url = "/example-book/" +# cname = "myproject.rs" +# input-404 = "not-found.md" diff --git a/docs/docusaurus.config.js b/docs/docusaurus.config.js index 5547069fbe..95df299b5f 100644 --- a/docs/docusaurus.config.js +++ b/docs/docusaurus.config.js @@ -69,7 +69,7 @@ const config = { defaultLocale: "en", locales: ["en"], }, - plugins: ["./plugins/perspective-loader"], + // plugins: ["./plugins/perspective-loader"], presets: [ [ "classic", diff --git a/docs/md/SUMMARY.md b/docs/md/SUMMARY.md new file mode 100644 index 0000000000..c9b1d3ee1f --- /dev/null +++ b/docs/md/SUMMARY.md @@ -0,0 +1,44 @@ +# Summary + +# Overview + +- [`Table`](./explanation/table.md) + - [Schema and column types](./explanation/table/schema.md) + - [Loading data](./explanation/table/loading_data.md) + - [`index` and `limit` options](./explanation/table/options.md) + - [`update()` and `remove()` streaming methods](./explanation/table/update_and_remove.md) + - [`clear()` and `replace()` start-over methods](./explanation/table/clear_and_replace.md) +- [JavaScript](./explanation/javascript.md) + - [Module Structure](./explanation/javascript_module_structure.md) + - [Build options](./explanation/javascript_builds.md) +- [Python](./explanation/python.md) + +# Getting Started + +- [Construct a `Table`](./how_to/table.md) +- [Rust](./how_to/rust.md) +- [JavaScript](./how_to/javascript.md) + - [Installation via NPM](./how_to/javascript/installation.md) + - [Importing with a bundler or without a bundler](./how_to/javascript/importing.md) + - [`perspective` data engine library](./how_to/javascript/worker.md) + - [Serializing data](./how_to/javascript/serializing.md) + - [Cleaning up](./how_to/javascript/deleting.md) + - [Hosting a `WebSocketServer` in Node.js](./how_to/javascript/nodejs_server.md) + - [`perspective-viewer` Custom Element library](./how_to/javascript/viewer.md) + - [Theming](./how_to/javascript/theming.md) + - [Loading data from a `Table`](./how_to/javascript/loading_data.md) + - [Loading data from a virtual `Table`](./how_to/javascript/loading_virtual_data.md) + - [Saving and restoring UI state](./how_to/javascript/save_restore.md) + - [Listening for events](./how_to/javascript/events.md) +- [Python](./how_to/python.md) + - [Installation](./how_to/python/installation.md) + - [Loading data into a `Table`](./how_to/python/table.md) + - [Callbacks and events](./how_to/python/callbacks.md) + - [Multithreading](./how_to/python/multithreading.md) + - [Hosting a WebSocket server](./how_to/python/websocket.md) + - [`PerspectiveWidget` for JupyterLab](./how_to/python/jupyterlab.md) + - [Tutorial: A `tornado` server with virtual `perspective-viewer`](./tutorials/python/tornado.md) + +# API + +- [Crate documentation on `docs.rs` ](./api_reference.md) diff --git a/docs/md/api_reference.md b/docs/md/api_reference.md new file mode 100644 index 0000000000..c5836cf30b --- /dev/null +++ b/docs/md/api_reference.md @@ -0,0 +1,9 @@ +# API Reference + +Perspective's complete API is hosted on `docs.rs`: + +- [`perspective-client`](https://docs.rs/perspective-client/latest/perspective_client/index.html) + covers `Table` and `View` data engine API methods common for Rust, + JavaScript and Python. +- [`perspective-rs`](https://docs.rs/perspective-client/latest/perspective_client/index.html) + adds Rust-specific documentation for the Rust crate entrypoint. diff --git a/docs/md/explanation.md b/docs/md/explanation.md new file mode 100644 index 0000000000..b1303b40c8 --- /dev/null +++ b/docs/md/explanation.md @@ -0,0 +1 @@ +# Explanation diff --git a/docs/md/explanation/javascript.md b/docs/md/explanation/javascript.md new file mode 100644 index 0000000000..ffea687dde --- /dev/null +++ b/docs/md/explanation/javascript.md @@ -0,0 +1,15 @@ +Perspective's JavaScript library offers a configurable UI powered by the same +fast streaming data engine, just re-compiled to WebAssembly. A simple example +which loads an [Apache Arrow](https://arrow.apache.org/) and computes a "Group +By" operation, returning a new Arrow: + +```javascript +import perspective from "@finos/perspective"; + +const table = await perspective.table(apache_arrow_data); +const view = await table.view({ group_by: ["CounterParty", "Security"] }); +const arrow = await view.to_arrow(); +``` + +[More Examples](https://github.com/finos/perspective/tree/master/examples) are +available on GitHub. diff --git a/docs/md/explanation/javascript_builds.md b/docs/md/explanation/javascript_builds.md new file mode 100644 index 0000000000..26efe0ae4c --- /dev/null +++ b/docs/md/explanation/javascript_builds.md @@ -0,0 +1,42 @@ +# JavaScript Builds + +Perspective requires the browser to have access to Perspective's `.wasm` +binaries _in addition_ to the bundled `.js` files, and as a result the build +process requires a few extra steps. To ease integration, Perspective's NPM +releases come with multiple prebuilt configurations. + +## Browser + +### ESM Builds + +The recommended builds for production use are packaged as ES Modules and require +a _bootstrapping_ step in order to acquire the `.wasm` binaries and initialize +Perspective's JavaScript with them. However, because they have no hard-coded +dependencies on the `.wasm` paths, they are ideal for use with JavaScript +bundlers such as ESBuild, Rollup, Vite or Webpack. + +### CDN Builds + +Perspective's CDN builds are good for non-bundled scenarios, such as importing +directly from a ` +``` + +## Node.js builds + +The Node.js runtime for the `@finos/perspective` module runs in-process by +default and does not implement a `child_process` interface. Hence, there is no +`worker()` method, and the module object itself directly exports the full +`perspective` API. + +```javascript +const perspective = require("@finos/perspective"); +``` + +In Node.js, perspective does not run in a WebWorker (as this API does not exist +in Node.js), so no need to call the `.worker()` factory function - the +`perspective` library exports the functions directly and run synchronously in +the main process. diff --git a/docs/md/how_to/javascript/installation.md b/docs/md/how_to/javascript/installation.md new file mode 100644 index 0000000000..0083fecc9d --- /dev/null +++ b/docs/md/how_to/javascript/installation.md @@ -0,0 +1,31 @@ +# JavaScript NPM Installation + +Perspective releases contain several different builds for use in most +environments. + +## Browser + +Perspective's WebAssembly data engine is available via NPM in the same package +as its Node.js counterpart, `@finos/perspective`. The Perspective Viewer UI +(which has no Node.js component) must be installed separately: + +```bash +$ npm add @finos/perspective @finos/perspective-viewer +``` + +By itself, `@finos/perspective-viewer` does not provide any visualizations, only +the UI framework. Perspective _Plugins_ provide visualizations and must be +installed separately. All Plugins are optional - but a `` +without Plugins would be rather boring! + +```bash +$ npm add @finos/perspective-viewer-d3fc @finos/perspective-viewer-datagrid @finos/perspective-viewer-openlayers +``` + +## Node.js + +To use Perspective from a Node.js server, simply install via NPM. + +```bash +$ npm add @finos/perspective +``` diff --git a/docs/md/how_to/javascript/loading_data.md b/docs/md/how_to/javascript/loading_data.md new file mode 100644 index 0000000000..5a4d76df91 --- /dev/null +++ b/docs/md/how_to/javascript/loading_data.md @@ -0,0 +1,38 @@ +# Loading data from a Table + +Data can be loaded into `` in the form of a `Table()` or a +`Promise` via the `load()` method. + +```javascript +// Create a new worker, then a new table promise on that worker. +const worker = await perspective.worker(); +const table = await worker.table(data); + +// Bind a viewer element to this table. +await viewer.load(table); +``` + +## Sharing a `Table` between multiple ``s + +Multiple ``s can share a `table()` by passing the `table()` +into the `load()` method of each viewer. Each `perspective-viewer` will update +when the underlying `table()` is updated, but `table.delete()` will fail until +all `perspective-viewer` instances referencing it are also deleted: + +```javascript +const viewer1 = document.getElementById("viewer1"); +const viewer2 = document.getElementById("viewer2"); + +// Create a new WebWorker +const worker = await perspective.worker(); + +// Create a table in this worker +const table = await worker.table(data); + +// Load the same table in 2 different elements +await viewer1.load(table); +await viewer2.load(table); + +// Both `viewer1` and `viewer2` will reflect this update +await table.update([{ x: 5, y: "e", z: true }]); +``` diff --git a/docs/md/how_to/javascript/loading_virtual_data.md b/docs/md/how_to/javascript/loading_virtual_data.md new file mode 100644 index 0000000000..da5ec808f4 --- /dev/null +++ b/docs/md/how_to/javascript/loading_virtual_data.md @@ -0,0 +1,37 @@ +### Loading data from a virtual `Table` + +Loading a virtual (server-only) [`Table`] works just like loading a local/Web +Worker [`Table`] - just pass the virtual [`Table`] to `viewer.load()`. In the +browser: + +```javascript +const elem = document.getElementsByTagName("perspective-viewer")[0]; + +// Bind to the server's worker instead of instantiating a Web Worker. +const websocket = await perspective.websocket( + window.location.origin.replace("http", "ws") +); + +// Bind the viewer to the preloaded data source. `table` and `view` objects +// live on the server. +const server_table = await websocket.open_table("table_one"); +await elem.load(server_table); +``` + +Alternatively, data can be _cloned_ from a server-side virtual `Table` into a +client-side WebAssemblt `Table`. The browser clone will be synced via delta +updates transferred via Apache Arrow IPC format, but local `View`s created will +be calculated locally on the client browser. + +```javascript +const worker = await perspective.worker(); +const server_view = await server_table.view(); +const client_table = worker.table(server_view); +await elem.load(client_table); +``` + +`` instances bound in this way are otherwise no different +than ``s which rely on a Web Worker, and can even share a +host application with Web Worker-bound `table()`s. The same `promise`-based API +is used to communicate with the server-instantiated `view()`, only in this case +it is over a websocket. diff --git a/docs/md/how_to/javascript/nodejs_server.md b/docs/md/how_to/javascript/nodejs_server.md new file mode 100644 index 0000000000..9a0ce66496 --- /dev/null +++ b/docs/md/how_to/javascript/nodejs_server.md @@ -0,0 +1,38 @@ +# Server-only via `WebSocketServer()` and Node.js + +For exceptionally large datasets, a `Client` can be bound to a +`perspective.table()` instance running in Node.js/Python/Rust remotely, rather +than creating one in a Web Worker and downloading the entire data set. This +trades off network bandwidth and server resource requirements for a smaller +browser memory and CPU footprint. + +An example in Node.js: + +```javascript +const { WebSocketServer, table } = require("@finos/perspective"); +const fs = require("fs"); + +// Start a WS/HTTP host on port 8080. The `assets` property allows +// the `WebSocketServer()` to also serves the file structure rooted in this +// module's directory. +const host = new WebSocketServer({ assets: [__dirname], port: 8080 }); + +// Read an arrow file from the file system and host it as a named table. +const arr = fs.readFileSync(__dirname + "/superstore.lz4.arrow"); +await table(arr, { name: "table_one" }); +``` + +... and the [`Client`] implementation in the browser: + +```javascript +const elem = document.getElementsByTagName("perspective-viewer")[0]; + +// Bind to the server's worker instead of instantiating a Web Worker. +const websocket = await perspective.websocket( + window.location.origin.replace("http", "ws") +); + +// Create a virtual `Table` to the preloaded data source. `table` and `view` +// objects live on the server. +const server_table = await websocket.open_table("table_one"); +``` diff --git a/docs/md/how_to/javascript/save_restore.md b/docs/md/how_to/javascript/save_restore.md new file mode 100644 index 0000000000..32f79fde59 --- /dev/null +++ b/docs/md/how_to/javascript/save_restore.md @@ -0,0 +1,99 @@ +# Saving and restoring UI state. + +`` is _persistent_, in that its entire state (sans the data +itself) can be serialized or deserialized. This include all column, filter, +pivot, expressions, etc. properties, as well as datagrid style settings, config +panel visibility, and more. This overloaded feature covers a range of use cases: + +- Setting a ``'s initial state after a `load()` call. +- Updating a single or subset of properties, without modifying others. +- Resetting some or all properties to their data-relative default. +- Persisting a user's configuration to `localStorage` or a server. + +## Serializing and deserializing the viewer state + +To retrieve the entire state as a JSON-ready JavaScript object, use the `save()` +method. `save()` also supports a few other formats such as `"arraybuffer"` and +`"string"` (base64, not JSON), which you may choose for size at the expense of +easy migration/manual-editing. + +```javascript +const json_token = await elem.save(); +const string_token = await elem.save("string"); +``` + +For any format, the serialized token can be restored to any +`` with a `Table` of identical schema, via the `restore()` +method. Note that while the data for a token returned from `save()` may differ, +generally its schema may not, as many other settings depend on column names and +types. + +```javascript +await elem.restore(json_token); +await elem.restore(string_token); +``` + +As `restore()` dispatches on the token's type, it is important to make sure that +these types match! A common source of error occurs when passing a +JSON-stringified token to `restore()`, which will assume base64-encoded msgpack +when a string token is used. + +```javascript +// This will error! +await elem.restore(JSON.stringify(json_token)); +``` + +### Updating individual properties + +Using the JSON format, every facet of a ``'s configuration +can be manipulated from JavaScript using the `restore()` method. The valid +structure of properties is described via the +[`ViewerConfig`](https://github.com/finos/perspective/blob/ebced4caa/rust/perspective-viewer/src/ts/viewer.ts#L16) +and embedded +[`ViewConfig`](https://github.com/finos/perspective/blob/ebced4caa19435a2a57d4687be7e428a4efc759b/packages/perspective/index.d.ts#L140) +type declarations, and [`View`](view.md) chapter of the documentation which has +several interactive examples for each `ViewConfig` property. + +```javascript +// Set the plugin (will also update `columns` to plugin-defaults) +await elem.restore({ plugin: "X Bar" }); + +// Update plugin and columns (only draws once) +await elem.restore({ plugin: "X Bar", columns: ["Sales"] }); + +// Open the config panel +await elem.restore({ settings: true }); + +// Create an expression +await elem.restore({ + columns: ['"Sales" + 100'], + expressions: { "New Column": '"Sales" + 100' }, +}); + +// ERROR if the column does not exist in the schema or expressions +// await elem.restore({columns: ["\"Sales\" + 100"], expressions: {}}); + +// Add a filter +await elem.restore({ filter: [["Sales", "<", 100]] }); + +// Add a sort, don't remove filter +await elem.restore({ sort: [["Prodit", "desc"]] }); + +// Reset just filter, preserve sort +await elem.restore({ filter: undefined }); + +// Reset all properties to default e.g. after `load()` +await elem.reset(); +``` + +Another effective way to quickly create a token for a desired configuration is +to simply copy the token returned from `save()` after settings the view manually +in the browser. The JSON format is human-readable and should be quite easy to +tweak once generated, as `save()` will return even the default settings for all +properties. You can call `save()` in your application code, or e.g. through the +Chrome developer console: + +```javascript +// Copy to clipboard +copy(await document.querySelector("perspective-viewer").save()); +``` diff --git a/docs/md/how_to/javascript/serializing.md b/docs/md/how_to/javascript/serializing.md new file mode 100644 index 0000000000..7332fa836e --- /dev/null +++ b/docs/md/how_to/javascript/serializing.md @@ -0,0 +1,21 @@ +### Serializing data + +The `view()` allows for serialization of data to JavaScript through the +`to_json()`, `to_ndjson()`, `to_columns()`, `to_csv()`, and `to_arrow()` methods +(the same data formats supported by the `Client::table` factory function). These +methods return a `promise` for the calculated data: + +```javascript +const view = await table.view({ group_by: ["State"], columns: ["Sales"] }); + +// JavaScript Objects +console.log(await view.to_json()); +console.log(await view.to_columns()); + +// String +console.log(await view.to_csv()); +console.log(await view.to_ndjson()); + +// ArrayBuffer +console.log(await view.to_arrow()); +``` diff --git a/docs/md/how_to/javascript/theming.md b/docs/md/how_to/javascript/theming.md new file mode 100644 index 0000000000..b3c1dc4d04 --- /dev/null +++ b/docs/md/how_to/javascript/theming.md @@ -0,0 +1,67 @@ +# Theming + +Theming is supported in `perspective-viewer` and its accompanying plugins. A +number of themes come bundled with `perspective-viewer`; you can import any of +these themes directly into your app, and the `perspective-viewer`s will be +themed accordingly: + +```javascript +// Themes based on Thought Merchants's Prospective design +import "@finos/perspective-viewer/dist/css/pro.css"; +import "@finos/perspective-viewer/dist/css/pro-dark.css"; + +// Other themes +import "@finos/perspective-viewer/dist/css/solarized.css"; +import "@finos/perspective-viewer/dist/css/solarized-dark.css"; +import "@finos/perspective-viewer/dist/css/monokai.css"; +import "@finos/perspective-viewer/dist/css/vaporwave.css"; +``` + +Alternatively, you may use `themes.css`, which bundles all default themes + +```javascript +import "@finos/perspective-viewer/dist/css/themes.css"; +``` + +If you choose not to bundle the themes yourself, they are available through +[CDN](https://cdn.jsdelivr.net/npm/@finos/perspective-viewer/dist/css/). These +can be directly linked in your HTML file: + +```html + +``` + +Note the `crossorigin="anonymous"` attribute. When including a theme from a +cross-origin context, this attribute may be required to allow +`` to detect the theme. If this fails, additional themes are +added to the `document` after `` init, or for any other +reason theme auto-detection fails, you may manually inform +`` of the available theme names with the `.resetThemes()` +method. + +```javascript +// re-auto-detect themes +viewer.resetThemes(); + +// Set available themes explicitly (they still must be imported as CSS!) +viewer.resetThemes(["Pro Light", "Pro Dark"]); +``` + +`` will default to the first loaded theme when initialized. +You may override this via `.restore()`, or provide an initial theme by setting +the `theme` attribute: + +```html + +``` + +or + +```javascript +const viewer = document.querySelector("perspective-viewer"); +await viewer.restore({ theme: "Pro Dark" }); +``` diff --git a/docs/md/how_to/javascript/viewer.md b/docs/md/how_to/javascript/viewer.md new file mode 100644 index 0000000000..ef9c86ae5e --- /dev/null +++ b/docs/md/how_to/javascript/viewer.md @@ -0,0 +1,16 @@ +# `` Custom Element library + +`` provides a complete graphical UI for configuring the +`perspective` library and formatting its output to the provided visualization +plugins. + +Once imported and initialized in JavaScript, the `` Web +Component will be available in any standard HTML on your site. A simple example: + +```html + + +``` diff --git a/docs/md/how_to/javascript/worker.md b/docs/md/how_to/javascript/worker.md new file mode 100644 index 0000000000..dc3d9adbe2 --- /dev/null +++ b/docs/md/how_to/javascript/worker.md @@ -0,0 +1,44 @@ +# Accessing the Perspective engine via a `Client` instance + +An instance of a `Client` is needed to talk to a Perspective `Server`, of which +there are a few varieties available in JavaScript. + +## Web Worker (Browser) + +Perspective's Web Worker client is actually a `Client` and `Server` rolled into +one. Instantiating this `Client` will also create a _dedicated_ Perspective +`Server` in a Web Worker process. + +To use it, you'll need to instantiate a Web Worker `perspective` engine via the +`worker()` method. This will create a new Web Worker (browser) and load the +WebAssembly binary. All calculation and data accumulation will occur in this +separate process. + +```javascript +const client = await perspective.worker(); +``` + +The `worker` symbol will expose the full `perspective` API for one managed Web +Worker process. You are free to create as many as your browser supports, but be +sure to keep track of the `worker` instances themselves, as you'll need them to +interact with your data in each instance. + +## Websocket (Browser) + +Alternatively, with a Perspective server running in Node.js, Python or Rust, you +can create a _virtual_ `Client` via the `websocket()` method. + +```javascript +const client = perspective.websocket("http://localhost:8080/"); +``` + +## Node.js + +The Node.js runtime for the `@finos/perspective` module runs in-process by +default and does not implement a `child_process` interface, so no need to call +the `.worker()` factory function. Instead, the `perspective` library exports the +functions directly and run synchronously in the main process. + +```javascript +const client = require("@finos/perspective"); +``` diff --git a/docs/md/how_to/python.md b/docs/md/how_to/python.md new file mode 100644 index 0000000000..4193ef4bec --- /dev/null +++ b/docs/md/how_to/python.md @@ -0,0 +1 @@ +# Python diff --git a/docs/md/how_to/python/callbacks.md b/docs/md/how_to/python/callbacks.md new file mode 100644 index 0000000000..cc6b28d12c --- /dev/null +++ b/docs/md/how_to/python/callbacks.md @@ -0,0 +1,34 @@ +# Callbacks and Events + +`perspective.Table` allows for `on_update` and `on_delete` callbacks to be +set—simply call `on_update` or `on_delete` with a reference to a function or a +lambda without any parameters: + +```python +def update_callback(): + print("Updated!") + +# set the update callback +on_update_id = view.on_update(update_callback) + + +def delete_callback(): + print("Deleted!") + +# set the delete callback +on_delete_id = view.on_delete(delete_callback) + +# set a lambda as a callback +view.on_delete(lambda: print("Deleted x2!")) +``` + +If the callback is a named reference to a function, it can be removed with +`remove_update` or `remove_delete`: + +```python +view.remove_update(on_update_id) +view.remove_delete(on_delete_id) +``` + +Callbacks defined with a lambda function cannot be removed, as lambda functions +have no identifier. diff --git a/docs/md/how_to/python/installation.md b/docs/md/how_to/python/installation.md new file mode 100644 index 0000000000..ae2558a9a6 --- /dev/null +++ b/docs/md/how_to/python/installation.md @@ -0,0 +1,27 @@ +# Installation + +`perspective-python` contains full bindings to the Perspective API, a JupyterLab +widget, and WebSocket handlers for several webserver libraries that allow you to +host Perspective using server-side Python. + +## PyPI + +`perspective-python` can be installed from [PyPI](https://pypi.org) via `pip`: + +```bash +pip install perspective-python +``` + +That's it! If JupyterLab is installed in this Python environment, you'll also +get the `perspective.widget.PerspectiveWidget` class when you import +`perspective` in a Jupyter Lab kernel. + + diff --git a/docs/md/how_to/python/jupyterlab.md b/docs/md/how_to/python/jupyterlab.md new file mode 100644 index 0000000000..33f6aea33b --- /dev/null +++ b/docs/md/how_to/python/jupyterlab.md @@ -0,0 +1,61 @@ +# `PerspectiveWidget` for JupyterLab + +Building on top of the API provided by `perspective.Table`, the +`PerspectiveWidget` is a JupyterLab plugin that offers the entire functionality +of Perspective within the Jupyter environment. It supports the same API +semantics of ``, along with the additional data types +supported by `perspective.Table`. `PerspectiveWidget` takes keyword arguments +for the managed `View`: + +```python +from perspective.widget import PerspectiveWidget +w = perspective.PerspectiveWidget( + data, + plugin="X Bar", + aggregates={"datetime": "any"}, + sort=[["date", "desc"]] +) +``` + +## Creating a widget + +A widget is created through the `PerspectiveWidget` constructor, which takes as +its first, required parameter a `perspective.Table`, a dataset, a schema, or +`None`, which serves as a special value that tells the Widget to defer loading +any data until later. In maintaining consistency with the Javascript API, +Widgets cannot be created with empty dictionaries or lists — `None` should be +used if the intention is to await data for loading later on. A widget can be +constructed from a dataset: + +```python +from perspective.widget import PerspectiveWidget +PerspectiveWidget(data, group_by=["date"]) +``` + +.. or a schema: + +```python +PerspectiveWidget({"a": int, "b": str}) +``` + +.. or an instance of a `perspective.Table`: + +```python +table = perspective.table(data) +PerspectiveWidget(table) +``` + +## Updating a widget + +`PerspectiveWidget` shares a similar API to the `` Custom +Element, and has similar `save()` and `restore()` methods that +serialize/deserialize UI state for the widget. + + diff --git a/docs/md/how_to/python/multithreading.md b/docs/md/how_to/python/multithreading.md new file mode 100644 index 0000000000..5ad1e4d1c4 --- /dev/null +++ b/docs/md/how_to/python/multithreading.md @@ -0,0 +1,18 @@ +# Multi-threading + +Perspective's server API releases the GIL when called (though it may be retained +for some portion of the `Client` call to encode RPC messages). It also +dispatches to an internal thread pool for some operations, enabling better +parallelism and overall better server performance. However, Perspective's Python +interface itself will still process queries in a single queue. To enable +parallel query processing, call `set_loop_callback` with a multi-threaded +executor such as `concurrent.futures.ThreadPoolExecutor`: + +```python +def perspective_thread(): + server = perspective.Server() + loop = tornado.ioloop.IOLoop() + with concurrent.futures.ThreadPoolExecutor() as executor: + server.set_loop_callback(loop.run_in_executor, executor) + loop.start() +``` diff --git a/docs/md/how_to/python/table.md b/docs/md/how_to/python/table.md new file mode 100644 index 0000000000..c15ecbaf93 --- /dev/null +++ b/docs/md/how_to/python/table.md @@ -0,0 +1,84 @@ +# Loading data into a Table + +A `Table` can be created from a dataset or a schema, the specifics of which are +[discussed](#loading-data-with-table) in the JavaScript section of the user's +guide. In Python, however, Perspective supports additional data types that are +commonly used when processing data: + +- `pandas.DataFrame` +- `polars.DataFrame` +- `bytes` (encoding an Apache Arrow) +- `objects` (either extracting a repr or via reference) +- `str` (encoding as a CSV) + +A `Table` is created in a similar fashion to its JavaScript equivalent: + +```python +from datetime import date, datetime +import numpy as np +import pandas as pd +import perspective + +data = pd.DataFrame({ + "int": np.arange(100), + "float": [i * 1.5 for i in range(100)], + "bool": [True for i in range(100)], + "date": [date.today() for i in range(100)], + "datetime": [datetime.now() for i in range(100)], + "string": [str(i) for i in range(100)] +}) + +table = perspective.table(data, index="float") +``` + +Likewise, a `View` can be created via the `view()` method: + +```python +view = table.view(group_by=["float"], filter=[["bool", "==", True]]) +column_data = view.to_columns() +row_data = view.to_json() +``` + +## Polars Support + +Polars `DataFrame` types work similarly to Apache Arrow input, which Perspective +uses to interface with Polars. + +```python +df = polars.DataFrame({"a": [1,2,3,4,5]}) +table = perspective.table(df) +``` + +## Pandas Support + +Perspective's `Table` can be constructed from `pandas.DataFrame` objects. +Internally, this just uses +[`pyarrow::from_pandas`](https://arrow.apache.org/docs/python/pandas.html), +which dictates behavior of this feature including type support. + +If the dataframe does not have an index set, an integer-typed column named +`"index"` is created. If you want to preserve the indexing behavior of the +dataframe passed into Perspective, simply create the `Table` with +`index="index"` as a keyword argument. This tells Perspective to once again +treat the index as a primary key: + +```python +data.set_index("datetime") +table = perspective.table(data, index="index") +``` + +## Time Zone Handling + +When parsing `"datetime"` strings, times are assumed _local time_ unless an +explicit timezone offset is parsed. All `"datetime"` columns (regardless of +input time zone) are _output_ to the user as `datetime.datetime` objects in +_local time_ according to the Python runtime. + +This behavior is consistent with Perspective's behavior in JavaScript. For more +details, see this in-depth +[explanation](https://github.com/finos/perspective/pull/867) of +`perspective-python` semantics around time zone handling. + +``` + +``` diff --git a/docs/md/how_to/python/websocket.md b/docs/md/how_to/python/websocket.md new file mode 100644 index 0000000000..bf60ae8a2e --- /dev/null +++ b/docs/md/how_to/python/websocket.md @@ -0,0 +1,129 @@ +# Hosting a WebSocket server + +An in-memory `Server` "hosts" all `perspective.Table` and `perspective.View` +instances created by its connected `Client`s. Hosted tables/views can have their +methods called from other sources than the Python server, i.e. by a +`perspective-viewer` running in a JavaScript client over the network, +interfacing with `perspective-python` through the websocket API. + +The server has full control of all hosted `Table` and `View` instances, and can +call any public API method on hosted instances. This makes it extremely easy to +stream data to a hosted `Table` using `.update()`: + +```python +server = perspective.Server() +client = server.new_local_client() +table = client.table(data, name="data_source") + +for i in range(10): + # updates continue to propagate automatically + table.update(new_data) +``` + +The `name` provided is important, as it enables Perspective in JavaScript to +look up a `Table` and get a handle to it over the network. Otherwise, `name` +will be assigned randomlu and the `Client` must look this up with +`CLient.get_hosted_table_names()` + +## Client/Server Replicated Mode + +Using Tornado and +[`PerspectiveTornadoHandler`](python.md#perspectivetornadohandler), as well as +`Perspective`'s JavaScript library, we can set up "distributed" Perspective +instances that allows multiple browser `perspective-viewer` clients to read from +a common `perspective-python` server, as in the +[Tornado Example Project](https://github.com/finos/perspective/tree/master/examples/python-tornado). + +This architecture works by maintaining two `Tables`—one on the server, and one +on the client that mirrors the server's `Table` automatically using `on_update`. +All updates to the table on the server are automatically applied to each client, +which makes this architecture a natural fit for streaming dashboards and other +distributed use-cases. In conjunction with [multithreading](#multi-threading), +distributed Perspective offers consistently high performance over large numbers +of clients and large datasets. + +_*server.py*_ + +```python +from perspective import Server +from perspective.hadnlers.tornado import PerspectiveTornadoHandler + +# Create an instance of Server, and host a Table +SERVER = Server() +CLIENT = SERVER.new_local_client() + +# The Table is exposed at `localhost:8888/websocket` with the name `data_source` +client.table(data, name = "data_source") + +app = tornado.web.Application([ + # create a websocket endpoint that the client JavaScript can access + (r"/websocket", PerspectiveTornadoHandler, {"perspective_server": SERVER}) +]) + +# Start the Tornado server +app.listen(8888) +loop = tornado.ioloop.IOLoop.current() +loop.start() +``` + +Instead of calling `load(server_table)`, create a `View` using `server_table` +and pass that into `viewer.load()`. This will automatically register an +`on_update` callback that synchronizes state between the server and the client. + +_*index.html*_ + +```html + + + +``` + +For a more complex example that offers distributed editing of the server +dataset, see +[client_server_editing.html](https://github.com/finos/perspective/blob/master/examples/python-tornado/client_server_editing.html). + +We also provide examples for Starlette/FastAPI and AIOHTTP: + +- [Starlette Example Project](https://github.com/finos/perspective/tree/master/examples/python-starlette). +- [AIOHTTP Example Project](https://github.com/finos/perspective/tree/master/examples/python-aiohttp). + +## Server-only Mode + +The server setup is identical to +[Client/Server Replicated Mode](#client-server-replicated-mode) above, but +instead of creating a `View`, the client calls `load(server_table)`: In Python, +use `Server` and `PerspectiveTornadoHandler` to create a websocket server that +exposes a `Table`. In this example, `table` is a proxy for the `Table` we +created on the server. All API methods are available on _proxies_, the.g.us +calling `view()`, `schema()`, `update()` on `table` will pass those operations +to the Python `Table`, execute the commands, and return the result back to +Javascript. + +```html + +``` + +```javascript +const websocket = perspective.websocket("ws://localhost:8888/websocket"); +const table = websocket.open_table("data_source"); +document.getElementById("viewer").load(table); +``` diff --git a/docs/md/how_to/rust.md b/docs/md/how_to/rust.md new file mode 100644 index 0000000000..837deaea9d --- /dev/null +++ b/docs/md/how_to/rust.md @@ -0,0 +1,28 @@ +# Rust + +Install via `cargo`: + +```bash +cargo add perspective +``` + +# Example + +Initialize a server and client + +```rust +let server = Server::default(); +let client = server.new_local_client(); +``` + +Load an Arrow + +```rust +let mut file = File::open(std::path::Path::new(ROOT_PATH).join(ARROW_FILE_PATH))?; +let mut feather = Vec::with_capacity(file.metadata()?.len() as usize); +file.read_to_end(&mut feather)?; +let data = UpdateData::Arrow(feather.into()); +let mut options = TableInitOptions::default(); +options.set_name("my_data_source"); +client.table(data.into(), options).await?; +``` diff --git a/docs/md/how_to/table.md b/docs/md/how_to/table.md new file mode 100644 index 0000000000..b135e11d16 --- /dev/null +++ b/docs/md/how_to/table.md @@ -0,0 +1,47 @@ +# Construct a Table + +Examples of constructing an empty `Table` from a schema. + +
+ +JavaScript: + +```javascript +var schema = { + x: "integer", + y: "string", + z: "boolean", +}; + +const table2 = await worker.table(schema); +``` + +
+
+ +Python: + +```python +from datetime import date, datetime + +schema = { + "x": "integer", + "y": "string", + "z": "boolean", +} + +table2 = perspective.table(schema) +``` + +
+
+ +Rust: + +```rust +let data = TableData::Schema(vec![(" a".to_string(), ColumnType::FLOAT)]); +let options = TableInitOptions::default(); +let table = client.table(data.into(), options).await?; +``` + +
diff --git a/docs/md/javascript.md b/docs/md/javascript.md new file mode 100644 index 0000000000..52258b5098 --- /dev/null +++ b/docs/md/javascript.md @@ -0,0 +1 @@ +# JavaScript diff --git a/docs/md/perspective.css b/docs/md/perspective.css new file mode 100644 index 0000000000..af094bd422 --- /dev/null +++ b/docs/md/perspective.css @@ -0,0 +1,3 @@ +perspective-viewer { + height: 500px; +} diff --git a/examples/react-example/webpack.config.js b/docs/md/perspective.js similarity index 65% rename from examples/react-example/webpack.config.js rename to docs/md/perspective.js index 1b2f1e8b6e..4365884f1c 100644 --- a/examples/react-example/webpack.config.js +++ b/docs/md/perspective.js @@ -10,43 +10,20 @@ // ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ // ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ -const PerspectivePlugin = require("@finos/perspective-webpack-plugin"); -const HtmlWebPackPlugin = require("html-webpack-plugin"); -const path = require("path"); +import "https://cdn.jsdelivr.net/npm/@finos/perspective-viewer/dist/cdn/perspective-viewer.js"; +import "https://cdn.jsdelivr.net/npm/@finos/perspective-viewer-datagrid/dist/cdn/perspective-viewer-datagrid.js"; +import "https://cdn.jsdelivr.net/npm/@finos/perspective-viewer-d3fc/dist/cdn/perspective-viewer-d3fc.js"; +import perspective from "https://cdn.jsdelivr.net/npm/@finos/perspective/dist/cdn/perspective.js"; -module.exports = { - mode: "development", - devtool: "source-map", - resolve: { - extensions: [".ts", ".tsx", ".js"], - }, +const WASM_URL = + "https://cdn.jsdelivr.net/npm/superstore-arrow/superstore.lz4.arrow"; - plugins: [ - new HtmlWebPackPlugin({ - title: "Perspective React Example", - template: "./src/index.html", - }), - new PerspectivePlugin(), - ], +const worker = await perspective.worker(); - module: { - rules: [ - { - test: /\.ts(x?)$/, - exclude: /node_modules/, - loader: "ts-loader", - }, - { - test: /\.css$/, - exclude: /node_modules/, - use: [{ loader: "style-loader" }, { loader: "css-loader" }], - }, - ], - }, - devServer: { - static: [ - path.join(__dirname, "dist"), - path.join(__dirname, "../../node_modules/superstore-arrow"), - ], - }, -}; +const table = await fetch(WASM_URL) + .then((x) => x.arrayBuffer()) + .then((x) => worker.table(x)); + +for (const viewer of document.querySelectorAll("perspective-viewer")) { + viewer.load(table); +} diff --git a/docs/md/tutorials/python/tornado.md b/docs/md/tutorials/python/tornado.md new file mode 100644 index 0000000000..0cfde7a1ef --- /dev/null +++ b/docs/md/tutorials/python/tornado.md @@ -0,0 +1,109 @@ +# Tutorial: A tornado server in Python + +Perspective ships with a pre-built Tornado handler that makes integration with +`tornado.websockets` extremely easy. This allows you to run an instance of +`Perspective` on a server using Python, open a websocket to a `Table`, and +access the `Table` in JavaScript and through ``. All +instructions sent to the `Table` are processed in Python, which executes the +commands, and returns its output through the websocket back to Javascript. + +### Python setup + +Make sure Perspective and Tornado are installed! + +```bash +pip install perspective-python tornado +``` + +To use the handler, we need to first have a `Server`, a `Client` and an instance +of a `Table`: + +```python +import perspective + +SERVER = perspective.Server() +CLIENT = SERVER.new_local_client() +``` + +Once the server has been created, create a `Table` instance with a name. The +name that you host the table under is important — it acts as a unique accessor +on the JavaScript side, which will look for a Table hosted at the websocket with +the name you specify. + +```python +TABLE = client.table(data, name="data_source_one") +``` + +After the server and table setup is complete, create a websocket endpoint and +provide it a reference to `PerspectiveTornadoHandler`. You must provide the +configuration object in the route tuple, and it must contain +`"perspective_server"`, which is a reference to the `Server` you just created. + +```python +from perspective.handlers.tornado import PerspectiveTornadoHandler + +app = tornado.web.Application([ + + # ... other handlers ... + + # Create a websocket endpoint that the client JavaScript can access + (r"/websocket", PerspectiveTornadoHandler, {"perspective_server": SERVER, "check_origin": True}) +]) +``` + +Optionally, the configuration object can also include `check_origin`, a boolean +that determines whether the websocket accepts requests from origins other than +where the server is hosted. See +[Tornado docs](https://www.tornadoweb.org/en/stable/websocket.html#tornado.websocket.WebSocketHandler.check_origin) +for more details. + +### JavaScript setup + +Once the server is up and running, you can access the Table you just hosted +using `perspective.websocket` and `open_table()`. First, create a client that +expects a Perspective server to accept connections at the specified URL: + +```javascript +import "@finos/perspective-viewer"; +import "@finos/perspective-viewer-datagrid"; +import perspective from "@finos/perspective"; + +const websocket = await perspective.websocket("ws://localhost:8888/websocket"); +``` + +Next open the `Table` we created on the server by name: + +```javascript +const table = await websocket.open_table("data_source_one"); +``` + +`table` is a proxy for the `Table` we created on the server. All operations that +are possible through the JavaScript API are possible on the Python API as well, +thus calling `view()`, `schema()`, `update()` etc. on `const table` will pass +those operations to the Python `Table`, execute the commands, and return the +result back to JavaScript. Similarly, providing this `table` to a +`` instance will allow virtual rendering: + +```javascript +const viewer = document.createElement("perspective-viewer"); +viewer.style.height = "500px"; +document.body.appendChild(viewer); +await viewer.load(table); +``` + +`perspective.websocket` expects a Websocket URL where it will send instructions. +When `open_table` is called, the name to a hosted Table is passed through, and a +request is sent through the socket to fetch the Table. No actual `Table` +instance is passed inbetween the runtimes; all instructions are proxied through +websockets. + +This provides for great flexibility — while `Perspective.js` is full of +features, browser WebAssembly runtimes currently have some performance +restrictions on memory and CPU feature utilization, and the architecture in +general suffers when the dataset itself is too large to download to the client +in full. + +The Python runtime does not suffer from memory limitations, utilizes Apache +Arrow internal threadpools for threading and parallel processing, and generates +architecture optimized code, which currently makes it more suitable as a +server-side runtime than `node.js`. diff --git a/docs/package.json b/docs/package.json index 6c0ba45ccc..4b3e213a18 100644 --- a/docs/package.json +++ b/docs/package.json @@ -21,15 +21,14 @@ "@finos/perspective-viewer": "workspace:^", "@finos/perspective-viewer-d3fc": "workspace:^", "@finos/perspective-viewer-datagrid": "workspace:^", - "@finos/perspective-webpack-plugin": "workspace:^", "@mdx-js/react": "^3.0.0", "blocks": "workspace:^", "clsx": "^2.0.0", "mkdirp": "3", "prism-react-renderer": "^1.3.3", "puppeteer": "^23", - "react": "^18.0.0", - "react-dom": "^18.0.0", + "react": "^18", + "react-dom": "^18", "superstore-arrow": "3.0.0" }, "devDependencies": { diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 0000000000..8bb2dc5609 --- /dev/null +++ b/examples/README.md @@ -0,0 +1,19 @@ +# Examples + +The projects in the directory are self-contained Perspective examples. However, +they are designed to run within the Perspective source repository tree. This +makes it easy for Perspective OSS developers to test changes and validate that +our Examples continue to work, but it also means these examples will most not +work without properly linking in the Perspective build environment. + +In order to _run_ a project in this directory as written: + +1. Install and build Perspective from source. +2. Run the project with `pnpm run start $PROJECT_NAME` from the repository root + (_not_ the `/examples` directory). + +# Optional + +Generally, the changes necessary to make these examples run _without_ the +Perspective source repository are minor path or metadata corrections. Your +results may vary. diff --git a/examples/blocks/README.md b/examples/blocks/README.md new file mode 100644 index 0000000000..203462c511 --- /dev/null +++ b/examples/blocks/README.md @@ -0,0 +1,4 @@ +# Bloc.ks + +Examples designed to be hosted on GitHub Pages. See the individual projects in +the `src/` directory for more info. diff --git a/examples/blocks/src/citibike/citibike.js b/examples/blocks/src/citibike/citibike.js index 6f73be549d..73e395603c 100644 --- a/examples/blocks/src/citibike/citibike.js +++ b/examples/blocks/src/citibike/citibike.js @@ -85,7 +85,7 @@ async function main() { } // Start a recurring asyn call to `get_feed` and update the `table` with the response. - get_feed("station_status", table.update); + get_feed("station_status", table.update.bind(table)); window.workspace.tables.set("citibike", Promise.resolve(table)); const layout = await get_layout(); diff --git a/examples/blocks/src/citibike/index.html b/examples/blocks/src/citibike/index.html index 8d03e0f15f..ee1c82d697 100644 --- a/examples/blocks/src/citibike/index.html +++ b/examples/blocks/src/citibike/index.html @@ -3,20 +3,14 @@ - diff --git a/examples/blocks/src/covid/index.html b/examples/blocks/src/covid/index.html index 2c19d0e007..e67fa518b3 100644 --- a/examples/blocks/src/covid/index.html +++ b/examples/blocks/src/covid/index.html @@ -2,8 +2,8 @@ - - + + diff --git a/examples/blocks/src/fractal/index.html b/examples/blocks/src/fractal/index.html index 5854e8a37c..46e0a1ad10 100644 --- a/examples/blocks/src/fractal/index.html +++ b/examples/blocks/src/fractal/index.html @@ -2,8 +2,8 @@ - - + + diff --git a/examples/blocks/src/market/index.html b/examples/blocks/src/market/index.html index 96141ef1c4..adb99186db 100644 --- a/examples/blocks/src/market/index.html +++ b/examples/blocks/src/market/index.html @@ -2,8 +2,8 @@ - - + + diff --git a/examples/blocks/src/movies/index.html b/examples/blocks/src/movies/index.html index 6919cb21e2..e7f45997ae 100644 --- a/examples/blocks/src/movies/index.html +++ b/examples/blocks/src/movies/index.html @@ -1,22 +1,17 @@ - - + +