Update docs

Signed-off-by: Andrew Stein <[email protected]>

.github/workflows/build.yaml

@@ -168,8 +168,6 @@ jobs:
                       packages/perspective-viewer-d3fc/dist
                       packages/perspective-viewer-datagrid/dist
                       packages/perspective-viewer-openlayers/dist
-                      packages/perspective-esbuild-plugin/dist
-                      packages/perspective-webpack-plugin/dist
                       packages/perspective-cli/dist
                       packages/perspective-workspace/dist
 
@@ -947,12 +945,6 @@ jobs:
             - run: pnpm pack --pack-destination=../..
               working-directory: ./packages/perspective-cli
 
-            - run: pnpm pack --pack-destination=../..
-              working-directory: ./packages/perspective-webpack-plugin
-
-            - run: pnpm pack --pack-destination=../..
-              working-directory: ./packages/perspective-esbuild-plugin
-
             - run: pnpm pack --pack-destination=../..
               working-directory: ./packages/perspective-jupyterlab
 

README.md

@@ -1,6 +1,10 @@
 <br />
-<a href="https://github.com/finos/perspective/blob/master/docs/static/svg/perspective-logo-light.svg?raw=true#gh-light-mode-only"><img src="https://github.com/finos/perspective/raw/master/docs/static/svg/perspective-logo-light.svg?raw=true#gh-light-mode-only" alt="Perspective" width="260"></a>
-<a href="https://github.com/finos/perspective/blob/master/docs/static/svg/perspective-logo-dark.svg?raw=true#gh-dark-mode-only"><img src="https://github.com/finos/perspective/raw/master/docs/static/svg/perspective-logo-dark.svg?raw=true#gh-dark-mode-only" alt="Perspective" width="260"></a>
+<a href="https://perspective.finos.org">
+<picture>
+<source media="(prefers-color-scheme: dark)" srcset="https://github.com/finos/perspective/raw/master/docs/static/svg/perspective-logo-dark.svg?raw=true">
+<img width="260" src="https://github.com/finos/perspective/raw/master/docs/static/svg/perspective-logo-light.svg?raw=true">
+</picture>
+</a>
 <br/><br/>
 
 [![Build Status](https://img.shields.io/github/actions/workflow/status/finos/perspective/build.yaml?event=push&style=for-the-badge)](https://github.com/finos/perspective/actions/workflows/build.yaml)

cpp/perspective/CMakeLists.txt

@@ -231,7 +231,7 @@ if(PSP_WASM_BUILD)
             ")
         endif()
     else()
-        set(OPT_FLAGS " -O3 -g0 -flto --emit-tsd=perspective-server.d.ts ")
+        set(OPT_FLAGS " -O3 -g0 --emit-tsd=perspective-server.d.ts ")
         if (PSP_WASM_EXCEPTIONS)
             set(OPT_FLAGS "${OPT_FLAGS} -fwasm-exceptions ")
         endif()

docs/book.toml

@@ -0,0 +1,42 @@
+#  ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+#  ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+#  ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+#  ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+#  ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+#  ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+#  ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+#  ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+#  ┃ This file is part of the Perspective library, distributed under the terms ┃
+#  ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+#  ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+[book]
+authors = ["Andrew Stein"]
+language = "en"
+multilingual = false
+src = "md"
+title = "Perspective"
+
+[build]
+build-dir = "static/guide"
+
+[output.html]
+# theme = "my-theme"
+# default-theme = "light"
+# preferred-dark-theme = "navy"
+# smart-punctuation = true
+# mathjax-support = false
+copy-fonts = true
+git-repository-url = "https://github.com/finos/perspective"
+git-repository-icon = "fa-github"
+site-url = "https://perspective.finos.org/guide/"
+additional-css = [
+    "md/perspective.css",
+    "node_modules/@finos/perspective-viewer/dist/css/themes.css",
+]
+# additional-js = []
+# no-section-label = false
+# edit-url-template = "https://github.com/rust-lang/mdBook/edit/master/guide/{path}"
+# site-url = "/guide/"
+# cname = "myproject.rs"
+# input-404 = "not-found.md"

docs/md/SUMMARY.md

@@ -0,0 +1,60 @@
+# Summary
+
+[What is Perspective](./perspective.md)
+
+# Overview
+
+-   [Data Architecture](./explanation/architecture.md)
+    -   [Client-only](./explanation/architecture/client_only.md)
+    -   [Client/Server replicated](./explanation/architecture/client_server.md)
+    -   [Server only](./explanation/architecture/server_only.md)
+-   [`Table`](./explanation/table.md)
+    -   [Construct an empty `Table` from a schema](./explanation/table/constructing_schema.md)
+    -   [Schema and column types](./explanation/table/schema.md)
+    -   [Loading data](./explanation/table/loading_data.md)
+    -   [`index` and `limit` options](./explanation/table/options.md)
+    -   [`update()` and `remove()` streaming methods](./explanation/table/update_and_remove.md)
+    -   [`clear()` and `replace()` start-over methods](./explanation/table/clear_and_replace.md)
+-   [`View`](./explanation/view.md)
+    -   [Querying data](./explanation/view/querying.md)
+        -   [`group_by`](./explanation/view/config/group_by.md)
+        -   [`split_by`](./explanation/view/config/split_by.md)
+        -   [`aggregates`](./explanation/view/config/aggregates.md)
+        -   [`columns`](./explanation/view/config/columns.md)
+        -   [`sort`](./explanation/view/config/sort.md)
+        -   [`filter`](./explanation/view/config/filter.md)
+        -   [`expressions`](./explanation/view/config/expressions.md)
+    -   [Flattening a View into a Table](./explanation/view/config/flattening.md)
+-   [JavaScript](./explanation/javascript.md)
+    -   [Module Structure](./explanation/javascript_module_structure.md)
+    -   [Build options](./explanation/javascript_builds.md)
+-   [Python](./explanation/python.md)
+
+# Getting Started
+
+-   [Rust](./how_to/rust.md)
+-   [JavaScript](./how_to/javascript.md)
+    -   [Installation via NPM](./how_to/javascript/installation.md)
+    -   [Importing with or without a bundler](./how_to/javascript/importing.md)
+    -   [`perspective` data engine library](./how_to/javascript/worker.md)
+        -   [Serializing data](./how_to/javascript/serializing.md)
+        -   [Cleaning up resources](./how_to/javascript/deleting.md)
+        -   [Hosting a `WebSocketServer` in Node.js](./how_to/javascript/nodejs_server.md)
+    -   [`perspective-viewer` Custom Element library](./how_to/javascript/viewer.md)
+        -   [Theming](./how_to/javascript/theming.md)
+        -   [Loading data from a `Table`](./how_to/javascript/loading_data.md)
+        -   [Loading data from a virtual `Table`](./how_to/javascript/loading_virtual_data.md)
+        -   [Saving and restoring UI state](./how_to/javascript/save_restore.md)
+        -   [Listening for events](./how_to/javascript/events.md)
+-   [Python](./how_to/python.md)
+    -   [Installation](./how_to/python/installation.md)
+    -   [Loading data into a `Table`](./how_to/python/table.md)
+    -   [Callbacks and events](./how_to/python/callbacks.md)
+    -   [Multithreading](./how_to/python/multithreading.md)
+    -   [Hosting a WebSocket server](./how_to/python/websocket.md)
+    -   [`PerspectiveWidget` for JupyterLab](./how_to/python/jupyterlab.md)
+    -   [Tutorial: A `tornado` server with virtual `perspective-viewer`](./tutorials/python/tornado.md)
+
+# API
+
+-   [Crate documentation on `docs.rs` ](./api_reference.md)

docs/md/api_reference.md

@@ -0,0 +1,9 @@
+# API Reference
+
+Perspective's complete API is hosted on `docs.rs`:
+
+-   [`perspective-client`](https://docs.rs/perspective-client/latest/perspective_client/index.html)
+    covers `Table` and `View` data engine API methods common for Rust,
+    JavaScript and Python.
+-   [`perspective-rs`](https://docs.rs/perspective-client/latest/perspective_client/index.html)
+    adds Rust-specific documentation for the Rust crate entrypoint.

docs/md/explanation.md

@@ -0,0 +1 @@
+# Explanation

docs/md/explanation/architecture.md

@@ -0,0 +1,15 @@
+# Data Architecture
+
+Application developers can choose from
+[Client (WebAssembly)](./architecture/client_only.md),
+[Server (Python/Node)](./architecture/server_only.md) or
+[Client/Server Replicated](./architecture/client_server.md) designs to bind
+data, and a web application can use one or a mix of these designs as needed. By
+serializing to Apache Arrow, tables are duplicated and synchronized across
+runtimes efficiently.
+
+Perspective is a multi-language platform. The examples in this section use
+Python and JavaScript as an example, but the same general principles apply to
+any `Client`/`Server` combination.
+
+<img src="./architecture/architecture.svg" />

docs/md/explanation/architecture/client_only.md

@@ -0,0 +1,39 @@
+# Client-only
+
+<img src="./architecture.sub1.svg" />
+
+_For static datasets, datasets provided by the user, and simple server-less and
+read-only web applications._
+
+In this design, Perspective is run as a client Browser WebAssembly library, the
+dataset is downloaded entirely to the client and all calculations and UI
+interactions are performed locally. Interactive performance is very good, using
+WebAssembly engine for near-native runtime plus WebWorker isolation for parallel
+rendering within the browser. Operations like scrolling and creating new views
+are responsive. However, the entire dataset must be downloaded to the client.
+Perspective is not a typical browser component, and datset sizes of 1gb+ in
+Apache Arrow format will load fine with good interactive performance!
+
+Horizontal scaling is a non-issue, since here is no concurrent state to scale,
+and only uses client-side computation via WebAssembly client. Client-only
+perspective can support as many concurrent users as can download the web
+application itself. Once the data is loaded, no server connection is needed and
+all operations occur in the client browser, imparting no additional runtime cost
+on the server beyond initial load. This also means updates and edits are local
+to the browser client and will be lost when the page is refreshed, unless
+otherwise persisted by your application.
+
+As the client-only design starts with creating a client-side Perspective
+`Table`, data can be provided by any standard web service in any Perspective
+compatible format (JSON, CSV or Apache Arrow).
+
+#### Javascript client
+
+```javascript
+const worker = await perspective.worker();
+const table = await worker.table(csv);
+
+const viewer = document.createElement("perspective-viewer");
+document.body.appendChild(viewer);
+await viewer.load(table);
+```

docs/md/explanation/architecture/client_server.md

@@ -0,0 +1,58 @@
+# Client/Server replicated
+
+<img src="./architecture.sub2.svg" />
+
+_For medium-sized, real-time, synchronized and/or editable data sets with many
+concurrent users._
+
+The dataset is instantiated in-memory with a Python or Node.js Perspective
+server, and web applications create duplicates of these tables in a local
+WebAssembly client in the browser, synchonized efficiently to the server via
+Apache Arrow. This design scales well with additional concurrent users, as
+browsers only need to download the initial data set and subsequent update
+deltas, while operations like scrolling, pivots, sorting, etc. are performed on
+the client.
+
+Python servers can make especially good use of additional threads, as
+Perspective will release the GIL for almost all operations. Interactive
+performance on the client is very good and identical to client-only
+architecture. Updates and edits are seamlessly synchonized across clients via
+their virtual server counterparts using websockets and Apache Arrow.
+
+#### Python and Tornado server
+
+```python
+from perspective import Server, PerspectiveTornadoHandler
+
+server = Server()
+client = server.new_local_client()
+client.table(csv, name="my_table")
+routes = [(
+    r"/websocket",
+    perspective.handlers.tornado.PerspectiveTornadoHandler,
+    {"perspective_server": server},
+)]
+
+app = tornado.web.Application(routes)
+app.listen(8080)
+loop = tornado.ioloop.IOLoop.current()
+loop.start()
+```
+
+#### Javascript client
+
+Perspective's websocket client interfaces with the Python server. then
+_replicates_ the server-side Table.
+
+```javascript
+const websocket = await perspective.websocket("ws://localhost:8080");
+const server_table = await websocket.open_table("my_table");
+const server_view = await server_table.view();
+
+const worker = await perspective.worker();
+const client_table = await worker.table(server_view);
+
+const viewer = document.createElement("perspective-viewer");
+document.body.appendChild(viewer);
+await viewer.load(client_table);
+```

docs/md/explanation/architecture/server_only.md

@@ -0,0 +1,33 @@
+# Server-only
+
+<img src="./architecture.sub3.svg" />
+
+_For extremely large datasets with a small number of concurrent users._
+
+The dataset is instantiated in-memory with a Python or Node.js server, and web
+applications connect virtually. Has very good initial load performance, since no
+data is downloaded. Group-by and other operations will run column-parallel if
+configured.
+
+But interactive performance is poor, as every user interaction must page the
+server to render. Operations like scrolling are not as responsive and can be
+impacted by network latency. Web applications must be "always connected" to the
+server via WebSocket. Disconnecting will prevent any interaction, scrolling,
+etc. of the UI. Does not use WebAssembly.
+
+Each connected browser will impact server performance as long as the connection
+is open, which in turn impacts interactive performance of every client. This
+ultimately limits the horizontal scalabity of this architecture. Since each
+client reads the perspective `Table` virtually, changes like edits and updates
+are automatically reflected to all clients and persist across browser refresh.
+Using the same Python server as the previous design, we can simply skip the
+intermediate WebAssembly `Table` and pass the virtual table directly to `load()`
+
+```javascript
+const websocket = await perspective.websocket("ws://localhost:8080");
+const server_table = await websocket.open_table("my_table");
+
+const viewer = document.createElement("perspective-viewer");
+document.body.appendChild(viewer);
+await viewer.load(server_table);
+```

docs/md/explanation/javascript.md

@@ -0,0 +1,15 @@
+Perspective's JavaScript library offers a configurable UI powered by the same
+fast streaming data engine, just re-compiled to WebAssembly. A simple example
+which loads an [Apache Arrow](https://arrow.apache.org/) and computes a "Group
+By" operation, returning a new Arrow:
+
+```javascript
+import perspective from "@finos/perspective";
+
+const table = await perspective.table(apache_arrow_data);
+const view = await table.view({ group_by: ["CounterParty", "Security"] });
+const arrow = await view.to_arrow();
+```
+
+[More Examples](https://github.com/finos/perspective/tree/master/examples) are
+available on GitHub.

docs/md/explanation/javascript_builds.md

@@ -0,0 +1,42 @@
+# JavaScript Builds
+
+Perspective requires the browser to have access to Perspective's `.wasm`
+binaries _in addition_ to the bundled `.js` files, and as a result the build
+process requires a few extra steps. To ease integration, Perspective's NPM
+releases come with multiple prebuilt configurations.
+
+## Browser
+
+### ESM Builds
+
+The recommended builds for production use are packaged as ES Modules and require
+a _bootstrapping_ step in order to acquire the `.wasm` binaries and initialize
+Perspective's JavaScript with them. However, because they have no hard-coded
+dependencies on the `.wasm` paths, they are ideal for use with JavaScript
+bundlers such as ESBuild, Rollup, Vite or Webpack.
+
+### CDN Builds <!-- Explanation -->
+
+Perspective's CDN builds are good for non-bundled scenarios, such as importing
+directly from a `<script>` tag with a browser-side `import`. CDN builds _do not_
+require _bootstrapping_ the WebAssembly binaries, but they also generally _do
+not_ work with bundlers such as `WebPack`.
+
+### Inline Builds <!-- Explanation -->
+
+<span class="warning">Inline builds are deprecated and will be removed in a
+future release.</span>
+
+Perspective's _Inline_ Builds are a last-ditch effort at compatibility. They
+work by _inlining_ WebAssembly binary content as a base64-encoded string. While
+inline builds work with most bundlers and _do not_ require bootstrapping, there
+is an inherent file-size and boot-performance penalty when using this
+inefficient build method.
+
+Prefer your bundler's inlining features and Perspective ESM builds to this one
+where possible.
+
+## Node.js
+
+There is a Node.js build as well for `@finos/perspective` data engine, which
+shouldn't require any special instructions to use.