From 0507bd60633aab866fbc48cbd3762b3f9e13f1a8 Mon Sep 17 00:00:00 2001 From: Igor Dykhta Date: Tue, 22 Oct 2024 21:10:01 +0300 Subject: [PATCH] [faat] deckgl-arrow-layers module (#2680) * deckgl-arrow-layers-module Signed-off-by: Ihor Dykhta --- .../js/lib/keplergl/kepler.gl.js | 12 +- package.json | 1 + src/deckgl-arrow-layers/babel.config.js | 51 +++ src/deckgl-arrow-layers/package.json | 61 +++ src/deckgl-arrow-layers/src/constants.ts | 20 + src/deckgl-arrow-layers/src/index.ts | 8 + .../src/layers/geo-arrow-arc-layer.ts | 201 ++++++++++ .../src/layers/geo-arrow-scatterplot-layer.ts | 264 +++++++++++++ .../src/layers/geo-arrow-text-layer.ts | 263 +++++++++++++ src/deckgl-arrow-layers/src/types.ts | 73 ++++ src/deckgl-arrow-layers/src/utils/picking.ts | 66 ++++ src/deckgl-arrow-layers/src/utils/utils.ts | 365 ++++++++++++++++++ src/deckgl-arrow-layers/src/utils/validate.ts | 70 ++++ .../tsconfig.production.json | 28 ++ src/deckgl-arrow-layers/webpack/umd.js | 103 +++++ yarn.lock | 126 +++++- 16 files changed, 1701 insertions(+), 11 deletions(-) create mode 100644 src/deckgl-arrow-layers/babel.config.js create mode 100644 src/deckgl-arrow-layers/package.json create mode 100644 src/deckgl-arrow-layers/src/constants.ts create mode 100644 src/deckgl-arrow-layers/src/index.ts create mode 100644 src/deckgl-arrow-layers/src/layers/geo-arrow-arc-layer.ts create mode 100644 src/deckgl-arrow-layers/src/layers/geo-arrow-scatterplot-layer.ts create mode 100644 src/deckgl-arrow-layers/src/layers/geo-arrow-text-layer.ts create mode 100644 src/deckgl-arrow-layers/src/types.ts create mode 100644 src/deckgl-arrow-layers/src/utils/picking.ts create mode 100644 src/deckgl-arrow-layers/src/utils/utils.ts create mode 100644 src/deckgl-arrow-layers/src/utils/validate.ts create mode 100644 src/deckgl-arrow-layers/tsconfig.production.json create mode 100644 src/deckgl-arrow-layers/webpack/umd.js diff --git a/bindings/kepler.gl-jupyter/js/lib/keplergl/kepler.gl.js b/bindings/kepler.gl-jupyter/js/lib/keplergl/kepler.gl.js index b80865d22f..d322b971b8 100644 --- a/bindings/kepler.gl-jupyter/js/lib/keplergl/kepler.gl.js +++ b/bindings/kepler.gl-jupyter/js/lib/keplergl/kepler.gl.js @@ -142,8 +142,9 @@ class KeplerGlJupyter { }, data: { fields: d.fields, - // rows: d.allData - ...(d.dataContainer instanceof ArrowDataContainer ? {cols: d.dataContainer._cols} : {rows: d.allData}) + ...(d.dataContainer instanceof ArrowDataContainer + ? {cols: d.dataContainer._cols} + : {rows: d.allData}) } })), config, @@ -153,12 +154,7 @@ class KeplerGlJupyter { } } -export function addDataConfigToKeplerGl({ - data: inputData, - config, - options, - store -}) { +export function addDataConfigToKeplerGl({data: inputData, config, options, store}) { const data = inputData ? dataToDatasets(inputData) : []; log(data); diff --git a/package.json b/package.json index 1da461868a..c78ff37caf 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,7 @@ "./src/styles", "./src/localization", "./src/deckgl-layers", + "./src/deckgl-arrow-layers", "./src/layers", "./src/schemas", "./src/table", diff --git a/src/deckgl-arrow-layers/babel.config.js b/src/deckgl-arrow-layers/babel.config.js new file mode 100644 index 0000000000..eb16706b69 --- /dev/null +++ b/src/deckgl-arrow-layers/babel.config.js @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: MIT +// Copyright contributors to the kepler.gl project + +const KeplerPackage = require('./package'); + +const PRESETS = ['@babel/preset-env', '@babel/preset-react', '@babel/preset-typescript']; +const PLUGINS = [ + ['@babel/plugin-transform-typescript', {isTSX: true, allowDeclareFields: true}], + '@babel/plugin-transform-modules-commonjs', + '@babel/plugin-transform-class-properties', + '@babel/plugin-transform-optional-chaining', + '@babel/plugin-transform-logical-assignment-operators', + '@babel/plugin-transform-nullish-coalescing-operator', + '@babel/plugin-transform-export-namespace-from', + [ + '@babel/transform-runtime', + { + regenerator: true + } + ], + [ + 'search-and-replace', + { + rules: [ + { + search: '__PACKAGE_VERSION__', + replace: KeplerPackage.version + } + ] + } + ] +]; +const ENV = { + test: { + plugins: ['istanbul'] + }, + debug: { + sourceMaps: 'inline', + retainLines: true + } +}; + +module.exports = function babel(api) { + api.cache(true); + + return { + presets: PRESETS, + plugins: PLUGINS, + env: ENV + }; +}; diff --git a/src/deckgl-arrow-layers/package.json b/src/deckgl-arrow-layers/package.json new file mode 100644 index 0000000000..dd58464440 --- /dev/null +++ b/src/deckgl-arrow-layers/package.json @@ -0,0 +1,61 @@ +{ + "name": "@kepler.gl/deckgl-arrow-layers", + "author": "Shan He ", + "version": "3.0.0", + "description": "Deck.gl layers with GeoArrow and GeoParquet support", + "license": "MIT", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "keywords": [ + "babel", + "es6", + "react", + "webgl", + "visualization", + "deck.gl" + ], + "repository": { + "type": "git", + "url": "https://github.com/keplergl/kepler.gl.git" + }, + "scripts": { + "build": "rm -fr dist && babel src --out-dir dist --source-maps inline --extensions '.ts,.tsx,.js,.jsx' --ignore '**/*.d.ts'", + "build:umd": "NODE_OPTIONS=--openssl-legacy-provider webpack --config ./webpack/umd.js --progress --env.prod", + "build:types": "tsc --project ./tsconfig.production.json", + "prepublish": "babel-node ../../scripts/license-header/bin --license ../../FILE-HEADER && yarn build && yarn build:types", + "stab": "mkdir -p dist && touch dist/index.js" + }, + "files": [ + "dist", + "umd" + ], + "dependencies": { + "@geoarrow/geoarrow-js": "^0.3.0", + "@math.gl/core": "^4.0.0", + "@math.gl/polygon": "^4.0.0", + "@math.gl/types": "^4.0.0", + "apache-arrow": ">=15", + "threads": "^1.7.0" + }, + "peerDependencies": { + "@deck.gl/aggregation-layers": "^8.9.27", + "@deck.gl/core": "^8.9.27", + "@deck.gl/geo-layers": "^8.9.27", + "@deck.gl/layers": "^8.9.27" + }, + "nyc": { + "sourceMap": false, + "instrument": false + }, + "maintainers": [ + "Shan He " + ], + "engines": { + "node": ">=18" + }, + "volta": { + "node": "18.18.2", + "yarn": "4.4.0" + }, + "packageManager": "yarn@4.4.0" +} diff --git a/src/deckgl-arrow-layers/src/constants.ts b/src/deckgl-arrow-layers/src/constants.ts new file mode 100644 index 0000000000..96e195b46a --- /dev/null +++ b/src/deckgl-arrow-layers/src/constants.ts @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: MIT +// Copyright contributors to the kepler.gl project + +// deck.gl-community +// SPDX-License-Identifier: MIT +// Copyright (c) vis.gl contributors + +/** + * Enum holding GeoArrow extension type names + */ +export enum EXTENSION_NAME { + POINT = 'geoarrow.point', + LINESTRING = 'geoarrow.linestring', + POLYGON = 'geoarrow.polygon', + MULTIPOINT = 'geoarrow.multipoint', + MULTILINESTRING = 'geoarrow.multilinestring', + MULTIPOLYGON = 'geoarrow.multipolygon' +} + +export const DEFAULT_COLOR: [number, number, number, number] = [0, 0, 0, 255]; diff --git a/src/deckgl-arrow-layers/src/index.ts b/src/deckgl-arrow-layers/src/index.ts new file mode 100644 index 0000000000..d5352f441a --- /dev/null +++ b/src/deckgl-arrow-layers/src/index.ts @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: MIT +// Copyright contributors to the kepler.gl project + +export {EXTENSION_NAME} from './constants'; + +export {GeoArrowScatterplotLayer} from './layers/geo-arrow-scatterplot-layer'; +export {GeoArrowTextLayer} from './layers/geo-arrow-text-layer'; +export {GeoArrowArcLayer} from './layers/geo-arrow-arc-layer'; diff --git a/src/deckgl-arrow-layers/src/layers/geo-arrow-arc-layer.ts b/src/deckgl-arrow-layers/src/layers/geo-arrow-arc-layer.ts new file mode 100644 index 0000000000..c2dea7a29b --- /dev/null +++ b/src/deckgl-arrow-layers/src/layers/geo-arrow-arc-layer.ts @@ -0,0 +1,201 @@ +// deck.gl-community +// SPDX-License-Identifier: MIT +// Copyright (c) vis.gl contributors + +import { + CompositeLayer, + CompositeLayerProps, + DefaultProps, + GetPickingInfoParams, + Layer, + LayersList, + assert +} from '@deck.gl/core/typed'; +import {ArcLayer} from '@deck.gl/layers/typed'; +import type {ArcLayerProps} from '@deck.gl/layers/typed'; +import * as arrow from 'apache-arrow'; +import * as ga from '@geoarrow/geoarrow-js'; +import {assignAccessor, extractAccessorsFromProps} from '../utils/utils'; +import {child} from '@geoarrow/geoarrow-js'; +import {GeoArrowExtraPickingProps, computeChunkOffsets, getPickingInfo} from '../utils/picking'; +import {ColorAccessor, FloatAccessor, GeoArrowPickingInfo} from '../types'; +import {validateAccessors} from '../utils/validate'; + +/** All properties supported by GeoArrowArcLayer */ +export type GeoArrowArcLayerProps = Omit< + ArcLayerProps, + | 'data' + | 'getSourcePosition' + | 'getTargetPosition' + | 'getSourceColor' + | 'getTargetColor' + | 'getWidth' + | 'getHeight' + | 'getTilt' +> & + _GeoArrowArcLayerProps & + CompositeLayerProps; + +/** Properties added by GeoArrowArcLayer */ +type _GeoArrowArcLayerProps = { + data: arrow.Table; + + /** + * Method called to retrieve the source position of each object. + */ + getSourcePosition: ga.vector.PointVector; + + /** + * Method called to retrieve the target position of each object. + */ + getTargetPosition: ga.vector.PointVector; + + /** + * The rgba color is in the format of `[r, g, b, [a]]`. + * @default [0, 0, 0, 255] + */ + getSourceColor?: ColorAccessor; + + /** + * The rgba color is in the format of `[r, g, b, [a]]`. + * @default [0, 0, 0, 255] + */ + getTargetColor?: ColorAccessor; + + /** + * The line width of each object, in units specified by `widthUnits`. + * @default 1 + */ + getWidth?: FloatAccessor; + + /** + * Multiplier of layer height. `0` will make the layer flat. + * @default 1 + */ + getHeight?: FloatAccessor; + + /** + * Use to tilt the arc to the side if you have multiple arcs with the same source and target positions. + * @default 0 + */ + getTilt?: FloatAccessor; + + /** + * If `true`, validate the arrays provided (e.g. chunk lengths) + * @default true + */ + _validate?: boolean; +}; + +// Remove data from the upstream default props +const { + data: _data, + getSourcePosition: _getSourcePosition, + getTargetPosition: _getTargetPosition, + ..._defaultProps +} = ArcLayer.defaultProps; + +// Default props added by us +const ourDefaultProps = { + _validate: true +}; + +// @ts-expect-error +const defaultProps: DefaultProps = { + ..._defaultProps, + ...ourDefaultProps +}; + +export class GeoArrowArcLayer extends CompositeLayer< + GeoArrowArcLayerProps & ExtraProps +> { + static defaultProps = defaultProps; + static layerName = 'GeoArrowArcLayer'; + + getPickingInfo( + params: GetPickingInfoParams & { + sourceLayer: {props: GeoArrowExtraPickingProps}; + } + ): GeoArrowPickingInfo { + return getPickingInfo(params, this.props.data); + } + + renderLayers(): Layer | LayersList | null { + return this._renderLayersPoint(); + } + + _renderLayersPoint(): Layer | LayersList | null { + const { + data: table, + getSourcePosition: sourcePosition, + getTargetPosition: targetPosition + } = this.props; + + if (this.props._validate) { + validateAccessors(this.props, table); + + // Note: below we iterate over table batches anyways, so this layer won't + // work as-is if data/table is null + assert(ga.vector.isPointVector(sourcePosition)); + assert(ga.vector.isPointVector(targetPosition)); + } + + // Exclude manually-set accessors + const [accessors, otherProps] = extractAccessorsFromProps(this.props, [ + 'getSourcePosition', + 'getTargetPosition' + ]); + const tableOffsets = computeChunkOffsets(table.data); + + const layers: ArcLayer[] = []; + for (let recordBatchIdx = 0; recordBatchIdx < table.batches.length; recordBatchIdx++) { + const sourceData = sourcePosition.data[recordBatchIdx]; + const sourceValues = child.getPointChild(sourceData).values; + const targetData = targetPosition.data[recordBatchIdx]; + const targetValues = child.getPointChild(targetData).values; + + const props: ArcLayerProps = { + // Note: because this is a composite layer and not doing the rendering + // itself, we still have to pass in our defaultProps + ...ourDefaultProps, + ...otherProps, + + // used for picking purposes + recordBatchIdx, + tableOffsets, + + id: `${this.props.id}-geoarrow-arc-${recordBatchIdx}`, + data: { + // @ts-expect-error passed through to enable use by function accessors + data: table.batches[recordBatchIdx], + length: sourceData.length, + attributes: { + getSourcePosition: { + value: sourceValues, + size: sourceData.type.listSize + }, + getTargetPosition: { + value: targetValues, + size: targetData.type.listSize + } + } + } + }; + + for (const [propName, propInput] of Object.entries(accessors)) { + assignAccessor({ + props, + propName, + propInput, + chunkIdx: recordBatchIdx + }); + } + + const SubLayerClass = this.getSubLayerClass('geo-arrow-arc-layer', ArcLayer); + const layer = new SubLayerClass(this.getSubLayerProps(props)); + layers.push(layer); + } + + return layers; + } +} diff --git a/src/deckgl-arrow-layers/src/layers/geo-arrow-scatterplot-layer.ts b/src/deckgl-arrow-layers/src/layers/geo-arrow-scatterplot-layer.ts new file mode 100644 index 0000000000..2f2cca2fbd --- /dev/null +++ b/src/deckgl-arrow-layers/src/layers/geo-arrow-scatterplot-layer.ts @@ -0,0 +1,264 @@ +// SPDX-License-Identifier: MIT +// Copyright contributors to the kepler.gl project + +// deck.gl-community +// SPDX-License-Identifier: MIT +// Copyright (c) vis.gl contributors + +import { + CompositeLayer, + CompositeLayerProps, + DefaultProps, + GetPickingInfoParams, + Layer, + LayersList, + assert +} from '@deck.gl/core/typed'; +import {ScatterplotLayer} from '@deck.gl/layers/typed'; +import type {ScatterplotLayerProps} from '@deck.gl/layers/typed'; +import * as arrow from 'apache-arrow'; +import * as ga from '@geoarrow/geoarrow-js'; +import { + assignAccessor, + extractAccessorsFromProps, + getGeometryVector, + invertOffsets +} from '../utils/utils'; +import {GeoArrowExtraPickingProps, computeChunkOffsets, getPickingInfo} from '../utils/picking'; +import {ColorAccessor, FloatAccessor, GeoArrowPickingInfo} from '../types'; +import {EXTENSION_NAME} from '../constants'; +import {validateAccessors} from '../utils/validate'; + +/** All properties supported by GeoArrowScatterplotLayer */ +export type GeoArrowScatterplotLayerProps = Omit< + ScatterplotLayerProps, + 'data' | 'getPosition' | 'getRadius' | 'getFillColor' | 'getLineColor' +> & + _GeoArrowScatterplotLayerProps & + CompositeLayerProps; + +/** Properties added by GeoArrowScatterplotLayer */ +type _GeoArrowScatterplotLayerProps = { + data: arrow.Table; + + /** + * If `true`, validate the arrays provided (e.g. chunk lengths) + * @default true + */ + _validate?: boolean; + /** + * Center position accessor. + * If not provided, will be inferred by finding a column with extension type + * `"geoarrow.point"` or `"geoarrow.multipoint"`. + */ + getPosition?: ga.vector.PointVector | ga.vector.MultiPointVector; + /** + * Radius accessor. + * @default 1 + */ + getRadius?: FloatAccessor; + /** + * Fill color accessor. + * @default [0, 0, 0, 255] + */ + getFillColor?: ColorAccessor; + /** + * Stroke color accessor. + * @default [0, 0, 0, 255] + */ + getLineColor?: ColorAccessor; + /** + * Stroke width accessor. + * @default 1 + */ + getLineWidth?: FloatAccessor; +}; + +// Remove data and getPosition from the upstream default props +const { + data: _data, + getPosition: _getPosition, + ..._upstreamDefaultProps +} = ScatterplotLayer.defaultProps; + +// Default props added by us +const ourDefaultProps = { + _validate: true +}; + +// @ts-expect-error +const defaultProps: DefaultProps = { + ..._upstreamDefaultProps, + ...ourDefaultProps +}; + +export class GeoArrowScatterplotLayer extends CompositeLayer< + GeoArrowScatterplotLayerProps & ExtraProps +> { + static defaultProps = defaultProps; + static layerName = 'GeoArrowScatterplotLayer'; + + getPickingInfo( + params: GetPickingInfoParams & { + sourceLayer: {props: GeoArrowExtraPickingProps}; + } + ): GeoArrowPickingInfo { + return getPickingInfo(params, this.props.data); + } + + renderLayers(): Layer | LayersList | null { + const {data: table} = this.props; + + if (this.props.getPosition !== undefined) { + const geometryColumn = this.props.getPosition; + if (geometryColumn !== undefined && ga.vector.isPointVector(geometryColumn)) { + return this._renderLayersPoint(geometryColumn); + } + + if (geometryColumn !== undefined && ga.vector.isMultiPointVector(geometryColumn)) { + return this._renderLayersMultiPoint(geometryColumn); + } + + throw new Error('getPosition should pass in an arrow Vector of Point or MultiPoint type'); + } else { + const pointVector = getGeometryVector(table, EXTENSION_NAME.POINT); + if (pointVector !== null) { + return this._renderLayersPoint(pointVector); + } + + const multiPointVector = getGeometryVector(table, EXTENSION_NAME.MULTIPOINT); + if (multiPointVector !== null) { + return this._renderLayersMultiPoint(multiPointVector); + } + } + + throw new Error('getPosition not GeoArrow point or multipoint'); + } + + _renderLayersPoint(geometryColumn: ga.vector.PointVector): Layer | LayersList | null { + const {data: table} = this.props; + + if (this.props._validate) { + assert(ga.vector.isPointVector(geometryColumn)); + validateAccessors(this.props, table); + } + + // Exclude manually-set accessors + const [accessors, otherProps] = extractAccessorsFromProps(this.props, ['getPosition']); + const tableOffsets = computeChunkOffsets(table.data); + + const layers: ScatterplotLayer[] = []; + for (let recordBatchIdx = 0; recordBatchIdx < table.batches.length; recordBatchIdx++) { + const geometryData = geometryColumn.data[recordBatchIdx]; + const flatCoordsData = ga.child.getPointChild(geometryData); + const flatCoordinateArray = flatCoordsData.values; + + const props: ScatterplotLayerProps = { + // Note: because this is a composite layer and not doing the rendering + // itself, we still have to pass in our defaultProps + ...ourDefaultProps, + ...otherProps, + + // used for picking purposes + recordBatchIdx, + tableOffsets, + + id: `${this.props.id}-geoarrow-scatterplot-${recordBatchIdx}`, + data: { + // @ts-expect-error + data: table.batches[recordBatchIdx], + length: geometryData.length, + attributes: { + getPosition: { + value: flatCoordinateArray, + size: geometryData.type.listSize + } + } + } + }; + + for (const [propName, propInput] of Object.entries(accessors)) { + assignAccessor({ + props, + propName, + propInput, + chunkIdx: recordBatchIdx + }); + } + + const layer = new ScatterplotLayer(this.getSubLayerProps(props)); + layers.push(layer); + } + + return layers; + } + + _renderLayersMultiPoint( + geometryColumn: ga.vector.MultiPointVector + ): Layer | LayersList | null { + const {data: table} = this.props; + + // TODO: validate that if nested, accessor props have the same nesting + // structure as the main geometry column. + if (this.props._validate) { + assert(ga.vector.isMultiPointVector(geometryColumn)); + validateAccessors(this.props, table); + } + + // Exclude manually-set accessors + const [accessors, otherProps] = extractAccessorsFromProps(this.props, ['getPosition']); + const tableOffsets = computeChunkOffsets(table.data); + + const layers: ScatterplotLayer[] = []; + for (let recordBatchIdx = 0; recordBatchIdx < table.batches.length; recordBatchIdx++) { + const multiPointData = geometryColumn.data[recordBatchIdx]; + const pointData = ga.child.getMultiPointChild(multiPointData); + const geomOffsets = multiPointData.valueOffsets; + const flatCoordsData = ga.child.getPointChild(pointData); + const flatCoordinateArray = flatCoordsData.values; + + const props: ScatterplotLayerProps = { + // Note: because this is a composite layer and not doing the rendering + // itself, we still have to pass in our defaultProps + ...ourDefaultProps, + ...otherProps, + + // used for picking purposes + recordBatchIdx, + tableOffsets, + + id: `${this.props.id}-geoarrow-scatterplot-${recordBatchIdx}`, + data: { + // @ts-expect-error + data: table.batches[recordBatchIdx], + // Map from expanded multi-geometry index to original index + // Used both in picking and for function callbacks + invertedGeomOffsets: invertOffsets(geomOffsets), + // Note: this needs to be the length one level down. + length: pointData.length, + attributes: { + getPosition: { + value: flatCoordinateArray, + size: pointData.type.listSize + } + } + } + }; + + for (const [propName, propInput] of Object.entries(accessors)) { + assignAccessor({ + props, + propName, + propInput, + chunkIdx: recordBatchIdx, + geomCoordOffsets: geomOffsets + }); + } + + const layer = new ScatterplotLayer(this.getSubLayerProps(props)); + layers.push(layer); + } + + return layers; + } +} diff --git a/src/deckgl-arrow-layers/src/layers/geo-arrow-text-layer.ts b/src/deckgl-arrow-layers/src/layers/geo-arrow-text-layer.ts new file mode 100644 index 0000000000..089a282e92 --- /dev/null +++ b/src/deckgl-arrow-layers/src/layers/geo-arrow-text-layer.ts @@ -0,0 +1,263 @@ +// deck.gl-community +// SPDX-License-Identifier: MIT +// Copyright (c) vis.gl contributors + +import { + CompositeLayer, + CompositeLayerProps, + DefaultProps, + GetPickingInfoParams, + Layer, + LayersList, + assert +} from '@deck.gl/core/typed'; +import {TextLayer} from '@deck.gl/layers/typed'; +import type {TextLayerProps} from '@deck.gl/layers'; +import * as arrow from 'apache-arrow'; +import * as ga from '@geoarrow/geoarrow-js'; +import { + assignAccessor, + expandArrayToCoords, + extractAccessorsFromProps, + getGeometryVector +} from '../utils/utils'; +import {GeoArrowExtraPickingProps, computeChunkOffsets, getPickingInfo} from '../utils/picking'; +import {ColorAccessor, FloatAccessor, GeoArrowPickingInfo} from '../types'; +import {EXTENSION_NAME} from '../constants'; +import {validateAccessors} from '../utils/validate'; + +/** All properties supported by GeoArrowTextLayer */ +export type GeoArrowTextLayerProps = Omit< + TextLayerProps, + // We remove background for now because there are special requirements for + // using binary attributes with background + // https://deck.gl/docs/api-reference/layers/text-layer#use-binary-attributes-with-background + | 'background' + | 'data' + | 'getBackgroundColor' + | 'getBorderColor' + | 'getBorderWidth' + | 'getText' + | 'getPosition' + | 'getColor' + | 'getSize' + | 'getAngle' + | 'getTextAnchor' + | 'getAlignmentBaseline' + | 'getPixelOffset' +> & + _GeoArrowTextLayerProps & + CompositeLayerProps; + +/** Properties added by GeoArrowTextLayer */ +type _GeoArrowTextLayerProps = { + data: arrow.Table; + + /** Background color accessor. + * @default [255, 255, 255, 255] + */ + getBackgroundColor?: ColorAccessor; + /** Border color accessor. + * @default [0, 0, 0, 255] + */ + getBorderColor?: ColorAccessor; + /** Border width accessor. + * @default 0 + */ + getBorderWidth?: FloatAccessor; + /** + * Label text accessor + */ + getText: arrow.Vector; + /** + * Anchor position accessor + */ + getPosition?: ga.vector.PointVector; + /** + * Label color accessor + * @default [0, 0, 0, 255] + */ + getColor?: ColorAccessor; + /** + * Label size accessor + * @default 32 + */ + getSize?: FloatAccessor; + /** + * Label rotation accessor, in degrees + * @default 0 + */ + getAngle?: FloatAccessor; + /** + * Horizontal alignment accessor + * @default 'middle' + */ + getTextAnchor?: arrow.Vector | 'start' | 'middle' | 'end'; + /** + * Vertical alignment accessor + * @default 'center' + */ + getAlignmentBaseline?: arrow.Vector | 'top' | 'center' | 'bottom'; + /** + * Label offset from the anchor position, [x, y] in pixels + * @default [0, 0] + */ + getPixelOffset?: arrow.Vector> | [number, number]; + + /** + * If `true`, validate the arrays provided (e.g. chunk lengths) + * @default true + */ + _validate?: boolean; +}; + +// Remove data and getPosition from the upstream default props +const { + data: _data, + getPosition: _getPosition, + getText: _getText, + getTextAnchor: _getTextAnchor, + getAlignmentBaseline: _getAlignmentBaseline, + getPixelOffset: _getPixelOffset, + ..._defaultProps +} = TextLayer.defaultProps; + +// Default props added by us +const ourDefaultProps: Pick< + GeoArrowTextLayerProps, + 'getTextAnchor' | 'getAlignmentBaseline' | 'getPixelOffset' | '_validate' +> = { + getTextAnchor: 'middle', + getAlignmentBaseline: 'center', + getPixelOffset: [0, 0], + _validate: true +}; + +// @ts-expect-error Type 'Uint8Array' is not assignable to type 'RGBAColor' +const defaultProps: DefaultProps = { + ..._defaultProps, + ...ourDefaultProps +}; + +export class GeoArrowTextLayer extends CompositeLayer< + GeoArrowTextLayerProps & ExtraProps +> { + static defaultProps = defaultProps; + static layerName = 'GeoArrowTextLayer'; + + getPickingInfo( + params: GetPickingInfoParams & { + sourceLayer: {props: GeoArrowExtraPickingProps}; + } + ): GeoArrowPickingInfo { + return getPickingInfo(params, this.props.data); + } + + renderLayers(): Layer | LayersList | null { + const {data: table} = this.props; + + if (this.props.getPosition !== undefined) { + const geometryColumn = this.props.getPosition; + if (geometryColumn !== undefined && ga.vector.isPointVector(geometryColumn)) { + return this._renderLayersPoint(geometryColumn); + } + + throw new Error('getPosition should pass in an arrow Vector of Point type'); + } else { + const pointVector = getGeometryVector(table, EXTENSION_NAME.POINT); + if (pointVector !== null) { + return this._renderLayersPoint(pointVector); + } + } + + throw new Error('getPosition not GeoArrow point'); + } + + _renderLayersPoint(geometryColumn: ga.vector.PointVector): Layer | LayersList | null { + const {data: table} = this.props; + + if (this.props._validate) { + assert(ga.vector.isPointVector(geometryColumn)); + validateAccessors(this.props, table); + } + + // Exclude manually-set accessors + const [accessors, otherProps] = extractAccessorsFromProps(this.props, [ + 'getPosition', + 'getText' + ]); + const tableOffsets = computeChunkOffsets(table.data); + + const layers: TextLayer[] = []; + for (let recordBatchIdx = 0; recordBatchIdx < table.batches.length; recordBatchIdx++) { + const geometryData = geometryColumn.data[recordBatchIdx]; + const flatCoordsData = ga.child.getPointChild(geometryData); + const flatCoordinateArray = flatCoordsData.values; + const textData = this.props.getText.data[recordBatchIdx]; + const textValues = textData.values; + // ! TODO valueOffsets are only present for string columns + const characterOffsets = textData.valueOffsets; + + const props: TextLayerProps = { + // Note: because this is a composite layer and not doing the rendering + // itself, we still have to pass in our defaultProps + ...ourDefaultProps, + ...otherProps, + + // used for picking purposes + // @ts-expect-error + recordBatchIdx, + tableOffsets, + + id: `${this.props.id}-geoarrow-text-layer-${recordBatchIdx}`, + data: { + data: table.batches[recordBatchIdx], + length: geometryData.length, + startIndices: characterOffsets, + attributes: { + // Positions need to be expanded to be one per character! + getPosition: { + value: expandArrayToCoords( + flatCoordinateArray, + geometryData.type.listSize, + characterOffsets + ), + size: geometryData.type.listSize + }, + // TODO: support non-ascii characters + getText: { + value: textValues + // size: 1, + } + } + }, + // TODO privide more robust data comparators + dataComparator: (d1, d2) => { + return d1.data === d2.data; + }, + _subLayerProps: { + characters: { + dataComparator: (d1, d2) => { + return d1.data === d2.data; + } + } + } + }; + + for (const [propName, propInput] of Object.entries(accessors)) { + assignAccessor({ + props, + propName, + propInput, + chunkIdx: recordBatchIdx, + geomCoordOffsets: characterOffsets + }); + } + + const layer = new TextLayer(this.getSubLayerProps(props)); + layers.push(layer); + } + + return layers; + } +} diff --git a/src/deckgl-arrow-layers/src/types.ts b/src/deckgl-arrow-layers/src/types.ts new file mode 100644 index 0000000000..8ef5af81cb --- /dev/null +++ b/src/deckgl-arrow-layers/src/types.ts @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: MIT +// Copyright contributors to the kepler.gl project + +// deck.gl-community +// SPDX-License-Identifier: MIT +// Copyright (c) vis.gl contributors + +import type {BinaryAttribute, Color, PickingInfo} from '@deck.gl/core/typed'; +import {TypedArray} from '@math.gl/types'; +import * as arrow from 'apache-arrow'; + +/** + * An individual layer's data + */ +export type GeoArrowLayerData = { + data: T; + length: number; + attributes?: Record; +}; + +/** + * Internal type for layer data handling, used in `wrapAccessorFunction`. + */ +export type _GeoArrowInternalLayerData = GeoArrowLayerData & { + /** + * A lookup table from expanded multi-geometry index to original index. + * + * This is omitted from the user-facing type because in `wrapAccessorFunction` + * in `utils.ts` we apply the lookup from "exploded" row to "original" row. + */ + invertedGeomOffsets?: Uint8Array | Uint16Array | Uint32Array; +}; + +export type AccessorContext = { + /** The current row index of the current iteration */ + index: number; + /** The value of the `data` prop */ + data: GeoArrowLayerData; + /** A pre-allocated array. The accessor function can optionally fill data into this array and return it, + * instead of creating a new array for every object. In some browsers this improves performance significantly + * by reducing garbage collection. */ + target: number[]; +}; + +/** + * Internal type for layer data handling, used in `wrapAccessorFunction`. + */ +export type _InternalAccessorContext = AccessorContext & { + /** The value of the `data` prop */ + data: _GeoArrowInternalLayerData; +}; + +/** Function that returns a value for each object. */ +export type AccessorFunction = ( + /** Contextual information of the current element. */ + objectInfo: AccessorContext +) => Out; + +/** Either a uniform value for all objects, or a function that returns a value for each object. */ +export type Accessor = Out | AccessorFunction; + +export type GeoArrowPickingInfo = PickingInfo & { + object?: arrow.StructRowProxy; +}; + +export type FloatAccessor = arrow.Vector | Accessor; +export type TimestampAccessor = arrow.Vector>; +export type ColorAccessor = + | arrow.Vector> + | Accessor; +export type NormalAccessor = + | arrow.Vector> + | Accessor>>; diff --git a/src/deckgl-arrow-layers/src/utils/picking.ts b/src/deckgl-arrow-layers/src/utils/picking.ts new file mode 100644 index 0000000000..cc66a8de75 --- /dev/null +++ b/src/deckgl-arrow-layers/src/utils/picking.ts @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: MIT +// Copyright contributors to the kepler.gl project + +// deck.gl-community +// SPDX-License-Identifier: MIT +// Copyright (c) vis.gl contributors + +import * as arrow from 'apache-arrow'; +import {GetPickingInfoParams} from '@deck.gl/core/typed'; +import {GeoArrowPickingInfo} from '../types'; + +export interface GeoArrowExtraPickingProps { + recordBatchIdx: number; + tableOffsets: Uint32Array; + data: { + invertedGeomOffsets?: Uint8Array | Uint16Array | Uint32Array; + }; +} + +export function getPickingInfo( + { + info, + sourceLayer + }: GetPickingInfoParams & { + sourceLayer: {props: GeoArrowExtraPickingProps}; + }, + table: arrow.Table +): GeoArrowPickingInfo { + // Geometry index as rendered + let index = info.index; + + // if a Multi- geometry dataset, map from the rendered index back to the + // feature index + if (sourceLayer.props.data.invertedGeomOffsets) { + index = sourceLayer.props.data.invertedGeomOffsets[index]; + } + + const recordBatchIdx = sourceLayer.props.recordBatchIdx; + const tableOffsets = sourceLayer.props.tableOffsets; + + const batch = table.batches[recordBatchIdx]; + const row = batch.get(index); + if (row === null) { + return info; + } + + const currentBatchOffset = tableOffsets[recordBatchIdx]; + + // Update index to be _global_ index, not within the specific record batch + index += currentBatchOffset; + return { + ...info, + index, + object: row + }; +} + +// This is vendored from Arrow JS because it's a private API +export function computeChunkOffsets( + chunks: ReadonlyArray> +) { + return chunks.reduce((offsets, chunk, index) => { + offsets[index + 1] = offsets[index] + chunk.length; + return offsets; + }, new Uint32Array(chunks.length + 1)); +} diff --git a/src/deckgl-arrow-layers/src/utils/utils.ts b/src/deckgl-arrow-layers/src/utils/utils.ts new file mode 100644 index 0000000000..f257bbd270 --- /dev/null +++ b/src/deckgl-arrow-layers/src/utils/utils.ts @@ -0,0 +1,365 @@ +// SPDX-License-Identifier: MIT +// Copyright contributors to the kepler.gl project + +// deck.gl-community +// SPDX-License-Identifier: MIT +// Copyright (c) vis.gl contributors + +import {assert} from '@deck.gl/core/typed'; +import * as arrow from 'apache-arrow'; +import * as ga from '@geoarrow/geoarrow-js'; +import {AccessorContext, AccessorFunction, _InternalAccessorContext} from '../types'; + +export type TypedArray = + | Uint8Array + | Uint8ClampedArray + | Uint16Array + | Uint32Array + | Int8Array + | Int16Array + | Int32Array + | Float32Array + | Float64Array; + +export function findGeometryColumnIndex( + schema: arrow.Schema, + extensionName: string, + geometryColumnName?: string | null +): number | null { + const index = schema.fields.findIndex( + field => + field.name === geometryColumnName || + field.metadata.get('ARROW:extension:name') === extensionName + ); + return index !== -1 ? index : null; +} + +/** + * Returns `true` if the input is a reference to a column in the table + */ +export function isColumnReference(input: any): input is string { + return typeof input === 'string'; +} + +function isDataInterleavedCoords( + data: arrow.Data +): data is arrow.Data> { + // TODO: also check 2 or 3d? Float64? + return data.type instanceof arrow.FixedSizeList; +} + +function isDataSeparatedCoords( + data: arrow.Data +): data is arrow.Data> { + // TODO: also check child names? Float64? + return data.type instanceof arrow.Struct; +} + +/** + * Convert geoarrow Struct coordinates to FixedSizeList coords + * + * The GeoArrow spec allows for either separated or interleaved coords, but at + * this time deck.gl only supports interleaved. + */ +// TODO: this hasn't been tested yet +// @ts-expect-error +function convertStructToFixedSizeList( + coords: + | arrow.Data> + | arrow.Data> +): arrow.Data> { + if (isDataInterleavedCoords(coords)) { + return coords; + } else if (isDataSeparatedCoords(coords)) { + // TODO: support 3d + const interleavedCoords = new Float64Array(coords.length * 2); + const [xChild, yChild] = coords.children; + for (let i = 0; i < coords.length; i++) { + interleavedCoords[i * 2] = xChild.values[i]; + interleavedCoords[i * 2 + 1] = yChild.values[i]; + } + + const childDataType = new arrow.Float64(); + const dataType = new arrow.FixedSizeList(2, new arrow.Field('coords', childDataType)); + + const interleavedCoordsData = arrow.makeData({ + type: childDataType, + length: interleavedCoords.length + }); + + const data = arrow.makeData({ + type: dataType, + length: coords.length, + nullCount: coords.nullCount, + nullBitmap: coords.nullBitmap, + child: interleavedCoordsData + }); + return data; + } + + assert(false); +} + +type AssignAccessorProps = { + /** The object on which to assign the resolved accesor */ + props: Record; + /** The name of the prop to set */ + propName: string; + /** The user-supplied input to the layer. Must either be a scalar value or a reference to a column in the table. */ + propInput: any; + /** Numeric index in the table */ + chunkIdx: number; + /** a map from the geometry index to the coord offsets for that geometry. */ + geomCoordOffsets?: Int32Array | null; +}; + +/** + * A wrapper around a user-provided accessor function + * + * For layers like Scatterplot, Path, and Polygon, we automatically handle + * "exploding" the table when multi-geometry input are provided. This means that + * the upstream `index` value passed to the user will be the correct row index + * _only_ for non-exploded data. + * + * With this function, we simplify the user usage by automatically converting + * back from "exploded" index back to the original row index. + */ +function wrapAccessorFunction( + objectInfo: _InternalAccessorContext, + userAccessorFunction: AccessorFunction +): Out { + const {index, data} = objectInfo; + let newIndex = index; + if (data.invertedGeomOffsets !== undefined) { + newIndex = data.invertedGeomOffsets[index]; + } + const newObjectData = { + data: data.data, + length: data.length, + attributes: data.attributes + }; + const newObjectInfo = { + index: newIndex, + data: newObjectData, + target: objectInfo.target + }; + return userAccessorFunction(newObjectInfo); +} + +/** + * Resolve accessor and assign to props object + * + * This is useful as a helper function because a scalar prop is set at the top + * level while a vectorized prop is set inside data.attributes + * + */ +export function assignAccessor(args: AssignAccessorProps) { + const {props, propName, propInput, chunkIdx, geomCoordOffsets} = args; + + if (propInput === undefined) { + return; + } + + if (propInput instanceof arrow.Vector) { + const columnData = propInput.data[chunkIdx]; + + if (arrow.DataType.isFixedSizeList(columnData)) { + assert(columnData.children.length === 1); + let values = columnData.children[0].values; + + if (geomCoordOffsets) { + values = expandArrayToCoords(values, columnData.type.listSize, geomCoordOffsets); + } + + props.data.attributes[propName] = { + value: values, + size: columnData.type.listSize, + // Set to `true` to signify that colors are already 0-255, and deck/luma + // does not need to rescale + // https://github.com/visgl/deck.gl/blob/401d624c0529faaa62125714c376b3ba3b8f379f/docs/api-reference/core/attribute-manager.md?plain=1#L66 + normalized: true + }; + } else if (arrow.DataType.isFloat(columnData)) { + let values = columnData.values; + + if (geomCoordOffsets) { + values = expandArrayToCoords(values, 1, geomCoordOffsets); + } + + props.data.attributes[propName] = { + value: values, + size: 1 + }; + } + } else if (typeof propInput === 'function') { + props[propName] = (object: any, objectInfo: AccessorContext) => { + // Special case that doesn't have the same parameters + if (propName === 'getPolygonOffset') { + return propInput(object, objectInfo); + } + + return wrapAccessorFunction(objectInfo, propInput); + }; + } else { + props[propName] = propInput; + } +} + +/** + * Expand an array from "one element per geometry" to "one element per coordinate" + * + * @param input: the input array to expand + * @param size : the number of nested elements in the input array per geometry. So for example, for RGB data this would be 3, for RGBA this would be 4. For radius, this would be 1. + * @param geomOffsets : an offsets array mapping from the geometry to the coordinate indexes. So in the case of a LineStringArray, this is retrieved directly from the GeoArrow storage. In the case of a PolygonArray, this comes from the resolved indexes that need to be given to the SolidPolygonLayer anyways. + * + * @return {TypedArray} values expanded to be per-coordinate + */ +export function expandArrayToCoords( + input: T, + size: number, + geomOffsets: Int32Array +): T { + const numCoords = geomOffsets[geomOffsets.length - 1]; + // @ts-expect-error + const outputArray: T = new input.constructor(numCoords * size); + + // geomIdx is an index into the geomOffsets array + // geomIdx is also the geometry/table index + for (let geomIdx = 0; geomIdx < geomOffsets.length - 1; geomIdx++) { + // geomOffsets maps from the geometry index to the coord index + // So here we get the range of coords that this geometry covers + const lastCoordIdx = geomOffsets[geomIdx]; + const nextCoordIdx = geomOffsets[geomIdx + 1]; + + // Iterate over this range of coord indices + for (let coordIdx = lastCoordIdx; coordIdx < nextCoordIdx; coordIdx++) { + // Iterate over size + for (let i = 0; i < size; i++) { + // Copy from the geometry index in `input` to the coord index in + // `output` + outputArray[coordIdx * size + i] = input[geomIdx * size + i]; + } + } + } + + return outputArray; +} + +/** + * Get a geometry vector with the specified extension type name from the table. + */ +export function getGeometryVector( + table: arrow.Table, + geoarrowTypeName: string +): arrow.Vector | null { + const geometryColumnIdx = findGeometryColumnIndex(table.schema, geoarrowTypeName); + + if (geometryColumnIdx === null) { + return null; + // throw new Error(`No column found with extension type ${geoarrowTypeName}`); + } + + return table.getChildAt(geometryColumnIdx); +} + +export function getListNestingLevels(data: arrow.Data): number { + let nestingLevels = 0; + if (arrow.DataType.isList(data.type)) { + nestingLevels += 1; + data = data.children[0]; + } + return nestingLevels; +} + +export function getMultiLineStringResolvedOffsets(data: ga.data.MultiLineStringData): Int32Array { + const geomOffsets = data.valueOffsets; + const lineStringData = ga.child.getMultiLineStringChild(data); + const ringOffsets = lineStringData.valueOffsets; + + const resolvedRingOffsets = new Int32Array(geomOffsets.length); + for (let i = 0; i < resolvedRingOffsets.length; ++i) { + // Perform the lookup into the ringIndices array using the geomOffsets + // array + resolvedRingOffsets[i] = ringOffsets[geomOffsets[i]]; + } + + return resolvedRingOffsets; +} + +export function getPolygonResolvedOffsets(data: ga.data.PolygonData): Int32Array { + const geomOffsets = data.valueOffsets; + const ringData = ga.child.getPolygonChild(data); + const ringOffsets = ringData.valueOffsets; + + const resolvedRingOffsets = new Int32Array(geomOffsets.length); + for (let i = 0; i < resolvedRingOffsets.length; ++i) { + // Perform the lookup into the ringIndices array using the geomOffsets + // array + resolvedRingOffsets[i] = ringOffsets[geomOffsets[i]]; + } + + return resolvedRingOffsets; +} + +export function getMultiPolygonResolvedOffsets(data: ga.data.MultiPolygonData): Int32Array { + const polygonData = ga.child.getMultiPolygonChild(data); + const ringData = ga.child.getPolygonChild(polygonData); + + const geomOffsets = data.valueOffsets; + const polygonOffsets = polygonData.valueOffsets; + const ringOffsets = ringData.valueOffsets; + + const resolvedRingOffsets = new Int32Array(geomOffsets.length); + for (let i = 0; i < resolvedRingOffsets.length; ++i) { + resolvedRingOffsets[i] = ringOffsets[polygonOffsets[geomOffsets[i]]]; + } + + return resolvedRingOffsets; +} + +/** + * Invert offsets so that lookup can go in the opposite direction + */ +export function invertOffsets(offsets: Int32Array): Uint8Array | Uint16Array | Uint32Array { + const largestOffset = offsets[offsets.length - 1]; + + const arrayConstructor = + offsets.length < Math.pow(2, 8) + ? Uint8Array + : offsets.length < Math.pow(2, 16) + ? Uint16Array + : Uint32Array; + + const invertedOffsets = new arrayConstructor(largestOffset); + for (let arrayIdx = 0; arrayIdx < offsets.length - 1; arrayIdx++) { + const thisOffset = offsets[arrayIdx]; + const nextOffset = offsets[arrayIdx + 1]; + for (let offset = thisOffset; offset < nextOffset; offset++) { + invertedOffsets[offset] = arrayIdx; + } + } + + return invertedOffsets; +} + +// TODO: better typing +export function extractAccessorsFromProps( + props: Record, + excludeKeys: string[] +): [Record, Record] { + const accessors: Record = {}; + const otherProps: Record = {}; + for (const [key, value] of Object.entries(props)) { + if (excludeKeys.includes(key)) { + continue; + } + + if (key.startsWith('get')) { + accessors[key] = value; + } else { + otherProps[key] = value; + } + } + + return [accessors, otherProps]; +} diff --git a/src/deckgl-arrow-layers/src/utils/validate.ts b/src/deckgl-arrow-layers/src/utils/validate.ts new file mode 100644 index 0000000000..e9f52972c0 --- /dev/null +++ b/src/deckgl-arrow-layers/src/utils/validate.ts @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: MIT +// Copyright contributors to the kepler.gl project + +// deck.gl-community +// SPDX-License-Identifier: MIT +// Copyright (c) vis.gl contributors + +import {assert} from '@deck.gl/core/typed'; +import * as arrow from 'apache-arrow'; + +export function validateAccessors(props: Record, table: arrow.Table): void { + const vectorAccessors: arrow.Vector[] = []; + const colorVectorAccessors: arrow.Vector[] = []; + for (const [accessorName, accessorValue] of Object.entries(props)) { + // Is it an accessor + if (accessorName.startsWith('get')) { + // Is it a vector accessor + if (accessorValue instanceof arrow.Vector) { + vectorAccessors.push(accessorValue); + + // Is it a color vector accessor + if (accessorName.endsWith('Color')) { + colorVectorAccessors.push(accessorValue); + } + } + } + } + + validateVectorAccessors(table, vectorAccessors); + for (const colorVectorAccessor of colorVectorAccessors) { + validateColorVector(colorVectorAccessor); + } +} + +/** + * Provide validation for accessors provided + * + * - Assert that all vectors have the same number of chunks as the main table + * - Assert that all chunks in each vector have the same number of rows as the + * relevant batch in the main table. + * + */ +export function validateVectorAccessors(table: arrow.Table, vectorAccessors: arrow.Vector[]) { + // Check the same number of chunks as the table's batches + for (const vectorAccessor of vectorAccessors) { + assert(table.batches.length === vectorAccessor.data.length); + } + + // Check that each table batch/vector data has the same number of rows + for (const vectorAccessor of vectorAccessors) { + for (let i = 0; i < table.batches.length; i++) { + assert(table.batches[i].numRows === vectorAccessor.data[i].length); + } + } +} + +export function validateColorVector(vector: arrow.Vector) { + // Assert the color vector is a FixedSizeList + assert(arrow.DataType.isFixedSizeList(vector.type)); + + // Assert it has 3 or 4 values + assert(vector.type.listSize === 3 || vector.type.listSize === 4); + + // Assert the child type is an integer + assert(arrow.DataType.isInt(vector.type.children[0])); + + // Assert the child type is a Uint8 + // Property 'type' does not exist on type 'Int_'. Did you mean 'TType'? + assert(vector.type.children[0].type.bitWidth === 8); +} diff --git a/src/deckgl-arrow-layers/tsconfig.production.json b/src/deckgl-arrow-layers/tsconfig.production.json new file mode 100644 index 0000000000..7abf03b517 --- /dev/null +++ b/src/deckgl-arrow-layers/tsconfig.production.json @@ -0,0 +1,28 @@ +{ + "compilerOptions": { + "target": "es2020", + "allowJs": false, + "checkJs": false, + "jsx": "react", + "module": "esnext", + "moduleResolution": "node", + "declaration": true, + "emitDeclarationOnly": true, + "noImplicitAny": false, + "noImplicitReturns": true, + "noImplicitThis": true, + "noUnusedLocals": true, + "outDir": "dist", + "sourceMap": true, + "strictNullChecks": true, + "suppressImplicitAnyIndexErrors": false, + "esModuleInterop": true, + "allowSyntheticDefaultImports": true, + "skipLibCheck": true, + "strict": true, + "resolveJsonModule": true, + "isolatedModules": true, + "baseUrl": "./src" //TODO change once all dependencies are isolated + }, + "include": ["src"] +} diff --git a/src/deckgl-arrow-layers/webpack/umd.js b/src/deckgl-arrow-layers/webpack/umd.js new file mode 100644 index 0000000000..c7c03dd3cf --- /dev/null +++ b/src/deckgl-arrow-layers/webpack/umd.js @@ -0,0 +1,103 @@ +// SPDX-License-Identifier: MIT +// Copyright contributors to the kepler.gl project + +const resolve = require('path').resolve; +const join = require('path').join; + +// Import package.json to read version +const KeplerPackage = require('../package'); + +const SRC_DIR = resolve(__dirname, '../src'); +const OUTPUT_DIR = resolve(__dirname, '../umd'); + +const LIBRARY_BUNDLE_CONFIG = () => ({ + entry: { + KeplerGl: join(SRC_DIR, 'index.ts') + }, + + // Silence warnings about big bundles + stats: { + warnings: false + }, + + output: { + // Generate the bundle in dist folder + path: OUTPUT_DIR, + filename: 'keplergl.min.js', + globalObject: 'this', + library: '[name]', + libraryTarget: 'umd' + }, + + // let's put everything in + externals: { + react: { + root: 'React', + commonjs2: 'react', + commonjs: 'react', + amd: 'react', + umd: 'react' + }, + 'react-dom': { + root: 'ReactDOM', + commonjs2: 'react-dom', + commonjs: 'react-dom', + amd: 'react-dom', + umd: 'react-dom' + }, + redux: { + root: 'Redux', + commonjs2: 'redux', + commonjs: 'redux', + amd: 'redux', + umd: 'redux' + }, + 'react-redux': { + root: 'ReactRedux', + commonjs2: 'react-redux', + commonjs: 'react-redux', + amd: 'react-redux', + umd: 'react-redux' + }, + 'styled-components': { + commonjs: 'styled-components', + commonjs2: 'styled-components', + amd: 'styled-components', + root: 'styled' + } + }, + resolve: { + extensions: ['.tsx', '.ts', '.js'], + modules: ['node_modules', SRC_DIR] + }, + module: { + rules: [ + { + test: /\.(js|ts|tsx)$/, + loader: 'babel-loader', + include: [SRC_DIR], + options: { + plugins: [ + [ + 'search-and-replace', + { + rules: [ + { + search: '__PACKAGE_VERSION__', + replace: KeplerPackage.version + } + ] + } + ] + ] + } + } + ] + }, + + node: { + fs: 'empty' + } +}); + +module.exports = env => LIBRARY_BUNDLE_CONFIG(env); diff --git a/yarn.lock b/yarn.lock index 6b22ef4f37..4510326656 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2411,6 +2411,19 @@ __metadata: languageName: node linkType: hard +"@geoarrow/geoarrow-js@npm:^0.3.0": + version: 0.3.1 + resolution: "@geoarrow/geoarrow-js@npm:0.3.1" + dependencies: + "@math.gl/polygon": "npm:^4.0.0" + proj4: "npm:^2.9.2" + threads: "npm:^1.7.0" + peerDependencies: + apache-arrow: ">=15" + checksum: 10c0/0f7cb53ab43651cdca5ab7e0055ba1a7a3b1ce69f86c6ddabc2d5375b7426895a12ea3b35e0c2aba208b0660b916c7c5e7f7635c0830e7a5d1fc4a03146b0907 + languageName: node + linkType: hard + "@hubble.gl/core@npm:1.2.0-alpha.6": version: 1.2.0-alpha.6 resolution: "@hubble.gl/core@npm:1.2.0-alpha.6" @@ -2946,6 +2959,24 @@ __metadata: languageName: unknown linkType: soft +"@kepler.gl/deckgl-arrow-layers@workspace:src/deckgl-arrow-layers": + version: 0.0.0-use.local + resolution: "@kepler.gl/deckgl-arrow-layers@workspace:src/deckgl-arrow-layers" + dependencies: + "@geoarrow/geoarrow-js": "npm:^0.3.0" + "@math.gl/core": "npm:^4.0.0" + "@math.gl/polygon": "npm:^4.0.0" + "@math.gl/types": "npm:^4.0.0" + apache-arrow: "npm:>=15" + threads: "npm:^1.7.0" + peerDependencies: + "@deck.gl/aggregation-layers": ^8.9.27 + "@deck.gl/core": ^8.9.27 + "@deck.gl/geo-layers": ^8.9.27 + "@deck.gl/layers": ^8.9.27 + languageName: unknown + linkType: soft + "@kepler.gl/deckgl-layers@npm:3.0.0, @kepler.gl/deckgl-layers@workspace:src/deckgl-layers": version: 0.0.0-use.local resolution: "@kepler.gl/deckgl-layers@workspace:src/deckgl-layers" @@ -3935,6 +3966,13 @@ __metadata: languageName: node linkType: hard +"@math.gl/types@npm:^4.0.0": + version: 4.1.0 + resolution: "@math.gl/types@npm:4.1.0" + checksum: 10c0/3c4dfa5ac5c9e2cef24d31f56b89c1dde785a5d70fd1a7030386346cb7dd4fa2cce5ba983b89842c1971492e30870dd22a078d64893f9c66887e38367bf992fa + languageName: node + linkType: hard + "@math.gl/web-mercator@npm:^3.5.1, @math.gl/web-mercator@npm:^3.5.5, @math.gl/web-mercator@npm:^3.6.2": version: 3.6.3 resolution: "@math.gl/web-mercator@npm:3.6.3" @@ -6511,7 +6549,7 @@ __metadata: languageName: node linkType: hard -"apache-arrow@npm:>= 15.0.0, apache-arrow@npm:>=15.0.0": +"apache-arrow@npm:>= 15.0.0, apache-arrow@npm:>=15, apache-arrow@npm:>=15.0.0": version: 17.0.0 resolution: "apache-arrow@npm:17.0.0" dependencies: @@ -8065,7 +8103,7 @@ __metadata: languageName: node linkType: hard -"callsites@npm:^3.0.0": +"callsites@npm:^3.0.0, callsites@npm:^3.1.0": version: 3.1.0 resolution: "callsites@npm:3.1.0" checksum: 10c0/fff92277400eb06c3079f9e74f3af120db9f8ea03bad0e84d9aede54bbe2d44a56cccb5f6cf12211f93f52306df87077ecec5b712794c5a9b5dac6d615a3f301 @@ -9565,6 +9603,18 @@ __metadata: languageName: node linkType: hard +"debug@npm:^4.2.0": + version: 4.3.7 + resolution: "debug@npm:4.3.7" + dependencies: + ms: "npm:^2.1.3" + peerDependenciesMeta: + supports-color: + optional: true + checksum: 10c0/1471db19c3b06d485a622d62f65947a19a23fbd0dd73f7fd3eafb697eec5360cde447fb075919987899b1a2096e85d35d4eb5a4de09a57600ac9cf7e6c8e768b + languageName: node + linkType: hard + "debug@npm:^4.3.2": version: 4.3.6 resolution: "debug@npm:4.3.6" @@ -11194,6 +11244,13 @@ __metadata: languageName: node linkType: hard +"esm@npm:^3.2.25": + version: 3.2.25 + resolution: "esm@npm:3.2.25" + checksum: 10c0/8e60e8075506a7ce28681c30c8f54623fe18a251c364cd481d86719fc77f58aa055b293d80632d9686d5408aaf865ffa434897dc9fd9153c8b3f469fad23f094 + languageName: node + linkType: hard + "espree@npm:^9.6.0, espree@npm:^9.6.1": version: 9.6.1 resolution: "espree@npm:9.6.1" @@ -14153,6 +14210,13 @@ __metadata: languageName: node linkType: hard +"is-observable@npm:^2.1.0": + version: 2.1.0 + resolution: "is-observable@npm:2.1.0" + checksum: 10c0/f6ae9e136f66ad59c4faa4661112c389b398461cdeb0ef5bc3c505989469b77b2ba4602e2abc54a635d65f616eec9b5a40cd7d2c1f96b2cc4748b56635eba1c6 + languageName: node + linkType: hard + "is-path-cwd@npm:^2.0.0": version: 2.2.0 resolution: "is-path-cwd@npm:2.2.0" @@ -16534,6 +16598,13 @@ __metadata: languageName: node linkType: hard +"mgrs@npm:1.0.0": + version: 1.0.0 + resolution: "mgrs@npm:1.0.0" + checksum: 10c0/a360853be5a3b4f4734dbf0a193851c08039ccb6077362ab0f890cccd170ef88b59585129757da9fea4d7a313d7dc3ee88f37f594fc1ae3e4e8efc5353144d77 + languageName: node + linkType: hard + "micromark@npm:~2.11.0": version: 2.11.4 resolution: "micromark@npm:2.11.4" @@ -17034,7 +17105,7 @@ __metadata: languageName: node linkType: hard -"ms@npm:2.1.3, ms@npm:^2.0.0, ms@npm:^2.1.1": +"ms@npm:2.1.3, ms@npm:^2.0.0, ms@npm:^2.1.1, ms@npm:^2.1.3": version: 2.1.3 resolution: "ms@npm:2.1.3" checksum: 10c0/d924b57e7312b3b63ad21fc5b3dc0af5e78d61a1fc7cfb5457edaf26326bf62be5307cc87ffb6862ef1c2b33b0233cdb5d4f01c4c958cc0d660948b65a287a48 @@ -17689,6 +17760,13 @@ __metadata: languageName: node linkType: hard +"observable-fns@npm:^0.6.1": + version: 0.6.1 + resolution: "observable-fns@npm:0.6.1" + checksum: 10c0/bf25d5b3e4040233e886800c48b347361d9c7a1179f345590e671c2dd5ea9b4447bd5037f8ed40b2bb6fd7e205f0c5450eff15f48efdf91b3dec027007cf2834 + languageName: node + linkType: hard + "obuf@npm:^1.0.0, obuf@npm:^1.1.2": version: 1.1.2 resolution: "obuf@npm:1.1.2" @@ -18817,6 +18895,16 @@ __metadata: languageName: node linkType: hard +"proj4@npm:^2.9.2": + version: 2.12.1 + resolution: "proj4@npm:2.12.1" + dependencies: + mgrs: "npm:1.0.0" + wkt-parser: "npm:^1.3.3" + checksum: 10c0/0a6f5f87137aa4297f02a95fab41f2cf5d39924f0e9800999be90e901b2d77a0b4fcab44a0c664839f4828c5a433fe81ec4c270a8fce62728f980ab4c8becdd4 + languageName: node + linkType: hard + "promise-inflight@npm:^1.0.1": version: 1.0.1 resolution: "promise-inflight@npm:1.0.1" @@ -22651,6 +22739,22 @@ __metadata: languageName: node linkType: hard +"threads@npm:^1.7.0": + version: 1.7.0 + resolution: "threads@npm:1.7.0" + dependencies: + callsites: "npm:^3.1.0" + debug: "npm:^4.2.0" + is-observable: "npm:^2.1.0" + observable-fns: "npm:^0.6.1" + tiny-worker: "npm:>= 2" + dependenciesMeta: + tiny-worker: + optional: true + checksum: 10c0/34114075c5eb253e937c01e0e51babb71eb1571c2fd4a8170645aa20c10186422324537d017d64bfd3de655cbb6647bf70d1d602260273b1e9750719d67f359a + languageName: node + linkType: hard + "through2-filter@npm:^3.0.0": version: 3.1.0 resolution: "through2-filter@npm:3.1.0" @@ -22742,6 +22846,15 @@ __metadata: languageName: node linkType: hard +"tiny-worker@npm:>= 2": + version: 2.3.0 + resolution: "tiny-worker@npm:2.3.0" + dependencies: + esm: "npm:^3.2.25" + checksum: 10c0/3106cace86e673216426a517e96fb72ce642ba79002554e4c6bceb585ba77cf5e5e68b452c752cada6136ae94fdbf11c56943a70de6c6bc6a2a3a9ae439746c9 + languageName: node + linkType: hard + "tinycolor2@npm:^1.4.1": version: 1.6.0 resolution: "tinycolor2@npm:1.6.0" @@ -24571,6 +24684,13 @@ __metadata: languageName: node linkType: hard +"wkt-parser@npm:^1.3.3": + version: 1.3.3 + resolution: "wkt-parser@npm:1.3.3" + checksum: 10c0/900fcdaea02828407742c4ea370b46a04c48ed95f63cde3db8e5b90c5a1b80da9827b43eb58940dd8cf790a971f9b8c42022b1548bce29a11153e699eef5bcb4 + languageName: node + linkType: hard + "word-wrap@npm:^1.2.5": version: 1.2.5 resolution: "word-wrap@npm:1.2.5"