[Feat] support GeoArrow format

examples/webpack.config.local.js

@@ -146,6 +146,12 @@ function makeLocalDevConfig(env, EXAMPLE_DIR = LIB_DIR, externals = {}) {
           use: ['source-map-loader'],
           enforce: 'pre',
           exclude: [/node_modules\/react-palm/, /node_modules\/react-data-grid/]
+        },
+        // for compiling apache-arrow ESM module
+        {
+          test: /\.mjs$/,
+          include: /node_modules\/apache-arrow/,
+          type: 'javascript/auto'
         }
       ]
     },

jest.setup.js

@@ -20,6 +20,7 @@
 
 import '@testing-library/jest-dom';
 import * as Utils from '@kepler.gl/utils';
+require('@loaders.gl/polyfills');
 
 jest.mock('mapbox-gl/dist/mapbox-gl', () => ({
   Map: () => ({})

package.json

@@ -112,7 +112,7 @@
     "@hubble.gl/core": "1.2.0-alpha.6",
     "@hubble.gl/react": "1.2.0-alpha.6",
     "@kepler.gl/components": "3.0.0-alpha.1",
-    "@loaders.gl/polyfills": "^4.0.0",
+    "@loaders.gl/polyfills": "^4.0.2",
     "@types/mapbox__geo-viewport": "^0.4.1",
     "@typescript-eslint/parser": "^5.27.0",
     "eslint-config-developit": "^1.2.0",
@@ -140,7 +140,7 @@
     "@babel/traverse": "^7.12.1",
     "@cfaester/enzyme-adapter-react-18": "^0.7.0",
     "@deck.gl/test-utils": "^8.9.12",
-    "@loaders.gl/polyfills": "^4.0.0",
+    "@loaders.gl/polyfills": "^4.0.2",
     "@luma.gl/test-utils": "^8.5.19",
     "@nebula.gl/layers": "1.0.2-alpha.1",
     "@probe.gl/env": "^3.5.0",
@@ -214,12 +214,12 @@
     "webpack-stats-plugin": "^0.2.1"
   },
   "resolutions": {
-    "@loaders.gl/core": "^4.0.0",
-    "@loaders.gl/csv": "^4.0.0",
-    "@loaders.gl/gltf": "^4.0.0",
-    "@loaders.gl/json": "^4.0.0",
-    "@loaders.gl/loader-utils": "^4.0.0",
-    "@loaders.gl/polyfills": "^4.0.0",
+    "@loaders.gl/core": "^4.0.2",
+    "@loaders.gl/csv": "^4.0.2",
+    "@loaders.gl/gltf": "^4.0.2",
+    "@loaders.gl/json": "^4.0.2",
+    "@loaders.gl/loader-utils": "^4.0.2",
+    "@loaders.gl/polyfills": "^4.0.2",
     "@luma.gl/constants": "8.5.19",
     "@luma.gl/core": "8.5.19",
     "@luma.gl/experimental": "8.5.19",

src/constants/src/default-settings.ts

@@ -430,7 +430,8 @@ export const ALL_FIELD_TYPES = keyMirror({
   timestamp: null,
   point: null,
   array: null,
-  object: null
+  object: null,
+  geoarrow: null
 });
 
 // Data Table
@@ -509,6 +510,10 @@ export const FIELD_TYPE_DISPLAY = {
     label: 'geo',
     color: BLUE2
   },
+  [ALL_FIELD_TYPES.geoarrow]: {
+    label: 'geo',
+    color: BLUE2
+  },
   [ALL_FIELD_TYPES.integer]: {
     label: 'int',
     color: YELLOW
@@ -757,6 +762,17 @@ export const FIELD_OPTS = {
       tooltip: []
     }
   },
+  [ALL_FIELD_TYPES.geoarrow]: {
+    type: 'geometry',
+    scale: {
+      ...notSupportedScaleOpts,
+      ...notSupportAggrOpts
+    },
+    format: {
+      legend: d => '...',
+      tooltip: []
+    }
+  },
   [ALL_FIELD_TYPES.object]: {
     type: 'numerical',
     scale: {},
@@ -1122,7 +1138,8 @@ export const DATASET_FORMATS = keyMirror({
   row: null,
   geojson: null,
   csv: null,
-  keplergl: null
+  keplergl: null,
+  arrow: null
 });
 
 export const MAP_CONTROLS = keyMirror({

src/deckgl-layers/babel.config.js

@@ -24,6 +24,7 @@ const PRESETS = ['@babel/preset-env', '@babel/preset-react', '@babel/preset-type
 const PLUGINS = [
   ['@babel/plugin-transform-typescript', {isTSX: true, allowDeclareFields: true}],
   '@babel/plugin-transform-modules-commonjs',
+  '@babel/plugin-proposal-class-properties', // for static class properties
   '@babel/plugin-proposal-optional-chaining',
   [
     '@babel/transform-runtime',

src/deckgl-layers/src/deckgl-extensions/filter-arrow-layer.ts

@@ -0,0 +1,57 @@
+import {Layer, LayerExtension} from '@deck.gl/core';
+import {LayerContext} from '@deck.gl/core/lib/layer';
+import GL from '@luma.gl/constants';
+
+import shaderModule from './filter-shader-module';
+
+const VALUE_FILTERED = 1;
+
+const defaultProps = {
+  getFiltered: {type: 'accessor', value: VALUE_FILTERED}
+};
+
+export type FilterArrowExtensionProps = {
+  getFiltered?: () => number;
+};
+
+/**
+ * FilterArrowExtension - a deck.gl extension to filter arrow layer
+ *
+ * A simple extension to filter arrow layer based on the result of CPU filteredIndex,
+ * so we can avoid filtering on the raw Arrow table and recreating geometry attributes.
+ * Specifically, an attribute `filtered` is added to the layer to indicate whether the feature has been Filtered
+ * the shader module is modified to discard the feature if filtered value is 0
+ * the accessor getFiltered is used to get the value of `filtered` based on the value `filteredIndex` in Arrowlayer
+ */
+export default class FilterArrowExtension extends LayerExtension {
+  static defaultProps = defaultProps;
+  static extensionName = 'FilterArrowExtension';
+
+  getShaders(extension: any) {
+    return {
+      modules: [shaderModule],
+      defines: {}
+    };
+  }
+
+  initializeState(this: Layer<FilterArrowExtensionProps>, context: LayerContext, extension: this) {
+    const attributeManager = this.getAttributeManager();
+    if (attributeManager) {
+      attributeManager.add({
+        filtered: {
+          size: 1,
+          type: GL.FLOAT,
+          accessor: 'getFiltered',
+          shaderAttributes: {
+            filtered: {
+              divisor: 0
+            },
+            instanceFiltered: {
+              divisor: 1
+            }
+          }
+        }
+      });
+    }
+  }
+}

src/deckgl-layers/src/deckgl-extensions/filter-shader-module.ts

@@ -0,0 +1,39 @@
+import {project} from '@deck.gl/core';
+
+const vs = `
+  #ifdef NON_INSTANCED_MODEL
+    #define FILTER_ARROW_ATTRIB filtered
+  #else
+    #define FILTER_ARROW_ATTRIB instanceFiltered
+  #endif
+  attribute float FILTER_ARROW_ATTRIB;
+`;
+
+const fs = ``;
+
+const inject = {
+  'vs:#decl': `
+    varying float is_filtered;
+  `,
+  'vs:#main-end': `
+    is_filtered = FILTER_ARROW_ATTRIB;
+  `,
+  'fs:#decl': `
+    varying float is_filtered;
+  `,
+  'fs:DECKGL_FILTER_COLOR': `
+    // abandon the fragments if  it is not filtered
+    if (is_filtered == 0.) {
+      discard;
+    }
+  `
+};
+
+export default {
+  name: 'filter-arrow',
+  dependencies: [project],
+  vs: vs,
+  fs: fs,
+  inject: inject,
+  getUniforms: () => {}
+};

src/deckgl-layers/src/index.ts

@@ -9,6 +9,7 @@ export {default as EnhancedGridLayer} from './grid-layer/enhanced-cpu-grid-layer
 export {default as EnhancedHexagonLayer} from './hexagon-layer/enhanced-hexagon-layer';
 export {default as EnhancedLineLayer} from './line-layer/line-layer';
 export {default as SvgIconLayer} from './svg-icon-layer/svg-icon-layer';
+export {default as FilterArrowExtension} from './deckgl-extensions/filter-arrow-layer';
 
 export * from './layer-utils/shader-utils';
 

src/layers/package.json

@@ -42,14 +42,15 @@
     "@kepler.gl/table": "3.0.0-alpha.1",
     "@kepler.gl/types": "3.0.0-alpha.1",
     "@kepler.gl/utils": "3.0.0-alpha.1",
-    "@loaders.gl/core": "^4.0.0",
-    "@loaders.gl/gis": "^4.0.0",
-    "@loaders.gl/gltf": "^4.0.0",
-    "@loaders.gl/wkt": "^4.0.0",
+    "@loaders.gl/arrow": "^4.0.2",
+    "@loaders.gl/core": "^4.0.2",
+    "@loaders.gl/gis": "^4.0.2",
+    "@loaders.gl/gltf": "^4.0.2",
+    "@loaders.gl/wkt": "^4.0.2",
     "@luma.gl/constants": "^8.5.19",
     "@mapbox/geojson-normalize": "0.0.1",
-    "@nebula.gl/layers": "1.0.2-alpha.1",
     "@nebula.gl/edit-modes": "1.0.2-alpha.1",
+    "@nebula.gl/layers": "1.0.2-alpha.1",
     "@turf/bbox": "^6.0.1",
     "@turf/helpers": "^6.1.4",
     "@types/geojson": "^7946.0.7",

src/layers/src/base-layer.ts

@@ -1428,6 +1428,16 @@ class Layer {
     // implemented in subclasses
     return () => null;
   }
+
+  getColumnAccessor(dataContainer?: DataContainerInterface): (...args: any[]) => any {
+    // implemented in subclasses
+    return () => null;
+  }
+
+  getFieldAccessor(dataContainer?: DataContainerInterface): (...args: any[]) => any {
+    // implemented in subclasses
+    return () => null;
+  }
 }
 
 export default Layer;