diff --git a/bench/benchmarks/dds_parse_bucket.js b/bench/benchmarks/dds_parse_bucket.js new file mode 100644 index 00000000000..64a74239cf9 --- /dev/null +++ b/bench/benchmarks/dds_parse_bucket.js @@ -0,0 +1,205 @@ +'use strict'; + +var VT = require('vector-tile'); +var Protobuf = require('pbf'); +var assert = require('assert'); + +var WorkerTile = require('../../js/source/worker_tile'); +var Worker = require('../../js/source/worker'); +var ajax = require('../../js/util/ajax'); +var Style = require('../../js/style/style'); +var util = require('../../js/util/util'); +var Evented = require('../../js/util/evented'); +var config = require('../../js/util/config'); +var coordinates = require('../lib/coordinates'); +var formatNumber = require('../lib/format_number'); + +var SAMPLE_COUNT = 10; + +module.exports = function run(options) { + config.ACCESS_TOKEN = options.accessToken; + + var evented = util.extend({}, Evented); + + var stylesheetURL = 'https://api.mapbox.com/styles/v1/mapbox/streets-v9?access_token=' + options.accessToken; + ajax.getJSON(stylesheetURL, function(err, stylesheet) { + if (err) return evented.fire('error', {error: err}); + + evented.fire('log', { + message: 'preloading assets', + color: 'dark' + }); + + stylesheet.layers.forEach(function (layer) { + if (layer.type === 'fill' && layer.paint) { + layer.paint['fill-color'] = { + property: 'level', + stops: [[0, 'white'], [100, 'blue']] + }; + } + }); + + preloadAssets(stylesheet, function(err, assets) { + if (err) return evented.fire('error', {error: err}); + + evented.fire('log', { + message: 'starting first test', + color: 'dark' + }); + + function getGlyphs(params, callback) { + callback(null, assets.glyphs[JSON.stringify(params)]); + } + + function getIcons(params, callback) { + callback(null, assets.icons[JSON.stringify(params)]); + } + + function getTile(url, callback) { + callback(null, assets.tiles[url]); + } + + var timeSum = 0; + var timeCount = 0; + + asyncTimesSeries(SAMPLE_COUNT, function(callback) { + runSample(stylesheet, getGlyphs, getIcons, getTile, function(err, time) { + if (err) return evented.fire('error', { error: err }); + timeSum += time; + timeCount++; + evented.fire('log', { message: formatNumber(time) + ' ms' }); + callback(); + }); + }, function(err) { + if (err) { + evented.fire('error', { error: err }); + + } else { + var timeAverage = timeSum / timeCount; + evented.fire('end', { + message: formatNumber(timeAverage) + ' ms', + score: timeAverage + }); + } + }); + }); + + }); + + return evented; +}; + +function preloadAssets(stylesheet, callback) { + var assets = { + glyphs: {}, + icons: {}, + tiles: {} + }; + + var style = new Style(stylesheet); + + style.on('load', function() { + function getGlyphs(params, callback) { + style['get glyphs'](params, function(err, glyphs) { + assets.glyphs[JSON.stringify(params)] = glyphs; + callback(err, glyphs); + }); + } + + function getIcons(params, callback) { + style['get icons'](params, function(err, icons) { + assets.icons[JSON.stringify(params)] = icons; + callback(err, icons); + }); + } + + function getTile(url, callback) { + ajax.getArrayBuffer(url, function(err, response) { + assets.tiles[url] = response; + callback(err, response); + }); + } + + style.update([], {transition: true}); + + runSample(stylesheet, getGlyphs, getIcons, getTile, function(err) { + style._remove(); + callback(err, assets); + }); + }); + + style.on('error', function(event) { + callback(event.error); + }); + +} + +function runSample(stylesheet, getGlyphs, getIcons, getTile, callback) { + var timeStart = performance.now(); + + var layerFamilies = createLayerFamilies(stylesheet.layers); + + util.asyncAll(coordinates, function(coordinate, eachCallback) { + var url = 'https://a.tiles.mapbox.com/v4/mapbox.mapbox-terrain-v2,mapbox.mapbox-streets-v6/' + coordinate.zoom + '/' + coordinate.row + '/' + coordinate.column + '.vector.pbf?access_token=' + config.ACCESS_TOKEN; + + var workerTile = new WorkerTile({ + coord: coordinate, + zoom: coordinate.zoom, + tileSize: 512, + overscaling: 1, + angle: 0, + pitch: 0, + showCollisionBoxes: false, + source: 'composite', + uid: url + }); + + var actor = { + send: function(action, params, sendCallback) { + setTimeout(function() { + if (action === 'get icons') { + getIcons(params, sendCallback); + } else if (action === 'get glyphs') { + getGlyphs(params, sendCallback); + } else assert(false); + }, 0); + } + }; + + getTile(url, function(err, response) { + if (err) throw err; + var data = new VT.VectorTile(new Protobuf(new Uint8Array(response))); + workerTile.parse(data, layerFamilies, actor, null, function(err) { + if (err) return callback(err); + eachCallback(); + }); + }); + }, function(err) { + var timeEnd = performance.now(); + callback(err, timeEnd - timeStart); + }); +} + +function asyncTimesSeries(times, work, callback) { + if (times > 0) { + work(function(err) { + if (err) callback(err); + else asyncTimesSeries(times - 1, work, callback); + }); + } else { + callback(); + } +} + +var createLayerFamiliesCacheKey; +var createLayerFamiliesCacheValue; +function createLayerFamilies(layers) { + if (layers !== createLayerFamiliesCacheKey) { + var worker = new Worker({addEventListener: function() {} }); + worker['set layers'](layers); + + createLayerFamiliesCacheKey = layers; + createLayerFamiliesCacheValue = worker.layerFamilies; + } + return createLayerFamiliesCacheValue; +} diff --git a/bench/benchmarks/dds_update_bucket.js b/bench/benchmarks/dds_update_bucket.js new file mode 100644 index 00000000000..618e16eb467 --- /dev/null +++ b/bench/benchmarks/dds_update_bucket.js @@ -0,0 +1,241 @@ +'use strict'; + +var VT = require('vector-tile'); +var Protobuf = require('pbf'); +var assert = require('assert'); + +var WorkerTile = require('../../js/source/worker_tile'); +var Worker = require('../../js/source/worker'); +var ajax = require('../../js/util/ajax'); +var Style = require('../../js/style/style'); +var util = require('../../js/util/util'); +var Evented = require('../../js/util/evented'); +var config = require('../../js/util/config'); +var coordinates = require('../lib/coordinates'); +var formatNumber = require('../lib/format_number'); + +var SAMPLE_COUNT = 10; + +module.exports = function run(options) { + config.ACCESS_TOKEN = options.accessToken; + + var evented = util.extend({}, Evented); + + var stylesheetURL = 'https://api.mapbox.com/styles/v1/mapbox/streets-v9?access_token=' + options.accessToken; + ajax.getJSON(stylesheetURL, function(err, stylesheet) { + if (err) return evented.fire('error', {error: err}); + + evented.fire('log', { + message: 'preloading assets', + color: 'dark' + }); + + stylesheet.layers.forEach(function (layer) { + if (layer.type === 'fill' && layer.paint) { + layer.paint['fill-color'] = { + property: 'level', + stops: [[0, 'white'], [100, 'blue']] + }; + } + }); + + preloadAssets(stylesheet, function(err, assets) { + if (err) return evented.fire('error', {error: err}); + + evented.fire('log', { + message: 'starting first test', + color: 'dark' + }); + + function getGlyphs(params, callback) { + callback(null, assets.glyphs[JSON.stringify(params)]); + } + + function getIcons(params, callback) { + callback(null, assets.icons[JSON.stringify(params)]); + } + + function getTile(url, callback) { + callback(null, assets.tiles[url]); + } + + function getWorkerTile(coordinate, url) { + return assets.workerTiles[url]; + } + + var timeSum = 0; + var timeCount = 0; + + asyncTimesSeries(SAMPLE_COUNT, function(callback) { + runSample(stylesheet, getGlyphs, getIcons, getTile, getWorkerTile, function(err, time) { + if (err) return evented.fire('error', { error: err }); + timeSum += time; + timeCount++; + evented.fire('log', { message: formatNumber(time) + ' ms' }); + callback(); + }); + }, function(err) { + if (err) { + evented.fire('error', { error: err }); + + } else { + var timeAverage = timeSum / timeCount; + evented.fire('end', { + message: formatNumber(timeAverage) + ' ms', + score: timeAverage + }); + } + }); + }); + + }); + + return evented; +}; + +function preloadAssets(stylesheet, callback) { + var assets = { + glyphs: {}, + icons: {}, + tiles: {}, + workerTiles: {} + }; + + var style = new Style(stylesheet); + + style.on('load', function() { + function getGlyphs(params, callback) { + style['get glyphs'](params, function(err, glyphs) { + assets.glyphs[JSON.stringify(params)] = glyphs; + callback(err, glyphs); + }); + } + + function getIcons(params, callback) { + style['get icons'](params, function(err, icons) { + assets.icons[JSON.stringify(params)] = icons; + callback(err, icons); + }); + } + + function getTile(url, callback) { + ajax.getArrayBuffer(url, function(err, response) { + assets.tiles[url] = response; + callback(err, response); + }); + } + + function getWorkerTile(coordinate, url) { + var workerTile = assets.workerTiles[url] = new WorkerTile({ + coord: coordinate, + zoom: coordinate.zoom, + tileSize: 512, + overscaling: 1, + angle: 0, + pitch: 0, + showCollisionBoxes: false, + source: 'composite', + uid: url + }); + return workerTile; + } + + style.update([], {transition: true}); + + runSample(stylesheet, getGlyphs, getIcons, getTile, getWorkerTile, function(err) { + style._remove(); + callback(err, assets); + }); + }); + + style.on('error', function(event) { + callback(event.error); + }); + +} + +function runSample(stylesheet, getGlyphs, getIcons, getTile, getWorkerTile, callback) { + var timeStart = performance.now(); + + var layerFamilies = createLayerFamilies(stylesheet.layers); + + util.asyncAll(coordinates, function(coordinate, eachCallback) { + var url = 'https://a.tiles.mapbox.com/v4/mapbox.mapbox-terrain-v2,mapbox.mapbox-streets-v6/' + coordinate.zoom + '/' + coordinate.row + '/' + coordinate.column + '.vector.pbf?access_token=' + config.ACCESS_TOKEN; + + var workerTile = getWorkerTile(coordinate, url); + + var actor = { + send: function(action, params, sendCallback) { + setTimeout(function() { + if (action === 'get icons') { + getIcons(params, sendCallback); + } else if (action === 'get glyphs') { + getGlyphs(params, sendCallback); + } else assert(false); + }, 0); + } + }; + + if (!workerTile.data) { + // preload data + getTile(url, function(err, response) { + if (err) throw err; + var data = new VT.VectorTile(new Protobuf(new Uint8Array(response))); + + // Copy the property data from the vector tile features so we + // can use it in updateProperties() during the actual benchmark + // In the real use case, the updated property data would be + // provided by client code and it wouldn't be pre-tiled, so a + // custom source would have to figure out how to generate this + // tile-specific payload, probably by joining on an id or + // something. + workerTile.__propertyData = {}; + for (var layerId in data.layers) { + var layer = data.layers[layerId]; + workerTile.__propertyData[layerId] = []; + for (var i = 0; i < layer.length; i++) { + var properties = layer.feature(i).properties; + workerTile.__propertyData[layerId].push(properties); + } + } + + workerTile.parse(data, layerFamilies, actor, null, function(err) { + if (err) return callback(err); + eachCallback(); + }); + }); + } else { + workerTile.updateProperties(workerTile.__propertyData, layerFamilies, actor, null, function(err) { + if (err) return callback(err); + eachCallback(); + }); + } + }, function(err) { + var timeEnd = performance.now(); + callback(err, timeEnd - timeStart); + }); +} + +function asyncTimesSeries(times, work, callback) { + if (times > 0) { + work(function(err) { + if (err) callback(err); + else asyncTimesSeries(times - 1, work, callback); + }); + } else { + callback(); + } +} + +var createLayerFamiliesCacheKey; +var createLayerFamiliesCacheValue; +function createLayerFamilies(layers) { + if (layers !== createLayerFamiliesCacheKey) { + var worker = new Worker({addEventListener: function() {} }); + worker['set layers'](layers); + + createLayerFamiliesCacheKey = layers; + createLayerFamiliesCacheValue = worker.layerFamilies; + } + return createLayerFamiliesCacheValue; +} diff --git a/bench/index.js b/bench/index.js index 538df4a5a71..6b5e96a087e 100644 --- a/bench/index.js +++ b/bench/index.js @@ -15,6 +15,8 @@ function main() { var benchmarks = { buffer: require('./benchmarks/buffer'), + 'dds-parse-bucket': require('./benchmarks/dds_parse_bucket'), + 'dds-update-bucket': require('./benchmarks/dds_update_bucket'), fps: require('./benchmarks/fps'), 'frame-duration': require('./benchmarks/frame_duration'), 'query-point': require('./benchmarks/query_point'), diff --git a/js/data/bucket.js b/js/data/bucket.js index e9849cb3a1f..076d529ff1a 100644 --- a/js/data/bucket.js +++ b/js/data/bucket.js @@ -138,6 +138,21 @@ Bucket.prototype.populateArrays = function() { this.trimArrays(); }; +/** + * Update paint arrays with the given feature properties, leaving geometries + * as-is. + * @private + */ +Bucket.prototype.updatePaintArrays = function(interfaceName, propertiesList) { + this.recalculateStyleLayers(); + + for (var i = 0; i < propertiesList.length; i++) { + var range = this._featureIndexToArrayRange[i]; + if (!range) continue; + this.populatePaintArrays(interfaceName, {zoom: this.zoom}, propertiesList[i], range); + } +}; + /** * Check if there is enough space available in the current array group for * `vertexLength` vertices. If not, append a new array group. Should be called @@ -202,6 +217,10 @@ Bucket.prototype.prepareArrayGroup = function(programName, numVertices) { * @private */ Bucket.prototype.createArrays = function() { + // mapping from `feature.index` to the start & end vertex array indexes. + // `feature.index` is the index into the _original_ source layer's feature + // list (as opposed to this bucket's post-filtered list). + this._featureIndexToArrayRange = {}; this.arrayGroups = {}; this.paintVertexArrayTypes = {}; @@ -344,16 +363,34 @@ Bucket.prototype.recalculateStyleLayers = function() { } }; -Bucket.prototype.populatePaintArrays = function(interfaceName, globalProperties, featureProperties, startGroup, startIndex) { + +/** + * @typedef {object} ArrayRange + * @property {number} arrayRange.startGroup + * @property {number} arrayRange.startVertex + * @property {number} arrayRange.endGroup + * @property {number} arrayRange.endVertex + * @private + */ + +Bucket.prototype.populatePaintArrays = function(interfaceName, globalProperties, featureProperties, arrayRange, featureIndex) { + if (typeof featureIndex !== 'undefined') { + this._featureIndexToArrayRange[featureIndex] = arrayRange; + } + for (var l = 0; l < this.childLayers.length; l++) { var layer = this.childLayers[l]; var groups = this.arrayGroups[interfaceName]; - for (var g = startGroup.index; g < groups.length; g++) { + + for (var g = arrayRange.startGroup; g <= arrayRange.endGroup; g++) { var group = groups[g]; var length = group.layoutVertexArray.length; var paintArray = group.paintVertexArrays[layer.id]; paintArray.resize(length); + var start = g === arrayRange.startGroup ? arrayRange.startVertex : 0; + var end = g === arrayRange.endGroup ? arrayRange.endVertex : length - 1; + var attributes = this.paintAttributes[interfaceName][layer.id].attributes; for (var m = 0; m < attributes.length; m++) { var attribute = attributes[m]; @@ -362,8 +399,7 @@ Bucket.prototype.populatePaintArrays = function(interfaceName, globalProperties, var multiplier = attribute.multiplier || 1; var components = attribute.components || 1; - var start = g === startGroup.index ? startIndex : 0; - for (var i = start; i < length; i++) { + for (var i = start; i <= end; i++) { var vertex = paintArray.get(i); for (var c = 0; c < components; c++) { var memberName = components > 1 ? (attribute.name + c) : attribute.name; diff --git a/js/data/bucket/circle_bucket.js b/js/data/bucket/circle_bucket.js index 8971f0eafc3..0716a07a041 100644 --- a/js/data/bucket/circle_bucket.js +++ b/js/data/bucket/circle_bucket.js @@ -116,5 +116,18 @@ CircleBucket.prototype.addFeature = function(feature) { } } - this.populatePaintArrays('circle', globalProperties, feature.properties, startGroup, startIndex); + var groups = this.arrayGroups['circle']; + var range = { + startGroup: startGroup.index, + startVertex: startIndex, + endGroup: groups.length - 1, + endVertex: groups[groups.length - 1].layoutVertexArray.length - 1 + }; + + this.populatePaintArrays('circle', globalProperties, feature.properties, range, feature.index); }; + +CircleBucket.prototype.updateFeatureProperties = function (propertiesList) { + this.updatePaintArrays('circle', propertiesList); +}; + diff --git a/js/data/bucket/fill_bucket.js b/js/data/bucket/fill_bucket.js index 9089a7c2c09..e9819598f07 100644 --- a/js/data/bucket/fill_bucket.js +++ b/js/data/bucket/fill_bucket.js @@ -61,13 +61,22 @@ FillBucket.prototype.addFeature = function(feature) { var polygons = classifyRings(lines, EARCUT_MAX_RINGS); var startGroup = this.prepareArrayGroup('fill', 0); - var startIndex = startGroup.layoutVertexArray.length; + var startVertex = startGroup.layoutVertexArray.length; + + var range = { + startGroup: this.arrayGroups['fill'].length - 1, + startVertex: startVertex + }; for (var i = 0; i < polygons.length; i++) { this.addPolygon(polygons[i]); } - this.populatePaintArrays('fill', {zoom: this.zoom}, feature.properties, startGroup, startIndex); + var endGroupIndex = this.arrayGroups['fill'].length - 1; + range.endGroup = endGroupIndex; + range.endVertex = this.arrayGroups['fill'][endGroupIndex].layoutVertexArray.length - 1; + + this.populatePaintArrays('fill', {zoom: this.zoom}, feature.properties, range, feature.index); }; FillBucket.prototype.addPolygon = function(polygon) { @@ -107,3 +116,8 @@ FillBucket.prototype.addPolygon = function(polygon) { group.elementArray.emplaceBack(triangleIndices[i] + startIndex); } }; + +FillBucket.prototype.updateFeatureProperties = function (propertiesList) { + this.updatePaintArrays('fill', propertiesList); +}; + diff --git a/js/source/worker_tile.js b/js/source/worker_tile.js index f7e70e299ba..cc0b427bd7e 100644 --- a/js/source/worker_tile.js +++ b/js/source/worker_tile.js @@ -110,9 +110,9 @@ WorkerTile.prototype.parse = function(data, layerFamilies, actor, rawTileData, c } } - var buckets = [], + var buckets = this.buckets = [], symbolBuckets = this.symbolBuckets = [], - otherBuckets = []; + otherBuckets = this.otherBuckets = []; featureIndex.bucketLayerIDs = {}; @@ -231,6 +231,56 @@ WorkerTile.prototype.parse = function(data, layerFamilies, actor, rawTileData, c } }; +// Proof of concept for updating only the properties (reusing existing, +// already-parsed geometries) of a vector tile. +// +// PLEASE NOTE: In practice, this might make more sense as part of a separate, +// custom source type, rather than being baked into WorkerTile. It's actually +// pretty independent of the rest of the code here, except that it relies on +// the changes to WorkerTile#parse wherein `this.buckets` and +// `this.otherBuckets` are saved for reuse. +// +// Params analogous to WorkerTile#parse, except that `data`, rather than being +// a `VectorTile`, is an object like: +// { +// source-layer-name: [{ /* feature 0 properties */}, {/* feature 1 properties */}, ...], +// another-layer-name: [...] +// } +// where feature 0, 1, 2 are the features, in order, in this vector tile. + +WorkerTile.prototype.updateProperties = function(data, layerFamilies, actor, rawTileData, callback) { + // load up data cached from initial parse() + var buckets = this.buckets, + otherBuckets = this.otherBuckets; + + var tile = this; + + // immediately parse non-symbol buckets (they have no dependencies) + for (var i = otherBuckets.length - 1; i >= 0; i--) { + if (!otherBuckets[i].updateFeatureProperties) continue; + var properties = data[otherBuckets[i].layer.sourceLayer]; + otherBuckets[i].updateFeatureProperties(properties || []); + } + + // this will probably be async once we include the symbol stuff + done(); + + function done() { + tile.status = 'done'; + + if (tile.redoPlacementAfterDone) { + tile.redoPlacement(tile.angle, tile.pitch, null); + tile.redoPlacementAfterDone = false; + } + + var nonEmptyBuckets = buckets.filter(isBucketNonEmpty); + callback(null, { + buckets: nonEmptyBuckets.map(serializeBucket) + }, getTransferables(nonEmptyBuckets)); + } +}; + + WorkerTile.prototype.redoPlacement = function(angle, pitch, showCollisionBoxes) { if (this.status !== 'done') { this.redoPlacementAfterDone = true; diff --git a/test/js/data/bucket.test.js b/test/js/data/bucket.test.js index 6c6d1691890..ca426a4330d 100644 --- a/test/js/data/bucket.test.js +++ b/test/js/data/bucket.test.js @@ -42,7 +42,13 @@ test('Bucket', function(t) { group.layoutVertexArray.emplaceBack(point.x * 2, point.y * 2); group.elementArray.emplaceBack(1, 2, 3); group.elementArray2.emplaceBack(point.x, point.y); - this.populatePaintArrays('test', {}, feature.properties, group, startIndex); + var range = { + startGroup: group.index, + startVertex: startIndex, + endGroup: group.index, + endVertex: group.layoutVertexArray.length - 1 + }; + this.populatePaintArrays('test', {}, feature.properties, range, feature.index); }; return Class; @@ -319,5 +325,36 @@ test('Bucket', function(t) { t.end(); }); + t.test('update feature properties', function(t) { + var bucket = create(); + + bucket.features = [createFeature(17, 42)]; + // this represents the feature's original, pre-filtered index + bucket.features[0].index = 3; + bucket.populateArrays(); + + + var testVertex = bucket.arrayGroups.test[0].layoutVertexArray; + t.equal(testVertex.length, 1); + var v0 = testVertex.get(0); + t.equal(v0.a_box0, 34); + t.equal(v0.a_box1, 84); + var testPaintVertex = bucket.arrayGroups.test[0].paintVertexArrays.layerid; + t.equal(testPaintVertex.length, 1); + var p0 = testPaintVertex.get(0); + t.equal(p0.a_map, 17); + + bucket.updatePaintArrays('test', [{ x: 0 }, { x: 1 }, { x: 2 }, { x: 3 }]); + + v0 = testVertex.get(0); + p0 = testPaintVertex.get(0); + t.equal(v0.a_box0, 34); + t.equal(v0.a_box1, 84); + t.equal(p0.a_map, 3); + + t.end(); + }); + + t.end(); });