Exit mbtiles read stream if we hit a tile maximum

CHANGELOG.md

@@ -1,3 +1,7 @@
+# 0.5.0
+
+- add `--maxTiles` option which limits the number of mbtiles tiles used when generating stats
+
 # 0.4.0
 
 - Upgrade [email protected]

README.md

@@ -59,21 +59,25 @@ Usage
   Output is logged to the console as a JSON string.
 
 Options
-  --attributes, -a Specify attributes to analyze. The provided value
-                   will be parsed as an array, split on commas.
+  --attributes, -a  Specify attributes to analyze. The provided value
+                    will be parsed as an array, split on commas.
+  --maxTiles, -m    The maximum number of tiles to generate statistics from
+                    an mbtiles file. Default to all tiles. Provided value will be parsed
+                    as an integer.
 
 Example
   mapbox-geostats population-centers.geojson --attributes name,pop > output.json
+  mapbox-geostats cities.mbtiles --maxTiles 100000 > output.json
 ```
 
 ## Node
 
 ```js
-var geostats = require('@mapbox/mapbox-geostats');
+const geostats = require('@mapbox/mapbox-geostats');
 
-geostats(filePath, options).then(function (stats) {
+geostats(filePath, options).then(stats => {
   // Do something with the stats
-}).catch(function (err) {
+}).catch(err => {
   // Do something with the error
 });
 ```
@@ -89,6 +93,7 @@ Returns a Promise that resolves with a stats object, whose structure is describe
 `options` (*optional*) is an optional object that can have the following properties:
 
 - `attributes`: An array of strings identifying attributes that you want analyzed and reported. By default, all attributes are analyzed and reported until we reach the limitations described above.
+- `maxTiles`: An integer setting the maximum number of tiles to count when generating stats from an MBTiles file (does not affect any other formats) - helpful for very large MBTiles files where stats generation takes a long time. Default is to analyze _every_ tile.
 
 ## Output: the stats
 

bin/mapbox-geostats

@@ -12,16 +12,21 @@ const help = `
     Output is logged to the console as a JSON string.
 
   Options
-    --attributes, -a Specify attributes to analyze. The provided value
-                     will be parsed as an array, split on commas.
+    --attributes, -a  Specify attributes to analyze. The provided value
+                      will be parsed as an array, split on commas.
+    --maxTiles, -m    The maximum number of tiles to generate statistics from
+                      an mbtiles file. Default to all tiles. Provided value will be parsed
+                      as an integer.
 
   Example
     mapbox-geostats cities.geojson --attributes name,pop > output.json
+    mapbox-geostats cities.mbtiles --maxTiles 100000 > output.json
   `;
 
 const cli = meow(help, {
   alias: {
     a: 'attributes',
+    m: 'maxTiles'
   },
 });
 
@@ -33,6 +38,10 @@ if (cli.flags.attributes) {
   options.attributes = cli.flags.attributes.split(',');
 }
 
+if (cli.flags.maxTiles) {
+  options.maxTiles = +cli.flags.maxTiles;
+}
+
 if (!input) {
   console.log(cli.help);
 } else {

index.js

@@ -23,6 +23,7 @@ function buildGeoStats(filePath, options) {
     : Constants.ATTRIBUTES_MAX_REPORT;
   options.maxValuesToCount = Math.floor(Constants.VALUES_MAX_COUNT / divisor);
   options.maxValuesToReport = Math.floor(Constants.VALUES_MAX_REPORT / divisor);
+  options.maxTilesToCount = (options.maxTiles) ? options.maxTiles : Constants.MBTILES_MAX_TILE_COUNT;
 
   return getFileType(filePath)
     .then(function (fileType) {

lib/constants.js

@@ -13,6 +13,8 @@ module.exports = {
   VALUE_STRING_MAX_LENGTH: 256,
   NAME_TRUNCATE_LENGTH: 256,
 
+  MBTILES_MAX_TILE_COUNT: Infinity,
+
   FILETYPE_GEOJSON: 'geojson',
   FILETYPE_SHAPEFILE: 'shp',
   FILETYPE_CSV: 'csv',

lib/tile-analyze.js

@@ -64,18 +64,32 @@ module.exports = function(filePath, options) {
 
   function analyzeSourceStream(source) {
     return new Promise((resolve, reject) => {
-      const zxyStream = source.createZXYStream();
+      let count = 1;
+
+      const zxyStream = source.createZXYStream({ maxTiles: 10 });
       const readStream = tilelive.createReadStream(source, { type: 'list' });
+      const analysisStream = new TileAnalyzeStream(analyzeTile);
+
       zxyStream.on('error', reject)
         .pipe(readStream)
-        .pipe(new TileAnalyzeStream(analyzeTile))
+        .pipe(analysisStream)
         .on('error', reject)
         .on('end', () => {
           resolve(Object.assign(stats, {
             layers: _.values(layerMap),
           }));
         })
         .resume();
+
+      // kill the stream when we've hit a maximum number of tiles
+      readStream.on('data', function() {
+        if (count >= options.maxTiles) {
+          readStream.unpipe(analysisStream);
+          readStream.end();
+          analysisStream.end();
+        }
+        count++;
+      });
     });
   }
 

test/test.js

@@ -408,3 +408,23 @@ test('MBTiles with two layers', t => {
     t.end();
   }).catch(t.threw);
 });
+
+test('MBTiles limits max tiles', t => {
+  // the four-tiles.mbtiles fixture has four tiles at z1, each tile has a different
+  // "color" property, meaning if all four tiles are counted, there will be four unique
+  // values for "color"
+  Promise.all([
+    geostats(fixturePath('src/four-tiles.mbtiles'), { maxTiles: 1 }),
+    geostats(fixturePath('src/four-tiles.mbtiles'), { maxTiles: 2 }),
+    geostats(fixturePath('src/four-tiles.mbtiles'), { maxTiles: 3 }),
+    geostats(fixturePath('src/four-tiles.mbtiles'), { maxTiles: 4 }),
+    geostats(fixturePath('src/four-tiles.mbtiles')),
+  ]).then((output) => {
+    t.equal(output[0].layers[0].attributes[0].values.length, 1, 'expected one value');
+    t.equal(output[1].layers[0].attributes[0].values.length, 2, 'expected two values');
+    t.equal(output[2].layers[0].attributes[0].values.length, 3, 'expected three values');
+    t.equal(output[3].layers[0].attributes[0].values.length, 4, 'expected all four values');
+    t.equal(output[4].layers[0].attributes[0].values.length, 4, 'expected all four values');
+    t.end();
+  }).catch(t.threw);
+});