initial commit

.gitignore

@@ -0,0 +1,8 @@
+node_modules
+package-lock.json
+*.bz2
+*.gz
+*.db
+*.tar
+*.geojson
+*.csv

.jshintignore

@@ -0,0 +1 @@
+node_modules

.jshintrc

@@ -0,0 +1,10 @@
+{
+  "node": true,
+
+  "curly": true,
+  "latedef": true,
+  "quotmark": true,
+  "undef": true,
+  "unused": true,
+  "trailing": true
+}

README.md

@@ -0,0 +1,154 @@
+[BETA] This repo is still a work-in-progress.
+
+## CLI
+
+```bas
+wof <cmd> [args]
+
+Commands:
+  wof feature  misc functions for working with feature streams
+  wof git      interact with git repositories
+  wof sqlite   interact with SQLite databases
+
+Options:
+  --version      Show version number                                   [boolean]
+  --verbose, -v  enable verbose logging               [boolean] [default: false]
+  --help         Show help                                             [boolean]
+```
+
+### Git
+
+interact with git repositories
+
+```bash
+  wof git export <repo>  export features from a git repo
+  wof git list <repo>    list paths in a git repo
+```
+
+#### Supported Repository Types
+
+Both 'bare' and 'checkout' style clones are supported, these commands use the `git archive` command and so do not require the geojson files to be present in the filesystem.
+
+If you don't need a working copy of the files you can save disk space by using a `--bare` style clone.
+
+The following command uses only `661M` of disk space, compared to `3.8G` for the equivalent 'checkout' style clone which also extracts ~400k files to the filesystem.
+
+```bash
+git clone \
+  --bare \
+  --depth 1 \
+  --single-branch \
+  --branch master \
+  https://github.com/whosonfirst-data/whosonfirst-data-admin-us.git us-bare
+```
+
+#### Git List
+
+list paths in a git repo
+
+```bash
+wof git list /data/wof
+
+... list of geojson files
+```
+
+```bash
+wof git list /data/wof --path data/101/914/241/101914241.geojson
+
+... list of geojson files matching the path argument
+```
+
+#### Git Export
+
+export features from a git repo
+
+```bash
+wof git export /data/wof
+
+... stream of geojson features
+```
+
+```bash
+wof git export /data/wof --path data/101/914/241/101914241.geojson
+
+... stream of geojson features matching the path argument
+```
+
+### SQLite
+
+interact with SQLite databases
+
+```bash
+  wof sqlite export <db>  export features from a SQLite database
+  wof sqlite import <db>  import features into a SQLite database
+```
+
+#### SQLite Export
+
+export features from a SQLite database
+
+```bash
+wof sqlite export wof.db
+
+... stream of geojson features
+```
+
+#### SQLite Import
+
+import features into a SQLite database
+
+```bash
+cat 101914243.geojson | wof sqlite import --rm wof.db
+```
+
+### Feature Tools
+
+misc functions for working with feature streams
+
+```bash
+  wof feature format      reformat a feature stream
+  wof feature properties  output feature properties
+  wof feature stats       generate aggregate stats
+```
+
+#### Feature Reformatting
+
+reformat a feature stream
+
+```bash
+cat jsonstream | bin/whosonfirst.js feature format
+
+... stream of minified json, one item per line
+```
+
+```bash
+cat jsonstream | bin/whosonfirst.js feature format --open $'[\n' --sep $'\n,\n' --close $'\n]'
+
+... stream of minified json, with array header, footer and separator
+```
+
+```bash
+cat 101914243.geojson | bin/whosonfirst.js feature format --indent 2
+
+... stream of pretty printed json
+```
+
+#### Feature Properties
+
+output feature properties
+
+```bash
+cat 101914243.geojson | bin/whosonfirst.js feature properties -p 'wof:id' -p 'wof:placetype'
+
+{"wof:id":101914243,"wof:placetype":"locality"}
+```
+
+#### Feature Stats
+
+generate aggregate stats
+
+```bash
+cat 101914243.geojson | bin/whosonfirst.js feature stats
+
+{"locality":1}
+```

bin/cmd/bundle.js

@@ -0,0 +1,8 @@
+module.exports = {
+  command: 'bundle',
+  describe: 'interact with tar bundles',
+  builder: (yargs) => yargs
+    .commandDir('bundle')
+    .usage('$0 <cmd> [args]')
+    .demandCommand(1, '')
+}

bin/cmd/bundle/export.js

@@ -0,0 +1,30 @@
+const path = require('path')
+const all = require('require-all')
+const stream = all(path.join(__dirname, '../../../stream'))
+
+module.exports = {
+  command: 'export <file>',
+  describe: 'export features from a bundle file',
+  builder: (yargs) => {
+    // mandatory params
+    yargs.positional('file', {
+      type: 'string',
+      describe: 'Location of bundle file.'
+    })
+
+    // optional params
+    yargs.option('alt', {
+      type: 'boolean',
+      default: true,
+      describe: 'Include alt-geometries.'
+    })
+  },
+  handler: (argv) => {
+    let args = ['-f', argv.file, '--include', '*.geojson']
+    if (!argv.alt) { args = [...args, '--exclude', '*-alt-*'] }
+
+    // create export stream
+    stream.bsdtar.extract(...args)
+      .pipe(process.stdout)
+  }
+}

bin/cmd/bundle/import.js

@@ -0,0 +1,51 @@
+const fs = require('fs')
+const path = require('path')
+const all = require('require-all')
+const stream = all(path.join(__dirname, '../../../stream'))
+
+module.exports = {
+  command: 'import <file>',
+  describe: 'import features into a bundle file',
+  builder: (yargs) => {
+    // mandatory params
+    yargs.positional('file', {
+      type: 'string',
+      describe: 'Location of bundle file.'
+    })
+
+    // optional params
+    yargs.option('unlink', {
+      type: 'boolean',
+      default: false,
+      alias: 'rm',
+      describe: 'Delete db file before import if it already exists.'
+    })
+  },
+  handler: (argv) => {
+    // auto-detect format
+    let compressor
+    if (argv.file.endsWith('tar.bz2')) { compressor = 'bzip2' }
+    if (argv.file.endsWith('tar.gz')) { compressor = 'gzip' }
+    if (argv.file.endsWith('tar')) { compressor = 'cat' }
+    if (!compressor) {
+      console.error(`unsupported file extension: ${argv.file}`)
+      process.exit(1)
+    }
+
+    // log info about selected compressor
+    if (argv.verbose) { console.error(`compressor ${compressor}`) }
+
+    // unlink file
+    if (argv.unlink) {
+      if (argv.verbose) { console.error(`unlink ${argv.file}`) }
+      fs.existsSync(argv.file) && fs.unlinkSync(argv.file)
+    }
+
+    // create import stream
+    process.stdin
+      .pipe(stream.json.parse())
+      .pipe(stream.tar.createWriteStream())
+      .pipe(stream.shell.duplex(compressor))
+      .pipe(fs.createWriteStream(argv.file))
+  }
+}

bin/cmd/bundle/list.js

@@ -0,0 +1,36 @@
+const path = require('path')
+const all = require('require-all')
+const stream = all(path.join(__dirname, '../../../stream'))
+
+module.exports = {
+  command: 'list <file>',
+  describe: 'list features in a bundle file',
+  builder: (yargs) => {
+    // mandatory params
+    yargs.positional('file', {
+      type: 'string',
+      describe: 'Location of bundle file.'
+    })
+
+    // optional params
+    yargs.option('alt', {
+      type: 'boolean',
+      default: true,
+      describe: 'Include alt-geometries.'
+    })
+    yargs.option('all', {
+      type: 'boolean',
+      default: false,
+      describe: 'List all files (not only *.geojson files).'
+    })
+  },
+  handler: (argv) => {
+    const extension = argv.all ? '*.*' : '*.geojson'
+    let args = ['-f', argv.file, '--include', extension]
+    if (!argv.alt) { args = [...args, '--exclude', '*-alt-*'] }
+
+    // create export stream
+    stream.bsdtar.list(...args)
+      .pipe(process.stdout)
+  }
+}

bin/cmd/feature.js

@@ -0,0 +1,8 @@
+module.exports = {
+  command: 'feature',
+  describe: 'misc functions for working with feature streams',
+  builder: (yargs) => yargs
+    .commandDir('feature')
+    .usage('$0 <cmd> [args]')
+    .demandCommand(1, '')
+}

bin/cmd/feature/format.js

@@ -0,0 +1,36 @@
+const path = require('path')
+const all = require('require-all')
+const stream = all(path.join(__dirname, '../../../stream'))
+
+module.exports = {
+  command: 'format',
+  describe: 'reformat a feature stream',
+  builder: (yargs) => {
+    yargs.option('open', {
+      type: 'string',
+      default: '',
+      describe: 'Tokens to emit at the start of the stream.'
+    })
+    yargs.option('sep', {
+      type: 'string',
+      default: '\n',
+      describe: 'Tokens to emit after each recod in the stream.'
+    })
+    yargs.option('close', {
+      type: 'string',
+      default: '',
+      describe: 'Tokens to emit at the end of the stream.'
+    })
+    yargs.option('indent', {
+      type: 'number',
+      default: 0,
+      describe: 'Pretty print using this number of space characters.'
+    })
+  },
+  handler: (yargs) => {
+    process.stdin
+      .pipe(stream.json.parse())
+      .pipe(stream.json.stringify(yargs.open, yargs.sep, yargs.close, yargs.indent))
+      .pipe(process.stdout)
+  }
+}