diff --git a/lib/parameters.js b/lib/parameters.js index 18136d42d..259f99de6 100644 --- a/lib/parameters.js +++ b/lib/parameters.js @@ -96,7 +96,7 @@ function getFullFileList(peliasConfig, args) { if (_.isEmpty(files)) { // no specific files listed, so return all .csv and .geojson files - return glob.sync( args.dirPath + '/**/*.{csv,geojson}' ); + return glob.sync( args.dirPath + '/**/*.{csv,geojson,geojson.gz,csv.gz}' ); } else { // otherwise return the requested files with full path return files.map(function(file) { diff --git a/lib/streams/recordStream.js b/lib/streams/recordStream.js index 552429418..6bd7a8359 100644 --- a/lib/streams/recordStream.js +++ b/lib/streams/recordStream.js @@ -5,6 +5,7 @@ const csvParse = require('csv-parse').parse; const combinedStream = require('combined-stream'); const through = require('through2'); const split = require('split2'); +const zlib = require('zlib'); const logger = require('pelias-logger').get('openaddresses'); const config = require('pelias-config').generate(); @@ -26,13 +27,13 @@ function getIdPrefix(filename, dirPath) { // of the directory tree to create the id if (filename.indexOf(dirPath) !== -1) { var subpath = _.replace(filename, dirPath, ''); - var prefix = _.replace(subpath, /\.(csv|geojson)/, ''); + var prefix = _.replace(_.replace(subpath, /\.(csv|geojson)/, ''), /\.gz/, ''); return _.trim(prefix, '/'); } } // if the dirPath doesn't contain this file, return the basename without extension - return path.basename(path.basename(filename, '.csv'), '.geojson'); + return path.basename(path.basename(path.basename(filename, '.gz'), '.csv'), '.geojson'); } /** @@ -98,7 +99,11 @@ function geojsonStream(stream) { } function fileStreamDispatcher(stream, filePath) { - if (filePath.endsWith('.geojson')) { + if (filePath.endsWith('.gz')) { + stream = stream.pipe(zlib.createGunzip()); + } + + if (/\.geojson(\.gz)?/.test(filePath)) { return geojsonStream(stream); }