feat(import): add support to .gz data

lib/parameters.js

@@ -96,7 +96,7 @@ function getFullFileList(peliasConfig, args) {
 
   if (_.isEmpty(files)) {
     // no specific files listed, so return all .csv and .geojson files
-    return glob.sync( args.dirPath + '/**/*.{csv,geojson}' );
+    return glob.sync( args.dirPath + '/**/*.{csv,geojson,geojson.gz,csv.gz}' );
   } else {
     // otherwise return the requested files with full path
     return files.map(function(file) {

lib/streams/recordStream.js

@@ -5,6 +5,7 @@ const csvParse = require('csv-parse').parse;
 const combinedStream = require('combined-stream');
 const through = require('through2');
 const split = require('split2');
+const zlib = require('zlib');
 
 const logger = require('pelias-logger').get('openaddresses');
 const config = require('pelias-config').generate();
@@ -26,13 +27,13 @@ function getIdPrefix(filename, dirPath) {
     // of the directory tree to create the id
     if (filename.indexOf(dirPath) !== -1) {
       var subpath = _.replace(filename, dirPath, '');
-      var prefix = _.replace(subpath, /\.(csv|geojson)/, '');
+      var prefix = _.replace(_.replace(subpath, /\.(csv|geojson)/, ''), /\.gz/, '');
       return _.trim(prefix, '/');
     }
   }
 
   // if the dirPath doesn't contain this file, return the basename without extension
-  return path.basename(path.basename(filename, '.csv'), '.geojson');
+  return path.basename(path.basename(path.basename(filename, '.gz'), '.csv'), '.geojson');
 }
 
 /**
@@ -98,7 +99,11 @@ function geojsonStream(stream) {
 }
 
 function fileStreamDispatcher(stream, filePath) {
-  if (filePath.endsWith('.geojson')) {
+  if (filePath.endsWith('.gz')) {
+    stream = stream.pipe(zlib.createGunzip());
+  }
+
+  if (/\.geojson(\.gz)?/.test(filePath)) {
     return geojsonStream(stream);
   }