diff --git a/README.md b/README.md index e4ff0db6..9107b9c3 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,12 @@ See [the config](https://github.com/pelias/config) documentation for details on } ``` +The importer has the possibility to download or not the OSM venues. +This ability is managed by the parameter "importVenues" as described below: + +| key | required | default | description | +| `imports.openstreetmap.importVenues` | no | true | set to `true` to include venues in the data download and import process | + ### Environment Settings - `imports.openstreetmap.datapath` - this is the directory which you downloaded the pbf file to diff --git a/config/features.js b/config/features.js index 35b262ea..9cebfd65 100644 --- a/config/features.js +++ b/config/features.js @@ -4,8 +4,13 @@ imports. @see: https://github.com/pelias/pbf2json for more info. **/ +// default tags imported var tags = [ - 'addr:housenumber+addr:street', + 'addr:housenumber+addr:street' +]; + +// tags corresponding to venues +var venue_tags = [ 'amenity+name', 'building+name', 'shop+name', @@ -40,4 +45,4 @@ var tags = [ 'aeroway~airport+name' ]; -module.exports = tags; +module.exports = {tags,venue_tags}; diff --git a/schema.js b/schema.js index e632bf67..fe09750e 100644 --- a/schema.js +++ b/schema.js @@ -6,6 +6,7 @@ const Joi = require('joi'); // datapath: string (required) // leveldbpath: string (required) // import: array of objects containing filename (optional) +// importVenues: boolean (optional) // download: array of objects containing sourceURL (optional) // deduplicate: boolean (optional) module.exports = Joi.object().keys({ @@ -14,7 +15,8 @@ module.exports = Joi.object().keys({ datapath: Joi.string(), leveldbpath: Joi.string(), import: Joi.array().items(Joi.object().keys({ - filename: Joi.string() + filename: Joi.string(), + importVenues: Joi.boolean().default(true).truthy('yes').falsy('no').insensitive(true) }).requiredKeys('filename').unknown(true)), download: Joi.array().items(Joi.object().keys({ sourceURL: Joi.string() diff --git a/stream/multiple_pbfs.js b/stream/multiple_pbfs.js index a5e452d5..00ec050b 100644 --- a/stream/multiple_pbfs.js +++ b/stream/multiple_pbfs.js @@ -8,7 +8,11 @@ function createCombinedStream(){ var defaultPath= require('pelias-config').generate().imports.openstreetmap; defaultPath.import.forEach(function( importObject){ - var conf = {file: path.join(defaultPath.datapath, importObject.filename), leveldb: defaultPath.leveldbpath}; + var conf = { + file: path.join(defaultPath.datapath, importObject.filename), + leveldb: defaultPath.leveldbpath, + importVenues: importObject.importVenues + }; fullStream.append(function(next){ logger.info('Creating read stream for: ' + conf.file); next(pbf.parser(conf)); diff --git a/stream/pbf.js b/stream/pbf.js index a80481d3..f5ec2a87 100644 --- a/stream/pbf.js +++ b/stream/pbf.js @@ -6,7 +6,7 @@ var fs = require('fs'), pbf2json = require('pbf2json'), - settings = require('pelias-config').generate(), + settings = require('pelias-config').generate(require('../schema')), features = require('../config/features'), path = require('path'); @@ -44,7 +44,13 @@ function config(opts){ // Use default parser tags if(!opts.tags){ - opts.tags = features; + // check if we import venues + opts.importVenues = settings.imports.openstreetmap.import[0].importVenues; + if(opts.importVenues){ + opts.tags = features.tags.concat(features.venue_tags); + } else { + opts.tags = features.tags; + } } return opts; } diff --git a/test/config/features.js b/test/config/features.js index f989d57a..a538675c 100644 --- a/test/config/features.js +++ b/test/config/features.js @@ -6,59 +6,68 @@ module.exports.tests = {}; // test exports module.exports.tests.interface = function(test, common) { test('interface: features', function(t) { - t.true(Array.isArray(features), 'valid taglist'); + t.true(Array.isArray(features.tags), 'valid taglist'); t.end(); }); }; // ensure some tags are excluded module.exports.tests.blacklist = function(test, common) { - test('blacklist', function(t) { + test('blacklist default tags', function(t) { // see: https://github.com/pelias/openstreetmap/pull/280 - t.true( features.indexOf('aeroway+name') <0 ); - t.true( features.indexOf('aeroway~gate+name') <0 ); - t.true( features.indexOf('railway+name') <0 ); - t.true( features.indexOf('railway~rail+name') <0 ); + t.true( features.tags.indexOf('aeroway+name') <0 ); + t.true( features.tags.indexOf('aeroway~gate+name') <0 ); + t.true( features.tags.indexOf('railway+name') <0 ); + t.true( features.tags.indexOf('railway~rail+name') <0 ); t.end(); }); }; // ensure some tags are included +// we exclude by default tags corresponding to venues module.exports.tests.whitelist = function(test, common) { - test('whitelist', function(t) { - t.false( features.indexOf('addr:housenumber+addr:street') <0 ); - t.false( features.indexOf('amenity+name') <0 ); - t.false( features.indexOf('building+name') <0 ); - t.false( features.indexOf('shop+name') <0 ); - t.false( features.indexOf('office+name') <0 ); - t.false( features.indexOf('public_transport+name') <0 ); - t.false( features.indexOf('cuisine+name') <0 ); - t.false( features.indexOf('railway~tram_stop+name') <0 ); - t.false( features.indexOf('railway~station+name') <0 ); - t.false( features.indexOf('railway~halt+name') <0 ); - t.false( features.indexOf('railway~subway_entrance+name') <0 ); - t.false( features.indexOf('railway~train_station_entrance+name') <0 ); - t.false( features.indexOf('sport+name') <0 ); - t.false( features.indexOf('natural+name') <0 ); - t.false( features.indexOf('tourism+name') <0 ); - t.false( features.indexOf('leisure+name') <0 ); - t.false( features.indexOf('historic+name') <0 ); - t.false( features.indexOf('man_made+name') <0 ); - t.false( features.indexOf('landuse+name') <0 ); - t.false( features.indexOf('waterway+name') <0 ); - t.false( features.indexOf('aerialway+name') <0 ); - t.false( features.indexOf('craft+name') <0 ); - t.false( features.indexOf('military+name') <0 ); - t.false( features.indexOf('aeroway~terminal+name') <0 ); - t.false( features.indexOf('aeroway~aerodrome+name') <0 ); - t.false( features.indexOf('aeroway~helipad+name') <0 ); - t.false( features.indexOf('aeroway~airstrip+name') <0 ); - t.false( features.indexOf('aeroway~heliport+name') <0 ); - t.false( features.indexOf('aeroway~areodrome+name') <0 ); - t.false( features.indexOf('aeroway~spaceport+name') <0 ); - t.false( features.indexOf('aeroway~landing_strip+name') <0 ); - t.false( features.indexOf('aeroway~airfield+name') <0 ); - t.false( features.indexOf('aeroway~airport+name') <0 ); + test('whitelist default tags', function(t) { + t.false( features.tags.indexOf('addr:housenumber+addr:street') <0 ); + t.true( features.venue_tags.indexOf('amenity+name') <0 ); + t.end(); + }); +}; + +// ensure some venue tags are included +module.exports.tests.whitelist = function(test, common) { + test('whitelist venue tags', function(t) { + t.false( features.venue_tags.indexOf('amenity+name') <0 ); + t.false( features.venue_tags.indexOf('building+name') <0 ); + t.false( features.venue_tags.indexOf('shop+name') <0 ); + t.false( features.venue_tags.indexOf('office+name') <0 ); + t.false( features.venue_tags.indexOf('public_transport+name') <0 ); + t.false( features.venue_tags.indexOf('cuisine+name') <0 ); + t.false( features.venue_tags.indexOf('railway~tram_stop+name') <0 ); + t.false( features.venue_tags.indexOf('railway~station+name') <0 ); + t.false( features.venue_tags.indexOf('railway~halt+name') <0 ); + t.false( features.venue_tags.indexOf('railway~subway_entrance+name') <0 ); + t.false( features.venue_tags.indexOf('railway~train_station_entrance+name') <0 ); + t.false( features.venue_tags.indexOf('sport+name') <0 ); + t.false( features.venue_tags.indexOf('natural+name') <0 ); + t.false( features.venue_tags.indexOf('tourism+name') <0 ); + t.false( features.venue_tags.indexOf('leisure+name') <0 ); + t.false( features.venue_tags.indexOf('historic+name') <0 ); + t.false( features.venue_tags.indexOf('man_made+name') <0 ); + t.false( features.venue_tags.indexOf('landuse+name') <0 ); + t.false( features.venue_tags.indexOf('waterway+name') <0 ); + t.false( features.venue_tags.indexOf('aerialway+name') <0 ); + t.false( features.venue_tags.indexOf('craft+name') <0 ); + t.false( features.venue_tags.indexOf('military+name') <0 ); + t.false( features.venue_tags.indexOf('aeroway~terminal+name') <0 ); + t.false( features.venue_tags.indexOf('aeroway~aerodrome+name') <0 ); + t.false( features.venue_tags.indexOf('aeroway~helipad+name') <0 ); + t.false( features.venue_tags.indexOf('aeroway~airstrip+name') <0 ); + t.false( features.venue_tags.indexOf('aeroway~heliport+name') <0 ); + t.false( features.venue_tags.indexOf('aeroway~areodrome+name') <0 ); + t.false( features.venue_tags.indexOf('aeroway~spaceport+name') <0 ); + t.false( features.venue_tags.indexOf('aeroway~landing_strip+name') <0 ); + t.false( features.venue_tags.indexOf('aeroway~airfield+name') <0 ); + t.false( features.venue_tags.indexOf('aeroway~airport+name') <0 ); t.end(); }); }; diff --git a/test/stream/pbf.js b/test/stream/pbf.js index 58c0cddf..4e6c31ae 100644 --- a/test/stream/pbf.js +++ b/test/stream/pbf.js @@ -94,6 +94,26 @@ module.exports.tests.parser = function(test, common) { }); }; +// Disable venues import +module.exports.tests.parser = function(test, common) { + test('Disable venues import', function(t) { + var localFakeGeneratedConfig = JSON.parse(JSON.stringify(fakeGeneratedConfig)); // "JSON.parse" to get a deep copy + var localFakeConfig = { + generate: function fakeGenerate() { + return localFakeGeneratedConfig; + } + }; + var pbf = proxyquire('../../stream/pbf', {'pelias-config': localFakeConfig}); + localFakeGeneratedConfig.imports.openstreetmap.import[0].importVenues = false; + var conf = pbf.config(); + var expected = { + importVenues: false + }; + t.equal(conf.importVenues, expected.importVenues, 'disable importVenues'); + t.end(); + }); +}; + module.exports.all = function (tape, common) { function test(name, testFunction) {