multicore http server

.jshintrc

@@ -2,7 +2,7 @@
   "node": true,
   "curly": true,
   "eqeqeq": true,
-  "esversion": 6,
+  "esversion": 8,
   "freeze": true,
   "immed": true,
   "indent": 2,

Dockerfile

@@ -1,5 +1,5 @@
 # base image
-FROM pelias/libpostal_baseimage
+FROM pelias/baseimage
 
 # dependencies
 RUN apt-get update && \

api/extract.js

@@ -1,31 +1,28 @@
-
-var sqlite3 = require('sqlite3'),
-    requireDir = require('require-dir'),
-    pretty = require('../lib/pretty'),
-    query = requireDir('../query'),
-    analyze = require('../lib/analyze');
+const Database = require('better-sqlite3');
+const query = { extract: require('../query/extract') };
+const analyze = require('../lib/analyze');
+const asyncForEach = require('../lib/asyncForEach');
 
 // export setup method
 function setup( addressDbPath, streetDbPath ){
 
   // connect to db
-  sqlite3.verbose();
-  var db = new sqlite3.Database( addressDbPath, sqlite3.OPEN_READONLY );
+  const db = new Database(addressDbPath, { readonly: true });
 
   // attach street database
-  query.attach( db, streetDbPath, 'street' );
+  db.exec(`ATTACH DATABASE '${streetDbPath}' as 'street'`);
 
   // query method
-  var q = function( coord, names, cb ){
+  var q = async function( coord, names, cb ){
 
     var point = {
       lat: parseFloat( coord.lat ),
       lon: parseFloat( coord.lon )
     };
 
     var normalized = [];
-    names.forEach( function( name ){
-      normalized = normalized.concat( analyze.street( name ) );
+    await asyncForEach(names, async (name) => {
+      normalized = normalized.concat( await analyze.street( name ) );
     });
 
     // error checking
@@ -34,7 +31,13 @@ function setup( addressDbPath, streetDbPath ){
     if( !normalized.length ){ return cb( 'invalid names' ); }
 
     // perform a db lookup for the specified street
-    query.extract( db, point, normalized, cb );
+    try {
+      const rows = query.extract( db, point, normalized );
+      cb(null, rows);
+    } catch (err) {
+      // an error occurred
+      return cb(err, null);
+    }
   };
 
   // close method to close db

api/near.js

@@ -1,24 +1,21 @@
-
-var sqlite3 = require('sqlite3'),
-    polyline = require('@mapbox/polyline'),
-    requireDir = require('require-dir'),
-    query = requireDir('../query'),
-    project = require('../lib/project'),
-    proximity = require('../lib/proximity');
+const Database = require('better-sqlite3');
+const polyline = require('@mapbox/polyline');
+const query = { near: require('../query/near') };
+const project = require('../lib/project');
+const proximity = require('../lib/proximity');
 
 // polyline precision
-var PRECISION = 6;
+const PRECISION = 6;
 
 // export setup method
 function setup( streetDbPath ){
 
   // connect to db
-  sqlite3.verbose();
   // @todo: this is required as the query uses the 'street.' prefix for tables
-  var db = new sqlite3.Database( ':memory:', sqlite3.OPEN_READONLY );
+  const db = new Database('/tmp/path', { memory: true });
 
   // attach street database
-  query.attach( db, streetDbPath, 'street' );
+  db.exec(`ATTACH DATABASE '${streetDbPath}' as 'street'`);
 
   // query method
   var q = function( coord, cb ){
@@ -32,11 +29,12 @@ function setup( streetDbPath ){
     if( isNaN( point.lat ) ){ return cb( 'invalid latitude' ); }
     if( isNaN( point.lon ) ){ return cb( 'invalid longitude' ); }
 
-    // perform a db lookup for nearby streets
-    query.near( db, point, function( err, res ){
+    try {
+      // perform a db lookup for nearby streets
+      const res = query.near( db, point );
 
-      // an error occurred or no results were found
-      if( err || !res || !res.length ){ return cb( err, null ); }
+      // no results were found
+      if( !res || !res.length ){ return cb( null, null ); }
 
       // decode polylines
       res.forEach( function( street, i ){
@@ -48,7 +46,10 @@ function setup( streetDbPath ){
 
       // return streets ordered ASC by distance from point
       cb( null, ordered );
-    });
+    } catch (err) {
+      // an error occurred
+      return cb(err, null);
+    }
   };
 
   // return methods

api/oa.js

@@ -1,19 +1,17 @@
-
-var sqlite3 = require('sqlite3'),
-    requireDir = require('require-dir'),
-    stream = requireDir('../stream', { recurse: true }),
-    query = requireDir('../query');
+const Database = require('better-sqlite3');
+const requireDir = require('require-dir');
+const stream = requireDir('../stream', { recurse: true });
+const query = requireDir('../query');
 
 // export method
 function oa(dataStream, addressDbPath, streetDbPath, done){
 
   // connect to db
-  sqlite3.verbose();
-  var db = new sqlite3.Database( process.argv[2] );
+  const db = new Database(addressDbPath);
 
   query.configure(db); // configure database
   query.tables.address(db); // create tables only if not already created
-  query.attach(db, process.argv[3], 'street'); // attach street database
+  db.exec(`ATTACH DATABASE '${streetDbPath}' as 'street'`);
 
   dataStream
     .pipe( stream.oa.parse() ) // parse openaddresses csv data

api/osm.js

@@ -1,19 +1,17 @@
-
-var sqlite3 = require('sqlite3'),
-    requireDir = require('require-dir'),
-    stream = requireDir('../stream', { recurse: true }),
-    query = requireDir('../query');
+const Database = require('better-sqlite3');
+const requireDir = require('require-dir');
+const stream = requireDir('../stream', { recurse: true });
+const query = requireDir('../query');
 
 // export method
 function osm(dataStream, addressDbPath, streetDbPath, done){
 
   // connect to db
-  sqlite3.verbose();
-  var db = new sqlite3.Database( process.argv[2] );
+  const db = new Database(addressDbPath);
 
   query.configure(db); // configure database
   query.tables.address(db); // create tables only if not already created
-  query.attach(db, process.argv[3], 'street'); // attach street database
+  db.exec(`ATTACH DATABASE '${streetDbPath}' as 'street'`);
 
   dataStream
     .pipe( stream.split() ) // split file on newline

api/polyline.js

@@ -1,15 +1,13 @@
-
-var sqlite3 = require('sqlite3'),
-    requireDir = require('require-dir'),
-    stream = requireDir('../stream', { recurse: true }),
-    query = requireDir('../query');
+const Database = require('better-sqlite3');
+const requireDir = require('require-dir');
+const stream = requireDir('../stream', { recurse: true });
+const query = requireDir('../query');
 
 // export method
 function polyline(dataStream, streetDbPath, done){
 
   // connect to db
-  sqlite3.verbose();
-  var db = new sqlite3.Database(streetDbPath);
+  const db = new Database(streetDbPath);
 
   query.configure(db); // configure database
   query.tables.street(db, true); // reset database and create tables

api/search.js

@@ -1,27 +1,25 @@
-
-var sqlite3 = require('sqlite3'),
-    requireDir = require('require-dir'),
-    query = requireDir('../query'),
-    project = require('../lib/project'),
-    geodesic = require('../lib/geodesic'),
-    analyze = require('../lib/analyze');
+const Database = require('better-sqlite3');
+const requireDir = require('require-dir');
+const query = requireDir('../query');
+const project = require('../lib/project');
+const geodesic = require('../lib/geodesic');
+const analyze = require('../lib/analyze');
 
 // export setup method
 function setup( addressDbPath, streetDbPath ){
 
   // connect to db
-  sqlite3.verbose();
-  var db = new sqlite3.Database( addressDbPath, sqlite3.OPEN_READONLY );
+  const db = new Database(addressDbPath, {readonly: true});
 
   // attach street database
-  query.attach( db, streetDbPath, 'street' );
+  db.exec(`ATTACH DATABASE '${streetDbPath}' as 'street'`);
 
   // enable memmapping of database pages
-  db.run('PRAGMA mmap_size=268435456;');
-  db.run('PRAGMA street.mmap_size=268435456;');
+  db.exec('PRAGMA mmap_size=268435456;');
+  db.exec('PRAGMA street.mmap_size=268435456;');
 
   // query method
-  var q = function( coord, number, street, cb ){
+  var q = async function( coord, number, street, cb ){
 
     var point = {
       lat: parseFloat( coord.lat ),
@@ -33,7 +31,7 @@ function setup( addressDbPath, streetDbPath ){
 
     var normalized = {
       number: analyze.housenumber( number ),
-      street: analyze.street( street )
+      street: await analyze.street( street )
     };
 
     // error checking
@@ -42,14 +40,16 @@ function setup( addressDbPath, streetDbPath ){
     if( isNaN( normalized.number ) ){ return cb( 'invalid number' ); }
     if( !normalized.street.length ){ return cb( 'invalid street' ); }
 
-    // perform a db lookup for the specified street
-    // @todo: perofmance: only query for part of the table
-    query.search( db, point, normalized.number, normalized.street, function( err, res ){
+    try {
+
+      // perform a db lookup for the specified street
+      // @todo: perofmance: only query for part of the table
+      const res = query.search( db, point, normalized.number, normalized.street );
 
       // @note: results can be from multiple different street ids.
 
-      // an error occurred or no results were found
-      if( err || !res || !res.length ){ return cb( err, null ); }
+      // no results were found
+      if( !res || !res.length ){ return cb( null, null ); }
 
       // try to find an exact match
       var match = res.find( function( row ){
@@ -132,21 +132,24 @@ function setup( addressDbPath, streetDbPath ){
 
       // if distance = 0 then we can simply use either A or B (they are the same lat/lon)
       // else we interpolate between the two positions
-      var point = A;
+      var interpolatedPoint = A;
       if( distance > 0 ){
         var ratio = ((normalized.number - before.housenumber) / (after.housenumber - before.housenumber));
-        point = geodesic.interpolate( distance, ratio, A, B );
+        interpolatedPoint = geodesic.interpolate( distance, ratio, A, B );
       }
 
       // return interpolated address
       return cb( null, {
         type: 'interpolated',
         source: 'mixed',
         number: '' + Math.floor( normalized.number ),
-        lat: parseFloat( project.toDeg( point.lat ).toFixed(7) ),
-        lon: parseFloat( project.toDeg( point.lon ).toFixed(7) )
+        lat: parseFloat( project.toDeg( interpolatedPoint.lat ).toFixed(7) ),
+        lon: parseFloat( project.toDeg( interpolatedPoint.lon ).toFixed(7) )
       });
-    });
+    } catch (err) {
+      // an error occurred
+      return cb(err, null);
+    }
   };
 
   // close method to close db

api/street.js

@@ -1,14 +1,12 @@
-
-var sqlite3 = require('sqlite3'),
-    requireDir = require('require-dir'),
-    query = requireDir('../query');
+const Database = require('better-sqlite3');
+const requireDir = require('require-dir');
+const query = requireDir('../query');
 
 // export setup method
 function setup( streetDbPath ){
 
   // connect to db
-  sqlite3.verbose();
-  var db = new sqlite3.Database( streetDbPath, sqlite3.OPEN_READONLY );
+  const db = new Database(streetDbPath, { readonly: true });
 
   // query method
   var q = function( ids, cb ){
@@ -24,15 +22,19 @@ function setup( streetDbPath ){
     });
     if( fail ){ return cb( 'non-numeric id' ); }
 
-    // perform a db lookup for the specified street
-    query.street( db, ids, function( err, res ){
+    try {
+      // perform a db lookup for the specified street
+      const res = query.street( db, ids );
 
-      // an error occurred or no results were found
-      if( err || !res ){ return cb( err, null ); }
+      // results were found
+      if( !res ){ return cb( null, null ); }
 
       // call callback
-      cb( err, res );
-    });
+      cb( null, res );
+    } catch (err) {
+      // an error occurred
+      return cb(err, null);
+    }
   };
 
   // close method to close db

api/tiger.js

@@ -1,19 +1,19 @@
-
-var sqlite3 = require('sqlite3'),
-    requireDir = require('require-dir'),
-    stream = requireDir('../stream', { recurse: true }),
-    query = requireDir('../query');
+const Database = require('better-sqlite3');
+const requireDir = require('require-dir');
+const stream = requireDir('../stream', { recurse: true });
+const query = requireDir('../query');
 
 // export method
 function tiger(dataStream, addressDbPath, streetDbPath, done){
 
   // connect to db
-  sqlite3.verbose();
-  var db = new sqlite3.Database( process.argv[2] );
+  var db = new Database( addressDbPath, {
+    verbose: console.log
+  });
 
   query.configure(db); // configure database
   query.tables.address(db); // create tables only if not already created
-  query.attach(db, process.argv[3], 'street'); // attach street database
+  db.exec(`ATTACH DATABASE '${streetDbPath}' as 'street'`);
 
   dataStream
     .pipe( stream.tiger.parse() ) // convert tiger data to generic model