feat(GID): Require GID to be stored in the _id field

This change implements the 5th and final step in the GID migration plan outlined in pelias/pelias#672 (comment). It adjusts the expectations of the API so that it requires the full GID, rather than the source_id, to be stored in the `_id` field in Elasticsearch. In the process, the `source_id` field is also no longer used in `helper/geojsonify.js`, making this the first step towards removing the `source_id` field (pelias/schema#383). Connects pelias/pelias#672
pelias · Oct 4, 2019 · c867946 · c867946
1 parent b85288a
commit c867946
Show file tree

Hide file tree

Showing 4 changed files with 47 additions and 73 deletions.
diff --git a/helper/geojsonify.js b/helper/geojsonify.js
@@ -6,6 +6,7 @@ const _ = require('lodash');
 const Document = require('pelias-model').Document;
 const codec = require('pelias-model').codec;
 const field = require('./fieldValue');
+const decode_gid = require('./decode_gid');
 
 function geojsonifyPlaces( params, docs ){
 
@@ -40,13 +41,14 @@ function geojsonifyPlaces( params, docs ){
 }
 
 function geojsonifyPlace(params, place) {
+  const gid_components = decode_gid(place._id);
   // setup the base doc
   const doc = {
-    id: place.source_id,
-    gid: new Document(place.source, place.layer, place.source_id).getGid(),
+    id: gid_components.id,
+    gid: new Document(place.source, place.layer, gid_components.id).getGid(),
     layer: place.layer,
     source: place.source,
-    source_id: place.source_id,
+    source_id: gid_components.id,
     bounding_box: place.bounding_box,
     lat: parseFloat(place.center_point.lat),
     lng: parseFloat(place.center_point.lon)

diff --git a/sanitizer/_ids.js b/sanitizer/_ids.js
@@ -1,8 +1,7 @@
-var _ = require('lodash'),
-    check = require('check-types'),
-    type_mapping = require('../helper/type_mapping');
-
-var ID_DELIM = ':';
+const _ = require('lodash');
+const check = require('check-types');
+const type_mapping = require('../helper/type_mapping');
+const decode_gid = require('../helper/decode_gid');
 
 // validate inputs, convert types and apply defaults id generally looks like
 // 'geonames:venue:4163334' (source:layer:id) so, all three are required
@@ -18,23 +17,22 @@ var targetError = function(target, target_list) {
 };
 
 function sanitizeId(rawId, messages) {
-  var parts = rawId.split(ID_DELIM);
+  const gid = decode_gid(rawId);
 
-  if ( parts.length < 3 ) {
+  if (!gid ) {
     messages.errors.push( formatError(rawId) );
     return;
   }
 
-  var source = parts[0];
-  var layer = parts[1];
-  var id = parts.slice(2).join(ID_DELIM);
+  const { source, layer, id } = gid;
 
   // check if any parts of the gid are empty
   if (_.includes([source, layer, id], '')) {
     messages.errors.push( formatError(rawId) );
     return;
   }
-  var valid_values = Object.keys(type_mapping.source_mapping);
+
+  const valid_values = Object.keys(type_mapping.source_mapping);
   if (!_.includes(valid_values, source)) {
     messages.errors.push( targetError(source, valid_values) );
     return;

diff --git a/test/unit/fixture/dedupe_elasticsearch_results.js b/test/unit/fixture/dedupe_elasticsearch_results.js
@@ -57,7 +57,6 @@ module.exports = [
     'address_parts': {},
     'alpha3': 'USA',
     'source': 'openstreetmap',
-    'source_id': 'node:357289197',
     'category': [
       'education'
     ],
@@ -118,7 +117,6 @@ module.exports = [
     'address_parts': {},
     'alpha3': 'USA',
     'source': 'geonames',
-    'source_id': '5219083',
     'category': [
       'education'
     ],
@@ -197,7 +195,6 @@ module.exports = [
     'address_parts': {},
     'alpha3': 'USA',
     'source': 'geonames',
-    'source_id': '5183465',
     'category': [
       'entertainment',
       'recreation'
@@ -277,7 +274,6 @@ module.exports = [
     'address_parts': {},
     'alpha3': 'USA',
     'source': 'openstreetmap',
-    'source_id': 'node:368338500',
     'category': [
       'education'
     ],
@@ -339,7 +335,6 @@ module.exports = [
     'address_parts': {},
     'alpha3': 'USA',
     'source': 'openstreetmap',
-    'source_id': 'way:84969670',
     'category': [
       'education'
     ],
@@ -409,7 +404,6 @@ module.exports = [
     'address_parts': {},
     'alpha3': 'USA',
     'source': 'geonames',
-    'source_id': '5192545',
     'category': [
       'education'
     ],
@@ -488,7 +482,6 @@ module.exports = [
     'address_parts': {},
     'alpha3': 'USA',
     'source': 'geonames',
-    'source_id': '5198085',
     'category': [
       'education'
     ],
@@ -558,7 +551,6 @@ module.exports = [
     'address_parts': {},
     'alpha3': 'USA',
     'source': 'geonames',
-    'source_id': '5208101',
     'category': [
       'education'
     ],
@@ -638,7 +630,6 @@ module.exports = [
     'address_parts': {},
     'alpha3': 'USA',
     'source': 'openstreetmap',
-    'source_id': 'way:161088588',
     'category': [
       'education'
     ],
@@ -717,7 +708,6 @@ module.exports = [
     'address_parts': {},
     'alpha3': 'USA',
     'source': 'geonames',
-    'source_id': '5200263',
     'category': [
       'education'
     ],
@@ -788,7 +778,6 @@ module.exports = [
     'address_parts': {},
     'alpha3': 'USA',
     'source': 'openstreetmap',
-    'source_id': 'way:34212977',
     'category': [
       'education'
     ],
@@ -867,7 +856,6 @@ module.exports = [
     'address_parts': {},
     'alpha3': 'USA',
     'source': 'openstreetmap',
-    'source_id': 'node:357330916',
     'category': [
       'education'
     ],
@@ -937,7 +925,6 @@ module.exports = [
     'address_parts': {},
     'alpha3': 'USA',
     'source': 'openstreetmap',
-    'source_id': 'node:357330919',
     'category': [
       'education'
     ],
@@ -1007,7 +994,6 @@ module.exports = [
     'address_parts': {},
     'alpha3': 'USA',
     'source': 'geonames',
-    'source_id': '5197082',
     'category': [
       'education'
     ],