feat(version): remove _type (#128)

Remove references to `_type` from this repo in order to support elasticsearch v8. Prior to version 6 `_type` was recommended to be "_doc", in es7 it was optional (but if specified it was required to be "_doc"), in es8 it's not allowed at all. See: https://www.elastic.co/guide/en/elasticsearch/reference/7.17/removal-of-types.html BREAKING CHANGE: Pelias no longer supports elasticsearch v6 This change drops support for elasticsearch v6 in order to support v8. The current recommended version remains v7.
pelias · Mar 13, 2024 · b67ee4b · b67ee4b
1 parent 58fac7e
commit b67ee4b
Show file tree

Hide file tree

Showing 7 changed files with 18 additions and 47 deletions.
diff --git a/README.md b/README.md
@@ -48,14 +48,11 @@ const stream = streamify([1, 2, 3])
   .pipe(dbclient()); // put documents into elasticsearch
 
 stream.on('finish', () => {
-  // let's assume that documents with the same type but another timestamp (for example old copies)
-  // have to be deleted
   const client = new elasticsearch.Client(config.esclient);
   elasticDeleteQuery(client);
 
   const options = {
     index: config.schema.indexName,
-    type: 'venue',
     body: {
       query: {
         "bool": {

diff --git a/package.json b/package.json
@@ -38,7 +38,7 @@
   "dependencies": {
     "@hapi/joi": "^16.0.0",
     "elasticsearch": "^16.0.0",
-    "pelias-config": "^5.2.0",
+    "pelias-config": "^6.0.0",
     "pelias-logger": "^1.2.1",
     "through2": "^3.0.0"
   },

diff --git a/src/BatchManager.js b/src/BatchManager.js
@@ -55,27 +55,18 @@ BatchManager.prototype._dispatch = function( batch, next ){
       this._stats.inc( 'indexed', batch._slots.length );
       this._stats.inc( 'batch_ok', 1 );
 
-      var types = {};
       var failures = 0;
 
-      batch._slots.forEach( function( task ){
+      batch._slots.forEach(( task ) => {
         if( task.status < 299 ){
-          const type = task.data.layer || task.cmd.index._type;
-          if( !types.hasOwnProperty( type ) ){
-            types[ type ] = 0;
-          }
-          types[ type ]++;
+          this._stats.inc( task.data.layer || 'default', 1 );
         } else {
           failures++;
         }
       });
 
       this._stats.inc( 'batch_retries', batch.retries );
       this._stats.inc( 'failed_records', failures );
-
-      for( var type in types ){
-        this._stats.inc( type, types[type] );
-      }
     }
 
     // console.log( 'batch complete', err, batch._slots.length );

diff --git a/src/Task.js b/src/Task.js
@@ -5,7 +5,6 @@ function Task( record ){
   this.cmd = {
     index: {
       _index: record._index,
-      _type: record._type,
       _id: record._id
     }
   };

diff --git a/src/configValidation.js b/src/configValidation.js
@@ -15,8 +15,7 @@ const schema = Joi.object().keys({
     requestTimeout: Joi.number().integer().min(0)
   }).unknown(true),
   schema: Joi.object().keys({
-    indexName: Joi.string().required(),
-    typeName: Joi.string().required()
+    indexName: Joi.string().required()
   })
 }).unknown(true);
 

diff --git a/test/configValidation.js b/test/configValidation.js
@@ -11,8 +11,7 @@ module.exports.tests.validate = function(test, common) {
     var config = {
       esclient: {},
       schema: {
-        indexName: 'example_index',
-        typeName: 'example_type'
+        indexName: 'example_index'
       }
     };
 
@@ -30,8 +29,7 @@ module.exports.tests.validate = function(test, common) {
       },
       esclient: {},
       schema: {
-        indexName: 'example_index',
-        typeName: 'example_type'
+        indexName: 'example_index'
       }
     };
 
@@ -51,8 +49,7 @@ module.exports.tests.validate = function(test, common) {
         },
         esclient: {},
         schema: {
-          indexName: 'example_index',
-          typeName: 'example_type'
+          indexName: 'example_index'
         }
       };
 
@@ -74,8 +71,7 @@ module.exports.tests.validate = function(test, common) {
       },
       esclient: {},
       schema: {
-        indexName: 'example_index',
-        typeName: 'example_type'
+        indexName: 'example_index'
       }
     };
 
@@ -94,8 +90,7 @@ module.exports.tests.validate = function(test, common) {
       },
       esclient: {},
       schema: {
-        indexName: 'example_index',
-        typeName: 'example_type'
+        indexName: 'example_index'
       }
     };
 
@@ -115,8 +110,7 @@ module.exports.tests.validate = function(test, common) {
         },
         esclient: {},
         schema: {
-          indexName: 'example_index',
-          typeName: 'example_type'
+          indexName: 'example_index'
         }
       };
 
@@ -138,8 +132,7 @@ module.exports.tests.validate = function(test, common) {
       },
       esclient: {},
       schema: {
-        indexName: 'example_index',
-        typeName: 'example_type'
+        indexName: 'example_index'
       }
     };
 
@@ -160,8 +153,7 @@ module.exports.tests.validate = function(test, common) {
         },
         esclient: value,
         schema: {
-          indexName: 'example_index',
-          typeName: 'example_type'
+          indexName: 'example_index'
         }
       };
 
@@ -186,8 +178,7 @@ module.exports.tests.validate = function(test, common) {
           requestTimeout: value
         },
         schema: {
-          indexName: 'example_index',
-          typeName: 'example_type'
+          indexName: 'example_index'
         }
       };
 
@@ -210,8 +201,7 @@ module.exports.tests.validate = function(test, common) {
         requestTimeout: 17.3
       },
       schema: {
-        indexName: 'example_index',
-        typeName: 'example_type'
+        indexName: 'example_index'
       }
     };
 
@@ -233,8 +223,7 @@ module.exports.tests.validate = function(test, common) {
         requestTimeout: -1
       },
       schema: {
-        indexName: 'example_index',
-        typeName: 'example_type'
+        indexName: 'example_index'
       }
     };
 
@@ -315,8 +304,7 @@ module.exports.tests.validate = function(test, common) {
       },
       esclient: {},
       schema: {
-        indexName: 'example_index',
-        typeName: 'example_type'
+        indexName: 'example_index'
       }
     };
 
@@ -344,8 +332,7 @@ module.exports.tests.validate = function(test, common) {
         requestTimeout: 17
       },
       schema: {
-        indexName: 'example_index',
-        typeName: 'example_type'
+        indexName: 'example_index'
       }
     };
 
@@ -374,8 +361,7 @@ module.exports.tests.validate = function(test, common) {
         requestTimeout: 17
       },
       schema: {
-        indexName: 'example_index',
-        typeName: 'example_type'
+        indexName: 'example_index'
       }
     };
 

diff --git a/test/stream.js b/test/stream.js
@@ -44,7 +44,6 @@ module.exports.tests.functional_example = function(test, common) {
     stream.pipe(assertStream);
     stream.write({
       _index: 'foo',
-      _type: 'foo',
       _id: 'foo',
       data: {}
     });