From b67ee4b6f911fa3669c70d86296ab76dbb29c8c1 Mon Sep 17 00:00:00 2001 From: Peter Johnson Date: Wed, 13 Mar 2024 14:02:36 +0100 Subject: [PATCH] feat(version): remove `_type` (#128) Remove references to `_type` from this repo in order to support elasticsearch v8. Prior to version 6 `_type` was recommended to be "_doc", in es7 it was optional (but if specified it was required to be "_doc"), in es8 it's not allowed at all. See: https://www.elastic.co/guide/en/elasticsearch/reference/7.17/removal-of-types.html BREAKING CHANGE: Pelias no longer supports elasticsearch v6 This change drops support for elasticsearch v6 in order to support v8. The current recommended version remains v7. --- README.md | 3 --- package.json | 2 +- src/BatchManager.js | 13 ++----------- src/Task.js | 1 - src/configValidation.js | 3 +-- test/configValidation.js | 42 ++++++++++++++-------------------------- test/stream.js | 1 - 7 files changed, 18 insertions(+), 47 deletions(-) diff --git a/README.md b/README.md index 10f4b6b..1fcd274 100644 --- a/README.md +++ b/README.md @@ -48,14 +48,11 @@ const stream = streamify([1, 2, 3]) .pipe(dbclient()); // put documents into elasticsearch stream.on('finish', () => { - // let's assume that documents with the same type but another timestamp (for example old copies) - // have to be deleted const client = new elasticsearch.Client(config.esclient); elasticDeleteQuery(client); const options = { index: config.schema.indexName, - type: 'venue', body: { query: { "bool": { diff --git a/package.json b/package.json index 9d1d8da..7c68415 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,7 @@ "dependencies": { "@hapi/joi": "^16.0.0", "elasticsearch": "^16.0.0", - "pelias-config": "^5.2.0", + "pelias-config": "^6.0.0", "pelias-logger": "^1.2.1", "through2": "^3.0.0" }, diff --git a/src/BatchManager.js b/src/BatchManager.js index dee5b92..29a05f9 100644 --- a/src/BatchManager.js +++ b/src/BatchManager.js @@ -55,16 +55,11 @@ BatchManager.prototype._dispatch = function( batch, next ){ this._stats.inc( 'indexed', batch._slots.length ); this._stats.inc( 'batch_ok', 1 ); - var types = {}; var failures = 0; - batch._slots.forEach( function( task ){ + batch._slots.forEach(( task ) => { if( task.status < 299 ){ - const type = task.data.layer || task.cmd.index._type; - if( !types.hasOwnProperty( type ) ){ - types[ type ] = 0; - } - types[ type ]++; + this._stats.inc( task.data.layer || 'default', 1 ); } else { failures++; } @@ -72,10 +67,6 @@ BatchManager.prototype._dispatch = function( batch, next ){ this._stats.inc( 'batch_retries', batch.retries ); this._stats.inc( 'failed_records', failures ); - - for( var type in types ){ - this._stats.inc( type, types[type] ); - } } // console.log( 'batch complete', err, batch._slots.length ); diff --git a/src/Task.js b/src/Task.js index 3d5302c..63434d2 100644 --- a/src/Task.js +++ b/src/Task.js @@ -5,7 +5,6 @@ function Task( record ){ this.cmd = { index: { _index: record._index, - _type: record._type, _id: record._id } }; diff --git a/src/configValidation.js b/src/configValidation.js index 4bb812f..5c89f86 100644 --- a/src/configValidation.js +++ b/src/configValidation.js @@ -15,8 +15,7 @@ const schema = Joi.object().keys({ requestTimeout: Joi.number().integer().min(0) }).unknown(true), schema: Joi.object().keys({ - indexName: Joi.string().required(), - typeName: Joi.string().required() + indexName: Joi.string().required() }) }).unknown(true); diff --git a/test/configValidation.js b/test/configValidation.js index f006314..dbed9ab 100644 --- a/test/configValidation.js +++ b/test/configValidation.js @@ -11,8 +11,7 @@ module.exports.tests.validate = function(test, common) { var config = { esclient: {}, schema: { - indexName: 'example_index', - typeName: 'example_type' + indexName: 'example_index' } }; @@ -30,8 +29,7 @@ module.exports.tests.validate = function(test, common) { }, esclient: {}, schema: { - indexName: 'example_index', - typeName: 'example_type' + indexName: 'example_index' } }; @@ -51,8 +49,7 @@ module.exports.tests.validate = function(test, common) { }, esclient: {}, schema: { - indexName: 'example_index', - typeName: 'example_type' + indexName: 'example_index' } }; @@ -74,8 +71,7 @@ module.exports.tests.validate = function(test, common) { }, esclient: {}, schema: { - indexName: 'example_index', - typeName: 'example_type' + indexName: 'example_index' } }; @@ -94,8 +90,7 @@ module.exports.tests.validate = function(test, common) { }, esclient: {}, schema: { - indexName: 'example_index', - typeName: 'example_type' + indexName: 'example_index' } }; @@ -115,8 +110,7 @@ module.exports.tests.validate = function(test, common) { }, esclient: {}, schema: { - indexName: 'example_index', - typeName: 'example_type' + indexName: 'example_index' } }; @@ -138,8 +132,7 @@ module.exports.tests.validate = function(test, common) { }, esclient: {}, schema: { - indexName: 'example_index', - typeName: 'example_type' + indexName: 'example_index' } }; @@ -160,8 +153,7 @@ module.exports.tests.validate = function(test, common) { }, esclient: value, schema: { - indexName: 'example_index', - typeName: 'example_type' + indexName: 'example_index' } }; @@ -186,8 +178,7 @@ module.exports.tests.validate = function(test, common) { requestTimeout: value }, schema: { - indexName: 'example_index', - typeName: 'example_type' + indexName: 'example_index' } }; @@ -210,8 +201,7 @@ module.exports.tests.validate = function(test, common) { requestTimeout: 17.3 }, schema: { - indexName: 'example_index', - typeName: 'example_type' + indexName: 'example_index' } }; @@ -233,8 +223,7 @@ module.exports.tests.validate = function(test, common) { requestTimeout: -1 }, schema: { - indexName: 'example_index', - typeName: 'example_type' + indexName: 'example_index' } }; @@ -315,8 +304,7 @@ module.exports.tests.validate = function(test, common) { }, esclient: {}, schema: { - indexName: 'example_index', - typeName: 'example_type' + indexName: 'example_index' } }; @@ -344,8 +332,7 @@ module.exports.tests.validate = function(test, common) { requestTimeout: 17 }, schema: { - indexName: 'example_index', - typeName: 'example_type' + indexName: 'example_index' } }; @@ -374,8 +361,7 @@ module.exports.tests.validate = function(test, common) { requestTimeout: 17 }, schema: { - indexName: 'example_index', - typeName: 'example_type' + indexName: 'example_index' } }; diff --git a/test/stream.js b/test/stream.js index 9d0afff..64ad301 100644 --- a/test/stream.js +++ b/test/stream.js @@ -44,7 +44,6 @@ module.exports.tests.functional_example = function(test, common) { stream.pipe(assertStream); stream.write({ _index: 'foo', - _type: 'foo', _id: 'foo', data: {} });