From c099dbff0fbb991538f9b8ce2784063d1b2019ab Mon Sep 17 00:00:00 2001
From: Jared Noble <jsnoble1@gmail.com>
Date: Thu, 13 Jul 2023 14:53:53 -0700
Subject: [PATCH 01/14] explicitly list test resources to be used for each test

---
 e2e/package.json                              |  6 ++---
 package.json                                  | 22 ++++---------------
 packages/elasticsearch-api/package.json       | 14 ++++++------
 packages/elasticsearch-store/package.json     | 10 ++++-----
 .../src/test-helpers/elasticsearch.ts         | 15 ++++++++++++-
 packages/scripts/src/helpers/config.ts        | 22 +++++++++++++++++++
 packages/scripts/src/helpers/misc.ts          | 10 ++++++++-
 packages/teraslice/package.json               |  6 ++---
 8 files changed, 67 insertions(+), 38 deletions(-)

diff --git a/e2e/package.json b/e2e/package.json
index 5d994dfbf26..b6729bcc928 100644
--- a/e2e/package.json
+++ b/e2e/package.json
@@ -24,9 +24,9 @@
         "logs": "./scripts/logs.sh",
         "logs-follow": "./scripts/logs.sh -f",
         "setup": "yarn --silent",
-        "test": "ts-scripts test --suite e2e --",
-        "test:debug": "ts-scripts test --suite e2e --debug --",
-        "test:watch": "ts-scripts test --suite e2e --watch --"
+        "test": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite e2e --",
+        "test:debug": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite e2e --debug --",
+        "test:watch": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite e2e --watch --"
     },
     "resolutions": {
         "ms": "^2.1.3"
diff --git a/package.json b/package.json
index 60c331fceee..f312431dabe 100644
--- a/package.json
+++ b/package.json
@@ -68,24 +68,10 @@
         "target": "es2019",
         "tests": {
             "suites": {
-                "e2e": [
-                    "elasticsearch",
-                    "kafka"
-                ],
-                "elasticsearch": [
-                    "elasticsearch"
-                ],
-                "search": [
-                    "elasticsearch",
-                    "opensearch"
-                ],
-                "restrained": [
-                    "restrained_elasticsearch",
-                    "restrained_opensearch"
-                ],
-                "kafka": [
-                    "kafka"
-                ],
+                "e2e": [],
+                "elasticsearch": [],
+                "search": [],
+                "restrained": [],
                 "unit": []
             }
         },
diff --git a/packages/elasticsearch-api/package.json b/packages/elasticsearch-api/package.json
index ff690a07521..2e1e8188599 100644
--- a/packages/elasticsearch-api/package.json
+++ b/packages/elasticsearch-api/package.json
@@ -13,13 +13,13 @@
     "main": "index.js",
     "typings": "types/index.d.ts",
     "scripts": {
-        "test": "ts-scripts test . --",
-        "test:7": "ELASTICSEARCH_VERSION='7.9.3' yarn run test",
-        "test:8": "ELASTICSEARCH_VERSION='8.1.2' yarn run test",
-        "test:debug": "ts-scripts test --debug . --",
-        "test:legacy": "LEGACY_CLIENT=true yarn run test:debug",
-        "test:opensearch": "TEST_OPENSEARCH='true' RESTRAINED_OPENSEARCH='true' yarn run test",
-        "test:watch": "ts-scripts test --watch . --"
+        "test": "TEST_RESTRAINED_ELASTICSEARCH='true' ts-scripts test . --",
+        "test:7": "TEST_RESTRAINED_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' yarn run test",
+        "test:8": "TEST_RESTRAINED_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='8.1.2' yarn run test",
+        "test:debug": "TEST_RESTRAINED_ELASTICSEARCH='true' ts-scripts test --debug . --",
+        "test:legacy": "TEST_RESTRAINED_ELASTICSEARCH='true' LEGACY_CLIENT=true yarn run test:debug",
+        "test:opensearch": "TEST_RESTRAINED_OPENSEARCH='true' RESTRAINED_OPENSEARCH='true' yarn run test",
+        "test:watch": "TEST_RESTRAINED_ELASTICSEARCH='true' ts-scripts test --watch . --"
     },
     "dependencies": {
         "@terascope/types": "^0.11.2",
diff --git a/packages/elasticsearch-store/package.json b/packages/elasticsearch-store/package.json
index b6eeb3827ff..540661a7f10 100644
--- a/packages/elasticsearch-store/package.json
+++ b/packages/elasticsearch-store/package.json
@@ -19,11 +19,11 @@
     "scripts": {
         "build": "tsc --build",
         "build:watch": "yarn build --watch",
-        "test": "ts-scripts test . --",
-        "test:7": "ELASTICSEARCH_VERSION='7.9.3' yarn run test",
-        "test:8": "ELASTICSEARCH_VERSION='8.1.2' yarn run test",
-        "test:debug": "ts-scripts test --debug . --",
-        "test:legacy": "LEGACY_CLIENT=true yarn run test",
+        "test": "TEST_ELASTICSEARCH='true' ts-scripts test . --",
+        "test:7": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' yarn run test",
+        "test:8": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='8.1.2' yarn run test",
+        "test:debug": "TEST_ELASTICSEARCH='true' ts-scripts test --debug . --",
+        "test:legacy": "TEST_ELASTICSEARCH='true' LEGACY_CLIENT=true yarn run test",
         "test:opensearch1": "TEST_OPENSEARCH='true' yarn run test",
         "test:opensearch2": "TEST_OPENSEARCH='true' OPENSEARCH_VERSION='2.8.0' yarn run test",
         "test:watch": "ts-scripts test --watch . --"
diff --git a/packages/elasticsearch-store/src/test-helpers/elasticsearch.ts b/packages/elasticsearch-store/src/test-helpers/elasticsearch.ts
index ca371ef25c2..20f2544dab0 100644
--- a/packages/elasticsearch-store/src/test-helpers/elasticsearch.ts
+++ b/packages/elasticsearch-store/src/test-helpers/elasticsearch.ts
@@ -25,7 +25,7 @@ export async function makeClient(): Promise<Client> {
         host = OPENSEARCH_HOST;
     }
 
-    if (process.env.RESTRAINED_OPENSEARCH) {
+    if (process.env.TEST_RESTRAINED_OPENSEARCH) {
         host = RESTRAINED_OPENSEARCH_HOST;
     }
 
@@ -229,6 +229,19 @@ export function getTestENVClientInfo()
             minorVersion
         };
     }
+
+    if (process.env.TEST_RESTRAINED_OPENSEARCH != null) {
+        const version = OPENSEARCH_VERSION;
+        const [majorVersion, minorVersion] = parseVersion(version);
+
+        return {
+            host: RESTRAINED_OPENSEARCH_HOST,
+            distribution: ElasticsearchDistribution.opensearch,
+            version,
+            majorVersion,
+            minorVersion
+        };
+    }
     const version = ELASTICSEARCH_VERSION;
     const [majorVersion, minorVersion] = parseVersion(version);
 
diff --git a/packages/scripts/src/helpers/config.ts b/packages/scripts/src/helpers/config.ts
index b26a2e21dc2..63ab7c969f7 100644
--- a/packages/scripts/src/helpers/config.ts
+++ b/packages/scripts/src/helpers/config.ts
@@ -2,6 +2,7 @@ import { address } from 'ip';
 import {
     toBoolean, toSafeString, isCI, toIntegerOrThrow
 } from '@terascope/utils';
+import { Service } from './interfaces.js';
 
 const forceColor = process.env.FORCE_COLOR || '1';
 export const FORCE_COLOR = toBoolean(forceColor)
@@ -112,3 +113,24 @@ export const JEST_MAX_WORKERS = process.env.JEST_MAX_WORKERS
     : undefined;
 
 export const NPM_DEFAULT_REGISTRY = 'https://registry.npmjs.org/';
+
+const {
+    TEST_OPENSEARCH = undefined,
+    TEST_ELASTICSEARCH = undefined,
+    TEST_KAFKA = undefined,
+    TEST_MINIO = undefined,
+    TEST_RESTRAINED_OPENSEARCH = undefined,
+    TEST_RESTRAINED_ELASTICSEARCH = undefined,
+    TEST_RABBITMQ = undefined
+} = process.env;
+
+export const ENV_SERVICES = [
+    toBoolean(TEST_OPENSEARCH) ? Service.Opensearch : undefined,
+    toBoolean(TEST_ELASTICSEARCH) ? Service.Elasticsearch : undefined,
+    toBoolean(TEST_KAFKA) ? Service.Kafka : undefined,
+    toBoolean(TEST_MINIO) ? Service.Minio : undefined,
+    toBoolean(TEST_RESTRAINED_OPENSEARCH) ? Service.RestrainedOpensearch : undefined,
+    toBoolean(TEST_RESTRAINED_ELASTICSEARCH) ? Service.RestrainedElasticsearch : undefined,
+    toBoolean(TEST_RABBITMQ) ? Service.RabbitMQ : undefined,
+]
+    .filter(Boolean);
diff --git a/packages/scripts/src/helpers/misc.ts b/packages/scripts/src/helpers/misc.ts
index be69da43963..2841bc81ecd 100644
--- a/packages/scripts/src/helpers/misc.ts
+++ b/packages/scripts/src/helpers/misc.ts
@@ -5,7 +5,10 @@ import defaultsDeep from 'lodash/defaultsDeep';
 import { isPlainObject, get, toTitleCase } from '@terascope/utils';
 import sortPackageJson from 'sort-package-json';
 import { PackageInfo, RootPackageInfo, Service } from './interfaces';
-import { NPM_DEFAULT_REGISTRY, DEV_TAG, DEV_DOCKER_IMAGE } from './config';
+import {
+    NPM_DEFAULT_REGISTRY, DEV_TAG, DEV_DOCKER_IMAGE,
+    ENV_SERVICES
+} from './config';
 import signale from './signale';
 
 let rootDir: string | undefined;
@@ -87,6 +90,11 @@ export function getAvailableTestSuites(): string[] {
 
 export function getServicesForSuite(suite: string): Service[] {
     const services = getRootInfo().terascope.tests.suites[suite] || [];
+
+    if (ENV_SERVICES.length) {
+        services.push(...ENV_SERVICES);
+    }
+
     const invalidServices = services.filter((name) => !Object.values(Service).includes(name));
     if (invalidServices.length) {
         const actual = invalidServices.join(', ');
diff --git a/packages/teraslice/package.json b/packages/teraslice/package.json
index f0e93719ecc..7333da75fef 100644
--- a/packages/teraslice/package.json
+++ b/packages/teraslice/package.json
@@ -28,9 +28,9 @@
     ],
     "scripts": {
         "start": "node service.js",
-        "test": "ts-scripts test . --",
-        "test:debug": "ts-scripts test --debug . --",
-        "test:watch": "ts-scripts test --watch . --"
+        "test": "TEST_ELASTICSEARCH='true' ts-scripts test . --",
+        "test:debug": "TEST_ELASTICSEARCH='true' ts-scripts test --debug . --",
+        "test:watch": "TEST_ELASTICSEARCH='true' ts-scripts test --watch . --"
     },
     "resolutions": {
         "debug": "^4.3.4",

From ba23cdac356da016a247c2e545c9b0a1a89f3914 Mon Sep 17 00:00:00 2001
From: Jared Noble <jsnoble1@gmail.com>
Date: Thu, 13 Jul 2023 14:57:27 -0700
Subject: [PATCH 02/14] fix type

---
 packages/scripts/src/helpers/config.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/scripts/src/helpers/config.ts b/packages/scripts/src/helpers/config.ts
index 63ab7c969f7..06ee3ccb454 100644
--- a/packages/scripts/src/helpers/config.ts
+++ b/packages/scripts/src/helpers/config.ts
@@ -133,4 +133,4 @@ export const ENV_SERVICES = [
     toBoolean(TEST_RESTRAINED_ELASTICSEARCH) ? Service.RestrainedElasticsearch : undefined,
     toBoolean(TEST_RABBITMQ) ? Service.RabbitMQ : undefined,
 ]
-    .filter(Boolean);
+    .filter(Boolean) as Service[];

From c56cf3d95be824ab296effc3d69ee765d2aa2d28 Mon Sep 17 00:00:00 2001
From: Jared Noble <jsnoble1@gmail.com>
Date: Thu, 13 Jul 2023 15:05:50 -0700
Subject: [PATCH 03/14] fix file type reference

---
 packages/scripts/src/helpers/config.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/scripts/src/helpers/config.ts b/packages/scripts/src/helpers/config.ts
index 06ee3ccb454..46af1dd5804 100644
--- a/packages/scripts/src/helpers/config.ts
+++ b/packages/scripts/src/helpers/config.ts
@@ -2,7 +2,7 @@ import { address } from 'ip';
 import {
     toBoolean, toSafeString, isCI, toIntegerOrThrow
 } from '@terascope/utils';
-import { Service } from './interfaces.js';
+import { Service } from './interfaces';
 
 const forceColor = process.env.FORCE_COLOR || '1';
 export const FORCE_COLOR = toBoolean(forceColor)

From 5ad1ac6193aa6eb133417ed630f1cda39e132303 Mon Sep 17 00:00:00 2001
From: Jared Noble <jsnoble1@gmail.com>
Date: Fri, 14 Jul 2023 08:34:52 -0700
Subject: [PATCH 04/14] fix unit tests for services startup

---
 package.json                          | 5 ++++-
 packages/scripts/test/service-spec.ts | 4 ++--
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/package.json b/package.json
index f312431dabe..acede75b8f6 100644
--- a/package.json
+++ b/package.json
@@ -72,7 +72,10 @@
                 "elasticsearch": [],
                 "search": [],
                 "restrained": [],
-                "unit": []
+                "unit": [],
+                "_for_testing_": [
+                    "elasticsearch"
+                ]
             }
         },
         "docker": {
diff --git a/packages/scripts/test/service-spec.ts b/packages/scripts/test/service-spec.ts
index fb1cd276854..4c51e92207c 100644
--- a/packages/scripts/test/service-spec.ts
+++ b/packages/scripts/test/service-spec.ts
@@ -23,14 +23,14 @@ describe('services', () => {
 
     describe('pullServices', () => {
         it('should throw error if service image is invalid', async () => {
-            await expect(services.pullServices('e2e', options))
+            await expect(services.pullServices('_for_testing_', options))
                 .rejects.toThrowWithMessage(TSError, /w*Failed to pull services for test suite*\w/);
         });
     });
 
     describe('ensureServices', () => {
         it('should throw if service has an incorrect setting', async () => {
-            await expect(services.ensureServices('e2e', options))
+            await expect(services.ensureServices('_for_testing_', options))
                 .rejects.toThrowWithMessage(TSError, /w*Unable to find image*\w/);
         });
 

From 4942aaaec1cfa1a70543bd2ae9ab6b23f394c4eb Mon Sep 17 00:00:00 2001
From: Jared Noble <jsnoble1@gmail.com>
Date: Fri, 14 Jul 2023 09:50:08 -0700
Subject: [PATCH 05/14] fix double references

---
 packages/elasticsearch-api/package.json   |  8 ++++----
 packages/elasticsearch-store/package.json | 10 +++++-----
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/packages/elasticsearch-api/package.json b/packages/elasticsearch-api/package.json
index 2e1e8188599..29b9b27c6a3 100644
--- a/packages/elasticsearch-api/package.json
+++ b/packages/elasticsearch-api/package.json
@@ -14,11 +14,11 @@
     "typings": "types/index.d.ts",
     "scripts": {
         "test": "TEST_RESTRAINED_ELASTICSEARCH='true' ts-scripts test . --",
-        "test:7": "TEST_RESTRAINED_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' yarn run test",
-        "test:8": "TEST_RESTRAINED_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='8.1.2' yarn run test",
+        "test:7": "TEST_RESTRAINED_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' ts-scripts test . --",
+        "test:8": "TEST_RESTRAINED_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='8.1.2' ts-scripts test . --",
         "test:debug": "TEST_RESTRAINED_ELASTICSEARCH='true' ts-scripts test --debug . --",
-        "test:legacy": "TEST_RESTRAINED_ELASTICSEARCH='true' LEGACY_CLIENT=true yarn run test:debug",
-        "test:opensearch": "TEST_RESTRAINED_OPENSEARCH='true' RESTRAINED_OPENSEARCH='true' yarn run test",
+        "test:legacy": "TEST_RESTRAINED_ELASTICSEARCH='true' LEGACY_CLIENT=true ts-scripts test . --",
+        "test:opensearch": "TEST_RESTRAINED_OPENSEARCH='true' RESTRAINED_OPENSEARCH='true' ts-scripts test . --",
         "test:watch": "TEST_RESTRAINED_ELASTICSEARCH='true' ts-scripts test --watch . --"
     },
     "dependencies": {
diff --git a/packages/elasticsearch-store/package.json b/packages/elasticsearch-store/package.json
index 540661a7f10..0564ff1a77d 100644
--- a/packages/elasticsearch-store/package.json
+++ b/packages/elasticsearch-store/package.json
@@ -20,12 +20,12 @@
         "build": "tsc --build",
         "build:watch": "yarn build --watch",
         "test": "TEST_ELASTICSEARCH='true' ts-scripts test . --",
-        "test:7": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' yarn run test",
-        "test:8": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='8.1.2' yarn run test",
+        "test:7": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' ts-scripts test . --",
+        "test:8": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='8.1.2' ts-scripts test . --",
         "test:debug": "TEST_ELASTICSEARCH='true' ts-scripts test --debug . --",
-        "test:legacy": "TEST_ELASTICSEARCH='true' LEGACY_CLIENT=true yarn run test",
-        "test:opensearch1": "TEST_OPENSEARCH='true' yarn run test",
-        "test:opensearch2": "TEST_OPENSEARCH='true' OPENSEARCH_VERSION='2.8.0' yarn run test",
+        "test:legacy": "TEST_ELASTICSEARCH='true' LEGACY_CLIENT=true ts-scripts test . --",
+        "test:opensearch1": "TEST_OPENSEARCH='true' ts-scripts test . --",
+        "test:opensearch2": "TEST_OPENSEARCH='true' OPENSEARCH_VERSION='2.8.0' ts-scripts test . --",
         "test:watch": "ts-scripts test --watch . --"
     },
     "dependencies": {

From 93241fe7889300387e26ee1b76e1381299e952e7 Mon Sep 17 00:00:00 2001
From: Jared Noble <jsnoble1@gmail.com>
Date: Tue, 18 Jul 2023 08:49:57 -0700
Subject: [PATCH 06/14] test fixes

---
 packages/elasticsearch-api/package.json       |  2 +-
 .../test/bulk-send-dlq-spec.js                | 34 ++++---------------
 .../test/bulk-send-limit-spec.js              | 24 ++-----------
 .../src/elasticsearch-client/create-client.ts |  7 ++--
 .../src/test-helpers/config.ts                |  2 +-
 .../src/test-helpers/elasticsearch.ts         | 11 ++++--
 6 files changed, 24 insertions(+), 56 deletions(-)

diff --git a/packages/elasticsearch-api/package.json b/packages/elasticsearch-api/package.json
index 29b9b27c6a3..d96dd89f27a 100644
--- a/packages/elasticsearch-api/package.json
+++ b/packages/elasticsearch-api/package.json
@@ -18,7 +18,7 @@
         "test:8": "TEST_RESTRAINED_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='8.1.2' ts-scripts test . --",
         "test:debug": "TEST_RESTRAINED_ELASTICSEARCH='true' ts-scripts test --debug . --",
         "test:legacy": "TEST_RESTRAINED_ELASTICSEARCH='true' LEGACY_CLIENT=true ts-scripts test . --",
-        "test:opensearch": "TEST_RESTRAINED_OPENSEARCH='true' RESTRAINED_OPENSEARCH='true' ts-scripts test . --",
+        "test:opensearch": "TEST_RESTRAINED_OPENSEARCH='true' ts-scripts test --debug . --",
         "test:watch": "TEST_RESTRAINED_ELASTICSEARCH='true' ts-scripts test --watch . --"
     },
     "dependencies": {
diff --git a/packages/elasticsearch-api/test/bulk-send-dlq-spec.js b/packages/elasticsearch-api/test/bulk-send-dlq-spec.js
index 4f7873992c2..a8562597051 100644
--- a/packages/elasticsearch-api/test/bulk-send-dlq-spec.js
+++ b/packages/elasticsearch-api/test/bulk-send-dlq-spec.js
@@ -9,37 +9,18 @@ const { ElasticsearchTestHelpers } = require('elasticsearch-store');
 const elasticsearchAPI = require('../index');
 
 const {
-    makeClient, cleanupIndex,
-    EvenDateData, TEST_INDEX_PREFIX,
-    createMappingFromDatatype
+    makeClient, cleanupIndex, EvenDateData,
+    TEST_INDEX_PREFIX, createMappingFromDatatype,
+    formatUploadData
 } = ElasticsearchTestHelpers;
 
 const THREE_MINUTES = 3 * 60 * 1000;
 
 jest.setTimeout(THREE_MINUTES + 60000);
 
-function formatUploadData(
-    index, data, isES8ClientTest = false
-) {
-    const results = [];
-
-    data.forEach((record, i) => {
-        const meta = { _index: index, _id: i + 1 };
-
-        if (!isES8ClientTest) {
-            meta._type = '_doc';
-        }
-
-        results.push({ action: { index: meta }, data: record });
-    });
-
-    return results;
-}
-
 describe('bulkSend', () => {
     let client;
     let api;
-    let isElasticsearch8 = false;
 
     beforeAll(async () => {
         client = await makeClient();
@@ -52,7 +33,6 @@ describe('bulkSend', () => {
         beforeAll(async () => {
             await cleanupIndex(client, index);
             api = elasticsearchAPI(client, logger, { _dead_letter_action: 'kafka_dead_letter' });
-            isElasticsearch8 = api.isElasticsearch8();
 
             const overrides = {
                 settings: {
@@ -79,7 +59,7 @@ describe('bulkSend', () => {
 
             const docs = cloneDeep(EvenDateData.data.slice(0, 2));
 
-            const result = await diffApi.bulkSend(formatUploadData(index, docs, isElasticsearch8));
+            const result = await diffApi.bulkSend(formatUploadData(index, docs, true));
 
             expect(result).toBe(2);
         });
@@ -91,7 +71,7 @@ describe('bulkSend', () => {
 
             docs[0].bytes = 'this is a bad value';
 
-            await expect(diffApi.bulkSend(formatUploadData(index, docs, isElasticsearch8)))
+            await expect(diffApi.bulkSend(formatUploadData(index, docs, true)))
                 .rejects.toThrow();
         });
 
@@ -101,7 +81,7 @@ describe('bulkSend', () => {
 
             docs[0].bytes = 'this is a bad value';
 
-            const result = await api.bulkSend(formatUploadData(index, docs, isElasticsearch8));
+            const result = await api.bulkSend(formatUploadData(index, docs, true));
 
             // 1 good doc  - so only 1 row affected
             expect(result).toBe(1);
@@ -113,7 +93,7 @@ describe('bulkSend', () => {
         it('should return a count if not un-retryable records if dlq is set', async () => {
             const docs = cloneDeep(EvenDateData.data.slice(0, 2));
 
-            const result = await api.bulkSend(formatUploadData(index, docs, isElasticsearch8));
+            const result = await api.bulkSend(formatUploadData(index, docs, true));
 
             expect(result).toBe(2);
         });
diff --git a/packages/elasticsearch-api/test/bulk-send-limit-spec.js b/packages/elasticsearch-api/test/bulk-send-limit-spec.js
index 724ada1f85d..f72293b5138 100644
--- a/packages/elasticsearch-api/test/bulk-send-limit-spec.js
+++ b/packages/elasticsearch-api/test/bulk-send-limit-spec.js
@@ -10,35 +10,16 @@ const elasticsearchAPI = require('../index');
 
 const {
     makeClient, cleanupIndex, waitForData,
-    EvenDateData, TEST_INDEX_PREFIX
+    EvenDateData, TEST_INDEX_PREFIX, formatUploadData
 } = ElasticsearchTestHelpers;
 
 const THREE_MINUTES = 3 * 60 * 1000;
 
 jest.setTimeout(THREE_MINUTES + 60000);
 
-function formatUploadData(
-    index, data, isES8ClientTest = false
-) {
-    const results = [];
-
-    data.forEach((record) => {
-        const meta = { _index: index };
-
-        if (!isES8ClientTest) {
-            meta._type = '_doc';
-        }
-
-        results.push({ action: { index: meta }, data: record });
-    });
-
-    return results;
-}
-
 describe('bulkSend', () => {
     let client;
     let api;
-    let isElasticsearch8 = false;
 
     beforeAll(async () => {
         client = await makeClient();
@@ -51,7 +32,6 @@ describe('bulkSend', () => {
         beforeAll(async () => {
             await cleanupIndex(client, index);
             api = elasticsearchAPI(client, logger);
-            isElasticsearch8 = api.isElasticsearch8();
         });
 
         afterAll(async () => {
@@ -62,7 +42,7 @@ describe('bulkSend', () => {
             const chunkedData = chunk(EvenDateData.data, 50);
 
             await pMap(chunkedData, async (cData) => {
-                const formattedData = formatUploadData(index, cData, isElasticsearch8);
+                const formattedData = formatUploadData(index, cData, true);
                 return api.bulkSend(formattedData);
             }, { concurrency: 9 });
 
diff --git a/packages/elasticsearch-store/src/elasticsearch-client/create-client.ts b/packages/elasticsearch-store/src/elasticsearch-client/create-client.ts
index c6784da76c6..ec2f3700bcd 100644
--- a/packages/elasticsearch-store/src/elasticsearch-client/create-client.ts
+++ b/packages/elasticsearch-store/src/elasticsearch-client/create-client.ts
@@ -11,7 +11,8 @@ import { Client } from './client';
 import { logWrapper } from './log-wrapper';
 import { ClientConfig } from './interfaces';
 
-const clientList = [opensearch1, opensearch2, elasticsearch7, elasticsearch6, elasticsearch8];
+// const clientList = [opensearch1, opensearch2, elasticsearch7, elasticsearch6, elasticsearch8];
+const clientList = [opensearch1];
 
 export async function createClient(
     config: ClientConfig,
@@ -71,6 +72,8 @@ async function getClientMetadata(
                 } as any;
             }
         } catch (err) {
+            console.log('\n\n err', err.message, '\n\n', config)
+            logger.error(err)
             if (logger.level() === 10) {
                 logger.error(err);
             }
@@ -97,7 +100,7 @@ export async function getBaseClient(
         if (distribution === ElasticsearchDistribution.opensearch) {
             if (majorVersion === 1) {
                 const client = new opensearch1.Client(config as any);
-
+                client.indices.create
                 logger.debug('Creating an opensearch client v1');
 
                 return client;
diff --git a/packages/elasticsearch-store/src/test-helpers/config.ts b/packages/elasticsearch-store/src/test-helpers/config.ts
index fefc05c5f5a..4cc68fd3a44 100644
--- a/packages/elasticsearch-store/src/test-helpers/config.ts
+++ b/packages/elasticsearch-store/src/test-helpers/config.ts
@@ -10,7 +10,7 @@ const {
     OPENSEARCH_VERSION = '1.3.0',
     OPENSEARCH_HOST = `http://${OPENSEARCH_USER}:${OPENSEARCH_PASSWORD}@${OPENSEARCH_HOSTNAME}`,
     RESTRAINED_OPENSEARCH_PORT = process.env.RESTRAINED_OPENSEARCH_PORT || '49206',
-    RESTRAINED_OPENSEARCH_HOST = `http://${OPENSEARCH_USER}:${OPENSEARCH_PASSWORD}@http://localhost:${RESTRAINED_OPENSEARCH_PORT}`,
+    RESTRAINED_OPENSEARCH_HOST = `http://${OPENSEARCH_USER}:${OPENSEARCH_PASSWORD}@localhost:${RESTRAINED_OPENSEARCH_PORT}`,
 } = process.env;
 
 export {
diff --git a/packages/elasticsearch-store/src/test-helpers/elasticsearch.ts b/packages/elasticsearch-store/src/test-helpers/elasticsearch.ts
index 20f2544dab0..ab2bbac79dd 100644
--- a/packages/elasticsearch-store/src/test-helpers/elasticsearch.ts
+++ b/packages/elasticsearch-store/src/test-helpers/elasticsearch.ts
@@ -145,7 +145,7 @@ export async function populateIndex(
 }
 
 export function formatUploadData(
-    index: string, data: any[]
+    index: string, data: any[], apiCompatibility = false
 ): Record<string, any>[] {
     const results: any[] = [];
 
@@ -159,8 +159,13 @@ export function formatUploadData(
         if (DataEntity.isDataEntity(record) && record.getKey()) {
             meta._id = record.getKey();
         }
-
-        results.push({ index: meta }, record);
+        // This format is used by elasticsearch-api and elasticsearch-assets
+        if (apiCompatibility) {
+            results.push({ action: { index: meta }, data: record });
+        } else {
+            // this is used for raw elasticsearch bulk queries
+            results.push({ index: meta }, record);
+        }
     });
 
     return results;

From 06e436e0b0d6cac81dea990780b451e533b559db Mon Sep 17 00:00:00 2001
From: Jared Noble <jsnoble1@gmail.com>
Date: Tue, 18 Jul 2023 08:55:23 -0700
Subject: [PATCH 07/14] fix createClient

---
 .../src/elasticsearch-client/create-client.ts                 | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/packages/elasticsearch-store/src/elasticsearch-client/create-client.ts b/packages/elasticsearch-store/src/elasticsearch-client/create-client.ts
index ec2f3700bcd..e7b5bab372e 100644
--- a/packages/elasticsearch-store/src/elasticsearch-client/create-client.ts
+++ b/packages/elasticsearch-store/src/elasticsearch-client/create-client.ts
@@ -72,8 +72,6 @@ async function getClientMetadata(
                 } as any;
             }
         } catch (err) {
-            console.log('\n\n err', err.message, '\n\n', config)
-            logger.error(err)
             if (logger.level() === 10) {
                 logger.error(err);
             }
@@ -100,7 +98,7 @@ export async function getBaseClient(
         if (distribution === ElasticsearchDistribution.opensearch) {
             if (majorVersion === 1) {
                 const client = new opensearch1.Client(config as any);
-                client.indices.create
+
                 logger.debug('Creating an opensearch client v1');
 
                 return client;

From 43061f902056123cf82ce14133bed28ce82d339d Mon Sep 17 00:00:00 2001
From: Jared Noble <jsnoble1@gmail.com>
Date: Wed, 19 Jul 2023 07:45:32 -0700
Subject: [PATCH 08/14] fix tests, pass dynamic search host test env, add ci
 tests

---
 .github/workflows/test.yml                    | 16 ++++++++---
 packages/elasticsearch-store/package.json     |  4 +--
 packages/scripts/src/helpers/config.ts        | 27 ++++++++++++++++---
 .../scripts/src/helpers/test-runner/utils.ts  |  5 ++++
 packages/teraslice/package.json               |  5 +++-
 packages/teraslice/test/test.config.js        |  6 +++--
 .../execution-controller-spec.js              |  1 +
 .../teraslice/test/workers/helpers/configs.js |  6 ++---
 8 files changed, 55 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 1dc7f7e8bfa..1b783cc4ecf 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -79,10 +79,20 @@ jobs:
           YARN_SETUP_ARGS: "--prod=false --silent"
 
       - name: Test elasticsearch 6
-        run: yarn --silent test --suite elasticsearch --elasticsearch-version 6.8.6 --elasticsearch-api-version 6.5
+        run: yarn --silent test
+        working-directory: ./packages/teraslice
 
       - name: Test elasticsearch 7
-        run: yarn --silent test --suite elasticsearch --elasticsearch-version 7.2.1 --elasticsearch-api-version 7.0
+        run:  yarn --silent test:elasticsearch7
+        working-directory: ./packages/teraslice
+
+      - name: Test opensearch 1.3
+        run: yarn --silent test:opensearch1
+        working-directory: ./packages/teraslice
+
+      - name: Test opensearch 2.8
+        run: yarn --silent test:opensearch2
+        working-directory: ./packages/teraslice
 
   elasticsearch-store-tests:
     runs-on: ubuntu-latest
@@ -116,7 +126,7 @@ jobs:
         working-directory: ./packages/elasticsearch-store
 
       - name: Test elasticsearch 7
-        run: yarn --silent test:7
+        run: yarn --silent test:elasticsearch7
         working-directory: ./packages/elasticsearch-store
 
       - name: Test opensearch 1.3
diff --git a/packages/elasticsearch-store/package.json b/packages/elasticsearch-store/package.json
index 0564ff1a77d..3ab0712ee9f 100644
--- a/packages/elasticsearch-store/package.json
+++ b/packages/elasticsearch-store/package.json
@@ -20,8 +20,8 @@
         "build": "tsc --build",
         "build:watch": "yarn build --watch",
         "test": "TEST_ELASTICSEARCH='true' ts-scripts test . --",
-        "test:7": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' ts-scripts test . --",
-        "test:8": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='8.1.2' ts-scripts test . --",
+        "test:elasticsearch7": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' ts-scripts test . --",
+        "test:elasticsearch8": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='8.1.2' ts-scripts test . --",
         "test:debug": "TEST_ELASTICSEARCH='true' ts-scripts test --debug . --",
         "test:legacy": "TEST_ELASTICSEARCH='true' LEGACY_CLIENT=true ts-scripts test . --",
         "test:opensearch1": "TEST_OPENSEARCH='true' ts-scripts test . --",
diff --git a/packages/scripts/src/helpers/config.ts b/packages/scripts/src/helpers/config.ts
index 46af1dd5804..25d2184bdd7 100644
--- a/packages/scripts/src/helpers/config.ts
+++ b/packages/scripts/src/helpers/config.ts
@@ -124,13 +124,32 @@ const {
     TEST_RABBITMQ = undefined
 } = process.env;
 
+const testOpensearch = toBoolean(TEST_OPENSEARCH);
+const testElasticsearch = toBoolean(TEST_ELASTICSEARCH);
+const testRestrainedOpensearch = toBoolean(TEST_RESTRAINED_OPENSEARCH);
+const testRestrainedElasticsearch = toBoolean(TEST_RESTRAINED_ELASTICSEARCH);
+
 export const ENV_SERVICES = [
-    toBoolean(TEST_OPENSEARCH) ? Service.Opensearch : undefined,
-    toBoolean(TEST_ELASTICSEARCH) ? Service.Elasticsearch : undefined,
+    testOpensearch ? Service.Opensearch : undefined,
+    testElasticsearch ? Service.Elasticsearch : undefined,
     toBoolean(TEST_KAFKA) ? Service.Kafka : undefined,
     toBoolean(TEST_MINIO) ? Service.Minio : undefined,
-    toBoolean(TEST_RESTRAINED_OPENSEARCH) ? Service.RestrainedOpensearch : undefined,
-    toBoolean(TEST_RESTRAINED_ELASTICSEARCH) ? Service.RestrainedElasticsearch : undefined,
+    testRestrainedOpensearch ? Service.RestrainedOpensearch : undefined,
+    testRestrainedElasticsearch ? Service.RestrainedElasticsearch : undefined,
     toBoolean(TEST_RABBITMQ) ? Service.RabbitMQ : undefined,
 ]
     .filter(Boolean) as Service[];
+
+let testHost;
+
+if (testElasticsearch) {
+    testHost = ELASTICSEARCH_HOST;
+} else if (testOpensearch) {
+    testHost = OPENSEARCH_HOST;
+} else if (testRestrainedOpensearch) {
+    testHost = RESTRAINED_OPENSEARCH_HOST;
+} else if (testRestrainedElasticsearch) {
+    testHost = RESTRAINED_ELASTICSEARCH_HOST;
+}
+
+export const SEARCH_TEST_HOST = testHost;
diff --git a/packages/scripts/src/helpers/test-runner/utils.ts b/packages/scripts/src/helpers/test-runner/utils.ts
index 4646cbf2764..cc33e2aa176 100644
--- a/packages/scripts/src/helpers/test-runner/utils.ts
+++ b/packages/scripts/src/helpers/test-runner/utils.ts
@@ -83,6 +83,7 @@ export function getEnv(options: TestOptions, suite?: string): ExecEnv {
             ELASTICSEARCH_HOST: config.ELASTICSEARCH_HOST,
             ELASTICSEARCH_VERSION: options.elasticsearchVersion,
             ELASTICSEARCH_API_VERSION: options.elasticsearchAPIVersion,
+            SEARCH_TEST_HOST: `${config.SEARCH_TEST_HOST}`
         });
     }
 
@@ -92,6 +93,7 @@ export function getEnv(options: TestOptions, suite?: string): ExecEnv {
             ELASTICSEARCH_HOST: config.RESTRAINED_ELASTICSEARCH_HOST,
             ELASTICSEARCH_VERSION: options.elasticsearchVersion,
             ELASTICSEARCH_API_VERSION: options.elasticsearchAPIVersion,
+            SEARCH_TEST_HOST: `${config.SEARCH_TEST_HOST}`
         });
     }
 
@@ -132,6 +134,8 @@ export function getEnv(options: TestOptions, suite?: string): ExecEnv {
             OPENSEARCH_HOST: config.OPENSEARCH_HOST,
             DISABLE_SECURITY_PLUGIN: true,
             DISABLE_INSTALL_DEMO_CONFIG: true,
+            SEARCH_TEST_HOST: `${config.SEARCH_TEST_HOST}`
+
         });
     }
 
@@ -145,6 +149,7 @@ export function getEnv(options: TestOptions, suite?: string): ExecEnv {
             RESTRAINED_OPENSEARCH_HOST: config.RESTRAINED_OPENSEARCH_HOST,
             DISABLE_SECURITY_PLUGIN: true,
             DISABLE_INSTALL_DEMO_CONFIG: true,
+            SEARCH_TEST_HOST: `${config.SEARCH_TEST_HOST}`
         });
     }
 
diff --git a/packages/teraslice/package.json b/packages/teraslice/package.json
index 7333da75fef..ef262401b89 100644
--- a/packages/teraslice/package.json
+++ b/packages/teraslice/package.json
@@ -30,7 +30,10 @@
         "start": "node service.js",
         "test": "TEST_ELASTICSEARCH='true' ts-scripts test . --",
         "test:debug": "TEST_ELASTICSEARCH='true' ts-scripts test --debug . --",
-        "test:watch": "TEST_ELASTICSEARCH='true' ts-scripts test --watch . --"
+        "test:watch": "TEST_ELASTICSEARCH='true' ts-scripts test --watch . --",
+        "test:elasticsearch7": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' ts-scripts test . --",
+        "test:opensearch1": "TEST_OPENSEARCH='true' ts-scripts test . --",
+        "test:opensearch2": "TEST_OPENSEARCH='true' OPENSEARCH_VERSION='2.8.0' ts-scripts test . --"
     },
     "resolutions": {
         "debug": "^4.3.4",
diff --git a/packages/teraslice/test/test.config.js b/packages/teraslice/test/test.config.js
index 9f0acfb18e8..a72aa38c2d3 100644
--- a/packages/teraslice/test/test.config.js
+++ b/packages/teraslice/test/test.config.js
@@ -7,17 +7,19 @@ const {
     ELASTICSEARCH_HOST = 'http://localhost:9200',
     TERASLICE_CLUSTER_NAME = newId(`${TEST_INDEX_PREFIX}teraslice`, true, 2),
     ELASTICSEARCH_VERSION = '6.8',
-    ELASTICSEARCH_API_VERSION = '6.5'
+    ELASTICSEARCH_API_VERSION = '6.5',
+    SEARCH_TEST_HOST = ELASTICSEARCH_HOST
 } = process.env;
 
 process.env.TERASLICE_CLUSTER_NAME = TERASLICE_CLUSTER_NAME;
 process.env.ELASTICSEARCH_HOST = ELASTICSEARCH_HOST;
 process.env.ELASTICSEARCH_API_VERSION = ELASTICSEARCH_API_VERSION;
 process.env.ELASTICSEARCH_VERSION = ELASTICSEARCH_VERSION;
+process.env.SEARCH_TEST_HOST = SEARCH_TEST_HOST;
 
 module.exports = {
     TEST_INDEX_PREFIX,
-    ELASTICSEARCH_HOST,
+    SEARCH_TEST_HOST,
     TERASLICE_CLUSTER_NAME,
     ELASTICSEARCH_VERSION,
     ELASTICSEARCH_API_VERSION
diff --git a/packages/teraslice/test/workers/execution-controller/execution-controller-spec.js b/packages/teraslice/test/workers/execution-controller/execution-controller-spec.js
index 44c66ee71b7..a2b0949c418 100644
--- a/packages/teraslice/test/workers/execution-controller/execution-controller-spec.js
+++ b/packages/teraslice/test/workers/execution-controller/execution-controller-spec.js
@@ -33,6 +33,7 @@ describe('ExecutionController', () => {
             exController.isExecutionFinished = true;
 
             await testContext.addExStore();
+
             ({ exStore } = testContext.stores);
 
             testContext.attachCleanup(() => exController.shutdown().catch(() => {
diff --git a/packages/teraslice/test/workers/helpers/configs.js b/packages/teraslice/test/workers/helpers/configs.js
index 4d02cef52c5..7b5da1a2023 100644
--- a/packages/teraslice/test/workers/helpers/configs.js
+++ b/packages/teraslice/test/workers/helpers/configs.js
@@ -5,7 +5,7 @@ const Chance = require('chance');
 const pickBy = require('lodash/pickBy');
 const { newId } = require('../../../lib/utils/id_utils');
 
-const { ELASTICSEARCH_HOST, ELASTICSEARCH_API_VERSION } = process.env;
+const { SEARCH_TEST_HOST, ELASTICSEARCH_API_VERSION } = process.env;
 
 const opsPath = path.join(__dirname, '..', 'fixtures', 'ops');
 
@@ -103,7 +103,7 @@ const newSysConfig = (options = {}) => {
             connectors: {
                 elasticsearch: {
                     default: {
-                        host: [ELASTICSEARCH_HOST],
+                        host: [SEARCH_TEST_HOST],
                         apiVersion: ELASTICSEARCH_API_VERSION,
                         requestTimeout: timeout,
                         deadTimeout: timeout
@@ -111,7 +111,7 @@ const newSysConfig = (options = {}) => {
                 },
                 'elasticsearch-next': {
                     default: {
-                        node: [ELASTICSEARCH_HOST],
+                        node: [SEARCH_TEST_HOST],
                         requestTimeout: timeout,
                         deadTimeout: timeout
                     }

From 12453b8d9bbd398a73017b6f36e9b9fd59a720dd Mon Sep 17 00:00:00 2001
From: Jared Noble <jsnoble1@gmail.com>
Date: Wed, 19 Jul 2023 08:35:44 -0700
Subject: [PATCH 09/14] fix sync issue

---
 packages/elasticsearch-store/package.json | 2 +-
 packages/teraslice/package.json           | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/packages/elasticsearch-store/package.json b/packages/elasticsearch-store/package.json
index 3ab0712ee9f..f8c83402ee2 100644
--- a/packages/elasticsearch-store/package.json
+++ b/packages/elasticsearch-store/package.json
@@ -20,9 +20,9 @@
         "build": "tsc --build",
         "build:watch": "yarn build --watch",
         "test": "TEST_ELASTICSEARCH='true' ts-scripts test . --",
+        "test:debug": "TEST_ELASTICSEARCH='true' ts-scripts test --debug . --",
         "test:elasticsearch7": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' ts-scripts test . --",
         "test:elasticsearch8": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='8.1.2' ts-scripts test . --",
-        "test:debug": "TEST_ELASTICSEARCH='true' ts-scripts test --debug . --",
         "test:legacy": "TEST_ELASTICSEARCH='true' LEGACY_CLIENT=true ts-scripts test . --",
         "test:opensearch1": "TEST_OPENSEARCH='true' ts-scripts test . --",
         "test:opensearch2": "TEST_OPENSEARCH='true' OPENSEARCH_VERSION='2.8.0' ts-scripts test . --",
diff --git a/packages/teraslice/package.json b/packages/teraslice/package.json
index ef262401b89..082fcc10bec 100644
--- a/packages/teraslice/package.json
+++ b/packages/teraslice/package.json
@@ -30,10 +30,10 @@
         "start": "node service.js",
         "test": "TEST_ELASTICSEARCH='true' ts-scripts test . --",
         "test:debug": "TEST_ELASTICSEARCH='true' ts-scripts test --debug . --",
-        "test:watch": "TEST_ELASTICSEARCH='true' ts-scripts test --watch . --",
         "test:elasticsearch7": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' ts-scripts test . --",
         "test:opensearch1": "TEST_OPENSEARCH='true' ts-scripts test . --",
-        "test:opensearch2": "TEST_OPENSEARCH='true' OPENSEARCH_VERSION='2.8.0' ts-scripts test . --"
+        "test:opensearch2": "TEST_OPENSEARCH='true' OPENSEARCH_VERSION='2.8.0' ts-scripts test . --",
+        "test:watch": "TEST_ELASTICSEARCH='true' ts-scripts test --watch . --"
     },
     "resolutions": {
         "debug": "^4.3.4",

From e8c17dbedbe28b05f7c4af23cfa1ae3ae1476c77 Mon Sep 17 00:00:00 2001
From: Jared Noble <jsnoble1@gmail.com>
Date: Thu, 20 Jul 2023 09:00:42 -0700
Subject: [PATCH 10/14] remove default empty string values for metrics. add
 explicit mappings

---
 .../lib/storage/backends/mappings/ex.json     | 45 +++++++++++++++++++
 .../execution-analytics.js                    |  4 +-
 2 files changed, 47 insertions(+), 2 deletions(-)

diff --git a/packages/teraslice/lib/storage/backends/mappings/ex.json b/packages/teraslice/lib/storage/backends/mappings/ex.json
index 78d7b70a4dd..afc81801cd3 100644
--- a/packages/teraslice/lib/storage/backends/mappings/ex.json
+++ b/packages/teraslice/lib/storage/backends/mappings/ex.json
@@ -43,6 +43,51 @@
                 "_slicer_stats": {
                     "type": "object"
                 },
+                "_slicer_stats.workers_available": {
+                    "type": "integer"
+                },
+                "_slicer_stats.workers_active": {
+                    "type": "integer"
+                },
+                "_slicer_stats.workers_joined": {
+                    "type": "integer"
+                },
+                "_slicer_stats.workers_reconnected": {
+                    "type": "integer"
+                },
+                "_slicer_stats.workers_disconnected": {
+                    "type": "integer"
+                },
+                "_slicer_stats.job_duration": {
+                    "type": "integer"
+                },
+                "_slicer_stats.failed": {
+                    "type": "integer"
+                },
+                "_slicer_stats.subslices": {
+                    "type": "integer"
+                },
+                "_slicer_stats.queued": {
+                    "type": "integer"
+                },
+                "_slicer_stats.slice_range_expansion": {
+                    "type": "integer"
+                },
+                "_slicer_stats.processed": {
+                    "type": "integer"
+                },
+                "_slicer_stats.slicers": {
+                    "type": "integer"
+                },
+                "_slicer_stats.subslice_by_key": {
+                    "type": "integer"
+                },
+                "_slicer_stats.started": {
+                    "type": "date"
+                },
+                "_slicer_stats.queuing_complete": {
+                    "type": "date"
+                },
                 "_created": {
                     "type": "date"
                 },
diff --git a/packages/teraslice/lib/workers/execution-controller/execution-analytics.js b/packages/teraslice/lib/workers/execution-controller/execution-analytics.js
index 9f36259bd5b..a5c9c35a6be 100644
--- a/packages/teraslice/lib/workers/execution-controller/execution-analytics.js
+++ b/packages/teraslice/lib/workers/execution-controller/execution-analytics.js
@@ -47,8 +47,8 @@ class ExecutionAnalytics {
             processed: 0,
             slicers: 0,
             subslice_by_key: 0,
-            started: '',
-            queuing_complete: ''
+            started: undefined,
+            queuing_complete: undefined
         };
 
         this.pushedAnalytics = {

From dee26b2fd03cf33e7cc3df98803faeb183d0a229 Mon Sep 17 00:00:00 2001
From: Jared Noble <jsnoble1@gmail.com>
Date: Thu, 20 Jul 2023 09:38:17 -0700
Subject: [PATCH 11/14] revert testing changes

---
 .../src/elasticsearch-client/create-client.ts                  | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/packages/elasticsearch-store/src/elasticsearch-client/create-client.ts b/packages/elasticsearch-store/src/elasticsearch-client/create-client.ts
index e7b5bab372e..c6784da76c6 100644
--- a/packages/elasticsearch-store/src/elasticsearch-client/create-client.ts
+++ b/packages/elasticsearch-store/src/elasticsearch-client/create-client.ts
@@ -11,8 +11,7 @@ import { Client } from './client';
 import { logWrapper } from './log-wrapper';
 import { ClientConfig } from './interfaces';
 
-// const clientList = [opensearch1, opensearch2, elasticsearch7, elasticsearch6, elasticsearch8];
-const clientList = [opensearch1];
+const clientList = [opensearch1, opensearch2, elasticsearch7, elasticsearch6, elasticsearch8];
 
 export async function createClient(
     config: ClientConfig,

From 6dcbd9f05a036ca42aadca19c106be508f13d1d8 Mon Sep 17 00:00:00 2001
From: Jared Noble <jsnoble1@gmail.com>
Date: Thu, 20 Jul 2023 09:38:35 -0700
Subject: [PATCH 12/14] bump: (minor) @terascope/scripts@0.53.0

---
 packages/scripts/package.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/scripts/package.json b/packages/scripts/package.json
index 464a2c48da4..c3867674cac 100644
--- a/packages/scripts/package.json
+++ b/packages/scripts/package.json
@@ -1,7 +1,7 @@
 {
     "name": "@terascope/scripts",
     "displayName": "Scripts",
-    "version": "0.52.0",
+    "version": "0.53.0",
     "description": "A collection of terascope monorepo scripts",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/scripts#readme",
     "bugs": {

From b3ce635dfcf32e016fedfb629bfab0982935bd95 Mon Sep 17 00:00:00 2001
From: Jared Noble <jsnoble1@gmail.com>
Date: Thu, 20 Jul 2023 09:39:43 -0700
Subject: [PATCH 13/14] release: (patch) teraslice@0.86.1

bump: (patch) @terascope/elasticsearch-api@3.6.2, @terascope/teraslice-state-storage@0.39.2

bump: (patch) elasticsearch-store@0.67.3, terafoundation@0.45.3
---
 e2e/package.json                              | 2 +-
 package.json                                  | 2 +-
 packages/elasticsearch-api/package.json       | 4 ++--
 packages/elasticsearch-store/package.json     | 2 +-
 packages/terafoundation/package.json          | 4 ++--
 packages/teraslice-state-storage/package.json | 4 ++--
 packages/teraslice/package.json               | 6 +++---
 7 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/e2e/package.json b/e2e/package.json
index b6729bcc928..cea28cde2f8 100644
--- a/e2e/package.json
+++ b/e2e/package.json
@@ -33,7 +33,7 @@
     },
     "devDependencies": {
         "bunyan": "^1.8.15",
-        "elasticsearch-store": "^0.67.2",
+        "elasticsearch-store": "^0.67.3",
         "fs-extra": "^10.1.0",
         "ms": "^2.1.3",
         "nanoid": "^3.3.4",
diff --git a/package.json b/package.json
index acede75b8f6..46d338b9c02 100644
--- a/package.json
+++ b/package.json
@@ -1,7 +1,7 @@
 {
     "name": "teraslice-workspace",
     "displayName": "Teraslice",
-    "version": "0.86.0",
+    "version": "0.86.1",
     "private": true,
     "homepage": "https://github.com/terascope/teraslice",
     "bugs": {
diff --git a/packages/elasticsearch-api/package.json b/packages/elasticsearch-api/package.json
index d96dd89f27a..cd0d3f1282f 100644
--- a/packages/elasticsearch-api/package.json
+++ b/packages/elasticsearch-api/package.json
@@ -1,7 +1,7 @@
 {
     "name": "@terascope/elasticsearch-api",
     "displayName": "Elasticsearch API",
-    "version": "3.6.1",
+    "version": "3.6.2",
     "description": "Elasticsearch client api used across multiple services, handles retries and exponential backoff",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/elasticsearch-api#readme",
     "bugs": {
@@ -31,7 +31,7 @@
         "@opensearch-project/opensearch": "^1.2.0",
         "@types/elasticsearch": "^5.0.40",
         "elasticsearch": "^15.4.1",
-        "elasticsearch-store": "^0.67.2",
+        "elasticsearch-store": "^0.67.3",
         "elasticsearch6": "npm:@elastic/elasticsearch@^6.7.0",
         "elasticsearch7": "npm:@elastic/elasticsearch@^7.0.0",
         "elasticsearch8": "npm:@elastic/elasticsearch@^8.0.0"
diff --git a/packages/elasticsearch-store/package.json b/packages/elasticsearch-store/package.json
index f8c83402ee2..ec9787c1dda 100644
--- a/packages/elasticsearch-store/package.json
+++ b/packages/elasticsearch-store/package.json
@@ -1,7 +1,7 @@
 {
     "name": "elasticsearch-store",
     "displayName": "Elasticsearch Store",
-    "version": "0.67.2",
+    "version": "0.67.3",
     "description": "An API for managing an elasticsearch index, with versioning and migration support.",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/elasticsearch-store#readme",
     "bugs": {
diff --git a/packages/terafoundation/package.json b/packages/terafoundation/package.json
index 349d42a21ed..624ddedb3c3 100644
--- a/packages/terafoundation/package.json
+++ b/packages/terafoundation/package.json
@@ -1,7 +1,7 @@
 {
     "name": "terafoundation",
     "displayName": "Terafoundation",
-    "version": "0.45.2",
+    "version": "0.45.3",
     "description": "A Clustering and Foundation tool for Terascope Tools",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/terafoundation#readme",
     "bugs": {
@@ -36,7 +36,7 @@
         "convict-format-with-moment": "^6.2.0",
         "convict-format-with-validator": "^6.2.0",
         "elasticsearch": "^15.4.1",
-        "elasticsearch-store": "^0.67.2",
+        "elasticsearch-store": "^0.67.3",
         "js-yaml": "^4.1.0",
         "mongoose": "~6.6.5",
         "nanoid": "^3.3.4",
diff --git a/packages/teraslice-state-storage/package.json b/packages/teraslice-state-storage/package.json
index a5e44432039..3155f511c80 100644
--- a/packages/teraslice-state-storage/package.json
+++ b/packages/teraslice-state-storage/package.json
@@ -1,7 +1,7 @@
 {
     "name": "@terascope/teraslice-state-storage",
     "displayName": "Teraslice State Storage",
-    "version": "0.39.1",
+    "version": "0.39.2",
     "description": "State storage operation api for teraslice",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/teraslice-state-storage#readme",
     "bugs": {
@@ -23,7 +23,7 @@
         "test:watch": "ts-scripts test --watch . --"
     },
     "dependencies": {
-        "@terascope/elasticsearch-api": "^3.6.1",
+        "@terascope/elasticsearch-api": "^3.6.2",
         "@terascope/utils": "^0.46.0"
     },
     "engines": {
diff --git a/packages/teraslice/package.json b/packages/teraslice/package.json
index 082fcc10bec..fded01f1e48 100644
--- a/packages/teraslice/package.json
+++ b/packages/teraslice/package.json
@@ -1,7 +1,7 @@
 {
     "name": "teraslice",
     "displayName": "Teraslice",
-    "version": "0.86.0",
+    "version": "0.86.1",
     "description": "Distributed computing platform for processing JSON data",
     "homepage": "https://github.com/terascope/teraslice#readme",
     "bugs": {
@@ -40,7 +40,7 @@
         "ms": "^2.1.3"
     },
     "dependencies": {
-        "@terascope/elasticsearch-api": "^3.6.1",
+        "@terascope/elasticsearch-api": "^3.6.2",
         "@terascope/job-components": "^0.59.0",
         "@terascope/teraslice-messaging": "^0.29.0",
         "@terascope/utils": "^0.46.0",
@@ -64,7 +64,7 @@
         "semver": "^7.3.8",
         "socket.io": "^1.7.4",
         "socket.io-client": "^1.7.4",
-        "terafoundation": "^0.45.2",
+        "terafoundation": "^0.45.3",
         "uuid": "^9.0.0"
     },
     "devDependencies": {

From e099edc39ab9765480b091a1f90ff9c2dd66fd76 Mon Sep 17 00:00:00 2001
From: Jared Noble <jsnoble1@gmail.com>
Date: Thu, 20 Jul 2023 11:17:03 -0700
Subject: [PATCH 14/14] brief doc overview how to use test-harness

---
 docs/packages/scripts/overview.md | 34 +++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)

diff --git a/docs/packages/scripts/overview.md b/docs/packages/scripts/overview.md
index 5aa6a570790..31c9d37a200 100644
--- a/docs/packages/scripts/overview.md
+++ b/docs/packages/scripts/overview.md
@@ -13,3 +13,37 @@ yarn global add @terascope/scripts
 # Using npm
 npm install --global @terascope/scripts
 ```
+
+
+## Test Runner
+Many tests require the use of various database's for testing. Built into the test framework we provide the means to test against supported databases including elasticsearch, opensearch, kafka, minio(S3) and rabbitmq.
+
+To enable these databases into the tests include the appropriate environmental variables to the test scripts in the package.json
+
+Example:
+```
+"test": "TEST_ELASTICSEARCH='true' ts-scripts test . --",
+"test:elasticsearch7": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' ts-scripts test . --",
+```
+
+In the above example, we specify `TEST_ELASTICSEARCH` which alerts the test harness to setup and wait for an elasticsearch instance to be available before any tests are run. It defaults to `v6.8.6` which can be overridden by the `ELASTICSEARCH_VERSION` variable
+
+All defaults and variables that are used can be found at `/packages/scripts/src/helpers/config.ts` file. These values are exported and a few of them are set in the test process environment for client setup.
+
+Multiple databases can be setup at the same time by using the correct environmental variables
+
+```
+"test": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite e2e --",
+```
+
+In the above scenario, both elasticsearch and kafka are setup before the tests are run
+
+List of environmental variables to setup a database:
+
+- "TEST_ELASTICSEARCH"
+- "TEST_OPENSEARCH"
+- "TEST_KAFKA"
+- "TEST_MINIO"
+- "TEST_RESTRAINED_OPENSEARCH" (this contains bulk queue limits to test api bulk overflows)
+- "TEST_RESTRAINED_ELASTICSEARCH" (this contains bulk queue limits to test api bulk overflows)
+- "TEST_RABBITMQ"