Merge pull request #3376 from terascope/test-resource-fixes

test runner resource fix
terascope · Jul 20, 2023 · 9e7fc55 · 9e7fc55
2 parents 3d0259c + e099edc
commit 9e7fc55
Show file tree

Hide file tree

Showing 23 changed files with 234 additions and 118 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -79,10 +79,20 @@ jobs:
           YARN_SETUP_ARGS: "--prod=false --silent"
 
       - name: Test elasticsearch 6
-        run: yarn --silent test --suite elasticsearch --elasticsearch-version 6.8.6 --elasticsearch-api-version 6.5
+        run: yarn --silent test
+        working-directory: ./packages/teraslice
 
       - name: Test elasticsearch 7
-        run: yarn --silent test --suite elasticsearch --elasticsearch-version 7.2.1 --elasticsearch-api-version 7.0
+        run:  yarn --silent test:elasticsearch7
+        working-directory: ./packages/teraslice
+
+      - name: Test opensearch 1.3
+        run: yarn --silent test:opensearch1
+        working-directory: ./packages/teraslice
+
+      - name: Test opensearch 2.8
+        run: yarn --silent test:opensearch2
+        working-directory: ./packages/teraslice
 
   elasticsearch-store-tests:
     runs-on: ubuntu-latest
@@ -116,7 +126,7 @@ jobs:
         working-directory: ./packages/elasticsearch-store
 
       - name: Test elasticsearch 7
-        run: yarn --silent test:7
+        run: yarn --silent test:elasticsearch7
         working-directory: ./packages/elasticsearch-store
 
       - name: Test opensearch 1.3

diff --git a/docs/packages/scripts/overview.md b/docs/packages/scripts/overview.md
@@ -13,3 +13,37 @@ yarn global add @terascope/scripts
 # Using npm
 npm install --global @terascope/scripts
 ```
+
+
+## Test Runner
+Many tests require the use of various database's for testing. Built into the test framework we provide the means to test against supported databases including elasticsearch, opensearch, kafka, minio(S3) and rabbitmq.
+
+To enable these databases into the tests include the appropriate environmental variables to the test scripts in the package.json
+
+Example:
+```
+"test": "TEST_ELASTICSEARCH='true' ts-scripts test . --",
+"test:elasticsearch7": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' ts-scripts test . --",
+```
+
+In the above example, we specify `TEST_ELASTICSEARCH` which alerts the test harness to setup and wait for an elasticsearch instance to be available before any tests are run. It defaults to `v6.8.6` which can be overridden by the `ELASTICSEARCH_VERSION` variable
+
+All defaults and variables that are used can be found at `/packages/scripts/src/helpers/config.ts` file. These values are exported and a few of them are set in the test process environment for client setup.
+
+Multiple databases can be setup at the same time by using the correct environmental variables
+
+```
+"test": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite e2e --",
+```
+
+In the above scenario, both elasticsearch and kafka are setup before the tests are run
+
+List of environmental variables to setup a database:
+
+- "TEST_ELASTICSEARCH"
+- "TEST_OPENSEARCH"
+- "TEST_KAFKA"
+- "TEST_MINIO"
+- "TEST_RESTRAINED_OPENSEARCH" (this contains bulk queue limits to test api bulk overflows)
+- "TEST_RESTRAINED_ELASTICSEARCH" (this contains bulk queue limits to test api bulk overflows)
+- "TEST_RABBITMQ"
diff --git a/e2e/package.json b/e2e/package.json
@@ -24,16 +24,16 @@
         "logs": "./scripts/logs.sh",
         "logs-follow": "./scripts/logs.sh -f",
         "setup": "yarn --silent",
-        "test": "ts-scripts test --suite e2e --",
-        "test:debug": "ts-scripts test --suite e2e --debug --",
-        "test:watch": "ts-scripts test --suite e2e --watch --"
+        "test": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite e2e --",
+        "test:debug": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite e2e --debug --",
+        "test:watch": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite e2e --watch --"
     },
     "resolutions": {
         "ms": "^2.1.3"
     },
     "devDependencies": {
         "bunyan": "^1.8.15",
-        "elasticsearch-store": "^0.67.2",
+        "elasticsearch-store": "^0.67.3",
         "fs-extra": "^10.1.0",
         "ms": "^2.1.3",
         "nanoid": "^3.3.4",

diff --git a/package.json b/package.json
@@ -1,7 +1,7 @@
 {
     "name": "teraslice-workspace",
     "displayName": "Teraslice",
-    "version": "0.86.0",
+    "version": "0.86.1",
     "private": true,
     "homepage": "https://github.com/terascope/teraslice",
     "bugs": {
@@ -68,25 +68,14 @@
         "target": "es2019",
         "tests": {
             "suites": {
-                "e2e": [
-                    "elasticsearch",
-                    "kafka"
-                ],
-                "elasticsearch": [
+                "e2e": [],
+                "elasticsearch": [],
+                "search": [],
+                "restrained": [],
+                "unit": [],
+                "_for_testing_": [
                     "elasticsearch"
-                ],
-                "search": [
-                    "elasticsearch",
-                    "opensearch"
-                ],
-                "restrained": [
-                    "restrained_elasticsearch",
-                    "restrained_opensearch"
-                ],
-                "kafka": [
-                    "kafka"
-                ],
-                "unit": []
+                ]
             }
         },
         "docker": {

diff --git a/packages/elasticsearch-api/package.json b/packages/elasticsearch-api/package.json
@@ -1,7 +1,7 @@
 {
     "name": "@terascope/elasticsearch-api",
     "displayName": "Elasticsearch API",
-    "version": "3.6.1",
+    "version": "3.6.2",
     "description": "Elasticsearch client api used across multiple services, handles retries and exponential backoff",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/elasticsearch-api#readme",
     "bugs": {
@@ -13,13 +13,13 @@
     "main": "index.js",
     "typings": "types/index.d.ts",
     "scripts": {
-        "test": "ts-scripts test . --",
-        "test:7": "ELASTICSEARCH_VERSION='7.9.3' yarn run test",
-        "test:8": "ELASTICSEARCH_VERSION='8.1.2' yarn run test",
-        "test:debug": "ts-scripts test --debug . --",
-        "test:legacy": "LEGACY_CLIENT=true yarn run test:debug",
-        "test:opensearch": "TEST_OPENSEARCH='true' RESTRAINED_OPENSEARCH='true' yarn run test",
-        "test:watch": "ts-scripts test --watch . --"
+        "test": "TEST_RESTRAINED_ELASTICSEARCH='true' ts-scripts test . --",
+        "test:7": "TEST_RESTRAINED_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' ts-scripts test . --",
+        "test:8": "TEST_RESTRAINED_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='8.1.2' ts-scripts test . --",
+        "test:debug": "TEST_RESTRAINED_ELASTICSEARCH='true' ts-scripts test --debug . --",
+        "test:legacy": "TEST_RESTRAINED_ELASTICSEARCH='true' LEGACY_CLIENT=true ts-scripts test . --",
+        "test:opensearch": "TEST_RESTRAINED_OPENSEARCH='true' ts-scripts test --debug . --",
+        "test:watch": "TEST_RESTRAINED_ELASTICSEARCH='true' ts-scripts test --watch . --"
     },
     "dependencies": {
         "@terascope/types": "^0.11.2",
@@ -31,7 +31,7 @@
         "@opensearch-project/opensearch": "^1.2.0",
         "@types/elasticsearch": "^5.0.40",
         "elasticsearch": "^15.4.1",
-        "elasticsearch-store": "^0.67.2",
+        "elasticsearch-store": "^0.67.3",
         "elasticsearch6": "npm:@elastic/elasticsearch@^6.7.0",
         "elasticsearch7": "npm:@elastic/elasticsearch@^7.0.0",
         "elasticsearch8": "npm:@elastic/elasticsearch@^8.0.0"

diff --git a/packages/elasticsearch-api/test/bulk-send-dlq-spec.js b/packages/elasticsearch-api/test/bulk-send-dlq-spec.js
@@ -9,37 +9,18 @@ const { ElasticsearchTestHelpers } = require('elasticsearch-store');
 const elasticsearchAPI = require('../index');
 
 const {
-    makeClient, cleanupIndex,
-    EvenDateData, TEST_INDEX_PREFIX,
-    createMappingFromDatatype
+    makeClient, cleanupIndex, EvenDateData,
+    TEST_INDEX_PREFIX, createMappingFromDatatype,
+    formatUploadData
 } = ElasticsearchTestHelpers;
 
 const THREE_MINUTES = 3 * 60 * 1000;
 
 jest.setTimeout(THREE_MINUTES + 60000);
 
-function formatUploadData(
-    index, data, isES8ClientTest = false
-) {
-    const results = [];
-
-    data.forEach((record, i) => {
-        const meta = { _index: index, _id: i + 1 };
-
-        if (!isES8ClientTest) {
-            meta._type = '_doc';
-        }
-
-        results.push({ action: { index: meta }, data: record });
-    });
-
-    return results;
-}
-
 describe('bulkSend', () => {
     let client;
     let api;
-    let isElasticsearch8 = false;
 
     beforeAll(async () => {
         client = await makeClient();
@@ -52,7 +33,6 @@ describe('bulkSend', () => {
         beforeAll(async () => {
             await cleanupIndex(client, index);
             api = elasticsearchAPI(client, logger, { _dead_letter_action: 'kafka_dead_letter' });
-            isElasticsearch8 = api.isElasticsearch8();
 
             const overrides = {
                 settings: {
@@ -79,7 +59,7 @@ describe('bulkSend', () => {
 
             const docs = cloneDeep(EvenDateData.data.slice(0, 2));
 
-            const result = await diffApi.bulkSend(formatUploadData(index, docs, isElasticsearch8));
+            const result = await diffApi.bulkSend(formatUploadData(index, docs, true));
 
             expect(result).toBe(2);
         });
@@ -91,7 +71,7 @@ describe('bulkSend', () => {
 
             docs[0].bytes = 'this is a bad value';
 
-            await expect(diffApi.bulkSend(formatUploadData(index, docs, isElasticsearch8)))
+            await expect(diffApi.bulkSend(formatUploadData(index, docs, true)))
                 .rejects.toThrow();
         });
 
@@ -101,7 +81,7 @@ describe('bulkSend', () => {
 
             docs[0].bytes = 'this is a bad value';
 
-            const result = await api.bulkSend(formatUploadData(index, docs, isElasticsearch8));
+            const result = await api.bulkSend(formatUploadData(index, docs, true));
 
             // 1 good doc  - so only 1 row affected
             expect(result).toBe(1);
@@ -113,7 +93,7 @@ describe('bulkSend', () => {
         it('should return a count if not un-retryable records if dlq is set', async () => {
             const docs = cloneDeep(EvenDateData.data.slice(0, 2));
 
-            const result = await api.bulkSend(formatUploadData(index, docs, isElasticsearch8));
+            const result = await api.bulkSend(formatUploadData(index, docs, true));
 
             expect(result).toBe(2);
         });

diff --git a/packages/elasticsearch-api/test/bulk-send-limit-spec.js b/packages/elasticsearch-api/test/bulk-send-limit-spec.js
@@ -10,35 +10,16 @@ const elasticsearchAPI = require('../index');
 
 const {
     makeClient, cleanupIndex, waitForData,
-    EvenDateData, TEST_INDEX_PREFIX
+    EvenDateData, TEST_INDEX_PREFIX, formatUploadData
 } = ElasticsearchTestHelpers;
 
 const THREE_MINUTES = 3 * 60 * 1000;
 
 jest.setTimeout(THREE_MINUTES + 60000);
 
-function formatUploadData(
-    index, data, isES8ClientTest = false
-) {
-    const results = [];
-
-    data.forEach((record) => {
-        const meta = { _index: index };
-
-        if (!isES8ClientTest) {
-            meta._type = '_doc';
-        }
-
-        results.push({ action: { index: meta }, data: record });
-    });
-
-    return results;
-}
-
 describe('bulkSend', () => {
     let client;
     let api;
-    let isElasticsearch8 = false;
 
     beforeAll(async () => {
         client = await makeClient();
@@ -51,7 +32,6 @@ describe('bulkSend', () => {
         beforeAll(async () => {
             await cleanupIndex(client, index);
             api = elasticsearchAPI(client, logger);
-            isElasticsearch8 = api.isElasticsearch8();
         });
 
         afterAll(async () => {
@@ -62,7 +42,7 @@ describe('bulkSend', () => {
             const chunkedData = chunk(EvenDateData.data, 50);
 
             await pMap(chunkedData, async (cData) => {
-                const formattedData = formatUploadData(index, cData, isElasticsearch8);
+                const formattedData = formatUploadData(index, cData, true);
                 return api.bulkSend(formattedData);
             }, { concurrency: 9 });
 

diff --git a/packages/elasticsearch-store/package.json b/packages/elasticsearch-store/package.json
@@ -1,7 +1,7 @@
 {
     "name": "elasticsearch-store",
     "displayName": "Elasticsearch Store",
-    "version": "0.67.2",
+    "version": "0.67.3",
     "description": "An API for managing an elasticsearch index, with versioning and migration support.",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/elasticsearch-store#readme",
     "bugs": {
@@ -19,13 +19,13 @@
     "scripts": {
         "build": "tsc --build",
         "build:watch": "yarn build --watch",
-        "test": "ts-scripts test . --",
-        "test:7": "ELASTICSEARCH_VERSION='7.9.3' yarn run test",
-        "test:8": "ELASTICSEARCH_VERSION='8.1.2' yarn run test",
-        "test:debug": "ts-scripts test --debug . --",
-        "test:legacy": "LEGACY_CLIENT=true yarn run test",
-        "test:opensearch1": "TEST_OPENSEARCH='true' yarn run test",
-        "test:opensearch2": "TEST_OPENSEARCH='true' OPENSEARCH_VERSION='2.8.0' yarn run test",
+        "test": "TEST_ELASTICSEARCH='true' ts-scripts test . --",
+        "test:debug": "TEST_ELASTICSEARCH='true' ts-scripts test --debug . --",
+        "test:elasticsearch7": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' ts-scripts test . --",
+        "test:elasticsearch8": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='8.1.2' ts-scripts test . --",
+        "test:legacy": "TEST_ELASTICSEARCH='true' LEGACY_CLIENT=true ts-scripts test . --",
+        "test:opensearch1": "TEST_OPENSEARCH='true' ts-scripts test . --",
+        "test:opensearch2": "TEST_OPENSEARCH='true' OPENSEARCH_VERSION='2.8.0' ts-scripts test . --",
         "test:watch": "ts-scripts test --watch . --"
     },
     "dependencies": {

diff --git a/packages/elasticsearch-store/src/test-helpers/config.ts b/packages/elasticsearch-store/src/test-helpers/config.ts
@@ -10,7 +10,7 @@ const {
     OPENSEARCH_VERSION = '1.3.0',
     OPENSEARCH_HOST = `http://${OPENSEARCH_USER}:${OPENSEARCH_PASSWORD}@${OPENSEARCH_HOSTNAME}`,
     RESTRAINED_OPENSEARCH_PORT = process.env.RESTRAINED_OPENSEARCH_PORT || '49206',
-    RESTRAINED_OPENSEARCH_HOST = `http://${OPENSEARCH_USER}:${OPENSEARCH_PASSWORD}@http://localhost:${RESTRAINED_OPENSEARCH_PORT}`,
+    RESTRAINED_OPENSEARCH_HOST = `http://${OPENSEARCH_USER}:${OPENSEARCH_PASSWORD}@localhost:${RESTRAINED_OPENSEARCH_PORT}`,
 } = process.env;
 
 export {