Merge branch 'master' into eslint-esm-changes

.github/workflows/test.yml

@@ -10,7 +10,7 @@ env:
   NODE_VERSIONS_EXT_STORAGE: '[18]'
 
 jobs:
-  compute:
+  compute-node-version-vars:
     runs-on: ubuntu-latest
     outputs:
       NODE_VERSIONS: ${{ steps.step1.outputs.NODE_VERSIONS }}
@@ -29,15 +29,15 @@ jobs:
 
   verify-build:
     runs-on: ubuntu-latest
-    needs: compute
+    needs: compute-node-version-vars
     steps:
     - name: Check out code
       uses: actions/checkout@v4
 
     - name: Setup Node
       uses: actions/setup-node@v4
       with:
-        node-version: ${{ fromJSON(needs.compute.outputs.NODE_VERSION_MAIN) }}
+        node-version: ${{ fromJSON(needs.compute-node-version-vars.outputs.NODE_VERSION_MAIN) }}
         cache: 'yarn'
 
     - name: Install and build packages
@@ -56,10 +56,10 @@ jobs:
 
   linux-unit-tests:
     runs-on: ubuntu-latest
-    needs: compute
+    needs: compute-node-version-vars
     strategy:
       matrix:
-        node-version: ${{ fromJSON(needs.compute.outputs.NODE_VERSIONS) }}
+        node-version: ${{ fromJSON(needs.compute-node-version-vars.outputs.NODE_VERSIONS) }}
     steps:
       - name: Check out code
         uses: actions/checkout@v4
@@ -80,10 +80,10 @@ jobs:
 
   linux-unit-esm-tests:
     runs-on: ubuntu-latest
-    needs: compute
+    needs: compute-node-version-vars
     strategy:
       matrix:
-        node-version: ${{ fromJSON(needs.compute.outputs.NODE_VERSIONS) }}
+        node-version: ${{ fromJSON(needs.compute-node-version-vars.outputs.NODE_VERSIONS) }}
     steps:
       - name: Check out code
         uses: actions/checkout@v4
@@ -106,12 +106,12 @@ jobs:
 
   teraslice-elasticsearch-tests:
     runs-on: ubuntu-latest
-    needs: [compute, verify-build, cache-docker-images]
+    needs: [compute-node-version-vars, verify-build, cache-docker-images]
     strategy:
       # opensearch is finiky, keep testing others if it fails
       fail-fast: false
       matrix:
-        node-version: ${{ fromJSON(needs.compute.outputs.NODE_VERSIONS) }}
+        node-version: ${{ fromJSON(needs.compute-node-version-vars.outputs.NODE_VERSIONS) }}
         search-version: [elasticsearch6, elasticsearch7, opensearch1, opensearch2]
     steps:
       - name: Check out code
@@ -153,12 +153,12 @@ jobs:
 
   elasticsearch-store-tests:
     runs-on: ubuntu-latest
-    needs: [compute, verify-build, cache-docker-images]
+    needs: [compute-node-version-vars, verify-build, cache-docker-images]
     strategy:
       # opensearch is finiky, keep testing others if it fails
       fail-fast: false
       matrix:
-        node-version: ${{ fromJSON(needs.compute.outputs.NODE_VERSIONS) }}
+        node-version: ${{ fromJSON(needs.compute-node-version-vars.outputs.NODE_VERSIONS) }}
         search-version: [elasticsearch6, elasticsearch7, opensearch1, opensearch2]
     steps:
       - name: Check out code
@@ -200,7 +200,7 @@ jobs:
 
   lint-and-sync:
     runs-on: ubuntu-latest
-    needs: [compute, verify-build, cache-docker-images]
+    needs: [compute-node-version-vars, verify-build, cache-docker-images]
     # will remove the checkout, build and setup when the artifact is made to just
     # test the linting and syncing of the codebase
     steps:
@@ -210,7 +210,7 @@ jobs:
     - name: Setup Node
       uses: actions/setup-node@v4
       with:
-        node-version: ${{ fromJSON(needs.compute.outputs.NODE_VERSION_MAIN) }}
+        node-version: ${{ fromJSON(needs.compute-node-version-vars.outputs.NODE_VERSION_MAIN) }}
         cache: 'yarn'
 
     - name: Install and build packages
@@ -226,12 +226,12 @@ jobs:
 
   elasticsearch-api-tests:
     runs-on: ubuntu-latest
-    needs: [compute, verify-build, cache-docker-images]
+    needs: [compute-node-version-vars, verify-build, cache-docker-images]
     strategy:
       # opensearch is finiky, keep testing others if it fails
       fail-fast: false
       matrix:
-        node-version: ${{ fromJSON(needs.compute.outputs.NODE_VERSIONS) }}
+        node-version: ${{ fromJSON(needs.compute-node-version-vars.outputs.NODE_VERSIONS) }}
         search-version: [elasticsearch6, elasticsearch7, opensearch1, opensearch2]
     steps:
       - name: Check out code
@@ -273,12 +273,12 @@ jobs:
 
   e2e-tests:
     runs-on: ubuntu-latest
-    needs: [compute, verify-build, cache-docker-images]
+    needs: [compute-node-version-vars, verify-build, cache-docker-images]
     strategy:
       # opensearch is finiky, keep testing others if it fails
       fail-fast: false
       matrix:
-        node-version: ${{ fromJSON(needs.compute.outputs.NODE_VERSIONS) }}
+        node-version: ${{ fromJSON(needs.compute-node-version-vars.outputs.NODE_VERSIONS) }}
         search-version: [elasticsearch6, elasticsearch7, opensearch1, opensearch2]
     steps:
       - name: Check out code
@@ -324,12 +324,12 @@ jobs:
 
   e2e-k8s-tests:
     runs-on: ubuntu-latest
-    needs: [compute, verify-build, cache-docker-images]
+    needs: [compute-node-version-vars, verify-build, cache-docker-images]
     strategy:
       # opensearch is finiky, keep testing others if it fails
       fail-fast: false
       matrix:
-        node-version: ${{ fromJSON(needs.compute.outputs.NODE_VERSIONS_EXT_K8S) }}
+        node-version: ${{ fromJSON(needs.compute-node-version-vars.outputs.NODE_VERSIONS_EXT_K8S) }}
     steps:
       - name: Check out code
         uses: actions/checkout@v4
@@ -379,12 +379,12 @@ jobs:
 
   e2e-k8s-v2-tests:
     runs-on: ubuntu-latest
-    needs: [compute, verify-build, cache-docker-images]
+    needs: [compute-node-version-vars, verify-build, cache-docker-images]
     strategy:
       # opensearch is finiky, keep testing others if it fails
       fail-fast: false
       matrix:
-        node-version: ${{ fromJSON(needs.compute.outputs.NODE_VERSIONS_EXT_K8S) }}
+        node-version: ${{ fromJSON(needs.compute-node-version-vars.outputs.NODE_VERSIONS_EXT_K8S) }}
     steps:
       - name: Check out code
         uses: actions/checkout@v4
@@ -434,12 +434,12 @@ jobs:
 
   e2e-external-storage-tests:
     runs-on: ubuntu-latest
-    needs: [compute, verify-build, cache-docker-images]
+    needs: [compute-node-version-vars, verify-build, cache-docker-images]
     strategy:
       # opensearch is finiky, keep testing others if it fails
       fail-fast: false
       matrix:
-        node-version: ${{ fromJSON(needs.compute.outputs.NODE_VERSIONS_EXT_STORAGE) }}
+        node-version: ${{ fromJSON(needs.compute-node-version-vars.outputs.NODE_VERSIONS_EXT_STORAGE) }}
     steps:
       - name: Check out code
         uses: actions/checkout@v4
@@ -484,12 +484,12 @@ jobs:
 
   e2e-external-storage-tests-encrypted:
     runs-on: ubuntu-latest
-    needs: [compute, verify-build, cache-docker-images]
+    needs: [compute-node-version-vars, verify-build, cache-docker-images]
     strategy:
       # opensearch is finiky, keep testing others if it fails
       fail-fast: false
       matrix:
-        node-version: ${{ fromJSON(needs.compute.outputs.NODE_VERSIONS_EXT_STORAGE) }}
+        node-version: ${{ fromJSON(needs.compute-node-version-vars.outputs.NODE_VERSIONS_EXT_STORAGE) }}
     steps:
       - name: Check out code
         uses: actions/checkout@v4

e2e/package.json

@@ -31,9 +31,9 @@
         "test:debug": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite e2e --debug --",
         "test:elasticsearch6": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite e2e --",
         "test:elasticsearch7": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' TEST_KAFKA='true' ts-scripts test --suite e2e --",
-        "test:k8s": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' KAFKA_VERSION='3.1' TEST_KAFKA='true' TEST_PLATFORM='kubernetes' ts-scripts test --suite e2e --",
-        "test:k8sNoBuild": "SKIP_DOCKER_BUILD_IN_E2E='true' TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' KAFKA_VERSION='3.1' TEST_KAFKA='true' TEST_PLATFORM='kubernetes' ts-scripts test --suite e2e --",
-        "test:k8sV2": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' KAFKA_VERSION='3.1' TEST_KAFKA='true' TEST_PLATFORM='kubernetesV2' ts-scripts test --suite e2e --",
+        "test:k8s": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' TEST_KAFKA='true' TEST_PLATFORM='kubernetes' ts-scripts test --suite e2e --",
+        "test:k8sNoBuild": "SKIP_DOCKER_BUILD_IN_E2E='true' TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' TEST_KAFKA='true' TEST_PLATFORM='kubernetes' ts-scripts test --suite e2e --",
+        "test:k8sV2": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' TEST_KAFKA='true' TEST_PLATFORM='kubernetesV2' ts-scripts test --suite e2e --",
         "test:opensearch1": "TEST_OPENSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite e2e --",
         "test:opensearch2": "TEST_OPENSEARCH='true' OPENSEARCH_VERSION='2.8.0' TEST_KAFKA='true' ts-scripts test --suite e2e --",
         "test:s3AssetStorage": "TEST_OPENSEARCH='true' TEST_KAFKA='true' TEST_MINIO='true' ASSET_STORAGE_CONNECTION_TYPE='s3' ASSET_STORAGE_CONNECTION='default' ts-scripts test --suite e2e --",
@@ -45,7 +45,7 @@
     "devDependencies": {
         "@terascope/types": "^1.0.0",
         "bunyan": "^1.8.15",
-        "elasticsearch-store": "^1.0.0",
+        "elasticsearch-store": "^1.0.3",
         "fs-extra": "^11.2.0",
         "ms": "^2.1.3",
         "nanoid": "^3.3.4",

e2e/test/cases/kafka/kafka-spec.ts

@@ -51,7 +51,7 @@ describe('kafka', () => {
             terasliceHarness.waitForExStatus(sender, 'completed')
         ]);
 
-        await terasliceHarness.waitForIndexCount(specIndex, total);
+        await terasliceHarness.waitForIndexCount(specIndex, total, 45 * 1000);
         await reader.stop();
 
         await terasliceHarness.waitForExStatus(reader, 'stopped');

package.json

@@ -1,7 +1,7 @@
 {
     "name": "teraslice-workspace",
     "displayName": "Teraslice",
-    "version": "2.1.0",
+    "version": "2.2.0",
     "private": true,
     "homepage": "https://github.com/terascope/teraslice",
     "bugs": {

packages/data-mate/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@terascope/data-mate",
     "displayName": "Data-Mate",
-    "version": "1.0.0",
+    "version": "1.0.3",
     "description": "Library of data validations/transformations",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/data-mate#readme",
     "repository": {
@@ -47,7 +47,7 @@
         "uuid": "^9.0.1",
         "valid-url": "^1.0.9",
         "validator": "^13.12.0",
-        "xlucene-parser": "^1.0.0"
+        "xlucene-parser": "^1.0.2"
     },
     "devDependencies": {
         "@types/ip6addr": "^0.2.6",

packages/data-mate/src/index.ts

@@ -1,4 +1,3 @@
-import { cloneDeep } from '@terascope/utils';
 import { DataFrame } from './data-frame/index.js';
 import {
     jexl, extract, extractConfig,
@@ -11,12 +10,15 @@ import {
 import { AggregationFrame } from './aggregation-frame/AggregationFrame.js';
 
 // import are immutable, so we rename and clone to alter methods
-const FieldTransform = cloneDeep(FTransform);
-const RecordTransform = cloneDeep(RTransform);
-
+const FieldTransform = {
+    ...FTransform
+};
 FieldTransform.repository.extract = extractConfig;
 FieldTransform.extract = extract;
 
+const RecordTransform = {
+    ...RTransform
+};
 RecordTransform.repository.transformRecord = transformRecordConfig;
 RecordTransform.transformRecord = transformRecord;
 

packages/data-mate/src/transforms/field-transform.ts

@@ -1294,7 +1294,7 @@ export function toISO8601(
     return _makeIso(input, args);
 }
 
-interface FormatDateConfig {
+export interface FormatDateConfig {
     format: string;
     resolution?: 'seconds' | 'milliseconds';
 }
@@ -1355,7 +1355,7 @@ export function formatDate(
     return _formatDate(input, args);
 }
 
-interface ParseDateConfig {
+export interface ParseDateConfig {
     format: string;
 }
 

packages/elasticsearch-api/package.json

@@ -33,7 +33,7 @@
         "@opensearch-project/opensearch": "^1.2.0",
         "@types/elasticsearch": "^5.0.43",
         "elasticsearch": "^15.4.1",
-        "elasticsearch-store": "^1.0.0",
+        "elasticsearch-store": "^1.0.3",
         "elasticsearch6": "npm:@elastic/elasticsearch@^6.7.0",
         "elasticsearch7": "npm:@elastic/elasticsearch@^7.0.0",
         "elasticsearch8": "npm:@elastic/elasticsearch@^8.0.0"

packages/elasticsearch-store/package.json

@@ -1,7 +1,7 @@
 {
     "name": "elasticsearch-store",
     "displayName": "Elasticsearch Store",
-    "version": "1.0.0",
+    "version": "1.0.3",
     "description": "An API for managing an elasticsearch index, with versioning and migration support.",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/elasticsearch-store#readme",
     "bugs": {
@@ -30,7 +30,7 @@
         "test:watch": "ts-scripts test --watch . --"
     },
     "dependencies": {
-        "@terascope/data-mate": "^1.0.0",
+        "@terascope/data-mate": "^1.0.3",
         "@terascope/data-types": "^1.0.0",
         "@terascope/types": "^1.0.0",
         "@terascope/utils": "^1.0.0",
@@ -42,7 +42,7 @@
         "opensearch2": "npm:@opensearch-project/opensearch@^2.2.1",
         "setimmediate": "^1.0.5",
         "uuid": "^9.0.1",
-        "xlucene-translator": "^1.0.0"
+        "xlucene-translator": "^1.0.2"
     },
     "devDependencies": {
         "@types/uuid": "^9.0.8"

packages/terafoundation/package.json

@@ -1,7 +1,7 @@
 {
     "name": "terafoundation",
     "displayName": "Terafoundation",
-    "version": "1.2.0",
+    "version": "1.2.3",
     "description": "A Clustering and Foundation tool for Terascope Tools",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/terafoundation#readme",
     "bugs": {
@@ -37,7 +37,7 @@
         "convict-format-with-moment": "^6.2.0",
         "convict-format-with-validator": "^6.2.0",
         "elasticsearch": "^15.4.1",
-        "elasticsearch-store": "^1.0.0",
+        "elasticsearch-store": "^1.0.3",
         "express": "^4.19.2",
         "js-yaml": "^4.1.0",
         "nanoid": "^3.3.4",

packages/teraslice-messaging/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@terascope/teraslice-messaging",
     "displayName": "Teraslice Messaging",
-    "version": "1.2.0",
+    "version": "1.3.0",
     "description": "An internal teraslice messaging library using socket.io",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/teraslice-messaging#readme",
     "bugs": {

packages/teraslice-messaging/src/execution-controller/interfaces.ts

@@ -1,4 +1,5 @@
 import { Logger } from '@terascope/utils';
+import http from 'node:http';
 
 export interface ClientOptions {
     executionControllerUrl: string;
@@ -16,9 +17,13 @@ export interface ServerOptions {
     workerDisconnectTimeout: number;
     networkLatencyBuffer?: number;
     actionTimeout: number;
+    requestListener?: RequestListener;
     logger?: Logger;
 }
 
+export interface RequestListener {
+    (request: http.IncomingMessage, response: http.ServerResponse): void;
+}
 export interface Worker {
     workerId: string;
 }