diff --git a/.gitignore b/.gitignore index 94226a14be6..1e7d06957b0 100644 --- a/.gitignore +++ b/.gitignore @@ -100,4 +100,6 @@ services/github/pod-github/src/github.graphql dev/tool/report.csv bundle/* bundle.js.map -tests/profiles \ No newline at end of file +tests/profiles +_* +**/bundle/model.json diff --git a/.nvmrc b/.nvmrc index 7ea6a59d347..7b3bc8194ea 100644 --- a/.nvmrc +++ b/.nvmrc @@ -1 +1 @@ -v20.11.0 +v20.13 diff --git a/.vscode/launch.json b/.vscode/launch.json index 10b8d91685b..eb104121540 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -63,16 +63,13 @@ "REKONI_URL": "http://localhost:4004", "FRONT_URL": "http://localhost:8080", "ACCOUNTS_URL": "http://localhost:3000", + "MODEL_JSON": "${workspaceRoot}/models/all/bundle/model.json", // "SERVER_PROVIDER":"uweb" "SERVER_PROVIDER":"ws", "MODEL_VERSION": "0.6.287", // "VERSION": "0.6.289", "ELASTIC_INDEX_NAME": "local_storage_index", - "UPLOAD_URL": "/files", - - // "RETRANSLATE_URL": "http://127.0.0.1:4500", - //"RETRANSLATE_URL": "https://208.167.249.201", - // "RETRANSLATE_TOKEN": "" + "UPLOAD_URL": "/files", }, "runtimeArgs": ["--nolazy", "-r", "ts-node/register"], "runtimeVersion": "20", @@ -92,8 +89,9 @@ "DB_URL": "mongodb://localhost:27017", // "DB_URL": "postgresql://postgres:example@localhost:5432", "SERVER_SECRET": "secret", - "TRANSACTOR_URL": "ws://localhost:3333", + "TRANSACTOR_URL": "ws://host.docker.internal:3333,ws://host.docker.internal:3331;;pg", "ACCOUNTS_URL": "http://localhost:3000", + "REGION_INFO": "|Mongo;pg|Postgree", "ACCOUNT_PORT": "3000", "FRONT_URL": "http://localhost:8080", "SES_URL": "", @@ -117,8 +115,9 @@ "args": ["src/__start.ts"], "env": { "MONGO_URL": "mongodb://localhost:27017", - "DB_URL": "mongodb://localhost:27017", - // "DB_URL": "postgresql://postgres:example@localhost:5432", + // "DB_URL": "mongodb://localhost:27017", + "REGION": "pg", + "DB_URL": "postgresql://postgres:example@localhost:5432", "SERVER_SECRET": "secret", "TRANSACTOR_URL": "ws://localhost:3333", "ACCOUNTS_URL": "http://localhost:3000", diff --git a/common/config/rush/command-line.json b/common/config/rush/command-line.json index 37f873388bc..21a9639e85a 100644 --- a/common/config/rush/command-line.json +++ b/common/config/rush/command-line.json @@ -156,9 +156,9 @@ }, { "commandKind": "phased", - "name": "revalidate", + "name": "dorevalidate", "phases": ["_phase:validate"], - "summary": "revalidate", + "summary": "dorevalidate", "enableParallelism": true, "incremental": false }, @@ -279,6 +279,14 @@ "description": "Clean typescript incremental cache", "safeForSimultaneousRushProcesses": true, "shellCommand": "find .|grep tsconfig.tsbuildinfo | xargs rm | pwd" + }, + { + "commandKind": "global", + "name": "revalidate", + "summary": "Clean Validate cache and to validate again", + "description": "Clean typescript incremental cache", + "safeForSimultaneousRushProcesses": true, + "shellCommand": "find .|grep tsBuildInfoFile.info | xargs rm | pwd && rush dorevalidate" }, { "commandKind": "global", @@ -288,6 +296,14 @@ "safeForSimultaneousRushProcesses": true, "shellCommand": "npx node ./common/scripts/show_version.js" }, + { + "commandKind": "global", + "name": "show-model", + "summary": "show model", + "description": "show model", + "safeForSimultaneousRushProcesses": true, + "shellCommand": "cd ./models/all && rushx show-model" + }, { "commandKind": "global", "name": "deps-clean", diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index bdabd4ef779..1f324f952c9 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -506,9 +506,6 @@ dependencies: '@rush-temp/model-server-training': specifier: file:./projects/model-server-training.tgz version: file:projects/model-server-training.tgz - '@rush-temp/model-server-translate': - specifier: file:./projects/model-server-translate.tgz - version: file:projects/model-server-translate.tgz '@rush-temp/model-server-view': specifier: file:./projects/model-server-view.tgz version: file:projects/model-server-view.tgz @@ -721,7 +718,7 @@ dependencies: version: file:projects/s3.tgz(esbuild@0.20.1)(ts-node@10.9.2) '@rush-temp/server': specifier: file:./projects/server.tgz - version: file:projects/server.tgz(esbuild@0.20.1) + version: file:projects/server.tgz(esbuild@0.20.1)(ts-node@10.9.2) '@rush-temp/server-activity': specifier: file:./projects/server-activity.tgz version: file:projects/server-activity.tgz(esbuild@0.20.1)(ts-node@10.9.2) @@ -824,6 +821,9 @@ dependencies: '@rush-temp/server-hr-resources': specifier: file:./projects/server-hr-resources.tgz version: file:projects/server-hr-resources.tgz(@types/node@20.11.19)(esbuild@0.20.1)(ts-node@10.9.2) + '@rush-temp/server-indexer': + specifier: file:./projects/server-indexer.tgz + version: file:projects/server-indexer.tgz(esbuild@0.20.1)(ts-node@10.9.2) '@rush-temp/server-inventory': specifier: file:./projects/server-inventory.tgz version: file:projects/server-inventory.tgz(esbuild@0.20.1)(ts-node@10.9.2) @@ -1040,9 +1040,6 @@ dependencies: '@rush-temp/training-resources': specifier: file:./projects/training-resources.tgz version: file:projects/training-resources.tgz(@types/node@20.11.19)(esbuild@0.20.1)(postcss-load-config@4.0.2)(postcss@8.4.35)(ts-node@10.9.2) - '@rush-temp/translate': - specifier: file:./projects/translate.tgz - version: file:projects/translate.tgz(esbuild@0.20.1)(ts-node@10.9.2) '@rush-temp/ui': specifier: file:./projects/ui.tgz version: file:projects/ui.tgz(@types/node@20.11.19)(esbuild@0.20.1)(postcss-load-config@4.0.2)(postcss@8.4.35)(ts-node@10.9.2) @@ -1611,7 +1608,7 @@ dependencies: specifier: ^29.7.0 version: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2) jest-environment-jsdom: - specifier: 29.7.0 + specifier: ^29.7.0 version: 29.7.0(bufferutil@4.0.8)(utf-8-validate@6.0.4) jimp: specifier: ^0.16.1 @@ -17454,11 +17451,11 @@ packages: http-proxy-agent: 5.0.0 https-proxy-agent: 5.0.1 is-potential-custom-element-name: 1.0.1 - nwsapi: 2.2.7 + nwsapi: 2.2.13 parse5: 7.1.2 saxes: 6.0.0 symbol-tree: 3.2.4 - tough-cookie: 4.1.3 + tough-cookie: 4.1.4 w3c-xmlserializer: 4.0.0 webidl-conversions: 7.0.0 whatwg-encoding: 2.0.0 @@ -19222,8 +19219,8 @@ packages: engines: {node: '>=0.10.0'} dev: false - /nwsapi@2.2.7: - resolution: {integrity: sha512-ub5E4+FBPKwAZx0UwIQOjYWGHTEq5sPqHQNRN8Z9e4A7u3Tj1weLJsL59yH9vmvqEtBHaOmT6cYQKIZOxp35FQ==} + /nwsapi@2.2.13: + resolution: {integrity: sha512-cTGB9ptp9dY9A5VbMSe7fQBcl/tt22Vcqdq8+eN93rblOuE0aCFu4aZ2vMwct/2t+lFnosm8RkQW1I0Omb1UtQ==} dev: false /nypm@0.3.6: @@ -20615,12 +20612,6 @@ packages: prosemirror-state: 1.4.3 dev: false - /prosemirror-model@1.19.4: - resolution: {integrity: sha512-RPmVXxUfOhyFdayHawjuZCxiROsm9L4FCUA6pWI+l7n2yCBsWy9VpdE1hpDHUS8Vad661YLY9AzqfjLhAKQ4iQ==} - dependencies: - orderedmap: 2.1.1 - dev: false - /prosemirror-model@1.22.3: resolution: {integrity: sha512-V4XCysitErI+i0rKFILGt/xClnFJaohe/wrrlT2NSZ+zk8ggQfDH4x2wNK7Gm0Hp4CIoWizvXFP7L9KMaCuI0Q==} dependencies: @@ -23108,8 +23099,8 @@ packages: engines: {node: '>=6'} dev: false - /tough-cookie@4.1.3: - resolution: {integrity: sha512-aX/y5pVRkfRnfmuX+OdbSdXvPe6ieKX/G2s7e98f4poJHnqH3281gDPm/metm6E/WRamfx7WC4HUqkWHfQHprw==} + /tough-cookie@4.1.4: + resolution: {integrity: sha512-Loo5UUvLD9ScZ6jh8beX1T6sO1w2/MpCRpEP7V280GKMVUQ0Jzar2U3UJPsrdbziLEMMhu3Ujnq//rhiFuIeag==} engines: {node: '>=6'} dependencies: psl: 1.9.0 @@ -24912,7 +24903,7 @@ packages: dev: false file:projects/account-service.tgz: - resolution: {integrity: sha512-Z1uqHeIat92THVFMgHd4yBhVLKJ76ix4IRnjesgUmD/L2OYdIPydaA2vA1Rxr9QqSjhJynMwBPW2dvHP3s40QQ==, tarball: file:projects/account-service.tgz} + resolution: {integrity: sha512-yJ3m7Xo8Fp4h8l/PFxhe9+Z3mytzypkwpy/AAJaSo6qqOIk99GoAQs3fGA5KTMrmBvB4j23BTaNhxfyh+KWNqw==, tarball: file:projects/account-service.tgz} name: '@rush-temp/account-service' version: 0.0.0 dependencies: @@ -24962,7 +24953,7 @@ packages: dev: false file:projects/account.tgz(@types/node@20.11.19)(esbuild@0.20.1)(ts-node@10.9.2): - resolution: {integrity: sha512-dp4BS+teiLPEg/gxddOhYSuaxjuzascxr49DMYfpm4OZEFHdTOHlO3bJXGddaDLgCbT4OcG77W0rdFo38T5/bA==, tarball: file:projects/account.tgz} + resolution: {integrity: sha512-+cnBQYOzJLNcW19m0ILPrDSFaP1C0v0zOnMu5RAWRBhYUe94PZW0NBlz8RhGtnTAJi/BdhKQq3O8ZPrO2rjQVg==, tarball: file:projects/account.tgz} id: file:projects/account.tgz name: '@rush-temp/account' version: 0.0.0 @@ -24995,7 +24986,6 @@ packages: - '@types/node' - babel-jest - babel-plugin-macros - - encoding - esbuild - gcp-metadata - kerberos @@ -25478,7 +25468,7 @@ packages: dev: false file:projects/auth-providers.tgz(@types/node@20.11.19)(esbuild@0.20.1)(ts-node@10.9.2): - resolution: {integrity: sha512-0YbyLxnpaSZfdCfdlt5T9LI/TmahO7fbLohsHvXQVFb2J1InAIyu1WWzbS92CLYLHxQtJmSIOAbPVbRV1wPWbw==, tarball: file:projects/auth-providers.tgz} + resolution: {integrity: sha512-qWWHjVWGqQee3gYpGf2P9eG1UXscJOePyJ8AFmLGyBEJo3VpjtRFVbeU1C4FSQ0z0osIUBX98iM+ZwwY58n4/A==, tarball: file:projects/auth-providers.tgz} id: file:projects/auth-providers.tgz name: '@rush-temp/auth-providers' version: 0.0.0 @@ -28324,13 +28314,14 @@ packages: dev: false file:projects/model-all.tgz: - resolution: {integrity: sha512-p89ECEYZCIf96+OOoZP6ORuDGkYGjQ7KepJHwazf6p8iqmVrSAqFG5sTZKoM6WzRmtZiJo0KAPRsJPJDHNamXQ==, tarball: file:projects/model-all.tgz} + resolution: {integrity: sha512-FHdenDxGsdlbQ6948ecdTtmUq5jHHwbY/ZXllcSYgV5WoGxFG/kJvPH6mqrh2y5MMNDIkcQTkf2J65mAxxdL8w==, tarball: file:projects/model-all.tgz} name: '@rush-temp/model-all' version: 0.0.0 dependencies: '@types/node': 20.11.19 '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0)(eslint@8.56.0)(typescript@5.3.3) '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.3.3) + esbuild: 0.20.1 eslint: 8.56.0 eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0)(eslint-plugin-import@2.29.1)(eslint-plugin-n@15.7.0)(eslint-plugin-promise@6.1.1)(eslint@8.56.0)(typescript@5.3.3) eslint-plugin-import: 2.29.1(eslint@8.56.0) @@ -28687,7 +28678,7 @@ packages: dev: false file:projects/model-love.tgz: - resolution: {integrity: sha512-vHa71QNYOwrIifwIvVUmNWd+qlSgxZ0ErEgrx9uBSnJCxff9iqRhtmrFlfJ27bvi/jiBWOro9XYt5liL7gOGhg==, tarball: file:projects/model-love.tgz} + resolution: {integrity: sha512-UiC6UZqvrfHuUWOu6yZA1F++CSp4lgs3TWAlYXv6yLUAl6tn8m+X5OWbELYe3uPJxKYE/1GGEXL/A8VdnbyRiw==, tarball: file:projects/model-love.tgz} name: '@rush-temp/model-love' version: 0.0.0 dependencies: @@ -29420,25 +29411,6 @@ packages: - supports-color dev: false - file:projects/model-server-translate.tgz: - resolution: {integrity: sha512-Q3KS0+BXrUZvvHVamGHduMNg40BkztvfBie7vfc33sZVblr6XN3fcwt78rQHTzQ1KM5ILdQhkg3wOrObqdnDsg==, tarball: file:projects/model-server-translate.tgz} - name: '@rush-temp/model-server-translate' - version: 0.0.0 - dependencies: - '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0)(eslint@8.56.0)(typescript@5.3.3) - '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.3.3) - eslint: 8.56.0 - eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0)(eslint-plugin-import@2.29.1)(eslint-plugin-n@15.7.0)(eslint-plugin-promise@6.1.1)(eslint@8.56.0)(typescript@5.3.3) - eslint-plugin-import: 2.29.1(eslint@8.56.0) - eslint-plugin-n: 15.7.0(eslint@8.56.0) - eslint-plugin-promise: 6.1.1(eslint@8.56.0) - prettier: 3.2.5 - prettier-plugin-svelte: 3.2.1(prettier@3.2.5)(svelte@4.2.11) - typescript: 5.3.3 - transitivePeerDependencies: - - supports-color - dev: false - file:projects/model-server-view.tgz: resolution: {integrity: sha512-SYVCAS/VCR78LEOhLAyl4VamECJKsUlG64SfXtpOdcVvIDY19PM0xb1U8IWHC6xeON6YAh5Fr1QFoI1okHYT8Q==, tarball: file:projects/model-server-view.tgz} name: '@rush-temp/model-server-view' @@ -30142,7 +30114,7 @@ packages: dev: false file:projects/pod-account.tgz: - resolution: {integrity: sha512-9wMWikXlXBfJQxuSgyWDrZZpIXzg3KTzD3krxImy0o7TRSTyB2whxQdQcA9HtuibsC8CVTyH5p7Ij2RsEAQadQ==, tarball: file:projects/pod-account.tgz} + resolution: {integrity: sha512-stE21nBKo9wG2CZvTtEOSSD91zMmqVa68rq5XFNvohR3yhPRjfl0HHRl2vz8opfQUFqMoOfCk3WJu3LBiR4NTg==, tarball: file:projects/pod-account.tgz} name: '@rush-temp/pod-account' version: 0.0.0 dependencies: @@ -30191,7 +30163,7 @@ packages: dev: false file:projects/pod-ai-bot.tgz(bufferutil@4.0.8)(utf-8-validate@6.0.4)(zod@3.23.8): - resolution: {integrity: sha512-86BRJHqw7YBa4uTCvjyITwGUenWTJ6iSwZHZAI9Tf3gotChn1AhfK0HVVgGVQE+TGmW5R+oLx8s4NBMGBiZx1A==, tarball: file:projects/pod-ai-bot.tgz} + resolution: {integrity: sha512-j8F0CnCN1Dl84mhnG8FIqGksrq9b7KwNf6RwpuaGRrdnclqJ1Hfj+PgCW3mG5YzXqMYYENVF87zkx5+MHmY2Iw==, tarball: file:projects/pod-ai-bot.tgz} id: file:projects/pod-ai-bot.tgz name: '@rush-temp/pod-ai-bot' version: 0.0.0 @@ -30715,7 +30687,7 @@ packages: dev: false file:projects/pod-server.tgz(utf-8-validate@6.0.4): - resolution: {integrity: sha512-R9l8BNTCz6ne+bPSak0RNPiBELxMu56UARkyUUSnUAlmUoJtM5NBZQrtH6Wd13FfQGKtH5FAEMPujhUD8WonJg==, tarball: file:projects/pod-server.tgz} + resolution: {integrity: sha512-R0fObVPXSToiK7ioPeEbZ1gZFFjswVFLPCojWYsoNsHg9TL+kTNUdST63PJr4GCr8XVv9VplfYYcEs7qSARvAw==, tarball: file:projects/pod-server.tgz} id: file:projects/pod-server.tgz name: '@rush-temp/pod-server' version: 0.0.0 @@ -30797,7 +30769,7 @@ packages: dev: false file:projects/pod-sign.tgz: - resolution: {integrity: sha512-T12TaQ811UfPf2jFD/gt98GcUz9CPPFFWJxVwkoEuv4xJZACYN3zU262fRqYoH654osCBPepxbOKZnhYUCwWtg==, tarball: file:projects/pod-sign.tgz} + resolution: {integrity: sha512-TMB8kS/tFThMV9qh1o7nGBgyFK98OUUv4TI3yJj2/0HEMO6DKtAOFH4NPP3C5cvRebdd0vIQiYXwrWAdPOHe7A==, tarball: file:projects/pod-sign.tgz} name: '@rush-temp/pod-sign' version: 0.0.0 dependencies: @@ -30850,7 +30822,7 @@ packages: dev: false file:projects/pod-telegram-bot.tgz(bufferutil@4.0.8)(utf-8-validate@6.0.4): - resolution: {integrity: sha512-ytBZ1LMObeiEgq1Nroue4mWRtiaYvd8Jgm5cEBE1muqps1ZNCrRSoE+2kCYrDOv7MlJBViid2fMU+3H2hGn2Eg==, tarball: file:projects/pod-telegram-bot.tgz} + resolution: {integrity: sha512-rNp3oKDqEjjL21fr2ElG+ESbHMrAaNjHFNMbUhWsODoK0+UKWohlYMui9TPXaN3SwDiAcE/zrmp0Lq2JFO9QDw==, tarball: file:projects/pod-telegram-bot.tgz} id: file:projects/pod-telegram-bot.tgz name: '@rush-temp/pod-telegram-bot' version: 0.0.0 @@ -32308,7 +32280,7 @@ packages: dev: false file:projects/server-backup.tgz(esbuild@0.20.1)(ts-node@10.9.2): - resolution: {integrity: sha512-1G/CeRC6a8hTyfEkae5eIXGQ65v4y07RK1DO+W8WH7uhKgLRLPChyjLIoYa2/GpsxPTV1NyTZ63M3hd8HvSCrw==, tarball: file:projects/server-backup.tgz} + resolution: {integrity: sha512-RCYMutijYbZbV1jzpWwwzQL9bZS8uzTOYX0eZgxmh3aYyU1Ub4v4TixQpdiiH72ujj0V2hPn9bM2hpYaxzbD1w==, tarball: file:projects/server-backup.tgz} id: file:projects/server-backup.tgz name: '@rush-temp/server-backup' version: 0.0.0 @@ -32531,7 +32503,7 @@ packages: dev: false file:projects/server-collaboration.tgz(esbuild@0.20.1)(ts-node@10.9.2): - resolution: {integrity: sha512-2uvEc7pys9XZfhJOz5RfmCWiMrUKjMd3dGm2rCnA19u+1yXuP8quDT3nONQZj5RZqReuJQKIUNjOFUTB0GS7hw==, tarball: file:projects/server-collaboration.tgz} + resolution: {integrity: sha512-pOLHByqV1nJAxe649ImcvLOKMS+W3qR6xGNedfBy6fjhabFQccTOtWaEqv1AsxxaD1kYdYxOdlK3nzcUixBe7Q==, tarball: file:projects/server-collaboration.tgz} id: file:projects/server-collaboration.tgz name: '@rush-temp/server-collaboration' version: 0.0.0 @@ -32686,7 +32658,7 @@ packages: dev: false file:projects/server-core.tgz(esbuild@0.20.1)(ts-node@10.9.2): - resolution: {integrity: sha512-6dpLEnoGUlT2PaIX4Zz1WahwcTsn+uevw8HkUaGnGJX3XbG25kWaZvKFZTY3b7AThZAfvKQFMvau5QmyzUVI/Q==, tarball: file:projects/server-core.tgz} + resolution: {integrity: sha512-NJlQNbZS7co2qKOPf/GLPbc0VgghD4JenfX92uaXNCPM50WDKcnnxrgelP7LJmkK01ZU7oZKd6WfoCMWr06u5w==, tarball: file:projects/server-core.tgz} id: file:projects/server-core.tgz name: '@rush-temp/server-core' version: 0.0.0 @@ -33110,6 +33082,38 @@ packages: - ts-node dev: false + file:projects/server-indexer.tgz(esbuild@0.20.1)(ts-node@10.9.2): + resolution: {integrity: sha512-rMMlimV5wzW1ZPKeGdkdpeODEn0cyYDQJWti1GmagrteQed+ZH0DL51uPP0z7aioLG/ODxgFZKj8GWD+tMSrnA==, tarball: file:projects/server-indexer.tgz} + id: file:projects/server-indexer.tgz + name: '@rush-temp/server-indexer' + version: 0.0.0 + dependencies: + '@types/jest': 29.5.12 + '@types/node': 20.11.19 + '@types/uuid': 8.3.4 + '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0)(eslint@8.56.0)(typescript@5.6.2) + '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.6.2) + eslint: 8.56.0 + eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0)(eslint-plugin-import@2.29.1)(eslint-plugin-n@15.7.0)(eslint-plugin-promise@6.1.1)(eslint@8.56.0)(typescript@5.6.2) + eslint-plugin-import: 2.29.1(eslint@8.56.0) + eslint-plugin-n: 15.7.0(eslint@8.56.0) + eslint-plugin-promise: 6.1.1(eslint@8.56.0) + fast-equals: 5.0.1 + jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2) + prettier: 3.2.5 + ts-jest: 29.1.2(esbuild@0.20.1)(jest@29.7.0)(typescript@5.6.2) + typescript: 5.6.2 + transitivePeerDependencies: + - '@babel/core' + - '@jest/types' + - babel-jest + - babel-plugin-macros + - esbuild + - node-notifier + - supports-color + - ts-node + dev: false + file:projects/server-inventory-resources.tgz(@types/node@20.11.19)(esbuild@0.20.1)(ts-node@10.9.2): resolution: {integrity: sha512-FqIWmm06u2ZlVzQCDza2QMPFhqaM09EkdvQPu8x2KxkDQdpmz9aqDaNmXEWVyrnnPg4e4XADIJZ/rqH8OvjoSw==, tarball: file:projects/server-inventory-resources.tgz} id: file:projects/server-inventory-resources.tgz @@ -33363,7 +33367,7 @@ packages: dev: false file:projects/server-pipeline.tgz: - resolution: {integrity: sha512-LN4wkDOP73wfb36R2//O2OFtR/sdhJYdtmMX4hDSj+xE9yAFDHjFefFavF3Azabsd/QKV4fGcmy0xmXiDGrGzA==, tarball: file:projects/server-pipeline.tgz} + resolution: {integrity: sha512-gaF/29tMBmt1rcHqrdJ87g7xHkb39eb8oZfDEVzyeK7qwf1vRjOYE0el4Pp/5OvkBrZSQGEwKgfUK3lHI3iLJQ==, tarball: file:projects/server-pipeline.tgz} name: '@rush-temp/server-pipeline' version: 0.0.0 dependencies: @@ -33614,7 +33618,7 @@ packages: dev: false file:projects/server-storage.tgz(esbuild@0.20.1): - resolution: {integrity: sha512-j0j1hf0/E2qzPNGrT5ek+dWocNZjrc7NJs5Z3Fw0Nbf1tjiAF8li/QBrOJ1juzoeGPq1yskFk+TPZdVByuDOlg==, tarball: file:projects/server-storage.tgz} + resolution: {integrity: sha512-ncjCQHykYVRqVpX1ROs+t8NspUUa2/X/JwM2feCjwRCfvq2oGMO/JN1PZrGJt5egzPWNwoMe/0NsuXJ9kxBc2g==, tarball: file:projects/server-storage.tgz} id: file:projects/server-storage.tgz name: '@rush-temp/server-storage' version: 0.0.0 @@ -34235,41 +34239,35 @@ packages: - ts-node dev: false - file:projects/server.tgz(esbuild@0.20.1): - resolution: {integrity: sha512-N4bHcWHyLdxRZYmnVe9zkuGfJnryYNFG1oZdFzvN3PlwoWjRrEa9AO+BVqZcMnTYcJmIBx9aSnDG3SRV3CsW4w==, tarball: file:projects/server.tgz} + file:projects/server.tgz(esbuild@0.20.1)(ts-node@10.9.2): + resolution: {integrity: sha512-PwdTpzGoD5SYmDTSD4hJT7ZkG2b05H030bVz5I+vz5f/AmgAzowxfxa0pbMujI6tEiU+o6hDiK25PQXn2xduHA==, tarball: file:projects/server.tgz} id: file:projects/server.tgz name: '@rush-temp/server' version: 0.0.0 dependencies: '@types/jest': 29.5.12 '@types/node': 20.11.19 - '@types/ws': 8.5.11 '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0)(eslint@8.56.0)(typescript@5.3.3) '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.3.3) cross-env: 7.0.3 - elastic-apm-node: 3.26.0 eslint: 8.56.0 eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0)(eslint-plugin-import@2.29.1)(eslint-plugin-n@15.7.0)(eslint-plugin-promise@6.1.1)(eslint@8.56.0)(typescript@5.3.3) eslint-plugin-import: 2.29.1(eslint@8.56.0) eslint-plugin-n: 15.7.0(eslint@8.56.0) eslint-plugin-promise: 6.1.1(eslint@8.56.0) - got: 11.8.6 jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2) prettier: 3.2.5 - prettier-plugin-svelte: 3.2.1(prettier@3.2.5)(svelte@4.2.11) ts-jest: 29.1.2(esbuild@0.20.1)(jest@29.7.0)(typescript@5.3.3) - ts-node: 10.9.2(@types/node@20.11.19)(typescript@5.3.3) typescript: 5.3.3 transitivePeerDependencies: - '@babel/core' - '@jest/types' - - '@swc/core' - - '@swc/wasm' - babel-jest - babel-plugin-macros - esbuild - node-notifier - supports-color + - ts-node dev: false file:projects/setting-assets.tgz(esbuild@0.20.1)(ts-node@10.9.2): @@ -35213,7 +35211,7 @@ packages: dev: false file:projects/text.tgz(@types/node@20.11.19)(bufferutil@4.0.8)(esbuild@0.20.1)(ts-node@10.9.2)(utf-8-validate@6.0.4)(y-protocols@1.0.6): - resolution: {integrity: sha512-vr1gGzEpiIsGo9J12StYEoGh94lXxkPBNCUWYnnTBxOKC62boqNgk2mq13flPrITbd3rWlkNnDSMxgwADN4rsg==, tarball: file:projects/text.tgz} + resolution: {integrity: sha512-krv6M+6hKhpAuRanIAZPdRq+Cxxn7sIxmq443+UrSNHL2zp1gxWbivmsG89t+n8B19PRbUNhmhRk77Uc78gmxg==, tarball: file:projects/text.tgz} id: file:projects/text.tgz name: '@rush-temp/text' version: 0.0.0 @@ -35254,7 +35252,6 @@ packages: markdown-it: 14.0.0 prettier: 3.2.5 prosemirror-codemark: 0.4.2 - prosemirror-model: 1.19.4 ts-jest: 29.1.2(esbuild@0.20.1)(jest@29.7.0)(typescript@5.3.3) typescript: 5.3.3 y-prosemirror: 1.2.12(y-protocols@1.0.6)(yjs@13.6.19) @@ -35711,38 +35708,6 @@ packages: - ts-node dev: false - file:projects/translate.tgz(esbuild@0.20.1)(ts-node@10.9.2): - resolution: {integrity: sha512-3n/OVZcAxY7QVTHt+xyHDI0pMXwwIkvURVj8MKUJSya9HTxX3HRULJTobYSix+nNQjLURgFu62ESbl/7bOJwKg==, tarball: file:projects/translate.tgz} - id: file:projects/translate.tgz - name: '@rush-temp/translate' - version: 0.0.0 - dependencies: - '@types/jest': 29.5.12 - '@types/node': 20.11.19 - '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0)(eslint@8.56.0)(typescript@5.3.3) - '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.3.3) - eslint: 8.56.0 - eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0)(eslint-plugin-import@2.29.1)(eslint-plugin-n@15.7.0)(eslint-plugin-promise@6.1.1)(eslint@8.56.0)(typescript@5.3.3) - eslint-plugin-import: 2.29.1(eslint@8.56.0) - eslint-plugin-n: 15.7.0(eslint@8.56.0) - eslint-plugin-promise: 6.1.1(eslint@8.56.0) - got: 11.8.6 - jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2) - prettier: 3.2.5 - prettier-plugin-svelte: 3.2.1(prettier@3.2.5)(svelte@4.2.11) - ts-jest: 29.1.2(esbuild@0.20.1)(jest@29.7.0)(typescript@5.3.3) - typescript: 5.3.3 - transitivePeerDependencies: - - '@babel/core' - - '@jest/types' - - babel-jest - - babel-plugin-macros - - esbuild - - node-notifier - - supports-color - - ts-node - dev: false - file:projects/ui.tgz(@types/node@20.11.19)(esbuild@0.20.1)(postcss-load-config@4.0.2)(postcss@8.4.35)(ts-node@10.9.2): resolution: {integrity: sha512-umESBjjPj7ES3uF9YcS31H5dwqZtMATByltYeDc+XG+7ovD1SOM11UAjBpHCqj026RvvqcSjE8lAQP1zRXxCoA==, tarball: file:projects/ui.tgz} id: file:projects/ui.tgz diff --git a/dev/.env b/dev/.env index 8306222d285..6650314f86f 100644 --- a/dev/.env +++ b/dev/.env @@ -1,2 +1,3 @@ STORAGE_CONFIG="minio|minio?accessKey=minioadmin&secretKey=minioadmin" -MONGO_URL=mongodb://mongodb:27017?compressors=snappy \ No newline at end of file +MONGO_URL=mongodb://mongodb:27017?compressors=snappy +DB_URL_PG=postgresql://postgres:example@postgres:5432 \ No newline at end of file diff --git a/dev/docker-compose.yaml b/dev/docker-compose.yaml index 7d215c3f174..b9784be4ce9 100644 --- a/dev/docker-compose.yaml +++ b/dev/docker-compose.yaml @@ -25,7 +25,7 @@ services: environment: - POSTGRES_PASSWORD=example volumes: - - db:/data/db + - dbpg:/data/db ports: - 5432:5432 restart: unless-stopped @@ -75,7 +75,8 @@ services: - SERVER_SECRET=secret # - DB_URL=postgresql://postgres:example@postgres:5432 - DB_URL=${MONGO_URL} - - TRANSACTOR_URL=ws://host.docker.internal:3333 + - REGION_INFO=|Mongo;pg|Postgres + - TRANSACTOR_URL=ws://host.docker.internal:3333,ws://host.docker.internal:3331;;pg - SES_URL= - STORAGE_CONFIG=${STORAGE_CONFIG} - FRONT_URL=http://host.docker.internal:8087 @@ -105,7 +106,6 @@ services: - MONGO_URL=${MONGO_URL} - DB_URL=${MONGO_URL} # - DB_URL=postgresql://postgres:example@postgres:5432 - - TRANSACTOR_URL=ws://transactor:3333;ws://host.docker.internal:3333 - SES_URL= - STORAGE_CONFIG=${STORAGE_CONFIG} - FRONT_URL=http://host.docker.internal:8087 @@ -118,6 +118,32 @@ services: # - INIT_SCRIPT_URL=https://raw.githubusercontent.com/hcengineering/init/main/script.yaml # - INIT_WORKSPACE=onboarding restart: unless-stopped + workspacepg: + image: hardcoreeng/workspace + extra_hosts: + - "host.docker.internal:host-gateway" + links: + - postgres + - minio + volumes: + - ./branding.json:/var/cfg/branding.json + environment: + # - WS_OPERATION=create + - SERVER_SECRET=secret + - MONGO_URL=${MONGO_URL} + - DB_URL=postgresql://postgres:example@postgres:5432 + - SES_URL= + - REGION=pg + - STORAGE_CONFIG=${STORAGE_CONFIG} + - FRONT_URL=http://host.docker.internal:8087 + - RESERVED_DB_NAMES=telegram,gmail,github + - MODEL_ENABLED=* + - ACCOUNTS_URL=http://host.docker.internal:3000 + - BRANDING_PATH=/var/cfg/branding.json + # - PARALLEL=2 + # - INIT_SCRIPT_URL=https://raw.githubusercontent.com/hcengineering/init/main/script.yaml + # - INIT_WORKSPACE=onboarding + restart: unless-stopped collaborator: image: hardcoreeng/collaborator extra_hosts: @@ -212,6 +238,44 @@ services: - BRANDING_PATH=/var/cfg/branding.json - SUPPORT_WORKSPACE=support restart: unless-stopped + transactor_pg: + image: hardcoreeng/transactor + extra_hosts: + - "host.docker.internal:host-gateway" + links: + - postgres + - elastic + - minio + - rekoni + - account + # - apm-server + ports: + - 3331:3331 + volumes: + - ./branding.json:/var/cfg/branding.json + environment: + # - SERVER_PROVIDER=uweb + # - UWS_HTTP_MAX_HEADERS_SIZE="32768" + - UV_THREADPOOL_SIZE=10 + - SERVER_PORT=3331 + - SERVER_SECRET=secret + - ENABLE_COMPRESSION=true + - ELASTIC_URL=http://host.docker.internal:9200 + - DB_URL=postgresql://postgres:example@postgres:5432 + - MONGO_URL=${MONGO_URL} + - 'MONGO_OPTIONS={"appName": "transactor-pg", "maxPoolSize": 1}' + - METRICS_CONSOLE=false + - METRICS_FILE=metrics.txt + - STORAGE_CONFIG=${STORAGE_CONFIG} + - REKONI_URL=http://host.docker.internal:4004 + - FRONT_URL=http://host.docker.internal:8087 + # - APM_SERVER_URL=http://apm-server:8200 + - SES_URL='' + - ACCOUNTS_URL=http://host.docker.internal:3000 + - LAST_NAME_FIRST=true + - ELASTIC_INDEX_NAME=local_storage_index + - BRANDING_PATH=/var/cfg/branding.json + restart: unless-stopped rekoni: image: hardcoreeng/rekoni-service restart: unless-stopped @@ -322,5 +386,6 @@ services: # memory: 300M volumes: db: + dbpg: files: elastic: diff --git a/models/server-translate/config/rig.json b/dev/import-tool/config/rig.json similarity index 85% rename from models/server-translate/config/rig.json rename to dev/import-tool/config/rig.json index 2f6be366054..78cc5a17334 100644 --- a/models/server-translate/config/rig.json +++ b/dev/import-tool/config/rig.json @@ -1,5 +1,5 @@ { "$schema": "https://developer.microsoft.com/json-schemas/rig-package/rig.schema.json", "rigPackageName": "@hcengineering/platform-rig", - "rigProfile": "model" + "rigProfile": "node" } diff --git a/server/translate/jest.config.js b/dev/import-tool/jest.config.js similarity index 100% rename from server/translate/jest.config.js rename to dev/import-tool/jest.config.js diff --git a/dev/import-tool/package.json b/dev/import-tool/package.json index 79602a42d97..e268b1d4f8e 100644 --- a/dev/import-tool/package.json +++ b/dev/import-tool/package.json @@ -14,7 +14,7 @@ "_phase:bundle": "rushx bundle", "_phase:docker-build": "rushx docker:build", "_phase:docker-staging": "rushx docker:staging", - "bundle": "mkdir -p bundle && esbuild src/__start.ts --bundle --keep-names --sourcemap=external --platform=node --define:process.env.MODEL_VERSION=$(node ../../common/scripts/show_version.js) --define:process.env.GIT_REVISION=$(../../common/scripts/git_version.sh) --log-level=error --outfile=bundle/bundle.js", + "bundle": "mkdir -p bundle && esbuild src/__start.ts --bundle --keep-names --platform=node --define:process.env.MODEL_VERSION=$(node ../../common/scripts/show_version.js) --define:process.env.GIT_REVISION=$(../../common/scripts/git_version.sh) --outfile=bundle/bundle.js --log-level=error --sourcemap=external", "docker:build": "../../common/scripts/docker_build.sh hardcoreeng/import-tool", "docker:tbuild": "docker build -t hardcoreeng/import-tool . --platform=linux/amd64 && ../../common/scripts/docker_tag_push.sh hardcoreeng/import-tool", "docker:staging": "../../common/scripts/docker_tag.sh hardcoreeng/import-tool staging", diff --git a/dev/prod/webpack.config.js b/dev/prod/webpack.config.js index 93b5b21edae..3669c434cd6 100644 --- a/dev/prod/webpack.config.js +++ b/dev/prod/webpack.config.js @@ -296,8 +296,26 @@ module.exports = [ // for some systems, watching many files can result in a lot of CPU or memory usage // https://webpack.js.org/configuration/watch/#watchoptionsignored // don't use this pattern, if you have a monorepo with linked packages - ignored: /node_modules/, + ignored: [ + "**/node_modules", + "**/.git", + "**/common/temp", + "**/.rush", + "**/.rush/**", + "**/rush-logs", + "**/lib/*.js", + "**/lib/*.js.map", + "**/types/*.d.ts", + "**/.git/objects/**", + "**/node_modules/**", + "**/desktop/deploy/**", + "**/dist/**", + "**/.validate/**", + "**/.format/**", + "**/.build_dev/**" + ], aggregateTimeout: 100, + followSymlinks: false, poll: 250 }, devtool: prod ? 'source-map' : 'eval-source-map', // 'inline-source-map', diff --git a/dev/tool/package.json b/dev/tool/package.json index 8a2e38c948e..75ab7db4bd8 100644 --- a/dev/tool/package.json +++ b/dev/tool/package.json @@ -14,7 +14,7 @@ "_phase:bundle": "rushx bundle", "_phase:docker-build": "rushx docker:build", "_phase:docker-staging": "rushx docker:staging", - "bundle": "mkdir -p bundle && esbuild src/__start.ts --bundle --keep-names --sourcemap=external --platform=node --define:process.env.MODEL_VERSION=$(node ../../common/scripts/show_version.js) --define:process.env.GIT_REVISION=$(../../common/scripts/git_version.sh) --log-level=error --outfile=bundle/bundle.js", + "bundle": "mkdir -p bundle && esbuild src/__start.ts --bundle --keep-names --platform=node --define:process.env.MODEL_VERSION=$(node ../../common/scripts/show_version.js) --define:process.env.GIT_REVISION=$(../../common/scripts/git_version.sh) --outfile=bundle/bundle.js --log-level=error --sourcemap=external", "docker:build": "../../common/scripts/docker_build.sh hardcoreeng/tool", "docker:tbuild": "docker build -t hardcoreeng/tool . --platform=linux/amd64 && ../../common/scripts/docker_tag_push.sh hardcoreeng/tool", "docker:staging": "../../common/scripts/docker_tag.sh hardcoreeng/tool staging", diff --git a/models/all/package.json b/models/all/package.json index 8c84c1aeff9..e5b825c0fca 100644 --- a/models/all/package.json +++ b/models/all/package.json @@ -13,7 +13,10 @@ "format": "format src", "_phase:build": "compile transpile src", "_phase:format": "format src", - "_phase:validate": "compile validate" + "_phase:validate": "compile validate", + "show-model": "node ./bundle/bundle.js", + "_phase:bundle": "rushx bundle", + "bundle": "mkdir -p bundle && esbuild src/show.ts --bundle --keep-names --platform=node --define:process.env.MODEL_VERSION=$(node ../../common/scripts/show_version.js) --define:process.env.VERSION=$(node ../../common/scripts/show_tag.js) --define:process.env.GIT_REVISION=$(../../common/scripts/git_version.sh) --outfile=bundle/bundle.js --log-level=error && node ./bundle/bundle.js > ./bundle/model.json" }, "devDependencies": { "@hcengineering/platform-rig": "^0.6.0", @@ -27,7 +30,8 @@ "@typescript-eslint/parser": "^6.11.0", "eslint-config-standard-with-typescript": "^40.0.0", "prettier": "^3.1.0", - "typescript": "^5.3.3" + "typescript": "^5.3.3", + "esbuild": "^0.20.0" }, "dependencies": { "@hcengineering/model": "^0.6.11", @@ -85,7 +89,6 @@ "@hcengineering/model-support": "^0.6.0", "@hcengineering/model-server-request": "^0.6.0", "@hcengineering/model-server-view": "^0.6.0", - "@hcengineering/model-server-translate": "^0.6.0", "@hcengineering/model-server-activity": "^0.6.0", "@hcengineering/model-drive": "^0.6.0", "@hcengineering/model-guest": "^0.6.0", diff --git a/models/all/src/build.ts b/models/all/src/build.ts new file mode 100644 index 00000000000..a4e5b47a389 --- /dev/null +++ b/models/all/src/build.ts @@ -0,0 +1,7 @@ +import { writeFileSync } from 'fs' +import builder from './' +const enabled = (process.env.MODEL_ENABLED ?? '*').split(',').map((it) => it.trim()) +const disabled = (process.env.MODEL_DISABLED ?? '').split(',').map((it) => it.trim()) + +const model = JSON.stringify(builder(enabled, disabled).getTxes()) +writeFileSync(process.argv[2], model) diff --git a/models/all/src/index.ts b/models/all/src/index.ts index 8548b9751c2..a958108caff 100644 --- a/models/all/src/index.ts +++ b/models/all/src/index.ts @@ -77,7 +77,6 @@ import view, { viewId, createModel as viewModel } from '@hcengineering/model-vie import workbench, { workbenchId, createModel as workbenchModel } from '@hcengineering/model-workbench' import { desktopPreferencesId, createModel as desktopPreferencesModel } from '@hcengineering/model-desktop-preferences' -import { createModel as serverTranslate, translateId } from '@hcengineering/model-server-translate' import document, { documentId, createModel as documentModel } from '@hcengineering/model-document' import { serverDocumentId, createModel as serverDocumentModel } from '@hcengineering/model-server-document' @@ -426,7 +425,6 @@ export default function buildModel (enabled: string[] = ['*'], disabled: string[ [serverRequestModel, serverRequestId], [serverViewModel, serverViewId], [serverActivityModel, serverActivityId], - [serverTranslate, translateId], [serverDocumentModel, serverDocumentId], [serverGithubModel, serverGithubId], [serverLoveModel, serverLoveId], diff --git a/models/all/src/show.ts b/models/all/src/show.ts new file mode 100644 index 00000000000..0198730ed20 --- /dev/null +++ b/models/all/src/show.ts @@ -0,0 +1,6 @@ +import builder from './' +const enabled = (process.env.MODEL_ENABLED ?? '*').split(',').map((it) => it.trim()) +const disabled = (process.env.MODEL_DISABLED ?? '').split(',').map((it) => it.trim()) + +const model = JSON.stringify(builder(enabled, disabled).getTxes()) +console.log(model) diff --git a/models/love/package.json b/models/love/package.json index ac9a360544a..8c16971dbae 100644 --- a/models/love/package.json +++ b/models/love/package.json @@ -30,7 +30,6 @@ "dependencies": { "@hcengineering/platform": "^0.6.11", "@hcengineering/model-presentation": "^0.6.0", - "@hcengineering/presentation": "^0.6.3", "@hcengineering/contact": "^0.6.24", "@hcengineering/core": "^0.6.32", "@hcengineering/model-core": "^0.6.0", diff --git a/models/server-translate/package.json b/models/server-translate/package.json deleted file mode 100644 index c2ef7d82088..00000000000 --- a/models/server-translate/package.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "name": "@hcengineering/model-server-translate", - "version": "0.6.0", - "main": "lib/index.js", - "svelte": "src/index.ts", - "types": "types/index.d.ts", - "author": "Anticrm Platform Contributors", - "template": "@hcengineering/model-package", - "license": "EPL-2.0", - "scripts": { - "build": "compile", - "build:watch": "compile", - "format": "format src", - "_phase:build": "compile transpile src", - "_phase:format": "format src", - "_phase:validate": "compile validate" - }, - "devDependencies": { - "@hcengineering/platform-rig": "^0.6.0", - "@typescript-eslint/eslint-plugin": "^6.11.0", - "eslint-plugin-import": "^2.26.0", - "eslint-plugin-promise": "^6.1.1", - "eslint-plugin-n": "^15.4.0", - "eslint": "^8.54.0", - "@typescript-eslint/parser": "^6.11.0", - "eslint-config-standard-with-typescript": "^40.0.0", - "prettier": "^3.1.0", - "typescript": "^5.3.3" - }, - "dependencies": { - "@hcengineering/core": "^0.6.32", - "@hcengineering/model": "^0.6.11", - "@hcengineering/platform": "^0.6.11", - "@hcengineering/model-core": "^0.6.0", - "@hcengineering/translate": "^0.6.0" - } -} diff --git a/models/server-translate/src/index.ts b/models/server-translate/src/index.ts deleted file mode 100644 index d79e374e06b..00000000000 --- a/models/server-translate/src/index.ts +++ /dev/null @@ -1,37 +0,0 @@ -// -// Copyright © 2022 Hardcore Engineering Inc. -// -// Licensed under the Eclipse Public License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. You may -// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// -// See the License for the specific language governing permissions and -// limitations under the License. -// - -import { type Builder, Model, Prop, TypeString, UX } from '@hcengineering/model' -import { TConfiguration } from '@hcengineering/model-core' -import { getEmbeddedLabel } from '@hcengineering/platform' - -import core, { DOMAIN_CONFIGURATION } from '@hcengineering/core' -import translate, { type TranslateConfiguration } from '@hcengineering/translate/src/plugin' - -export { translateId } from '@hcengineering/translate/src/plugin' - -@Model(translate.class.TranslateConfiguration, core.class.Configuration, DOMAIN_CONFIGURATION) -@UX(getEmbeddedLabel('Retranslation')) -export class TTranslateConfiguration extends TConfiguration implements TranslateConfiguration { - @Prop(TypeString(), getEmbeddedLabel('Token')) - token!: string - - @Prop(TypeString(), getEmbeddedLabel('Token')) - endpoint!: string -} - -export function createModel (builder: Builder): void { - builder.createModel(TTranslateConfiguration) -} diff --git a/models/view/src/plugin.ts b/models/view/src/plugin.ts index aeb975a63d3..d456d2abee9 100644 --- a/models/view/src/plugin.ts +++ b/models/view/src/plugin.ts @@ -17,7 +17,7 @@ import { type Blob, type Doc, type Ref } from '@hcengineering/core' import { type IntlString, mergeIds, type Resource } from '@hcengineering/platform' import { type AnyComponent } from '@hcengineering/ui/src/types' import { type FilterFunction, type ViewAction, type ViewCategoryAction, viewId } from '@hcengineering/view' -import { type BlobMetadata, type FileOrBlob, type FilePreviewExtension } from '@hcengineering/presentation' +import { type BlobMetadata, type FileOrBlob, type FilePreviewExtension } from '@hcengineering/presentation/src/types' import { type PresentationMiddlewareFactory } from '@hcengineering/presentation/src/pipeline' import view from '@hcengineering/view-resources/src/plugin' diff --git a/packages/platform-rig/bin/compile.js b/packages/platform-rig/bin/compile.js index bff64655cbb..80bf3106a2d 100755 --- a/packages/platform-rig/bin/compile.js +++ b/packages/platform-rig/bin/compile.js @@ -4,6 +4,7 @@ const { spawn } = require('child_process') const esbuild = require('esbuild') const { copy } = require('esbuild-plugin-copy') +const fs = require('fs') async function execProcess(cmd, logFile, args, buildDir= '.build') { let compileRoot = dirname(dirname(process.argv[1])) @@ -89,6 +90,36 @@ function collectFiles(source) { return result } +function collectFileStats(source, result) { + if( !existsSync(source)) { + return + } + const files = readdirSync(source) + for (const f of files) { + const sourceFile = join(source, f) + const stat = lstatSync(sourceFile) + if (stat.isDirectory()) { + collectFileStats(sourceFile, result) + } else { + let ext = basename(sourceFile) + if (!ext.endsWith('.ts') && !ext.endsWith('.js') && !ext.endsWith('.svelte')) { + continue + } + result[sourceFile] = stat.mtime.getTime() + } + } +} + +function cleanNonModified(before, after) { + for( const [k,v] of Object.entries(before)) { + if( after[k] === v) { + // Same modify date, looks like not modified + console.log('clean file', k) + rmSync(k) + } + } +} + switch (args[0]) { case 'ui': { console.log('Nothing to compile to UI') @@ -97,9 +128,15 @@ switch (args[0]) { case 'transpile': { const filesToTranspile = collectFiles(join(process.cwd(), args[1])) let st = Date.now() - performESBuild(filesToTranspile) + const before = {} + const after = {} + collectFileStats('lib', before) + + performESBuild(filesToTranspile) .then(() => { console.log("Transpile time: ", Date.now() - st) + collectFileStats('lib', after) + cleanNonModified(before, after) }) break } @@ -125,7 +162,7 @@ switch (args[0]) { break } } -async function performESBuild(filesToTranspile) { +async function performESBuild(filesToTranspile) { await esbuild.build({ entryPoints: filesToTranspile, bundle: false, diff --git a/packages/text/package.json b/packages/text/package.json index c235dd88c67..cc70f410c76 100644 --- a/packages/text/package.json +++ b/packages/text/package.json @@ -33,10 +33,10 @@ "prettier": "^3.1.0", "typescript": "^5.3.3", "jest": "^29.7.0", - "jest-environment-jsdom": "29.7.0", "ts-jest": "^29.1.1", "@types/jest": "^29.5.5", - "@types/markdown-it": "~13.0.0" + "@types/markdown-it": "~13.0.0", + "jest-environment-jsdom": "^29.7.0" }, "dependencies": { "@hcengineering/core": "^0.6.32", diff --git a/plugins/login-resources/src/components/CreateWorkspaceForm.svelte b/plugins/login-resources/src/components/CreateWorkspaceForm.svelte index 0b1d99e773c..7a72ebe98b3 100644 --- a/plugins/login-resources/src/components/CreateWorkspaceForm.svelte +++ b/plugins/login-resources/src/components/CreateWorkspaceForm.svelte @@ -17,8 +17,8 @@ import { Status, Severity, OK } from '@hcengineering/platform' import Form from './Form.svelte' - import { createWorkspace, getAccount, goTo, setLoginInfo } from '../utils' - import { getCurrentLocation, navigate } from '@hcengineering/ui' + import { createWorkspace, getAccount, getRegionInfo, goTo, setLoginInfo, type RegionInfo } from '../utils' + import { getCurrentLocation, navigate, DropdownLabels } from '@hcengineering/ui' import login from '../plugin' import { workbenchId } from '@hcengineering/workbench' import { onMount } from 'svelte' @@ -40,9 +40,13 @@ let status: Status = OK let account: LoginInfo | undefined = undefined + let regions: RegionInfo[] = [] + let selectedRegion: string = '' onMount(async () => { account = await getAccount() + regions = (await getRegionInfo()) ?? [] + selectedRegion = regions[0]?.region if (account?.confirmed === false) { const loc = getCurrentLocation() loc.path[1] = 'confirmationSend' @@ -56,11 +60,11 @@ func: async () => { status = new Status(Severity.INFO, login.status.ConnectingToServer, {}) - const [loginStatus, result] = await createWorkspace(object.workspace) + const [loginStatus, result] = await createWorkspace(object.workspace, selectedRegion) status = loginStatus if (result !== undefined) { - setLoginInfo(result) + setLoginInfo(result as any) navigate({ path: [workbenchId, result.workspace] }) } @@ -68,6 +72,12 @@ } +{#if regions.length > 1} +
+ ({ id: it.region, label: it.name }))} /> +
+{/if} +
- {#each workspaces.filter((it) => search === '' || (it.workspaceName?.includes(search) ?? false) || it.workspace.includes(search)) as workspace} + {#each workspaces + .slice(0, 500) + .filter((it) => search === '' || (it.workspaceName?.includes(search) ?? false) || it.workspace.includes(search)) as workspace} {@const wsName = workspace.workspaceName ?? workspace.workspace} + {@const lastUsageDays = Math.round((Date.now() - workspace.lastVisit) / (1000 * 3600 * 24))}
- {#if isAdmin && wsName !== workspace.workspace} - + {#if isAdmin} + {workspace.workspace} + {#if workspace.region !== undefined} + at ({workspace.region}) + {/if} +
+ {#if workspace.backupInfo != null} + {@const sz = workspace.backupInfo.dataSize + workspace.backupInfo.blobsSize} + {@const szGb = Math.round((sz * 100) / 1024) / 100} + {#if szGb > 0} + - {Math.round((sz * 100) / 1024) / 100}Gb - + {:else} + - {Math.round(sz)}Mb - + {/if} + {/if} + ({lastUsageDays} days) +
{/if}
diff --git a/plugins/login-resources/src/utils.ts b/plugins/login-resources/src/utils.ts index b2994745c68..a9fbd412b11 100644 --- a/plugins/login-resources/src/utils.ts +++ b/plugins/login-resources/src/utils.ts @@ -160,7 +160,8 @@ export async function signUpOtp (email: string): Promise<[Status, OtpInfo | unde } export async function createWorkspace ( - workspaceName: string + workspaceName: string, + region?: string ): Promise<[Status, (LoginInfo & { workspace: string }) | undefined]> { const accountsUrl = getMetadata(login.metadata.AccountsUrl) @@ -179,7 +180,7 @@ export async function createWorkspace ( const request = { method: 'createWorkspace', - params: [workspaceName] + params: [workspaceName, region] } try { @@ -310,6 +311,53 @@ export async function getAccount (doNavigate: boolean = true): Promise { + const accountsUrl = getMetadata(login.metadata.AccountsUrl) + + if (accountsUrl === undefined) { + throw new Error('accounts url not specified') + } + + const token = getMetadata(presentation.metadata.Token) ?? fetchMetadataLocalStorage(login.metadata.LastToken) + if (token === undefined) { + if (doNavigate) { + const loc = getCurrentLocation() + loc.path[1] = 'login' + loc.path.length = 2 + navigate(loc) + } + return + } + + const request = { + method: 'getRegionInfo', + params: [] as any[] + } + + try { + const response = await fetch(accountsUrl, { + method: 'POST', + headers: { + Authorization: 'Bearer ' + token, + 'Content-Type': 'application/json' + }, + body: JSON.stringify(request) + }) + const result = await response.json() + if (result.error != null) { + throw new PlatformError(result.error) + } + return result.result + } catch (err: any) { + Analytics.handleError(err) + } +} + export async function selectWorkspace ( workspace: string, token?: string | null | undefined diff --git a/plugins/login/src/index.ts b/plugins/login/src/index.ts index fecbade5874..1e873843898 100644 --- a/plugins/login/src/index.ts +++ b/plugins/login/src/index.ts @@ -37,6 +37,8 @@ export interface Workspace { lastVisit: number backupInfo?: BackupStatus + + region?: string } /** diff --git a/plugins/workbench-resources/src/components/SelectWorkspaceMenu.svelte b/plugins/workbench-resources/src/components/SelectWorkspaceMenu.svelte index d42ec511af0..66ca71688f6 100644 --- a/plugins/workbench-resources/src/components/SelectWorkspaceMenu.svelte +++ b/plugins/workbench-resources/src/components/SelectWorkspaceMenu.svelte @@ -178,6 +178,9 @@
{wsName} + {#if ws.region != null && ws.region !== ''} + - ({ws.region}) + {/if} {#if isAdmin && ws.lastVisit != null && ws.lastVisit !== 0}
{#if ws.backupInfo != null} diff --git a/pods/account/package.json b/pods/account/package.json index 2cc2b9e9417..bb1b2681ccf 100644 --- a/pods/account/package.json +++ b/pods/account/package.json @@ -14,7 +14,7 @@ "_phase:bundle": "rushx bundle", "_phase:docker-build": "rushx docker:build", "_phase:docker-staging": "rushx docker:staging", - "bundle": "mkdir -p bundle && esbuild src/__start.ts --sourcemap=inline --external:*.node --external:snappy --bundle --define:process.env.MODEL_VERSION=$(node ../../common/scripts/show_version.js) --minify --platform=node > bundle/bundle.js", + "bundle": "mkdir -p bundle && esbuild src/__start.ts --external:*.node --external:snappy --keep-names --bundle --define:process.env.MODEL_VERSION=$(node ../../common/scripts/show_version.js) --platform=node --outfile=bundle/bundle.js --log-level=error --sourcemap=external", "docker:build": "../../common/scripts/docker_build.sh hardcoreeng/account", "docker:tbuild": "docker build -t hardcoreeng/account . --platform=linux/amd64 && ../../common/scripts/docker_tag_push.sh hardcoreeng/account", "docker:abuild": "docker build -t hardcoreeng/account . --platform=linux/arm64 && ../../common/scripts/docker_tag_push.sh hardcoreeng/account", @@ -64,7 +64,6 @@ "@koa/cors": "^5.0.0", "@hcengineering/server-token": "^0.6.11", "@hcengineering/server-core": "^0.6.1", - "@hcengineering/model-all": "^0.6.0", "@hcengineering/analytics": "^0.6.0", "@hcengineering/analytics-service": "^0.6.0" } diff --git a/pods/backup/Dockerfile b/pods/backup/Dockerfile index aab274c7863..2450bc6e2ca 100644 --- a/pods/backup/Dockerfile +++ b/pods/backup/Dockerfile @@ -13,6 +13,8 @@ ENV LD_PRELOAD=libjemalloc.so.2 ENV MALLOC_CONF=dirty_decay_ms:1000,narenas:2,background_thread:true COPY bundle/bundle.js ./ +COPY bundle/bundle.js.map ./ +COPY bundle/model.json ./ EXPOSE 3000 CMD [ "node", "bundle.js" ] diff --git a/pods/backup/package.json b/pods/backup/package.json index 98fe4a9746f..fdae32ea7f1 100644 --- a/pods/backup/package.json +++ b/pods/backup/package.json @@ -14,7 +14,8 @@ "_phase:bundle": "rushx bundle", "_phase:docker-build": "rushx docker:build", "_phase:docker-staging": "rushx docker:staging", - "bundle": "mkdir -p bundle && esbuild src/index.ts --bundle --sourcemap=inline --platform=node --external:*.node --external:bufferutil --external:snappy > bundle/bundle.js", + "get-model": "mkdir -p bundle && esbuild src/get-model.ts --bundle --keep-names --platform=node --define:process.env.MODEL_VERSION=$(node ../../common/scripts/show_version.js) --define:process.env.VERSION=$(node ../../common/scripts/show_tag.js) --define:process.env.GIT_REVISION=$(../../common/scripts/git_version.sh) --outfile=bundle/bundle.js --log-level=error && node ./bundle/bundle.js > ./bundle/model.json", + "bundle": "mkdir -p bundle && rushx get-model && esbuild src/index.ts --keep-names --bundle --platform=node --external:*.node --outfile=bundle/bundle.js --log-level=error --sourcemap=external", "docker:build": "../../common/scripts/docker_build.sh hardcoreeng/backup", "docker:tbuild": "docker build -t hardcoreeng/backup . --platform=linux/amd64 && ../../common/scripts/docker_tag_push.sh hardcoreeng/backup", "docker:staging": "../../common/scripts/docker_tag.sh hardcoreeng/backup staging", @@ -44,7 +45,8 @@ "typescript": "^5.3.3", "jest": "^29.7.0", "ts-jest": "^29.1.1", - "@types/jest": "^29.5.5" + "@types/jest": "^29.5.5", + "@hcengineering/model-all": "^0.6.0" }, "dependencies": { "@hcengineering/platform": "^0.6.11", @@ -57,7 +59,6 @@ "dotenv": "~16.0.0", "@hcengineering/backup-service": "^0.6.0", "@hcengineering/analytics": "^0.6.0", - "@hcengineering/analytics-service": "^0.6.0", - "@hcengineering/model-all": "^0.6.0" + "@hcengineering/analytics-service": "^0.6.0" } } diff --git a/pods/backup/src/get-model.ts b/pods/backup/src/get-model.ts new file mode 100644 index 00000000000..9f9b31b2936 --- /dev/null +++ b/pods/backup/src/get-model.ts @@ -0,0 +1 @@ +import '@hcengineering/model-all/src/show' diff --git a/pods/backup/src/index.ts b/pods/backup/src/index.ts index e746f59e23e..4aa4be81bb7 100644 --- a/pods/backup/src/index.ts +++ b/pods/backup/src/index.ts @@ -22,12 +22,8 @@ import { createBackupPipeline, getConfig } from '@hcengineering/server-pipeline' import { writeFile } from 'fs/promises' import { join } from 'path' -import builder from '@hcengineering/model-all' - -const enabled = (process.env.MODEL_ENABLED ?? '*').split(',').map((it) => it.trim()) -const disabled = (process.env.MODEL_DISABLED ?? '').split(',').map((it) => it.trim()) - -const model = JSON.parse(JSON.stringify(builder(enabled, disabled).getTxes())) as Tx[] +import { readFileSync } from 'node:fs' +const model = JSON.parse(readFileSync(process.env.MODEL_JSON ?? 'model.json').toString()) as Tx[] const metricsContext = new MeasureMetricsContext( 'backup', diff --git a/pods/collaborator/Dockerfile b/pods/collaborator/Dockerfile index 2f2ec250534..59f6041e61b 100644 --- a/pods/collaborator/Dockerfile +++ b/pods/collaborator/Dockerfile @@ -12,6 +12,7 @@ ENV LD_PRELOAD=libjemalloc.so.2 ENV MALLOC_CONF=dirty_decay_ms:1000,narenas:2,background_thread:true COPY bundle/bundle.js ./ +COPY bundle/bundle.js.map ./ EXPOSE 3078 CMD [ "node", "bundle.js" ] diff --git a/pods/collaborator/package.json b/pods/collaborator/package.json index 31898da99fb..47fc9d59959 100644 --- a/pods/collaborator/package.json +++ b/pods/collaborator/package.json @@ -13,7 +13,7 @@ "_phase:bundle": "rushx bundle", "_phase:docker-build": "rushx docker:build", "_phase:docker-staging": "rushx docker:staging", - "bundle": "mkdir -p bundle && esbuild src/__start.ts --bundle --platform=node --sourcemap=inline --keep-names --external:*.node --external:bufferutil --external:snappy > bundle/bundle.js", + "bundle": "mkdir -p bundle && esbuild src/__start.ts --bundle --platform=node --keep-names --external:*.node --external:bufferutil --external:snappy --outfile=bundle/bundle.js --log-level=error --sourcemap=external", "docker:build": "../../common/scripts/docker_build.sh hardcoreeng/collaborator", "docker:tbuild": "docker build -t hardcoreeng/collaborator . --platform=linux/amd64 && ../../common/scripts/docker_tag_push.sh hardcoreeng/collaborator", "docker:abuild": "docker build -t hardcoreeng/collaborator . --platform=linux/arm64 && ../../common/scripts/docker_tag_push.sh hardcoreeng/collaborator", diff --git a/pods/server/Dockerfile b/pods/server/Dockerfile index 20a45aca0cc..251577680cc 100644 --- a/pods/server/Dockerfile +++ b/pods/server/Dockerfile @@ -10,6 +10,7 @@ RUN apt-get clean ENV LD_PRELOAD=libjemalloc.so.2 ENV MALLOC_CONF=dirty_decay_ms:1000,narenas:2,background_thread:true +COPY bundle/model.json ./ COPY bundle/bundle.js ./ COPY bundle/bundle.js.map ./ diff --git a/pods/server/package.json b/pods/server/package.json index a792486a165..f4a303072ac 100644 --- a/pods/server/package.json +++ b/pods/server/package.json @@ -8,14 +8,15 @@ "template": "@hcengineering/node-package", "license": "EPL-2.0", "scripts": { - "start": "rush bundle --to @hcengineering/pod-server && cross-env NODE_ENV=production ELASTIC_INDEX_NAME=local_storage_index MODEL_VERSION=$(node ../../common/scripts/show_version.js) ACCOUNTS_URL=http://localhost:3000 REKONI_URL=http://localhost:4004 MONGO_URL=mongodb://localhost:27017 ELASTIC_URL=http://localhost:9200 FRONT_URL=http://localhost:8087 UPLOAD_URL=/upload MINIO_ENDPOINT=localhost MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin METRICS_CONSOLE=true SERVER_SECRET=secret OPERATION_PROFILING=false node bundle/bundle.js", - "start-u": "rush bundle --to @hcengineering/pod-server && ./bundle/ && cross-env NODE_ENV=production SERVER_PROVIDER=uweb ELASTIC_INDEX_NAME=local_storage_index MODEL_VERSION=$(node ../../common/scripts/show_version.js) ACCOUNTS_URL=http://localhost:3000 REKONI_URL=http://localhost:4004 MONGO_URL=mongodb://localhost:27017 ELASTIC_URL=http://localhost:9200 FRONT_URL=http://localhost:8087 UPLOAD_URL=/upload MINIO_ENDPOINT=localhost MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin METRICS_CONSOLE=true SERVER_SECRET=secret node bundle/bundle.js", - "start-flame": "rush bundle --to @hcengineering/pod-server && cross-env NODE_ENV=production ELASTIC_INDEX_NAME=local_storage_index MODEL_VERSION=$(node ../../common/scripts/show_version.js) ACCOUNTS_URL=http://localhost:3000 REKONI_URL=http://localhost:4004 MONGO_URL=mongodb://localhost:27017 ELASTIC_URL=http://localhost:9200 FRONT_URL=http://localhost:8087 UPLOAD_URL=/upload MINIO_ENDPOINT=localhost MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin METRICS_CONSOLE=true SERVER_SECRET=secret clinic flame --dest ./out -- node --nolazy -r ts-node/register --enable-source-maps src/__start.ts", + "start": "rush bundle --to @hcengineering/pod-server && cross-env NODE_ENV=production ELASTIC_INDEX_NAME=local_storage_index MODEL_VERSION=$(node ../../common/scripts/show_version.js) ACCOUNTS_URL=http://localhost:3000 REKONI_URL=http://localhost:4004 MONGO_URL=mongodb://localhost:27017 ELASTIC_URL=http://localhost:9200 FRONT_URL=http://localhost:8087 UPLOAD_URL=/upload MINIO_ENDPOINT=localhost MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin METRICS_CONSOLE=true SERVER_SECRET=secret OPERATION_PROFILING=false MODEL_JSON=../../models/all/bundle/model.json node bundle/bundle.js", + "start-u": "rush bundle --to @hcengineering/pod-server && ./bundle/ && cross-env NODE_ENV=production SERVER_PROVIDER=uweb ELASTIC_INDEX_NAME=local_storage_index MODEL_VERSION=$(node ../../common/scripts/show_version.js) ACCOUNTS_URL=http://localhost:3000 REKONI_URL=http://localhost:4004 MONGO_URL=mongodb://localhost:27017 ELASTIC_URL=http://localhost:9200 FRONT_URL=http://localhost:8087 UPLOAD_URL=/upload MINIO_ENDPOINT=localhost MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin METRICS_CONSOLE=true SERVER_SECRET=secret MODEL_JSON=../../models/all/bundle/model.json node bundle/bundle.js", + "start-flame": "rush bundle --to @hcengineering/pod-server && cross-env NODE_ENV=production ELASTIC_INDEX_NAME=local_storage_index MODEL_VERSION=$(node ../../common/scripts/show_version.js) ACCOUNTS_URL=http://localhost:3000 REKONI_URL=http://localhost:4004 MONGO_URL=mongodb://localhost:27017 ELASTIC_URL=http://localhost:9200 FRONT_URL=http://localhost:8087 UPLOAD_URL=/upload MINIO_ENDPOINT=localhost MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin METRICS_CONSOLE=true SERVER_SECRET=secret MODEL_JSON=../../models/all/bundle/model.json clinic flame --dest ./out -- node --nolazy -r ts-node/register --enable-source-maps src/__start.ts", "build": "compile", "_phase:bundle": "rushx bundle", "_phase:docker-build": "rushx docker:build", "_phase:docker-staging": "rushx docker:staging", - "bundle": "mkdir -p bundle && esbuild src/__start.ts --sourcemap=inline --bundle --keep-names --platform=node --external:*.node --external:bufferutil --external:snappy --external:utf-8-validate --external:msgpackr-extract --define:process.env.MODEL_VERSION=$(node ../../common/scripts/show_version.js) --define:process.env.VERSION=$(node ../../common/scripts/show_tag.js) --define:process.env.GIT_REVISION=$(../../common/scripts/git_version.sh) --outfile=bundle/bundle.js --log-level=error --sourcemap=external", + "get-model": "mkdir -p bundle && esbuild src/get-model.ts --bundle --keep-names --platform=node --define:process.env.MODEL_VERSION=$(node ../../common/scripts/show_version.js) --define:process.env.VERSION=$(node ../../common/scripts/show_tag.js) --define:process.env.GIT_REVISION=$(../../common/scripts/git_version.sh) --outfile=bundle/bundle.js --log-level=error && node ./bundle/bundle.js > ./bundle/model.json", + "bundle": "mkdir -p bundle && rushx get-model && esbuild src/__start.ts --bundle --keep-names --platform=node --external:*.node --external:bufferutil --external:snappy --external:utf-8-validate --external:msgpackr-extract --define:process.env.MODEL_VERSION=$(node ../../common/scripts/show_version.js) --define:process.env.VERSION=$(node ../../common/scripts/show_tag.js) --define:process.env.GIT_REVISION=$(../../common/scripts/git_version.sh) --outfile=bundle/bundle.js --log-level=error --sourcemap=external", "docker:build": "../../common/scripts/docker_build.sh hardcoreeng/transactor", "docker:tbuild": "docker build -t hardcoreeng/transactor . --platform=linux/amd64 && ../../common/scripts/docker_tag_push.sh hardcoreeng/transactor", "docker:abuild": "docker build -t hardcoreeng/transactor . --platform=linux/arm64 && ../../common/scripts/docker_tag_push.sh hardcoreeng/transactor", @@ -48,7 +49,8 @@ "typescript": "^5.3.3", "jest": "^29.7.0", "ts-jest": "^29.1.1", - "@types/jest": "^29.5.5" + "@types/jest": "^29.5.5", + "@hcengineering/model-all": "^0.6.0" }, "dependencies": { "@hcengineering/core": "^0.6.32", @@ -63,7 +65,6 @@ "@hcengineering/server-token": "^0.6.11", "@hcengineering/middleware": "^0.6.0", "@hcengineering/minio": "^0.6.0", - "@hcengineering/translate": "^0.6.0", "@hcengineering/analytics": "^0.6.0", "@hcengineering/analytics-service": "^0.6.0", "@hcengineering/contact": "^0.6.24", @@ -73,7 +74,6 @@ "@hcengineering/pod-telegram-bot": "^0.6.0", "@hcengineering/server-ai-bot": "^0.6.0", "@hcengineering/server-ai-bot-resources": "^0.6.0", - "@hcengineering/model-all": "^0.6.0", "ws": "^8.18.0", "bufferutil": "^4.0.8", "msgpackr": "^1.11.0", diff --git a/pods/server/src/__start.ts b/pods/server/src/__start.ts index c6d7f2e76d5..916e6977f2a 100644 --- a/pods/server/src/__start.ts +++ b/pods/server/src/__start.ts @@ -9,7 +9,7 @@ import contactPlugin from '@hcengineering/contact' import { MeasureMetricsContext, newMetrics, setOperationLogProfiling } from '@hcengineering/core' import notification from '@hcengineering/notification' import { setMetadata } from '@hcengineering/platform' -import { getMetricsContext, serverConfigFromEnv } from '@hcengineering/server' +import { serverConfigFromEnv } from '@hcengineering/server' import serverAiBot from '@hcengineering/server-ai-bot' import serverCore, { type StorageConfiguration, loadBrandingMap } from '@hcengineering/server-core' import serverNotification from '@hcengineering/server-notification' @@ -20,6 +20,7 @@ import { startHttpServer } from '@hcengineering/server-ws' import { join } from 'path' import { start } from '.' import { profileStart, profileStop } from './inspector' +import { getMetricsContext } from './metrics' configureAnalytics(process.env.SENTRY_DSN, {}) Analytics.setTag('application', 'transactor') diff --git a/pods/server/src/get-model.ts b/pods/server/src/get-model.ts new file mode 100644 index 00000000000..9f9b31b2936 --- /dev/null +++ b/pods/server/src/get-model.ts @@ -0,0 +1 @@ +import '@hcengineering/model-all/src/show' diff --git a/server/server/src/metrics.ts b/pods/server/src/metrics.ts similarity index 93% rename from server/server/src/metrics.ts rename to pods/server/src/metrics.ts index eaa8cdbb3a8..8b5714223fd 100644 --- a/server/server/src/metrics.ts +++ b/pods/server/src/metrics.ts @@ -1,4 +1,4 @@ -import { MeasureContext, MeasureMetricsContext, metricsToString, newMetrics } from '@hcengineering/core' +import { type MeasureContext, MeasureMetricsContext, metricsToString, newMetrics } from '@hcengineering/core' import { writeFile } from 'fs/promises' const metricsFile = process.env.METRICS_FILE diff --git a/pods/server/src/server.ts b/pods/server/src/server.ts index 4a71e2921a5..b708ce9e7b0 100644 --- a/pods/server/src/server.ts +++ b/pods/server/src/server.ts @@ -15,22 +15,19 @@ // import { type Branding, type BrandingMap, type Tx, type WorkspaceIdWithUrl } from '@hcengineering/core' -import { buildStorageFromConfig, getMetricsContext } from '@hcengineering/server' +import { buildStorageFromConfig } from '@hcengineering/server-storage' +import { getMetricsContext } from './metrics' -import { ClientSession, startSessionManager, type ServerFactory, type Session } from '@hcengineering/server' -import { type Pipeline, type StorageConfiguration } from '@hcengineering/server-core' +import { ClientSession, startSessionManager } from '@hcengineering/server' +import { type Pipeline, type ServerFactory, type Session, type StorageConfiguration } from '@hcengineering/server-core' import { type Token } from '@hcengineering/server-token' import { serverAiBotId } from '@hcengineering/server-ai-bot' import { createAIBotAdapter } from '@hcengineering/server-ai-bot-resources' import { createServerPipeline, registerServerPlugins, registerStringLoaders } from '@hcengineering/server-pipeline' -import builder from '@hcengineering/model-all' - -const enabled = (process.env.MODEL_ENABLED ?? '*').split(',').map((it) => it.trim()) -const disabled = (process.env.MODEL_DISABLED ?? '').split(',').map((it) => it.trim()) - -const model = JSON.parse(JSON.stringify(builder(enabled, disabled).getTxes())) as Tx[] +import { readFileSync } from 'node:fs' +const model = JSON.parse(readFileSync(process.env.MODEL_JSON ?? 'model.json').toString()) as Tx[] registerStringLoaders() diff --git a/pods/workspace/Dockerfile b/pods/workspace/Dockerfile index 1086d64b386..e648d52bde0 100644 --- a/pods/workspace/Dockerfile +++ b/pods/workspace/Dockerfile @@ -12,5 +12,6 @@ ENV LD_PRELOAD=libjemalloc.so.2 ENV MALLOC_CONF=dirty_decay_ms:1000,narenas:2,background_thread:true COPY bundle/bundle.js ./ +COPY bundle/bundle.js.map ./ CMD [ "node", "bundle.js" ] diff --git a/pods/workspace/package.json b/pods/workspace/package.json index ed5178438ee..30b42c16888 100644 --- a/pods/workspace/package.json +++ b/pods/workspace/package.json @@ -14,7 +14,7 @@ "_phase:bundle": "rushx bundle", "_phase:docker-build": "rushx docker:build", "_phase:docker-staging": "rushx docker:staging", - "bundle": "mkdir -p bundle && esbuild src/__start.ts --sourcemap=inline --external:*.node --external:snappy --bundle --define:process.env.MODEL_VERSION=$(node ../../common/scripts/show_version.js) --minify --platform=node > bundle/bundle.js", + "bundle": "mkdir -p bundle && esbuild src/__start.ts --external:*.node --external:snappy --bundle --define:process.env.MODEL_VERSION=$(node ../../common/scripts/show_version.js) --minify --platform=node --outfile=bundle/bundle.js --log-level=error --sourcemap=external", "docker:build": "../../common/scripts/docker_build.sh hardcoreeng/workspace", "docker:tbuild": "docker build -t hardcoreeng/workspace . --platform=linux/amd64 && ../../common/scripts/docker_tag_push.sh hardcoreeng/workspace", "docker:abuild": "docker build -t hardcoreeng/workspace . --platform=linux/arm64 && ../../common/scripts/docker_tag_push.sh hardcoreeng/workspace", diff --git a/rush.json b/rush.json index 14a26885d97..17df5d89717 100644 --- a/rush.json +++ b/rush.json @@ -505,6 +505,11 @@ "projectFolder": "server/core", "shouldPublish": false }, + { + "packageName": "@hcengineering/server-indexer", + "projectFolder": "server/indexer", + "shouldPublish": false + }, { "packageName": "@hcengineering/server-token", "projectFolder": "server/token", @@ -1526,16 +1531,6 @@ "projectFolder": "server-plugins/request-resources", "shouldPublish": false }, - { - "packageName": "@hcengineering/translate", - "projectFolder": "server/translate", - "shouldPublish": false - }, - { - "packageName": "@hcengineering/model-server-translate", - "projectFolder": "models/server-translate", - "shouldPublish": false - }, { "packageName": "@hcengineering/server-view", "projectFolder": "server-plugins/view", diff --git a/server-plugins/collaboration/package.json b/server-plugins/collaboration/package.json index f29c2ef5e4e..a5d892745ef 100644 --- a/server-plugins/collaboration/package.json +++ b/server-plugins/collaboration/package.json @@ -40,6 +40,7 @@ "dependencies": { "@hcengineering/core": "^0.6.32", "@hcengineering/platform": "^0.6.11", - "@hcengineering/server-core": "^0.6.1" + "@hcengineering/server-core": "^0.6.1", + "@hcengineering/server-indexer": "^0.6.0" } } diff --git a/server-plugins/collaboration/src/fulltext.ts b/server-plugins/collaboration/src/fulltext.ts index 7a9bcf9d005..f07fbab1fc4 100644 --- a/server-plugins/collaboration/src/fulltext.ts +++ b/server-plugins/collaboration/src/fulltext.ts @@ -27,20 +27,18 @@ import core, { collaborativeDocParse, getFullTextIndexableAttributes } from '@hcengineering/core' +import { ContentTextAdapter, DbAdapter, IndexedDoc, StorageAdapter } from '@hcengineering/server-core' + import { - ContentTextAdapter, - DbAdapter, DocUpdateHandler, FullTextPipeline, FullTextPipelineStage, - IndexedDoc, - StorageAdapter, collabStageId, contentStageId, docKey, docUpdKey, fieldStateId -} from '@hcengineering/server-core' +} from '@hcengineering/server-indexer' /** * @public diff --git a/server/account-service/package.json b/server/account-service/package.json index 182635f2122..5471ff7c06e 100644 --- a/server/account-service/package.json +++ b/server/account-service/package.json @@ -54,7 +54,6 @@ "koa-bodyparser": "^4.4.1", "@koa/cors": "^5.0.0", "@hcengineering/server-tool": "^0.6.0", - "@hcengineering/server-client": "^0.6.0", "@hcengineering/server-token": "^0.6.11", "@hcengineering/analytics": "^0.6.0" } diff --git a/server/account-service/src/index.ts b/server/account-service/src/index.ts index bafb25415fd..22da0613ac8 100644 --- a/server/account-service/src/index.ts +++ b/server/account-service/src/index.ts @@ -16,7 +16,6 @@ import { Analytics } from '@hcengineering/analytics' import { registerProviders } from '@hcengineering/auth-providers' import { metricsAggregate, type BrandingMap, type MeasureContext } from '@hcengineering/core' import platform, { Severity, Status, addStringsLoader, setMetadata } from '@hcengineering/platform' -import serverClientPlugin from '@hcengineering/server-client' import serverToken, { decodeToken } from '@hcengineering/server-token' import toolPlugin from '@hcengineering/server-tool' import cors from '@koa/cors' @@ -81,7 +80,6 @@ export function serveAccount (measureCtx: MeasureContext, brandings: BrandingMap if (initScriptUrl !== undefined) { setMetadata(toolPlugin.metadata.InitScriptURL, initScriptUrl) } - setMetadata(serverClientPlugin.metadata.UserAgent, 'AccountService') const accountsDb = getAccountDB(dbUrl) diff --git a/server/account/package.json b/server/account/package.json index 8d42e62b23f..72b6058a8ab 100644 --- a/server/account/package.json +++ b/server/account/package.json @@ -56,7 +56,6 @@ "@hcengineering/analytics": "^0.6.0", "@hcengineering/server-storage": "^0.6.0", "@hcengineering/server-core": "^0.6.1", - "@hcengineering/server-pipeline": "^0.6.0", - "@hcengineering/model-all": "^0.6.0" + "@hcengineering/server-pipeline": "^0.6.0" } } diff --git a/server/account/src/operations.ts b/server/account/src/operations.ts index 773b2d82f5f..71a82adefc7 100644 --- a/server/account/src/operations.ts +++ b/server/account/src/operations.ts @@ -63,6 +63,7 @@ import type { ObjectId, OtpInfo, Query, + RegionInfo, UpgradeStatistic, Workspace, WorkspaceEvent, @@ -75,6 +76,7 @@ import { cleanEmail, EndpointKind, getEndpoint, + getRegions, hashWithSalt, isEmail, toAccountInfo, @@ -383,6 +385,17 @@ function decodeToken (ctx: MeasureContext, token: string): Token { } } +/** + * @public + */ +export async function getRegionInfo ( + ctx: MeasureContext, + db: AccountDB, + branding: Branding | null +): Promise { + return getRegions() +} + /** * @public */ @@ -911,10 +924,21 @@ async function generateWorkspaceRecord ( email: string, branding: Branding | null, workspaceName: string, - fixedWorkspace?: string + fixedWorkspace?: string, + region?: string ): Promise { type WorkspaceData = Omit const brandingKey = branding?.key ?? 'huly' + + const reg = getRegions().find((it) => it.region === (region ?? '')) + if (reg === undefined) { + throw new PlatformError( + new Status(Severity.ERROR, platform.status.InternalServerError, { + workspace: fixedWorkspace, + region: region ?? '' + }) + ) + } if (fixedWorkspace !== undefined) { const ws = await db.workspace.find({ workspaceUrl: fixedWorkspace }) @@ -932,6 +956,7 @@ async function generateWorkspaceRecord ( workspaceName, accounts: [], disabled: true, + region: region ?? '', mode: 'pending-creation', progress: 0, createdOn: Date.now(), @@ -969,6 +994,7 @@ async function generateWorkspaceRecord ( workspaceName, accounts: [], disabled: true, + region: region ?? '', mode: 'pending-creation', progress: 0, createdOn: Date.now(), @@ -1009,12 +1035,13 @@ export async function createWorkspace ( branding: Branding | null, email: string, workspaceName: string, - workspace?: string + workspace?: string, + region?: string ): Promise { // We need to search for duplicate workspaceUrl // Safe generate workspace record. return await createQueue.exec(async () => { - return await generateWorkspaceRecord(db, email, branding, workspaceName, workspace) + return await generateWorkspaceRecord(db, email, branding, workspaceName, workspace, region) }) } @@ -1276,7 +1303,8 @@ export async function createUserWorkspace ( db: AccountDB, branding: Branding | null, token: string, - workspaceName: string + workspaceName: string, + region?: string ): Promise { const { email } = decodeToken(ctx, token) @@ -1298,7 +1326,7 @@ export async function createUserWorkspace ( ) } } - const workspaceInfo = await createWorkspace(ctx, db, branding, email, workspaceName, undefined) + const workspaceInfo = await createWorkspace(ctx, db, branding, email, workspaceName, undefined, region) // Update last workspace time. await db.account.updateOne({ _id: userAccount._id }, { lastWorkspace: Date.now() }) @@ -2445,6 +2473,7 @@ export function getMethods (): Record { checkJoin: wrap(checkJoin), signUpJoin: wrap(signUpJoin), selectWorkspace: wrap(selectWorkspace), + getRegionInfo: wrap(getRegionInfo), getUserWorkspaces: wrap(getUserWorkspaces), getInviteLink: wrap(getInviteLink), getAccountInfo: wrap(getAccountInfo), diff --git a/server/account/src/types.ts b/server/account/src/types.ts index aa94e9e987d..272465917e0 100644 --- a/server/account/src/types.ts +++ b/server/account/src/types.ts @@ -105,6 +105,11 @@ export interface WorkspaceLoginInfo extends LoginInfo { progress?: number } +export interface RegionInfo { + region: string + name: string +} + /** * @public */ diff --git a/server/account/src/utils.ts b/server/account/src/utils.ts index f24ad042aca..e572034cb75 100644 --- a/server/account/src/utils.ts +++ b/server/account/src/utils.ts @@ -13,16 +13,16 @@ // limitations under the License. // import { groupByArray, MeasureContext } from '@hcengineering/core' -import { getMetadata } from '@hcengineering/platform' import { getMongoClient } from '@hcengineering/mongo' // TODO: get rid of this import later +import { getMetadata } from '@hcengineering/platform' import { getDBClient } from '@hcengineering/postgres' import { pbkdf2Sync } from 'crypto' import { MongoAccountDB } from './collections/mongo' import { PostgresAccountDB } from './collections/postgres' -import type { Account, AccountInfo, AccountDB, WorkspaceInfo } from './types' import { accountPlugin } from './plugin' +import type { Account, AccountDB, AccountInfo, RegionInfo, WorkspaceInfo } from './types' /** * @public @@ -99,7 +99,15 @@ export enum EndpointKind { External } -export const getEndpoint = (ctx: MeasureContext, workspaceInfo: WorkspaceInfo, kind: EndpointKind): string => { +const toTransactor = (line: string): { internalUrl: string, region: string, externalUrl: string } => { + const [internalUrl, externalUrl, region] = line + .split(';') + .map((it) => it.trim()) + .map((it) => (it.length === 0 ? undefined : it)) + return { internalUrl: internalUrl ?? '', region: region ?? '', externalUrl: externalUrl ?? internalUrl ?? '' } +} + +const getEndpoints = (): string[] => { const transactorsUrl = getMetadata(accountPlugin.metadata.Transactors) if (transactorsUrl === undefined) { throw new Error('Please provide transactor endpoint url') @@ -112,13 +120,22 @@ export const getEndpoint = (ctx: MeasureContext, workspaceInfo: WorkspaceInfo, k if (endpoints.length === 0) { throw new Error('Please provide transactor endpoint url') } + return endpoints +} - const toTransactor = (line: string): { internalUrl: string, region: string, externalUrl: string } => { - const [internalUrl, externalUrl, region] = line.split(';') - return { internalUrl, region: region ?? '', externalUrl: externalUrl ?? internalUrl } +export const getRegions = (): RegionInfo[] => { + if (process.env.REGION_INFO !== undefined) { + return process.env.REGION_INFO.split(';') + .map((it) => it.split('|')) + .map((it) => ({ region: it[0], name: it[1] })) } + return getEndpoints() + .map(toTransactor) + .map((it) => ({ region: it.region, name: it.region })) +} - const byRegions = groupByArray(endpoints.map(toTransactor), (it) => it.region) +export const getEndpoint = (ctx: MeasureContext, workspaceInfo: WorkspaceInfo, kind: EndpointKind): string => { + const byRegions = groupByArray(getEndpoints().map(toTransactor), (it) => it.region) let transactors = (byRegions.get(workspaceInfo.region ?? '') ?? []) .map((it) => (kind === EndpointKind.Internal ? it.internalUrl : it.externalUrl)) .flat() @@ -126,10 +143,11 @@ export const getEndpoint = (ctx: MeasureContext, workspaceInfo: WorkspaceInfo, k // This is really bad if (transactors.length === 0) { ctx.error('No transactors for the target region, will use default region', { group: workspaceInfo.region }) + + transactors = (byRegions.get('') ?? []) + .map((it) => (kind === EndpointKind.Internal ? it.internalUrl : it.externalUrl)) + .flat() } - transactors = (byRegions.get('') ?? []) - .map((it) => (kind === EndpointKind.Internal ? it.internalUrl : it.externalUrl)) - .flat() if (transactors.length === 0) { ctx.error('No transactors for the default region') diff --git a/server/backup/package.json b/server/backup/package.json index 16a0bc55bc7..808422a6220 100644 --- a/server/backup/package.json +++ b/server/backup/package.json @@ -51,6 +51,7 @@ "@hcengineering/server-tool": "^0.6.0", "@hcengineering/server-client": "^0.6.0", "@hcengineering/server-token": "^0.6.11", - "@hcengineering/server-core": "^0.6.1" + "@hcengineering/server-core": "^0.6.1", + "@hcengineering/server-indexer": "^0.6.0" } } diff --git a/server/backup/src/backup.ts b/server/backup/src/backup.ts index c0c07aa7ae0..e3ab794c66f 100644 --- a/server/backup/src/backup.ts +++ b/server/backup/src/backup.ts @@ -40,7 +40,8 @@ import core, { type Tx } from '@hcengineering/core' import { BlobClient, createClient } from '@hcengineering/server-client' -import { fullTextPushStagePrefix, type StorageAdapter } from '@hcengineering/server-core' +import { type StorageAdapter } from '@hcengineering/server-core' +import { fullTextPushStagePrefix } from '@hcengineering/server-indexer' import { generateToken } from '@hcengineering/server-token' import { connect } from '@hcengineering/server-tool' import { createWriteStream, existsSync, mkdirSync } from 'node:fs' diff --git a/server/core/package.json b/server/core/package.json index e0dd7304abe..9459e3c8264 100644 --- a/server/core/package.json +++ b/server/core/package.json @@ -38,11 +38,12 @@ "@hcengineering/core": "^0.6.32", "@hcengineering/platform": "^0.6.11", "@hcengineering/text": "^0.6.5", + "@hcengineering/rpc": "^0.6.5", "@hcengineering/analytics": "^0.6.0", + "@hcengineering/server-token": "^0.6.11", "@hcengineering/query": "^0.6.12", "fast-equals": "^5.0.1", "@hcengineering/storage": "^0.6.0", - "html-to-text": "^9.0.3", "uuid": "^8.3.2" } } diff --git a/server/core/src/configuration.ts b/server/core/src/configuration.ts index bc1c3b3fceb..77803c60e03 100644 --- a/server/core/src/configuration.ts +++ b/server/core/src/configuration.ts @@ -16,16 +16,7 @@ import { type MeasureContext } from '@hcengineering/core' import { type DbAdapterFactory } from './adapter' -import { type FullTextPipelineStage } from './indexer/types' -import { type StorageAdapter } from './storage' -import type { - ContentTextAdapter, - ContentTextAdapterFactory, - FullTextAdapter, - FullTextAdapterFactory, - ServiceAdapterConfig, - SessionFindAll -} from './types' +import type { ContentTextAdapterFactory, ServiceAdapterConfig } from './types' /** * @public @@ -44,15 +35,6 @@ export interface ContentTextAdapterConfiguration { url: string } -/** - * @public - */ -export type FullTextPipelineStageFactory = ( - adapter: FullTextAdapter, - storageFindAll: SessionFindAll, - storageAdapter: StorageAdapter, - contentAdapter: ContentTextAdapter -) => FullTextPipelineStage[] /** * @public */ @@ -61,11 +43,6 @@ export interface DbConfiguration { domains: Record defaultAdapter: string metrics: MeasureContext - fulltextAdapter: { - factory: FullTextAdapterFactory - url: string - stages: FullTextPipelineStageFactory - } contentAdapters: Record serviceAdapters: Record defaultContentAdapter: string diff --git a/server/core/src/index.ts b/server/core/src/index.ts index 567f7d13949..af391578711 100644 --- a/server/core/src/index.ts +++ b/server/core/src/index.ts @@ -19,18 +19,17 @@ export * from './adapter' export * from './base' export * from './benchmark' export * from './configuration' -export * from './fulltext' -export * from './indexer' export * from './limitter' export * from './mem' export * from './pipeline' export { default, serverCoreId } from './plugin' -export * from './server' export * from './storage' export * from './types' export * from './utils' +export * from './content' export * from './dbAdapterManager' export * from './domainHelper' +export * from './nullAdapter' export * from './service' export * from './triggers' diff --git a/server/server/src/nullAdapter.ts b/server/core/src/nullAdapter.ts similarity index 81% rename from server/server/src/nullAdapter.ts rename to server/core/src/nullAdapter.ts index 4537a65822b..74ff2a7abd4 100644 --- a/server/server/src/nullAdapter.ts +++ b/server/core/src/nullAdapter.ts @@ -13,8 +13,9 @@ // limitations under the License. // -import { Hierarchy, MeasureContext, ModelDb, WorkspaceId } from '@hcengineering/core' -import { DbAdapter, DummyDbAdapter } from '@hcengineering/server-core' +import { type Hierarchy, type MeasureContext, type ModelDb, type WorkspaceId } from '@hcengineering/core' +import type { DbAdapter } from './adapter' +import { DummyDbAdapter } from './mem' /** * @public diff --git a/server/core/src/server/index.ts b/server/core/src/server/index.ts deleted file mode 100644 index a7c77653e3c..00000000000 --- a/server/core/src/server/index.ts +++ /dev/null @@ -1,27 +0,0 @@ -// -// Copyright © 2020, 2021 Anticrm Platform Contributors. -// Copyright © 2021 Hardcore Engineering Inc. -// -// Licensed under the Eclipse Public License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. You may -// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// -// See the License for the specific language governing permissions and -// limitations under the License. -// - -import { DummyStorageAdapter, type StorageAdapter } from '../storage' - -/** - * @public - */ -export function createNullStorageFactory (): StorageAdapter { - return new DummyStorageAdapter() -} - -export { AggregatorStorageAdapter, buildStorage } from './aggregator' -export { QueryJoiner } from './utils' diff --git a/server/core/src/server/utils.ts b/server/core/src/server/utils.ts deleted file mode 100644 index 74a6a70ba6c..00000000000 --- a/server/core/src/server/utils.ts +++ /dev/null @@ -1,107 +0,0 @@ -// -// Copyright © 2024 Hardcore Engineering Inc. -// -// Licensed under the Eclipse Public License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. You may -// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// -// See the License for the specific language governing permissions and -// limitations under the License. -// - -import { - type Class, - type Doc, - type DocumentQuery, - type FindOptions, - type FindResult, - type MeasureContext, - type Ref -} from '@hcengineering/core' - -import { deepEqual } from 'fast-equals' -import type { Middleware } from '../types' - -interface Query { - _class: Ref> - query: DocumentQuery - result: FindResult | Promise> | undefined - options?: FindOptions - callbacks: number - max: number -} -/** - * @public - */ -export class QueryJoiner { - private readonly queries: Map>, Query[]> = new Map>, Query[]>() - - constructor (readonly _findAll: Middleware['findAll']) {} - - async findAll( - ctx: MeasureContext, - _class: Ref>, - query: DocumentQuery, - options?: FindOptions - ): Promise> { - // Will find a query or add + 1 to callbacks - const q = this.findQuery(_class, query, options) ?? this.createQuery(_class, query, options) - if (q.result === undefined) { - q.result = this._findAll(ctx, _class, query, options) - } - if (q.result instanceof Promise) { - q.result = await q.result - q.callbacks-- - } - this.removeFromQueue(q) - - return q.result as FindResult - } - - private findQuery( - _class: Ref>, - query: DocumentQuery, - options?: FindOptions - ): Query | undefined { - const queries = this.queries.get(_class) - if (queries === undefined) return - for (const q of queries) { - if (!deepEqual(query, q.query) || !deepEqual(options, q.options)) { - continue - } - q.callbacks++ - q.max++ - return q - } - } - - private createQuery(_class: Ref>, query: DocumentQuery, options?: FindOptions): Query { - const queries = this.queries.get(_class) ?? [] - const q: Query = { - _class, - query, - result: undefined, - options: options as FindOptions, - callbacks: 1, - max: 1 - } - - queries.push(q) - this.queries.set(_class, queries) - return q - } - - private removeFromQueue (q: Query): void { - if (q.callbacks === 0) { - const queries = this.queries.get(q._class) ?? [] - this.queries.set( - q._class, - queries.filter((it) => it !== q) - ) - } - } -} diff --git a/server/core/src/types.ts b/server/core/src/types.ts index b807324679c..e5fe6f50f7c 100644 --- a/server/core/src/types.ts +++ b/server/core/src/types.ts @@ -44,6 +44,8 @@ import { } from '@hcengineering/core' import type { Asset, Resource } from '@hcengineering/platform' import type { LiveQuery } from '@hcengineering/query' +import type { Request, Response } from '@hcengineering/rpc' +import type { Token } from '@hcengineering/server-token' import { type Readable } from 'stream' import type { DbAdapter, DomainHelper } from './adapter' import { type StorageAdapter } from './storage' @@ -518,3 +520,206 @@ export interface StorageConfiguration { default: string storages: StorageConfig[] } + +/** + * @public + */ +export interface SessionRequest { + id: string + params: any + start: number +} + +/** + * @public + */ +export interface StatisticsElement { + find: number + tx: number +} + +export interface ClientSessionCtx { + ctx: MeasureContext + sendResponse: (msg: any) => Promise + sendError: (msg: any, error: any) => Promise +} + +/** + * @public + */ +export interface Session { + createTime: number + getUser: () => string + pipeline: () => Pipeline + ping: (ctx: ClientSessionCtx) => Promise + findAll: ( + ctx: ClientSessionCtx, + _class: Ref>, + query: DocumentQuery, + options?: FindOptions + ) => Promise + findAllRaw: ( + ctx: MeasureContext, + _class: Ref>, + query: DocumentQuery, + options?: FindOptions + ) => Promise> + tx: (ctx: ClientSessionCtx, tx: Tx) => Promise + + // Session restore information + sessionId: string + sessionInstanceId?: string + workspaceClosed?: boolean + + requests: Map + + binaryMode: boolean + useCompression: boolean + total: StatisticsElement + current: StatisticsElement + mins5: StatisticsElement + + lastRequest: number + lastPing: number + + isUpgradeClient: () => boolean + + getMode: () => string + + broadcast: (ctx: MeasureContext, socket: ConnectionSocket, tx: Tx[]) => void +} + +/** + * @public + */ +export interface ConnectionSocket { + id: string + isClosed: boolean + close: () => void + send: (ctx: MeasureContext, msg: Response, binary: boolean, compression: boolean) => void + data: () => Record + + readRequest: (buffer: Buffer, binary: boolean) => Request + + checkState: () => boolean +} + +/** + * @public + */ +export let LOGGING_ENABLED = true + +/** + * @public + */ +export function disableLogging (): void { + LOGGING_ENABLED = false +} + +/** + * @public + */ +export interface Workspace { + context: MeasureContext + id: string + pipeline: Promise + sessions: Map + upgrade: boolean + + closing?: Promise + softShutdown: number + workspaceInitCompleted: boolean + + workspaceId: WorkspaceId + workspaceName: string + branding: Branding | null +} + +export interface AddSessionActive { + session: Session + context: MeasureContext + workspaceId: string +} +export type AddSessionResponse = AddSessionActive | { upgrade: true } | { error: any, terminate?: boolean } + +/** + * @public + */ +export interface SessionManager { + workspaces: Map + sessions: Map + + createSession: ( + token: Token, + pipeline: Pipeline, + workspaceId: WorkspaceIdWithUrl, + branding: Branding | null + ) => Session + + addSession: ( + ctx: MeasureContext, + ws: ConnectionSocket, + token: Token, + rawToken: string, + pipelineFactory: PipelineFactory, + sessionId: string | undefined, + accountsUrl: string + ) => Promise + + broadcastAll: (workspace: Workspace, tx: Tx[], targets?: string[]) => void + + close: (ctx: MeasureContext, ws: ConnectionSocket, workspaceId: string) => Promise + + closeAll: ( + wsId: string, + workspace: Workspace, + code: number, + reason: 'upgrade' | 'shutdown', + ignoreSocket?: ConnectionSocket + ) => Promise + + forceClose: (wsId: string, ignoreSocket?: ConnectionSocket) => Promise + + closeWorkspaces: (ctx: MeasureContext) => Promise + + scheduleMaintenance: (timeMinutes: number) => void + + profiling?: { + start: () => void + stop: () => Promise + } + + handleRequest: ( + requestCtx: MeasureContext, + service: S, + ws: ConnectionSocket, + request: Request, + workspace: string // wsId, toWorkspaceString() + ) => void +} + +/** + * @public + */ +export type HandleRequestFunction = ( + rctx: MeasureContext, + service: S, + ws: ConnectionSocket, + msg: Request, + workspaceId: string +) => void + +/** + * @public + */ + +export type ServerFactory = ( + sessions: SessionManager, + handleRequest: HandleRequestFunction, + ctx: MeasureContext, + pipelineFactory: PipelineFactory, + port: number, + enableCompression: boolean, + accountsUrl: string, + externalStorage: StorageAdapter +) => () => Promise diff --git a/models/server-translate/.eslintrc.js b/server/indexer/.eslintrc.js similarity index 85% rename from models/server-translate/.eslintrc.js rename to server/indexer/.eslintrc.js index c1cf82cba08..ce90fb9646f 100644 --- a/models/server-translate/.eslintrc.js +++ b/server/indexer/.eslintrc.js @@ -1,5 +1,5 @@ module.exports = { - extends: ['./node_modules/@hcengineering/platform-rig/profiles/model/eslint.config.json'], + extends: ['./node_modules/@hcengineering/platform-rig/profiles/node/eslint.config.json'], parserOptions: { tsconfigRootDir: __dirname, project: './tsconfig.json' diff --git a/models/server-translate/.npmignore b/server/indexer/.npmignore similarity index 100% rename from models/server-translate/.npmignore rename to server/indexer/.npmignore diff --git a/models/server-translate/CHANGELOG.json b/server/indexer/CHANGELOG.json similarity index 74% rename from models/server-translate/CHANGELOG.json rename to server/indexer/CHANGELOG.json index 200ec36b8d0..bab1ce33b4f 100644 --- a/models/server-translate/CHANGELOG.json +++ b/server/indexer/CHANGELOG.json @@ -1,9 +1,9 @@ { - "name": "@hcengineering/model-server-core", + "name": "@hcengineering/server-core", "entries": [ { "version": "0.6.0", - "tag": "@hcengineering/model-server-core_v0.6.0", + "tag": "@hcengineering/server-core_v0.6.0", "date": "Sun, 08 Aug 2021 10:14:57 GMT", "comments": { "dependency": [ diff --git a/models/server-translate/CHANGELOG.md b/server/indexer/CHANGELOG.md similarity index 76% rename from models/server-translate/CHANGELOG.md rename to server/indexer/CHANGELOG.md index 500a56bf3d8..7a1b917dea7 100644 --- a/models/server-translate/CHANGELOG.md +++ b/server/indexer/CHANGELOG.md @@ -1,4 +1,4 @@ -# Change Log - @hcengineering/model-server-core +# Change Log - @hcengineering/server-core This log was last generated on Sun, 08 Aug 2021 10:14:57 GMT and should not be manually modified. diff --git a/server/translate/config/rig.json b/server/indexer/config/rig.json similarity index 56% rename from server/translate/config/rig.json rename to server/indexer/config/rig.json index 0110930f55e..78cc5a17334 100644 --- a/server/translate/config/rig.json +++ b/server/indexer/config/rig.json @@ -1,4 +1,5 @@ { "$schema": "https://developer.microsoft.com/json-schemas/rig-package/rig.schema.json", - "rigPackageName": "@hcengineering/platform-rig" + "rigPackageName": "@hcengineering/platform-rig", + "rigProfile": "node" } diff --git a/server/indexer/jest.config.js b/server/indexer/jest.config.js new file mode 100644 index 00000000000..2cfd408b679 --- /dev/null +++ b/server/indexer/jest.config.js @@ -0,0 +1,7 @@ +module.exports = { + preset: 'ts-jest', + testEnvironment: 'node', + testMatch: ['**/?(*.)+(spec|test).[jt]s?(x)'], + roots: ["./src"], + coverageReporters: ["text-summary", "html"] +} diff --git a/server/translate/package.json b/server/indexer/package.json similarity index 67% rename from server/translate/package.json rename to server/indexer/package.json index 7bf3f2ea673..3fe5507df36 100644 --- a/server/translate/package.json +++ b/server/indexer/package.json @@ -1,23 +1,19 @@ { - "name": "@hcengineering/translate", + "name": "@hcengineering/server-indexer", "version": "0.6.0", "main": "lib/index.js", "svelte": "src/index.ts", "types": "types/index.d.ts", - "files": [ - "lib/**/*", - "types/**/*", - "tsconfig.json" - ], "author": "Anticrm Platform Contributors", + "template": "@hcengineering/node-package", "license": "EPL-2.0", "scripts": { "build": "compile", "build:watch": "compile", "format": "format src", - "test": "jest --passWithNoTests --silent", + "test": "jest --passWithNoTests --silent --forceExit", "_phase:build": "compile transpile src", - "_phase:test": "jest --passWithNoTests --silent", + "_phase:test": "jest --passWithNoTests --silent --forceExit", "_phase:format": "format src", "_phase:validate": "compile validate" }, @@ -35,12 +31,18 @@ "typescript": "^5.3.3", "jest": "^29.7.0", "ts-jest": "^29.1.1", - "@types/jest": "^29.5.5" + "@types/jest": "^29.5.5", + "@types/uuid": "^8.3.1" }, "dependencies": { "@hcengineering/core": "^0.6.32", "@hcengineering/platform": "^0.6.11", "@hcengineering/server-core": "^0.6.1", - "@hcengineering/server": "^0.6.4" + "@hcengineering/server-token": "^0.6.11", + "@hcengineering/text": "^0.6.5", + "@hcengineering/analytics": "^0.6.0", + "@hcengineering/query": "^0.6.12", + "fast-equals": "^5.0.1", + "@hcengineering/storage": "^0.6.0" } } diff --git a/server/core/src/fulltext.ts b/server/indexer/src/fulltext.ts similarity index 96% rename from server/core/src/fulltext.ts rename to server/indexer/src/fulltext.ts index 8a2436c14ed..1eaa7df8fc8 100644 --- a/server/core/src/fulltext.ts +++ b/server/indexer/src/fulltext.ts @@ -44,11 +44,10 @@ import core, { isIndexedAttribute, toFindResult } from '@hcengineering/core' +import type { FullTextAdapter, IndexedDoc, SessionFindAll, StorageAdapter, WithFind } from '@hcengineering/server-core' +import { getScoringConfig, mapSearchResultDoc } from './mapper' import { type FullTextIndexPipeline } from './indexer' import { createStateDoc } from './indexer/utils' -import { getScoringConfig, mapSearchResultDoc } from './mapper' -import { type StorageAdapter } from './storage' -import type { FullTextAdapter, IndexedDoc, SessionFindAll, WithFind } from './types' /** * @public @@ -58,7 +57,7 @@ export class FullTextIndex implements WithFind { constructor ( private readonly hierarchy: Hierarchy, - private readonly adapter: FullTextAdapter, + private readonly adapter: FullTextAdapter | undefined, private readonly storageFindAll: SessionFindAll, readonly storageAdapter: StorageAdapter | undefined, readonly workspace: WorkspaceId, @@ -205,7 +204,7 @@ export class FullTextIndex implements WithFind { let { docs, pass } = await this.indexer.search(classes, findQuery, fullTextLimit) if (docs.length === 0 && pass) { - docs = await this.adapter.search(classes, findQuery, fullTextLimit) + docs = (await this.adapter?.search(classes, findQuery, fullTextLimit)) ?? [] } const indexedDocMap = new Map, IndexedDoc>() @@ -269,10 +268,10 @@ export class FullTextIndex implements WithFind { } async searchFulltext (ctx: MeasureContext, query: SearchQuery, options: SearchOptions): Promise { - const resultRaw = await this.adapter.searchString(query, { + const resultRaw = (await this.adapter?.searchString(query, { ...options, scoring: getScoringConfig(this.hierarchy, query.classes ?? []) - }) + })) ?? { docs: [] } const result: SearchResult = { ...resultRaw, diff --git a/server/indexer/src/index.ts b/server/indexer/src/index.ts new file mode 100644 index 00000000000..7fcb9ec2d34 --- /dev/null +++ b/server/indexer/src/index.ts @@ -0,0 +1,46 @@ +// +// Copyright © 2024 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import type { + ContentTextAdapter, + FullTextAdapter, + FullTextAdapterFactory, + SessionFindAll +} from '@hcengineering/server-core' +import type { StorageAdapter } from '@hcengineering/storage' +import type { FullTextPipelineStage } from './indexer' + +export * from './fulltext' +export * from './indexer' +export * from './rekoni' +export * from './ydoc' + +/** + * @public + */ +export type FullTextPipelineStageFactory = ( + adapter: FullTextAdapter, + storageFindAll: SessionFindAll, + storageAdapter: StorageAdapter, + contentAdapter: ContentTextAdapter +) => FullTextPipelineStage[] + +export interface FulltextDBConfiguration { + fulltextAdapter?: { + factory: FullTextAdapterFactory + url: string + stages: FullTextPipelineStageFactory + } +} diff --git a/server/core/src/indexer/content.ts b/server/indexer/src/indexer/content.ts similarity index 94% rename from server/core/src/indexer/content.ts rename to server/indexer/src/indexer/content.ts index d3803ced013..f6e7b720c72 100644 --- a/server/core/src/indexer/content.ts +++ b/server/indexer/src/indexer/content.ts @@ -13,6 +13,7 @@ // limitations under the License. // +import { Analytics } from '@hcengineering/analytics' import core, { getFullTextIndexableAttributes, type Blob, @@ -25,9 +26,12 @@ import core, { type Ref, type WorkspaceId } from '@hcengineering/core' -import { type DbAdapter } from '../adapter' -import { type StorageAdapter } from '../storage' -import { type ContentTextAdapter, type IndexedDoc } from '../types' +import { + type ContentTextAdapter, + type DbAdapter, + type IndexedDoc, + type StorageAdapter +} from '@hcengineering/server-core' import { contentStageId, fieldStateId, @@ -36,8 +40,6 @@ import { type FullTextPipelineStage } from './types' import { docKey, docUpdKey } from './utils' -import { Analytics } from '@hcengineering/analytics' - /** * @public */ @@ -114,10 +116,8 @@ export class ContentRetrievalStage implements FullTextPipelineStage { const readable = await this.storageAdapter?.get(this.metrics, this.workspace, ref) if (readable !== undefined) { - let textContent = await this.metrics.with( - 'fetch', - {}, - async () => await this.contentAdapter.content(ref, contentType, readable) + let textContent = await this.metrics.with('fetch', {}, () => + this.contentAdapter.content(ref, contentType, readable) ) readable?.destroy() diff --git a/server/core/src/indexer/field.ts b/server/indexer/src/indexer/field.ts similarity index 98% rename from server/core/src/indexer/field.ts rename to server/indexer/src/indexer/field.ts index e32f8bd3ce2..0e4ce1373b3 100644 --- a/server/core/src/indexer/field.ts +++ b/server/indexer/src/indexer/field.ts @@ -26,9 +26,8 @@ import core, { type MeasureContext, type Ref } from '@hcengineering/core' +import { type DbAdapter, type IndexedDoc, type SessionFindAll } from '@hcengineering/server-core' import { deepEqual } from 'fast-equals' -import { type DbAdapter } from '../adapter' -import { type IndexedDoc, type SessionFindAll } from '../types' import { contentStageId, fieldStateId, @@ -37,7 +36,6 @@ import { type FullTextPipelineStage } from './types' import { collectPropagate, docKey, docUpdKey, getContent, getCustomAttrKeys, isFullTextAttribute } from './utils' - /** * @public */ diff --git a/server/core/src/indexer/fulltext.ts b/server/indexer/src/indexer/fulltext.ts similarity index 76% rename from server/core/src/indexer/fulltext.ts rename to server/indexer/src/indexer/fulltext.ts index 071dece2bce..b49f8765e9f 100644 --- a/server/core/src/indexer/fulltext.ts +++ b/server/indexer/src/indexer/fulltext.ts @@ -31,37 +31,37 @@ import core, { WorkspaceEvent } from '@hcengineering/core' import { PlatformError, unknownError } from '@hcengineering/platform' -import { BaseMiddleware } from '../base' -import type { DbConfiguration } from '../configuration' -import { createContentAdapter } from '../content' -import { FullTextIndex } from '../fulltext' -import { FullTextIndexPipeline } from '../indexer' import type { + DbConfiguration, FullTextAdapter, Middleware, MiddlewareCreator, PipelineContext, SessionFindAll, TxMiddlewareResult -} from '../types' +} from '@hcengineering/server-core' +import { BaseMiddleware, createContentAdapter } from '@hcengineering/server-core' +import type { FulltextDBConfiguration } from '..' +import { FullTextIndex } from '../fulltext' +import { FullTextIndexPipeline } from '../indexer' /** * @public */ export class FullTextMiddleware extends BaseMiddleware implements Middleware { private fulltext!: FullTextIndex - private fulltextAdapter!: FullTextAdapter + private fulltextAdapter?: FullTextAdapter constructor ( context: PipelineContext, next: Middleware | undefined, - readonly conf: DbConfiguration, + readonly conf: DbConfiguration & FulltextDBConfiguration, readonly upgrade?: boolean ) { super(context, next) } - static create (conf: DbConfiguration, upgrade: boolean): MiddlewareCreator { + static create (conf: DbConfiguration & FulltextDBConfiguration, upgrade: boolean): MiddlewareCreator { return async (ctx, context, next): Promise => { const middleware = new FullTextMiddleware(context, next, conf, upgrade) await middleware.init(ctx) @@ -76,24 +76,23 @@ export class FullTextMiddleware extends BaseMiddleware implements Middleware { if (this.context.storageAdapter == null) { throw new PlatformError(unknownError('Storage adapter should be specified')) } - const fullTextCtx = this.conf.metrics.newChild('🗒️ fulltext', {}) - this.fulltextAdapter = await fullTextCtx.with( - 'init', - {}, - async (ctx) => - await this.conf.fulltextAdapter.factory(this.conf.fulltextAdapter.url, this.context.workspace, fullTextCtx) - ) + const fullTextCtx = ctx.newChild('🗒️ fulltext', {}) + if (this.conf.fulltextAdapter !== undefined) { + const fta = this.conf.fulltextAdapter + this.fulltextAdapter = await fullTextCtx.with( + 'init', + {}, + async (ctx) => await fta.factory(fta.url, this.context.workspace, fullTextCtx) + ) + } - const contentAdapter = await ctx.with( - 'create content adapter', - {}, - async (ctx) => - await createContentAdapter( - this.conf.contentAdapters, - this.conf.defaultContentAdapter, - this.context.workspace, - fullTextCtx.newChild('content', {}) - ) + const contentAdapter = await ctx.with('create content adapter', {}, (ctx) => + createContentAdapter( + this.conf.contentAdapters, + this.conf.defaultContentAdapter, + this.context.workspace, + fullTextCtx.newChild('content', {}) + ) ) const defaultAdapter = this.context.adapterManager.getDefaultAdapter() const findAll: SessionFindAll = (ctx, _class, query, options) => { @@ -101,12 +100,10 @@ export class FullTextMiddleware extends BaseMiddleware implements Middleware { } // TODO: Extract storage adapter to context - const stages = this.conf.fulltextAdapter.stages( - this.fulltextAdapter, - findAll, - this.context.storageAdapter, - contentAdapter - ) + const stages = + this.conf.fulltextAdapter !== undefined && this.fulltextAdapter !== undefined + ? this.conf.fulltextAdapter.stages(this.fulltextAdapter, findAll, this.context.storageAdapter, contentAdapter) + : [] const indexer = new FullTextIndexPipeline( defaultAdapter, @@ -170,6 +167,6 @@ export class FullTextMiddleware extends BaseMiddleware implements Middleware { async close (): Promise { await this.fulltext.close() - await this.fulltextAdapter.close() + await this.fulltextAdapter?.close() } } diff --git a/server/core/src/indexer/fulltextPush.ts b/server/indexer/src/indexer/fulltextPush.ts similarity index 99% rename from server/core/src/indexer/fulltextPush.ts rename to server/indexer/src/indexer/fulltextPush.ts index 5678c22172b..26bf4f9c18a 100644 --- a/server/core/src/indexer/fulltextPush.ts +++ b/server/indexer/src/indexer/fulltextPush.ts @@ -33,10 +33,9 @@ import core, { toIdMap, type WorkspaceId } from '@hcengineering/core' -import { jsonToText, markupToJSON } from '@hcengineering/text' -import { type DbAdapter } from '../adapter' +import { type DbAdapter, type FullTextAdapter, type IndexedDoc, type SessionFindAll } from '@hcengineering/server-core' import { updateDocWithPresenter } from '../mapper' -import { type FullTextAdapter, type IndexedDoc, type SessionFindAll } from '../types' +import { jsonToText, markupToJSON } from '@hcengineering/text' import { contentStageId, type DocUpdateHandler, diff --git a/server/core/src/indexer/index.ts b/server/indexer/src/indexer/index.ts similarity index 100% rename from server/core/src/indexer/index.ts rename to server/indexer/src/indexer/index.ts diff --git a/server/core/src/indexer/indexer.ts b/server/indexer/src/indexer/indexer.ts similarity index 99% rename from server/core/src/indexer/indexer.ts rename to server/indexer/src/indexer/indexer.ts index 25a29839d8b..c27d7e1bafb 100644 --- a/server/core/src/indexer/indexer.ts +++ b/server/indexer/src/indexer/indexer.ts @@ -33,10 +33,8 @@ import core, { setObjectValue, systemAccountEmail } from '@hcengineering/core' -import { type DbAdapter } from '../adapter' -import { RateLimiter } from '../limitter' -import type { IndexedDoc } from '../types' -import { SessionDataImpl } from '../utils' +import type { DbAdapter, IndexedDoc } from '@hcengineering/server-core' +import { RateLimiter, SessionDataImpl } from '@hcengineering/server-core' import { type FullTextPipeline, type FullTextPipelineStage } from './types' export * from './content' diff --git a/server/core/src/indexer/summary.ts b/server/indexer/src/indexer/summary.ts similarity index 98% rename from server/core/src/indexer/summary.ts rename to server/indexer/src/indexer/summary.ts index b71fbc3546d..6a6212317dd 100644 --- a/server/core/src/indexer/summary.ts +++ b/server/indexer/src/indexer/summary.ts @@ -29,9 +29,8 @@ import core, { type Ref } from '@hcengineering/core' import { translate } from '@hcengineering/platform' +import { type DbAdapter, type IndexedDoc, type SessionFindAll } from '@hcengineering/server-core' import { jsonToText, markupToJSON } from '@hcengineering/text' -import { type DbAdapter } from '../adapter' -import { type IndexedDoc, type SessionFindAll } from '../types' import { contentStageId, type DocUpdateHandler, diff --git a/server/core/src/indexer/types.ts b/server/indexer/src/indexer/types.ts similarity index 96% rename from server/core/src/indexer/types.ts rename to server/indexer/src/indexer/types.ts index 8a798d27cb8..e55b2dc4f15 100644 --- a/server/core/src/indexer/types.ts +++ b/server/indexer/src/indexer/types.ts @@ -24,8 +24,7 @@ import { type ModelDb, type Ref } from '@hcengineering/core' -import type { DbAdapter } from '../adapter' -import type { IndexedDoc } from '../types' +import type { DbAdapter, IndexedDoc } from '@hcengineering/server-core' /** * @public diff --git a/server/core/src/indexer/utils.ts b/server/indexer/src/indexer/utils.ts similarity index 98% rename from server/core/src/indexer/utils.ts rename to server/indexer/src/indexer/utils.ts index c315787084c..8ca1d1ab38e 100644 --- a/server/core/src/indexer/utils.ts +++ b/server/indexer/src/indexer/utils.ts @@ -26,7 +26,7 @@ import core, { type Ref, type Space } from '@hcengineering/core' -import plugin from '../plugin' +import plugin from '@hcengineering/server-core' import { type FullTextPipeline } from './types' export { docKey, docUpdKey, extractDocKey, isFullTextAttribute } from '@hcengineering/core' diff --git a/server/core/src/mapper.ts b/server/indexer/src/mapper.ts similarity index 99% rename from server/core/src/mapper.ts rename to server/indexer/src/mapper.ts index 271a06a2a0d..12b4070428f 100644 --- a/server/core/src/mapper.ts +++ b/server/indexer/src/mapper.ts @@ -11,14 +11,13 @@ import { } from '@hcengineering/core' import { getResource, type Resource } from '@hcengineering/platform' -import plugin from './plugin' -import { +import plugin, { type ClassSearchConfigProps, type IndexedDoc, type SearchPresenter, type SearchPresenterFunc, type SearchScoring -} from './types' +} from '@hcengineering/server-core' interface IndexedReader { get: (attribute: string) => any diff --git a/server/server/src/rekoni.ts b/server/indexer/src/rekoni.ts similarity index 88% rename from server/server/src/rekoni.ts rename to server/indexer/src/rekoni.ts index 8b354bf0932..495500c4b87 100644 --- a/server/server/src/rekoni.ts +++ b/server/indexer/src/rekoni.ts @@ -1,5 +1,5 @@ -import { MeasureContext, WorkspaceId } from '@hcengineering/core' -import { ContentTextAdapter } from '@hcengineering/server-core' +import { type MeasureContext, type WorkspaceId } from '@hcengineering/core' +import { type ContentTextAdapter } from '@hcengineering/server-core' import { generateToken } from '@hcengineering/server-token' /** @@ -15,7 +15,7 @@ export async function createRekoniAdapter ( content: async (name: string, type: string, doc): Promise => { try { // Node doesn't support Readable with fetch. - const chunks: Buffer[] = [] + const chunks: any[] = [] let len = 0 await new Promise((resolve, reject) => { doc.on('data', (chunk) => { @@ -33,7 +33,7 @@ export async function createRekoniAdapter ( }) }) - const body = Buffer.concat(chunks) + const body: Buffer = Buffer.concat(chunks) const r = await ( await fetch(`${url}/toText?name=${encodeURIComponent(name)}&type=${encodeURIComponent(type)}`, { method: 'POST', diff --git a/server/server/src/ydoc.ts b/server/indexer/src/ydoc.ts similarity index 80% rename from server/server/src/ydoc.ts rename to server/indexer/src/ydoc.ts index 788068d2760..4b6fbaa9d4b 100644 --- a/server/server/src/ydoc.ts +++ b/server/indexer/src/ydoc.ts @@ -1,7 +1,8 @@ -import { MeasureContext, WorkspaceId } from '@hcengineering/core' -import { ContentTextAdapter } from '@hcengineering/server-core' +import { type MeasureContext, type WorkspaceId } from '@hcengineering/core' +import { type ContentTextAdapter } from '@hcengineering/server-core' import { pmNodeToText, yDocContentToNodes } from '@hcengineering/text' -import { Readable } from 'stream' +import { Readable } from 'node:stream' +import { Buffer } from 'node:buffer' /** * @public @@ -13,12 +14,12 @@ export async function createYDocAdapter ( ): Promise { return { content: async (_name: string, _type: string, data: Readable | Buffer | string): Promise => { - const chunks: Buffer[] = [] + const chunks: any[] = [] if (data instanceof Readable) { await new Promise((resolve) => { data.on('readable', () => { - let chunk + let chunk: any while ((chunk = data.read()) !== null) { const b = chunk as Buffer chunks.push(b) diff --git a/models/server-translate/tsconfig.json b/server/indexer/tsconfig.json similarity index 90% rename from models/server-translate/tsconfig.json rename to server/indexer/tsconfig.json index 9765b6086db..f017cc597c2 100644 --- a/models/server-translate/tsconfig.json +++ b/server/indexer/tsconfig.json @@ -1,5 +1,5 @@ { - "extends": "./node_modules/@hcengineering/platform-rig/profiles/model/tsconfig.json", + "extends": "./node_modules/@hcengineering/platform-rig/profiles/node/tsconfig.json", "compilerOptions": { "rootDir": "./src", diff --git a/server/middleware/src/queryJoin.ts b/server/middleware/src/queryJoin.ts index be84b4e0006..6e157ad20e8 100644 --- a/server/middleware/src/queryJoin.ts +++ b/server/middleware/src/queryJoin.ts @@ -22,7 +22,88 @@ import { type MeasureContext, Ref } from '@hcengineering/core' -import { BaseMiddleware, Middleware, type PipelineContext, QueryJoiner } from '@hcengineering/server-core' +import { BaseMiddleware, Middleware, type PipelineContext } from '@hcengineering/server-core' +import { deepEqual } from 'fast-equals' + +interface Query { + _class: Ref> + query: DocumentQuery + result: FindResult | Promise> | undefined + options?: FindOptions + callbacks: number + max: number +} +/** + * @public + */ +export class QueryJoiner { + private readonly queries: Map>, Query[]> = new Map>, Query[]>() + + constructor (readonly _findAll: Middleware['findAll']) {} + + async findAll( + ctx: MeasureContext, + _class: Ref>, + query: DocumentQuery, + options?: FindOptions + ): Promise> { + // Will find a query or add + 1 to callbacks + const q = this.findQuery(_class, query, options) ?? this.createQuery(_class, query, options) + if (q.result === undefined) { + q.result = this._findAll(ctx, _class, query, options) + } + if (q.result instanceof Promise) { + q.result = await q.result + q.callbacks-- + } + this.removeFromQueue(q) + + return q.result as FindResult + } + + private findQuery( + _class: Ref>, + query: DocumentQuery, + options?: FindOptions + ): Query | undefined { + const queries = this.queries.get(_class) + if (queries === undefined) return + for (const q of queries) { + if (!deepEqual(query, q.query) || !deepEqual(options, q.options)) { + continue + } + q.callbacks++ + q.max++ + return q + } + } + + private createQuery(_class: Ref>, query: DocumentQuery, options?: FindOptions): Query { + const queries = this.queries.get(_class) ?? [] + const q: Query = { + _class, + query, + result: undefined, + options: options as FindOptions, + callbacks: 1, + max: 1 + } + + queries.push(q) + this.queries.set(_class, queries) + return q + } + + private removeFromQueue (q: Query): void { + if (q.callbacks === 0) { + const queries = this.queries.get(q._class) ?? [] + this.queries.set( + q._class, + queries.filter((it) => it !== q) + ) + } + } +} /** * @public diff --git a/server/minio/src/index.ts b/server/minio/src/index.ts index d0c2888bbbc..ba5fdb97863 100644 --- a/server/minio/src/index.ts +++ b/server/minio/src/index.ts @@ -100,6 +100,9 @@ export class MinioService implements StorageAdapter { async listBuckets (ctx: MeasureContext): Promise { if (this.opt.rootBucket !== undefined) { const info = new Map() + if (!(await this.client.bucketExists(this.opt.rootBucket))) { + return [] + } const stream = this.client.listObjects(this.opt.rootBucket, '', false) await new Promise((resolve, reject) => { stream.on('end', () => { diff --git a/server/server-pipeline/package.json b/server/server-pipeline/package.json index 2fb6976ce17..a74e3d3b0c7 100644 --- a/server/server-pipeline/package.json +++ b/server/server-pipeline/package.json @@ -40,6 +40,7 @@ "@hcengineering/core": "^0.6.32", "@hcengineering/platform": "^0.6.11", "@hcengineering/server-core": "^0.6.1", + "@hcengineering/server-indexer": "^0.6.0", "@hcengineering/server-attachment": "^0.6.1", "@hcengineering/server-attachment-resources": "^0.6.0", "@hcengineering/server-collaboration": "^0.6.0", @@ -99,7 +100,6 @@ "@hcengineering/server-training-resources": "^0.1.0", "@hcengineering/middleware": "^0.6.0", "@hcengineering/minio": "^0.6.0", - "@hcengineering/translate": "^0.6.0", "@hcengineering/login-assets": "^0.6.0", "@hcengineering/onboard-assets": "^0.6.0", "@hcengineering/view-assets": "^0.6.11", diff --git a/server/server-pipeline/src/indexing.ts b/server/server-pipeline/src/indexing.ts index 2b884ab170b..784adfb12c6 100644 --- a/server/server-pipeline/src/indexing.ts +++ b/server/server-pipeline/src/indexing.ts @@ -2,18 +2,21 @@ import { type Branding, type MeasureContext, type WorkspaceId } from '@hcengineering/core' import { CollaborativeContentRetrievalStage } from '@hcengineering/server-collaboration' import { - ContentRetrievalStage, - FullSummaryStage, - FullTextPushStage, - IndexedFieldStage, - globalIndexer, type ContentTextAdapter, type FullTextAdapter, - type FullTextPipelineStage, type SessionFindAll, type StorageAdapter } from '@hcengineering/server-core' +import { + ContentRetrievalStage, + FullSummaryStage, + FullTextPushStage, + IndexedFieldStage, + globalIndexer, + type FullTextPipelineStage +} from '@hcengineering/server-indexer' + export function createIndexStages ( fullText: MeasureContext, workspace: WorkspaceId, @@ -46,15 +49,6 @@ export function createIndexStages ( contentAdapter ) ) - - // // Add any => english language translation - // const retranslateStage = new LibRetranslateStage(fullText.newChild('retranslate', {}), workspace) - // retranslateStage.clearExcept = stages.map(it => it.stageId) - // for (const st of stages) { - // // Clear retranslation on content change. - // st.updateFields.push((doc, upd) => retranslateStage.update(doc, upd)) - // } - // stages.push(retranslateStage) // Summary stage const summaryStage = new FullSummaryStage(storageFindAll) diff --git a/server/server-pipeline/src/pipeline.ts b/server/server-pipeline/src/pipeline.ts index a255451f033..4865ed6587e 100644 --- a/server/server-pipeline/src/pipeline.ts +++ b/server/server-pipeline/src/pipeline.ts @@ -39,21 +39,13 @@ import { } from '@hcengineering/middleware' import { createMongoAdapter, createMongoTxAdapter } from '@hcengineering/mongo' import { createPostgresAdapter, createPostgresTxAdapter } from '@hcengineering/postgres' -import { - buildStorageFromConfig, - createNullAdapter, - createRekoniAdapter, - createStorageDataAdapter, - createYDocAdapter, - storageConfigFromEnv -} from '@hcengineering/server' import { createBenchmarkAdapter, createInMemoryAdapter, + createNullAdapter, createPipeline, DummyDbAdapter, DummyFullTextAdapter, - FullTextMiddleware, type DbAdapterFactory, type DbConfiguration, type Middleware, @@ -64,6 +56,13 @@ import { type StorageAdapter, type StorageConfiguration } from '@hcengineering/server-core' +import { + createRekoniAdapter, + createYDocAdapter, + FullTextMiddleware, + type FulltextDBConfiguration +} from '@hcengineering/server-indexer' +import { buildStorageFromConfig, createStorageDataAdapter, storageConfigFromEnv } from '@hcengineering/server-storage' import { createIndexStages } from './indexing' /** @@ -112,7 +111,7 @@ export function createServerPipeline ( externalStorage: StorageAdapter }, - extensions?: Partial + extensions?: Partial & Partial ): PipelineFactory { return (ctx, workspace, upgrade, broadcast, branding) => { const metricsCtx = opt.usePassedCtx === true ? ctx : metrics @@ -306,12 +305,12 @@ export function getConfig ( externalStorage: StorageAdapter }, - extensions?: Partial + extensions?: Partial ): DbConfiguration { const metricsCtx = opt.usePassedCtx === true ? ctx : metrics const wsMetrics = metricsCtx.newChild('🧲 session', {}) const [dbUrl, mongoUrl] = dbUrls.split(';') - const conf: DbConfiguration = { + const conf: DbConfiguration & FulltextDBConfiguration = { domains: { [DOMAIN_TX]: 'Tx', [DOMAIN_TRANSIENT]: 'InMemory', diff --git a/server/server-storage/package.json b/server/server-storage/package.json index ca69b912464..6a08bb122a1 100644 --- a/server/server-storage/package.json +++ b/server/server-storage/package.json @@ -47,6 +47,8 @@ "@hcengineering/minio": "^0.6.0", "@hcengineering/s3": "^0.6.0", "@hcengineering/datalake": "^0.6.0", + "@hcengineering/storage": "^0.6.0", + "@hcengineering/analytics": "^0.6.0", "@hcengineering/server-token": "^0.6.11" } } diff --git a/server/core/src/server/aggregator.ts b/server/server-storage/src/aggregator.ts similarity index 98% rename from server/core/src/server/aggregator.ts rename to server/server-storage/src/aggregator.ts index 61a4e38dacb..b17b63a43c8 100644 --- a/server/core/src/server/aggregator.ts +++ b/server/server-storage/src/aggregator.ts @@ -22,9 +22,11 @@ import { } from '@hcengineering/storage' import { Analytics } from '@hcengineering/analytics' -import { type RawDBAdapter } from '../adapter' -import serverCore from '../plugin' -import { type StorageConfig, type StorageConfiguration } from '../types' +import serverCore, { + type RawDBAdapter, + type StorageConfig, + type StorageConfiguration +} from '@hcengineering/server-core' class NoSuchKeyError extends Error { code: string diff --git a/server/server-storage/src/index.ts b/server/server-storage/src/index.ts index d4e01475fcf..823da0a904d 100644 --- a/server/server-storage/src/index.ts +++ b/server/server-storage/src/index.ts @@ -14,5 +14,6 @@ // limitations under the License. // +export * from './aggregator' export * from './blobStorage' export * from './starter' diff --git a/server/server-storage/src/starter.ts b/server/server-storage/src/starter.ts index 97c769cac3b..a01e70a886f 100644 --- a/server/server-storage/src/starter.ts +++ b/server/server-storage/src/starter.ts @@ -2,13 +2,8 @@ import { DatalakeService, type DatalakeConfig } from '@hcengineering/datalake' import { MinioConfig, MinioService, addMinioFallback } from '@hcengineering/minio' import { createRawMongoDBAdapter } from '@hcengineering/mongo' import { S3Service, type S3Config } from '@hcengineering/s3' -import { - AggregatorStorageAdapter, - StorageAdapter, - StorageConfiguration, - buildStorage, - type StorageConfig -} from '@hcengineering/server-core' +import { StorageAdapter, StorageConfiguration, type StorageConfig } from '@hcengineering/server-core' +import { AggregatorStorageAdapter, buildStorage } from './aggregator' /* diff --git a/server/core/src/__tests__/aggregator.spec.ts b/server/server-storage/src/tests/aggregator.spec.ts similarity index 96% rename from server/core/src/__tests__/aggregator.spec.ts rename to server/server-storage/src/tests/aggregator.spec.ts index e85f46a584f..12ccf3df988 100644 --- a/server/core/src/__tests__/aggregator.spec.ts +++ b/server/server-storage/src/tests/aggregator.spec.ts @@ -1,6 +1,6 @@ import { MeasureMetricsContext, type MeasureContext, type WorkspaceId } from '@hcengineering/core' import type { StorageAdapter } from '@hcengineering/storage' -import { AggregatorStorageAdapter } from '../server' +import { AggregatorStorageAdapter } from '../aggregator' import { MemRawDBAdapter, MemStorageAdapter } from './memAdapters' describe('aggregator tests', () => { diff --git a/server/core/src/__tests__/memAdapters.ts b/server/server-storage/src/tests/memAdapters.ts similarity index 98% rename from server/core/src/__tests__/memAdapters.ts rename to server/server-storage/src/tests/memAdapters.ts index 02060086e2d..cd56f602451 100644 --- a/server/core/src/__tests__/memAdapters.ts +++ b/server/server-storage/src/tests/memAdapters.ts @@ -16,9 +16,9 @@ import core, { type WorkspaceId } from '@hcengineering/core' import { genMinModel } from '@hcengineering/core/src/__tests__/minmodel' +import type { RawDBAdapter, RawDBAdapterStream } from '@hcengineering/server-core' import type { BlobStorageIterator, BucketInfo, StorageAdapter, UploadedObjectInfo } from '@hcengineering/storage' import { Readable } from 'stream' -import type { RawDBAdapter, RawDBAdapterStream } from '../adapter' export class MemStorageAdapter implements StorageAdapter { files = new Map() diff --git a/server/server/package.json b/server/server/package.json index b7adc78232e..c6154efc9e7 100644 --- a/server/server/package.json +++ b/server/server/package.json @@ -30,7 +30,6 @@ "eslint-plugin-promise": "^6.1.1", "eslint-plugin-n": "^15.4.0", "eslint": "^8.54.0", - "ts-node": "^10.8.0", "@typescript-eslint/parser": "^6.11.0", "eslint-config-standard-with-typescript": "^40.0.0", "prettier": "^3.1.0", @@ -45,9 +44,6 @@ "@hcengineering/server-core": "^0.6.1", "@hcengineering/server-token": "^0.6.11", "@hcengineering/rpc": "^0.6.5", - "@hcengineering/middleware": "^0.6.0", - "@hcengineering/text": "^0.6.5", - "@hcengineering/server-storage": "^0.6.0", "@hcengineering/analytics": "^0.6.0" } } diff --git a/server/server/src/client.ts b/server/server/src/client.ts index 1001fa2e3a4..3c6de5cf213 100644 --- a/server/server/src/client.ts +++ b/server/server/src/client.ts @@ -35,15 +35,18 @@ import core, { type WorkspaceIdWithUrl } from '@hcengineering/core' import { PlatformError, unknownError } from '@hcengineering/platform' -import { BackupClientOps, SessionDataImpl, createBroadcastEvent, type Pipeline } from '@hcengineering/server-core' -import { type Token } from '@hcengineering/server-token' import { + BackupClientOps, + SessionDataImpl, + createBroadcastEvent, type ClientSessionCtx, type ConnectionSocket, + type Pipeline, type Session, type SessionRequest, type StatisticsElement -} from './types' +} from '@hcengineering/server-core' +import { type Token } from '@hcengineering/server-token' import { handleSend } from './utils' /** diff --git a/server/server/src/index.ts b/server/server/src/index.ts index fdd13aaaf6e..2fe0c8e1473 100644 --- a/server/server/src/index.ts +++ b/server/server/src/index.ts @@ -14,15 +14,9 @@ // limitations under the License. // -export { buildStorageFromConfig, createStorageDataAdapter, storageConfigFromEnv } from '@hcengineering/server-storage' +export * from './blobs' export * from './client' -export * from './metrics' -export * from './nullAdapter' -export * from './rekoni' export * from './sessionManager' export * from './starter' -export * from './types' -export * from './utils' -export * from './ydoc' export * from './stats' -export * from './blobs' +export * from './utils' diff --git a/server/server/src/sessionManager.ts b/server/server/src/sessionManager.ts index 67787b78a22..5c2b350126a 100644 --- a/server/server/src/sessionManager.ts +++ b/server/server/src/sessionManager.ts @@ -34,18 +34,20 @@ import core, { } from '@hcengineering/core' import { unknownError, type Status } from '@hcengineering/platform' import { type HelloRequest, type HelloResponse, type Request, type Response } from '@hcengineering/rpc' -import type { Pipeline, PipelineFactory, StorageAdapter } from '@hcengineering/server-core' -import { type Token } from '@hcengineering/server-token' - import { LOGGING_ENABLED, + Pipeline, + PipelineFactory, + ServerFactory, + SessionManager, + StorageAdapter, type ClientSessionCtx, type ConnectionSocket, - type ServerFactory, type Session, - type SessionManager, type Workspace -} from './types' +} from '@hcengineering/server-core' +import { type Token } from '@hcengineering/server-token' + import { sendResponse } from './utils' const ticksPerSecond = 20 @@ -1013,6 +1015,25 @@ class TSessionManager implements SessionManager { } } } + +export function createSessionManager ( + ctx: MeasureContext, + sessionFactory: ( + token: Token, + pipeline: Pipeline, + workspaceId: WorkspaceIdWithUrl, + branding: Branding | null + ) => Session, + brandingMap: BrandingMap, + timeouts: Timeouts, + profiling?: { + start: () => void + stop: () => Promise + } +): SessionManager { + return new TSessionManager(ctx, sessionFactory, timeouts, brandingMap ?? null, profiling) +} + /** * @public */ @@ -1038,14 +1059,14 @@ export function startSessionManager ( } } & Partial ): () => Promise { - const sessions = new TSessionManager( + const sessions = createSessionManager( ctx, opt.sessionFactory, + opt.brandingMap, { pingTimeout: opt.pingTimeout ?? 10000, reconnectTimeout: 500 }, - opt.brandingMap, opt.profiling ) return opt.serverFactory( diff --git a/server/server/src/stats.ts b/server/server/src/stats.ts index d0474ab8fac..b77d2c506c5 100644 --- a/server/server/src/stats.ts +++ b/server/server/src/stats.ts @@ -6,8 +6,8 @@ import { type MetricsData, toWorkspaceString } from '@hcengineering/core' -import os from 'os' -import { type SessionManager } from './types' +import { type SessionManager } from '@hcengineering/server-core' +import os from 'node:os' /** * @public diff --git a/server/server/src/types.ts b/server/server/src/types.ts deleted file mode 100644 index 817f7dad402..00000000000 --- a/server/server/src/types.ts +++ /dev/null @@ -1,210 +0,0 @@ -import { - type Branding, - type Class, - type Doc, - type DocumentQuery, - type FindOptions, - type FindResult, - type MeasureContext, - type Ref, - type Tx, - type WorkspaceId, - type WorkspaceIdWithUrl -} from '@hcengineering/core' -import { type Request, type Response } from '@hcengineering/rpc' -import { type Pipeline, type PipelineFactory, type StorageAdapter } from '@hcengineering/server-core' -import { type Token } from '@hcengineering/server-token' - -/** - * @public - */ -export interface SessionRequest { - id: string - params: any - start: number -} - -/** - * @public - */ -export interface StatisticsElement { - find: number - tx: number -} - -export interface ClientSessionCtx { - ctx: MeasureContext - sendResponse: (msg: any) => Promise - sendError: (msg: any, error: any) => Promise -} - -/** - * @public - */ -export interface Session { - createTime: number - getUser: () => string - pipeline: () => Pipeline - ping: (ctx: ClientSessionCtx) => Promise - findAll: ( - ctx: ClientSessionCtx, - _class: Ref>, - query: DocumentQuery, - options?: FindOptions - ) => Promise - findAllRaw: ( - ctx: MeasureContext, - _class: Ref>, - query: DocumentQuery, - options?: FindOptions - ) => Promise> - tx: (ctx: ClientSessionCtx, tx: Tx) => Promise - - // Session restore information - sessionId: string - sessionInstanceId?: string - workspaceClosed?: boolean - - requests: Map - - binaryMode: boolean - useCompression: boolean - total: StatisticsElement - current: StatisticsElement - mins5: StatisticsElement - - lastRequest: number - lastPing: number - - isUpgradeClient: () => boolean - - getMode: () => string - - broadcast: (ctx: MeasureContext, socket: ConnectionSocket, tx: Tx[]) => void -} - -/** - * @public - */ -export interface ConnectionSocket { - id: string - isClosed: boolean - close: () => void - send: (ctx: MeasureContext, msg: Response, binary: boolean, compression: boolean) => void - data: () => Record - - readRequest: (buffer: Buffer, binary: boolean) => Request - - checkState: () => boolean -} - -/** - * @public - */ -export let LOGGING_ENABLED = true - -/** - * @public - */ -export function disableLogging (): void { - LOGGING_ENABLED = false -} - -/** - * @public - */ -export interface Workspace { - context: MeasureContext - id: string - pipeline: Promise - sessions: Map - upgrade: boolean - - closing?: Promise - softShutdown: number - workspaceInitCompleted: boolean - - workspaceId: WorkspaceId - workspaceName: string - branding: Branding | null -} - -export interface AddSessionActive { - session: Session - context: MeasureContext - workspaceId: string -} -export type AddSessionResponse = AddSessionActive | { upgrade: true } | { error: any, terminate?: boolean } - -/** - * @public - */ -export interface SessionManager { - workspaces: Map - sessions: Map - - createSession: ( - token: Token, - pipeline: Pipeline, - workspaceId: WorkspaceIdWithUrl, - branding: Branding | null - ) => Session - - addSession: ( - ctx: MeasureContext, - ws: ConnectionSocket, - token: Token, - rawToken: string, - pipelineFactory: PipelineFactory, - sessionId: string | undefined, - accountsUrl: string - ) => Promise - - broadcastAll: (workspace: Workspace, tx: Tx[], targets?: string[]) => void - - close: (ctx: MeasureContext, ws: ConnectionSocket, workspaceId: string) => Promise - - closeAll: ( - wsId: string, - workspace: Workspace, - code: number, - reason: 'upgrade' | 'shutdown', - ignoreSocket?: ConnectionSocket - ) => Promise - - forceClose: (wsId: string, ignoreSocket?: ConnectionSocket) => Promise - - closeWorkspaces: (ctx: MeasureContext) => Promise - - scheduleMaintenance: (timeMinutes: number) => void - - profiling?: { - start: () => void - stop: () => Promise - } -} - -/** - * @public - */ -export type HandleRequestFunction = ( - rctx: MeasureContext, - service: S, - ws: ConnectionSocket, - msg: Request, - workspaceId: string -) => void -/** - * @public - */ - -export type ServerFactory = ( - sessions: SessionManager, - handleRequest: HandleRequestFunction, - ctx: MeasureContext, - pipelineFactory: PipelineFactory, - port: number, - enableCompression: boolean, - accountsUrl: string, - externalStorage: StorageAdapter -) => () => Promise diff --git a/server/server/src/utils.ts b/server/server/src/utils.ts index a3f4f709d64..57df91fa66c 100644 --- a/server/server/src/utils.ts +++ b/server/server/src/utils.ts @@ -1,7 +1,31 @@ -import { toFindResult, type FindResult, type MeasureContext } from '@hcengineering/core' +// +// Copyright © 2024 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { type FindResult, type MeasureContext } from '@hcengineering/core' + +import type { + AddSessionActive, + AddSessionResponse, + ConnectionSocket, + HandleRequestFunction, + Session +} from '@hcengineering/server-core' + +import { toFindResult } from '@hcengineering/core' import { type Response } from '@hcengineering/rpc' import type { Token } from '@hcengineering/server-token' -import type { AddSessionActive, AddSessionResponse, ConnectionSocket, HandleRequestFunction, Session } from './types' export interface WebsocketData { connectionSocket?: ConnectionSocket diff --git a/server/translate/.eslintrc.js b/server/translate/.eslintrc.js deleted file mode 100644 index 72235dc2833..00000000000 --- a/server/translate/.eslintrc.js +++ /dev/null @@ -1,7 +0,0 @@ -module.exports = { - extends: ['./node_modules/@hcengineering/platform-rig/profiles/default/eslint.config.json'], - parserOptions: { - tsconfigRootDir: __dirname, - project: './tsconfig.json' - } -} diff --git a/server/translate/.npmignore b/server/translate/.npmignore deleted file mode 100644 index e3ec093c383..00000000000 --- a/server/translate/.npmignore +++ /dev/null @@ -1,4 +0,0 @@ -* -!/lib/** -!CHANGELOG.md -/lib/**/__tests__/ diff --git a/server/translate/src/index.ts b/server/translate/src/index.ts deleted file mode 100644 index 35ab48b2407..00000000000 --- a/server/translate/src/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export { default, translateId } from './plugin' -export * from './retranslate' -export * from './types' diff --git a/server/translate/src/plugin.ts b/server/translate/src/plugin.ts deleted file mode 100644 index 0924d0337f7..00000000000 --- a/server/translate/src/plugin.ts +++ /dev/null @@ -1,38 +0,0 @@ -// -// Copyright © 2022 Hardcore Engineering Inc. -// -// Licensed under the Eclipse Public License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. You may -// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// -// See the License for the specific language governing permissions and -// limitations under the License. -// - -import type { Plugin } from '@hcengineering/platform' -import { plugin } from '@hcengineering/platform' - -import type { Class, Ref } from '@hcengineering/core' -import type { TranslateConfiguration } from './types' - -export * from './types' - -/** - * @public - */ -export const translateId = 'translate' as Plugin - -/** - * @public - */ -const translatePlugin = plugin(translateId, { - class: { - TranslateConfiguration: '' as Ref> - } -}) - -export default translatePlugin diff --git a/server/translate/src/retranslate.ts b/server/translate/src/retranslate.ts deleted file mode 100644 index d460c5bc4a6..00000000000 --- a/server/translate/src/retranslate.ts +++ /dev/null @@ -1,228 +0,0 @@ -// -// Copyright © 2022 Hardcore Engineering Inc. -// -// Licensed under the Eclipse Public License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. You may -// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// -// See the License for the specific language governing permissions and -// limitations under the License. -// - -import { - Class, - Doc, - DocIndexState, - DocumentQuery, - DocumentUpdate, - MeasureContext, - Ref, - WorkspaceId, - type LowLevelStorage -} from '@hcengineering/core' -import { - contentStageId, - docKey, - DocUpdateHandler, - docUpdKey, - extractDocKey, - fieldStateId, - FullTextPipeline, - IndexedDoc -} from '@hcengineering/server-core' - -import { translateStateId, TranslationStage } from './types' - -/** - * @public - */ -export class LibRetranslateStage implements TranslationStage { - require = [fieldStateId, contentStageId] - stageId = translateStateId - - updateFields: DocUpdateHandler[] = [] - - langExtra = 'en' - - clearExcept?: string[] = undefined - - enabled = false - - token: string = '' - endpoint: string = '' - - constructor (readonly workspaceId: WorkspaceId) {} - - async initialize (ctx: MeasureContext, storage: LowLevelStorage, pipeline: FullTextPipeline): Promise {} - - async search ( - _classes: Ref>[], - query: DocumentQuery, - size: number | undefined, - from?: number - ): Promise<{ docs: IndexedDoc[], pass: boolean }> { - return { docs: [], pass: true } - } - - async update (doc: DocIndexState, update: DocumentUpdate): Promise { - for (const [k] of Array.from(Object.keys(update))) { - const { _class, attr, docId, extra } = extractDocKey(k) - if (!extra.includes('en')) { - // Fill translation document update request. - ;(update as any)[docUpdKey(attr, { _class, docId, extra: [...extra, ''] })] = '' - } - } - } - - async collect (toIndex: DocIndexState[], pipeline: FullTextPipeline, metrics: MeasureContext): Promise { - if (!this.enabled) { - return - } - for (const doc of toIndex) { - if (pipeline.cancelling) { - return - } - await this.retranslate(doc, pipeline) - } - } - - async isEnglish (text: string): Promise { - let english = false - try { - if (text.length > 0) { - const langResponse = await ( - await fetch(this.endpoint + '/detect', { - headers: { - 'Content-Type': 'application/json' - }, - body: JSON.stringify({ - q: text, - api_key: this.token - }) - }) - ).json() - english = JSON.parse(langResponse.body).some((it: any) => it.language === 'en' && it.confidence * 100 > 90) - } - } catch (err: any) { - // Coult not detect language - // console.error(err) - } - return english - } - - async retranslate (doc: DocIndexState, pipeline: FullTextPipeline): Promise { - // Copy content attributes as well. - const update: DocumentUpdate = {} - const elasticUpdate: Partial = {} - - if (pipeline.cancelling) { - return - } - try { - for (const [attrKey, v] of Object.entries(doc.attributes)) { - if (pipeline.cancelling) { - return - } - const { _class, attr, docId, extra } = extractDocKey(attrKey) - - // Translate only non english value and non child value. - if (!extra.includes(this.langExtra) && docId === undefined) { - const enContent = doc.attributes[`${docKey(attr, { _class, docId })}`] - - let sourceContent = v as string - if (extra.includes('base64')) { - sourceContent = Buffer.from(sourceContent, 'base64').toString() - } - - // If value is cleared - if (enContent === undefined || enContent === '') { - let toTranslate = `${sourceContent}\n` - // Remove extra spaces and extra new lines. - toTranslate = toTranslate - .split(/ |\t|\f/) - .filter((it) => it) - .join(' ') - .split(/\n+/) - .join('\n') - - let english = false - try { - if (toTranslate.length > 0) { - english = await this.isEnglish(toTranslate) - } - } catch (err: any) { - // Coult not detect language - console.error(err) - } - - let translatedText = '' - if (!english) { - try { - const st = Date.now() - console.log('retranslate:begin: ', doc._id, attr) - const translation = await ( - await fetch(this.endpoint + '/translate', { - headers: { - 'Content-Type': 'application/json' - }, - body: JSON.stringify({ - q: toTranslate, - source: 'auto', - target: 'en', - format: 'text', - api_key: this.token - }) - }) - ).json() - - const response: any = JSON.parse(translation.body) - console.log('retranslate:', doc._id, attr, Date.now() - st, response.translatedText.length) - translatedText = response.translatedText - } catch (err: any) { - console.error(err) - } - } else { - translatedText = '' - console.log('retranslate: Not required', doc._id, attr) - } - const base64Content = Buffer.from(translatedText).toString('base64') - ;(update as any)[`${docUpdKey(attr, { _class, extra: [...extra, this.langExtra] })}`] = base64Content - elasticUpdate[`${docKey(attr, { _class, extra: [...extra, this.langExtra] })}`] = base64Content - - if (doc.attachedTo != null) { - const parentUpdate: DocumentUpdate = {} - - ;(parentUpdate as any)[docUpdKey(attr, { _class, docId: doc._id, extra: [...extra, this.langExtra] })] = - base64Content - - await pipeline.update(doc.attachedTo as Ref, false, parentUpdate) - } - } - } - } - } catch (err: any) { - const wasError = doc.attributes.error !== undefined - - await pipeline.update(doc._id, false, { [docKey('error')]: JSON.stringify(err) }) - if (wasError) { - return - } - // Print error only first time, and update it in doc index - console.error(err) - return - } - - await pipeline.update(doc._id, true, update, true) - } - - async remove (docs: DocIndexState[], pipeline: FullTextPipeline): Promise { - // will be handled by field processor - for (const doc of docs) { - await pipeline.update(doc._id, true, {}) - } - } -} diff --git a/server/translate/src/types.ts b/server/translate/src/types.ts deleted file mode 100644 index 5df965dcf13..00000000000 --- a/server/translate/src/types.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { FullTextPipelineStage } from '@hcengineering/server-core' - -import { Configuration } from '@hcengineering/core' - -/** - * @public - */ -export const translateStateId = 'trn-v2' - -/** - * @public - */ -export interface TranslationStage extends FullTextPipelineStage {} - -/** - * @public - */ -export interface TranslateConfiguration extends Configuration { - token: string - endpoint: string -} diff --git a/server/translate/tsconfig.json b/server/translate/tsconfig.json deleted file mode 100644 index 59e4fd42978..00000000000 --- a/server/translate/tsconfig.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "extends": "./node_modules/@hcengineering/platform-rig/profiles/default/tsconfig.json", - - "compilerOptions": { - "rootDir": "./src", - "outDir": "./lib", - "declarationDir": "./types", - "tsBuildInfoFile": ".build/build.tsbuildinfo" - } -} \ No newline at end of file diff --git a/server/workspace-service/src/index.ts b/server/workspace-service/src/index.ts index b3e353a437a..3f850a09ef7 100644 --- a/server/workspace-service/src/index.ts +++ b/server/workspace-service/src/index.ts @@ -68,12 +68,6 @@ export function serveWorkspaceAccount ( } setMetadata(serverClientPlugin.metadata.Endpoint, accountUri) - const transactorUri = process.env.TRANSACTOR_URL - if (transactorUri === undefined) { - console.log('Please provide transactor url') - process.exit(1) - } - const serverSecret = process.env.SERVER_SECRET if (serverSecret === undefined) { console.log('Please provide server secret') diff --git a/server/ws/src/server_http.ts b/server/ws/src/server_http.ts index d98a0858892..3fa0b4dcbdc 100644 --- a/server/ws/src/server_http.ts +++ b/server/ws/src/server_http.ts @@ -22,15 +22,19 @@ import { getFile, getFileRange, getStatistics, - LOGGING_ENABLED, processRequest, wipeStatistics, type BlobResponse, + type WebsocketData +} from '@hcengineering/server' +import { + LOGGING_ENABLED, type ConnectionSocket, type HandleRequestFunction, type SessionManager, - type WebsocketData -} from '@hcengineering/server' + type PipelineFactory, + type StorageAdapter +} from '@hcengineering/server-core' import { decodeToken, type Token } from '@hcengineering/server-token' import cors from 'cors' import express, { type Response as ExpressResponse } from 'express' @@ -38,7 +42,6 @@ import http, { type IncomingMessage } from 'http' import os from 'os' import { WebSocketServer, type RawData, type WebSocket } from 'ws' -import { type PipelineFactory, type StorageAdapter } from '@hcengineering/server-core' import 'bufferutil' import 'utf-8-validate' let profiling = false diff --git a/services/github/pod-github/package.json b/services/github/pod-github/package.json index bda9ce568f0..39de6218322 100644 --- a/services/github/pod-github/package.json +++ b/services/github/pod-github/package.json @@ -17,7 +17,7 @@ "_phase:bundle": "rushx bundle", "_phase:docker-build": "rushx docker:build", "_phase:docker-staging": "rushx docker:staging", - "bundle": "mkdir -p bundle && esbuild src/index.ts --bundle --sourcemap=external --platform=node --outfile=bundle/bundle.js --log-level=error", + "bundle": "mkdir -p bundle && esbuild src/index.ts --bundle --platform=node --outfile=bundle/bundle.js --log-level=error --sourcemap=external", "docker:build": "../../../common/scripts/docker_build.sh hardcoreeng/github", "docker:staging": "../../../common/scripts/docker_tag.sh hardcoreeng/github staging", "docker:push": "../../../common/scripts/docker_tag.sh hardcoreeng/github", diff --git a/services/sign/pod-sign/package.json b/services/sign/pod-sign/package.json index b5736af48ff..5a22f67a359 100644 --- a/services/sign/pod-sign/package.json +++ b/services/sign/pod-sign/package.json @@ -56,6 +56,7 @@ "@hcengineering/server-token": "^0.6.11", "@hcengineering/server-core": "^0.6.1", "@hcengineering/server": "^0.6.4", + "@hcengineering/server-storage": "^0.6.0", "@hcengineering/core": "^0.6.32", "@hcengineering/platform": "^0.6.11", "@hcengineering/client": "^0.6.18", diff --git a/services/sign/pod-sign/src/main.ts b/services/sign/pod-sign/src/main.ts index 3ff63692496..5d5601df28d 100644 --- a/services/sign/pod-sign/src/main.ts +++ b/services/sign/pod-sign/src/main.ts @@ -3,7 +3,7 @@ // import { setMetadata } from '@hcengineering/platform' -import { storageConfigFromEnv } from '@hcengineering/server' +import { storageConfigFromEnv } from '@hcengineering/server-storage' import serverClient from '@hcengineering/server-client' import { loadBrandingMap } from '@hcengineering/server-core' import serverToken from '@hcengineering/server-token' diff --git a/services/sign/pod-sign/src/server.ts b/services/sign/pod-sign/src/server.ts index 1e910e0f650..941ec77143e 100644 --- a/services/sign/pod-sign/src/server.ts +++ b/services/sign/pod-sign/src/server.ts @@ -19,7 +19,7 @@ import express, { type Express, type NextFunction, type Request, type Response } import { Token } from '@hcengineering/server-token' import { type Server } from 'http' import { StorageConfiguration } from '@hcengineering/server-core' -import { buildStorageFromConfig } from '@hcengineering/server' +import { buildStorageFromConfig } from '@hcengineering/server-storage' import { MeasureMetricsContext, generateId } from '@hcengineering/core' import { ApiError } from './error'