From c55ed0c4cce842639ff4d7797885757f46e2f6e6 Mon Sep 17 00:00:00 2001
From: Hassan Abdel-Rahman <hassan.abdelrahman@gmail.com>
Date: Mon, 13 Jan 2025 16:22:00 -0500
Subject: [PATCH 01/13] Multiple worker support

---
 .github/workflows/manual-deploy.yml           |  28 ++-
 README.md                                     |   7 +-
 .../matrix/helpers/isolated-realm-server.ts   |  35 +++-
 packages/realm-server/main.ts                 | 103 ++++++-----
 packages/realm-server/package.json            |  10 +-
 .../{Dockerfile => realm-server.Dockerfile}   |   0
 packages/realm-server/scripts/start-all.sh    |   4 +-
 .../realm-server/scripts/start-development.sh |   1 +
 .../realm-server/scripts/start-test-realms.sh |   9 +-
 .../scripts/start-worker-development.sh       |  30 +++
 .../scripts/start-worker-production.sh        |  21 +++
 .../scripts/start-worker-staging.sh           |  21 +++
 .../realm-server/scripts/start-worker-test.sh |  24 +++
 packages/realm-server/worker-manager.ts       | 174 ++++++++++++++++++
 packages/realm-server/worker.Dockerfile       |  24 +++
 packages/realm-server/worker.ts               |  28 ++-
 pnpm-lock.yaml                                |   6 +
 17 files changed, 445 insertions(+), 80 deletions(-)
 rename packages/realm-server/{Dockerfile => realm-server.Dockerfile} (100%)
 create mode 100755 packages/realm-server/scripts/start-worker-development.sh
 create mode 100755 packages/realm-server/scripts/start-worker-production.sh
 create mode 100755 packages/realm-server/scripts/start-worker-staging.sh
 create mode 100755 packages/realm-server/scripts/start-worker-test.sh
 create mode 100644 packages/realm-server/worker-manager.ts
 create mode 100644 packages/realm-server/worker.Dockerfile

diff --git a/.github/workflows/manual-deploy.yml b/.github/workflows/manual-deploy.yml
index 8b37f081b4..584dc4391a 100644
--- a/.github/workflows/manual-deploy.yml
+++ b/.github/workflows/manual-deploy.yml
@@ -74,10 +74,21 @@ jobs:
     with:
       repository: "boxel-realm-server-${{ inputs.environment }}"
       environment: ${{ inputs.environment }}
-      dockerfile: "packages/realm-server/Dockerfile"
+      dockerfile: "packages/realm-server/realm-server.Dockerfile"
       build-args: |
         "realm_server_script=start:${{ inputs.environment }}"
 
+  build-worker:
+    name: Build worker Docker image
+    uses: cardstack/gh-actions/.github/workflows/docker-ecr.yml@main
+    secrets: inherit
+    with:
+      repository: "boxel-realm-server-${{ inputs.environment }}"
+      environment: ${{ inputs.environment }}
+      dockerfile: "packages/realm-server/worker.Dockerfile"
+      build-args: |
+        "worker_script=start:worker-${{ inputs.environment }}"
+
   build-pg-migration:
     name: Build pg-migration Docker image
     uses: cardstack/gh-actions/.github/workflows/docker-ecr.yml@main
@@ -110,9 +121,22 @@ jobs:
     steps:
       - run: sleep 240
 
+  deploy-worker:
+    name: Deploy worker
+    needs: [build-realm-server, deploy-host, post-migrate-db]
+    uses: cardstack/gh-actions/.github/workflows/ecs-deploy.yml@main
+    secrets: inherit
+    with:
+      container-name: "boxel-worker"
+      environment: ${{ inputs.environment }}
+      cluster: ${{ inputs.environment }}
+      service-name: "boxel-worker-${{ inputs.environment }}"
+      image: ${{ needs.build-worker.outputs.image }}
+      wait-for-service-stability: false
+
   deploy-realm-server:
     name: Deploy realm server
-    needs: [build-realm-server, deploy-host, post-migrate-db]
+    needs: [deploy-worker]
     uses: cardstack/gh-actions/.github/workflows/ecs-deploy.yml@main
     secrets: inherit
     with:
diff --git a/README.md b/README.md
index 97cd97f074..a6f4a7e816 100644
--- a/README.md
+++ b/README.md
@@ -73,7 +73,7 @@ Live reloads are not available in this mode, however, if you use start the serve
 
 #### Using `start:all`
 
-Instead of running `pnpm start:base`, you can alternatively use `pnpm start:all` which also serves a few other realms on other ports--this is convenient if you wish to switch between the app and the tests without having to restart servers. Here's what is spun up with `start:all`:
+Instead of running `pnpm start:base`, you can alternatively use `pnpm start:all` which also serves a few other realms on other ports--this is convenient if you wish to switch between the app and the tests without having to restart servers. Use the environment variable `WORKER_COUNT` to add additional workers. By default there is 1 worker for each realm server. Here's what is spun up with `start:all`:
 
 | Port  | Description                                                   | Running `start:all` | Running `start:base` |
 | ----- | ------------------------------------------------------------- | ------------------- | -------------------- |
@@ -82,13 +82,16 @@ Instead of running `pnpm start:base`, you can alternatively use `pnpm start:all`
 | :4201 | `/seed` seed realm                                            | ✅                  | 🚫                   |
 | :4202 | `/test` host test realm, `/node-test` node test realm         | ✅                  | 🚫                   |
 | :4205 | `/test` realm for matrix client tests (playwright controlled) | 🚫                  | 🚫                   |
+| :4210 | Development Worker Manager (spins up 1 worker by default)     | ✅                  | 🚫                   |
+| :4211 | Test Worker Manager (spins up 1 worker by default)            | ✅                  | 🚫                   |
+| :4212 | Test Worker Manager for matrix client tests (playwright controlled - 1 worker) | ✅                  | 🚫                   |
 | :5001 | Mail user interface for viewing emails sent to local SMTP     | ✅                  | 🚫                   |
 | :5435 | Postgres DB                                                   | ✅                  | 🚫                   |
 | :8008 | Matrix synapse server                                         | ✅                  | 🚫                   |
 
 #### Using `start:development`
 
-You can also use `start:development` if you want the functionality of `start:all`, but without running the test realms. `start:development` will enable you to open http://localhost:4201 and allow to select between the cards in the /base and /experiments realm.
+You can also use `start:development` if you want the functionality of `start:all`, but without running the test realms. `start:development` will enable you to open http://localhost:4201 and allow to select between the cards in the /base and /experiments realm. In order to use `start:development` you must also make sure to run `start:worker-development` in order to start the workers (which are normally started in `start:all`.
 
 ### Card Pre-rendering
 
diff --git a/packages/matrix/helpers/isolated-realm-server.ts b/packages/matrix/helpers/isolated-realm-server.ts
index 7c631d3891..c72c61ccfa 100644
--- a/packages/matrix/helpers/isolated-realm-server.ts
+++ b/packages/matrix/helpers/isolated-realm-server.ts
@@ -31,6 +31,36 @@ export async function startServer() {
   process.env.MATRIX_URL = 'http://localhost:8008';
   process.env.REALM_SERVER_MATRIX_USERNAME = 'realm_server';
 
+  let worker = spawn(
+    'ts-node',
+    [
+      `--transpileOnly`,
+      'worker-manager',
+      `--port=4212`,
+      `--matrixURL='http://localhost:8008'`,
+      `--distURL="${process.env.HOST_URL ?? 'http://localhost:4200'}"`,
+
+      `--fromUrl='http://localhost:4205/test/'`,
+      `--toUrl='http://localhost:4205/test/'`,
+      `--fromUrl='https://cardstack.com/base/'`,
+      `--toUrl='http://localhost:4201/base/'`,
+    ],
+    {
+      cwd: realmServerDir,
+      stdio: ['pipe', 'pipe', 'pipe', 'ipc'],
+    },
+  );
+  if (worker.stdout) {
+    worker.stdout.on('data', (data: Buffer) =>
+      console.log(`worker: ${data.toString()}`),
+    );
+  }
+  if (worker.stderr) {
+    worker.stderr.on('data', (data: Buffer) =>
+      console.error(`worker: ${data.toString()}`),
+    );
+  }
+
   let realmServer = spawn(
     'ts-node',
     [
@@ -40,13 +70,14 @@ export async function startServer() {
       `--matrixURL='http://localhost:8008'`,
       `--realmsRootPath='${dir.name}'`,
       `--seedPath='${seedPath}'`,
+      `--workerManagerPort=4212`,
       `--migrateDB`,
       `--useRegistrationSecretFunction`,
 
       `--path='${testRealmDir}'`,
       `--username='test_realm'`,
-      `--fromUrl='/test/'`,
-      `--toUrl='/test/'`,
+      `--fromUrl='http://localhost:4205/test/'`,
+      `--toUrl='http://localhost:4205/test/'`,
       `--fromUrl='https://cardstack.com/base/'`,
       `--toUrl='http://localhost:4201/base/'`,
     ],
diff --git a/packages/realm-server/main.ts b/packages/realm-server/main.ts
index aa7745d41c..f253e7cacb 100644
--- a/packages/realm-server/main.ts
+++ b/packages/realm-server/main.ts
@@ -11,13 +11,12 @@ import { NodeAdapter } from './node-realm';
 import yargs from 'yargs';
 import { RealmServer } from './server';
 import { resolve } from 'path';
-import { spawn } from 'child_process';
+import { createConnection, type Socket } from 'net';
 import { makeFastBootIndexRunner } from './fastboot';
 import { shimExternals } from './lib/externals';
 import * as Sentry from '@sentry/node';
 import { PgAdapter, PgQueuePublisher } from '@cardstack/postgres';
 import { MatrixClient } from '@cardstack/runtime-common/matrix-client';
-import flattenDeep from 'lodash/flattenDeep';
 import 'decorator-transforms/globals';
 
 let log = logger('main');
@@ -68,6 +67,7 @@ let {
   useRegistrationSecretFunction,
   seedPath,
   migrateDB,
+  workerManagerPort,
 } = yargs(process.argv.slice(2))
   .usage('Start realm server')
   .options({
@@ -130,6 +130,11 @@ let {
         'The flag should be set when running matrix tests where the synapse instance is torn down and restarted multiple times during the life of the realm server.',
       type: 'boolean',
     },
+    workerManagerPort: {
+      description:
+        'The port the worker manager is running on. used to wait for the workers to be ready',
+      type: 'number',
+    },
   })
   .parseSync();
 
@@ -165,8 +170,8 @@ let virtualNetwork = new VirtualNetwork();
 shimExternals(virtualNetwork);
 
 let urlMappings = fromUrls.map((fromUrl, i) => [
-  new URL(String(fromUrl), `http://localhost:${port}`),
-  new URL(String(toUrls[i]), `http://localhost:${port}`),
+  new URL(String(fromUrl)),
+  new URL(String(toUrls[i])),
 ]);
 for (let [from, to] of urlMappings) {
   virtualNetwork.addURLMapping(from, to);
@@ -185,7 +190,9 @@ let autoMigrate = migrateDB || undefined;
     manager.getOptions.bind(manager),
   );
 
-  await startWorker({ autoMigrate });
+  if (workerManagerPort != null) {
+    await waitForWorkerManager(workerManagerPort);
+  }
 
   for (let [i, path] of paths.entries()) {
     let url = hrefs[i][0];
@@ -324,51 +331,49 @@ let autoMigrate = migrateDB || undefined;
   process.exit(-3);
 });
 
-async function startWorker(opts?: { autoMigrate?: true }) {
-  let worker = spawn(
-    'ts-node',
-    [
-      '--transpileOnly',
-      'worker',
-      `--port=${port}`,
-      `--matrixURL='${matrixURL}'`,
-      `--distURL='${distURL}'`,
-      ...(opts?.autoMigrate ? [`--migrateDB`] : []),
-      ...flattenDeep(
-        urlMappings.map(([from, to]) => [
-          `--fromUrl='${from}'`,
-          `--toUrl='${to}'`,
-        ]),
-      ),
-    ],
-    {
-      stdio: ['pipe', 'pipe', 'pipe', 'ipc'],
-    },
-  );
+let workerReadyDeferred: Deferred<boolean> | undefined;
+async function waitForWorkerManager(port: number) {
+  const workerManager = await new Promise<Socket>((r) => {
+    let socket = createConnection({ port }, () => {
+      log.info(`Connected to worker manager on port ${port}`);
+      r(socket);
+    });
+  });
 
-  if (worker.stdout) {
-    worker.stdout.on('data', (data: Buffer) =>
-      log.info(`worker: ${data.toString()}`),
-    );
-  }
-  if (worker.stderr) {
-    worker.stderr.on('data', (data: Buffer) =>
-      console.error(`worker: ${data.toString()}`),
-    );
-  }
+  workerManager.on('data', (data) => {
+    let res = data.toString();
+    if (!workerReadyDeferred) {
+      throw new Error(
+        `received unsolicited message from worker manager on port ${port}`,
+      );
+    }
+    switch (res) {
+      case 'ready':
+      case 'not-ready':
+        workerReadyDeferred.fulfill(res === 'ready' ? true : false);
+        break;
+      default:
+        workerReadyDeferred.reject(
+          `unexpected response from worker manager: ${res}`,
+        );
+    }
+  });
 
-  let timeout = await Promise.race([
-    new Promise<void>((r) => {
-      worker.on('message', (message) => {
-        if (message === 'ready') {
-          r();
-        }
-      });
-    }),
-    new Promise<true>((r) => setTimeout(() => r(true), 30_000)),
-  ]);
-  if (timeout) {
-    console.error(`timed-out waiting for worker to start. Stopping server`);
-    process.exit(-2);
+  try {
+    let isReady = false;
+    let timeout = Date.now() + 30_000;
+    do {
+      workerReadyDeferred = new Deferred();
+      workerManager.write('ready?');
+      isReady = await workerReadyDeferred.promise;
+    } while (!isReady && Date.now() < timeout);
+    if (!isReady) {
+      throw new Error(
+        `timed out trying to connect to worker manager on port ${port}`,
+      );
+    }
+  } finally {
+    workerManager.end();
   }
+  log.info('workers are ready');
 }
diff --git a/packages/realm-server/package.json b/packages/realm-server/package.json
index 5994be340c..4a3b02add5 100644
--- a/packages/realm-server/package.json
+++ b/packages/realm-server/package.json
@@ -24,6 +24,7 @@
     "@types/lodash": "^4.14.182",
     "@types/mime-types": "^2.1.1",
     "@types/node": "^18.18.5",
+    "@types/pluralize": "^0.0.30",
     "@types/qs": "^6.9.17",
     "@types/qunit": "^2.11.3",
     "@types/sane": "^2.0.1",
@@ -52,6 +53,7 @@
     "loglevel": "^1.8.1",
     "mime-types": "^2.1.35",
     "npm-run-all": "^4.1.5",
+    "pluralize": "^8.0.0",
     "prettier": "^2.8.4",
     "prettier-plugin-ember-template-tag": "^1.1.0",
     "qs": "^6.13.0",
@@ -85,12 +87,16 @@
     "setup:catalog-in-deployment": "mkdir -p /persistent/catalog && rsync --dry-run --itemize-changes --size-only --recursive --delete ../catalog-realm/. /persistent/catalog/ && rsync --size-only --recursive --delete ../catalog-realm/. /persistent/catalog/",
     "start": "PGPORT=5435 NODE_NO_WARNINGS=1 ts-node --transpileOnly main",
     "start:base": "./scripts/start-base.sh",
-    "start:test-realms": "./scripts/start-test-realms.sh",
+    "start:test-realms": "./scripts/start-test-realms.sh --workerManagerPort=4211",
     "start:all": "./scripts/start-all.sh",
     "start:without-matrix": "./scripts/start-without-matrix.sh",
     "start:staging": "./scripts/start-staging.sh",
-    "start:development": "./scripts/start-development.sh",
+    "start:development": "./scripts/start-development.sh --workerManagerPort=4210",
     "start:production": "./scripts/start-production.sh",
+    "start:worker-development": "./scripts/start-worker-development.sh",
+    "start:worker-test": "./scripts/start-worker-test.sh",
+    "start:worker-staging": "./scripts/start-worker-staging.sh",
+    "start:worker-production": "./scripts/start-worker-production.sh",
     "start:services-for-matrix-tests": "./scripts/start-services-for-matrix-tests.sh",
     "wait": "sleep 10000000",
     "lint": "concurrently \"pnpm:lint:*(!fix)\" --names \"lint:\"",
diff --git a/packages/realm-server/Dockerfile b/packages/realm-server/realm-server.Dockerfile
similarity index 100%
rename from packages/realm-server/Dockerfile
rename to packages/realm-server/realm-server.Dockerfile
diff --git a/packages/realm-server/scripts/start-all.sh b/packages/realm-server/scripts/start-all.sh
index 67c0e1c111..5eb97d3694 100755
--- a/packages/realm-server/scripts/start-all.sh
+++ b/packages/realm-server/scripts/start-all.sh
@@ -1,8 +1,8 @@
 #! /bin/sh
 
 NODE_NO_WARNINGS=1 start-server-and-test \
-  'run-p start:pg start:matrix start:smtp start:development' \
+  'run-p start:pg start:matrix start:smtp start:worker-development start:development' \
   'http-get://localhost:4201/base/_readiness-check?acceptHeader=application%2Fvnd.api%2Bjson|http-get://localhost:4201/experiments/_readiness-check?acceptHeader=application%2Fvnd.api%2Bjson|http://localhost:8008|http://localhost:5001' \
-  'run-p start:test-realms' \
+  'run-p start:worker-test start:test-realms' \
   'http-get://localhost:4202/node-test/_readiness-check?acceptHeader=application%2Fvnd.api%2Bjson' \
   'wait'
diff --git a/packages/realm-server/scripts/start-development.sh b/packages/realm-server/scripts/start-development.sh
index b7d0e2267a..f9704ea706 100755
--- a/packages/realm-server/scripts/start-development.sh
+++ b/packages/realm-server/scripts/start-development.sh
@@ -24,6 +24,7 @@ NODE_ENV=development \
   --realmsRootPath='./realms/localhost_4201' \
   --seedPath='../seed-realm' \
   --migrateDB \
+  $1 \
   \
   --path='../base' \
   --username='base_realm' \
diff --git a/packages/realm-server/scripts/start-test-realms.sh b/packages/realm-server/scripts/start-test-realms.sh
index 2b3695f4a7..9c42c331c4 100755
--- a/packages/realm-server/scripts/start-test-realms.sh
+++ b/packages/realm-server/scripts/start-test-realms.sh
@@ -23,15 +23,16 @@ NODE_ENV=test \
   --realmsRootPath='./realms/localhost_4202' \
   --matrixRegistrationSecretFile='../matrix/registration_secret.txt' \
   --migrateDB \
+  $1 \
   \
   --path='./tests/cards' \
   --username='node-test_realm' \
-  --fromUrl='/node-test/' \
-  --toUrl='/node-test/' \
+  --fromUrl='http://localhost:4202/node-test/' \
+  --toUrl='http://localhost:4202/node-test/' \
   \
   --path='../host/tests/cards' \
   --username='test_realm' \
-  --fromUrl='/test/' \
-  --toUrl='/test/' \
+  --fromUrl='http://localhost:4202/test/' \
+  --toUrl='http://localhost:4202/test/' \
   --fromUrl='https://cardstack.com/base/' \
   --toUrl='http://localhost:4201/base/'
diff --git a/packages/realm-server/scripts/start-worker-development.sh b/packages/realm-server/scripts/start-worker-development.sh
new file mode 100755
index 0000000000..eacbbec262
--- /dev/null
+++ b/packages/realm-server/scripts/start-worker-development.sh
@@ -0,0 +1,30 @@
+#! /bin/sh
+SCRIPTS_DIR="$(cd "$(dirname "$0")" && pwd)"
+. "$SCRIPTS_DIR/wait-for-pg.sh"
+
+wait_for_postgres
+
+NODE_ENV=development \
+  NODE_NO_WARNINGS=1 \
+  PGPORT=5435 \
+  PGDATABASE=boxel \
+  LOG_LEVELS='*=info' \
+  REALM_SECRET_SEED="shhh! it's a secret" \
+  ts-node \
+  --transpileOnly worker-manager \
+  --count="${WORKER_COUNT:-1}" \
+  --port=4210 \
+  --matrixURL='http://localhost:8008' \
+  --distURL="${HOST_URL:-http://localhost:4200}" \
+  \
+  --fromUrl='https://cardstack.com/base/' \
+  --toUrl='http://localhost:4201/base/' \
+  \
+  --fromUrl='http://localhost:4201/experiments/' \
+  --toUrl='http://localhost:4201/experiments/' \
+  \
+  --fromUrl='http://localhost:4201/seed/' \
+  --toUrl='http://localhost:4201/seed/' \
+  \
+  --fromUrl='http://localhost:4201/catalog/' \
+  --toUrl='http://localhost:4201/catalog/'
diff --git a/packages/realm-server/scripts/start-worker-production.sh b/packages/realm-server/scripts/start-worker-production.sh
new file mode 100755
index 0000000000..b37de0001d
--- /dev/null
+++ b/packages/realm-server/scripts/start-worker-production.sh
@@ -0,0 +1,21 @@
+#! /bin/sh
+
+NODE_NO_WARNINGS=1 \
+  LOG_LEVELS='*=info' \
+  ts-node \
+  --transpileOnly worker-manager \
+  --count="${WORKER_COUNT:-1}" \
+  --matrixURL='https://matrix.boxel.ai' \
+  --distURL='https://boxel-host.boxel.ai' \
+  \
+  --fromUrl='https://cardstack.com/base/' \
+  --toUrl='https://app.boxel.ai/base/' \
+  \
+  --fromUrl='https://app.boxel.ai/experiments/' \
+  --toUrl='https://app.boxel.ai/experiments/' \
+  \
+  --fromUrl='https://app.boxel.ai/seed/' \
+  --toUrl='https://app.boxel.ai/seed/' \
+  \
+  --fromUrl='https://app.boxel.ai/catalog/' \
+  --toUrl='https://app.boxel.ai/catalog/'
diff --git a/packages/realm-server/scripts/start-worker-staging.sh b/packages/realm-server/scripts/start-worker-staging.sh
new file mode 100755
index 0000000000..7a91268ad6
--- /dev/null
+++ b/packages/realm-server/scripts/start-worker-staging.sh
@@ -0,0 +1,21 @@
+#! /bin/sh
+
+NODE_NO_WARNINGS=1 \
+  LOG_LEVELS='perf=debug' \
+  ts-node \
+  --transpileOnly worker-manager \
+  --count="${WORKER_COUNT:-1}" \
+  --matrixURL='https://matrix-staging.stack.cards' \
+  --distURL='https://boxel-host-staging.stack.cards' \
+  \
+  --fromUrl='https://cardstack.com/base/' \
+  --toUrl='https://realms-staging.stack.cards/base/' \
+  \
+  --fromUrl='https://realms-staging.stack.cards/experiments/' \
+  --toUrl='https://realms-staging.stack.cards/experiments/' \
+  \
+  --fromUrl='https://realms-staging.stack.cards/seed/' \
+  --toUrl='https://realms-staging.stack.cards/seed/' \
+  \
+  --fromUrl='https://realms-staging.stack.cards/catalog/' \
+  --toUrl='https://realms-staging.stack.cards/catalog/'
diff --git a/packages/realm-server/scripts/start-worker-test.sh b/packages/realm-server/scripts/start-worker-test.sh
new file mode 100755
index 0000000000..c4b95c89e9
--- /dev/null
+++ b/packages/realm-server/scripts/start-worker-test.sh
@@ -0,0 +1,24 @@
+#! /bin/sh
+SCRIPTS_DIR="$(cd "$(dirname "$0")" && pwd)"
+. "$SCRIPTS_DIR/wait-for-pg.sh"
+
+wait_for_postgres
+
+NODE_ENV=test \
+  PGPORT=5435 \
+  PGDATABASE=boxel_test \
+  NODE_NO_WARNINGS=1 \
+  REALM_SECRET_SEED="shhh! it's a secret" \
+  ts-node \
+  --transpileOnly worker-manager \
+  --port=4211 \
+  --matrixURL='http://localhost:8008' \
+  --distURL="${HOST_URL:-http://localhost:4200}" \
+  \
+  --fromUrl='http://localhost:4202/node-test/' \
+  --toUrl='http://localhost:4202/node-test/' \
+  \
+  --fromUrl='http://localhost:4202/test/' \
+  --toUrl='http://localhost:4202/test/' \
+  --fromUrl='https://cardstack.com/base/' \
+  --toUrl='http://localhost:4201/base/'
diff --git a/packages/realm-server/worker-manager.ts b/packages/realm-server/worker-manager.ts
new file mode 100644
index 0000000000..ae54f520f8
--- /dev/null
+++ b/packages/realm-server/worker-manager.ts
@@ -0,0 +1,174 @@
+import './instrument';
+import './setup-logger'; // This should be first
+import { logger } from '@cardstack/runtime-common';
+import yargs from 'yargs';
+import * as Sentry from '@sentry/node';
+import { createServer } from 'net';
+import flattenDeep from 'lodash/flattenDeep';
+import { spawn } from 'child_process';
+import pluralize from 'pluralize';
+
+let log = logger('worker');
+
+const REALM_SECRET_SEED = process.env.REALM_SECRET_SEED;
+if (!REALM_SECRET_SEED) {
+  log.error(
+    `The REALM_SECRET_SEED environment variable is not set. Please make sure this env var has a value`,
+  );
+  process.exit(-1);
+}
+
+let {
+  port,
+  matrixURL,
+  count = 1,
+  distURL = process.env.HOST_URL ?? 'http://localhost:4200',
+  fromUrl: fromUrls,
+  toUrl: toUrls,
+} = yargs(process.argv.slice(2))
+  .usage('Start worker manager')
+  .options({
+    port: {
+      description: 'TCP port for worker to communicate readiness (for tests)',
+      type: 'number',
+    },
+    count: {
+      description: 'The number of workers to start',
+      type: 'number',
+    },
+    fromUrl: {
+      description: 'the source of the realm URL proxy',
+      demandOption: true,
+      type: 'array',
+    },
+    toUrl: {
+      description: 'the target of the realm URL proxy',
+      demandOption: true,
+      type: 'array',
+    },
+    distURL: {
+      description:
+        'the URL of a deployed host app. (This can be provided instead of the --distPath)',
+      type: 'string',
+    },
+    matrixURL: {
+      description: 'The matrix homeserver for the realm server',
+      demandOption: true,
+      type: 'string',
+    },
+  })
+  .parseSync();
+
+let isReady = false;
+if (port != null) {
+  // in tests we start a simple TCP server to communicate to the realm when
+  // the worker is ready to start processing jobs
+  let server = createServer((socket) => {
+    log.info(`realm connected to worker manager`);
+    socket.on('data', (data) => {
+      if (data.toString() === 'ready?') {
+        socket.write(isReady ? 'ready' : 'not-ready');
+      }
+    });
+    socket.on('close', (hadError) => {
+      log.info(`realm has disconnected${hadError ? ' due to an error' : ''}.`);
+    });
+    socket.on('error', (err: any) => {
+      console.error(`realm disconnected from worker manager: ${err.message}`);
+    });
+  });
+
+  server.listen(port, () => {
+    log.info(`worker manager listening for realm on port ${port}`);
+  });
+
+  const shutdown = () => {
+    log.info(`Shutting down server for worker manager...`);
+    server.close((err) => {
+      if (err) {
+        log.error(`Error while closing the server for worker manager:`, err);
+        process.exit(1);
+      }
+      log.info(`Server closed for worker manager.`);
+      process.exit(0);
+    });
+  };
+
+  process.on('SIGINT', shutdown);
+  process.on('SIGTERM', shutdown);
+  process.on('uncaughtException', (err) => {
+    log.error(`Uncaught exception in worker manager:`, err);
+    shutdown();
+  });
+}
+
+(async () => {
+  log.info(`starting ${count} ${pluralize('worker', count)}`);
+  let urlMappings = fromUrls.map((fromUrl, i) => [
+    new URL(String(fromUrl)),
+    new URL(String(toUrls[i])),
+  ]);
+
+  for (let i = 0; i < count; i++) {
+    await startWorker(urlMappings);
+  }
+  isReady = true;
+  log.info('All workers have been started');
+})().catch((e: any) => {
+  Sentry.captureException(e);
+  log.error(
+    `worker: Unexpected error encountered starting worker manager, stopping worker manager`,
+    e,
+  );
+  process.exit(1);
+});
+
+async function startWorker(urlMappings: URL[][]) {
+  let worker = spawn(
+    'ts-node',
+    [
+      '--transpileOnly',
+      'worker',
+      `--matrixURL='${matrixURL}'`,
+      `--distURL='${distURL}'`,
+      ...flattenDeep(
+        urlMappings.map(([from, to]) => [
+          `--fromUrl='${from.href}'`,
+          `--toUrl='${to.href}'`,
+        ]),
+      ),
+    ],
+    {
+      stdio: ['pipe', 'pipe', 'pipe', 'ipc'],
+    },
+  );
+
+  if (worker.stdout) {
+    worker.stdout.on('data', (data: Buffer) =>
+      log.info(`[worker ${worker.pid}]: ${data.toString()}`),
+    );
+  }
+  if (worker.stderr) {
+    worker.stderr.on('data', (data: Buffer) =>
+      log.error(`[worker ${worker.pid}]: ${data.toString()}`),
+    );
+  }
+
+  let timeout = await Promise.race([
+    new Promise<void>((r) => {
+      worker.on('message', (message) => {
+        if (message === 'ready') {
+          log.info(`[worker ${worker.pid}]: worker ready`);
+          r();
+        }
+      });
+    }),
+    new Promise<true>((r) => setTimeout(() => r(true), 30_000)),
+  ]);
+  if (timeout) {
+    console.error(
+      `timed-out waiting for worker pid ${worker.pid} to start. Stopping worker manager`,
+    );
+    process.exit(-2);
+  }
+}
diff --git a/packages/realm-server/worker.Dockerfile b/packages/realm-server/worker.Dockerfile
new file mode 100644
index 0000000000..47f1bc1b70
--- /dev/null
+++ b/packages/realm-server/worker.Dockerfile
@@ -0,0 +1,24 @@
+# syntax=docker/dockerfile:1
+
+FROM node:18.6.0-slim
+ARG worker_script
+ENV worker_script=$worker_script
+
+WORKDIR /realm-server
+
+RUN apt-get update && apt-get install -y ca-certificates curl unzip postgresql jq
+RUN npm install -g pnpm@8.10.5
+
+COPY pnpm-lock.yaml ./
+
+COPY patches/ ./patches
+COPY vendor/ ./vendor
+
+ADD . ./
+
+RUN CI=1 pnpm fetch
+RUN CI=1 pnpm install -r --offline
+
+EXPOSE 3000
+
+CMD pnpm --filter "./packages/realm-server" $worker_script
diff --git a/packages/realm-server/worker.ts b/packages/realm-server/worker.ts
index 2c1510cdb5..e4eb00f080 100644
--- a/packages/realm-server/worker.ts
+++ b/packages/realm-server/worker.ts
@@ -17,7 +17,7 @@ let log = logger('worker');
 
 const REALM_SECRET_SEED = process.env.REALM_SECRET_SEED;
 if (!REALM_SECRET_SEED) {
-  console.error(
+  log.error(
     `The REALM_SECRET_SEED environment variable is not set. Please make sure this env var has a value`,
   );
   process.exit(-1);
@@ -29,11 +29,10 @@ if (!REALM_SECRET_SEED) {
 // the '-' is the task ID.
 const ECS_CONTAINER_METADATA_URI = process.env.ECS_CONTAINER_METADATA_URI;
 let workerId = ECS_CONTAINER_METADATA_URI
-  ? ECS_CONTAINER_METADATA_URI.split('/').pop()!
-  : 'realm_worker';
+  ? `${ECS_CONTAINER_METADATA_URI.split('/').pop()!}-pid-${process.pid}`
+  : `worker-pid-${process.pid}`;
 
 let {
-  port,
   matrixURL,
   distURL = process.env.HOST_URL ?? 'http://localhost:4200',
   fromUrl: fromUrls,
@@ -42,11 +41,6 @@ let {
 } = yargs(process.argv.slice(2))
   .usage('Start worker')
   .options({
-    port: {
-      description: 'port number',
-      demandOption: true,
-      type: 'number',
-    },
     fromUrl: {
       description: 'the source of the realm URL proxy',
       demandOption: true,
@@ -68,17 +62,17 @@ let {
       type: 'boolean',
     },
     matrixURL: {
-      description: 'The matrix homeserver for the realm',
+      description: 'The matrix homeserver for the realm server',
       demandOption: true,
       type: 'string',
     },
   })
   .parseSync();
 
-log.info(`starting worker for port ${port}`);
+log.info(`starting worker with pid ${process.pid}`);
 
 if (fromUrls.length !== toUrls.length) {
-  console.error(
+  log.error(
     `Mismatched number of URLs, the --fromUrl params must be matched to the --toUrl params`,
   );
   process.exit(-1);
@@ -89,8 +83,8 @@ let virtualNetwork = new VirtualNetwork();
 shimExternals(virtualNetwork);
 
 let urlMappings = fromUrls.map((fromUrl, i) => [
-  new URL(String(fromUrl), `http://localhost:${port}`),
-  new URL(String(toUrls[i]), `http://localhost:${port}`),
+  new URL(String(fromUrl)),
+  new URL(String(toUrls[i])),
 ]);
 for (let [from, to] of urlMappings) {
   virtualNetwork.addURLMapping(from, to);
@@ -117,14 +111,14 @@ let autoMigrate = migrateDB || undefined;
   });
 
   await worker.run();
+  log.info(`worker started`);
   if (process.send) {
-    log.info(`worker on port ${port} is ready`);
     process.send('ready');
   }
 })().catch((e: any) => {
   Sentry.captureException(e);
-  console.error(
-    `worker on port ${port}: Unexpected error encountered starting realm, stopping server`,
+  log.error(
+    `worker: Unexpected error encountered starting worker, stopping worker`,
     e,
   );
   process.exit(1);
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index f379694f4e..4e78ed74e0 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -1813,6 +1813,9 @@ importers:
       '@types/node':
         specifier: ^18.18.5
         version: 18.18.5
+      '@types/pluralize':
+        specifier: ^0.0.30
+        version: 0.0.30
       '@types/qs':
         specifier: ^6.9.17
         version: 6.9.17
@@ -1897,6 +1900,9 @@ importers:
       npm-run-all:
         specifier: ^4.1.5
         version: 4.1.5
+      pluralize:
+        specifier: ^8.0.0
+        version: 8.0.0
       prettier:
         specifier: github:cardstack/prettier#glimmer-style-tag-in-template-support
         version: github.com/cardstack/prettier/60eccfdc598d682a931d3c569ffb0c4f92ef5db6

From e0a83a933bec73204714b977c65d38b937b02103 Mon Sep 17 00:00:00 2001
From: Hassan Abdel-Rahman <hassan.abdelrahman@gmail.com>
Date: Mon, 13 Jan 2025 17:25:20 -0500
Subject: [PATCH 02/13] update start-base script with full URL and cleanup
 comments

---
 README.md                                        |  2 +-
 packages/matrix/helpers/isolated-realm-server.ts | 10 +++++-----
 packages/realm-server/scripts/start-base.sh      |  2 +-
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/README.md b/README.md
index a6f4a7e816..65e7bfc633 100644
--- a/README.md
+++ b/README.md
@@ -84,7 +84,7 @@ Instead of running `pnpm start:base`, you can alternatively use `pnpm start:all`
 | :4205 | `/test` realm for matrix client tests (playwright controlled) | 🚫                  | 🚫                   |
 | :4210 | Development Worker Manager (spins up 1 worker by default)     | ✅                  | 🚫                   |
 | :4211 | Test Worker Manager (spins up 1 worker by default)            | ✅                  | 🚫                   |
-| :4212 | Test Worker Manager for matrix client tests (playwright controlled - 1 worker) | ✅                  | 🚫                   |
+| :4212 | Test Worker Manager for matrix client tests (playwright controlled - 1 worker) | ✅ | 🚫                   |
 | :5001 | Mail user interface for viewing emails sent to local SMTP     | ✅                  | 🚫                   |
 | :5435 | Postgres DB                                                   | ✅                  | 🚫                   |
 | :8008 | Matrix synapse server                                         | ✅                  | 🚫                   |
diff --git a/packages/matrix/helpers/isolated-realm-server.ts b/packages/matrix/helpers/isolated-realm-server.ts
index c72c61ccfa..0e82b60494 100644
--- a/packages/matrix/helpers/isolated-realm-server.ts
+++ b/packages/matrix/helpers/isolated-realm-server.ts
@@ -31,7 +31,7 @@ export async function startServer() {
   process.env.MATRIX_URL = 'http://localhost:8008';
   process.env.REALM_SERVER_MATRIX_USERNAME = 'realm_server';
 
-  let worker = spawn(
+  let workerManager = spawn(
     'ts-node',
     [
       `--transpileOnly`,
@@ -50,13 +50,13 @@ export async function startServer() {
       stdio: ['pipe', 'pipe', 'pipe', 'ipc'],
     },
   );
-  if (worker.stdout) {
-    worker.stdout.on('data', (data: Buffer) =>
+  if (workerManager.stdout) {
+    workerManager.stdout.on('data', (data: Buffer) =>
       console.log(`worker: ${data.toString()}`),
     );
   }
-  if (worker.stderr) {
-    worker.stderr.on('data', (data: Buffer) =>
+  if (workerManager.stderr) {
+    workerManager.stderr.on('data', (data: Buffer) =>
       console.error(`worker: ${data.toString()}`),
     );
   }
diff --git a/packages/realm-server/scripts/start-base.sh b/packages/realm-server/scripts/start-base.sh
index b58d55d57a..45442b474a 100755
--- a/packages/realm-server/scripts/start-base.sh
+++ b/packages/realm-server/scripts/start-base.sh
@@ -26,4 +26,4 @@ NODE_ENV=development \
   --path='../base' \
   --username='base_realm' \
   --fromUrl='https://cardstack.com/base/' \
-  --toUrl='/base/'
+  --toUrl='http://localhost:4201/base/'

From 50994bf26936d523ff7c41a77d3fc928dc2ca70e Mon Sep 17 00:00:00 2001
From: Hassan Abdel-Rahman <hassan.abdelrahman@gmail.com>
Date: Mon, 13 Jan 2025 17:43:57 -0500
Subject: [PATCH 03/13] add worker manager start script for matrix

---
 README.md                                     |  1 +
 packages/realm-server/package.json            |  3 ++-
 packages/realm-server/scripts/start-base.sh   |  1 +
 .../start-services-for-matrix-tests.sh        |  4 ++--
 .../scripts/start-without-matrix.sh           |  4 ++--
 .../realm-server/scripts/start-worker-base.sh | 20 +++++++++++++++++++
 6 files changed, 28 insertions(+), 5 deletions(-)
 create mode 100755 packages/realm-server/scripts/start-worker-base.sh

diff --git a/README.md b/README.md
index 65e7bfc633..1391391eb4 100644
--- a/README.md
+++ b/README.md
@@ -85,6 +85,7 @@ Instead of running `pnpm start:base`, you can alternatively use `pnpm start:all`
 | :4210 | Development Worker Manager (spins up 1 worker by default)     | ✅                  | 🚫                   |
 | :4211 | Test Worker Manager (spins up 1 worker by default)            | ✅                  | 🚫                   |
 | :4212 | Test Worker Manager for matrix client tests (playwright controlled - 1 worker) | ✅ | 🚫                   |
+| :4213 | Test Worker Manager for matrix client tests - base realm server (playwright controlled - 1 worker) | ✅ | 🚫                   |
 | :5001 | Mail user interface for viewing emails sent to local SMTP     | ✅                  | 🚫                   |
 | :5435 | Postgres DB                                                   | ✅                  | 🚫                   |
 | :8008 | Matrix synapse server                                         | ✅                  | 🚫                   |
diff --git a/packages/realm-server/package.json b/packages/realm-server/package.json
index 4a3b02add5..84b88d7e8b 100644
--- a/packages/realm-server/package.json
+++ b/packages/realm-server/package.json
@@ -86,7 +86,7 @@
     "setup:seed-in-deployment": "mkdir -p /persistent/seed && rsync --dry-run --itemize-changes --size-only --recursive --delete ../seed-realm/. /persistent/seed/ && rsync --size-only --recursive --delete ../seed-realm/. /persistent/seed/",
     "setup:catalog-in-deployment": "mkdir -p /persistent/catalog && rsync --dry-run --itemize-changes --size-only --recursive --delete ../catalog-realm/. /persistent/catalog/ && rsync --size-only --recursive --delete ../catalog-realm/. /persistent/catalog/",
     "start": "PGPORT=5435 NODE_NO_WARNINGS=1 ts-node --transpileOnly main",
-    "start:base": "./scripts/start-base.sh",
+    "start:base": "./scripts/start-base.sh --workerManagerPort=4213",
     "start:test-realms": "./scripts/start-test-realms.sh --workerManagerPort=4211",
     "start:all": "./scripts/start-all.sh",
     "start:without-matrix": "./scripts/start-without-matrix.sh",
@@ -94,6 +94,7 @@
     "start:development": "./scripts/start-development.sh --workerManagerPort=4210",
     "start:production": "./scripts/start-production.sh",
     "start:worker-development": "./scripts/start-worker-development.sh",
+    "start:worker-base": "./scripts/start-worker-base.sh",
     "start:worker-test": "./scripts/start-worker-test.sh",
     "start:worker-staging": "./scripts/start-worker-staging.sh",
     "start:worker-production": "./scripts/start-worker-production.sh",
diff --git a/packages/realm-server/scripts/start-base.sh b/packages/realm-server/scripts/start-base.sh
index 45442b474a..4bb090604b 100755
--- a/packages/realm-server/scripts/start-base.sh
+++ b/packages/realm-server/scripts/start-base.sh
@@ -22,6 +22,7 @@ NODE_ENV=development \
   --matrixURL='http://localhost:8008' \
   --realmsRootPath='./realms/localhost_4201' \
   --migrateDB \
+  $1 \
   \
   --path='../base' \
   --username='base_realm' \
diff --git a/packages/realm-server/scripts/start-services-for-matrix-tests.sh b/packages/realm-server/scripts/start-services-for-matrix-tests.sh
index e60030f660..ff2d0fa934 100755
--- a/packages/realm-server/scripts/start-services-for-matrix-tests.sh
+++ b/packages/realm-server/scripts/start-services-for-matrix-tests.sh
@@ -1,7 +1,7 @@
 #! /bin/sh
 NODE_NO_WARNINGS=1 start-server-and-test \
-  'run-p start:pg start:base' \
+  'run-p start:pg start:worker-base start:base' \
   'http-get://localhost:4201/base/_readiness-check?acceptHeader=application%2Fvnd.api%2Bjson' \
-  'run-p start:test-realms' \
+  'run-p start:worker-test start:test-realms' \
   'http-get://localhost:4202/test/_readiness-check?acceptHeader=application%2Fvnd.api%2Bjson|http-get://localhost:4202/node-test/_readiness-check?acceptHeader=application%2Fvnd.api%2Bjson' \
   'wait'
diff --git a/packages/realm-server/scripts/start-without-matrix.sh b/packages/realm-server/scripts/start-without-matrix.sh
index 20fee9fb8d..7dea229030 100755
--- a/packages/realm-server/scripts/start-without-matrix.sh
+++ b/packages/realm-server/scripts/start-without-matrix.sh
@@ -1,7 +1,7 @@
 #! /bin/sh
 NODE_NO_WARNINGS=1 start-server-and-test \
-  'run-p start:pg start:development' \
+  'run-p start:pg start:worker-development start:development' \
   'http-get://localhost:4201/base/_readiness-check?acceptHeader=application%2Fvnd.api%2Bjson|http-get://localhost:4201/experiments/_readiness-check?acceptHeader=application%2Fvnd.api%2Bjson' \
-  'run-p start:test-realms' \
+  'run-p start:worker-test start:test-realms' \
   'http-get://localhost:4202/node-test/_readiness-check?acceptHeader=application%2Fvnd.api%2Bjson' \
   'wait'
diff --git a/packages/realm-server/scripts/start-worker-base.sh b/packages/realm-server/scripts/start-worker-base.sh
new file mode 100755
index 0000000000..9325706168
--- /dev/null
+++ b/packages/realm-server/scripts/start-worker-base.sh
@@ -0,0 +1,20 @@
+#! /bin/sh
+SCRIPTS_DIR="$(cd "$(dirname "$0")" && pwd)"
+. "$SCRIPTS_DIR/wait-for-pg.sh"
+
+wait_for_postgres
+
+NODE_ENV=development \
+  NODE_NO_WARNINGS=1 \
+  PGPORT=5435 \
+  PGDATABASE=boxel_base \
+  REALM_SECRET_SEED="shhh! it's a secret" \
+  ts-node \
+  --transpileOnly worker-manager \
+  --count="${WORKER_COUNT:-1}" \
+  --port=4213 \
+  --matrixURL='http://localhost:8008' \
+  --distURL="${HOST_URL:-http://localhost:4200}" \
+  \
+  --fromUrl='https://cardstack.com/base/' \
+  --toUrl='http://localhost:4201/base/'

From 07abcbff19d0536ea358cf9a0ca82bfcedad32ac Mon Sep 17 00:00:00 2001
From: Hassan Abdel-Rahman <hassan.abdelrahman@gmail.com>
Date: Mon, 13 Jan 2025 18:16:50 -0500
Subject: [PATCH 04/13] stop the worker manager in the isolated realm server

---
 .../matrix/helpers/isolated-realm-server.ts   | 36 +++++++++++++++----
 packages/realm-server/worker-manager.ts       | 16 +++++++++
 2 files changed, 45 insertions(+), 7 deletions(-)

diff --git a/packages/matrix/helpers/isolated-realm-server.ts b/packages/matrix/helpers/isolated-realm-server.ts
index 0e82b60494..27466c8788 100644
--- a/packages/matrix/helpers/isolated-realm-server.ts
+++ b/packages/matrix/helpers/isolated-realm-server.ts
@@ -86,6 +86,7 @@ export async function startServer() {
       stdio: ['pipe', 'pipe', 'pipe', 'ipc'],
     },
   );
+  realmServer.unref();
   if (realmServer.stdout) {
     realmServer.stdout.on('data', (data: Buffer) =>
       console.log(`realm server: ${data.toString()}`),
@@ -122,25 +123,36 @@ export async function startServer() {
     );
   }
 
-  return new IsolatedRealmServer(realmServer, testRealmDir);
+  return new IsolatedRealmServer(realmServer, workerManager, testRealmDir);
 }
 
 export class IsolatedRealmServer {
-  private stopped: (() => void) | undefined;
+  private realmServerStopped: (() => void) | undefined;
+  private workerManagerStopped: (() => void) | undefined;
   private sqlResults: ((results: string) => void) | undefined;
   private sqlError: ((error: string) => void) | undefined;
 
   constructor(
     private realmServerProcess: ReturnType<typeof spawn>,
+    private workerManagerProcess: ReturnType<typeof spawn>,
     readonly realmPath: string, // useful for debugging
   ) {
+    workerManagerProcess.on('message', (message) => {
+      if (message === 'stopped') {
+        if (!this.workerManagerStopped) {
+          console.error(`received unprompted worker manager stop`);
+          return;
+        }
+        this.workerManagerStopped();
+      }
+    });
     realmServerProcess.on('message', (message) => {
       if (message === 'stopped') {
-        if (!this.stopped) {
+        if (!this.realmServerStopped) {
           console.error(`received unprompted server stop`);
           return;
         }
-        this.stopped();
+        this.realmServerStopped();
       } else if (
         typeof message === 'string' &&
         message.startsWith('sql-results:')
@@ -180,10 +192,20 @@ export class IsolatedRealmServer {
   }
 
   async stop() {
-    let stop = new Promise<void>((r) => (this.stopped = r));
+    let realmServerStop = new Promise<void>(
+      (r) => (this.realmServerStopped = r),
+    );
     this.realmServerProcess.send('stop');
-    await stop;
-    this.stopped = undefined;
+    await realmServerStop;
+    this.realmServerStopped = undefined;
     this.realmServerProcess.send('kill');
+
+    let workerManagerStop = new Promise<void>(
+      (r) => (this.workerManagerStopped = r),
+    );
+    this.workerManagerProcess.send('stop');
+    await workerManagerStop;
+    this.workerManagerStopped = undefined;
+    this.workerManagerProcess.send('kill');
   }
 }
diff --git a/packages/realm-server/worker-manager.ts b/packages/realm-server/worker-manager.ts
index ae54f520f8..6adec036b7 100644
--- a/packages/realm-server/worker-manager.ts
+++ b/packages/realm-server/worker-manager.ts
@@ -77,6 +77,7 @@ if (port != null) {
       console.error(`realm disconnected from worker manager: ${err.message}`);
     });
   });
+  server.unref();
 
   server.listen(port, () => {
     log.info(`worker manager listening for realm on port ${port}`);
@@ -100,6 +101,21 @@ if (port != null) {
     log.error(`Uncaught exception in worker manager:`, err);
     shutdown();
   });
+
+  process.on('message', (message) => {
+    if (message === 'stop') {
+      console.log(`stopping realm server on port ${port}...`);
+      server.close(() => {
+        console.log(`worker manager on port ${port} has stopped`);
+        if (process.send) {
+          process.send('stopped');
+        }
+      });
+    } else if (message === 'kill') {
+      console.log(`Ending worker manager process for ${port}...`);
+      process.exit(0);
+    }
+  });
 }
 
 (async () => {

From 80c033f4526ca614b2d5a1c81560d7664eba310f Mon Sep 17 00:00:00 2001
From: Hassan Abdel-Rahman <hassan.abdelrahman@gmail.com>
Date: Tue, 14 Jan 2025 09:18:12 -0500
Subject: [PATCH 05/13] spawn a new worker if one of the child workers dies

---
 packages/realm-server/worker-manager.ts | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/packages/realm-server/worker-manager.ts b/packages/realm-server/worker-manager.ts
index 6adec036b7..8304453599 100644
--- a/packages/realm-server/worker-manager.ts
+++ b/packages/realm-server/worker-manager.ts
@@ -60,6 +60,10 @@ let {
   .parseSync();
 
 let isReady = false;
+let isExiting = false;
+process.on('SIGINT', () => (isExiting = true));
+process.on('SIGTERM', () => (isExiting = true));
+
 if (port != null) {
   // in tests we start a simple TCP server to communicate to the realm when
   // the worker is ready to start processing jobs
@@ -159,6 +163,13 @@ async function startWorker(urlMappings: URL[][]) {
     },
   );
 
+  worker.on('exit', () => {
+    if (!isExiting) {
+      log.info(`worker ${worker.pid} exited. spawning replacement worker`);
+      startWorker(urlMappings);
+    }
+  });
+
   if (worker.stdout) {
     worker.stdout.on('data', (data: Buffer) =>
       log.info(`[worker ${worker.pid}]: ${data.toString()}`),

From 29d4207d8663c3133487c9f1d2ef687eba824ac8 Mon Sep 17 00:00:00 2001
From: Hassan Abdel-Rahman <hassan.abdelrahman@gmail.com>
Date: Tue, 14 Jan 2025 09:30:48 -0500
Subject: [PATCH 06/13] fix incorrect ECR repo in deploy script

---
 .github/workflows/manual-deploy.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/manual-deploy.yml b/.github/workflows/manual-deploy.yml
index 584dc4391a..e885031be7 100644
--- a/.github/workflows/manual-deploy.yml
+++ b/.github/workflows/manual-deploy.yml
@@ -83,7 +83,7 @@ jobs:
     uses: cardstack/gh-actions/.github/workflows/docker-ecr.yml@main
     secrets: inherit
     with:
-      repository: "boxel-realm-server-${{ inputs.environment }}"
+      repository: "boxel-worker-${{ inputs.environment }}"
       environment: ${{ inputs.environment }}
       dockerfile: "packages/realm-server/worker.Dockerfile"
       build-args: |

From f222f5fc2cc4a53cbc66ada430638df1e2c65eb3 Mon Sep 17 00:00:00 2001
From: Hassan Abdel-Rahman <hassan.abdelrahman@gmail.com>
Date: Tue, 14 Jan 2025 09:36:10 -0500
Subject: [PATCH 07/13] fix incorrect dep for deploy-worker step in deploy
 script

---
 .github/workflows/manual-deploy.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/manual-deploy.yml b/.github/workflows/manual-deploy.yml
index e885031be7..1f3c91c43c 100644
--- a/.github/workflows/manual-deploy.yml
+++ b/.github/workflows/manual-deploy.yml
@@ -123,7 +123,7 @@ jobs:
 
   deploy-worker:
     name: Deploy worker
-    needs: [build-realm-server, deploy-host, post-migrate-db]
+    needs: [build-worker, deploy-host, post-migrate-db]
     uses: cardstack/gh-actions/.github/workflows/ecs-deploy.yml@main
     secrets: inherit
     with:

From fbb96aa90195db2801612c173e4ccbbc80a790fb Mon Sep 17 00:00:00 2001
From: Hassan Abdel-Rahman <hassan.abdelrahman@gmail.com>
Date: Tue, 14 Jan 2025 09:46:25 -0500
Subject: [PATCH 08/13] set concurrency group correctly

---
 packages/runtime-common/realm-index-updater.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/runtime-common/realm-index-updater.ts b/packages/runtime-common/realm-index-updater.ts
index fbe2aacdc9..c37f40dbf4 100644
--- a/packages/runtime-common/realm-index-updater.ts
+++ b/packages/runtime-common/realm-index-updater.ts
@@ -102,7 +102,7 @@ export class RealmIndexUpdater {
       };
       let job = await this.#queue.publish<FromScratchResult>(
         `from-scratch-index`,
-        'indexing',
+        `indexing:${this.#realm.url}`,
         4 * 60,
         args,
       );
@@ -140,7 +140,7 @@ export class RealmIndexUpdater {
       };
       let job = await this.#queue.publish<IncrementalResult>(
         `incremental-index`,
-        'indexing',
+        `indexing:${this.#realm.url}`,
         4 * 60,
         args,
       );

From 0c1da8b1204c0a0c578956addf13b592e4f0ca5f Mon Sep 17 00:00:00 2001
From: Hassan Abdel-Rahman <hassan.abdelrahman@gmail.com>
Date: Tue, 14 Jan 2025 10:12:50 -0500
Subject: [PATCH 09/13] fix deploy realm server deps

---
 .github/workflows/manual-deploy.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/manual-deploy.yml b/.github/workflows/manual-deploy.yml
index 1f3c91c43c..4958da9b31 100644
--- a/.github/workflows/manual-deploy.yml
+++ b/.github/workflows/manual-deploy.yml
@@ -136,7 +136,7 @@ jobs:
 
   deploy-realm-server:
     name: Deploy realm server
-    needs: [deploy-worker]
+    needs: [deploy-worker, build-realm-server, deploy-host, post-migrate-db]
     uses: cardstack/gh-actions/.github/workflows/ecs-deploy.yml@main
     secrets: inherit
     with:

From fbee5d7642f31d7ad3c0b0e9fc9b61bae021f9bd Mon Sep 17 00:00:00 2001
From: Hassan Abdel-Rahman <hassan.abdelrahman@gmail.com>
Date: Tue, 14 Jan 2025 10:30:54 -0500
Subject: [PATCH 10/13] workaround for service stability wait in deploy

---
 .github/workflows/manual-deploy.yml | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/manual-deploy.yml b/.github/workflows/manual-deploy.yml
index 4958da9b31..bb07ec0073 100644
--- a/.github/workflows/manual-deploy.yml
+++ b/.github/workflows/manual-deploy.yml
@@ -114,12 +114,15 @@ jobs:
       image: ${{ needs.build-pg-migration.outputs.image }}
       wait-for-service-stability: false
 
+  # the wait-for-service-stability flag doesn't seem to work in
+  # aws-actions/amazon-ecs-deploy-task-definition@v2. we keep getting timeouts
+  # waiting for service stability. So we are manually waiting here.
   post-migrate-db:
     name: Wait for db-migration
     needs: [migrate-db]
     runs-on: ubuntu-latest
     steps:
-      - run: sleep 240
+      - run: sleep 180
 
   deploy-worker:
     name: Deploy worker
@@ -134,9 +137,20 @@ jobs:
       image: ${{ needs.build-worker.outputs.image }}
       wait-for-service-stability: false
 
+  # the wait-for-service-stability flag doesn't seem to work in
+  # aws-actions/amazon-ecs-deploy-task-definition@v2. we keep getting timeouts
+  # waiting for service stability. So we are manually waiting here.
+  post-deploy-worker:
+    name: Wait for worker
+    needs: [deploy-worker]
+    runs-on: ubuntu-latest
+    steps:
+      - run: sleep 180
+
   deploy-realm-server:
     name: Deploy realm server
-    needs: [deploy-worker, build-realm-server, deploy-host, post-migrate-db]
+    needs:
+      [post-deploy-worker, build-realm-server, deploy-host, post-migrate-db]
     uses: cardstack/gh-actions/.github/workflows/ecs-deploy.yml@main
     secrets: inherit
     with:

From 7f40d6c874b8d272c4894e85532ff097455dedeb Mon Sep 17 00:00:00 2001
From: Hassan Abdel-Rahman <hassan.abdelrahman@gmail.com>
Date: Tue, 14 Jan 2025 10:42:39 -0500
Subject: [PATCH 11/13] cancel old matrix tests in CI build

---
 .github/workflows/ci.yaml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 13dd2cdfcd..2202c70966 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -99,6 +99,9 @@ jobs:
       matrix:
         shardIndex: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]
         shardTotal: [12]
+    concurrency:
+      group: matrix-client-test-${{ matrix.shardIndex }}-${{ github.head_ref || github.run_id }}
+      cancel-in-progress: true
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/init

From 585b8a78507bbfb87e141ac4d7432f03f1f2beb4 Mon Sep 17 00:00:00 2001
From: Hassan Abdel-Rahman <hassan.abdelrahman@gmail.com>
Date: Tue, 14 Jan 2025 10:53:25 -0500
Subject: [PATCH 12/13] test CI

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index 1391391eb4..b7370c89c7 100644
--- a/README.md
+++ b/README.md
@@ -2,6 +2,7 @@
 
 For a quickstart, see [here](./QUICKSTART.md)
 
+
 ## Setup
 
 - you will want the [Glint](https://marketplace.visualstudio.com/items?itemName=typed-ember.glint-vscode) vscode extension

From 348026bf236f227f10f45a884f045f530eda6ba2 Mon Sep 17 00:00:00 2001
From: Hassan Abdel-Rahman <hassan.abdelrahman@gmail.com>
Date: Tue, 14 Jan 2025 10:54:20 -0500
Subject: [PATCH 13/13] cleanup

---
 README.md | 1 -
 1 file changed, 1 deletion(-)

diff --git a/README.md b/README.md
index b7370c89c7..1391391eb4 100644
--- a/README.md
+++ b/README.md
@@ -2,7 +2,6 @@
 
 For a quickstart, see [here](./QUICKSTART.md)
 
-
 ## Setup
 
 - you will want the [Glint](https://marketplace.visualstudio.com/items?itemName=typed-ember.glint-vscode) vscode extension