From 9a773234113899e622f3c7979a4d0cc8e48fdc09 Mon Sep 17 00:00:00 2001
From: Hassan Abdel-Rahman <hassan.abdelrahman@gmail.com>
Date: Fri, 8 Nov 2024 13:20:23 -0500
Subject: [PATCH 01/12] Update deployment to run migrations as a separate step

---
 .github/workflows/manual-deploy.yml           | 25 +++++++++++++++++--
 packages/postgres/Dockerfile                  | 10 ++++++++
 packages/postgres/package.json                | 10 +++++++-
 packages/postgres/pg-adapter.ts               | 13 +++++-----
 .../scripts/convert-to-sqlite.ts              |  0
 .../scripts/ensure-db-exists.sh               |  0
 packages/postgres/scripts/start-pg.sh         |  8 ++++++
 packages/postgres/scripts/stop-pg.sh          |  2 ++
 .../postgres/scripts/verify-migrations.sh     |  1 +
 packages/realm-server/main.ts                 |  9 ++++++-
 packages/realm-server/package.json            |  4 ---
 packages/realm-server/scripts/drop-all-dbs.sh |  1 +
 packages/realm-server/scripts/start-base.sh   |  1 +
 .../realm-server/scripts/start-development.sh |  1 +
 packages/realm-server/scripts/start-pg.sh     |  9 +------
 .../realm-server/scripts/start-test-realms.sh |  1 +
 packages/realm-server/scripts/stop-pg.sh      |  3 +--
 packages/realm-server/tests/helpers/index.ts  |  2 +-
 pnpm-lock.yaml                                | 13 +++++++---
 19 files changed, 84 insertions(+), 29 deletions(-)
 create mode 100644 packages/postgres/Dockerfile
 rename packages/{realm-server => postgres}/scripts/convert-to-sqlite.ts (100%)
 rename packages/{realm-server => postgres}/scripts/ensure-db-exists.sh (100%)
 create mode 100755 packages/postgres/scripts/start-pg.sh
 create mode 100755 packages/postgres/scripts/stop-pg.sh
 create mode 100755 packages/postgres/scripts/verify-migrations.sh
 mode change 100755 => 120000 packages/realm-server/scripts/start-pg.sh
 mode change 100755 => 120000 packages/realm-server/scripts/stop-pg.sh

diff --git a/.github/workflows/manual-deploy.yml b/.github/workflows/manual-deploy.yml
index 37ba35b325..29fa7cd7a9 100644
--- a/.github/workflows/manual-deploy.yml
+++ b/.github/workflows/manual-deploy.yml
@@ -24,7 +24,7 @@ jobs:
       dockerfile: "packages/ai-bot/Dockerfile"
 
   deploy-ai-bot:
-    needs: [build-ai-bot]
+    needs: [build-ai-bot, migrate-db]
     name: Deploy ai-bot to AWS ECS
     uses: cardstack/gh-actions/.github/workflows/ecs-deploy.yml@main
     secrets: inherit
@@ -77,9 +77,30 @@ jobs:
       build-args: |
         "realm_server_script=start:${{ inputs.environment }}"
 
+  build-pg-migration:
+    name: Build pg-migration Docker image
+    uses: cardstack/gh-actions/.github/workflows/docker-ecr.yml@main
+    secrets: inherit
+    with:
+      repository: "boxel-pg-migration-${{ inputs.environment }}"
+      environment: ${{ inputs.environment }}
+      dockerfile: "packages/postgres/Dockerfile"
+
+  migrate-db:
+    needs: [build-pg-migration]
+    name: Deploy pg-migrations to AWS ECS and run db migrations
+    uses: cardstack/gh-actions/.github/workflows/ecs-deploy.yml@main
+    secrets: inherit
+    with:
+      container-name: "boxel-pg-migration"
+      environment: ${{ inputs.environment }}
+      cluster: ${{ inputs.environment }}
+      service-name: "boxel-pg-migration-${{ inputs.environment }}"
+      image: ${{ needs.build-pg-migration.outputs.image }}
+
   deploy-realm-server:
     name: Deploy realm server
-    needs: [build-realm-server, deploy-host]
+    needs: [build-realm-server, deploy-host, migrate-db]
     uses: cardstack/gh-actions/.github/workflows/ecs-deploy.yml@main
     secrets: inherit
     with:
diff --git a/packages/postgres/Dockerfile b/packages/postgres/Dockerfile
new file mode 100644
index 0000000000..8f0a1d773a
--- /dev/null
+++ b/packages/postgres/Dockerfile
@@ -0,0 +1,10 @@
+FROM node:18.6.0-slim
+ARG CI=1
+RUN apt-get update && apt-get install -y postgresql
+RUN npm install -g pnpm@8.10.5
+WORKDIR /postgres
+COPY . .
+RUN pnpm install --frozen-lockfile
+
+WORKDIR /postgres
+CMD ./node_modules/.bin/node-pg-migrate --migrations-table migrations up
diff --git a/packages/postgres/package.json b/packages/postgres/package.json
index a8f54ba5aa..d52e3df112 100644
--- a/packages/postgres/package.json
+++ b/packages/postgres/package.json
@@ -5,14 +5,22 @@
   "dependencies": {
     "@cardstack/runtime-common": "workspace:*",
     "@sentry/node": "^8.31.0",
+    "@types/fs-extra": "^9.0.13",
     "@types/pg": "^8.11.5",
+    "fs-extra": "^10.1.0",
     "node-pg-migrate": "^6.2.2",
     "pg": "^8.11.5"
   },
   "devDependencies": {
-    "concurrently": "^8.0.1"
+    "concurrently": "^8.0.1",
+    "sql-parser-cst": "^0.28.0"
   },
   "scripts": {
+    "start:pg": "./scripts/start-pg.sh",
+    "stop:pg": "./scripts/stop-pg.sh",
+    "migrate": "PGDATABASE=boxel ./scripts/ensure-db-exists.sh && PGPORT=5435 PGDATABASE=boxel PGUSER=postgres node-pg-migrate --migrations-table migrations",
+    "make-schema": "./scripts/schema-dump.sh",
+    "drop-db": "docker exec boxel-pg dropdb -U postgres -w",
     "lint": "concurrently \"pnpm:lint:*(!fix)\" --names \"lint:\"",
     "lint:fix": "concurrently \"pnpm:lint:*:fix\" --names \"fix:\"",
     "lint:js": "eslint . --cache",
diff --git a/packages/postgres/pg-adapter.ts b/packages/postgres/pg-adapter.ts
index d391c44e5a..7780256ff2 100644
--- a/packages/postgres/pg-adapter.ts
+++ b/packages/postgres/pg-adapter.ts
@@ -25,10 +25,15 @@ type Config = ReturnType<typeof config>;
 export class PgAdapter implements DBAdapter {
   #isClosed = false;
   private pool: Pool;
-  private started = this.#startClient();
+  private started: Promise<void>;
   private config: Config;
 
-  constructor() {
+  constructor(opts?: { autoMigrate?: true }) {
+    if (opts?.autoMigrate) {
+      this.started = this.migrateDb();
+    } else {
+      this.started = Promise.resolve();
+    }
     this.config = config();
     let { user, host, database, password, port } = this.config;
     log.info(`connecting to DB ${this.url}`);
@@ -52,10 +57,6 @@ export class PgAdapter implements DBAdapter {
     return `${user}@${host}:${port}/${database}`;
   }
 
-  async #startClient() {
-    await this.migrateDb();
-  }
-
   async close() {
     log.info(`closing ${this.url}`);
     this.#isClosed = true;
diff --git a/packages/realm-server/scripts/convert-to-sqlite.ts b/packages/postgres/scripts/convert-to-sqlite.ts
similarity index 100%
rename from packages/realm-server/scripts/convert-to-sqlite.ts
rename to packages/postgres/scripts/convert-to-sqlite.ts
diff --git a/packages/realm-server/scripts/ensure-db-exists.sh b/packages/postgres/scripts/ensure-db-exists.sh
similarity index 100%
rename from packages/realm-server/scripts/ensure-db-exists.sh
rename to packages/postgres/scripts/ensure-db-exists.sh
diff --git a/packages/postgres/scripts/start-pg.sh b/packages/postgres/scripts/start-pg.sh
new file mode 100755
index 0000000000..5fee9072da
--- /dev/null
+++ b/packages/postgres/scripts/start-pg.sh
@@ -0,0 +1,8 @@
+#! /bin/sh
+if [ -z "$(docker ps -f name=boxel-pg --all --format '{{.Names}}')" ]; then
+  # running postgres on port 5435 so it doesn't collide with native postgres
+  # that may be running on your system
+  docker run --name boxel-pg -e POSTGRES_HOST_AUTH_METHOD=trust -p 5435:5432 -d postgres:16.3 >/dev/null
+else
+  docker start boxel-pg >/dev/null
+fi
diff --git a/packages/postgres/scripts/stop-pg.sh b/packages/postgres/scripts/stop-pg.sh
new file mode 100755
index 0000000000..6bf33ffcc1
--- /dev/null
+++ b/packages/postgres/scripts/stop-pg.sh
@@ -0,0 +1,2 @@
+#! /bin/sh
+docker stop boxel-pg >/dev/null
diff --git a/packages/postgres/scripts/verify-migrations.sh b/packages/postgres/scripts/verify-migrations.sh
new file mode 100755
index 0000000000..50d48845c0
--- /dev/null
+++ b/packages/postgres/scripts/verify-migrations.sh
@@ -0,0 +1 @@
+#! /bin/sh
diff --git a/packages/realm-server/main.ts b/packages/realm-server/main.ts
index 9536f15e11..ff38114c5f 100644
--- a/packages/realm-server/main.ts
+++ b/packages/realm-server/main.ts
@@ -66,6 +66,7 @@ let {
   username: usernames,
   useRegistrationSecretFunction,
   seedPath,
+  migrateDB,
 } = yargs(process.argv.slice(2))
   .usage('Start realm server')
   .options({
@@ -118,6 +119,11 @@ let {
       demandOption: true,
       type: 'array',
     },
+    migrateDB: {
+      description:
+        'When this flag is set the database will automatically migrate when server is started',
+      type: 'boolean',
+    },
     useRegistrationSecretFunction: {
       description:
         'The flag should be set when running matrix tests where the synapse instance is torn down and restarted multiple times during the life of the realm server.',
@@ -166,10 +172,11 @@ for (let [from, to] of urlMappings) {
 }
 let hrefs = urlMappings.map(([from, to]) => [from.href, to.href]);
 let dist: URL = new URL(distURL);
+let autoMigrate = migrateDB || undefined;
 
 (async () => {
   let realms: Realm[] = [];
-  let dbAdapter = new PgAdapter();
+  let dbAdapter = new PgAdapter({ autoMigrate });
   let queue = new PgQueuePublisher(dbAdapter);
   let manager = new RunnerOptionsManager();
   let { getIndexHTML } = await makeFastBootIndexRunner(
diff --git a/packages/realm-server/package.json b/packages/realm-server/package.json
index 28d8df755d..6d4503cc36 100644
--- a/packages/realm-server/package.json
+++ b/packages/realm-server/package.json
@@ -53,7 +53,6 @@
     "qs": "^6.13.0",
     "qunit": "^2.20.0",
     "sane": "^5.0.1",
-    "sql-parser-cst": "^0.28.0",
     "start-server-and-test": "^1.14.0",
     "supertest": "^6.2.4",
     "testem": "^3.10.1",
@@ -92,9 +91,6 @@
     "lint:js": "eslint . --cache",
     "lint:js:fix": "eslint . --fix",
     "lint:glint": "glint",
-    "migrate": "PGDATABASE=boxel ./scripts/ensure-db-exists.sh && PGPORT=5435 PGDATABASE=boxel PGUSER=postgres node-pg-migrate --migrations-table migrations",
-    "make-schema": "./scripts/schema-dump.sh",
-    "drop-db": "docker exec boxel-pg dropdb -U postgres -w",
     "drop-all-dbs": "./scripts/drop-all-dbs.sh"
   },
   "volta": {
diff --git a/packages/realm-server/scripts/drop-all-dbs.sh b/packages/realm-server/scripts/drop-all-dbs.sh
index 65640e1cf9..6814639bcb 100755
--- a/packages/realm-server/scripts/drop-all-dbs.sh
+++ b/packages/realm-server/scripts/drop-all-dbs.sh
@@ -2,6 +2,7 @@
 CURRENT_DIR="$(pwd)"
 SCRIPTS_DIR="$(cd "$(dirname "$0")" && pwd)"
 
+cd ${SCRIPTS_DIR}/../../postgres
 pnpm run drop-db boxel
 pnpm run drop-db boxel_test
 pnpm run drop-db boxel_base
diff --git a/packages/realm-server/scripts/start-base.sh b/packages/realm-server/scripts/start-base.sh
index ca291c5cbd..b58d55d57a 100755
--- a/packages/realm-server/scripts/start-base.sh
+++ b/packages/realm-server/scripts/start-base.sh
@@ -21,6 +21,7 @@ NODE_ENV=development \
   --port=4201 \
   --matrixURL='http://localhost:8008' \
   --realmsRootPath='./realms/localhost_4201' \
+  --migrateDB \
   \
   --path='../base' \
   --username='base_realm' \
diff --git a/packages/realm-server/scripts/start-development.sh b/packages/realm-server/scripts/start-development.sh
index 61f35635de..b7d0e2267a 100755
--- a/packages/realm-server/scripts/start-development.sh
+++ b/packages/realm-server/scripts/start-development.sh
@@ -23,6 +23,7 @@ NODE_ENV=development \
   --matrixURL='http://localhost:8008' \
   --realmsRootPath='./realms/localhost_4201' \
   --seedPath='../seed-realm' \
+  --migrateDB \
   \
   --path='../base' \
   --username='base_realm' \
diff --git a/packages/realm-server/scripts/start-pg.sh b/packages/realm-server/scripts/start-pg.sh
deleted file mode 100755
index 5fee9072da..0000000000
--- a/packages/realm-server/scripts/start-pg.sh
+++ /dev/null
@@ -1,8 +0,0 @@
-#! /bin/sh
-if [ -z "$(docker ps -f name=boxel-pg --all --format '{{.Names}}')" ]; then
-  # running postgres on port 5435 so it doesn't collide with native postgres
-  # that may be running on your system
-  docker run --name boxel-pg -e POSTGRES_HOST_AUTH_METHOD=trust -p 5435:5432 -d postgres:16.3 >/dev/null
-else
-  docker start boxel-pg >/dev/null
-fi
diff --git a/packages/realm-server/scripts/start-pg.sh b/packages/realm-server/scripts/start-pg.sh
new file mode 120000
index 0000000000..6f63d47939
--- /dev/null
+++ b/packages/realm-server/scripts/start-pg.sh
@@ -0,0 +1 @@
+../../postgres/scripts/start-pg.sh
\ No newline at end of file
diff --git a/packages/realm-server/scripts/start-test-realms.sh b/packages/realm-server/scripts/start-test-realms.sh
index 2689ec89d0..2b3695f4a7 100755
--- a/packages/realm-server/scripts/start-test-realms.sh
+++ b/packages/realm-server/scripts/start-test-realms.sh
@@ -22,6 +22,7 @@ NODE_ENV=test \
   --matrixURL='http://localhost:8008' \
   --realmsRootPath='./realms/localhost_4202' \
   --matrixRegistrationSecretFile='../matrix/registration_secret.txt' \
+  --migrateDB \
   \
   --path='./tests/cards' \
   --username='node-test_realm' \
diff --git a/packages/realm-server/scripts/stop-pg.sh b/packages/realm-server/scripts/stop-pg.sh
deleted file mode 100755
index 6bf33ffcc1..0000000000
--- a/packages/realm-server/scripts/stop-pg.sh
+++ /dev/null
@@ -1,2 +0,0 @@
-#! /bin/sh
-docker stop boxel-pg >/dev/null
diff --git a/packages/realm-server/scripts/stop-pg.sh b/packages/realm-server/scripts/stop-pg.sh
new file mode 120000
index 0000000000..12b3f29fc0
--- /dev/null
+++ b/packages/realm-server/scripts/stop-pg.sh
@@ -0,0 +1 @@
+../../postgres/scripts/stop-pg.sh
\ No newline at end of file
diff --git a/packages/realm-server/tests/helpers/index.ts b/packages/realm-server/tests/helpers/index.ts
index 0ecba1369c..32d9c39574 100644
--- a/packages/realm-server/tests/helpers/index.ts
+++ b/packages/realm-server/tests/helpers/index.ts
@@ -115,7 +115,7 @@ export function setupDB(
 
   const runBeforeHook = async () => {
     prepareTestDB();
-    dbAdapter = new PgAdapter();
+    dbAdapter = new PgAdapter({ autoMigrate: true });
     publisher = new PgQueuePublisher(dbAdapter);
     runner = new PgQueueRunner(dbAdapter, 'test-worker');
   };
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 0cedcda2c7..8cb6e65905 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -1692,9 +1692,15 @@ importers:
       '@sentry/node':
         specifier: ^8.31.0
         version: 8.31.0
+      '@types/fs-extra':
+        specifier: ^9.0.13
+        version: 9.0.13
       '@types/pg':
         specifier: ^8.11.5
         version: 8.11.5
+      fs-extra:
+        specifier: ^10.1.0
+        version: 10.1.0
       node-pg-migrate:
         specifier: ^6.2.2
         version: 6.2.2(pg@8.11.5)
@@ -1705,6 +1711,9 @@ importers:
       concurrently:
         specifier: ^8.0.1
         version: 8.2.2
+      sql-parser-cst:
+        specifier: ^0.28.0
+        version: 0.28.0
 
   packages/realm-server:
     devDependencies:
@@ -1858,9 +1867,6 @@ importers:
       sane:
         specifier: ^5.0.1
         version: 5.0.1
-      sql-parser-cst:
-        specifier: ^0.28.0
-        version: 0.28.0
       start-server-and-test:
         specifier: ^1.14.0
         version: 1.14.0
@@ -7039,7 +7045,6 @@ packages:
     resolution: {integrity: sha512-nEnwB++1u5lVDM2UI4c1+5R+FYaKfaAzS4OococimjVm3nQw3TuzH5UNsocrcTBbhnerblyHj4A49qXbIiZdpA==}
     dependencies:
       '@types/node': 18.18.5
-    dev: true
 
   /@types/glob@7.2.0:
     resolution: {integrity: sha512-ZUxbzKl0IfJILTS6t7ip5fQQM/J3TJYubDm3nMbgubNNYS62eXeUpoLUC8/7fJNiFYHTrGPQn7hspDUzIHX3UA==}

From fa29d5835d9691c7eb6e223aea3ad20fd55398fe Mon Sep 17 00:00:00 2001
From: Hassan Abdel-Rahman <hassan.abdelrahman@gmail.com>
Date: Fri, 8 Nov 2024 13:46:20 -0500
Subject: [PATCH 02/12] update the isolated realm's db to auto migrate

---
 packages/matrix/helpers/isolated-realm-server.ts | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/matrix/helpers/isolated-realm-server.ts b/packages/matrix/helpers/isolated-realm-server.ts
index 2b7df2ebe5..71e6b02f04 100644
--- a/packages/matrix/helpers/isolated-realm-server.ts
+++ b/packages/matrix/helpers/isolated-realm-server.ts
@@ -40,6 +40,7 @@ export async function startServer() {
       `--matrixURL='http://localhost:8008'`,
       `--realmsRootPath='${dir.name}'`,
       `--seedPath='${seedPath}'`,
+      `--migrateDB`,
       `--useRegistrationSecretFunction`,
 
       `--path='${testRealmDir}'`,

From f788e2bdc56e070475ee803b65f74d9ff599df96 Mon Sep 17 00:00:00 2001
From: Hassan Abdel-Rahman <hassan.abdelrahman@gmail.com>
Date: Fri, 8 Nov 2024 14:05:55 -0500
Subject: [PATCH 03/12] auto migrate in realm server tests and add option for
 worker to auto migrate

---
 packages/postgres/scripts/verify-migrations.sh           | 1 -
 packages/realm-server/main.ts                            | 5 +++--
 packages/realm-server/package.json                       | 2 +-
 .../scripts/{drop-all-dbs.sh => full-reset.sh}           | 0
 packages/realm-server/tests/billing-test.ts              | 2 +-
 packages/realm-server/tests/index-query-engine-test.ts   | 2 +-
 packages/realm-server/tests/index-writer-test.ts         | 2 +-
 packages/realm-server/tests/queue-test.ts                | 4 ++--
 packages/realm-server/worker.ts                          | 9 ++++++++-
 9 files changed, 17 insertions(+), 10 deletions(-)
 delete mode 100755 packages/postgres/scripts/verify-migrations.sh
 rename packages/realm-server/scripts/{drop-all-dbs.sh => full-reset.sh} (100%)

diff --git a/packages/postgres/scripts/verify-migrations.sh b/packages/postgres/scripts/verify-migrations.sh
deleted file mode 100755
index 50d48845c0..0000000000
--- a/packages/postgres/scripts/verify-migrations.sh
+++ /dev/null
@@ -1 +0,0 @@
-#! /bin/sh
diff --git a/packages/realm-server/main.ts b/packages/realm-server/main.ts
index ff38114c5f..ee8ab31994 100644
--- a/packages/realm-server/main.ts
+++ b/packages/realm-server/main.ts
@@ -184,7 +184,7 @@ let autoMigrate = migrateDB || undefined;
     manager.getOptions.bind(manager),
   );
 
-  await startWorker();
+  await startWorker({ autoMigrate });
 
   for (let [i, path] of paths.entries()) {
     let url = hrefs[i][0];
@@ -305,7 +305,7 @@ let autoMigrate = migrateDB || undefined;
   process.exit(-3);
 });
 
-async function startWorker() {
+async function startWorker(opts?: { autoMigrate?: true }) {
   let worker = spawn(
     'ts-node',
     [
@@ -314,6 +314,7 @@ async function startWorker() {
       `--port=${port}`,
       `--matrixURL='${matrixURL}'`,
       `--distURL='${distURL}'`,
+      ...(opts?.autoMigrate ? [`--migrateDB`] : []),
       ...flattenDeep(
         urlMappings.map(([from, to]) => [
           `--fromUrl='${from}'`,
diff --git a/packages/realm-server/package.json b/packages/realm-server/package.json
index 6d4503cc36..bacef6864f 100644
--- a/packages/realm-server/package.json
+++ b/packages/realm-server/package.json
@@ -91,7 +91,7 @@
     "lint:js": "eslint . --cache",
     "lint:js:fix": "eslint . --fix",
     "lint:glint": "glint",
-    "drop-all-dbs": "./scripts/drop-all-dbs.sh"
+    "full-reset": "./scripts/full-reset.sh"
   },
   "volta": {
     "extends": "../../package.json"
diff --git a/packages/realm-server/scripts/drop-all-dbs.sh b/packages/realm-server/scripts/full-reset.sh
similarity index 100%
rename from packages/realm-server/scripts/drop-all-dbs.sh
rename to packages/realm-server/scripts/full-reset.sh
diff --git a/packages/realm-server/tests/billing-test.ts b/packages/realm-server/tests/billing-test.ts
index 9894b8a250..10dfd92c92 100644
--- a/packages/realm-server/tests/billing-test.ts
+++ b/packages/realm-server/tests/billing-test.ts
@@ -155,7 +155,7 @@ module('billing', function (hooks) {
 
   hooks.beforeEach(async function () {
     prepareTestDB();
-    dbAdapter = new PgAdapter();
+    dbAdapter = new PgAdapter({ autoMigrate: true });
   });
 
   hooks.afterEach(async function () {
diff --git a/packages/realm-server/tests/index-query-engine-test.ts b/packages/realm-server/tests/index-query-engine-test.ts
index d6fc6787b9..489586984b 100644
--- a/packages/realm-server/tests/index-query-engine-test.ts
+++ b/packages/realm-server/tests/index-query-engine-test.ts
@@ -119,7 +119,7 @@ module('query', function (hooks) {
     ]);
     loader = new Loader(fetch, virtualNetwork.resolveImport);
 
-    dbAdapter = new PgAdapter();
+    dbAdapter = new PgAdapter({ autoMigrate: true });
     indexQueryEngine = new IndexQueryEngine(dbAdapter);
   });
 
diff --git a/packages/realm-server/tests/index-writer-test.ts b/packages/realm-server/tests/index-writer-test.ts
index 37851f2332..7af115c366 100644
--- a/packages/realm-server/tests/index-writer-test.ts
+++ b/packages/realm-server/tests/index-writer-test.ts
@@ -12,7 +12,7 @@ module('index-writer', function (hooks) {
 
   hooks.beforeEach(async function () {
     prepareTestDB();
-    adapter = new PgAdapter();
+    adapter = new PgAdapter({ autoMigrate: true });
     indexWriter = new IndexWriter(adapter);
     indexQueryEngine = new IndexQueryEngine(adapter);
   });
diff --git a/packages/realm-server/tests/queue-test.ts b/packages/realm-server/tests/queue-test.ts
index ff06043cd6..7160e5824f 100644
--- a/packages/realm-server/tests/queue-test.ts
+++ b/packages/realm-server/tests/queue-test.ts
@@ -21,7 +21,7 @@ module('queue', function (hooks) {
 
   hooks.beforeEach(async function () {
     prepareTestDB();
-    adapter = new PgAdapter();
+    adapter = new PgAdapter({ autoMigrate: true });
     publisher = new PgQueuePublisher(adapter);
     runner = new PgQueueRunner(adapter, 'q1');
     await runner.start();
@@ -50,7 +50,7 @@ module('queue', function (hooks) {
     let runner2: QueueRunner;
     let adapter2: PgAdapter;
     nestedHooks.beforeEach(async function () {
-      adapter2 = new PgAdapter();
+      adapter2 = new PgAdapter({ autoMigrate: true });
       runner2 = new PgQueueRunner(adapter2, 'q2');
       await runner2.start();
 
diff --git a/packages/realm-server/worker.ts b/packages/realm-server/worker.ts
index 7840f4448d..2c1510cdb5 100644
--- a/packages/realm-server/worker.ts
+++ b/packages/realm-server/worker.ts
@@ -38,6 +38,7 @@ let {
   distURL = process.env.HOST_URL ?? 'http://localhost:4200',
   fromUrl: fromUrls,
   toUrl: toUrls,
+  migrateDB,
 } = yargs(process.argv.slice(2))
   .usage('Start worker')
   .options({
@@ -61,6 +62,11 @@ let {
         'the URL of a deployed host app. (This can be provided instead of the --distPath)',
       type: 'string',
     },
+    migrateDB: {
+      description:
+        'When this flag is set the database will automatically migrate when server is started',
+      type: 'boolean',
+    },
     matrixURL: {
       description: 'The matrix homeserver for the realm',
       demandOption: true,
@@ -90,9 +96,10 @@ for (let [from, to] of urlMappings) {
   virtualNetwork.addURLMapping(from, to);
 }
 let dist: URL = new URL(distURL);
+let autoMigrate = migrateDB || undefined;
 
 (async () => {
-  let dbAdapter = new PgAdapter();
+  let dbAdapter = new PgAdapter({ autoMigrate });
   let queue = new PgQueueRunner(dbAdapter, workerId);
   let manager = new RunnerOptionsManager();
   let { getRunner } = await makeFastBootIndexRunner(

From 7dd080b6c90d40be5e22f50eec022d1b93884639 Mon Sep 17 00:00:00 2001
From: Hassan Abdel-Rahman <hassan.abdelrahman@gmail.com>
Date: Fri, 8 Nov 2024 15:10:23 -0500
Subject: [PATCH 04/12] add logging to Docker

---
 packages/postgres/Dockerfile | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/packages/postgres/Dockerfile b/packages/postgres/Dockerfile
index 8f0a1d773a..ddb882c9c0 100644
--- a/packages/postgres/Dockerfile
+++ b/packages/postgres/Dockerfile
@@ -1,10 +1,11 @@
 FROM node:18.6.0-slim
 ARG CI=1
-RUN apt-get update && apt-get install -y postgresql
+RUN apt-get update && apt-get install -y postgresql tree
 RUN npm install -g pnpm@8.10.5
 WORKDIR /postgres
 COPY . .
 RUN pnpm install --frozen-lockfile
 
 WORKDIR /postgres
-CMD ./node_modules/.bin/node-pg-migrate --migrations-table migrations up
+
+CMD tree . && ./node_modules/.bin/node-pg-migrate --migrations-table migrations up

From b4531010011b4ef629da9e308952aa4d0f2f6062 Mon Sep 17 00:00:00 2001
From: Hassan Abdel-Rahman <hassan.abdelrahman@gmail.com>
Date: Fri, 8 Nov 2024 15:24:13 -0500
Subject: [PATCH 05/12] fixing WORKDIR in dockerfile

---
 .github/workflows/manual-deploy.yml | 2 +-
 packages/postgres/Dockerfile        | 9 ++++-----
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/manual-deploy.yml b/.github/workflows/manual-deploy.yml
index 29fa7cd7a9..920ab687c3 100644
--- a/.github/workflows/manual-deploy.yml
+++ b/.github/workflows/manual-deploy.yml
@@ -88,7 +88,7 @@ jobs:
 
   migrate-db:
     needs: [build-pg-migration]
-    name: Deploy pg-migrations to AWS ECS and run db migrations
+    name: Deploy and run DB migrations
     uses: cardstack/gh-actions/.github/workflows/ecs-deploy.yml@main
     secrets: inherit
     with:
diff --git a/packages/postgres/Dockerfile b/packages/postgres/Dockerfile
index ddb882c9c0..078690d76f 100644
--- a/packages/postgres/Dockerfile
+++ b/packages/postgres/Dockerfile
@@ -1,11 +1,10 @@
 FROM node:18.6.0-slim
 ARG CI=1
-RUN apt-get update && apt-get install -y postgresql tree
+RUN apt-get update && apt-get install -y postgresql
 RUN npm install -g pnpm@8.10.5
-WORKDIR /postgres
+WORKDIR /boxel
 COPY . .
 RUN pnpm install --frozen-lockfile
 
-WORKDIR /postgres
-
-CMD tree . && ./node_modules/.bin/node-pg-migrate --migrations-table migrations up
+WORKDIR packages/postgres
+CMD ./packages/postgres/node_modules/.bin/node-pg-migrate --migrations-table migrations up

From 91dcb8e458d310145ee55fcdb4e8798df5c47267 Mon Sep 17 00:00:00 2001
From: Hassan Abdel-Rahman <hassan.abdelrahman@gmail.com>
Date: Fri, 8 Nov 2024 15:31:27 -0500
Subject: [PATCH 06/12] more docker debug

---
 packages/postgres/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/postgres/Dockerfile b/packages/postgres/Dockerfile
index 078690d76f..1facf3adaa 100644
--- a/packages/postgres/Dockerfile
+++ b/packages/postgres/Dockerfile
@@ -7,4 +7,4 @@ COPY . .
 RUN pnpm install --frozen-lockfile
 
 WORKDIR packages/postgres
-CMD ./packages/postgres/node_modules/.bin/node-pg-migrate --migrations-table migrations up
+CMD echo "`pwd`" && echo "`ls -la`" &&  ./node_modules/.bin/node-pg-migrate --migrations-table migrations up

From 7a60c87501bebecb4798b560d07c0e8ee3419772 Mon Sep 17 00:00:00 2001
From: Hassan Abdel-Rahman <hassan.abdelrahman@gmail.com>
Date: Mon, 11 Nov 2024 09:39:26 -0500
Subject: [PATCH 07/12] debug dockerfile

---
 packages/postgres/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/postgres/Dockerfile b/packages/postgres/Dockerfile
index 1facf3adaa..911b8809b2 100644
--- a/packages/postgres/Dockerfile
+++ b/packages/postgres/Dockerfile
@@ -7,4 +7,4 @@ COPY . .
 RUN pnpm install --frozen-lockfile
 
 WORKDIR packages/postgres
-CMD echo "`pwd`" && echo "`ls -la`" &&  ./node_modules/.bin/node-pg-migrate --migrations-table migrations up
+# CMD ./node_modules/.bin/node-pg-migrate --migrations-table migrations up

From 60d518625f874ffb6bd42bc78f83b24b412173ec Mon Sep 17 00:00:00 2001
From: Hassan Abdel-Rahman <hassan.abdelrahman@gmail.com>
Date: Mon, 11 Nov 2024 09:49:05 -0500
Subject: [PATCH 08/12] debug docker

---
 packages/postgres/Dockerfile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/postgres/Dockerfile b/packages/postgres/Dockerfile
index 911b8809b2..dc8c22911e 100644
--- a/packages/postgres/Dockerfile
+++ b/packages/postgres/Dockerfile
@@ -8,3 +8,4 @@ RUN pnpm install --frozen-lockfile
 
 WORKDIR packages/postgres
 # CMD ./node_modules/.bin/node-pg-migrate --migrations-table migrations up
+CMD /bin/sh

From 223f14b1475b449ddfbe98117c64ffc03996d899 Mon Sep 17 00:00:00 2001
From: Hassan Abdel-Rahman <hassan.abdelrahman@gmail.com>
Date: Mon, 11 Nov 2024 09:58:37 -0500
Subject: [PATCH 09/12] debug dockerfile

---
 packages/postgres/Dockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/postgres/Dockerfile b/packages/postgres/Dockerfile
index dc8c22911e..85aeebae89 100644
--- a/packages/postgres/Dockerfile
+++ b/packages/postgres/Dockerfile
@@ -6,6 +6,6 @@ WORKDIR /boxel
 COPY . .
 RUN pnpm install --frozen-lockfile
 
-WORKDIR packages/postgres
+WORKDIR /boxel/packages/postgres
 # CMD ./node_modules/.bin/node-pg-migrate --migrations-table migrations up
-CMD /bin/sh
+ENTRYPOINT ["tail", "-f", "/dev/null"]

From 3a1d91ea7388a7832e01bfbd832211098b6744a7 Mon Sep 17 00:00:00 2001
From: Hassan Abdel-Rahman <hassan.abdelrahman@gmail.com>
Date: Mon, 11 Nov 2024 10:12:30 -0500
Subject: [PATCH 10/12] set check-order in migration script

---
 packages/postgres/Dockerfile | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/packages/postgres/Dockerfile b/packages/postgres/Dockerfile
index 85aeebae89..732c5f8823 100644
--- a/packages/postgres/Dockerfile
+++ b/packages/postgres/Dockerfile
@@ -7,5 +7,4 @@ COPY . .
 RUN pnpm install --frozen-lockfile
 
 WORKDIR /boxel/packages/postgres
-# CMD ./node_modules/.bin/node-pg-migrate --migrations-table migrations up
-ENTRYPOINT ["tail", "-f", "/dev/null"]
+CMD ./node_modules/.bin/node-pg-migrate --check-order false --migrations-table migrations up

From c7f5ddd49d8cde4c8eed7b7fed15b16db7bc5f07 Mon Sep 17 00:00:00 2001
From: Hassan Abdel-Rahman <hassan.abdelrahman@gmail.com>
Date: Mon, 11 Nov 2024 10:31:33 -0500
Subject: [PATCH 11/12] don't allow container to exit after migrations are
 finished

---
 packages/postgres/Dockerfile | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/packages/postgres/Dockerfile b/packages/postgres/Dockerfile
index 732c5f8823..de2fe8fde9 100644
--- a/packages/postgres/Dockerfile
+++ b/packages/postgres/Dockerfile
@@ -7,4 +7,6 @@ COPY . .
 RUN pnpm install --frozen-lockfile
 
 WORKDIR /boxel/packages/postgres
-CMD ./node_modules/.bin/node-pg-migrate --check-order false --migrations-table migrations up
+
+# after the migration is done the container is available for manual DB introspection
+CMD ./node_modules/.bin/node-pg-migrate --check-order false --migrations-table migrations up && sleep infinity

From 0291bbe000bb2cf6457bf3b4e2ebdb4ef04b14fb Mon Sep 17 00:00:00 2001
From: Hassan Abdel-Rahman <hassan.abdelrahman@gmail.com>
Date: Mon, 11 Nov 2024 10:42:09 -0500
Subject: [PATCH 12/12] cleanup

---
 packages/postgres/Dockerfile | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/postgres/Dockerfile b/packages/postgres/Dockerfile
index de2fe8fde9..e324fd4628 100644
--- a/packages/postgres/Dockerfile
+++ b/packages/postgres/Dockerfile
@@ -8,5 +8,4 @@ RUN pnpm install --frozen-lockfile
 
 WORKDIR /boxel/packages/postgres
 
-# after the migration is done the container is available for manual DB introspection
 CMD ./node_modules/.bin/node-pg-migrate --check-order false --migrations-table migrations up && sleep infinity