Update deployment to run migrations as a separate step

.github/workflows/manual-deploy.yml

@@ -24,7 +24,7 @@ jobs:
       dockerfile: "packages/ai-bot/Dockerfile"
 
   deploy-ai-bot:
-    needs: [build-ai-bot]
+    needs: [build-ai-bot, migrate-db]
     name: Deploy ai-bot to AWS ECS
     uses: cardstack/gh-actions/.github/workflows/ecs-deploy.yml@main
     secrets: inherit
@@ -77,9 +77,30 @@ jobs:
       build-args: |
         "realm_server_script=start:${{ inputs.environment }}"
 
+  build-pg-migration:
+    name: Build pg-migration Docker image
+    uses: cardstack/gh-actions/.github/workflows/docker-ecr.yml@main
+    secrets: inherit
+    with:
+      repository: "boxel-pg-migration-${{ inputs.environment }}"
+      environment: ${{ inputs.environment }}
+      dockerfile: "packages/postgres/Dockerfile"
+
+  migrate-db:
+    needs: [build-pg-migration]
+    name: Deploy and run DB migrations
+    uses: cardstack/gh-actions/.github/workflows/ecs-deploy.yml@main
+    secrets: inherit
+    with:
+      container-name: "boxel-pg-migration"
+      environment: ${{ inputs.environment }}
+      cluster: ${{ inputs.environment }}
+      service-name: "boxel-pg-migration-${{ inputs.environment }}"
+      image: ${{ needs.build-pg-migration.outputs.image }}
+
   deploy-realm-server:
     name: Deploy realm server
-    needs: [build-realm-server, deploy-host]
+    needs: [build-realm-server, deploy-host, migrate-db]
     uses: cardstack/gh-actions/.github/workflows/ecs-deploy.yml@main
     secrets: inherit
     with:

packages/matrix/helpers/isolated-realm-server.ts

@@ -40,6 +40,7 @@ export async function startServer() {
       `--matrixURL='http://localhost:8008'`,
       `--realmsRootPath='${dir.name}'`,
       `--seedPath='${seedPath}'`,
+      `--migrateDB`,
       `--useRegistrationSecretFunction`,
 
       `--path='${testRealmDir}'`,

packages/postgres/Dockerfile

@@ -0,0 +1,11 @@
+FROM node:18.6.0-slim
+ARG CI=1
+RUN apt-get update && apt-get install -y postgresql
+RUN npm install -g [email protected]
+WORKDIR /boxel
+COPY . .
+RUN pnpm install --frozen-lockfile
+
+WORKDIR /boxel/packages/postgres
+
+CMD ./node_modules/.bin/node-pg-migrate --check-order false --migrations-table migrations up && sleep infinity

packages/postgres/package.json

@@ -5,14 +5,22 @@
   "dependencies": {
     "@cardstack/runtime-common": "workspace:*",
     "@sentry/node": "^8.31.0",
+    "@types/fs-extra": "^9.0.13",
     "@types/pg": "^8.11.5",
+    "fs-extra": "^10.1.0",
     "node-pg-migrate": "^6.2.2",
     "pg": "^8.11.5"
   },
   "devDependencies": {
-    "concurrently": "^8.0.1"
+    "concurrently": "^8.0.1",
+    "sql-parser-cst": "^0.28.0"
   },
   "scripts": {
+    "start:pg": "./scripts/start-pg.sh",
+    "stop:pg": "./scripts/stop-pg.sh",
+    "migrate": "PGDATABASE=boxel ./scripts/ensure-db-exists.sh && PGPORT=5435 PGDATABASE=boxel PGUSER=postgres node-pg-migrate --migrations-table migrations",
+    "make-schema": "./scripts/schema-dump.sh",
+    "drop-db": "docker exec boxel-pg dropdb -U postgres -w",
     "lint": "concurrently \"pnpm:lint:*(!fix)\" --names \"lint:\"",
     "lint:fix": "concurrently \"pnpm:lint:*:fix\" --names \"fix:\"",
     "lint:js": "eslint . --cache",

packages/postgres/pg-adapter.ts

@@ -25,10 +25,15 @@ type Config = ReturnType<typeof config>;
 export class PgAdapter implements DBAdapter {
   #isClosed = false;
   private pool: Pool;
-  private started = this.#startClient();
+  private started: Promise<void>;
   private config: Config;
 
-  constructor() {
+  constructor(opts?: { autoMigrate?: true }) {
+    if (opts?.autoMigrate) {
+      this.started = this.migrateDb();
+    } else {
+      this.started = Promise.resolve();
+    }
     this.config = config();
     let { user, host, database, password, port } = this.config;
     log.info(`connecting to DB ${this.url}`);
@@ -52,10 +57,6 @@ export class PgAdapter implements DBAdapter {
     return `${user}@${host}:${port}/${database}`;
   }
 
-  async #startClient() {
-    await this.migrateDb();
-  }
-
   async close() {
     log.info(`closing ${this.url}`);
     this.#isClosed = true;

packages/postgres/scripts/start-pg.sh

@@ -0,0 +1,8 @@
+#! /bin/sh
+if [ -z "$(docker ps -f name=boxel-pg --all --format '{{.Names}}')" ]; then
+  # running postgres on port 5435 so it doesn't collide with native postgres
+  # that may be running on your system
+  docker run --name boxel-pg -e POSTGRES_HOST_AUTH_METHOD=trust -p 5435:5432 -d postgres:16.3 >/dev/null
+else
+  docker start boxel-pg >/dev/null
+fi

packages/postgres/scripts/stop-pg.sh

@@ -0,0 +1,2 @@
+#! /bin/sh
+docker stop boxel-pg >/dev/null

packages/realm-server/main.ts

@@ -66,6 +66,7 @@ let {
   username: usernames,
   useRegistrationSecretFunction,
   seedPath,
+  migrateDB,
 } = yargs(process.argv.slice(2))
   .usage('Start realm server')
   .options({
@@ -118,6 +119,11 @@ let {
       demandOption: true,
       type: 'array',
     },
+    migrateDB: {
+      description:
+        'When this flag is set the database will automatically migrate when server is started',
+      type: 'boolean',
+    },
     useRegistrationSecretFunction: {
       description:
         'The flag should be set when running matrix tests where the synapse instance is torn down and restarted multiple times during the life of the realm server.',
@@ -166,18 +172,19 @@ for (let [from, to] of urlMappings) {
 }
 let hrefs = urlMappings.map(([from, to]) => [from.href, to.href]);
 let dist: URL = new URL(distURL);
+let autoMigrate = migrateDB || undefined;
 
 (async () => {
   let realms: Realm[] = [];
-  let dbAdapter = new PgAdapter();
+  let dbAdapter = new PgAdapter({ autoMigrate });
   let queue = new PgQueuePublisher(dbAdapter);
   let manager = new RunnerOptionsManager();
   let { getIndexHTML } = await makeFastBootIndexRunner(
     dist,
     manager.getOptions.bind(manager),
   );
 
-  await startWorker();
+  await startWorker({ autoMigrate });
 
   for (let [i, path] of paths.entries()) {
     let url = hrefs[i][0];
@@ -298,7 +305,7 @@ let dist: URL = new URL(distURL);
   process.exit(-3);
 });
 
-async function startWorker() {
+async function startWorker(opts?: { autoMigrate?: true }) {
   let worker = spawn(
     'ts-node',
     [
@@ -307,6 +314,7 @@ async function startWorker() {
       `--port=${port}`,
       `--matrixURL='${matrixURL}'`,
       `--distURL='${distURL}'`,
+      ...(opts?.autoMigrate ? [`--migrateDB`] : []),
       ...flattenDeep(
         urlMappings.map(([from, to]) => [
           `--fromUrl='${from}'`,

packages/realm-server/package.json

@@ -53,7 +53,6 @@
     "qs": "^6.13.0",
     "qunit": "^2.20.0",
     "sane": "^5.0.1",
-    "sql-parser-cst": "^0.28.0",
     "start-server-and-test": "^1.14.0",
     "supertest": "^6.2.4",
     "testem": "^3.10.1",
@@ -92,10 +91,7 @@
     "lint:js": "eslint . --cache",
     "lint:js:fix": "eslint . --fix",
     "lint:glint": "glint",
-    "migrate": "PGDATABASE=boxel ./scripts/ensure-db-exists.sh && PGPORT=5435 PGDATABASE=boxel PGUSER=postgres node-pg-migrate --migrations-table migrations",
-    "make-schema": "./scripts/schema-dump.sh",
-    "drop-db": "docker exec boxel-pg dropdb -U postgres -w",
-    "drop-all-dbs": "./scripts/drop-all-dbs.sh"
+    "full-reset": "./scripts/full-reset.sh"
   },
   "volta": {
     "extends": "../../package.json"

packages/realm-server/scripts/drop-all-dbs.sh → packages/realm-server/scripts/full-reset.sh

@@ -2,6 +2,7 @@
 CURRENT_DIR="$(pwd)"
 SCRIPTS_DIR="$(cd "$(dirname "$0")" && pwd)"
 
+cd ${SCRIPTS_DIR}/../../postgres
 pnpm run drop-db boxel
 pnpm run drop-db boxel_test
 pnpm run drop-db boxel_base

packages/realm-server/scripts/start-base.sh

@@ -21,6 +21,7 @@ NODE_ENV=development \
   --port=4201 \
   --matrixURL='http://localhost:8008' \
   --realmsRootPath='./realms/localhost_4201' \
+  --migrateDB \
   \
   --path='../base' \
   --username='base_realm' \

packages/realm-server/scripts/start-development.sh

@@ -23,6 +23,7 @@ NODE_ENV=development \
   --matrixURL='http://localhost:8008' \
   --realmsRootPath='./realms/localhost_4201' \
   --seedPath='../seed-realm' \
+  --migrateDB \
   \
   --path='../base' \
   --username='base_realm' \

packages/realm-server/scripts/start-pg.sh

@@ -0,0 +1 @@
+../../postgres/scripts/start-pg.sh

packages/realm-server/scripts/start-test-realms.sh

@@ -22,6 +22,7 @@ NODE_ENV=test \
   --matrixURL='http://localhost:8008' \
   --realmsRootPath='./realms/localhost_4202' \
   --matrixRegistrationSecretFile='../matrix/registration_secret.txt' \
+  --migrateDB \
   \
   --path='./tests/cards' \
   --username='node-test_realm' \

packages/realm-server/scripts/stop-pg.sh

@@ -0,0 +1 @@
+../../postgres/scripts/stop-pg.sh

packages/realm-server/tests/billing-test.ts

@@ -82,7 +82,7 @@ module('billing', function (hooks) {
 
   hooks.beforeEach(async function () {
     prepareTestDB();
-    dbAdapter = new PgAdapter();
+    dbAdapter = new PgAdapter({ autoMigrate: true });
   });
 
   hooks.afterEach(async function () {

packages/realm-server/tests/helpers/index.ts

@@ -120,7 +120,7 @@ export function setupDB(
 
   const runBeforeHook = async () => {
     prepareTestDB();
-    dbAdapter = new PgAdapter();
+    dbAdapter = new PgAdapter({ autoMigrate: true });
     publisher = new PgQueuePublisher(dbAdapter);
     runner = new PgQueueRunner(dbAdapter, 'test-worker');
   };

packages/realm-server/tests/index-query-engine-test.ts

@@ -119,7 +119,7 @@ module('query', function (hooks) {
     ]);
     loader = new Loader(fetch, virtualNetwork.resolveImport);
 
-    dbAdapter = new PgAdapter();
+    dbAdapter = new PgAdapter({ autoMigrate: true });
     indexQueryEngine = new IndexQueryEngine(dbAdapter);
   });
 

packages/realm-server/tests/index-writer-test.ts

@@ -12,7 +12,7 @@ module('index-writer', function (hooks) {
 
   hooks.beforeEach(async function () {
     prepareTestDB();
-    adapter = new PgAdapter();
+    adapter = new PgAdapter({ autoMigrate: true });
     indexWriter = new IndexWriter(adapter);
     indexQueryEngine = new IndexQueryEngine(adapter);
   });

packages/realm-server/tests/queue-test.ts

@@ -21,7 +21,7 @@ module('queue', function (hooks) {
 
   hooks.beforeEach(async function () {
     prepareTestDB();
-    adapter = new PgAdapter();
+    adapter = new PgAdapter({ autoMigrate: true });
     publisher = new PgQueuePublisher(adapter);
     runner = new PgQueueRunner(adapter, 'q1');
     await runner.start();
@@ -50,7 +50,7 @@ module('queue', function (hooks) {
     let runner2: QueueRunner;
     let adapter2: PgAdapter;
     nestedHooks.beforeEach(async function () {
-      adapter2 = new PgAdapter();
+      adapter2 = new PgAdapter({ autoMigrate: true });
       runner2 = new PgQueueRunner(adapter2, 'q2');
       await runner2.start();
 

packages/realm-server/worker.ts

@@ -38,6 +38,7 @@ let {
   distURL = process.env.HOST_URL ?? 'http://localhost:4200',
   fromUrl: fromUrls,
   toUrl: toUrls,
+  migrateDB,
 } = yargs(process.argv.slice(2))
   .usage('Start worker')
   .options({
@@ -61,6 +62,11 @@ let {
         'the URL of a deployed host app. (This can be provided instead of the --distPath)',
       type: 'string',
     },
+    migrateDB: {
+      description:
+        'When this flag is set the database will automatically migrate when server is started',
+      type: 'boolean',
+    },
     matrixURL: {
       description: 'The matrix homeserver for the realm',
       demandOption: true,
@@ -90,9 +96,10 @@ for (let [from, to] of urlMappings) {
   virtualNetwork.addURLMapping(from, to);
 }
 let dist: URL = new URL(distURL);
+let autoMigrate = migrateDB || undefined;
 
 (async () => {
-  let dbAdapter = new PgAdapter();
+  let dbAdapter = new PgAdapter({ autoMigrate });
   let queue = new PgQueueRunner(dbAdapter, workerId);
   let manager = new RunnerOptionsManager();
   let { getRunner } = await makeFastBootIndexRunner(