diff --git a/.github/workflows/manual-deploy.yml b/.github/workflows/manual-deploy.yml index 37ba35b325..920ab687c3 100644 --- a/.github/workflows/manual-deploy.yml +++ b/.github/workflows/manual-deploy.yml @@ -24,7 +24,7 @@ jobs: dockerfile: "packages/ai-bot/Dockerfile" deploy-ai-bot: - needs: [build-ai-bot] + needs: [build-ai-bot, migrate-db] name: Deploy ai-bot to AWS ECS uses: cardstack/gh-actions/.github/workflows/ecs-deploy.yml@main secrets: inherit @@ -77,9 +77,30 @@ jobs: build-args: | "realm_server_script=start:${{ inputs.environment }}" + build-pg-migration: + name: Build pg-migration Docker image + uses: cardstack/gh-actions/.github/workflows/docker-ecr.yml@main + secrets: inherit + with: + repository: "boxel-pg-migration-${{ inputs.environment }}" + environment: ${{ inputs.environment }} + dockerfile: "packages/postgres/Dockerfile" + + migrate-db: + needs: [build-pg-migration] + name: Deploy and run DB migrations + uses: cardstack/gh-actions/.github/workflows/ecs-deploy.yml@main + secrets: inherit + with: + container-name: "boxel-pg-migration" + environment: ${{ inputs.environment }} + cluster: ${{ inputs.environment }} + service-name: "boxel-pg-migration-${{ inputs.environment }}" + image: ${{ needs.build-pg-migration.outputs.image }} + deploy-realm-server: name: Deploy realm server - needs: [build-realm-server, deploy-host] + needs: [build-realm-server, deploy-host, migrate-db] uses: cardstack/gh-actions/.github/workflows/ecs-deploy.yml@main secrets: inherit with: diff --git a/packages/matrix/helpers/isolated-realm-server.ts b/packages/matrix/helpers/isolated-realm-server.ts index 2b7df2ebe5..71e6b02f04 100644 --- a/packages/matrix/helpers/isolated-realm-server.ts +++ b/packages/matrix/helpers/isolated-realm-server.ts @@ -40,6 +40,7 @@ export async function startServer() { `--matrixURL='http://localhost:8008'`, `--realmsRootPath='${dir.name}'`, `--seedPath='${seedPath}'`, + `--migrateDB`, `--useRegistrationSecretFunction`, `--path='${testRealmDir}'`, diff --git a/packages/postgres/Dockerfile b/packages/postgres/Dockerfile new file mode 100644 index 0000000000..e324fd4628 --- /dev/null +++ b/packages/postgres/Dockerfile @@ -0,0 +1,11 @@ +FROM node:18.6.0-slim +ARG CI=1 +RUN apt-get update && apt-get install -y postgresql +RUN npm install -g pnpm@8.10.5 +WORKDIR /boxel +COPY . . +RUN pnpm install --frozen-lockfile + +WORKDIR /boxel/packages/postgres + +CMD ./node_modules/.bin/node-pg-migrate --check-order false --migrations-table migrations up && sleep infinity diff --git a/packages/postgres/package.json b/packages/postgres/package.json index a8f54ba5aa..d52e3df112 100644 --- a/packages/postgres/package.json +++ b/packages/postgres/package.json @@ -5,14 +5,22 @@ "dependencies": { "@cardstack/runtime-common": "workspace:*", "@sentry/node": "^8.31.0", + "@types/fs-extra": "^9.0.13", "@types/pg": "^8.11.5", + "fs-extra": "^10.1.0", "node-pg-migrate": "^6.2.2", "pg": "^8.11.5" }, "devDependencies": { - "concurrently": "^8.0.1" + "concurrently": "^8.0.1", + "sql-parser-cst": "^0.28.0" }, "scripts": { + "start:pg": "./scripts/start-pg.sh", + "stop:pg": "./scripts/stop-pg.sh", + "migrate": "PGDATABASE=boxel ./scripts/ensure-db-exists.sh && PGPORT=5435 PGDATABASE=boxel PGUSER=postgres node-pg-migrate --migrations-table migrations", + "make-schema": "./scripts/schema-dump.sh", + "drop-db": "docker exec boxel-pg dropdb -U postgres -w", "lint": "concurrently \"pnpm:lint:*(!fix)\" --names \"lint:\"", "lint:fix": "concurrently \"pnpm:lint:*:fix\" --names \"fix:\"", "lint:js": "eslint . --cache", diff --git a/packages/postgres/pg-adapter.ts b/packages/postgres/pg-adapter.ts index d391c44e5a..7780256ff2 100644 --- a/packages/postgres/pg-adapter.ts +++ b/packages/postgres/pg-adapter.ts @@ -25,10 +25,15 @@ type Config = ReturnType; export class PgAdapter implements DBAdapter { #isClosed = false; private pool: Pool; - private started = this.#startClient(); + private started: Promise; private config: Config; - constructor() { + constructor(opts?: { autoMigrate?: true }) { + if (opts?.autoMigrate) { + this.started = this.migrateDb(); + } else { + this.started = Promise.resolve(); + } this.config = config(); let { user, host, database, password, port } = this.config; log.info(`connecting to DB ${this.url}`); @@ -52,10 +57,6 @@ export class PgAdapter implements DBAdapter { return `${user}@${host}:${port}/${database}`; } - async #startClient() { - await this.migrateDb(); - } - async close() { log.info(`closing ${this.url}`); this.#isClosed = true; diff --git a/packages/realm-server/scripts/convert-to-sqlite.ts b/packages/postgres/scripts/convert-to-sqlite.ts similarity index 100% rename from packages/realm-server/scripts/convert-to-sqlite.ts rename to packages/postgres/scripts/convert-to-sqlite.ts diff --git a/packages/realm-server/scripts/ensure-db-exists.sh b/packages/postgres/scripts/ensure-db-exists.sh similarity index 100% rename from packages/realm-server/scripts/ensure-db-exists.sh rename to packages/postgres/scripts/ensure-db-exists.sh diff --git a/packages/postgres/scripts/start-pg.sh b/packages/postgres/scripts/start-pg.sh new file mode 100755 index 0000000000..5fee9072da --- /dev/null +++ b/packages/postgres/scripts/start-pg.sh @@ -0,0 +1,8 @@ +#! /bin/sh +if [ -z "$(docker ps -f name=boxel-pg --all --format '{{.Names}}')" ]; then + # running postgres on port 5435 so it doesn't collide with native postgres + # that may be running on your system + docker run --name boxel-pg -e POSTGRES_HOST_AUTH_METHOD=trust -p 5435:5432 -d postgres:16.3 >/dev/null +else + docker start boxel-pg >/dev/null +fi diff --git a/packages/postgres/scripts/stop-pg.sh b/packages/postgres/scripts/stop-pg.sh new file mode 100755 index 0000000000..6bf33ffcc1 --- /dev/null +++ b/packages/postgres/scripts/stop-pg.sh @@ -0,0 +1,2 @@ +#! /bin/sh +docker stop boxel-pg >/dev/null diff --git a/packages/realm-server/main.ts b/packages/realm-server/main.ts index 9536f15e11..ee8ab31994 100644 --- a/packages/realm-server/main.ts +++ b/packages/realm-server/main.ts @@ -66,6 +66,7 @@ let { username: usernames, useRegistrationSecretFunction, seedPath, + migrateDB, } = yargs(process.argv.slice(2)) .usage('Start realm server') .options({ @@ -118,6 +119,11 @@ let { demandOption: true, type: 'array', }, + migrateDB: { + description: + 'When this flag is set the database will automatically migrate when server is started', + type: 'boolean', + }, useRegistrationSecretFunction: { description: 'The flag should be set when running matrix tests where the synapse instance is torn down and restarted multiple times during the life of the realm server.', @@ -166,10 +172,11 @@ for (let [from, to] of urlMappings) { } let hrefs = urlMappings.map(([from, to]) => [from.href, to.href]); let dist: URL = new URL(distURL); +let autoMigrate = migrateDB || undefined; (async () => { let realms: Realm[] = []; - let dbAdapter = new PgAdapter(); + let dbAdapter = new PgAdapter({ autoMigrate }); let queue = new PgQueuePublisher(dbAdapter); let manager = new RunnerOptionsManager(); let { getIndexHTML } = await makeFastBootIndexRunner( @@ -177,7 +184,7 @@ let dist: URL = new URL(distURL); manager.getOptions.bind(manager), ); - await startWorker(); + await startWorker({ autoMigrate }); for (let [i, path] of paths.entries()) { let url = hrefs[i][0]; @@ -298,7 +305,7 @@ let dist: URL = new URL(distURL); process.exit(-3); }); -async function startWorker() { +async function startWorker(opts?: { autoMigrate?: true }) { let worker = spawn( 'ts-node', [ @@ -307,6 +314,7 @@ async function startWorker() { `--port=${port}`, `--matrixURL='${matrixURL}'`, `--distURL='${distURL}'`, + ...(opts?.autoMigrate ? [`--migrateDB`] : []), ...flattenDeep( urlMappings.map(([from, to]) => [ `--fromUrl='${from}'`, diff --git a/packages/realm-server/package.json b/packages/realm-server/package.json index 28d8df755d..bacef6864f 100644 --- a/packages/realm-server/package.json +++ b/packages/realm-server/package.json @@ -53,7 +53,6 @@ "qs": "^6.13.0", "qunit": "^2.20.0", "sane": "^5.0.1", - "sql-parser-cst": "^0.28.0", "start-server-and-test": "^1.14.0", "supertest": "^6.2.4", "testem": "^3.10.1", @@ -92,10 +91,7 @@ "lint:js": "eslint . --cache", "lint:js:fix": "eslint . --fix", "lint:glint": "glint", - "migrate": "PGDATABASE=boxel ./scripts/ensure-db-exists.sh && PGPORT=5435 PGDATABASE=boxel PGUSER=postgres node-pg-migrate --migrations-table migrations", - "make-schema": "./scripts/schema-dump.sh", - "drop-db": "docker exec boxel-pg dropdb -U postgres -w", - "drop-all-dbs": "./scripts/drop-all-dbs.sh" + "full-reset": "./scripts/full-reset.sh" }, "volta": { "extends": "../../package.json" diff --git a/packages/realm-server/scripts/drop-all-dbs.sh b/packages/realm-server/scripts/full-reset.sh similarity index 97% rename from packages/realm-server/scripts/drop-all-dbs.sh rename to packages/realm-server/scripts/full-reset.sh index 65640e1cf9..6814639bcb 100755 --- a/packages/realm-server/scripts/drop-all-dbs.sh +++ b/packages/realm-server/scripts/full-reset.sh @@ -2,6 +2,7 @@ CURRENT_DIR="$(pwd)" SCRIPTS_DIR="$(cd "$(dirname "$0")" && pwd)" +cd ${SCRIPTS_DIR}/../../postgres pnpm run drop-db boxel pnpm run drop-db boxel_test pnpm run drop-db boxel_base diff --git a/packages/realm-server/scripts/start-base.sh b/packages/realm-server/scripts/start-base.sh index ca291c5cbd..b58d55d57a 100755 --- a/packages/realm-server/scripts/start-base.sh +++ b/packages/realm-server/scripts/start-base.sh @@ -21,6 +21,7 @@ NODE_ENV=development \ --port=4201 \ --matrixURL='http://localhost:8008' \ --realmsRootPath='./realms/localhost_4201' \ + --migrateDB \ \ --path='../base' \ --username='base_realm' \ diff --git a/packages/realm-server/scripts/start-development.sh b/packages/realm-server/scripts/start-development.sh index 61f35635de..b7d0e2267a 100755 --- a/packages/realm-server/scripts/start-development.sh +++ b/packages/realm-server/scripts/start-development.sh @@ -23,6 +23,7 @@ NODE_ENV=development \ --matrixURL='http://localhost:8008' \ --realmsRootPath='./realms/localhost_4201' \ --seedPath='../seed-realm' \ + --migrateDB \ \ --path='../base' \ --username='base_realm' \ diff --git a/packages/realm-server/scripts/start-pg.sh b/packages/realm-server/scripts/start-pg.sh deleted file mode 100755 index 5fee9072da..0000000000 --- a/packages/realm-server/scripts/start-pg.sh +++ /dev/null @@ -1,8 +0,0 @@ -#! /bin/sh -if [ -z "$(docker ps -f name=boxel-pg --all --format '{{.Names}}')" ]; then - # running postgres on port 5435 so it doesn't collide with native postgres - # that may be running on your system - docker run --name boxel-pg -e POSTGRES_HOST_AUTH_METHOD=trust -p 5435:5432 -d postgres:16.3 >/dev/null -else - docker start boxel-pg >/dev/null -fi diff --git a/packages/realm-server/scripts/start-pg.sh b/packages/realm-server/scripts/start-pg.sh new file mode 120000 index 0000000000..6f63d47939 --- /dev/null +++ b/packages/realm-server/scripts/start-pg.sh @@ -0,0 +1 @@ +../../postgres/scripts/start-pg.sh \ No newline at end of file diff --git a/packages/realm-server/scripts/start-test-realms.sh b/packages/realm-server/scripts/start-test-realms.sh index 2689ec89d0..2b3695f4a7 100755 --- a/packages/realm-server/scripts/start-test-realms.sh +++ b/packages/realm-server/scripts/start-test-realms.sh @@ -22,6 +22,7 @@ NODE_ENV=test \ --matrixURL='http://localhost:8008' \ --realmsRootPath='./realms/localhost_4202' \ --matrixRegistrationSecretFile='../matrix/registration_secret.txt' \ + --migrateDB \ \ --path='./tests/cards' \ --username='node-test_realm' \ diff --git a/packages/realm-server/scripts/stop-pg.sh b/packages/realm-server/scripts/stop-pg.sh deleted file mode 100755 index 6bf33ffcc1..0000000000 --- a/packages/realm-server/scripts/stop-pg.sh +++ /dev/null @@ -1,2 +0,0 @@ -#! /bin/sh -docker stop boxel-pg >/dev/null diff --git a/packages/realm-server/scripts/stop-pg.sh b/packages/realm-server/scripts/stop-pg.sh new file mode 120000 index 0000000000..12b3f29fc0 --- /dev/null +++ b/packages/realm-server/scripts/stop-pg.sh @@ -0,0 +1 @@ +../../postgres/scripts/stop-pg.sh \ No newline at end of file diff --git a/packages/realm-server/tests/billing-test.ts b/packages/realm-server/tests/billing-test.ts index e3b0dff9a1..4f5e271625 100644 --- a/packages/realm-server/tests/billing-test.ts +++ b/packages/realm-server/tests/billing-test.ts @@ -82,7 +82,7 @@ module('billing', function (hooks) { hooks.beforeEach(async function () { prepareTestDB(); - dbAdapter = new PgAdapter(); + dbAdapter = new PgAdapter({ autoMigrate: true }); }); hooks.afterEach(async function () { diff --git a/packages/realm-server/tests/helpers/index.ts b/packages/realm-server/tests/helpers/index.ts index 0b66054ae8..b118df6a2a 100644 --- a/packages/realm-server/tests/helpers/index.ts +++ b/packages/realm-server/tests/helpers/index.ts @@ -120,7 +120,7 @@ export function setupDB( const runBeforeHook = async () => { prepareTestDB(); - dbAdapter = new PgAdapter(); + dbAdapter = new PgAdapter({ autoMigrate: true }); publisher = new PgQueuePublisher(dbAdapter); runner = new PgQueueRunner(dbAdapter, 'test-worker'); }; diff --git a/packages/realm-server/tests/index-query-engine-test.ts b/packages/realm-server/tests/index-query-engine-test.ts index d6fc6787b9..489586984b 100644 --- a/packages/realm-server/tests/index-query-engine-test.ts +++ b/packages/realm-server/tests/index-query-engine-test.ts @@ -119,7 +119,7 @@ module('query', function (hooks) { ]); loader = new Loader(fetch, virtualNetwork.resolveImport); - dbAdapter = new PgAdapter(); + dbAdapter = new PgAdapter({ autoMigrate: true }); indexQueryEngine = new IndexQueryEngine(dbAdapter); }); diff --git a/packages/realm-server/tests/index-writer-test.ts b/packages/realm-server/tests/index-writer-test.ts index 37851f2332..7af115c366 100644 --- a/packages/realm-server/tests/index-writer-test.ts +++ b/packages/realm-server/tests/index-writer-test.ts @@ -12,7 +12,7 @@ module('index-writer', function (hooks) { hooks.beforeEach(async function () { prepareTestDB(); - adapter = new PgAdapter(); + adapter = new PgAdapter({ autoMigrate: true }); indexWriter = new IndexWriter(adapter); indexQueryEngine = new IndexQueryEngine(adapter); }); diff --git a/packages/realm-server/tests/queue-test.ts b/packages/realm-server/tests/queue-test.ts index ff06043cd6..7160e5824f 100644 --- a/packages/realm-server/tests/queue-test.ts +++ b/packages/realm-server/tests/queue-test.ts @@ -21,7 +21,7 @@ module('queue', function (hooks) { hooks.beforeEach(async function () { prepareTestDB(); - adapter = new PgAdapter(); + adapter = new PgAdapter({ autoMigrate: true }); publisher = new PgQueuePublisher(adapter); runner = new PgQueueRunner(adapter, 'q1'); await runner.start(); @@ -50,7 +50,7 @@ module('queue', function (hooks) { let runner2: QueueRunner; let adapter2: PgAdapter; nestedHooks.beforeEach(async function () { - adapter2 = new PgAdapter(); + adapter2 = new PgAdapter({ autoMigrate: true }); runner2 = new PgQueueRunner(adapter2, 'q2'); await runner2.start(); diff --git a/packages/realm-server/worker.ts b/packages/realm-server/worker.ts index 7840f4448d..2c1510cdb5 100644 --- a/packages/realm-server/worker.ts +++ b/packages/realm-server/worker.ts @@ -38,6 +38,7 @@ let { distURL = process.env.HOST_URL ?? 'http://localhost:4200', fromUrl: fromUrls, toUrl: toUrls, + migrateDB, } = yargs(process.argv.slice(2)) .usage('Start worker') .options({ @@ -61,6 +62,11 @@ let { 'the URL of a deployed host app. (This can be provided instead of the --distPath)', type: 'string', }, + migrateDB: { + description: + 'When this flag is set the database will automatically migrate when server is started', + type: 'boolean', + }, matrixURL: { description: 'The matrix homeserver for the realm', demandOption: true, @@ -90,9 +96,10 @@ for (let [from, to] of urlMappings) { virtualNetwork.addURLMapping(from, to); } let dist: URL = new URL(distURL); +let autoMigrate = migrateDB || undefined; (async () => { - let dbAdapter = new PgAdapter(); + let dbAdapter = new PgAdapter({ autoMigrate }); let queue = new PgQueueRunner(dbAdapter, workerId); let manager = new RunnerOptionsManager(); let { getRunner } = await makeFastBootIndexRunner( diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0cedcda2c7..8cb6e65905 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1692,9 +1692,15 @@ importers: '@sentry/node': specifier: ^8.31.0 version: 8.31.0 + '@types/fs-extra': + specifier: ^9.0.13 + version: 9.0.13 '@types/pg': specifier: ^8.11.5 version: 8.11.5 + fs-extra: + specifier: ^10.1.0 + version: 10.1.0 node-pg-migrate: specifier: ^6.2.2 version: 6.2.2(pg@8.11.5) @@ -1705,6 +1711,9 @@ importers: concurrently: specifier: ^8.0.1 version: 8.2.2 + sql-parser-cst: + specifier: ^0.28.0 + version: 0.28.0 packages/realm-server: devDependencies: @@ -1858,9 +1867,6 @@ importers: sane: specifier: ^5.0.1 version: 5.0.1 - sql-parser-cst: - specifier: ^0.28.0 - version: 0.28.0 start-server-and-test: specifier: ^1.14.0 version: 1.14.0 @@ -7039,7 +7045,6 @@ packages: resolution: {integrity: sha512-nEnwB++1u5lVDM2UI4c1+5R+FYaKfaAzS4OococimjVm3nQw3TuzH5UNsocrcTBbhnerblyHj4A49qXbIiZdpA==} dependencies: '@types/node': 18.18.5 - dev: true /@types/glob@7.2.0: resolution: {integrity: sha512-ZUxbzKl0IfJILTS6t7ip5fQQM/J3TJYubDm3nMbgubNNYS62eXeUpoLUC8/7fJNiFYHTrGPQn7hspDUzIHX3UA==}