From 423527818e5205be1159356d7680768cdcabae86 Mon Sep 17 00:00:00 2001 From: Shubham Padia Date: Sat, 6 Apr 2024 20:02:42 +0530 Subject: [PATCH] Databases: Support postgres schema in dev environment. Fixes #1818. This commit adds postgres schema support to the app logic. The dev environment uses synchronize function to create tables, and does not run the explicit migrations. We will add schema support for production in the next commit. --- .env.example | 3 +++ lib/env.ts | 3 +++ npm/src/db/constants.ts | 1 + npm/src/db/defaultDb.ts | 3 +++ npm/src/db/sql/sql.ts | 24 +++++++++++++++++++++--- npm/src/typings.ts | 3 +++ npm/test/db/db.test.ts | 37 +++++++++++++++++++++++++++++++++++-- 7 files changed, 69 insertions(+), 5 deletions(-) create mode 100644 npm/src/db/constants.ts diff --git a/.env.example b/.env.example index ef95f75289..b8122675c8 100644 --- a/.env.example +++ b/.env.example @@ -20,6 +20,9 @@ DB_PAGE_LIMIT=50 DB_ENCRYPTION_KEY= # Uncomment below if you wish to run DB migrations manually. #DB_MANUAL_MIGRATION=true +# Specify postgres schema to use. In production, you will need to create the schema first +# to use this option. Jackson will not create it for you. +POSTGRES_SCHEMA= # Admin Portal settings # SMTP details for Magic Links diff --git a/lib/env.ts b/lib/env.ts index 80aa50aec8..af2c3786fa 100644 --- a/lib/env.ts +++ b/lib/env.ts @@ -50,6 +50,9 @@ const db: DatabaseOption = { writeCapacityUnits: process.env.DB_DYNAMODB_RCUS ? Number(process.env.DB_DYNAMODB_WCUS) : undefined, }, manualMigration: process.env.DB_MANUAL_MIGRATION === 'true', + postgres: { + schema: process.env.POSTGRES_SCHEMA, + }, }; /** Indicates if the Jackson instance is hosted (i.e. not self-hosted) */ diff --git a/npm/src/db/constants.ts b/npm/src/db/constants.ts new file mode 100644 index 0000000000..93bed1e31c --- /dev/null +++ b/npm/src/db/constants.ts @@ -0,0 +1 @@ +export const DEFAULT_POSTGRES_SCHEMA = 'public'; diff --git a/npm/src/db/defaultDb.ts b/npm/src/db/defaultDb.ts index 6b935d7267..aab4d1e2c4 100644 --- a/npm/src/db/defaultDb.ts +++ b/npm/src/db/defaultDb.ts @@ -1,4 +1,5 @@ import { JacksonOption } from '../typings'; +import { DEFAULT_POSTGRES_SCHEMA } from './constants'; export default function defaultDb(opts: JacksonOption) { opts.db = opts.db || {}; @@ -12,6 +13,8 @@ export default function defaultDb(opts: JacksonOption) { opts.db.dynamodb.readCapacityUnits = opts.db.dynamodb.readCapacityUnits || 5; opts.db.dynamodb.writeCapacityUnits = opts.db.dynamodb.writeCapacityUnits || 5; opts.db.manualMigration = opts.db.manualMigration || false; + opts.db.postgres = opts.db.postgres || {}; + opts.db.postgres.schema = opts.db.postgres.schema || DEFAULT_POSTGRES_SCHEMA; return opts; } diff --git a/npm/src/db/sql/sql.ts b/npm/src/db/sql/sql.ts index eee5a494ea..bc5990ae21 100644 --- a/npm/src/db/sql/sql.ts +++ b/npm/src/db/sql/sql.ts @@ -6,8 +6,9 @@ import { DatabaseDriver, DatabaseOption, Index, Encrypted, Records, SortOrder } import { DataSource, DataSourceOptions, In, IsNull } from 'typeorm'; import * as dbutils from '../utils'; import * as mssql from './mssql'; +import { DEFAULT_POSTGRES_SCHEMA } from '../constants'; -class Sql implements DatabaseDriver { +export class Sql implements DatabaseDriver { private options: DatabaseOption; private dataSource!: DataSource; private storeRepository; @@ -26,6 +27,7 @@ class Sql implements DatabaseDriver { async init({ JacksonStore, JacksonIndex, JacksonTTL }): Promise { const sqlType = this.options.engine === 'planetscale' ? 'mysql' : this.options.type!; + const postgresSchema = this.options.postgres?.schema || DEFAULT_POSTGRES_SCHEMA; // Synchronize by default for non-planetscale engines only if migrations are not set to run let synchronize = !this.options.manualMigration; if (this.options.engine === 'planetscale') { @@ -53,14 +55,30 @@ class Sql implements DatabaseDriver { ...baseOpts, }); } else { - this.dataSource = new DataSource({ + const dataSourceOptions = { url: this.options.url, ssl: this.options.ssl, ...baseOpts, - }); + }; + + if (sqlType === 'postgres') { + dataSourceOptions['synchronize'] = false; + dataSourceOptions['schema'] = postgresSchema; + } + this.dataSource = new DataSource(dataSourceOptions); } + await this.dataSource.initialize(); + if (sqlType === 'postgres' && synchronize) { + // We skip synchronization for postgres databases because TypeORM + // does not create schemas if they don't exist, we manually run + // synchronize here if it is set to true. + const queryRunner = this.dataSource.createQueryRunner(); + await queryRunner.query(`CREATE SCHEMA IF NOT EXISTS ${postgresSchema}`); + this.dataSource.synchronize(); + } + break; } catch (err) { console.error(`error connecting to engine: ${this.options.engine}, type: ${sqlType} db: ${err}`); diff --git a/npm/src/typings.ts b/npm/src/typings.ts index 4835d23f00..469c7f6386 100644 --- a/npm/src/typings.ts +++ b/npm/src/typings.ts @@ -418,6 +418,9 @@ export interface DatabaseOption { writeCapacityUnits?: number; }; manualMigration?: boolean; + postgres?: { + schema?: string; + }; } export interface JacksonOption { diff --git a/npm/test/db/db.test.ts b/npm/test/db/db.test.ts index fe376b463e..008cc6de33 100644 --- a/npm/test/db/db.test.ts +++ b/npm/test/db/db.test.ts @@ -9,6 +9,7 @@ const dbObjs: { [key: string]: DatabaseDriver } = {}; const connectionStores: Storable[] = []; const ttlStores: Storable[] = []; const ttl = 2; +const non_default_schema = 'non_default'; const record1 = { id: '1', @@ -130,6 +131,12 @@ const dbs = [ ...postgresDbConfig, encryptionKey, }, + { + ...postgresDbConfig, + postgres: { + schema: non_default_schema, + }, + }, { ...mongoDbConfig, }, @@ -188,7 +195,11 @@ tap.before(async () => { for (const idx in dbs) { const opts = dbs[idx]; const db = await DB.new(opts, true); - dbObjs[opts.engine! + (opts.type ? opts.type : '')] = db; + if (opts.type === 'postgres' && opts['schema'] === non_default_schema) { + dbObjs[opts['schema'] + opts.engine! + (opts.type ? opts.type : '')] = db; + } else { + dbObjs[opts.engine! + (opts.type ? opts.type : '')] = db; + } const randomSession = Date.now(); connectionStores.push(db.store('saml:config:' + randomSession + randomBytes(4).toString('hex'))); @@ -201,15 +212,32 @@ tap.teardown(async () => { }); tap.test('dbs', async () => { + // We need this to ensure that the test runs atleast once. + // It is quite easy to skip the test by mistake in the future + // if one of the conditions change and it goes unnoticed. + let has_non_default_postgres_schema_test_ran = false; for (const idx in connectionStores) { const connectionStore = connectionStores[idx]; const ttlStore = ttlStores[idx]; const dbEngine = dbs[idx].engine!; - let dbType = dbEngine; + let dbType = dbEngine.toString(); if (dbs[idx].type) { dbType += ': ' + dbs[idx].type; } + tap.test('Test non default postgres schema', (t) => { + if (dbType === 'sql: postgres' && dbs[idx].postgres?.schema === non_default_schema) { + t.same( + connectionStore['db']['db']['dataSource']['createQueryBuilder']()['connection']['options'][ + 'schema' + ], + non_default_schema + ); + } + has_non_default_postgres_schema_test_ran = true; + t.end(); + }); + tap.test('put(): ' + dbType, async () => { await connectionStore.put( record1.id, @@ -527,4 +555,9 @@ tap.test('dbs', async () => { await value.close(); } }); + + tap.test('Ensure that the test for non default postgres schema has ran atleast once', (t) => { + t.same(has_non_default_postgres_schema_test_ran, true); + t.end(); + }); });