911: Setup migrations

backend/src/main/kotlin/app/ehrenamtskarte/backend/EntryPoint.kt

@@ -4,6 +4,8 @@ import app.ehrenamtskarte.backend.common.database.Database
 import app.ehrenamtskarte.backend.common.webservice.GraphQLHandler
 import app.ehrenamtskarte.backend.common.webservice.WebService
 import app.ehrenamtskarte.backend.config.BackendConfiguration
+import app.ehrenamtskarte.backend.migration.MigrationUtils
+import app.ehrenamtskarte.backend.migration.database.Migrations
 import app.ehrenamtskarte.backend.stores.importer.Importer
 import com.expediagroup.graphql.generator.extensions.print
 import com.github.ajalt.clikt.core.CliktCommand
@@ -17,6 +19,8 @@ import com.github.ajalt.clikt.parameters.options.option
 import com.github.ajalt.clikt.parameters.types.choice
 import com.github.ajalt.clikt.parameters.types.file
 import com.github.ajalt.clikt.parameters.types.int
+import org.jetbrains.exposed.sql.exists
+import org.jetbrains.exposed.sql.transactions.transaction
 import java.io.File
 import java.util.TimeZone
 
@@ -116,9 +120,47 @@ class Execute : CliktCommand(help = "Starts the webserver") {
     }
 }
 
+class Migrate : CliktCommand(help = "Migrates the database") {
+    private val config by requireObject<BackendConfiguration>()
+
+    override fun run() {
+        val db = Database.setupWithoutMigrationCheck(config)
+        MigrationUtils.applyRequiredMigrations(db)
+    }
+}
+
+class MigrateSkipBaseline : CliktCommand(
+    help = """
+    Applies all migrations except for the baseline step.
+
+    This command allows the production system to be upgraded to the new DB migration system.
+    It adds the migrations table without applying the baseline migration step.
+    It should be used only once when introducing the new DB migration system on the production server.
+    Once this is done, this command can be safely removed.
+    """.trimIndent(),
+) {
+    private val config by requireObject<BackendConfiguration>()
+
+    override fun run() {
+        val db = Database.setupWithoutMigrationCheck(config)
+        if (transaction { Migrations.exists() }) {
+            throw IllegalArgumentException("The migrations table has already been created. Use the migrate command instead.")
+        }
+        MigrationUtils.applyRequiredMigrations(db, skipBaseline = true)
+    }
+}
+
 fun main(args: Array<String>) {
     // Set the default time zone to UTC in order to make timestamps work properly in every configuration.
     TimeZone.setDefault(TimeZone.getTimeZone("UTC"))
 
-    Entry().subcommands(Execute(), Import(), ImportSingle(), CreateAdmin(), GraphQLExport()).main(args)
+    Entry().subcommands(
+        Execute(),
+        Import(),
+        ImportSingle(),
+        Migrate(),
+        MigrateSkipBaseline(),
+        CreateAdmin(),
+        GraphQLExport()
+    ).main(args)
 }

backend/src/main/kotlin/app/ehrenamtskarte/backend/auth/database/Schema.kt

@@ -11,7 +11,6 @@ import org.jetbrains.exposed.sql.Op
 import org.jetbrains.exposed.sql.and
 import org.jetbrains.exposed.sql.javatime.timestamp
 import org.jetbrains.exposed.sql.or
-import org.jetbrains.exposed.sql.transactions.TransactionManager
 
 object Administrators : IntIdTable() {
     val email = varchar("email", 100)
@@ -37,11 +36,6 @@ object Administrators : IntIdTable() {
     }
 }
 
-fun createEmailIndexIfNotExists() {
-    val sql = "CREATE UNIQUE INDEX IF NOT EXISTS email_lower_idx ON ${Administrators.nameInDatabaseCase()} (lower(${Administrators.email.nameInDatabaseCase()}))"
-    TransactionManager.current().exec(sql)
-}
-
 class AdministratorEntity(id: EntityID<Int>) : IntEntity(id) {
     companion object : IntEntityClass<AdministratorEntity>(Administrators)
 

backend/src/main/kotlin/app/ehrenamtskarte/backend/common/database/Database.kt

@@ -3,6 +3,11 @@ package app.ehrenamtskarte.backend.common.database
 import app.ehrenamtskarte.backend.auth.database.repos.AdministratorsRepository
 import app.ehrenamtskarte.backend.auth.webservice.schema.types.Role
 import app.ehrenamtskarte.backend.config.BackendConfiguration
+import app.ehrenamtskarte.backend.migration.assertDatabaseIsInSync
+import app.ehrenamtskarte.backend.projects.database.insertOrUpdateProjects
+import app.ehrenamtskarte.backend.regions.database.insertOrUpdateRegions
+import app.ehrenamtskarte.backend.stores.database.createOrReplaceStoreFunctions
+import app.ehrenamtskarte.backend.stores.database.insertOrUpdateCategories
 import org.jetbrains.exposed.sql.Database.Companion.connect
 import org.jetbrains.exposed.sql.DatabaseConfig
 import org.jetbrains.exposed.sql.StdOutSqlLogger
@@ -11,12 +16,6 @@ import org.jetbrains.exposed.sql.transactions.transaction
 import java.io.BufferedReader
 import java.io.InputStreamReader
 import java.util.stream.Collectors
-import app.ehrenamtskarte.backend.application.database.setupDatabase as setupDatabaseForApplication
-import app.ehrenamtskarte.backend.auth.database.setupDatabase as setupDatabaseForAuth
-import app.ehrenamtskarte.backend.projects.database.setupDatabase as setupDatabaseForProjects
-import app.ehrenamtskarte.backend.regions.database.setupDatabase as setupDatabaseForRegions
-import app.ehrenamtskarte.backend.stores.database.setupDatabase as setupDatabaseForStores
-import app.ehrenamtskarte.backend.verification.database.setupDatabase as setupDatabaseForVerification
 
 class Database {
 
@@ -46,8 +45,20 @@ class Database {
             }
         }
 
-        fun setup(config: BackendConfiguration) {
-            connect(
+        fun setup(config: BackendConfiguration): org.jetbrains.exposed.sql.Database {
+            val database = setupWithoutMigrationCheck(config)
+            transaction {
+                assertDatabaseIsInSync()
+                insertOrUpdateProjects(config)
+                insertOrUpdateRegions()
+                insertOrUpdateCategories(Companion::executeScript)
+                createOrReplaceStoreFunctions(Companion::executeScript)
+            }
+            return database
+        }
+
+        fun setupWithoutMigrationCheck(config: BackendConfiguration): org.jetbrains.exposed.sql.Database {
+            val database = connect(
                 config.postgres.url,
                 driver = "org.postgresql.Driver",
                 user = config.postgres.user,
@@ -57,22 +68,15 @@ class Database {
                     // Note(michael-markl): I believe this is postgres specific syntax.
                     it.prepareStatement("SET TIME ZONE 'UTC';").executeUpdate()
                 },
-                databaseConfig = if (config.production) {
-                    null
-                } else {
-                    DatabaseConfig.invoke {
+                databaseConfig = DatabaseConfig.invoke {
+                    // Nested transactions are helpful for applying migrations in subtransactions.
+                    useNestedTransactions = true
+                    if (!config.production) {
                         this.sqlLogger = StdOutSqlLogger
                     }
                 },
             )
-            transaction {
-                setupDatabaseForProjects(config)
-                setupDatabaseForRegions()
-                setupDatabaseForStores(Companion::executeScript)
-                setupDatabaseForVerification()
-                setupDatabaseForApplication()
-                setupDatabaseForAuth()
-            }
+            return database
         }
     }
 }

backend/src/main/kotlin/app/ehrenamtskarte/backend/migration/AssertDatabaseInSync.kt

@@ -0,0 +1,106 @@
+package app.ehrenamtskarte.backend.migration
+
+import app.ehrenamtskarte.backend.migration.database.Migrations
+import app.ehrenamtskarte.backend.migration.migrations.MigrationsRegistry
+import org.jetbrains.exposed.sql.ColumnDiff
+import org.jetbrains.exposed.sql.SchemaUtils
+import org.jetbrains.exposed.sql.Table
+import org.jetbrains.exposed.sql.transactions.TransactionManager
+import org.jetbrains.exposed.sql.vendors.currentDialect
+
+class DatabaseOutOfSyncException(suggestedMigrationStatements: List<String>? = null, comment: String? = null) :
+    Exception(
+        run {
+            var message = ""
+            if (comment != null) {
+                message += "\n$comment"
+            }
+            if (suggestedMigrationStatements != null) {
+                message += "\nThe following migrations are suggested:\n--- START OF SUGGESTED MIGRATIONS\n" +
+                    suggestedMigrationStatements.joinToString("\n") { "$it;" } +
+                    "\n--- END OF SUGGESTED MIGRATIONS"
+            }
+            message
+        },
+    )
+
+fun assertDatabaseIsInSync() {
+    val allTables = TablesRegistry.getAllTables()
+    val allMigrations = MigrationsRegistry.getAllMigrations()
+    val versionDb = Migrations.getCurrentVersionOrNull()
+    val versionCode = allMigrations.maxOfOrNull { it.version }
+    if (versionCode != versionDb) {
+        throw DatabaseOutOfSyncException(comment = "Latest migration versions do not match: Version on DB $versionDb - Code Version $versionCode")
+    }
+
+    var outOfSyncComment: String? = null
+
+    // Check if all tables in the DB appear in `allTables` and vice versa.
+    // We ignore spatial_ref_sys.
+    val tablesInDb =
+        currentDialect.allTablesNames().map { it.substringAfter(".") }
+            .filter { it != "spatial_ref_sys" }.toSet()
+    val tablesInCode = allTables.map { it.nameInDatabaseCase() }.toSet()
+    if (tablesInDb != tablesInCode) {
+        val tablesNotInCode = tablesInDb - tablesInCode
+        val tablesNotInDb = tablesInCode - tablesInDb
+        outOfSyncComment = "List of tables is out sync with database:"
+        if (tablesNotInCode.isNotEmpty()) {
+            outOfSyncComment += "\nUnknown tables found in DB: $tablesNotInCode"
+        }
+        if (tablesNotInDb.isNotEmpty()) {
+            outOfSyncComment += "\nTables missing in DB: $tablesNotInDb"
+        }
+    }
+
+    val statements = statementsRequiredToActualizeScheme(*allTables) + dropExcessiveColumns(*allTables)
+    if (statements.isNotEmpty() || outOfSyncComment != null) {
+        throw DatabaseOutOfSyncException(statements, comment = outOfSyncComment)
+    }
+}
+
+/**
+ * Checks whether there exist any excessive columns for the passed tables.
+ * Returns a list of SQL statements to drop these excessive columns.
+ */
+private fun dropExcessiveColumns(vararg tables: Table): List<String> {
+    val transaction = TransactionManager.current()
+
+    val statements = mutableListOf<String>()
+    val existingColumnsByTable = currentDialect.tableColumns(*tables)
+    for (table in tables) {
+        val columns = existingColumnsByTable[table] ?: continue
+        for (column in columns) {
+            val columnInCode = table.columns.singleOrNull { it.name.equals(column.name, ignoreCase = true) }
+            if (columnInCode == null) {
+                statements += "ALTER TABLE ${transaction.identity(table)} DROP ${column.name}"
+            }
+        }
+    }
+
+    return statements
+}
+
+/**
+ * Workaround for https://github.com/JetBrains/Exposed/issues/1486
+ */
+internal fun statementsRequiredToActualizeScheme(vararg tables: Table): List<String> {
+    val statements = SchemaUtils.statementsRequiredToActualizeScheme(*tables)
+    val existingColumnsByTable = currentDialect.tableColumns(*tables)
+    val allColumns = tables.map { table -> table.columns }.flatten()
+    val problematicColumns = allColumns.filter { column ->
+        val hasDefaultInCode = column.descriptionDdl().contains("DEFAULT (CURRENT_TIMESTAMP)")
+        val existingColumn = existingColumnsByTable[column.table]?.singleOrNull {
+            column.name.equals(it.name, true)
+        }
+        val hasDefaultInDb = existingColumn?.defaultDbValue == "CURRENT_TIMESTAMP"
+        hasDefaultInCode && hasDefaultInDb
+    }
+    val problematicStatements = problematicColumns.map {
+        currentDialect.modifyColumn(
+            it,
+            ColumnDiff(defaults = true, nullability = false, autoInc = false, caseSensitiveName = false),
+        ).single()
+    }
+    return statements.filter { it !in problematicStatements }
+}

backend/src/main/kotlin/app/ehrenamtskarte/backend/migration/Migration.kt

@@ -0,0 +1,23 @@
+package app.ehrenamtskarte.backend.migration
+
+import org.jetbrains.exposed.sql.Transaction
+
+typealias Statement = (Transaction.() -> Unit?)
+
+abstract class Migration {
+
+    val name: String
+    val version: Int
+
+    init {
+        val className = this::class.simpleName!!
+        val groups =
+            Regex("^V(\\d{4})_(.*)").matchEntire(className)?.groupValues ?: throw IllegalArgumentException(
+                "Migration class name $className doesn't match convention.",
+            )
+        version = groups[1].toInt()
+        name = groups[2]
+    }
+
+    abstract val migrate: Statement
+}