feat: encrypt the bot database at rest (#130)

Configures MVStore to encrypt the database at rest. I initially expected Nitrite to do this when the database is secured with a username and password but this doesn't seem to be the case. The bot will attempt to migrate existing databases automatically on startup. However, **existing backups are not migrated automatically** but new ones will be stored encrypted as well.

docs/self-hosting.md

@@ -25,7 +25,7 @@ You can also build it from scratch by cloning the repository and then running `m
 ```yaml
 services:
   v-rising-discord-bot:
-    image: ghcr.io/darkatra/v-rising-discord-bot:2.11.0-native # find the latest version here: https://github.com/DarkAtra/v-rising-discord-bot/releases
+    image: ghcr.io/darkatra/v-rising-discord-bot:2.12.0-native # find the latest version here: https://github.com/DarkAtra/v-rising-discord-bot/releases
     command: -Dagql.nativeTransport=false
     mem_reservation: 128M
     mem_limit: 256M

pom.xml

@@ -12,7 +12,7 @@
 
     <groupId>de.darkatra</groupId>
     <artifactId>v-rising-discord-bot</artifactId>
-    <version>2.11.6</version>
+    <version>2.12.0</version>
     <packaging>jar</packaging>
 
     <licenses>

src/main/kotlin/de/darkatra/vrising/discord/migration/DatabaseHelper.kt

@@ -3,8 +3,15 @@ package de.darkatra.vrising.discord.migration
 import org.dizitart.no2.Nitrite
 import org.dizitart.no2.collection.Document
 import org.dizitart.no2.collection.NitriteId
+import org.dizitart.no2.common.Constants
+import org.dizitart.no2.common.meta.Attributes
 import org.dizitart.no2.store.NitriteMap
 
 fun Nitrite.getNitriteMap(name: String): NitriteMap<NitriteId, Document> {
     return store.openMap(name, NitriteId::class.java, Document::class.java)
 }
+
+fun Nitrite.listAllCollectionNames(): List<String> {
+    return store.openMap<String, Attributes>(Constants.META_MAP_NAME, String::class.java, Attributes::class.java).keys()
+        .filter { key -> !key.startsWith("\$nitrite") }
+}

src/main/kotlin/de/darkatra/vrising/discord/persistence/DatabaseConfiguration.kt

@@ -2,17 +2,30 @@ package de.darkatra.vrising.discord.persistence
 
 import de.darkatra.vrising.discord.BotProperties
 import de.darkatra.vrising.discord.migration.SchemaEntityConverter
+import de.darkatra.vrising.discord.migration.getNitriteMap
+import de.darkatra.vrising.discord.migration.listAllCollectionNames
 import de.darkatra.vrising.discord.persistence.model.converter.ErrorEntityConverter
 import de.darkatra.vrising.discord.persistence.model.converter.PlayerActivityFeedEntityConverter
 import de.darkatra.vrising.discord.persistence.model.converter.PvpKillFeedEntityConverter
 import de.darkatra.vrising.discord.persistence.model.converter.ServerEntityConverter
 import de.darkatra.vrising.discord.persistence.model.converter.StatusMonitorEntityConverter
 import org.dizitart.no2.Nitrite
+import org.dizitart.no2.NitriteBuilder
+import org.dizitart.no2.exceptions.NitriteIOException
 import org.dizitart.no2.mvstore.MVStoreModule
+import org.dizitart.no2.store.StoreModule
+import org.slf4j.LoggerFactory
 import org.springframework.boot.context.properties.EnableConfigurationProperties
 import org.springframework.context.annotation.Bean
 import org.springframework.context.annotation.Configuration
+import java.nio.charset.StandardCharsets
+import java.nio.file.Files
 import java.nio.file.Path
+import kotlin.io.path.absolutePathString
+import kotlin.io.path.copyTo
+import kotlin.io.path.deleteIfExists
+import kotlin.io.path.exists
+import kotlin.io.path.inputStream
 
 @Configuration
 @EnableConfigurationProperties(BotProperties::class)
@@ -22,12 +35,61 @@ class DatabaseConfiguration(
 
     companion object {
 
+        private const val ENCRYPTED_MARKER = "H2encrypt"
+        private val logger by lazy { LoggerFactory.getLogger(DatabaseConfiguration::class.java) }
+
         fun buildNitriteDatabase(databaseFile: Path, username: String? = null, password: String? = null): Nitrite {
 
-            val storeModule = MVStoreModule.withConfig()
-                .filePath(databaseFile.toAbsolutePath().toFile())
-                .compress(true)
-                .build()
+            // version 2.12.0 introduced database encryption at rest. the following code attempts to perform the migration if necessary
+            val firstFewBytes = databaseFile.inputStream().readNBytes(ENCRYPTED_MARKER.length).toString(StandardCharsets.UTF_8)
+            if (firstFewBytes != ENCRYPTED_MARKER) {
+
+                // if the automated migration was aborted while writing the files to disc, restore the backup
+                val unencryptedDatabaseBackupFile = Path.of(System.getProperty("java.io.tmpdir")).resolve("v-rising-bot.db.unencrypted")
+                if (unencryptedDatabaseBackupFile.exists()) {
+                    logger.info("Found an unencrypted backup of the database at: ${unencryptedDatabaseBackupFile.absolutePathString()}")
+                    unencryptedDatabaseBackupFile.copyTo(databaseFile, overwrite = true)
+                    logger.info("Successfully restored the backup. Will re-attempt the migration.")
+                }
+
+                logger.info("Attempting to encrypt the bot database with the provided database password.")
+
+                // retry opening the database without encryption if we encounter an error
+                val unencryptedDatabase = try {
+                    getNitriteBuilder(getStoreModule(databaseFile, null)).openOrCreate(username, password)
+                } catch (e: NitriteIOException) {
+                    throw IllegalStateException("Could not encrypt the database.", e)
+                }
+
+                unencryptedDatabaseBackupFile.deleteIfExists()
+
+                // create an encrypted copy of the existing database
+                val tempDatabaseFile = Files.createTempFile("v-rising-bot", ".db")
+
+                val encryptedDatabase = getNitriteBuilder(getStoreModule(tempDatabaseFile, password)).openOrCreate(username, password)
+                for (collectionName in unencryptedDatabase.listAllCollectionNames()) {
+
+                    val oldCollection = unencryptedDatabase.getNitriteMap(collectionName)
+                    val newCollection = encryptedDatabase.getNitriteMap(collectionName)
+
+                    oldCollection.values().forEach { document -> newCollection.put(document.id, document) }
+                }
+                unencryptedDatabase.close()
+                encryptedDatabase.close()
+
+                databaseFile.copyTo(unencryptedDatabaseBackupFile)
+                tempDatabaseFile.copyTo(databaseFile, overwrite = true)
+
+                unencryptedDatabaseBackupFile.deleteIfExists()
+                tempDatabaseFile.deleteIfExists()
+
+                logger.info("Successfully encrypted the database.")
+            }
+
+            return getNitriteBuilder(getStoreModule(databaseFile, password)).openOrCreate(username, password)
+        }
+
+        private fun getNitriteBuilder(storeModule: StoreModule): NitriteBuilder {
 
             return Nitrite.builder()
                 .loadModule(storeModule)
@@ -38,7 +100,15 @@ class DatabaseConfiguration(
                 .registerEntityConverter(PvpKillFeedEntityConverter())
                 .registerEntityConverter(ServerEntityConverter())
                 .registerEntityConverter(StatusMonitorEntityConverter())
-                .openOrCreate(username, password)
+        }
+
+        private fun getStoreModule(databaseFile: Path, password: String?): MVStoreModule {
+
+            return MVStoreModule.withConfig()
+                .filePath(databaseFile.toAbsolutePath().toFile())
+                .encryptionKey(password?.let(String::toCharArray))
+                .compress(true)
+                .build()
         }
     }
 

src/main/resources/application.yml

@@ -9,6 +9,9 @@ logging:
   level:
     root: info
     com.ibasco.agql: warn
+    # nitrite is logging some of the exceptions before throwing - disable all nitrite logs since we already log all exceptions
+    nitrite: off
+    nitrite-mvstore: off
   pattern:
     console: "%clr(%d{yyyy-MM-dd'T'HH:mm:ss.SSSXXX}){faint} %clr(%5p) %clr(${PID:-}){magenta} %clr(---){faint} %clr([%15.15t]){faint} %clr(%-40.40logger{39}){cyan} %clr(:){faint} %m %mdc%n%wEx"
 

src/test/kotlin/de/darkatra/vrising/discord/DatabaseConfigurationTestUtils.kt

@@ -15,6 +15,9 @@ object DatabaseConfigurationTestUtils {
 
     val DATABASE_FILE_V1_2_x by lazy { DatabaseConfigurationTestUtils::class.java.getResource("/persistence/v1.2.db")!! }
     val DATABASE_FILE_V2_10_5 by lazy { DatabaseConfigurationTestUtils::class.java.getResource("/persistence/v2.10.5.db")!! }
+    val DATABASE_FILE_V2_10_5_WITH_PASSWORD by lazy { DatabaseConfigurationTestUtils::class.java.getResource("/persistence/v2.10.5-with-password.db")!! }
+    val DATABASE_FILE_V2_12_0_WITH_PASSWORD by lazy { DatabaseConfigurationTestUtils::class.java.getResource("/persistence/v2.12.0-with-password.db")!! }
+
     private val logger by lazy { LoggerFactory.getLogger(javaClass) }
 
     fun getTestDatabase(fromTemplate: URL? = null, username: String? = null, password: String? = null): Nitrite {

src/test/kotlin/de/darkatra/vrising/discord/migration/DatabaseMigrationServiceTest.kt

@@ -7,8 +7,12 @@ import de.darkatra.vrising.discord.persistence.model.Version
 import org.assertj.core.api.Assertions.assertThat
 import org.dizitart.no2.collection.Document
 import org.junit.jupiter.api.Test
+import org.junit.jupiter.api.extension.ExtendWith
+import org.springframework.boot.test.system.CapturedOutput
+import org.springframework.boot.test.system.OutputCaptureExtension
 import java.time.Instant
 
+@ExtendWith(OutputCaptureExtension::class)
 class DatabaseMigrationServiceTest {
 
     @Test
@@ -226,4 +230,87 @@ class DatabaseMigrationServiceTest {
             assertThat(server.statusMonitor!!.recentErrors).isEmpty()
         }
     }
+
+    @Test
+    fun `should migrate schema of password secured database from 2_10_5 to 2_11_0`(capturedOutput: CapturedOutput) {
+
+        DatabaseConfigurationTestUtils.getTestDatabase(DatabaseConfigurationTestUtils.DATABASE_FILE_V2_10_5_WITH_PASSWORD, "test", "test").use { database ->
+
+            assertThat(capturedOutput.out).contains("Successfully encrypted the database.")
+
+            val repository = database.getRepository(Schema::class.java)
+            repository.insert(Schema(appVersion = "V2.10.5"))
+
+            val databaseMigrationService = DatabaseMigrationService(
+                database = database,
+                appVersionFromPom = "2.11.0"
+            )
+
+            database.getRepository(Server::class.java).use { serverRepository ->
+                assertThat(serverRepository.size()).isEqualTo(0)
+            }
+
+            val oldDocument = database.getCollection("de.darkatra.vrising.discord.persistence.model.ServerStatusMonitor").use { oldCollection ->
+                assertThat(oldCollection.size()).isEqualTo(1)
+                oldCollection.find().first()
+            }
+
+            assertThat(databaseMigrationService.migrateToLatestVersion()).isTrue()
+
+            database.getCollection("de.darkatra.vrising.discord.persistence.model.ServerStatusMonitor").use { oldCollection ->
+                assertThat(oldCollection.size()).isEqualTo(0)
+            }
+
+            val server = database.getRepository(Server::class.java).use { serverRepository ->
+                assertThat(serverRepository.size()).isEqualTo(1)
+                serverRepository.find().first()
+            }
+            assertThat(server.id).isEqualTo(oldDocument["id"])
+            @Suppress("DEPRECATION")
+            assertThat(server.version).isEqualTo(Version(1, Instant.ofEpochMilli(oldDocument["version"] as Long)))
+            assertThat(server.discordServerId).isEqualTo(oldDocument["discordServerId"])
+            assertThat(server.hostname).isEqualTo(oldDocument["hostname"])
+            assertThat(server.queryPort).isEqualTo(oldDocument["queryPort"])
+            assertThat(server.apiHostname).isEqualTo(oldDocument["apiHostname"])
+            assertThat(server.apiPort).isEqualTo(oldDocument["apiPort"])
+            assertThat(server.apiUsername).isEqualTo(oldDocument["apiUsername"])
+            assertThat(server.apiPassword).isEqualTo(oldDocument["apiPassword"])
+            assertThat(server.pvpLeaderboard).isNull()
+            assertThat(server.playerActivityFeed).isNotNull()
+            assertThat(server.playerActivityFeed!!.status).isEqualTo(Status.ACTIVE)
+            assertThat(server.playerActivityFeed!!.discordChannelId).isEqualTo(oldDocument["playerActivityDiscordChannelId"])
+            assertThat(server.playerActivityFeed!!.lastUpdated).isNotNull()
+            assertThat(server.playerActivityFeed!!.currentFailedAttempts).isEqualTo(0)
+            assertThat(server.playerActivityFeed!!.recentErrors).isEmpty()
+            assertThat(server.pvpKillFeed).isNotNull()
+            assertThat(server.pvpKillFeed!!.status).isEqualTo(Status.ACTIVE)
+            assertThat(server.pvpKillFeed!!.discordChannelId).isEqualTo(oldDocument["pvpKillFeedDiscordChannelId"])
+            assertThat(server.pvpKillFeed!!.lastUpdated).isNotNull()
+            assertThat(server.pvpKillFeed!!.currentFailedAttempts).isEqualTo(0)
+            assertThat(server.pvpKillFeed!!.recentErrors).isEmpty()
+            assertThat(server.statusMonitor).isNotNull()
+            assertThat(server.statusMonitor!!.status).isNotNull()
+            assertThat(server.statusMonitor!!.status).isEqualTo(Status.ACTIVE)
+            assertThat(server.statusMonitor!!.discordChannelId).isEqualTo(oldDocument["discordChannelId"])
+            assertThat(server.statusMonitor!!.displayServerDescription).isEqualTo(oldDocument["displayServerDescription"])
+            assertThat(server.statusMonitor!!.displayPlayerGearLevel).isEqualTo(oldDocument["displayPlayerGearLevel"])
+            assertThat(server.statusMonitor!!.currentEmbedMessageId).isEqualTo(oldDocument["currentEmbedMessageId"])
+            assertThat(server.statusMonitor!!.currentFailedAttempts).isEqualTo(oldDocument["currentFailedAttempts"])
+            assertThat(server.statusMonitor!!.currentFailedApiAttempts).isEqualTo(oldDocument["currentFailedApiAttempts"])
+            assertThat(server.statusMonitor!!.recentErrors).isEmpty()
+        }
+    }
+
+    @Test
+    fun `should not attempt to encrypt an already encrypted database`(capturedOutput: CapturedOutput) {
+
+        DatabaseConfigurationTestUtils.getTestDatabase(DatabaseConfigurationTestUtils.DATABASE_FILE_V2_12_0_WITH_PASSWORD, "test", "test").use { database ->
+
+            assertThat(capturedOutput.out).doesNotContain("Successfully encrypted the database.")
+
+            database.getRepository(Server::class.java).use { serverRepository ->
+                assertThat(serverRepository.size()).isEqualTo(1)
+            }
+        }
+    }
 }