Database versioning and migration

.github/workflows/migration-check.yml

@@ -0,0 +1,29 @@
+name: Migration Check
+
+# Cancel workflow if there is a new change to the branch.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches:
+      - main
+  push:
+    branches:
+      - main
+
+jobs:
+  migration-check:
+    name: Forest database migration checks
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v3
+      - name: Setup sccache
+        uses: mozilla-actions/[email protected]
+        timeout-minutes: ${{ fromJSON(env.CACHE_TIMEOUT_MINUTES) }}
+        continue-on-error: true
+      - name: migration check
+        run: ./scripts/migration_check.sh

CHANGELOG.md

@@ -27,6 +27,8 @@
 
 ### Breaking
 
+- [#3203](https://github.com/ChainSafe/forest/issues/3203): Implemented database
+  versioning and migration
 - [#3189](https://github.com/ChainSafe/forest/issues/3189): Changed the database
   organisation to use multiple columns. The database will need to be recreated.
 - [#3220](https://github.com/ChainSafe/forest/pull/3220): Removed the

documentation/src/developer_documentation/db_migration.md

@@ -0,0 +1,10 @@
+# Steps to add support for a new database version:
+
+- Add a new enum variant for new database version in `DBVersion`.
+- Update `get_db_version` to include newly added enum variant.
+- Add version transition for each DBVersion in `migrate_db` method.
+- Add steps required for new migration in `migrate` method. In each migration
+  step, you can either do in place migration or use temp_db/ to migrate data
+  from existing db but finally it must atomically rename temp_db/ back to
+  existing db name.
+- Update `LATEST_DB_VERSION` to latest database version.

scripts/migration_check.sh

@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Function to sync using a specific tag
+function sync_with_tag() {
+    local tag=$1
+    echo "Syncing using tag ($tag)..."
+
+    # Create the download URL for the forest release
+    URL="https://github.com/ChainSafe/forest/releases/download/${tag}/forest-${tag}-linux-amd64.zip"
+    # Download the release using curl
+    curl -LJO "${URL}"
+
+    # Unzip the downloaded file
+    unzip "forest-${tag}-linux-amd64.zip"
+    cd "forest-${tag}"
+
+    # Run forest daemon 
+    ./forest --chain calibnet --encrypt-keystore false --auto-download-snapshot --detach
+
+    # Check if the sync succeeded for the tag
+    if ./forest-cli --chain calibnet sync wait; then
+        echo "Sync successful for tag: $tag"
+        pkill -9 forest
+        # clean up
+        cd ..
+        rm "forest-${tag}-linux-amd64.zip" "forest-${tag}" -rf
+        sleep 5s
+    else
+        echo "Sync failed for tag: $tag"
+        exit 1
+    fi
+}
+
+# DB Migration are supported v0.11.1 onwards
+START_TAG="v0.11.1"
+
+# Fetch the latest tags from the remote repository
+git fetch --tags
+
+# Get a list of all tags sorted chronologically
+tags=$(git tag --sort=creatordate)
+# Get latest tag
+LATEST_TAG=""
+
+# Database migration are not supported for forest version below `v0.11.1`
+is_tag_valid=false
+
+echo "Testing db migrations from v0.11.1 to latest, one by one"
+# Loop through each tag and sync with corresponding version
+for tag in $tags; do
+  # Check if the current tag matches the start tag
+  if [ "$tag" = "$START_TAG" ]; then
+    is_tag_valid=true
+  fi
+  if $is_tag_valid; then
+    # Run sync check with the current tag
+    sync_with_tag "$tag"
+  fi
+  LATEST_TAG="$tag"
+done
+
+echo "Testing db migration from v0.11.1 to latest, at once"
+# Sync calibnet with Forest `V0.11.1`
+sync_with_tag "$START_TAG"
+# Sync calibnet with latest version of Forest
+sync_with_tag "$LATEST_TAG"
+
+echo "Migration check completed successfully."

src/cli_shared/mod.rs

@@ -13,7 +13,14 @@ pub use tikv_jemallocator;
 
 /// Gets chain data directory
 pub fn chain_path(config: &crate::cli_shared::cli::Config) -> PathBuf {
-    PathBuf::from(&config.client.data_dir).join(config.chain.network.to_string())
+    let chain_path = PathBuf::from(&config.client.data_dir).join(config.chain.network.to_string());
+    // Use the dev database if it exists, else use versioned database
+    let dev_path = chain_path.join("dev");
+    if dev_path.exists() {
+        dev_path
+    } else {
+        chain_path.join(env!("CARGO_PKG_VERSION"))
+    }
 }
 
 pub mod snapshot;

src/daemon/mod.rs

@@ -15,6 +15,7 @@ use crate::cli_shared::{
 };
 use crate::db::{
     db_engine::{db_root, open_proxy_db},
+    migration::{check_if_another_db_exist, migrate_db, LATEST_DB_VERSION},
     rolling::DbGarbageCollector,
 };
 use crate::genesis::{get_network_name_from_genesis, import_chain, read_genesis_header};
@@ -150,6 +151,10 @@ pub(super) async fn start(
 
     let keystore = Arc::new(RwLock::new(keystore));
 
+    if let Some(db_path) = check_if_another_db_exist(&config) {
+        migrate_db(&config, db_path, LATEST_DB_VERSION).await?;
+    }
+
     let chain_data_path = chain_path(&config);
     let db = Arc::new(open_proxy_db(
         db_root(&chain_data_path),

src/db/migration/mod.rs

@@ -0,0 +1,129 @@
+// Copyright 2019-2023 ChainSafe Systems
+// SPDX-License-Identifier: Apache-2.0, MIT
+
+use super::db_engine::{db_root, open_proxy_db};
+use crate::chain::ChainStore;
+use crate::cli_shared::{chain_path, cli::Config};
+use crate::fil_cns::composition as cns;
+use crate::genesis::read_genesis_header;
+use crate::state_manager::StateManager;
+use crate::utils::proofs_api::paramfetch::{
+    ensure_params_downloaded, set_proofs_parameter_cache_dir_env,
+};
+use std::fs;
+use std::path::{Path, PathBuf};
+use std::sync::Arc;
+use tracing::info;
+
+pub const LATEST_DB_VERSION: DBVersion = DBVersion::V11;
+
+/// Database version for each forest version which supports db migration
+#[derive(Debug, Eq, PartialEq)]
+pub enum DBVersion {
+    V0, // Default DBVersion for any unknown db
+    V11,
+}
+
+/// Check to verify database migrations
+async fn migration_check(config: &Config, existing_chain_data_root: &Path) -> anyhow::Result<()> {
+    info!(
+        "Running database migration checks for: {}",
+        existing_chain_data_root.display()
+    );
+    // Set proof param dir env path, required for running validations
+    if cns::FETCH_PARAMS {
+        set_proofs_parameter_cache_dir_env(&config.client.data_dir);
+    }
+    ensure_params_downloaded().await?;
+
+    // Open db
+    let db = Arc::new(open_proxy_db(
+        db_root(existing_chain_data_root),
+        config.db_config().clone(),
+    )?);
+    let genesis = read_genesis_header(None, config.chain.genesis_bytes(), &db).await?;
+    let chain_store = Arc::new(ChainStore::new(
+        db,
+        Arc::clone(&config.chain),
+        genesis,
+        existing_chain_data_root,
+    )?);
+    let state_manager = Arc::new(StateManager::new(chain_store, Arc::clone(&config.chain))?);
+
+    let ts = state_manager.chain_store().heaviest_tipset();
+    let height = ts.epoch();
+    assert!(height.is_positive());
+    // re-compute 100 tipsets only
+    state_manager.validate_range((height - 100)..=height)?;
+    Ok(())
+}
+
+/// Migrate database to latest version
+pub async fn migrate_db(
+    config: &Config,
+    db_path: PathBuf,
+    target_version: DBVersion,
+) -> anyhow::Result<()> {
+    info!("Running database migrations...");
+    // Get DBVersion from existing db path
+    let mut current_version = get_db_version(&db_path);
+    // Run pre-migration checks, which includes:
+    // - re-compute 100 tipsets
+    migration_check(config, &db_path).await?;
+
+    // Iterate over all DBVersion's until database is migrated to lastest version
+    while current_version != target_version {
+        let next_version = match current_version {
+            DBVersion::V0 => DBVersion::V11,
+            _ => break,
+        };
+        // Execute the migration steps for itermediate version
+        migrate(&db_path, &next_version)?;
+        current_version = next_version;
+    }
+    // Run post-migration checks, which includes:
+    // - re-compute 100 tipsets
+    migration_check(config, &db_path).await?;
+
+    // Rename db to latest versioned db
+    fs::rename(db_path.as_path(), chain_path(config))?;
+
+    info!("Database Successfully Migrated to {:?}", target_version);
+    Ok(())
+}
+
+// TODO: Add Steps required for new migration
+/// Migrate to an intermediate db version
+fn migrate(_existing_db_path: &Path, next_version: &DBVersion) -> anyhow::Result<()> {
+    match next_version {
+        DBVersion::V11 => Ok(()),
+        _ => Ok(()),
+    }
+}
+
+/// Checks if another database already exist
+pub fn check_if_another_db_exist(config: &Config) -> Option<PathBuf> {
+    let dir = PathBuf::from(&config.client.data_dir).join(config.chain.network.to_string());
+    let paths = fs::read_dir(&dir).ok()?;
+    for dir in paths.flatten() {
+        let path = dir.path();
+        if path.is_dir() && path != chain_path(config) {
+            return Some(path);
+        }
+    }
+    None
+}
+
+/// Returns `DBVersion` for the given database path.
+fn get_db_version(db_path: &Path) -> DBVersion {
+    match db_path
+        .parent()
+        .and_then(|parent_path| parent_path.file_name())
+    {
+        Some(dir_name) => match dir_name.to_str() {
+            Some(name) if name.starts_with("0.11") => DBVersion::V11,
+            _ => DBVersion::V0, // Defaults to V0
+        },
+        None => DBVersion::V0, // Defaults to V0
+    }
+}

src/db/mod.rs

@@ -7,6 +7,7 @@ pub mod parity_db;
 pub mod parity_db_config;
 pub use memory::MemoryDB;
 
+pub mod migration;
 pub mod rolling;
 
 /// Interface used to store and retrieve settings from the database.
@@ -64,9 +65,8 @@ impl<DB: DBStatistics> DBStatistics for std::sync::Arc<DB> {
 }
 
 pub mod db_engine {
-    use std::path::{Path, PathBuf};
-
     use crate::db::rolling::*;
+    use std::path::{Path, PathBuf};
 
     pub type Db = crate::db::parity_db::ParityDb;
     pub type DbConfig = crate::db::parity_db_config::ParityDbConfig;