Merge pull request #57 from getAlby/feat/migrate-to-vss

feat: migrate from sqlite to VSS

bindings/ldk_node.udl

@@ -52,6 +52,7 @@ interface Builder {
 	Node build_with_fs_store();
 	void restore_encoded_channel_monitors(sequence<KeyValue> monitors);
 	void reset_state(ResetState what);
+	void migrate_storage(MigrateStorage what);
 	[Throws=BuildError]
 	Node build_with_vss_store(string vss_url, string store_id, string lnurl_auth_server_url, record<string, string> fixed_headers);
 	[Throws=BuildError]
@@ -651,3 +652,8 @@ enum ResetState {
  	"NetworkGraph",
  	"All",
  };
+
+enum MigrateStorage {
+ 	"VSS",
+ 	// "Sqlite",
+ };

src/builder.rs

@@ -17,6 +17,8 @@ use crate::io::utils::{read_node_metrics, write_node_metrics};
 use crate::io::vss_store::VssStore;
 use crate::io::{
 	NODE_METRICS_KEY, NODE_METRICS_PRIMARY_NAMESPACE, NODE_METRICS_SECONDARY_NAMESPACE,
+	PEER_INFO_PERSISTENCE_KEY, PEER_INFO_PERSISTENCE_PRIMARY_NAMESPACE,
+	PEER_INFO_PERSISTENCE_SECONDARY_NAMESPACE,
 };
 use crate::liquidity::LiquiditySource;
 use crate::logger::{log_error, log_info, FilesystemLogger, Logger};
@@ -26,13 +28,15 @@ use crate::peer_store::PeerStore;
 use crate::tx_broadcaster::TransactionBroadcaster;
 use crate::types::{
 	ChainMonitor, ChannelManager, DynStore, GossipSync, Graph, KeyValue, KeysManager,
-	MessageRouter, OnionMessenger, PeerManager, ResetState,
+	MessageRouter, MigrateStorage, OnionMessenger, PeerManager, ResetState,
 };
 use crate::wallet::persist::KVStoreWalletPersister;
 use crate::wallet::Wallet;
 use crate::{io, NodeMetrics};
 use crate::{LogLevel, Node};
+use lightning::util::persist::KVStore;
 
+use chrono::Local;
 use lightning::chain::{chainmonitor, BestBlock, Watch};
 use lightning::io::Cursor;
 use lightning::ln::channelmanager::{self, ChainParameters, ChannelManagerReadArgs};
@@ -48,6 +52,7 @@ use lightning::sign::EntropySource;
 use lightning::util::persist::{
 	read_channel_monitors, CHANNEL_MANAGER_PERSISTENCE_KEY,
 	CHANNEL_MANAGER_PERSISTENCE_PRIMARY_NAMESPACE, CHANNEL_MANAGER_PERSISTENCE_SECONDARY_NAMESPACE,
+	CHANNEL_MONITOR_PERSISTENCE_PRIMARY_NAMESPACE, CHANNEL_MONITOR_PERSISTENCE_SECONDARY_NAMESPACE,
 	NETWORK_GRAPH_PERSISTENCE_KEY, NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE,
 	NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, SCORER_PERSISTENCE_KEY,
 	SCORER_PERSISTENCE_PRIMARY_NAMESPACE, SCORER_PERSISTENCE_SECONDARY_NAMESPACE,
@@ -75,7 +80,7 @@ use std::convert::TryInto;
 use std::default::Default;
 use std::fmt;
 use std::fs;
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 use std::sync::atomic::AtomicBool;
 use std::sync::{Arc, Mutex, RwLock};
 use std::time::SystemTime;
@@ -190,6 +195,7 @@ pub struct NodeBuilder {
 	liquidity_source_config: Option<LiquiditySourceConfig>,
 	monitors_to_restore: Option<Vec<KeyValue>>,
 	reset_state: Option<ResetState>,
+	migrate_storage: Option<MigrateStorage>,
 }
 
 impl NodeBuilder {
@@ -207,6 +213,7 @@ impl NodeBuilder {
 		let liquidity_source_config = None;
 		let monitors_to_restore = None;
 		let reset_state = None;
+		let migrate_storage = None;
 		Self {
 			config,
 			entropy_source_config,
@@ -215,6 +222,7 @@ impl NodeBuilder {
 			liquidity_source_config,
 			monitors_to_restore,
 			reset_state,
+			migrate_storage,
 		}
 	}
 
@@ -230,6 +238,12 @@ impl NodeBuilder {
 		self
 	}
 
+	/// Alby: migrate storage on startup.
+	pub fn migrate_storage(&mut self, what: MigrateStorage) -> &mut Self {
+		self.migrate_storage = Some(what);
+		self
+	}
+
 	/// Configures the [`Node`] instance to source its wallet entropy from a seed file on disk.
 	///
 	/// If the given file does not exist a new random seed file will be generated and
@@ -495,6 +509,49 @@ impl NodeBuilder {
 
 		let vss_seed_bytes: [u8; 32] = vss_xprv.private_key.secret_bytes();
 
+		// Alby: move sqlite store for migration from sqlite to VSS
+		let mut migrate_from_store = None;
+		let migrate_to_vss = match self.migrate_storage {
+			Some(MigrateStorage::VSS) => true,
+			_ => false,
+		};
+		if migrate_to_vss {
+			// rename and read existing file
+			let storage_dir_path = config.storage_dir_path.clone();
+
+			// Get the current date and time
+			let now = Local::now();
+			let timestamp = now.format("%Y%m%d_%H%M%S").to_string();
+
+			// Create a backup filename based on the current date and time
+			let backup_filename = format!("ldk_node_data_{}.sqlite", timestamp);
+			let current_file_path =
+				Path::new(storage_dir_path.as_str()).join(io::sqlite_store::SQLITE_DB_FILE_NAME);
+			let backup_file_path =
+				Path::new(storage_dir_path.as_str()).join(backup_filename.clone());
+
+			// Rename the file, so that we start fresh
+			log_info!(
+				logger,
+				"Migrating to VSS - Moving sqlite db to backup file: {}",
+				backup_file_path.to_str().expect("Invalid backup file path")
+			);
+			fs::rename(&current_file_path, &backup_file_path).map_err(|e| {
+				log_error!(logger, "Failed to rename existing sqlite file: {}", e);
+				BuildError::KVStoreSetupFailed
+			})?;
+
+			// Read from the old file
+			migrate_from_store = Some(Arc::new(
+				SqliteStore::new(
+					storage_dir_path.into(),
+					Some(backup_filename),
+					Some(io::sqlite_store::KV_TABLE_NAME.to_string()),
+				)
+				.map_err(|_| BuildError::KVStoreSetupFailed)?,
+			) as Arc<DynStore>);
+		}
+
 		// Alby: use a secondary KV store for non-essential data (not needed by VSS)
 		let storage_dir_path = config.storage_dir_path.clone();
 		let secondary_kv_store = Arc::new(
@@ -512,6 +569,73 @@ impl NodeBuilder {
 				log_error!(logger, "Failed to setup VssStore: {}", e);
 				BuildError::KVStoreSetupFailed
 			})?;
+
+		// Alby: migrate from backed up sqlite store to VSS
+		if let Some(from_store) = migrate_from_store {
+			log_info!(logger, "Migrating to VSS - migrating store data");
+			// write essential data from old store to new store
+
+			let migrate_kv = |primary_namespace: &str,
+			                  secondary_namespace: &str,
+			                  key: &str|
+			 -> Result<(), BuildError> {
+				log_info!(
+					logger,
+					"Migrating key {} {} {}",
+					primary_namespace,
+					secondary_namespace,
+					key
+				);
+				let value =
+					from_store.read(primary_namespace, secondary_namespace, key).map_err(|e| {
+						log_error!(logger, "Failed to fetch value: {}", e);
+						BuildError::KVStoreSetupFailed
+					})?;
+				// write value to new store
+				vss_store.write(primary_namespace, secondary_namespace, key, &value).map_err(
+					|e| {
+						log_error!(logger, "Failed to migrate value: {}", e);
+						BuildError::KVStoreSetupFailed
+					},
+				)?;
+				Ok(())
+			};
+
+			let channel_monitor_keys = from_store
+				.list(
+					CHANNEL_MONITOR_PERSISTENCE_PRIMARY_NAMESPACE,
+					CHANNEL_MONITOR_PERSISTENCE_SECONDARY_NAMESPACE,
+				)
+				.map_err(|e| {
+					log_error!(logger, "Failed to fetch channel_monitor_keys: {}", e);
+					BuildError::KVStoreSetupFailed
+				})?;
+
+			for key in channel_monitor_keys {
+				migrate_kv(
+					CHANNEL_MONITOR_PERSISTENCE_PRIMARY_NAMESPACE,
+					CHANNEL_MONITOR_PERSISTENCE_SECONDARY_NAMESPACE,
+					key.as_str(),
+				)?;
+			}
+
+			// migrate channel manager
+			migrate_kv(
+				CHANNEL_MANAGER_PERSISTENCE_PRIMARY_NAMESPACE,
+				CHANNEL_MANAGER_PERSISTENCE_SECONDARY_NAMESPACE,
+				CHANNEL_MANAGER_PERSISTENCE_KEY,
+			)?;
+
+			// migrate peers
+			migrate_kv(
+				PEER_INFO_PERSISTENCE_PRIMARY_NAMESPACE,
+				PEER_INFO_PERSISTENCE_SECONDARY_NAMESPACE,
+				PEER_INFO_PERSISTENCE_KEY,
+			)?;
+
+			log_info!(logger, "Migration to VSS completed successfully");
+		}
+
 		build_with_store_internal(
 			config,
 			self.chain_data_source_config.as_ref(),
@@ -595,6 +719,11 @@ impl ArcedNodeBuilder {
 		self.inner.write().unwrap().reset_state(what);
 	}
 
+	/// Alby: migrate storage on startup.
+	pub fn migrate_storage(&self, what: MigrateStorage) {
+		self.inner.write().unwrap().migrate_storage(what);
+	}
+
 	/// Configures the [`Node`] instance to source its wallet entropy from a seed file on disk.
 	///
 	/// If the given file does not exist a new random seed file will be generated and

src/lib.rs

@@ -136,13 +136,13 @@ use payment::{
 	UnifiedQrPayment,
 };
 use peer_store::{PeerInfo, PeerStore};
-#[cfg(feature = "uniffi")]
-use types::ResetState;
 use types::{
 	Broadcaster, BumpTransactionEventHandler, ChainMonitor, ChannelManager, DynStore, Graph,
 	KeysManager, OnionMessenger, PeerManager, Router, Scorer, Sweeper, Wallet,
 };
 pub use types::{ChannelDetails, ChannelType, KeyValue, PeerDetails, TlvEntry, UserChannelId};
+#[cfg(feature = "uniffi")]
+use types::{MigrateStorage, ResetState};
 
 use logger::{log_error, log_info, log_trace, FilesystemLogger, Logger};
 

src/types.rs

@@ -415,3 +415,12 @@ pub enum ResetState {
 	/// All of the above.
 	All,
 }
+
+/// Options to migrate from one storage type to another.
+#[derive(Debug, Copy, Clone)]
+pub enum MigrateStorage {
+	/// SQLite to VSS
+	VSS,
+	// VSS to SQLite - currently unsupported
+	// Sqlite,
+}