From de7a305142b271f583c73564ba5f72daf35cd617 Mon Sep 17 00:00:00 2001 From: musitdev Date: Mon, 18 Mar 2024 18:17:38 +0100 Subject: [PATCH 1/9] put all tx files under txfiles folder --- .../node/src/txvalidation/download_manager.rs | 18 +-- crates/node/src/types/file.rs | 142 ++++++++++++------ crates/node/src/types/transaction.rs | 38 +---- crates/node/src/vmm/qemu.rs | 3 +- 4 files changed, 108 insertions(+), 93 deletions(-) diff --git a/crates/node/src/txvalidation/download_manager.rs b/crates/node/src/txvalidation/download_manager.rs index 17928e4e..e299cef1 100644 --- a/crates/node/src/txvalidation/download_manager.rs +++ b/crates/node/src/txvalidation/download_manager.rs @@ -40,7 +40,7 @@ pub async fn download_asset_file( let file_uri = asset_file.get_uri(); let mut resp = match tokio::time::timeout( tokio::time::Duration::from_secs(5), - http_client.get(asset_file.file.url).send(), + http_client.get(&asset_file.url).send(), ) .await { @@ -72,13 +72,13 @@ pub async fn download_asset_file( } resp.ok_or(eyre!( "Download no host found to download the file: {}", - asset_file.file.name + asset_file.name ))? } Err(err) => { return Err(eyre!( "Download file: {:?}, request send timeout.", - asset_file.file.name + asset_file.name )); } }; @@ -113,13 +113,13 @@ pub async fn download_asset_file( Ok(Err(_)) => { return Err(eyre!( "Download file: {:?}, connection timeout", - asset_file.file.name + asset_file.name )); } Err(err) => { return Err(eyre!( "Download file: {:?}, http error:{err}", - asset_file.file.name + asset_file.name )); } } @@ -127,11 +127,11 @@ pub async fn download_asset_file( fd.flush().await?; let checksum: crate::types::Hash = (&hasher.finalize()).into(); - if checksum != asset_file.file.checksum { + if checksum != asset_file.checksum { Err(eyre!( "download_file:{:?} bad checksum checksum:{checksum} set_file.checksum:{}.", - asset_file.file.name, - asset_file.file.checksum + asset_file.name, + asset_file.checksum )) } else { //rename to original name @@ -140,7 +140,7 @@ pub async fn download_asset_file( } else { Err(eyre!( "failed to download file: {:?} response status: {}", - asset_file.file.name, + asset_file.name, resp.status() )) } diff --git a/crates/node/src/types/file.rs b/crates/node/src/types/file.rs index 5d69e4e7..c5c267c0 100644 --- a/crates/node/src/types/file.rs +++ b/crates/node/src/types/file.rs @@ -1,4 +1,5 @@ use crate::types::transaction; +use crate::types::transaction::ProgramData; use crate::types::Hash; use eyre::Result; use serde::Deserialize; @@ -6,6 +7,11 @@ use serde::Serialize; use std::path::Path; use std::path::PathBuf; +// List of folder where the different file type are stored. +pub const IMAGES_DIR: &str = "images"; +pub const TX_FILES_DIR: &str = "txfiles"; +pub const VM_FILES_DIR: &str = "vmfiles"; + // Describe a file use by an executed task. #[derive(Clone, Debug)] pub struct TaskVmFile { @@ -22,6 +28,7 @@ impl TaskVmFile<()> { pub fn get_workspace_path(data_directory: &Path, tx_hash: Hash) -> PathBuf { PathBuf::new() .join(data_directory) + .join(TX_FILES_DIR) .join(tx_hash.to_string()) .join(gevulot_shim::WORKSPACE_NAME) } @@ -124,7 +131,8 @@ impl TaskVmFile { file_path = file_path.strip_prefix("/").unwrap(); // Unwrap tested in `is_absolute()`. } - let mut path = PathBuf::from(&self.extension.0.to_string()); + let mut path = PathBuf::from(TX_FILES_DIR); + path.push(self.extension.0.to_string()); path.push(file_path); path } @@ -187,42 +195,73 @@ pub struct DbFile { // AssetFile: Use to download the file asset associated to a Tx. #[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq)] pub struct AssetFile { - tx_hash: String, + pub name: String, + pub file_path: PathBuf, + pub url: String, + pub checksum: Hash, // Verify_exist: define if the exist() verification do a real file system verification. // Some file must be download even if there's already present. - verify_exist: bool, - pub file: DbFile, + pub verify_exist: bool, } + impl AssetFile { - pub fn new( - name: String, - url: String, - checksum: Hash, - tx_hash: String, - verify_exist: bool, - ) -> Self { - AssetFile { - tx_hash, - verify_exist, - file: DbFile { - name, - url, + pub fn new_from_program_data(data: &ProgramData, tx_hash: Hash) -> Result> { + match data { + ProgramData::Input { + file_name, + file_url, checksum, - }, + } => { + //verify the url is valide. + reqwest::Url::parse(&file_url)?; + let mut file_name_path = Path::new(&file_name); + if file_name_path.is_absolute() { + file_name_path = file_name_path.strip_prefix("/").unwrap(); // Unwrap tested in `is_absolute()`. + } + + // Build file path + let mut file_path = PathBuf::from(TX_FILES_DIR); + file_path.push(tx_hash.to_string()); + file_path.push(file_name_path); + Ok(Some(AssetFile { + name: file_name.to_string(), + file_path, + url: file_url.to_string(), + checksum: checksum.to_string().into(), + verify_exist: false, + })) + } + ProgramData::Output { .. } => { + /* ProgramData::Output as input means it comes from another + program execution -> skip this branch. */ + Ok(None) + } } } - // Get relative File path for downloaded files to be saved on the node. - // The path is is / - pub fn get_save_path(&self) -> PathBuf { - let mut file_path = Path::new(&self.file.name); - if file_path.is_absolute() { - file_path = file_path.strip_prefix("/").unwrap(); // Unwrap tested in `is_absolute()`. - } + // pub fn new( + // name: String, + // url: String, + // checksum: Hash, + // tx_hash: String, + // path_prefix: String, + // verify_exist: bool, + // ) -> Self { + // AssetFile { + // tx_hash, + // verify_exist, + // path_prefix, + // file: DbFile { + // name, + // url, + // checksum, + // }, + // } + // } - let mut path = PathBuf::from(&self.tx_hash); - path.push(file_path); - path + // Get relative File path for downloaded files to be saved on the node. + pub fn get_save_path(&self) -> &Path { + &self.file_path } // Get relative File path for downloaded files to be saved on the node. @@ -271,26 +310,28 @@ impl TxFile { } pub fn into_download_file(self, tx_hash: Hash) -> AssetFile { - let relative_path = self.get_relatif_path(tx_hash).to_str().unwrap().to_string(); - let url = format!("{}/{}", self.url, relative_path); + let relative_path = self.get_relatif_path(tx_hash); + let url = format!( + "{}/{}", + self.url, + relative_path.to_str().unwrap().to_string() + ); - let file_name = Path::new(&self.name).file_name().unwrap_or_default(); - let mut path = PathBuf::from(self.checksum.to_string()); - path.push(file_name); - AssetFile::new( - path.to_str().unwrap().to_string(), + AssetFile { + name: self.name, + file_path: relative_path.into(), url, - self.checksum, - tx_hash.to_string(), - true, - ) + checksum: self.checksum, + verify_exist: true, + } } // Relative File path for Proof or Verify Tx file. // The path is // pub fn get_relatif_path(&self, tx_hash: Hash) -> PathBuf { let file_name = Path::new(&self.name).file_name().unwrap_or_default(); - let mut path = PathBuf::from(tx_hash.to_string()); + let mut path = PathBuf::from(TX_FILES_DIR); + path.push(tx_hash.to_string()); path.push(self.checksum.to_string()); path.push(file_name); path @@ -327,15 +368,16 @@ impl TxFile { impl From> for AssetFile { fn from(file: TxFile) -> Self { - //image file has the image directory happened at the beginning. - let mut extention = PathBuf::from("images"); - extention.push(file.extention.0.to_string()); - AssetFile::new( - file.name, - file.url, - file.checksum, - extention.to_str().unwrap().to_string(), - false, - ) + let mut file_path = PathBuf::from(IMAGES_DIR); + file_path.push(file.extention.0.to_string()); //Tx hash + file_path.push(&file.name); + + AssetFile { + name: file.name, + file_path, + url: file.url, + checksum: file.checksum, + verify_exist: false, + } } } diff --git a/crates/node/src/types/transaction.rs b/crates/node/src/types/transaction.rs index 82fb5dfe..137e64a3 100644 --- a/crates/node/src/types/transaction.rs +++ b/crates/node/src/types/transaction.rs @@ -442,38 +442,12 @@ impl Transaction { TxFile::::try_from_prg_meta_data(prover).into(), TxFile::::try_from_prg_meta_data(verifier).into(), ]), - Payload::Run { workflow } => { - workflow - .steps - .iter() - .flat_map(|step| &step.inputs) - .filter_map(|input| { - match input { - ProgramData::Input { - file_name, - file_url, - checksum, - } => Some((file_name, file_url, checksum)), - ProgramData::Output { .. } => { - /* ProgramData::Output as input means it comes from another - program execution -> skip this branch. */ - None - } - } - }) - .map(|(file_name, file_url, checksum)| { - //verify the url is valide. - reqwest::Url::parse(file_url)?; - Ok(AssetFile::new( - file_name.to_string(), - file_url.clone(), - checksum.to_string().into(), - self.hash.to_string(), - false, - )) - }) - .collect() - } + Payload::Run { workflow } => workflow + .steps + .iter() + .flat_map(|step| &step.inputs) + .filter_map(|input| AssetFile::new_from_program_data(input, self.hash).transpose()) + .collect(), Payload::Proof { files, .. } | Payload::Verification { files, .. } => { //generated file during execution has already been moved. No Download. if self.state.is_from_tx_exec_result() { diff --git a/crates/node/src/vmm/qemu.rs b/crates/node/src/vmm/qemu.rs index 2cf36810..e61940cc 100644 --- a/crates/node/src/vmm/qemu.rs +++ b/crates/node/src/vmm/qemu.rs @@ -1,3 +1,4 @@ +use crate::types::file::IMAGES_DIR; use async_trait::async_trait; use eyre::Result; use gevulot_node::types::file::TaskVmFile; @@ -35,8 +36,6 @@ use crate::{ vmm::ResourceRequest, }; -const IMAGES_DIR: &str = "images"; - impl VMId for u32 { fn as_any(&self) -> &dyn Any { self From b4805ce91e5149612e0a6a6e2b975c291fbb2474 Mon Sep 17 00:00:00 2001 From: musitdev Date: Mon, 18 Mar 2024 18:39:21 +0100 Subject: [PATCH 2/9] pass clippy and correct comments --- .../node/src/txvalidation/download_manager.rs | 20 +++++------ crates/node/src/txvalidation/mod.rs | 6 ++-- crates/node/src/types/file.rs | 33 +++---------------- crates/node/src/types/transaction.rs | 3 +- 4 files changed, 18 insertions(+), 44 deletions(-) diff --git a/crates/node/src/txvalidation/download_manager.rs b/crates/node/src/txvalidation/download_manager.rs index e299cef1..676505b8 100644 --- a/crates/node/src/txvalidation/download_manager.rs +++ b/crates/node/src/txvalidation/download_manager.rs @@ -30,7 +30,7 @@ pub async fn download_asset_file( tracing::info!("download_file:{asset_file:?} local_directory_path:{local_directory_path:?} local_relative_file_path:{local_relative_file_path:?} http_peer_list:{http_peer_list:?}"); // Detect if the file already exist. If yes don't download. - if asset_file.exist(&local_relative_file_path).await { + if asset_file.exist(local_relative_file_path).await { tracing::trace!( "download_asset_file: File already exist, skip download: {:#?}", asset_file.get_save_path() @@ -50,7 +50,7 @@ pub async fn download_asset_file( .iter() .filter_map(|(peer, port)| { port.map(|port| { - //use parse to create an URL, no new method. + // Use parse to create an URL, no new method. let mut url = reqwest::Url::parse(&format!("{HTTP_SERVER_SCHEME}localhost")).unwrap(); //unwrap always succeed url.set_ip_host(peer.ip()).unwrap(); //unwrap always succeed @@ -84,7 +84,7 @@ pub async fn download_asset_file( }; if resp.status() == reqwest::StatusCode::OK { - let file_path = local_directory_path.join(&local_relative_file_path); + let file_path = local_directory_path.join(local_relative_file_path); // Ensure any necessary subdirectories exists. if let Some(parent) = file_path.parent() { if let Ok(false) = tokio::fs::try_exists(parent).await { @@ -92,15 +92,15 @@ pub async fn download_asset_file( } } - //create a tmp file during download. - //this way the file won't be available for download from the other nodes - //until it is completely written. + // Create a tmp file during download. + // This way the file won't be available for download from the other nodes + // until it is completely written. let mut tmp_file_path = file_path.clone(); tmp_file_path.set_extension("tmp"); let fd = tokio::fs::File::create(&tmp_file_path).await?; let mut fd = tokio::io::BufWriter::new(fd); - //create the Hasher to verify the Hash + // Create the Hasher to verify the Hash let mut hasher = blake3::Hasher::new(); loop { @@ -146,8 +146,8 @@ pub async fn download_asset_file( } } -//start the local server and serve the specified file path. -//Return the server task join handle. +// Start the local server and serve the specified file path. +// Return the server task join handle. pub async fn serve_files( mut bind_addr: SocketAddr, http_download_port: u16, @@ -207,7 +207,7 @@ async fn server_process_file( let file = match tokio::fs::File::open(&file_path).await { Ok(file) => file, Err(_) => { - //try to see if the file is currently being updated. + // Try to see if the file is currently being updated. file_path.set_extension("tmp"); let (status_code, message) = if file_path.as_path().exists() { ( diff --git a/crates/node/src/txvalidation/mod.rs b/crates/node/src/txvalidation/mod.rs index 29dc54ba..69a276ae 100644 --- a/crates/node/src/txvalidation/mod.rs +++ b/crates/node/src/txvalidation/mod.rs @@ -29,7 +29,7 @@ mod download_manager; mod event; // `ValidatedTxReceiver` provides a simple trait to decouple event based -// transaction handling from the execution part. +// Transaction handling from the execution part. #[async_trait::async_trait] pub trait ValidatedTxReceiver: Send + Sync { async fn send_new_tx(&mut self, tx: Transaction) -> eyre::Result<()>; @@ -61,8 +61,8 @@ pub enum EventProcessError { pub type CallbackSender = oneshot::Sender>; -//Sending Tx interface. -//Some marker type to define the sender source. +// Sending Tx interface. +// Some marker type to define the sender source. pub struct RpcSender; #[derive(Clone)] pub struct P2pSender; diff --git a/crates/node/src/types/file.rs b/crates/node/src/types/file.rs index c5c267c0..8b242bd3 100644 --- a/crates/node/src/types/file.rs +++ b/crates/node/src/types/file.rs @@ -53,7 +53,6 @@ impl TaskVmFile { let to = PathBuf::new() .join(data_dir) .join(tx_file.get_relatif_path()); - tracing::trace!("TaskVmFile copy_file_for_vm_exe to:{to:?}",); if !tokio::fs::try_exists(&to).await.unwrap_or(false) { let from = PathBuf::new().join(data_dir).join(&self.extension.0); if let Some(parent) = to.parent() { @@ -61,7 +60,7 @@ impl TaskVmFile { .await .map_err(|err| format!("mkdir {parent:?} fail:{err}"))?; } - tracing::trace!("TaskVmFile copy_file_for_vm_exe from:{from:?} to:{to:?}",); + tracing::debug!("TaskVmFile copy_file_for_vm_exe from:{from:?} to:{to:?}",); tokio::fs::copy(&from, &to) .await .map_err(|err| format!("copy file from:{from:?} to:{to:?} error:{err}"))?; @@ -213,7 +212,7 @@ impl AssetFile { checksum, } => { //verify the url is valide. - reqwest::Url::parse(&file_url)?; + reqwest::Url::parse(file_url)?; let mut file_name_path = Path::new(&file_name); if file_name_path.is_absolute() { file_name_path = file_name_path.strip_prefix("/").unwrap(); // Unwrap tested in `is_absolute()`. @@ -239,26 +238,6 @@ impl AssetFile { } } - // pub fn new( - // name: String, - // url: String, - // checksum: Hash, - // tx_hash: String, - // path_prefix: String, - // verify_exist: bool, - // ) -> Self { - // AssetFile { - // tx_hash, - // verify_exist, - // path_prefix, - // file: DbFile { - // name, - // url, - // checksum, - // }, - // } - // } - // Get relative File path for downloaded files to be saved on the node. pub fn get_save_path(&self) -> &Path { &self.file_path @@ -311,15 +290,11 @@ impl TxFile { pub fn into_download_file(self, tx_hash: Hash) -> AssetFile { let relative_path = self.get_relatif_path(tx_hash); - let url = format!( - "{}/{}", - self.url, - relative_path.to_str().unwrap().to_string() - ); + let url = format!("{}/{}", self.url, relative_path.to_str().unwrap()); AssetFile { name: self.name, - file_path: relative_path.into(), + file_path: relative_path, url, checksum: self.checksum, verify_exist: true, diff --git a/crates/node/src/types/transaction.rs b/crates/node/src/types/transaction.rs index 137e64a3..91dd058a 100644 --- a/crates/node/src/types/transaction.rs +++ b/crates/node/src/types/transaction.rs @@ -434,7 +434,6 @@ impl Transaction { } pub fn get_asset_list(&self) -> Result> { - tracing::trace!("get_asset_list Transaction { .filter_map(|input| AssetFile::new_from_program_data(input, self.hash).transpose()) .collect(), Payload::Proof { files, .. } | Payload::Verification { files, .. } => { - //generated file during execution has already been moved. No Download. + // Generated file during execution has already been moved. No Download. if self.state.is_from_tx_exec_result() { Ok(vec![]) } else { From 034fce9d5a6afe50f33564f4bc6ce99c57cfbc09 Mon Sep 17 00:00:00 2001 From: musitdev Date: Mon, 18 Mar 2024 18:17:38 +0100 Subject: [PATCH 3/9] put all tx files under txfiles folder --- .../node/src/txvalidation/download_manager.rs | 18 +-- crates/node/src/types/file.rs | 142 ++++++++++++------ crates/node/src/types/transaction.rs | 38 +---- crates/node/src/vmm/qemu.rs | 3 +- 4 files changed, 108 insertions(+), 93 deletions(-) diff --git a/crates/node/src/txvalidation/download_manager.rs b/crates/node/src/txvalidation/download_manager.rs index 17928e4e..e299cef1 100644 --- a/crates/node/src/txvalidation/download_manager.rs +++ b/crates/node/src/txvalidation/download_manager.rs @@ -40,7 +40,7 @@ pub async fn download_asset_file( let file_uri = asset_file.get_uri(); let mut resp = match tokio::time::timeout( tokio::time::Duration::from_secs(5), - http_client.get(asset_file.file.url).send(), + http_client.get(&asset_file.url).send(), ) .await { @@ -72,13 +72,13 @@ pub async fn download_asset_file( } resp.ok_or(eyre!( "Download no host found to download the file: {}", - asset_file.file.name + asset_file.name ))? } Err(err) => { return Err(eyre!( "Download file: {:?}, request send timeout.", - asset_file.file.name + asset_file.name )); } }; @@ -113,13 +113,13 @@ pub async fn download_asset_file( Ok(Err(_)) => { return Err(eyre!( "Download file: {:?}, connection timeout", - asset_file.file.name + asset_file.name )); } Err(err) => { return Err(eyre!( "Download file: {:?}, http error:{err}", - asset_file.file.name + asset_file.name )); } } @@ -127,11 +127,11 @@ pub async fn download_asset_file( fd.flush().await?; let checksum: crate::types::Hash = (&hasher.finalize()).into(); - if checksum != asset_file.file.checksum { + if checksum != asset_file.checksum { Err(eyre!( "download_file:{:?} bad checksum checksum:{checksum} set_file.checksum:{}.", - asset_file.file.name, - asset_file.file.checksum + asset_file.name, + asset_file.checksum )) } else { //rename to original name @@ -140,7 +140,7 @@ pub async fn download_asset_file( } else { Err(eyre!( "failed to download file: {:?} response status: {}", - asset_file.file.name, + asset_file.name, resp.status() )) } diff --git a/crates/node/src/types/file.rs b/crates/node/src/types/file.rs index 5d69e4e7..c5c267c0 100644 --- a/crates/node/src/types/file.rs +++ b/crates/node/src/types/file.rs @@ -1,4 +1,5 @@ use crate::types::transaction; +use crate::types::transaction::ProgramData; use crate::types::Hash; use eyre::Result; use serde::Deserialize; @@ -6,6 +7,11 @@ use serde::Serialize; use std::path::Path; use std::path::PathBuf; +// List of folder where the different file type are stored. +pub const IMAGES_DIR: &str = "images"; +pub const TX_FILES_DIR: &str = "txfiles"; +pub const VM_FILES_DIR: &str = "vmfiles"; + // Describe a file use by an executed task. #[derive(Clone, Debug)] pub struct TaskVmFile { @@ -22,6 +28,7 @@ impl TaskVmFile<()> { pub fn get_workspace_path(data_directory: &Path, tx_hash: Hash) -> PathBuf { PathBuf::new() .join(data_directory) + .join(TX_FILES_DIR) .join(tx_hash.to_string()) .join(gevulot_shim::WORKSPACE_NAME) } @@ -124,7 +131,8 @@ impl TaskVmFile { file_path = file_path.strip_prefix("/").unwrap(); // Unwrap tested in `is_absolute()`. } - let mut path = PathBuf::from(&self.extension.0.to_string()); + let mut path = PathBuf::from(TX_FILES_DIR); + path.push(self.extension.0.to_string()); path.push(file_path); path } @@ -187,42 +195,73 @@ pub struct DbFile { // AssetFile: Use to download the file asset associated to a Tx. #[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq)] pub struct AssetFile { - tx_hash: String, + pub name: String, + pub file_path: PathBuf, + pub url: String, + pub checksum: Hash, // Verify_exist: define if the exist() verification do a real file system verification. // Some file must be download even if there's already present. - verify_exist: bool, - pub file: DbFile, + pub verify_exist: bool, } + impl AssetFile { - pub fn new( - name: String, - url: String, - checksum: Hash, - tx_hash: String, - verify_exist: bool, - ) -> Self { - AssetFile { - tx_hash, - verify_exist, - file: DbFile { - name, - url, + pub fn new_from_program_data(data: &ProgramData, tx_hash: Hash) -> Result> { + match data { + ProgramData::Input { + file_name, + file_url, checksum, - }, + } => { + //verify the url is valide. + reqwest::Url::parse(&file_url)?; + let mut file_name_path = Path::new(&file_name); + if file_name_path.is_absolute() { + file_name_path = file_name_path.strip_prefix("/").unwrap(); // Unwrap tested in `is_absolute()`. + } + + // Build file path + let mut file_path = PathBuf::from(TX_FILES_DIR); + file_path.push(tx_hash.to_string()); + file_path.push(file_name_path); + Ok(Some(AssetFile { + name: file_name.to_string(), + file_path, + url: file_url.to_string(), + checksum: checksum.to_string().into(), + verify_exist: false, + })) + } + ProgramData::Output { .. } => { + /* ProgramData::Output as input means it comes from another + program execution -> skip this branch. */ + Ok(None) + } } } - // Get relative File path for downloaded files to be saved on the node. - // The path is is / - pub fn get_save_path(&self) -> PathBuf { - let mut file_path = Path::new(&self.file.name); - if file_path.is_absolute() { - file_path = file_path.strip_prefix("/").unwrap(); // Unwrap tested in `is_absolute()`. - } + // pub fn new( + // name: String, + // url: String, + // checksum: Hash, + // tx_hash: String, + // path_prefix: String, + // verify_exist: bool, + // ) -> Self { + // AssetFile { + // tx_hash, + // verify_exist, + // path_prefix, + // file: DbFile { + // name, + // url, + // checksum, + // }, + // } + // } - let mut path = PathBuf::from(&self.tx_hash); - path.push(file_path); - path + // Get relative File path for downloaded files to be saved on the node. + pub fn get_save_path(&self) -> &Path { + &self.file_path } // Get relative File path for downloaded files to be saved on the node. @@ -271,26 +310,28 @@ impl TxFile { } pub fn into_download_file(self, tx_hash: Hash) -> AssetFile { - let relative_path = self.get_relatif_path(tx_hash).to_str().unwrap().to_string(); - let url = format!("{}/{}", self.url, relative_path); + let relative_path = self.get_relatif_path(tx_hash); + let url = format!( + "{}/{}", + self.url, + relative_path.to_str().unwrap().to_string() + ); - let file_name = Path::new(&self.name).file_name().unwrap_or_default(); - let mut path = PathBuf::from(self.checksum.to_string()); - path.push(file_name); - AssetFile::new( - path.to_str().unwrap().to_string(), + AssetFile { + name: self.name, + file_path: relative_path.into(), url, - self.checksum, - tx_hash.to_string(), - true, - ) + checksum: self.checksum, + verify_exist: true, + } } // Relative File path for Proof or Verify Tx file. // The path is // pub fn get_relatif_path(&self, tx_hash: Hash) -> PathBuf { let file_name = Path::new(&self.name).file_name().unwrap_or_default(); - let mut path = PathBuf::from(tx_hash.to_string()); + let mut path = PathBuf::from(TX_FILES_DIR); + path.push(tx_hash.to_string()); path.push(self.checksum.to_string()); path.push(file_name); path @@ -327,15 +368,16 @@ impl TxFile { impl From> for AssetFile { fn from(file: TxFile) -> Self { - //image file has the image directory happened at the beginning. - let mut extention = PathBuf::from("images"); - extention.push(file.extention.0.to_string()); - AssetFile::new( - file.name, - file.url, - file.checksum, - extention.to_str().unwrap().to_string(), - false, - ) + let mut file_path = PathBuf::from(IMAGES_DIR); + file_path.push(file.extention.0.to_string()); //Tx hash + file_path.push(&file.name); + + AssetFile { + name: file.name, + file_path, + url: file.url, + checksum: file.checksum, + verify_exist: false, + } } } diff --git a/crates/node/src/types/transaction.rs b/crates/node/src/types/transaction.rs index 82fb5dfe..137e64a3 100644 --- a/crates/node/src/types/transaction.rs +++ b/crates/node/src/types/transaction.rs @@ -442,38 +442,12 @@ impl Transaction { TxFile::::try_from_prg_meta_data(prover).into(), TxFile::::try_from_prg_meta_data(verifier).into(), ]), - Payload::Run { workflow } => { - workflow - .steps - .iter() - .flat_map(|step| &step.inputs) - .filter_map(|input| { - match input { - ProgramData::Input { - file_name, - file_url, - checksum, - } => Some((file_name, file_url, checksum)), - ProgramData::Output { .. } => { - /* ProgramData::Output as input means it comes from another - program execution -> skip this branch. */ - None - } - } - }) - .map(|(file_name, file_url, checksum)| { - //verify the url is valide. - reqwest::Url::parse(file_url)?; - Ok(AssetFile::new( - file_name.to_string(), - file_url.clone(), - checksum.to_string().into(), - self.hash.to_string(), - false, - )) - }) - .collect() - } + Payload::Run { workflow } => workflow + .steps + .iter() + .flat_map(|step| &step.inputs) + .filter_map(|input| AssetFile::new_from_program_data(input, self.hash).transpose()) + .collect(), Payload::Proof { files, .. } | Payload::Verification { files, .. } => { //generated file during execution has already been moved. No Download. if self.state.is_from_tx_exec_result() { diff --git a/crates/node/src/vmm/qemu.rs b/crates/node/src/vmm/qemu.rs index 3c6bf0b8..b2bfc3dd 100644 --- a/crates/node/src/vmm/qemu.rs +++ b/crates/node/src/vmm/qemu.rs @@ -1,3 +1,4 @@ +use crate::types::file::IMAGES_DIR; use async_trait::async_trait; use eyre::Result; use gevulot_node::types::file::TaskVmFile; @@ -35,8 +36,6 @@ use crate::{ vmm::ResourceRequest, }; -const IMAGES_DIR: &str = "images"; - impl VMId for u32 { fn as_any(&self) -> &dyn Any { self From 764f1833235a21bbfc825bff4dc9ca386b1c7f60 Mon Sep 17 00:00:00 2001 From: musitdev Date: Mon, 18 Mar 2024 18:39:21 +0100 Subject: [PATCH 4/9] pass clippy and correct comments --- .../node/src/txvalidation/download_manager.rs | 20 +++++------ crates/node/src/txvalidation/mod.rs | 6 ++-- crates/node/src/types/file.rs | 33 +++---------------- crates/node/src/types/transaction.rs | 3 +- 4 files changed, 18 insertions(+), 44 deletions(-) diff --git a/crates/node/src/txvalidation/download_manager.rs b/crates/node/src/txvalidation/download_manager.rs index e299cef1..676505b8 100644 --- a/crates/node/src/txvalidation/download_manager.rs +++ b/crates/node/src/txvalidation/download_manager.rs @@ -30,7 +30,7 @@ pub async fn download_asset_file( tracing::info!("download_file:{asset_file:?} local_directory_path:{local_directory_path:?} local_relative_file_path:{local_relative_file_path:?} http_peer_list:{http_peer_list:?}"); // Detect if the file already exist. If yes don't download. - if asset_file.exist(&local_relative_file_path).await { + if asset_file.exist(local_relative_file_path).await { tracing::trace!( "download_asset_file: File already exist, skip download: {:#?}", asset_file.get_save_path() @@ -50,7 +50,7 @@ pub async fn download_asset_file( .iter() .filter_map(|(peer, port)| { port.map(|port| { - //use parse to create an URL, no new method. + // Use parse to create an URL, no new method. let mut url = reqwest::Url::parse(&format!("{HTTP_SERVER_SCHEME}localhost")).unwrap(); //unwrap always succeed url.set_ip_host(peer.ip()).unwrap(); //unwrap always succeed @@ -84,7 +84,7 @@ pub async fn download_asset_file( }; if resp.status() == reqwest::StatusCode::OK { - let file_path = local_directory_path.join(&local_relative_file_path); + let file_path = local_directory_path.join(local_relative_file_path); // Ensure any necessary subdirectories exists. if let Some(parent) = file_path.parent() { if let Ok(false) = tokio::fs::try_exists(parent).await { @@ -92,15 +92,15 @@ pub async fn download_asset_file( } } - //create a tmp file during download. - //this way the file won't be available for download from the other nodes - //until it is completely written. + // Create a tmp file during download. + // This way the file won't be available for download from the other nodes + // until it is completely written. let mut tmp_file_path = file_path.clone(); tmp_file_path.set_extension("tmp"); let fd = tokio::fs::File::create(&tmp_file_path).await?; let mut fd = tokio::io::BufWriter::new(fd); - //create the Hasher to verify the Hash + // Create the Hasher to verify the Hash let mut hasher = blake3::Hasher::new(); loop { @@ -146,8 +146,8 @@ pub async fn download_asset_file( } } -//start the local server and serve the specified file path. -//Return the server task join handle. +// Start the local server and serve the specified file path. +// Return the server task join handle. pub async fn serve_files( mut bind_addr: SocketAddr, http_download_port: u16, @@ -207,7 +207,7 @@ async fn server_process_file( let file = match tokio::fs::File::open(&file_path).await { Ok(file) => file, Err(_) => { - //try to see if the file is currently being updated. + // Try to see if the file is currently being updated. file_path.set_extension("tmp"); let (status_code, message) = if file_path.as_path().exists() { ( diff --git a/crates/node/src/txvalidation/mod.rs b/crates/node/src/txvalidation/mod.rs index 29dc54ba..69a276ae 100644 --- a/crates/node/src/txvalidation/mod.rs +++ b/crates/node/src/txvalidation/mod.rs @@ -29,7 +29,7 @@ mod download_manager; mod event; // `ValidatedTxReceiver` provides a simple trait to decouple event based -// transaction handling from the execution part. +// Transaction handling from the execution part. #[async_trait::async_trait] pub trait ValidatedTxReceiver: Send + Sync { async fn send_new_tx(&mut self, tx: Transaction) -> eyre::Result<()>; @@ -61,8 +61,8 @@ pub enum EventProcessError { pub type CallbackSender = oneshot::Sender>; -//Sending Tx interface. -//Some marker type to define the sender source. +// Sending Tx interface. +// Some marker type to define the sender source. pub struct RpcSender; #[derive(Clone)] pub struct P2pSender; diff --git a/crates/node/src/types/file.rs b/crates/node/src/types/file.rs index c5c267c0..8b242bd3 100644 --- a/crates/node/src/types/file.rs +++ b/crates/node/src/types/file.rs @@ -53,7 +53,6 @@ impl TaskVmFile { let to = PathBuf::new() .join(data_dir) .join(tx_file.get_relatif_path()); - tracing::trace!("TaskVmFile copy_file_for_vm_exe to:{to:?}",); if !tokio::fs::try_exists(&to).await.unwrap_or(false) { let from = PathBuf::new().join(data_dir).join(&self.extension.0); if let Some(parent) = to.parent() { @@ -61,7 +60,7 @@ impl TaskVmFile { .await .map_err(|err| format!("mkdir {parent:?} fail:{err}"))?; } - tracing::trace!("TaskVmFile copy_file_for_vm_exe from:{from:?} to:{to:?}",); + tracing::debug!("TaskVmFile copy_file_for_vm_exe from:{from:?} to:{to:?}",); tokio::fs::copy(&from, &to) .await .map_err(|err| format!("copy file from:{from:?} to:{to:?} error:{err}"))?; @@ -213,7 +212,7 @@ impl AssetFile { checksum, } => { //verify the url is valide. - reqwest::Url::parse(&file_url)?; + reqwest::Url::parse(file_url)?; let mut file_name_path = Path::new(&file_name); if file_name_path.is_absolute() { file_name_path = file_name_path.strip_prefix("/").unwrap(); // Unwrap tested in `is_absolute()`. @@ -239,26 +238,6 @@ impl AssetFile { } } - // pub fn new( - // name: String, - // url: String, - // checksum: Hash, - // tx_hash: String, - // path_prefix: String, - // verify_exist: bool, - // ) -> Self { - // AssetFile { - // tx_hash, - // verify_exist, - // path_prefix, - // file: DbFile { - // name, - // url, - // checksum, - // }, - // } - // } - // Get relative File path for downloaded files to be saved on the node. pub fn get_save_path(&self) -> &Path { &self.file_path @@ -311,15 +290,11 @@ impl TxFile { pub fn into_download_file(self, tx_hash: Hash) -> AssetFile { let relative_path = self.get_relatif_path(tx_hash); - let url = format!( - "{}/{}", - self.url, - relative_path.to_str().unwrap().to_string() - ); + let url = format!("{}/{}", self.url, relative_path.to_str().unwrap()); AssetFile { name: self.name, - file_path: relative_path.into(), + file_path: relative_path, url, checksum: self.checksum, verify_exist: true, diff --git a/crates/node/src/types/transaction.rs b/crates/node/src/types/transaction.rs index 137e64a3..91dd058a 100644 --- a/crates/node/src/types/transaction.rs +++ b/crates/node/src/types/transaction.rs @@ -434,7 +434,6 @@ impl Transaction { } pub fn get_asset_list(&self) -> Result> { - tracing::trace!("get_asset_list Transaction { .filter_map(|input| AssetFile::new_from_program_data(input, self.hash).transpose()) .collect(), Payload::Proof { files, .. } | Payload::Verification { files, .. } => { - //generated file during execution has already been moved. No Download. + // Generated file during execution has already been moved. No Download. if self.state.is_from_tx_exec_result() { Ok(vec![]) } else { From b1ad67649990d9190abeb9a4179c19df7b8e939f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tuomas=20M=C3=A4kinen?= <1947505+tuommaki@users.noreply.github.com> Date: Tue, 19 Mar 2024 09:30:29 +0200 Subject: [PATCH 5/9] Fix image filename (#153) When generating file name from a program deployment, use the `image_file_name` instead of program name / tag. --- crates/node/src/types/file.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/node/src/types/file.rs b/crates/node/src/types/file.rs index 5d69e4e7..14efa928 100644 --- a/crates/node/src/types/file.rs +++ b/crates/node/src/types/file.rs @@ -317,7 +317,7 @@ pub struct Image(Hash); impl TxFile { pub fn try_from_prg_meta_data(value: &transaction::ProgramMetadata) -> Self { TxFile::build( - value.name.clone(), + value.image_file_name.clone(), value.image_file_url.clone(), value.image_file_checksum.clone().into(), Image(value.hash), From 493b727923a3470d4430840bdb549c7d00b794a0 Mon Sep 17 00:00:00 2001 From: musitdev Date: Mon, 18 Mar 2024 18:17:38 +0100 Subject: [PATCH 6/9] put all tx files under txfiles folder --- .../node/src/txvalidation/download_manager.rs | 18 +-- crates/node/src/types/file.rs | 142 ++++++++++++------ crates/node/src/types/transaction.rs | 38 +---- crates/node/src/vmm/qemu.rs | 3 +- 4 files changed, 108 insertions(+), 93 deletions(-) diff --git a/crates/node/src/txvalidation/download_manager.rs b/crates/node/src/txvalidation/download_manager.rs index 17928e4e..e299cef1 100644 --- a/crates/node/src/txvalidation/download_manager.rs +++ b/crates/node/src/txvalidation/download_manager.rs @@ -40,7 +40,7 @@ pub async fn download_asset_file( let file_uri = asset_file.get_uri(); let mut resp = match tokio::time::timeout( tokio::time::Duration::from_secs(5), - http_client.get(asset_file.file.url).send(), + http_client.get(&asset_file.url).send(), ) .await { @@ -72,13 +72,13 @@ pub async fn download_asset_file( } resp.ok_or(eyre!( "Download no host found to download the file: {}", - asset_file.file.name + asset_file.name ))? } Err(err) => { return Err(eyre!( "Download file: {:?}, request send timeout.", - asset_file.file.name + asset_file.name )); } }; @@ -113,13 +113,13 @@ pub async fn download_asset_file( Ok(Err(_)) => { return Err(eyre!( "Download file: {:?}, connection timeout", - asset_file.file.name + asset_file.name )); } Err(err) => { return Err(eyre!( "Download file: {:?}, http error:{err}", - asset_file.file.name + asset_file.name )); } } @@ -127,11 +127,11 @@ pub async fn download_asset_file( fd.flush().await?; let checksum: crate::types::Hash = (&hasher.finalize()).into(); - if checksum != asset_file.file.checksum { + if checksum != asset_file.checksum { Err(eyre!( "download_file:{:?} bad checksum checksum:{checksum} set_file.checksum:{}.", - asset_file.file.name, - asset_file.file.checksum + asset_file.name, + asset_file.checksum )) } else { //rename to original name @@ -140,7 +140,7 @@ pub async fn download_asset_file( } else { Err(eyre!( "failed to download file: {:?} response status: {}", - asset_file.file.name, + asset_file.name, resp.status() )) } diff --git a/crates/node/src/types/file.rs b/crates/node/src/types/file.rs index 14efa928..b9324f85 100644 --- a/crates/node/src/types/file.rs +++ b/crates/node/src/types/file.rs @@ -1,4 +1,5 @@ use crate::types::transaction; +use crate::types::transaction::ProgramData; use crate::types::Hash; use eyre::Result; use serde::Deserialize; @@ -6,6 +7,11 @@ use serde::Serialize; use std::path::Path; use std::path::PathBuf; +// List of folder where the different file type are stored. +pub const IMAGES_DIR: &str = "images"; +pub const TX_FILES_DIR: &str = "txfiles"; +pub const VM_FILES_DIR: &str = "vmfiles"; + // Describe a file use by an executed task. #[derive(Clone, Debug)] pub struct TaskVmFile { @@ -22,6 +28,7 @@ impl TaskVmFile<()> { pub fn get_workspace_path(data_directory: &Path, tx_hash: Hash) -> PathBuf { PathBuf::new() .join(data_directory) + .join(TX_FILES_DIR) .join(tx_hash.to_string()) .join(gevulot_shim::WORKSPACE_NAME) } @@ -124,7 +131,8 @@ impl TaskVmFile { file_path = file_path.strip_prefix("/").unwrap(); // Unwrap tested in `is_absolute()`. } - let mut path = PathBuf::from(&self.extension.0.to_string()); + let mut path = PathBuf::from(TX_FILES_DIR); + path.push(self.extension.0.to_string()); path.push(file_path); path } @@ -187,42 +195,73 @@ pub struct DbFile { // AssetFile: Use to download the file asset associated to a Tx. #[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq)] pub struct AssetFile { - tx_hash: String, + pub name: String, + pub file_path: PathBuf, + pub url: String, + pub checksum: Hash, // Verify_exist: define if the exist() verification do a real file system verification. // Some file must be download even if there's already present. - verify_exist: bool, - pub file: DbFile, + pub verify_exist: bool, } + impl AssetFile { - pub fn new( - name: String, - url: String, - checksum: Hash, - tx_hash: String, - verify_exist: bool, - ) -> Self { - AssetFile { - tx_hash, - verify_exist, - file: DbFile { - name, - url, + pub fn new_from_program_data(data: &ProgramData, tx_hash: Hash) -> Result> { + match data { + ProgramData::Input { + file_name, + file_url, checksum, - }, + } => { + //verify the url is valide. + reqwest::Url::parse(&file_url)?; + let mut file_name_path = Path::new(&file_name); + if file_name_path.is_absolute() { + file_name_path = file_name_path.strip_prefix("/").unwrap(); // Unwrap tested in `is_absolute()`. + } + + // Build file path + let mut file_path = PathBuf::from(TX_FILES_DIR); + file_path.push(tx_hash.to_string()); + file_path.push(file_name_path); + Ok(Some(AssetFile { + name: file_name.to_string(), + file_path, + url: file_url.to_string(), + checksum: checksum.to_string().into(), + verify_exist: false, + })) + } + ProgramData::Output { .. } => { + /* ProgramData::Output as input means it comes from another + program execution -> skip this branch. */ + Ok(None) + } } } - // Get relative File path for downloaded files to be saved on the node. - // The path is is / - pub fn get_save_path(&self) -> PathBuf { - let mut file_path = Path::new(&self.file.name); - if file_path.is_absolute() { - file_path = file_path.strip_prefix("/").unwrap(); // Unwrap tested in `is_absolute()`. - } + // pub fn new( + // name: String, + // url: String, + // checksum: Hash, + // tx_hash: String, + // path_prefix: String, + // verify_exist: bool, + // ) -> Self { + // AssetFile { + // tx_hash, + // verify_exist, + // path_prefix, + // file: DbFile { + // name, + // url, + // checksum, + // }, + // } + // } - let mut path = PathBuf::from(&self.tx_hash); - path.push(file_path); - path + // Get relative File path for downloaded files to be saved on the node. + pub fn get_save_path(&self) -> &Path { + &self.file_path } // Get relative File path for downloaded files to be saved on the node. @@ -271,26 +310,28 @@ impl TxFile { } pub fn into_download_file(self, tx_hash: Hash) -> AssetFile { - let relative_path = self.get_relatif_path(tx_hash).to_str().unwrap().to_string(); - let url = format!("{}/{}", self.url, relative_path); + let relative_path = self.get_relatif_path(tx_hash); + let url = format!( + "{}/{}", + self.url, + relative_path.to_str().unwrap().to_string() + ); - let file_name = Path::new(&self.name).file_name().unwrap_or_default(); - let mut path = PathBuf::from(self.checksum.to_string()); - path.push(file_name); - AssetFile::new( - path.to_str().unwrap().to_string(), + AssetFile { + name: self.name, + file_path: relative_path.into(), url, - self.checksum, - tx_hash.to_string(), - true, - ) + checksum: self.checksum, + verify_exist: true, + } } // Relative File path for Proof or Verify Tx file. // The path is // pub fn get_relatif_path(&self, tx_hash: Hash) -> PathBuf { let file_name = Path::new(&self.name).file_name().unwrap_or_default(); - let mut path = PathBuf::from(tx_hash.to_string()); + let mut path = PathBuf::from(TX_FILES_DIR); + path.push(tx_hash.to_string()); path.push(self.checksum.to_string()); path.push(file_name); path @@ -327,15 +368,16 @@ impl TxFile { impl From> for AssetFile { fn from(file: TxFile) -> Self { - //image file has the image directory happened at the beginning. - let mut extention = PathBuf::from("images"); - extention.push(file.extention.0.to_string()); - AssetFile::new( - file.name, - file.url, - file.checksum, - extention.to_str().unwrap().to_string(), - false, - ) + let mut file_path = PathBuf::from(IMAGES_DIR); + file_path.push(file.extention.0.to_string()); //Tx hash + file_path.push(&file.name); + + AssetFile { + name: file.name, + file_path, + url: file.url, + checksum: file.checksum, + verify_exist: false, + } } } diff --git a/crates/node/src/types/transaction.rs b/crates/node/src/types/transaction.rs index 82fb5dfe..137e64a3 100644 --- a/crates/node/src/types/transaction.rs +++ b/crates/node/src/types/transaction.rs @@ -442,38 +442,12 @@ impl Transaction { TxFile::::try_from_prg_meta_data(prover).into(), TxFile::::try_from_prg_meta_data(verifier).into(), ]), - Payload::Run { workflow } => { - workflow - .steps - .iter() - .flat_map(|step| &step.inputs) - .filter_map(|input| { - match input { - ProgramData::Input { - file_name, - file_url, - checksum, - } => Some((file_name, file_url, checksum)), - ProgramData::Output { .. } => { - /* ProgramData::Output as input means it comes from another - program execution -> skip this branch. */ - None - } - } - }) - .map(|(file_name, file_url, checksum)| { - //verify the url is valide. - reqwest::Url::parse(file_url)?; - Ok(AssetFile::new( - file_name.to_string(), - file_url.clone(), - checksum.to_string().into(), - self.hash.to_string(), - false, - )) - }) - .collect() - } + Payload::Run { workflow } => workflow + .steps + .iter() + .flat_map(|step| &step.inputs) + .filter_map(|input| AssetFile::new_from_program_data(input, self.hash).transpose()) + .collect(), Payload::Proof { files, .. } | Payload::Verification { files, .. } => { //generated file during execution has already been moved. No Download. if self.state.is_from_tx_exec_result() { diff --git a/crates/node/src/vmm/qemu.rs b/crates/node/src/vmm/qemu.rs index 3c6bf0b8..b2bfc3dd 100644 --- a/crates/node/src/vmm/qemu.rs +++ b/crates/node/src/vmm/qemu.rs @@ -1,3 +1,4 @@ +use crate::types::file::IMAGES_DIR; use async_trait::async_trait; use eyre::Result; use gevulot_node::types::file::TaskVmFile; @@ -35,8 +36,6 @@ use crate::{ vmm::ResourceRequest, }; -const IMAGES_DIR: &str = "images"; - impl VMId for u32 { fn as_any(&self) -> &dyn Any { self From a9595fe1077c80d955437cc8a93998ae476a4401 Mon Sep 17 00:00:00 2001 From: musitdev Date: Mon, 18 Mar 2024 18:39:21 +0100 Subject: [PATCH 7/9] pass clippy and correct comments --- .../node/src/txvalidation/download_manager.rs | 20 +++++------ crates/node/src/txvalidation/mod.rs | 6 ++-- crates/node/src/types/file.rs | 33 +++---------------- crates/node/src/types/transaction.rs | 3 +- 4 files changed, 18 insertions(+), 44 deletions(-) diff --git a/crates/node/src/txvalidation/download_manager.rs b/crates/node/src/txvalidation/download_manager.rs index e299cef1..676505b8 100644 --- a/crates/node/src/txvalidation/download_manager.rs +++ b/crates/node/src/txvalidation/download_manager.rs @@ -30,7 +30,7 @@ pub async fn download_asset_file( tracing::info!("download_file:{asset_file:?} local_directory_path:{local_directory_path:?} local_relative_file_path:{local_relative_file_path:?} http_peer_list:{http_peer_list:?}"); // Detect if the file already exist. If yes don't download. - if asset_file.exist(&local_relative_file_path).await { + if asset_file.exist(local_relative_file_path).await { tracing::trace!( "download_asset_file: File already exist, skip download: {:#?}", asset_file.get_save_path() @@ -50,7 +50,7 @@ pub async fn download_asset_file( .iter() .filter_map(|(peer, port)| { port.map(|port| { - //use parse to create an URL, no new method. + // Use parse to create an URL, no new method. let mut url = reqwest::Url::parse(&format!("{HTTP_SERVER_SCHEME}localhost")).unwrap(); //unwrap always succeed url.set_ip_host(peer.ip()).unwrap(); //unwrap always succeed @@ -84,7 +84,7 @@ pub async fn download_asset_file( }; if resp.status() == reqwest::StatusCode::OK { - let file_path = local_directory_path.join(&local_relative_file_path); + let file_path = local_directory_path.join(local_relative_file_path); // Ensure any necessary subdirectories exists. if let Some(parent) = file_path.parent() { if let Ok(false) = tokio::fs::try_exists(parent).await { @@ -92,15 +92,15 @@ pub async fn download_asset_file( } } - //create a tmp file during download. - //this way the file won't be available for download from the other nodes - //until it is completely written. + // Create a tmp file during download. + // This way the file won't be available for download from the other nodes + // until it is completely written. let mut tmp_file_path = file_path.clone(); tmp_file_path.set_extension("tmp"); let fd = tokio::fs::File::create(&tmp_file_path).await?; let mut fd = tokio::io::BufWriter::new(fd); - //create the Hasher to verify the Hash + // Create the Hasher to verify the Hash let mut hasher = blake3::Hasher::new(); loop { @@ -146,8 +146,8 @@ pub async fn download_asset_file( } } -//start the local server and serve the specified file path. -//Return the server task join handle. +// Start the local server and serve the specified file path. +// Return the server task join handle. pub async fn serve_files( mut bind_addr: SocketAddr, http_download_port: u16, @@ -207,7 +207,7 @@ async fn server_process_file( let file = match tokio::fs::File::open(&file_path).await { Ok(file) => file, Err(_) => { - //try to see if the file is currently being updated. + // Try to see if the file is currently being updated. file_path.set_extension("tmp"); let (status_code, message) = if file_path.as_path().exists() { ( diff --git a/crates/node/src/txvalidation/mod.rs b/crates/node/src/txvalidation/mod.rs index 29dc54ba..69a276ae 100644 --- a/crates/node/src/txvalidation/mod.rs +++ b/crates/node/src/txvalidation/mod.rs @@ -29,7 +29,7 @@ mod download_manager; mod event; // `ValidatedTxReceiver` provides a simple trait to decouple event based -// transaction handling from the execution part. +// Transaction handling from the execution part. #[async_trait::async_trait] pub trait ValidatedTxReceiver: Send + Sync { async fn send_new_tx(&mut self, tx: Transaction) -> eyre::Result<()>; @@ -61,8 +61,8 @@ pub enum EventProcessError { pub type CallbackSender = oneshot::Sender>; -//Sending Tx interface. -//Some marker type to define the sender source. +// Sending Tx interface. +// Some marker type to define the sender source. pub struct RpcSender; #[derive(Clone)] pub struct P2pSender; diff --git a/crates/node/src/types/file.rs b/crates/node/src/types/file.rs index b9324f85..ef3a2c40 100644 --- a/crates/node/src/types/file.rs +++ b/crates/node/src/types/file.rs @@ -53,7 +53,6 @@ impl TaskVmFile { let to = PathBuf::new() .join(data_dir) .join(tx_file.get_relatif_path()); - tracing::trace!("TaskVmFile copy_file_for_vm_exe to:{to:?}",); if !tokio::fs::try_exists(&to).await.unwrap_or(false) { let from = PathBuf::new().join(data_dir).join(&self.extension.0); if let Some(parent) = to.parent() { @@ -61,7 +60,7 @@ impl TaskVmFile { .await .map_err(|err| format!("mkdir {parent:?} fail:{err}"))?; } - tracing::trace!("TaskVmFile copy_file_for_vm_exe from:{from:?} to:{to:?}",); + tracing::debug!("TaskVmFile copy_file_for_vm_exe from:{from:?} to:{to:?}",); tokio::fs::copy(&from, &to) .await .map_err(|err| format!("copy file from:{from:?} to:{to:?} error:{err}"))?; @@ -213,7 +212,7 @@ impl AssetFile { checksum, } => { //verify the url is valide. - reqwest::Url::parse(&file_url)?; + reqwest::Url::parse(file_url)?; let mut file_name_path = Path::new(&file_name); if file_name_path.is_absolute() { file_name_path = file_name_path.strip_prefix("/").unwrap(); // Unwrap tested in `is_absolute()`. @@ -239,26 +238,6 @@ impl AssetFile { } } - // pub fn new( - // name: String, - // url: String, - // checksum: Hash, - // tx_hash: String, - // path_prefix: String, - // verify_exist: bool, - // ) -> Self { - // AssetFile { - // tx_hash, - // verify_exist, - // path_prefix, - // file: DbFile { - // name, - // url, - // checksum, - // }, - // } - // } - // Get relative File path for downloaded files to be saved on the node. pub fn get_save_path(&self) -> &Path { &self.file_path @@ -311,15 +290,11 @@ impl TxFile { pub fn into_download_file(self, tx_hash: Hash) -> AssetFile { let relative_path = self.get_relatif_path(tx_hash); - let url = format!( - "{}/{}", - self.url, - relative_path.to_str().unwrap().to_string() - ); + let url = format!("{}/{}", self.url, relative_path.to_str().unwrap()); AssetFile { name: self.name, - file_path: relative_path.into(), + file_path: relative_path, url, checksum: self.checksum, verify_exist: true, diff --git a/crates/node/src/types/transaction.rs b/crates/node/src/types/transaction.rs index 137e64a3..91dd058a 100644 --- a/crates/node/src/types/transaction.rs +++ b/crates/node/src/types/transaction.rs @@ -434,7 +434,6 @@ impl Transaction { } pub fn get_asset_list(&self) -> Result> { - tracing::trace!("get_asset_list Transaction { .filter_map(|input| AssetFile::new_from_program_data(input, self.hash).transpose()) .collect(), Payload::Proof { files, .. } | Payload::Verification { files, .. } => { - //generated file during execution has already been moved. No Download. + // Generated file during execution has already been moved. No Download. if self.state.is_from_tx_exec_result() { Ok(vec![]) } else { From 37fa25a322bfd61ecf1b343545031b590201fffe Mon Sep 17 00:00:00 2001 From: musitdev Date: Mon, 18 Mar 2024 18:17:38 +0100 Subject: [PATCH 8/9] put all tx files under txfiles folder --- crates/node/src/types/file.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/crates/node/src/types/file.rs b/crates/node/src/types/file.rs index ef3a2c40..451efd6e 100644 --- a/crates/node/src/types/file.rs +++ b/crates/node/src/types/file.rs @@ -212,7 +212,7 @@ impl AssetFile { checksum, } => { //verify the url is valide. - reqwest::Url::parse(file_url)?; + reqwest::Url::parse(&file_url)?; let mut file_name_path = Path::new(&file_name); if file_name_path.is_absolute() { file_name_path = file_name_path.strip_prefix("/").unwrap(); // Unwrap tested in `is_absolute()`. @@ -290,11 +290,15 @@ impl TxFile { pub fn into_download_file(self, tx_hash: Hash) -> AssetFile { let relative_path = self.get_relatif_path(tx_hash); - let url = format!("{}/{}", self.url, relative_path.to_str().unwrap()); + let url = format!( + "{}/{}", + self.url, + relative_path.to_str().unwrap().to_string() + ); AssetFile { name: self.name, - file_path: relative_path, + file_path: relative_path.into(), url, checksum: self.checksum, verify_exist: true, From a391818ad40b65ea67a9d77e957137060dcf9e66 Mon Sep 17 00:00:00 2001 From: musitdev Date: Mon, 18 Mar 2024 18:39:21 +0100 Subject: [PATCH 9/9] pass clippy and correct comments --- crates/node/src/types/file.rs | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/crates/node/src/types/file.rs b/crates/node/src/types/file.rs index 451efd6e..ef3a2c40 100644 --- a/crates/node/src/types/file.rs +++ b/crates/node/src/types/file.rs @@ -212,7 +212,7 @@ impl AssetFile { checksum, } => { //verify the url is valide. - reqwest::Url::parse(&file_url)?; + reqwest::Url::parse(file_url)?; let mut file_name_path = Path::new(&file_name); if file_name_path.is_absolute() { file_name_path = file_name_path.strip_prefix("/").unwrap(); // Unwrap tested in `is_absolute()`. @@ -290,15 +290,11 @@ impl TxFile { pub fn into_download_file(self, tx_hash: Hash) -> AssetFile { let relative_path = self.get_relatif_path(tx_hash); - let url = format!( - "{}/{}", - self.url, - relative_path.to_str().unwrap().to_string() - ); + let url = format!("{}/{}", self.url, relative_path.to_str().unwrap()); AssetFile { name: self.name, - file_path: relative_path.into(), + file_path: relative_path, url, checksum: self.checksum, verify_exist: true,