diff --git a/src/error.rs b/src/error.rs index 46156d8..88a34c1 100644 --- a/src/error.rs +++ b/src/error.rs @@ -10,4 +10,14 @@ pub enum Error { #[error("IO error when calling Nix: {0}")] NixIO(#[from] std::io::Error), + + #[error("Erorr when sending data to a mpsc channel: {0}")] + Mpsc(Box>), +} + +// Cannot automatically derive using #[from] because of the Box +impl From> for Error { + fn from(e: std::sync::mpsc::SendError) -> Self { + Error::Mpsc(Box::new(e)) + } } diff --git a/src/lib.rs b/src/lib.rs index 2053949..7c2ad1d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,5 +1,6 @@ use ::std::sync::{Arc, Mutex}; use rayon::prelude::*; +use std::sync::mpsc; use error::Result; use nix::{DerivationDescription, FoundDrv}; @@ -9,109 +10,112 @@ pub mod nix; fn process( collected_paths: &Arc>>, - flake_ref: &str, - system: Option<&str>, - attribute_path: &str, - offline: &bool, + flake_ref: &String, + system: &Option, + attribute_path: String, + offline: bool, lib: &nix::lib::Lib, -) -> Vec { + // Sender channel to communicate DerivationDescription to the main thread + tx: mpsc::Sender, +) -> Result<()> { + // check if the build_input has already be processed + let done = { + let mut collected_paths = collected_paths.lock().unwrap(); + !collected_paths.insert(attribute_path.to_string()) + }; + + if done { + log::debug!("Skipping already processed derivation: {}", attribute_path); + return Ok(()); + } + log::debug!("Processing derivation: {:?}", attribute_path); // call describe_derivation to get the derivation description let description = - nix::describe_derivation(flake_ref, system, attribute_path, offline, lib).unwrap(); + nix::describe_derivation(flake_ref, system, &attribute_path, &offline, lib).unwrap(); + + // Send the DerivationDescription to the main thread + // TODO: Error handling + tx.send(description.clone())?; // use par_iter to call process on all children of this derivation - let children: Vec = description + description .build_inputs - .par_iter() - .map(|build_input| { - // check if the build_input has already be processed - let done = { - let mut collected_paths = collected_paths.lock().unwrap(); - match &build_input.output_path { - Some(output_path) => !collected_paths.insert(output_path.clone()), - None => false, - } - }; - - if done { - log::debug!( - "Skipping already processed derivation: {}", - build_input.attribute_path - ); - Vec::new() - } else { - process( - collected_paths, - flake_ref, - system, - &build_input.attribute_path, - offline, - lib, - ) - } + .into_par_iter() + .map(|build_input| -> Result<()> { + process( + collected_paths, + flake_ref, + system, + build_input.attribute_path, + offline, + lib, + tx.clone(), + ) }) - .flatten() - .collect(); + .collect::>>()?; - // combine the children and this derivation into a single Vec - let mut descriptions = children; - descriptions.push(description); - descriptions + Ok(()) } pub fn nixtract( - flake_ref: impl AsRef, - system: Option>, - attribute_path: Option>, - offline: &bool, -) -> Result> { - let flake_ref = flake_ref.as_ref(); - // Convert system to a Option<&str> - let system = system.as_ref().map(AsRef::as_ref); - let attribute_path = attribute_path.as_ref().map(AsRef::as_ref); + flake_ref: impl Into, + system: Option>, + attribute_path: Option>, + offline: bool, +) -> Result> { + // Convert the arguments to the expected types + let flake_ref = flake_ref.into(); + let system = system.map(Into::into); + let attribute_path = attribute_path.map(Into::into); // Writes the `lib.nix` file to the tempdir and stores its path let lib = nix::lib::Lib::new()?; + // Create a channel to communicate DerivationDescription to the main thread + let (tx, rx) = mpsc::channel(); + log::info!( "Starting nixtract with flake_ref: {}, system: {}, attribute_path: {:?}", flake_ref, system - .map(AsRef::as_ref) - .unwrap_or("builtins.currentSystem"), - attribute_path.map(AsRef::as_ref).unwrap_or("") + .clone() + .unwrap_or("builtins.currentSystem".to_owned()), + attribute_path.clone().unwrap_or_default() ); - let collected_paths: Arc>> = - Arc::new(Mutex::new(std::collections::HashSet::new())); + // Spawn a new rayon thread to call process on every foundDrv + rayon::spawn(move || { + let collected_paths: Arc>> = + Arc::new(Mutex::new(std::collections::HashSet::new())); - // call find_attribute_paths to get the initial set of derivations - let attribute_paths = - nix::find_attribute_paths(flake_ref, system, attribute_path, offline, &lib)?; + // call find_attribute_paths to get the initial set of derivations + let attribute_paths = + nix::find_attribute_paths(&flake_ref, &system, &attribute_path, &offline, &lib) + .unwrap(); - // Combine all AttributePaths into a single Vec - let mut derivations: Vec = Vec::new(); - for attribute_path in attribute_paths { - derivations.extend(attribute_path.found_drvs); - } + // Combine all AttributePaths into a single Vec + let mut derivations: Vec = Vec::new(); + for attribute_path in attribute_paths { + derivations.extend(attribute_path.found_drvs); + } - // call process on every foundDrv - let descriptions: Vec = derivations - .par_iter() - .map(|found_drv| { - process( + derivations.into_par_iter().for_each(|found_drv| { + match process( &collected_paths, - flake_ref, - system, - &found_drv.attribute_path, + &flake_ref, + &system, + found_drv.attribute_path, offline, &lib, - ) - }) - .flatten() - .collect(); + tx.clone(), + ) { + Ok(_) => {} + Err(e) => log::error!("Error processing derivation: {}", e), + } + }); + }); - Ok(descriptions) + Ok(rx.into_iter()) } diff --git a/src/main.rs b/src/main.rs index 897e169..d8bd5c2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -72,25 +72,30 @@ fn main() -> Result<(), Box> { opts.flake_ref, opts.system, opts.attribute_path, - &opts.offline, + opts.offline, )?; - // Print the results - let output = if opts.pretty { - serde_json::to_string_pretty(&results)? - } else { - serde_json::to_string(&results)? - }; - - match opts.output_path.as_deref() { - None | Some("-") => { - println!("{}", output); - } - Some(output_path) => { - log::info!("Writing results to {:?}", output_path); - std::fs::write(output_path, output)?; + // Create the out writer + let mut out_writer = match opts.output_path.as_deref() { + None | Some("-") => Box::new(std::io::stdout()) as Box, + Some(path) => { + let file = std::fs::File::create(path)?; + Box::new(file) as Box } }; + // Print the results + for result in results { + let output = if opts.pretty { + serde_json::to_string_pretty(&result)? + } else { + serde_json::to_string(&result)? + }; + + // Append to the out_writer + out_writer.write_all(output.as_bytes())?; + out_writer.write_all(b"\n")?; + } + Ok(()) } diff --git a/src/nix/describe_derivation.rs b/src/nix/describe_derivation.rs index a696698..7bc7dae 100644 --- a/src/nix/describe_derivation.rs +++ b/src/nix/describe_derivation.rs @@ -5,7 +5,7 @@ use serde::{Deserialize, Serialize}; use super::lib::Lib; use crate::error::{Error, Result}; -#[derive(Deserialize, Serialize, Debug, Eq, PartialEq)] +#[derive(Deserialize, Serialize, Debug, Eq, PartialEq, Clone)] #[serde(rename_all = "camelCase")] pub struct DerivationDescription { pub attribute_path: String, @@ -19,21 +19,21 @@ pub struct DerivationDescription { pub build_inputs: Vec, } -#[derive(Deserialize, Serialize, Debug, Eq, PartialEq)] +#[derive(Deserialize, Serialize, Debug, Eq, PartialEq, Clone)] #[serde(rename_all = "camelCase")] pub struct Output { pub name: String, pub output_path: Option, } -#[derive(Deserialize, Serialize, Debug, Eq, PartialEq)] +#[derive(Deserialize, Serialize, Debug, Eq, PartialEq, Clone)] #[serde(rename_all = "camelCase")] pub struct ParsedName { pub name: String, pub version: String, } -#[derive(Deserialize, Serialize, Debug, Eq, PartialEq)] +#[derive(Deserialize, Serialize, Debug, Eq, PartialEq, Clone)] #[serde(rename_all = "camelCase")] pub struct NixpkgsMetadata { pub description: String, @@ -44,7 +44,7 @@ pub struct NixpkgsMetadata { pub licenses: Option>, } -#[derive(Deserialize, Serialize, Debug, Eq, PartialEq)] +#[derive(Deserialize, Serialize, Debug, Eq, PartialEq, Clone)] #[serde(rename_all = "camelCase")] pub struct Source { pub git_repo_url: String, @@ -52,7 +52,7 @@ pub struct Source { pub rev: String, } -#[derive(Deserialize, Serialize, Debug, Eq, PartialEq)] +#[derive(Deserialize, Serialize, Debug, Eq, PartialEq, Clone)] #[serde(rename_all = "camelCase")] pub struct License { // Not all licenses in nixpkgs have an associated spdx id @@ -60,7 +60,7 @@ pub struct License { pub full_name: String, } -#[derive(Deserialize, Serialize, Debug, Eq, PartialEq)] +#[derive(Deserialize, Serialize, Debug, Eq, PartialEq, Clone)] #[serde(rename_all = "camelCase")] pub struct BuiltInput { pub attribute_path: String, @@ -69,9 +69,9 @@ pub struct BuiltInput { } pub fn describe_derivation( - flake_ref: impl AsRef, - system: Option>, - attribute_path: impl AsRef, + flake_ref: &String, + system: &Option, + attribute_path: &String, offline: &bool, lib: &Lib, ) -> Result { @@ -80,17 +80,17 @@ pub fn describe_derivation( // Create a scope so env_vars isn't needlessly mutable let env_vars: HashMap = { let mut res = HashMap::from([ - ("TARGET_FLAKE_REF".to_owned(), flake_ref.as_ref().to_owned()), + ("TARGET_FLAKE_REF".to_owned(), flake_ref.to_owned()), ( "TARGET_ATTRIBUTE_PATH".to_owned(), - attribute_path.as_ref().to_owned(), + attribute_path.to_owned(), ), ("NIXPKGS_ALLOW_UNFREE".to_owned(), "1".to_owned()), ("NIXPKGS_ALLOW_INSECURE".to_owned(), "1".to_owned()), ("NIXPKGS_ALLOW_BROKEN".to_owned(), "1".to_owned()), ]); if let Some(system) = system { - res.insert("TARGET_SYSTEM".to_owned(), system.as_ref().to_owned()); + res.insert("TARGET_SYSTEM".to_owned(), system.to_owned()); } res }; diff --git a/src/nix/find_attribute_paths.rs b/src/nix/find_attribute_paths.rs index e988d80..8bf81ab 100644 --- a/src/nix/find_attribute_paths.rs +++ b/src/nix/find_attribute_paths.rs @@ -20,9 +20,9 @@ pub struct FoundDrv { } pub fn find_attribute_paths( - flake_ref: impl AsRef, - system: Option>, - attribute_path: Option>, + flake_ref: &String, + system: &Option, + attribute_path: &Option, offline: &bool, lib: &Lib, ) -> Result> { @@ -31,19 +31,16 @@ pub fn find_attribute_paths( // Create a scope so env_vars isn't needlessly mutable let env_vars: HashMap = { let mut res = HashMap::from([ - ("TARGET_FLAKE_REF".to_owned(), flake_ref.as_ref().to_owned()), + ("TARGET_FLAKE_REF".to_owned(), flake_ref.to_owned()), ("NIXPKGS_ALLOW_UNFREE".to_owned(), "1".to_owned()), ("NIXPKGS_ALLOW_INSECURE".to_owned(), "1".to_owned()), ("NIXPKGS_ALLOW_BROKEN".to_owned(), "1".to_owned()), ]); if let Some(attribute_path) = attribute_path { - res.insert( - "TARGET_ATTRIBUTE_PATH".to_owned(), - attribute_path.as_ref().to_owned(), - ); + res.insert("TARGET_ATTRIBUTE_PATH".to_owned(), attribute_path.clone()); } if let Some(system) = system { - res.insert("TARGET_SYSTEM".to_owned(), system.as_ref().to_owned()); + res.insert("TARGET_SYSTEM".to_owned(), system.to_owned()); } res }; diff --git a/src/nix/lib.rs b/src/nix/lib.rs index 5b67ca0..cdbb3c4 100644 --- a/src/nix/lib.rs +++ b/src/nix/lib.rs @@ -1,5 +1,4 @@ //! Nix only accepts a file as included files, so we need to create a temporary file to pass to it -//! TODO: Allow sharing of a single lib instance so we don't copy the file so many times use crate::error::Result; use std::io::Write;