add support for redis as cache storage

Cargo.toml

@@ -3,13 +3,22 @@ name = "cade"
 version = "0.1.0"
 edition = "2021"
 
+[[bin]]
+name = "cade"
+path = "src/wrapper/main.rs"
+
+# [lib]
+# name = "common"
+# path = "src/lib.rs"
+
 [dependencies]
 serde = { version = "1.0.194", features = ["derive"] }
 config = "0.13.4"
 blake3 = "1.5.0"
 lz4 = "1.24.0"
 zstd = "0.13.0"
 pathdiff = "0.2.1"
+redis = "0.25.2"
 
 # needed for profiling
 [profile.release]

src/common/mod.rs

@@ -0,0 +1 @@
+pub mod compression;

src/lib.rs

@@ -0,0 +1 @@
+pub mod common;

src/cache/cache.rs → src/wrapper/cache/cache.rs

@@ -1,24 +1,29 @@
 use std::{path::Path, io};
-use crate::{compression::zstd, wrapper_config};
+use crate::config;
+use cade::common::compression::zstd;
 
-use super::{provider::CacheProvider, file_provider::FileCacheProvider};
+use super::{provider::CacheProvider, file_provider::FileCacheProvider, redis_provider::RedisProvider};
 
 /// This is a handler for caching.
 /// its purpose is to abstract the caching part from the rest of the logic
 pub struct Cache {
-    providers: Vec<Box<dyn CacheProvider + 'static>>,
+    providers: Vec<Box<dyn CacheProvider + 'static>>
 }
 
 impl Cache {
-    pub fn new(cache_configs: &Vec<wrapper_config::CacheConfig>) -> Cache {
+    pub fn new(config: &config::WrapperConfig) -> Cache {
         let mut providers:Vec<Box<dyn CacheProvider + 'static>> = Vec::new();
-        for config in cache_configs {
-            match config.variant.as_str() {
-                "filesystem" => {
-                    let path = Path::new(&config.path);
-                    let provider = FileCacheProvider::new(path, config.update_on_hit);
-                    providers.push(Box::new(provider));},
-                _ => panic!("Unknown cache provider: {}", config.variant),
+        for cache_config in &config.cache {
+            match cache_config {
+                config::CacheConfig::filesystem(filesystem_config) => {
+                    let path = Path::new(&filesystem_config.path);
+                    let provider = FileCacheProvider::new(path, filesystem_config.update_on_hit, config.panic_on_cache_content_mismatch);
+                    providers.push(Box::new(provider));
+                },
+                config::CacheConfig::redis(redis_config) => {
+                    let provider = RedisProvider::new(&redis_config.url, redis_config.update_on_hit, config.panic_on_cache_content_mismatch, redis_config.expire);
+                    providers.push(Box::new(provider));
+                }
             }
         }
         Cache {
@@ -55,6 +60,4 @@ impl Cache {
         let compressed_data = zstd::compress(&data);
         self.update_all_entry(category, key, &compressed_data);
     }
-
-
 }

src/cache/file_provider.rs → src/wrapper/cache/file_provider.rs

@@ -6,17 +6,26 @@ use super::provider::CacheProvider;
 
 pub struct FileCacheProvider {
     path: PathBuf,
-    update: bool
+    update: bool,
+    panic_on_cache_content_mismatch: bool
 }
 
 impl FileCacheProvider {
-    pub fn new(path: &Path, update: bool) -> FileCacheProvider {
+    pub fn new(path: &Path, update: bool, panic_on_cache_content_mismatch: bool) -> FileCacheProvider {
         if !path.exists() {
             fs::create_dir_all(path).expect(&format!("Unable to create directory '{}'!", path.to_str().unwrap()));
         }
         FileCacheProvider {
             path: path.to_path_buf(),
-            update: update
+            update: update,
+            panic_on_cache_content_mismatch: panic_on_cache_content_mismatch
+        }
+    }
+
+    fn get_path(&self, category: Option<&str>, key: &str) -> PathBuf {
+        match category {
+            Some(category) => self.path.join(category).join(key),
+            None => self.path.join(key),
         }
     }
 }
@@ -29,7 +38,7 @@ impl CacheProvider for FileCacheProvider {
 
     fn set_entry(&self, category: Option<&str>, key: &str, value: &Vec<u8>) {
         let path = self.get_path(category, key);
-        if path.exists() {
+        if self.panic_on_cache_content_mismatch && path.exists() && category != Some("obj") {
             let input_data = std::fs::read(&path).expect(&format!("Unable to read input file '{}'!", path.to_str().unwrap()));
             if input_data != *value {
                 panic!("content of '{}' does not match expected value! (hash collision?)", path.to_str().unwrap());
@@ -50,15 +59,4 @@ impl CacheProvider for FileCacheProvider {
     fn update(&self) -> bool {
         self.update
     }
-}
-
-impl FileCacheProvider {
-    fn get_path(&self, category: Option<&str>, key: &str) -> PathBuf {
-        let path = match category {
-            Some(category) => self.path.join(category).join(key),
-            None => self.path.join(key),
-        };
-
-        path
-    }
 }

src/wrapper/cache/mod.rs

@@ -0,0 +1,4 @@
+pub(crate) mod cache;
+mod provider;
+mod file_provider;
+mod redis_provider;

src/wrapper/cache/redis_provider.rs

@@ -0,0 +1,90 @@
+use std::io;
+
+use redis::{Commands, Expiry, RedisError};
+
+use super::provider::CacheProvider;
+
+pub struct RedisProvider {
+    update: bool,
+    panic_on_cache_content_mismatch: bool,
+    expire: Option<u32>,
+    client: redis::Client
+}
+
+impl RedisProvider {
+    pub fn new(url: &str, update: bool, panic_on_cache_content_mismatch: bool, expire: Option<u32>) -> RedisProvider {
+        let client = redis::Client::open(url).unwrap();
+
+        RedisProvider {
+            update: update,
+            panic_on_cache_content_mismatch: panic_on_cache_content_mismatch,
+            expire: expire,
+            client: client
+        }
+    }
+
+    fn get_key(&self, category: Option<&str>, key: &str) -> String {
+        match category {
+            Some(category) => format!("{}_{}", category, key),
+            None => key.to_string()
+        }
+    }
+}
+
+impl CacheProvider for RedisProvider {
+    fn get_entry(&self, category: Option<&str>, key: &str) -> io::Result<Vec<u8>> {
+        let full_key = self.get_key(category, key);
+
+        let mut con = self.client.get_connection().unwrap();
+
+        let ret: Result<Vec<u8>, RedisError>;
+        if self.expire.is_some() {
+            ret = con.get_ex(&full_key, Expiry::EX(self.expire.unwrap().try_into().unwrap()));
+        } else {
+            ret = con.get(&full_key);
+        }
+
+        match ret {
+            Ok(data) => {
+                // retrun err if data is empty
+                if data.len() == 0 {
+                    Err(io::Error::new(io::ErrorKind::NotFound, "Not found"))
+                } else {
+                    Ok(data)
+                }
+            },
+            Err(err) => Err(io::Error::new(io::ErrorKind::Other, err))
+        }
+    }
+
+    fn set_entry(&self, category: Option<&str>, key: &str, value: &Vec<u8>) {
+        let full_key = self.get_key(category, key);
+        let mut con = self.client.get_connection().unwrap();
+        if self.has_entry(category, key) {
+            let _:() = con.expire(&full_key, self.expire.unwrap().into()).unwrap();
+
+            if self.panic_on_cache_content_mismatch && category != Some("obj") {
+                let input_data = self.get_entry(category, key).expect(&format!("Unable to access redis key '{}'!", full_key));
+                if input_data != *value {
+                    panic!("content of '{}' does not match expected value! (hash collision?)", full_key);
+                }
+            }
+        } else {
+            if self.expire.is_some() {
+                let _:() = con.set_ex(&full_key, value, self.expire.unwrap().into()).unwrap();
+            } else {
+                let _:() = con.set(&full_key, value).unwrap();
+            }
+        }
+    }
+
+    fn has_entry(&self, category: Option<&str>, key: &str) -> bool {
+        let mut con = self.client.get_connection().unwrap();
+
+        con.exists(self.get_key(category, key)).unwrap()
+    }
+
+    fn update(&self) -> bool {
+        self.update
+    }
+}

src/cache_handler.rs → src/wrapper/cache_handler.rs

@@ -1,4 +1,4 @@
-use std::{io, process::{Output, Command}};
+use std::{io, process::{Command, Output}};
 
 pub trait CacheHandler {
     fn cache_lookup(&mut self, args: &Vec<String>) -> bool;
@@ -9,7 +9,6 @@ pub trait CacheHandler {
     fn get_stderr_key(&self) -> Option<&String>;
 
     fn execute(&mut self, args: &Vec<String>) -> io::Result<Output> {
-
         let status = Command::new(&args[0])
             .args(&args[1..])
             .output();

src/compiler/compile_handler.rs → src/wrapper/compiler/compile_handler.rs

@@ -1,7 +1,7 @@
 use std::{path::Path, env, fs, io, process::Output, str};
 use pathdiff::diff_paths;
 
-use crate::{cache_handler::CacheHandler, hash::{hash, Hasher}, cache::cache::Cache, wrapper_config};
+use crate::{cache_handler::CacheHandler, hash::{hash, Hasher}, cache::cache::Cache, config};
 
 use super::{response_file, dep_parser::{self, DepParser}, gcc, tasking};
 
@@ -21,7 +21,7 @@ pub struct Compiler<'a> {
     cache: &'a Cache,
     total_hash: Option<String>,
     source_hash:Option<String>,
-    config: &'a wrapper_config::WrapperConfig
+    config: &'a config::WrapperConfig
 }
 
 struct CompilerArgs {
@@ -154,11 +154,14 @@ fn handle_path_arg(arg: &str, prefix:&str, next_arg: &Option<&String>) -> (bool,
 }
 
 impl<'a> Compiler<'a> {
-    pub fn new(exe_name:&str, cache: &'a Cache, config: &'a wrapper_config::WrapperConfig) -> Compiler<'a> {
+    pub fn new(exe_name:&str, cache: &'a Cache, config: &'a config::WrapperConfig) -> Compiler<'a> {
         // Compiler{name: compiler.get_name()}
         let compiler: Box<dyn CompilerTrait>;
         match exe_name {
-            "gcc"  => { compiler = Box::new(gcc::Gcc{}); },
+            "gcc" |
+            "g++" |
+            "tricore-gcc" |
+            "tricore-g++" => { compiler = Box::new(gcc::Gcc{}); },
             "cctc" => { compiler = Box::new(tasking::Tasking{}); },
             _ => {
                 println!("Unknown compiler");
@@ -262,6 +265,10 @@ impl<'a> Compiler<'a> {
         let dep_hash = hash(&data);
         self.cache.set_entry(Some("dep"), &self.source_hash.as_ref().unwrap(), &dep_hash.as_bytes().to_vec());
 
+        if self.config.debug {
+            fs::write(self.parsed_args.out_file.as_ref().unwrap().to_owned() + ".cade_dep", data).unwrap();
+        }
+
         self.cache.set_entry(Some("dep"), &dep_hash, &data);
     }
 

src/compiler/dep_parser.rs → src/wrapper/compiler/dep_parser.rs

@@ -40,9 +40,9 @@ impl DepParser {
                             val += el2;
                             if el2.ends_with('\\') {
                                 val += " ";
-                            } else  if !map.contains(&val) {
+                            } else  if !map.contains(&val.to_lowercase()) {
                                 self.deps.push(val.to_owned());
-                                map.insert(val.to_owned());
+                                map.insert(val.to_lowercase());
                                 val = "".to_owned();
                             } else {
                                 val = "".to_owned();
@@ -52,9 +52,9 @@ impl DepParser {
                     }
                 } else {
                     // escaped
-                    if is_prereq && !map.contains(el) {
+                    if is_prereq && !map.contains(&el.to_lowercase()) {
                         self.deps.push(el.to_owned());
-                        map.insert(el.to_owned());
+                        map.insert(el.to_lowercase());
                     }
                     column += el.len();
                 }
@@ -69,7 +69,7 @@ impl DepParser {
     pub fn update_hash(&self, hasher: &mut Hasher) {
         for dep in &self.deps {
             // dbg!("create hash for: {}", dep);
-            hasher.update(&std::fs::read(dep).expect(&("Unable to read input file (".to_owned() + dep + ")!")));
+            hasher.update(&std::fs::read(dep.replace("\\ ", " ")).expect(&("Unable to read input file (".to_owned() + dep + ")!")));
         }
     }
 

src/wrapper_config.rs → src/wrapper/config.rs

@@ -9,14 +9,29 @@ pub enum CacheAccess {
 }
 
 #[derive(Deserialize)]
-pub struct CacheConfig {
-    pub variant: String,
+pub struct FilesystemConfig {
     pub path: String,
     pub access: CacheAccess,
     #[serde(default = "update_on_hit_default")]
     pub update_on_hit: bool
 }
 
+#[derive(Deserialize)]
+pub struct RedisConfig {
+    pub url: String,
+    pub expire: Option<u32>,
+    pub access: CacheAccess,
+    #[serde(default = "update_on_hit_default")]
+    pub update_on_hit: bool
+}
+
+#[derive(Deserialize)]
+#[allow(non_camel_case_types)]
+pub enum CacheConfig {
+    filesystem(FilesystemConfig),
+    redis(RedisConfig)
+}
+
 fn update_on_hit_default() -> bool {
     true
 }
@@ -25,6 +40,18 @@ fn update_on_hit_default() -> bool {
 pub struct WrapperConfig {
     pub base_dir: Option<String>,
     pub cache: Vec<CacheConfig>,
+    #[serde(default = "debug_default")]
+    pub debug: bool,
+    #[serde(default = "panic_on_cache_content_mismatch_default")]
+    pub panic_on_cache_content_mismatch: bool,
+}
+
+fn debug_default() -> bool {
+    false
+}
+
+fn panic_on_cache_content_mismatch_default() -> bool {
+    false
 }
 
 impl WrapperConfig {