initial draft: rabbitmq

vipyrsec · Aug 30, 2023 · 51b87ca · 51b87ca
1 parent b2d1e72
commit 51b87ca
Show file tree

Hide file tree

Showing 9 changed files with 1,289 additions and 471 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -4,17 +4,26 @@ version = "0.1.0"
 edition = "2021"
 
 [dependencies]
+async-compression = { version = "0.4.1", features = ["gzip", "tokio", "futures-io"] }
+async-tar = "0.4.2"
+async_zip = "0.0.15"
+chrono = "0.4.26"
 env_logger = "0.10.0"
 figment = { version = "0.10.10", features = ["env", "toml"] }
 flate2 = "1.0.26"
+futures = "0.3.28"
+futures-io = "0.3.28"
+lapin = "2.3.1"
 log = "0.4.17"
 once_cell = "1.18.0"
 parking_lot = "0.12.1"
-reqwest = { version = "0.11.18", features = ["blocking", "json", "serde_json", "gzip"] }
+reqwest = { version = "0.11.18", features = ["blocking", "json", "serde_json", "gzip", "stream"] }
 serde = { version = "1.0.180", features = ["derive"] }
+serde_json = "1.0.104"
 tar = "0.4.39"
 thiserror = "1.0.44"
 threadpool = "1.8.1"
+tokio = { version = "1.29.1", features = ["full"] }
 tracing = "0.1.37"
 tracing-subscriber = { version = "0.3.17", features = ["env-filter"] }
 yara = "0.20.0"

diff --git a/src/app_config.rs b/src/app_config.rs
@@ -19,6 +19,8 @@ pub struct AppConfig {
     pub username: String,
     pub password: String,
     pub max_scan_size: u64,
+    pub amqp: String,
+    pub prefetch: u16,
 }
 
 impl Default for AppConfig {
@@ -41,6 +43,8 @@ impl Default for AppConfig {
             bulk_size: 20,
             load_duration: 60,
             max_scan_size: 1.28e+8 as u64, // 128 MB
+            amqp: String::from("amqp://127.0.0.1:5672/%2f"),
+            prefetch: available_parallelism as u16,
         }
     }
 }