From 187f790387950310747f034ebd45065e59a6dab8 Mon Sep 17 00:00:00 2001 From: kraktus Date: Fri, 2 Sep 2022 23:25:48 +0200 Subject: [PATCH 1/3] fix: Allow to build scaphandre without warp10 refactor: use macro for exporters match from CLI --- src/exporters/mod.rs | 1 + src/lib.rs | 111 +++++++++++++++++++++++-------------------- src/main.rs | 1 + 3 files changed, 62 insertions(+), 51 deletions(-) diff --git a/src/exporters/mod.rs b/src/exporters/mod.rs index 6cc26d9e..713dc95c 100644 --- a/src/exporters/mod.rs +++ b/src/exporters/mod.rs @@ -8,6 +8,7 @@ pub mod qemu; pub mod riemann; pub mod stdout; pub mod utils; +#[cfg(feature = "warp10")] pub mod warpten; use crate::sensors::{utils::current_system_time_since_epoch, RecordGenerator, Topology}; use chrono::Utc; diff --git a/src/lib.rs b/src/lib.rs index da878f71..3f18355a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -9,9 +9,11 @@ pub mod exporters; pub mod sensors; use clap::ArgMatches; use colored::*; +#[cfg(feature = "warp10")] +use exporters::warpten::Warp10Exporter; use exporters::{ json::JSONExporter, prometheus::PrometheusExporter, qemu::QemuExporter, - riemann::RiemannExporter, stdout::StdoutExporter, warpten::Warp10Exporter, Exporter, + riemann::RiemannExporter, stdout::StdoutExporter, Exporter, }; use sensors::{powercap_rapl::PowercapRAPLSensor, Sensor}; use std::collections::HashMap; @@ -50,6 +52,19 @@ fn get_sensor(matches: &ArgMatches) -> Box { Box::new(sensor) } +macro_rules! declare_exporters { + ($header:tt, $exporter_match_flag:tt, $matches:tt, $($name:tt, $exporter:ty,)+) => {$( + if let Some(exporter_parameters) = $matches.subcommand_matches($name) { + $exporter_match_flag = true; + if $header { + scaphandre_header($name); + } + let mut exporter = <$exporter>::new(get_sensor(&$matches)); // FIXME + exporter.run(exporter_parameters.clone()); + } + )+} +} + /// Matches the sensor and exporter name and options requested from the command line and /// creates the appropriate instances. Launchs the standardized entrypoint of /// the choosen exporter: run() @@ -57,57 +72,47 @@ fn get_sensor(matches: &ArgMatches) -> Box { pub fn run(matches: ArgMatches) { loggerv::init_with_verbosity(matches.occurrences_of("v")).unwrap(); - let sensor_boxed = get_sensor(&matches); - let exporter_parameters; - let mut header = true; + let mut exporter_match_flag = false; if matches.is_present("no-header") { header = false; } - if let Some(stdout_exporter_parameters) = matches.subcommand_matches("stdout") { - if header { - scaphandre_header("stdout"); - } - exporter_parameters = stdout_exporter_parameters.clone(); - let mut exporter = StdoutExporter::new(sensor_boxed); - exporter.run(exporter_parameters); - } else if let Some(json_exporter_parameters) = matches.subcommand_matches("json") { - if header { - scaphandre_header("json"); - } - exporter_parameters = json_exporter_parameters.clone(); - let mut exporter = JSONExporter::new(sensor_boxed); - exporter.run(exporter_parameters); - } else if let Some(riemann_exporter_parameters) = matches.subcommand_matches("riemann") { - if header { - scaphandre_header("riemann"); - } - exporter_parameters = riemann_exporter_parameters.clone(); - let mut exporter = RiemannExporter::new(sensor_boxed); - exporter.run(exporter_parameters); - } else if let Some(prometheus_exporter_parameters) = matches.subcommand_matches("prometheus") { - if header { - scaphandre_header("prometheus"); - } - exporter_parameters = prometheus_exporter_parameters.clone(); - let mut exporter = PrometheusExporter::new(sensor_boxed); - exporter.run(exporter_parameters); - } else if let Some(qemu_exporter_parameters) = matches.subcommand_matches("qemu") { - if header { - scaphandre_header("qemu"); - } - exporter_parameters = qemu_exporter_parameters.clone(); - let mut exporter = QemuExporter::new(sensor_boxed); - exporter.run(exporter_parameters); - } else if let Some(warp10_exporter_parameters) = matches.subcommand_matches("warp10") { - if header { - scaphandre_header("warp10"); - } - exporter_parameters = warp10_exporter_parameters.clone(); - let mut exporter = Warp10Exporter::new(sensor_boxed); - exporter.run(exporter_parameters); - } else { + #[cfg(not(feature = "warp10"))] + declare_exporters!( + header, + exporter_match_flag, + matches, + "stdout", + StdoutExporter, + "json", + JSONExporter, + "riemann", + RiemannExporter, + "prometheus", + PrometheusExporter, + "qemu", + QemuExporter, + ); + #[cfg(feature = "warp10")] + declare_exporters!( + header, + exporter_match_flag, + matches, + "stdout", + StdoutExporter, + "json", + JSONExporter, + "riemann", + RiemannExporter, + "prometheus", + PrometheusExporter, + "qemu", + QemuExporter, + "warp10", // <-- Added + Warp10Exporter, // <-- Added + ); + if !exporter_match_flag { error!("Couldn't determine which exporter has been chosen."); } } @@ -136,10 +141,14 @@ pub fn get_exporters_options() -> HashMap "Prometheus exporter exposes power consumption metrics on an http endpoint (/metrics is default) in prometheus accepted format", "riemann" => "Riemann exporter sends power consumption metrics to a Riemann server", "qemu" => "Qemu exporter watches all Qemu/KVM virtual machines running on the host and exposes metrics of each of them in a dedicated folder", + #[cfg(feature = "warp10")] "warp10" => "Warp10 exporter sends data to a Warp10 host, through HTTP", _ => "Unknown exporter", } From ff29d2429a9faa681e62916cbd92c8b50543db97 Mon Sep 17 00:00:00 2001 From: kraktus Date: Fri, 2 Sep 2022 23:38:08 +0200 Subject: [PATCH 2/3] fix: Allow scaphandre to build without "containers" feature try to fix compilation --- src/exporters/mod.rs | 162 ++++++++++++++++++++++++----------------- src/exporters/utils.rs | 9 ++- src/sensors/utils.rs | 7 +- 3 files changed, 107 insertions(+), 71 deletions(-) diff --git a/src/exporters/mod.rs b/src/exporters/mod.rs index 713dc95c..eb99e4e4 100644 --- a/src/exporters/mod.rs +++ b/src/exporters/mod.rs @@ -13,13 +13,17 @@ pub mod warpten; use crate::sensors::{utils::current_system_time_since_epoch, RecordGenerator, Topology}; use chrono::Utc; use clap::ArgMatches; -use docker_sync::{container::Container, Docker}; -use k8s_sync::kubernetes::Kubernetes; -use k8s_sync::Pod; +#[cfg(feature = "containers")] +use { + docker_sync::{container::Container, Docker}, + k8s_sync::{kubernetes::Kubernetes, Pod}, + utils::{get_docker_client, get_kubernetes_client}, +}; + use std::collections::HashMap; use std::fmt; use std::time::Duration; -use utils::{get_docker_client, get_kubernetes_client, get_scaphandre_version}; +use utils::get_scaphandre_version; /// General metric definition. #[derive(Debug)] @@ -110,6 +114,12 @@ struct MetricGenerator { hostname: String, /// Tells MetricGenerator if it has to watch for qemu virtual machines. qemu: bool, + #[cfg(feature = "containers")] + cd: ContainerData, +} + +#[cfg(feature = "containers")] +struct ContainerData { /// Tells MetricGenerator if it has to watch for containers. watch_containers: bool, /// @@ -139,7 +149,7 @@ struct MetricGenerator { /// to use the following methods to avoid discrepancies between exporters. impl MetricGenerator { /// Returns a MetricGenerator instance that will host metrics. - + #[allow(unused_variables)] fn new( topology: Topology, hostname: String, @@ -147,49 +157,61 @@ impl MetricGenerator { watch_containers: bool, ) -> MetricGenerator { let data = Vec::new(); - let containers = vec![]; - let pods = vec![]; - let docker_version = String::from(""); - let mut docker_client = None; - //let kubernetes_version = String::from(""); - let mut kubernetes_client = None; - if watch_containers { - let mut container_runtime = false; - match get_docker_client() { - Ok(docker) => { - docker_client = Some(docker); + #[cfg(feature = "containers")] + { + let containers = vec![]; + let pods = vec![]; + let docker_version = String::from(""); + let mut docker_client = None; + //let kubernetes_version = String::from(""); + let mut kubernetes_client = None; + if watch_containers { + let mut container_runtime = false; + match get_docker_client() { + Ok(docker) => { + docker_client = Some(docker); + container_runtime = true; + } + Err(err) => { + info!("Couldn't connect to docker socket. Error: {}", err); + } + } + if let Ok(kubernetes) = get_kubernetes_client() { + kubernetes_client = Some(kubernetes); container_runtime = true; + } else { + info!("Couldn't connect to kubernetes API."); } - Err(err) => { - info!("Couldn't connect to docker socket. Error: {}", err); + if !container_runtime { + warn!("--containers was used but scaphandre couldn't connect to any container runtime."); } } - if let Ok(kubernetes) = get_kubernetes_client() { - kubernetes_client = Some(kubernetes); - container_runtime = true; - } else { - info!("Couldn't connect to kubernetes API."); - } - if !container_runtime { - warn!("--containers was used but scaphandre couldn't connect to any container runtime."); + let cd = ContainerData { + watch_containers, + containers_last_check: String::from(""), + pods_last_check: String::from(""), + containers, + docker_version, + docker_client, + watch_docker: true, + watch_kubernetes: true, + kubernetes_client, + pods, + }; + MetricGenerator { + data, + topology, + hostname, + qemu, + cd, } } + #[cfg(not(feature = "containers"))] MetricGenerator { data, topology, hostname, - containers, qemu, - containers_last_check: String::from(""), - docker_version, - docker_client, - watch_containers, - watch_docker: true, - kubernetes_client, - watch_kubernetes: true, - pods, - pods_last_check: String::from(""), - //kubernetes_version, } } @@ -570,17 +592,18 @@ impl MetricGenerator { } } - /// If *self.watch_docker* is true and *self.docker_client* is Some + /// If *self.cd.watch_docker* is true and *self.cd.docker_client* is Some /// gets the list of docker containers running on the machine, thanks - /// to *self.docker_client*. Stores the resulting vector as *self.containers*. - /// Updates *self.containers_last_check* to the current timestamp, if the + /// to *self.cd.docker_client*. Stores the resulting vector as *self.cd.containers*. + /// Updates *self.cd.containers_last_check* to the current timestamp, if the /// operation is successful. + #[cfg(feature = "containers")] fn gen_docker_containers_basic_metadata(&mut self) { - if self.watch_docker && self.docker_client.is_some() { - if let Some(docker) = self.docker_client.as_mut() { + if self.cd.watch_docker && self.cd.docker_client.is_some() { + if let Some(docker) = self.cd.docker_client.as_mut() { if let Ok(containers_result) = docker.get_containers(false) { - self.containers = containers_result; - self.containers_last_check = + self.cd.containers = containers_result; + self.cd.containers_last_check = current_system_time_since_epoch().as_secs().to_string(); } } else { @@ -589,45 +612,49 @@ impl MetricGenerator { } } - /// If *self.watch_kubernetes* is true, + /// If *self.cd.watch_kubernetes* is true, /// queries the local kubernetes API (if this is a kubernetes cluster node) - /// and retrieves the list of pods running on this node, thanks to *self.kubernetes_client*. - /// Stores the result as *self.pods* and updates *self.pods_last_check* if the operation is successfull. + /// and retrieves the list of pods running on this node, thanks to *self.cd.kubernetes_client*. + /// Stores the result as *self.cd.pods* and updates *self.cd.pods_last_check* if the operation is successfull. + #[cfg(feature = "containers")] fn gen_kubernetes_pods_basic_metadata(&mut self) { - if self.watch_kubernetes { - if let Some(kubernetes) = self.kubernetes_client.as_mut() { + if self.cd.watch_kubernetes { + if let Some(kubernetes) = self.cd.kubernetes_client.as_mut() { if let Ok(pods_result) = kubernetes.list_pods("".to_string()) { - self.pods = pods_result; - debug!("Found {} pods", &self.pods.len()); + self.cd.pods = pods_result; + debug!("Found {} pods", &self.cd.pods.len()); } else { info!("Failed getting pods list, despite client seems ok."); } } else { debug!("Kubernetes socket is not some."); } - self.pods_last_check = current_system_time_since_epoch().as_secs().to_string(); + self.cd.pods_last_check = current_system_time_since_epoch().as_secs().to_string(); } } /// Generate process metrics. fn gen_process_metrics(&mut self) { - if self.watch_containers { + #[cfg(feature = "containers")] + if self.cd.watch_containers { let now = current_system_time_since_epoch().as_secs().to_string(); - if self.watch_docker && self.docker_client.is_some() { - let last_check = self.containers_last_check.clone(); + if self.cd.watch_docker && self.cd.docker_client.is_some() { + let last_check = self.cd.containers_last_check.clone(); if last_check.is_empty() { - match self.docker_client.as_mut().unwrap().get_version() { + match self.cd.docker_client.as_mut().unwrap().get_version() { Ok(version_response) => { - self.docker_version = String::from(version_response.Version.as_str()); + self.cd.docker_version = + String::from(version_response.Version.as_str()); self.gen_docker_containers_basic_metadata(); } Err(error) => { info!("Couldn't query the docker socket: {}", error); - self.watch_docker = false; + self.cd.watch_docker = false; } } } else { match self + .cd .docker_client .as_mut() .unwrap() @@ -642,15 +669,15 @@ impl MetricGenerator { Err(err) => debug!("couldn't get docker events - {:?} - {}", err, err), } } - self.containers_last_check = + self.cd.containers_last_check = current_system_time_since_epoch().as_secs().to_string(); } - if self.watch_kubernetes && self.kubernetes_client.is_some() { - if self.pods_last_check.is_empty() { + if self.cd.watch_kubernetes && self.cd.kubernetes_client.is_some() { + if self.cd.pods_last_check.is_empty() { self.gen_kubernetes_pods_basic_metadata(); info!("First check done on pods."); } - let last_check = self.pods_last_check.clone(); + let last_check = self.cd.pods_last_check.clone(); if (now.parse::().unwrap() - last_check.parse::().unwrap()) > 20 { info!( "Just refreshed pod list ! last: {} now: {}, diff: {}", @@ -669,15 +696,18 @@ impl MetricGenerator { let mut attributes = HashMap::new(); - if self.watch_containers && (!self.containers.is_empty() || !self.pods.is_empty()) { + #[cfg(feature = "containers")] + if self.cd.watch_containers + && (!self.cd.containers.is_empty() || !self.cd.pods.is_empty()) + { let container_data = self .topology .proc_tracker .get_process_container_description( pid, - &self.containers, - self.docker_version.clone(), - &self.pods, + &self.cd.containers, + self.cd.docker_version.clone(), + &self.cd.pods, //self.kubernetes_version.clone(), ); diff --git a/src/exporters/utils.rs b/src/exporters/utils.rs index 99e99c6e..b6211939 100644 --- a/src/exporters/utils.rs +++ b/src/exporters/utils.rs @@ -2,8 +2,11 @@ //! //! The utils module provides common functions used by the exporters. use clap::crate_version; -use docker_sync::Docker; -use k8s_sync::{errors::KubernetesError, kubernetes::Kubernetes}; +#[cfg(feature = "containers")] +use { + docker_sync::Docker, + k8s_sync::{errors::KubernetesError, kubernetes::Kubernetes}, +}; /// Returns an Option containing the VM name of a qemu process. /// @@ -83,6 +86,7 @@ mod tests { } } +#[cfg(feature = "containers")] pub fn get_docker_client() -> Result { let docker = match Docker::connect() { Ok(docker) => docker, @@ -91,6 +95,7 @@ pub fn get_docker_client() -> Result { Ok(docker) } +#[cfg(feature = "containers")] pub fn get_kubernetes_client() -> Result { match Kubernetes::connect( Some(String::from("/root/.kube/config")), diff --git a/src/sensors/utils.rs b/src/sensors/utils.rs index 95f1b3b8..2ddb261b 100644 --- a/src/sensors/utils.rs +++ b/src/sensors/utils.rs @@ -1,9 +1,8 @@ -use docker_sync::container::Container; -use k8s_sync::Pod; use procfs::process::Process; use regex::Regex; -use std::collections::HashMap; use std::time::{Duration, SystemTime}; +#[cfg(feature = "containers")] +use {docker_sync::container::Container, k8s_sync::Pod, std::collections::HashMap}; #[derive(Debug, Clone)] /// Manages ProcessRecord instances. @@ -157,6 +156,7 @@ impl ProcessTracker { } /// Extracts the container_id from a cgroup path containing it. + #[cfg(feature = "containers")] fn extract_pod_id_from_cgroup_path(&self, pathname: String) -> Result { let mut container_id = String::from(pathname.split('/').last().unwrap()); if container_id.starts_with("docker-") { @@ -174,6 +174,7 @@ impl ProcessTracker { /// currently running docker containers on the machine. /// The *pods* slice contains the [Pod] items referencing currently /// running pods on the machine if it is a kubernetes cluster node. + #[cfg(feature = "containers")] pub fn get_process_container_description( &self, pid: i32, From a76464c9ead275ff03319596611f32b46795a8d4 Mon Sep 17 00:00:00 2001 From: kraktus Date: Sat, 3 Sep 2022 11:15:22 +0200 Subject: [PATCH 3/3] refactor: exporters complete path useless since already imported --- src/lib.rs | 27 ++++++--------------------- 1 file changed, 6 insertions(+), 21 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 3f18355a..6e8945a2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -121,32 +121,17 @@ pub fn run(matches: ArgMatches) { /// This function has to be updated to enable a new exporter. pub fn get_exporters_options() -> HashMap>> { let mut options = HashMap::new(); - options.insert( - String::from("stdout"), - exporters::stdout::StdoutExporter::get_options(), - ); - options.insert( - String::from("json"), - exporters::json::JSONExporter::get_options(), - ); + options.insert(String::from("stdout"), StdoutExporter::get_options()); + options.insert(String::from("json"), JSONExporter::get_options()); options.insert( String::from("prometheus"), - exporters::prometheus::PrometheusExporter::get_options(), - ); - options.insert( - String::from("riemann"), - exporters::riemann::RiemannExporter::get_options(), - ); - options.insert( - String::from("qemu"), - exporters::qemu::QemuExporter::get_options(), + PrometheusExporter::get_options(), ); + options.insert(String::from("riemann"), RiemannExporter::get_options()); + options.insert(String::from("qemu"), QemuExporter::get_options()); #[cfg(feature = "warp10")] { - options.insert( - String::from("warp10"), - exporters::warpten::Warp10Exporter::get_options(), - ); + options.insert(String::from("warp10"), Warp10Exporter::get_options()); } options