From 0f1e711cf613574efae9d2d0fdc4b71f9a7479b6 Mon Sep 17 00:00:00 2001 From: Simon Gerber Date: Sat, 27 Jan 2024 21:23:56 +0100 Subject: [PATCH 1/4] Add support for reclass option `ignore_class_notfound_regexp` The patterns provided in `ignore_class_notfound_regexp` (which is expected to be a list of strings) are checked when `ignore_class_notfound=true`. By default, `ignore_class_notfound_regexp` is set to the single pattern '.*', so that all missing classes are ignored. --- Cargo.toml | 1 + README.md | 2 +- src/config.rs | 29 +++++++++++++ src/node/mod.rs | 22 +++++++++- ...nder_tests_ignore_class_notfound_regexp.rs | 41 +++++++++++++++++++ .../classes/a.yml | 2 + .../classes/b.yml | 2 + .../classes/c.yml | 2 + .../classes/d.yml | 2 + .../nodes/n1.yml | 8 ++++ .../nodes/n2.yml | 2 + .../reclass-config.yml | 9 ++++ 12 files changed, 120 insertions(+), 2 deletions(-) create mode 100644 src/node/node_render_tests_ignore_class_notfound_regexp.rs create mode 100644 tests/inventory-class-notfound-regexp/classes/a.yml create mode 100644 tests/inventory-class-notfound-regexp/classes/b.yml create mode 100644 tests/inventory-class-notfound-regexp/classes/c.yml create mode 100644 tests/inventory-class-notfound-regexp/classes/d.yml create mode 100644 tests/inventory-class-notfound-regexp/nodes/n1.yml create mode 100644 tests/inventory-class-notfound-regexp/nodes/n2.yml create mode 100644 tests/inventory-class-notfound-regexp/reclass-config.yml diff --git a/Cargo.toml b/Cargo.toml index ce9787a..acdf4f7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,6 +22,7 @@ indexmap = "2.2.3" nom = "7.1.3" pyo3 = { version = "0.20.2", features = ["chrono"] } rayon = "1.8.1" +regex = "1.10.3" serde = { version = "1.0.197", features = ["derive"] } serde_json = "1.0.114" serde_yaml = "0.9.32" diff --git a/README.md b/README.md index ff990d0..f3a3b52 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ The implementation currently supports the following features of Kapicorp Reclass * The Reclass options `nodes_path` and `classes_path` * The Reclass option `ignore_class_notfound` +* The Reclass option `ignore_class_notfound_regexp` * Escaped parameter references * Merging referenced lists and dictionaries * Constant parameters @@ -27,7 +28,6 @@ The following Kapicorp Reclass features aren't supported: * Ignoring overwritten missing references * Inventory Queries -* The Reclass option `ignore_class_notfound_regexp` * The Reclass option `allow_none_override` can't be set to `False` * The Reclass `yaml_git` and `mixed` storage types * Any Reclass option which is not mentioned explicitly here or above diff --git a/src/config.rs b/src/config.rs index 269585a..fe9990d 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,5 +1,6 @@ use anyhow::{anyhow, Result}; use pyo3::prelude::*; +use regex::RegexSet; use std::collections::hash_map::DefaultHasher; use std::collections::HashSet; use std::hash::{Hash, Hasher}; @@ -63,6 +64,9 @@ pub struct Config { pub compose_node_name: bool, /// Python Reclass compatibility flags. See `CompatFlag` for available flags. #[pyo3(get)] + pub ignore_class_notfound_regexp: Vec, + pub ignore_class_notfound_regexset: RegexSet, + #[pyo3(get)] pub compatflags: HashSet, } @@ -114,6 +118,8 @@ impl Config { classes_path: to_lexical_normal(&cpath, true).display().to_string(), ignore_class_notfound: ignore_class_notfound.unwrap_or(false), compose_node_name: false, + ignore_class_notfound_regexp: vec![".*".to_string()], + ignore_class_notfound_regexset: RegexSet::new([".*"])?, compatflags: HashSet::new(), }) } @@ -158,6 +164,22 @@ impl Config { "Expected value of config key 'ignore_class_notfound' to be a boolean" ))?; } + "ignore_class_notfound_regexp" => { + let list = v.as_sequence().ok_or(anyhow!( + "Expected value of config key 'ignore_class_notfound_regexp' to be a list" + ))?; + self.ignore_class_notfound_regexp.clear(); + for val in list { + self.ignore_class_notfound_regexp.push( + val.as_str() + .ok_or(anyhow!( + "Expected entry of 'ignore_class_notfound_regexp' to be a string" + ))? + .to_string(), + ); + } + self.ignore_class_notfound_regexp.shrink_to_fit(); + } "compose_node_name" => { self.compose_node_name = v.as_bool().ok_or(anyhow!( "Expected value of config key 'compose_node_name' to be a boolean" @@ -185,6 +207,13 @@ impl Config { } } } + self.compile_ignore_class_notfound_patterns()?; + Ok(()) + } + + fn compile_ignore_class_notfound_patterns(&mut self) -> Result<()> { + self.ignore_class_notfound_regexset = RegexSet::new(&self.ignore_class_notfound_regexp) + .map_err(|e| anyhow!("while compiling ignore_class_notfound regex patterns: {e}"))?; Ok(()) } diff --git a/src/node/mod.rs b/src/node/mod.rs index 9ef17db..6cd7e05 100644 --- a/src/node/mod.rs +++ b/src/node/mod.rs @@ -155,9 +155,27 @@ impl Node { // Lookup path for provided class in r.classes, handling ignore_class_notfound let Some(classinfo) = r.classes.get(&cls) else { - if r.config.ignore_class_notfound { + // ignore_class_notfound_regexp is only applied if ignore_class_notfound == true. + // By default the regexset has a single pattern for .* so that all missing classes are + // ignored. + if r.config.ignore_class_notfound + && r.config.ignore_class_notfound_regexset.is_match(&cls) + { return Ok(None); } + if r.config.ignore_class_notfound { + // return an error informing the user that we didn't ignore the missing class + // based on the configured regex patterns. + eprintln!( + "Missing class '{cls}' not ignored due to configured regex patterns: [{}]", + r.config + .ignore_class_notfound_regexp + .iter() + .map(|s| format!("'{s}'")) + .collect::>() + .join(", ") + ); + } return Err(anyhow!("Class {cls} not found")); }; @@ -545,3 +563,5 @@ mod node_tests { #[cfg(test)] mod node_render_tests; +#[cfg(test)] +mod node_render_tests_ignore_class_notfound_regexp; diff --git a/src/node/node_render_tests_ignore_class_notfound_regexp.rs b/src/node/node_render_tests_ignore_class_notfound_regexp.rs new file mode 100644 index 0000000..90b617b --- /dev/null +++ b/src/node/node_render_tests_ignore_class_notfound_regexp.rs @@ -0,0 +1,41 @@ +use crate::types::Value; +use crate::{Config, Reclass}; + +#[test] +fn test_render_n1() { + let mut c = Config::new( + Some("./tests/inventory-class-notfound-regexp"), + None, + None, + None, + ) + .unwrap(); + c.load_from_file("reclass-config.yml").unwrap(); + let r = Reclass::new_from_config(c).unwrap(); + + let n1 = r.render_node("n1").unwrap(); + assert_eq!( + n1.classes, + vec!["service.foo", "service.bar", "missing", "a", "amissing"] + ); + assert_eq!( + n1.parameters.get(&"a".into()), + Some(&Value::Literal("a".into())) + ); +} + +#[test] +fn test_render_n2() { + let mut c = Config::new( + Some("./tests/inventory-class-notfound-regexp"), + None, + None, + None, + ) + .unwrap(); + c.load_from_file("reclass-config.yml").unwrap(); + let r = Reclass::new_from_config(c).unwrap(); + + let n2 = r.render_node("n2"); + assert!(n2.is_err()); +} diff --git a/tests/inventory-class-notfound-regexp/classes/a.yml b/tests/inventory-class-notfound-regexp/classes/a.yml new file mode 100644 index 0000000..a1b5258 --- /dev/null +++ b/tests/inventory-class-notfound-regexp/classes/a.yml @@ -0,0 +1,2 @@ +parameters: + a: a diff --git a/tests/inventory-class-notfound-regexp/classes/b.yml b/tests/inventory-class-notfound-regexp/classes/b.yml new file mode 100644 index 0000000..eae7f80 --- /dev/null +++ b/tests/inventory-class-notfound-regexp/classes/b.yml @@ -0,0 +1,2 @@ +parameters: + b: b diff --git a/tests/inventory-class-notfound-regexp/classes/c.yml b/tests/inventory-class-notfound-regexp/classes/c.yml new file mode 100644 index 0000000..9ddda85 --- /dev/null +++ b/tests/inventory-class-notfound-regexp/classes/c.yml @@ -0,0 +1,2 @@ +parameters: + c: c diff --git a/tests/inventory-class-notfound-regexp/classes/d.yml b/tests/inventory-class-notfound-regexp/classes/d.yml new file mode 100644 index 0000000..4d77361 --- /dev/null +++ b/tests/inventory-class-notfound-regexp/classes/d.yml @@ -0,0 +1,2 @@ +parameters: + d: d diff --git a/tests/inventory-class-notfound-regexp/nodes/n1.yml b/tests/inventory-class-notfound-regexp/nodes/n1.yml new file mode 100644 index 0000000..cab9a10 --- /dev/null +++ b/tests/inventory-class-notfound-regexp/nodes/n1.yml @@ -0,0 +1,8 @@ +classes: + - service.foo + - service.bar + - missing + - a + - amissing + +parameters: {} diff --git a/tests/inventory-class-notfound-regexp/nodes/n2.yml b/tests/inventory-class-notfound-regexp/nodes/n2.yml new file mode 100644 index 0000000..9c51f04 --- /dev/null +++ b/tests/inventory-class-notfound-regexp/nodes/n2.yml @@ -0,0 +1,2 @@ +classes: + - foo diff --git a/tests/inventory-class-notfound-regexp/reclass-config.yml b/tests/inventory-class-notfound-regexp/reclass-config.yml new file mode 100644 index 0000000..ed35fef --- /dev/null +++ b/tests/inventory-class-notfound-regexp/reclass-config.yml @@ -0,0 +1,9 @@ +# Add reclass-config.yml for Kapitan/Python reclass +nodes_uri: nodes +classes_uri: classes +compose_node_name: false +allow_none_override: true +ignore_class_notfound: true +ignore_class_notfound_regexp: + - service\..* + - .*missing.* From 26ae7b5454895b1246864216c07fef096cd02a3b Mon Sep 17 00:00:00 2001 From: Simon Gerber Date: Tue, 20 Feb 2024 16:32:53 +0100 Subject: [PATCH 2/4] Add Python tests for `ignore_class_notfound_regexp` --- tests/test_ignore_class_notfound_regexp.py | 26 ++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 tests/test_ignore_class_notfound_regexp.py diff --git a/tests/test_ignore_class_notfound_regexp.py b/tests/test_ignore_class_notfound_regexp.py new file mode 100644 index 0000000..f95b42c --- /dev/null +++ b/tests/test_ignore_class_notfound_regexp.py @@ -0,0 +1,26 @@ +import pytest + +import reclass_rs + + +def test_ignore_regexp_render_n1(): + r = reclass_rs.Reclass.from_config( + "./tests/inventory-class-notfound-regexp", "reclass-config.yml" + ) + assert r.config.ignore_class_notfound_regexp == ["service\\..*", ".*missing.*"] + + n1 = r.nodeinfo("n1") + + assert n1 is not None + + +def test_ignore_regexp_render_n1(): + r = reclass_rs.Reclass.from_config( + "./tests/inventory-class-notfound-regexp", "reclass-config.yml" + ) + assert r.config.ignore_class_notfound_regexp == ["service\\..*", ".*missing.*"] + + with pytest.raises( + ValueError, match="Error while rendering n2: Class foo not found" + ): + n2 = r.nodeinfo("n2") From 7973d58124dfedf19a8ab98bf9d06c759d6e25e9 Mon Sep 17 00:00:00 2001 From: Simon Gerber Date: Tue, 20 Feb 2024 16:33:08 +0100 Subject: [PATCH 3/4] Make `ignore_class_notfound_regexp` config fields private Instead of exposing the fields (which must be kept in sync) directly, we expose a getter & setter pair. The setter ensures that a new RegexSet is compiled after the internal `ignore_class_notfound_regexp` field is updated. We still expose the field read-only to Python through a generated PyO3 getter. --- src/config.rs | 33 +++++++++++++++++++++++++++++++-- src/node/mod.rs | 7 +++---- 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/src/config.rs b/src/config.rs index fe9990d..cb645f6 100644 --- a/src/config.rs +++ b/src/config.rs @@ -64,8 +64,8 @@ pub struct Config { pub compose_node_name: bool, /// Python Reclass compatibility flags. See `CompatFlag` for available flags. #[pyo3(get)] - pub ignore_class_notfound_regexp: Vec, - pub ignore_class_notfound_regexset: RegexSet, + ignore_class_notfound_regexp: Vec, + ignore_class_notfound_regexset: RegexSet, #[pyo3(get)] pub compatflags: HashSet, } @@ -211,6 +211,22 @@ impl Config { Ok(()) } + /// Returns the currently configured `ignore_class_notfound_regexp` pattern list. + pub fn get_ignore_class_notfound_regexp(&self) -> &Vec { + &self.ignore_class_notfound_regexp + } + + /// Updates the saved ignore_class_notfound_regexp pattern list with the provided list and + /// ensures that the precompiled RegexSet is updated to match the new pattern list. + pub fn set_ignore_class_notfound_regexp(&mut self, patterns: Vec) -> Result<()> { + self.ignore_class_notfound_regexp = patterns; + self.compile_ignore_class_notfound_patterns() + } + + pub(crate) fn is_class_ignored(&self, cls: &str) -> bool { + self.ignore_class_notfound && self.ignore_class_notfound_regexset.is_match(cls) + } + fn compile_ignore_class_notfound_patterns(&mut self) -> Result<()> { self.ignore_class_notfound_regexset = RegexSet::new(&self.ignore_class_notfound_regexp) .map_err(|e| anyhow!("while compiling ignore_class_notfound regex patterns: {e}"))?; @@ -315,4 +331,17 @@ mod tests { assert_eq!(cfg.classes_path, "./inventory/classes"); assert_eq!(cfg.ignore_class_notfound, false); } + + #[test] + fn test_config_update_ignore_class_notfound_patterns() { + let mut cfg = Config::new(Some("./inventory"), None, None, None).unwrap(); + assert_eq!(cfg.ignore_class_notfound_regexp, vec![".*"]); + + cfg.set_ignore_class_notfound_regexp(vec![".*foo".into(), "bar.*".into()]) + .unwrap(); + + assert!(cfg.ignore_class_notfound_regexset.is_match("thefooer")); + assert!(cfg.ignore_class_notfound_regexset.is_match("baring")); + assert!(!cfg.ignore_class_notfound_regexset.is_match("bazzer")); + } } diff --git a/src/node/mod.rs b/src/node/mod.rs index 6cd7e05..a937832 100644 --- a/src/node/mod.rs +++ b/src/node/mod.rs @@ -158,18 +158,17 @@ impl Node { // ignore_class_notfound_regexp is only applied if ignore_class_notfound == true. // By default the regexset has a single pattern for .* so that all missing classes are // ignored. - if r.config.ignore_class_notfound - && r.config.ignore_class_notfound_regexset.is_match(&cls) - { + if r.config.is_class_ignored(&cls) { return Ok(None); } + if r.config.ignore_class_notfound { // return an error informing the user that we didn't ignore the missing class // based on the configured regex patterns. eprintln!( "Missing class '{cls}' not ignored due to configured regex patterns: [{}]", r.config - .ignore_class_notfound_regexp + .get_ignore_class_notfound_regexp() .iter() .map(|s| format!("'{s}'")) .collect::>() From d2d8ba4a36f00348f5324d3ccf76c33eafc4c44a Mon Sep 17 00:00:00 2001 From: Simon Gerber Date: Tue, 20 Feb 2024 17:27:51 +0100 Subject: [PATCH 4/4] Add setter for `ignore_class_notfound_regexp()` in the Python interface --- src/lib.rs | 12 ++++++++++++ tests/test_ignore_class_notfound_regexp.py | 13 ++++++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index 81cfbe7..db8a7d5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -368,6 +368,18 @@ impl Reclass { .collect::>(); Ok(res) } + + /// Update the current Reclass instance's config object with the provided + /// `ignore_class_notfound_regexp` patterns + pub fn set_ignore_class_notfound_regexp(&mut self, patterns: Vec) -> PyResult<()> { + self.config + .set_ignore_class_notfound_regexp(patterns) + .map_err(|e| { + PyValueError::new_err(format!( + "Error while setting ignore_class_notfound_regexp: {e}" + )) + }) + } } impl Default for Reclass { diff --git a/tests/test_ignore_class_notfound_regexp.py b/tests/test_ignore_class_notfound_regexp.py index f95b42c..9ef1b83 100644 --- a/tests/test_ignore_class_notfound_regexp.py +++ b/tests/test_ignore_class_notfound_regexp.py @@ -14,7 +14,7 @@ def test_ignore_regexp_render_n1(): assert n1 is not None -def test_ignore_regexp_render_n1(): +def test_ignore_regexp_render_n2(): r = reclass_rs.Reclass.from_config( "./tests/inventory-class-notfound-regexp", "reclass-config.yml" ) @@ -24,3 +24,14 @@ def test_ignore_regexp_render_n1(): ValueError, match="Error while rendering n2: Class foo not found" ): n2 = r.nodeinfo("n2") + + +def test_ignore_regexp_update_config_render_n2(): + r = reclass_rs.Reclass.from_config( + "./tests/inventory-class-notfound-regexp", "reclass-config.yml" + ) + r.set_ignore_class_notfound_regexp([".*"]) + assert r.config.ignore_class_notfound_regexp == [".*"] + + n2 = r.nodeinfo("n2") + assert n2 is not None