From 74e5668101417ff25208dd24d450cc7598b6b6c4 Mon Sep 17 00:00:00 2001 From: up9cloud <8325632+up9cloud@users.noreply.github.com> Date: Tue, 19 Sep 2023 17:47:38 +0000 Subject: [PATCH 1/3] Support jsonc format Signed-off-by: T.C. <8325632+up9cloud@users.noreply.github.com> --- Cargo.toml | 4 +- README.md | 4 +- src/file/format/jsonc.rs | 61 +++++++++++ src/file/format/mod.rs | 17 +++ src/lib.rs | 2 +- tests/Settings-enum-test.jsonc | 4 + tests/Settings-invalid.jsonc | 4 + tests/Settings.jsonc | 23 ++++ tests/file_jsonc.rs | 195 +++++++++++++++++++++++++++++++++ 9 files changed, 311 insertions(+), 3 deletions(-) create mode 100644 src/file/format/jsonc.rs create mode 100644 tests/Settings-enum-test.jsonc create mode 100644 tests/Settings-invalid.jsonc create mode 100644 tests/Settings.jsonc create mode 100644 tests/file_jsonc.rs diff --git a/Cargo.toml b/Cargo.toml index 36267375..7aab950c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,11 +15,12 @@ edition = "2018" maintenance = { status = "actively-developed" } [features] -default = ["toml", "json", "yaml", "ini", "ron", "json5", "convert-case", "async"] +default = ["toml", "json", "yaml", "ini", "ron", "json5", "jsonc", "convert-case", "async"] json = ["serde_json"] yaml = ["yaml-rust"] ini = ["rust-ini"] json5 = ["json5_rs", "serde/derive"] +jsonc = ["jsonc-parser"] convert-case = ["convert_case"] preserve_order = ["indexmap", "toml/preserve_order", "serde_json/preserve_order", "ron/indexmap"] async = ["async-trait"] @@ -36,6 +37,7 @@ yaml-rust = { version = "0.4", optional = true } rust-ini = { version = "0.19", optional = true } ron = { version = "0.8", optional = true } json5_rs = { version = "0.4", optional = true, package = "json5" } +jsonc-parser = { version = "0.22.1", optional = true } indexmap = { version = "2.0.0", features = ["serde"], optional = true } convert_case = { version = "0.6", optional = true } pathdiff = "0.2" diff --git a/README.md b/README.md index badab891..23d02a06 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ - Set defaults - Set explicit values (to programmatically override) - - Read from [JSON], [TOML], [YAML], [INI], [RON], [JSON5] files + - Read from [JSON], [TOML], [YAML], [INI], [RON], [JSON5], [JSONC] files - Read from environment - Loosely typed — Configuration values may be read in any supported type, as long as there exists a reasonable conversion - Access nested fields using a formatted path — Uses a subset of JSONPath; currently supports the child ( `redis.port` ) and subscript operators ( `databases[0].name` ) @@ -22,6 +22,7 @@ [INI]: https://github.com/zonyitoo/rust-ini [RON]: https://github.com/ron-rs/ron [JSON5]: https://github.com/callum-oakley/json5-rs +[JSONC]: https://github.com/dprint/jsonc-parser Please note this library @@ -43,6 +44,7 @@ config = "0.13.1" - `toml` - Adds support for reading TOML files - `ron` - Adds support for reading RON files - `json5` - Adds support for reading JSON5 files + - `jsonc` - Adds support for reading JSONC files ### Support for custom formats diff --git a/src/file/format/jsonc.rs b/src/file/format/jsonc.rs new file mode 100644 index 00000000..c5be8200 --- /dev/null +++ b/src/file/format/jsonc.rs @@ -0,0 +1,61 @@ +use std::error::Error; + +use crate::error::{ConfigError, Unexpected}; +use crate::map::Map; +use crate::value::{Value, ValueKind}; + +use jsonc_parser::JsonValue; + +pub fn parse( + uri: Option<&String>, + text: &str, +) -> Result, Box> { + match jsonc_parser::parse_to_value(text, &Default::default())? { + Some(r) => match r { + JsonValue::String(ref value) => Err(Unexpected::Str(value.to_string())), + JsonValue::Number(value) => Err(Unexpected::Float(value.parse::().unwrap())), + JsonValue::Boolean(value) => Err(Unexpected::Bool(value)), + JsonValue::Object(o) => match from_jsonc_value(uri, JsonValue::Object(o)).kind { + ValueKind::Table(map) => Ok(map), + _ => unreachable!(), + }, + JsonValue::Array(_) => Err(Unexpected::Seq), + JsonValue::Null => Err(Unexpected::Unit), + }, + None => Err(Unexpected::Unit), + } + .map_err(|err| ConfigError::invalid_root(uri, err)) + .map_err(|err| Box::new(err) as Box) +} + +fn from_jsonc_value(uri: Option<&String>, value: JsonValue) -> Value { + let vk = match value { + JsonValue::Null => ValueKind::Nil, + JsonValue::String(v) => ValueKind::String(v.to_string()), + JsonValue::Number(ref value) => { + if let Ok(value) = value.parse::() { + ValueKind::I64(value) + } else if let Ok(value) = value.parse::() { + ValueKind::Float(value) + } else { + unreachable!(); + } + }, + JsonValue::Boolean(v) => ValueKind::Boolean(v), + JsonValue::Object(table) => { + let m = table + .into_iter() + .map(|(k, v)| (k, from_jsonc_value(uri, v))) + .collect(); + ValueKind::Table(m) + } + JsonValue::Array(array) => { + let l = array + .into_iter() + .map(|v| from_jsonc_value(uri, v)) + .collect(); + ValueKind::Array(l) + } + }; + Value::new(uri, vk) +} diff --git a/src/file/format/mod.rs b/src/file/format/mod.rs index 025e98a9..e4d22fcb 100644 --- a/src/file/format/mod.rs +++ b/src/file/format/mod.rs @@ -27,6 +27,9 @@ mod ron; #[cfg(feature = "json5")] mod json5; +#[cfg(feature = "jsonc")] +mod jsonc; + /// File formats provided by the library. /// /// Although it is possible to define custom formats using [`Format`] trait it is recommended to use FileFormat if possible. @@ -55,6 +58,10 @@ pub enum FileFormat { /// JSON5 (parsed with json5) #[cfg(feature = "json5")] Json5, + + /// JSONC (parsed with jsonc) + #[cfg(feature = "jsonc")] + Jsonc, } lazy_static! { @@ -81,6 +88,12 @@ lazy_static! { #[cfg(feature = "json5")] formats.insert(FileFormat::Json5, vec!["json5"]); + #[cfg(all(feature = "jsonc", feature = "json"))] + formats.insert(FileFormat::Jsonc, vec!["jsonc"]); + + #[cfg(all(feature = "jsonc", not(feature = "json")))] + formats.insert(FileFormat::Jsonc, vec!["jsonc", "json"]); + formats }; } @@ -117,6 +130,9 @@ impl FileFormat { #[cfg(feature = "json5")] FileFormat::Json5 => json5::parse(uri, text), + #[cfg(feature = "jsonc")] + FileFormat::Jsonc => jsonc::parse(uri, text), + #[cfg(all( not(feature = "toml"), not(feature = "json"), @@ -124,6 +140,7 @@ impl FileFormat { not(feature = "ini"), not(feature = "ron"), not(feature = "json5"), + not(feature = "jsonc"), ))] _ => unreachable!("No features are enabled, this library won't work without features"), } diff --git a/src/lib.rs b/src/lib.rs index 511c638a..afc2f518 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6,7 +6,7 @@ //! - Environment variables //! - String literals in well-known formats //! - Another Config instance -//! - Files: TOML, JSON, YAML, INI, RON, JSON5 and custom ones defined with Format trait +//! - Files: TOML, JSON, YAML, INI, RON, JSON5, JSONC and custom ones defined with Format trait //! - Manual, programmatic override (via a `.set` method on the Config instance) //! //! Additionally, Config supports: diff --git a/tests/Settings-enum-test.jsonc b/tests/Settings-enum-test.jsonc new file mode 100644 index 00000000..ab6bdb1f --- /dev/null +++ b/tests/Settings-enum-test.jsonc @@ -0,0 +1,4 @@ +{ + // foo + "bar": "bar is a lowercase param", +} diff --git a/tests/Settings-invalid.jsonc b/tests/Settings-invalid.jsonc new file mode 100644 index 00000000..ba2d7cba --- /dev/null +++ b/tests/Settings-invalid.jsonc @@ -0,0 +1,4 @@ +{ + "ok": true, + "error" +} diff --git a/tests/Settings.jsonc b/tests/Settings.jsonc new file mode 100644 index 00000000..1461aa0e --- /dev/null +++ b/tests/Settings.jsonc @@ -0,0 +1,23 @@ +{ + // c + /* c */ + "debug": true, + "debug_json": true, + "production": false, + "arr": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], + "place": { + "name": "Torre di Pisa", + "longitude": 43.7224985, + "latitude": 10.3970522, + "favorite": false, + "reviews": 3866, + "rating": 4.5, + "creator": { + "name": "John Smith", + "username": "jsmith", + "email": "jsmith@localhost" + }, + }, + "FOO": "FOO should be overridden", + "bar": "I am bar", +} diff --git a/tests/file_jsonc.rs b/tests/file_jsonc.rs new file mode 100644 index 00000000..80aec72e --- /dev/null +++ b/tests/file_jsonc.rs @@ -0,0 +1,195 @@ +#![cfg(feature = "jsonc")] + +use serde_derive::Deserialize; + +use config::{Config, File, FileFormat, Map, Value}; +use float_cmp::ApproxEqUlps; +use std::path::PathBuf; + +#[derive(Debug, Deserialize)] +struct Place { + name: String, + longitude: f64, + latitude: f64, + favorite: bool, + telephone: Option, + reviews: u64, + creator: Map, + rating: Option, +} + +#[derive(Debug, Deserialize)] +struct Settings { + debug: f64, + production: Option, + place: Place, + #[serde(rename = "arr")] + elements: Vec, +} + +fn make() -> Config { + Config::builder() + .add_source(File::new("tests/Settings", FileFormat::Jsonc)) + .build() + .unwrap() +} + +#[test] +fn test_file() { + let c = make(); + + // Deserialize the entire file as single struct + let s: Settings = c.try_deserialize().unwrap(); + + assert!(s.debug.approx_eq_ulps(&1.0, 2)); + assert_eq!(s.production, Some("false".to_string())); + assert_eq!(s.place.name, "Torre di Pisa"); + assert!(s.place.longitude.approx_eq_ulps(&43.722_498_5, 2)); + assert!(s.place.latitude.approx_eq_ulps(&10.397_052_2, 2)); + assert!(!s.place.favorite); + assert_eq!(s.place.reviews, 3866); + assert_eq!(s.place.rating, Some(4.5)); + assert_eq!(s.place.telephone, None); + assert_eq!(s.elements.len(), 10); + assert_eq!(s.elements[3], "4".to_string()); + if cfg!(feature = "preserve_order") { + assert_eq!( + s.place + .creator + .into_iter() + .collect::>(), + vec![ + ("name".to_string(), "John Smith".into()), + ("username".into(), "jsmith".into()), + ("email".into(), "jsmith@localhost".into()), + ] + ); + } else { + assert_eq!( + s.place.creator["name"].clone().into_string().unwrap(), + "John Smith".to_string() + ); + } +} + +#[test] +fn test_error_parse() { + let res = Config::builder() + .add_source(File::new("tests/Settings-invalid", FileFormat::Jsonc)) + .build(); + + let path_with_extension: PathBuf = ["tests", "Settings-invalid.jsonc"].iter().collect(); + + assert!(res.is_err()); + assert_eq!( + res.unwrap_err().to_string(), + format!( + "Expected a colon after the string or word in an object property on line 4 column 1. in {}", + path_with_extension.display() + ) + ); +} + +#[derive(Debug, Deserialize, PartialEq)] +enum EnumSettings { + Bar(String), +} + +#[derive(Debug, Deserialize, PartialEq)] +struct StructSettings { + foo: String, + bar: String, +} +#[derive(Debug, Deserialize, PartialEq)] +#[allow(non_snake_case)] +struct CapSettings { + FOO: String, +} + +#[test] +fn test_override_uppercase_value_for_struct() { + std::env::set_var("APP_FOO", "I HAVE BEEN OVERRIDDEN_WITH_UPPER_CASE"); + + let cfg = Config::builder() + .add_source(File::new("tests/Settings", FileFormat::Jsonc)) + .add_source(config::Environment::with_prefix("APP").separator("_")) + .build() + .unwrap(); + + let cap_settings = cfg.clone().try_deserialize::(); + let lower_settings = cfg.try_deserialize::().unwrap(); + + match cap_settings { + Ok(v) => { + // this assertion will ensure that the map has only lowercase keys + assert_ne!(v.FOO, "FOO should be overridden"); + assert_eq!( + lower_settings.foo, + "I HAVE BEEN OVERRIDDEN_WITH_UPPER_CASE".to_string() + ); + } + Err(e) => { + if e.to_string().contains("missing field `FOO`") { + assert_eq!( + lower_settings.foo, + "I HAVE BEEN OVERRIDDEN_WITH_UPPER_CASE".to_string() + ); + } else { + panic!("{}", e); + } + } + } +} + +#[test] +fn test_override_lowercase_value_for_struct() { + std::env::set_var("config_foo", "I have been overridden_with_lower_case"); + + let cfg = Config::builder() + .add_source(File::new("tests/Settings", FileFormat::Jsonc)) + .add_source(config::Environment::with_prefix("config").separator("_")) + .build() + .unwrap(); + + let values: StructSettings = cfg.try_deserialize().unwrap(); + assert_eq!( + values.foo, + "I have been overridden_with_lower_case".to_string() + ); + assert_ne!(values.foo, "I am bar".to_string()); +} + +#[test] +fn test_override_uppercase_value_for_enums() { + std::env::set_var("APPS_BAR", "I HAVE BEEN OVERRIDDEN_WITH_UPPER_CASE"); + + let cfg = Config::builder() + .add_source(File::new("tests/Settings-enum-test", FileFormat::Jsonc)) + .add_source(config::Environment::with_prefix("APPS").separator("_")) + .build() + .unwrap(); + let val: EnumSettings = cfg.try_deserialize().unwrap(); + + assert_eq!( + val, + EnumSettings::Bar("I HAVE BEEN OVERRIDDEN_WITH_UPPER_CASE".to_string()) + ); +} + +#[test] +fn test_override_lowercase_value_for_enums() { + std::env::set_var("test_bar", "I have been overridden_with_lower_case"); + + let cfg = Config::builder() + .add_source(File::new("tests/Settings-enum-test", FileFormat::Jsonc)) + .add_source(config::Environment::with_prefix("test").separator("_")) + .build() + .unwrap(); + + let param: EnumSettings = cfg.try_deserialize().unwrap(); + + assert_eq!( + param, + EnumSettings::Bar("I have been overridden_with_lower_case".to_string()) + ); +} From 026e3ac00fadd72fd57c248e566c90f75567cecb Mon Sep 17 00:00:00 2001 From: up9cloud <8325632+up9cloud@users.noreply.github.com> Date: Tue, 19 Sep 2023 22:00:13 +0000 Subject: [PATCH 2/3] fix ci (fmt, clippy, test case) Signed-off-by: T.C. <8325632+up9cloud@users.noreply.github.com> --- src/file/format/jsonc.rs | 12 ++++++------ tests/file_jsonc.rs | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/file/format/jsonc.rs b/src/file/format/jsonc.rs index c5be8200..14b14f9c 100644 --- a/src/file/format/jsonc.rs +++ b/src/file/format/jsonc.rs @@ -32,15 +32,15 @@ fn from_jsonc_value(uri: Option<&String>, value: JsonValue) -> Value { let vk = match value { JsonValue::Null => ValueKind::Nil, JsonValue::String(v) => ValueKind::String(v.to_string()), - JsonValue::Number(ref value) => { - if let Ok(value) = value.parse::() { - ValueKind::I64(value) - } else if let Ok(value) = value.parse::() { - ValueKind::Float(value) + JsonValue::Number(number) => { + if let Ok(v) = number.parse::() { + ValueKind::I64(v) + } else if let Ok(v) = number.parse::() { + ValueKind::Float(v) } else { unreachable!(); } - }, + } JsonValue::Boolean(v) => ValueKind::Boolean(v), JsonValue::Object(table) => { let m = table diff --git a/tests/file_jsonc.rs b/tests/file_jsonc.rs index 80aec72e..56b36234 100644 --- a/tests/file_jsonc.rs +++ b/tests/file_jsonc.rs @@ -52,7 +52,7 @@ fn test_file() { assert_eq!(s.place.telephone, None); assert_eq!(s.elements.len(), 10); assert_eq!(s.elements[3], "4".to_string()); - if cfg!(feature = "preserve_order") { + if cfg!(feature = "TODO: preserve_order") { assert_eq!( s.place .creator From fb5e297cc79ce2ea88c95fac99497bc74c9f8c19 Mon Sep 17 00:00:00 2001 From: up9cloud <8325632+up9cloud@users.noreply.github.com> Date: Wed, 21 Feb 2024 07:27:55 +0000 Subject: [PATCH 3/3] fix cargo fmt --- tests/file_ini.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/file_ini.rs b/tests/file_ini.rs index e17c67b5..7e0cfb98 100644 --- a/tests/file_ini.rs +++ b/tests/file_ini.rs @@ -48,7 +48,8 @@ fn test_file() { assert_eq!(s.place.telephone, None); if cfg!(feature = "preserve_order") { assert_eq!( - s.place.creator + s.place + .creator .into_iter() .collect::>(), vec![