From d9111683a27a010be56766349f1558c4c83ec04f Mon Sep 17 00:00:00 2001 From: Raman Gupta <rocketraman@gmail.com> Date: Thu, 28 Mar 2024 14:20:47 -0400 Subject: [PATCH] Support node normalization, add a path normalizer The purpose of the path normalizer is to normalize all inbound paths by making them lower case, and removing any "-" characters. Normalizing paths means sources with different idiomatic approaches to defining key values will all map correctly to the defined config classes. The only downside to this is multiple config attributes in the same class that differ only by case can no longer be disambiguated. This should be a rare case and the advantages are more than worth losing this "feature". We also add a LowercaseParameterMapper by default which can handle the normalized paths. --- .../com/sksamuel/hoplite/ConfigLoader.kt | 4 ++ .../sksamuel/hoplite/ConfigLoaderBuilder.kt | 25 ++++++++-- .../com/sksamuel/hoplite/ParameterMapper.kt | 5 ++ .../sksamuel/hoplite/internal/ConfigParser.kt | 48 ++++++++++--------- .../hoplite/internal/PropertySourceLoader.kt | 16 +++++-- .../main/kotlin/com/sksamuel/hoplite/nodes.kt | 12 +++++ ...vironmentVariableOverridePropertySource.kt | 6 +-- .../hoplite/transformer/NodeTransformer.kt | 11 +++++ .../hoplite/transformer/PathNormalizer.kt | 35 ++++++++++++++ .../hoplite/EmptyDecoderRegistryTest.kt | 2 + .../hoplite/transformer/PathNormalizerTest.kt | 44 +++++++++++++++++ 11 files changed, 176 insertions(+), 32 deletions(-) create mode 100644 hoplite-core/src/main/kotlin/com/sksamuel/hoplite/transformer/NodeTransformer.kt create mode 100644 hoplite-core/src/main/kotlin/com/sksamuel/hoplite/transformer/PathNormalizer.kt create mode 100644 hoplite-core/src/test/kotlin/com/sksamuel/hoplite/transformer/PathNormalizerTest.kt diff --git a/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/ConfigLoader.kt b/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/ConfigLoader.kt index 4c655bb1..581bac1c 100644 --- a/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/ConfigLoader.kt +++ b/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/ConfigLoader.kt @@ -9,6 +9,7 @@ import com.sksamuel.hoplite.fp.getOrElse import com.sksamuel.hoplite.internal.CascadeMode import com.sksamuel.hoplite.internal.ConfigParser import com.sksamuel.hoplite.internal.DecodeMode +import com.sksamuel.hoplite.transformer.NodeTransformer import com.sksamuel.hoplite.parsers.ParserRegistry import com.sksamuel.hoplite.preprocessor.Preprocessor import com.sksamuel.hoplite.report.Print @@ -27,6 +28,7 @@ class ConfigLoader( val propertySources: List<PropertySource>, val parserRegistry: ParserRegistry, val preprocessors: List<Preprocessor>, + val nodeTransformers: List<NodeTransformer>, val paramMappers: List<ParameterMapper>, val onFailure: List<(Throwable) -> Unit> = emptyList(), val decodeMode: DecodeMode = DecodeMode.Lenient, @@ -172,6 +174,7 @@ class ConfigLoader( cascadeMode = cascadeMode, preprocessors = preprocessors, preprocessingIterations = preprocessingIterations, + nodeTransformers = nodeTransformers, prefix = prefix, resolvers = resolvers, decoderRegistry = decoderRegistry, @@ -228,6 +231,7 @@ class ConfigLoader( cascadeMode = cascadeMode, preprocessors = preprocessors, preprocessingIterations = preprocessingIterations, + nodeTransformers = nodeTransformers, prefix = null, resolvers = resolvers, decoderRegistry = decoderRegistry, diff --git a/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/ConfigLoaderBuilder.kt b/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/ConfigLoaderBuilder.kt index 38a8f425..1fe8e723 100644 --- a/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/ConfigLoaderBuilder.kt +++ b/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/ConfigLoaderBuilder.kt @@ -5,6 +5,7 @@ import com.sksamuel.hoplite.decoder.DefaultDecoderRegistry import com.sksamuel.hoplite.env.Environment import com.sksamuel.hoplite.internal.CascadeMode import com.sksamuel.hoplite.internal.DecodeMode +import com.sksamuel.hoplite.transformer.NodeTransformer import com.sksamuel.hoplite.parsers.DefaultParserRegistry import com.sksamuel.hoplite.parsers.Parser import com.sksamuel.hoplite.preprocessor.EnvOrSystemPropertyPreprocessor @@ -51,6 +52,7 @@ class ConfigLoaderBuilder private constructor() { private val propertySources = mutableListOf<PropertySource>() private val preprocessors = mutableListOf<Preprocessor>() + private val nodeTransformers = mutableListOf<NodeTransformer>() private val resolvers = mutableListOf<Resolver>() private val paramMappers = mutableListOf<ParameterMapper>() private val parsers = mutableMapOf<String, Parser>() @@ -70,8 +72,8 @@ class ConfigLoaderBuilder private constructor() { /** * Returns a [ConfigLoaderBuilder] with all defaults applied. * - * This means that the default [Decoder]s, [Preprocessor]s, [ParameterMapper]s, [PropertySource]s, - * and [Parser]s are all registered. + * This means that the default [Decoder]s, [Preprocessor]s, [NodeTransformer]s, [ParameterMapper]s, + * [PropertySource]s, and [Parser]s are all registered. * * If you wish to avoid adding defaults, for example to avoid certain decoders or sources, then * use [empty] to obtain an empty ConfigLoaderBuilder and call the various addDefault methods manually. @@ -80,6 +82,7 @@ class ConfigLoaderBuilder private constructor() { return empty() .addDefaultDecoders() .addDefaultPreprocessors() + .addDefaultNodeTransformers() .addDefaultParamMappers() .addDefaultPropertySources() .addDefaultParsers() @@ -88,7 +91,7 @@ class ConfigLoaderBuilder private constructor() { /** * Returns a [ConfigLoaderBuilder] with all defaults applied, using resolvers in place of preprocessors. * - * This means that the default [Decoder]s, [Resolver]s, [ParameterMapper]s, [PropertySource]s, + * This means that the default [Decoder]s, [Resolver]s, [NodeTransformer]s, [ParameterMapper]s, [PropertySource]s, * and [Parser]s are all registered. * * If you wish to avoid adding defaults, for example to avoid certain decoders or sources, then @@ -102,6 +105,7 @@ class ConfigLoaderBuilder private constructor() { return empty() .addDefaultDecoders() .addDefaultResolvers() + .addDefaultNodeTransformers() .addDefaultParamMappers() .addDefaultPropertySources() .addDefaultParsers() @@ -205,6 +209,16 @@ class ConfigLoaderBuilder private constructor() { fun addDefaultPreprocessors() = addPreprocessors(defaultPreprocessors()) + fun addNodeTransformer(nodeTransformer: NodeTransformer): ConfigLoaderBuilder = apply { + this.nodeTransformers.add(nodeTransformer) + } + + fun addNodeTransformers(nodeTransformers: Iterable<NodeTransformer>): ConfigLoaderBuilder = apply { + this.nodeTransformers.addAll(nodeTransformers) + } + + fun addDefaultNodeTransformers() = addNodeTransformers(defaultNodeTransformers()) + fun addParser(ext: String, parser: Parser) = addFileExtensionMapping(ext, parser) fun addParsers(map: Map<String, Parser>) = addFileExtensionMappings(map) @@ -249,6 +263,7 @@ class ConfigLoaderBuilder private constructor() { return addDefaultDecoders() .addDefaultParsers() .addDefaultPreprocessors() + .addDefaultNodeTransformers() .addDefaultParamMappers() .addDefaultPropertySources() } @@ -372,6 +387,7 @@ class ConfigLoaderBuilder private constructor() { propertySources = propertySources.toList(), parserRegistry = DefaultParserRegistry(parsers), preprocessors = preprocessors.toList(), + nodeTransformers = nodeTransformers.toList(), paramMappers = paramMappers.toList(), onFailure = failureCallbacks.toList(), resolvers = resolvers, @@ -407,6 +423,8 @@ fun defaultPreprocessors(): List<Preprocessor> = listOf( LookupPreprocessor, ) +fun defaultNodeTransformers(): List<NodeTransformer> = emptyList() + fun defaultResolvers(): List<Resolver> = listOf( EnvVarContextResolver, SystemPropertyContextResolver, @@ -419,6 +437,7 @@ fun defaultResolvers(): List<Resolver> = listOf( fun defaultParamMappers(): List<ParameterMapper> = listOf( DefaultParamMapper, + LowercaseParamMapper, SnakeCaseParamMapper, KebabCaseParamMapper, AliasAnnotationParamMapper, diff --git a/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/ParameterMapper.kt b/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/ParameterMapper.kt index 6a45dfe0..5ce721d7 100644 --- a/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/ParameterMapper.kt +++ b/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/ParameterMapper.kt @@ -34,6 +34,11 @@ object DefaultParamMapper : ParameterMapper { setOfNotNull(param.name) } +object LowercaseParamMapper : ParameterMapper { + override fun map(param: KParameter, constructor: KFunction<Any>, kclass: KClass<*>): Set<String> = + setOfNotNull(param.name?.lowercase()) +} + /** * Disabled by default so that common ENVVAR PARAMS don't override your lower case * names unexpectedly. diff --git a/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/internal/ConfigParser.kt b/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/internal/ConfigParser.kt index 3b52b78a..bbf45f13 100644 --- a/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/internal/ConfigParser.kt +++ b/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/internal/ConfigParser.kt @@ -14,6 +14,7 @@ import com.sksamuel.hoplite.env.Environment import com.sksamuel.hoplite.fp.flatMap import com.sksamuel.hoplite.fp.invalid import com.sksamuel.hoplite.fp.valid +import com.sksamuel.hoplite.transformer.NodeTransformer import com.sksamuel.hoplite.parsers.ParserRegistry import com.sksamuel.hoplite.preprocessor.Preprocessor import com.sksamuel.hoplite.report.Print @@ -26,31 +27,32 @@ import com.sksamuel.hoplite.secrets.SecretsPolicy import kotlin.reflect.KClass class ConfigParser( - classpathResourceLoader: ClasspathResourceLoader, - parserRegistry: ParserRegistry, - allowEmptyTree: Boolean, - allowNullOverride: Boolean, - cascadeMode: CascadeMode, - preprocessors: List<Preprocessor>, - preprocessingIterations: Int, - private val prefix: String?, - private val resolvers: List<Resolver>, - private val decoderRegistry: DecoderRegistry, - private val paramMappers: List<ParameterMapper>, - private val flattenArraysToString: Boolean, - private val resolveTypesCaseInsensitive: Boolean, - private val allowUnresolvedSubstitutions: Boolean, - private val secretsPolicy: SecretsPolicy?, - private val decodeMode: DecodeMode, - private val useReport: Boolean, - private val obfuscator: Obfuscator, - private val reportPrintFn: Print, - private val environment: Environment?, - private val sealedTypeDiscriminatorField: String?, - private val contextResolverMode: ContextResolverMode, + classpathResourceLoader: ClasspathResourceLoader, + parserRegistry: ParserRegistry, + allowEmptyTree: Boolean, + allowNullOverride: Boolean, + cascadeMode: CascadeMode, + preprocessors: List<Preprocessor>, + preprocessingIterations: Int, + nodeTransformers: List<NodeTransformer>, + private val prefix: String?, + private val resolvers: List<Resolver>, + private val decoderRegistry: DecoderRegistry, + private val paramMappers: List<ParameterMapper>, + private val flattenArraysToString: Boolean, + private val resolveTypesCaseInsensitive: Boolean, + private val allowUnresolvedSubstitutions: Boolean, + private val secretsPolicy: SecretsPolicy?, + private val decodeMode: DecodeMode, + private val useReport: Boolean, + private val obfuscator: Obfuscator, + private val reportPrintFn: Print, + private val environment: Environment?, + private val sealedTypeDiscriminatorField: String?, + private val contextResolverMode: ContextResolverMode, ) { - private val loader = PropertySourceLoader(prefix, classpathResourceLoader, parserRegistry, allowEmptyTree) + private val loader = PropertySourceLoader(prefix, nodeTransformers, classpathResourceLoader, parserRegistry, allowEmptyTree) private val cascader = Cascader(cascadeMode, allowEmptyTree, allowNullOverride) private val preprocessing = Preprocessing(preprocessors, preprocessingIterations) private val decoding = Decoding(decoderRegistry, secretsPolicy) diff --git a/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/internal/PropertySourceLoader.kt b/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/internal/PropertySourceLoader.kt index 21a95efb..68cb70b4 100644 --- a/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/internal/PropertySourceLoader.kt +++ b/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/internal/PropertySourceLoader.kt @@ -1,6 +1,5 @@ package com.sksamuel.hoplite.internal -import com.sksamuel.hoplite.filter import com.sksamuel.hoplite.ClasspathResourceLoader import com.sksamuel.hoplite.ConfigFailure import com.sksamuel.hoplite.ConfigResult @@ -8,23 +7,29 @@ import com.sksamuel.hoplite.ConfigSource import com.sksamuel.hoplite.Node import com.sksamuel.hoplite.PropertySource import com.sksamuel.hoplite.PropertySourceContext +import com.sksamuel.hoplite.filter import com.sksamuel.hoplite.fp.NonEmptyList import com.sksamuel.hoplite.fp.flatMap import com.sksamuel.hoplite.fp.invalid import com.sksamuel.hoplite.fp.sequence import com.sksamuel.hoplite.fp.valid +import com.sksamuel.hoplite.transformer.NodeTransformer +import com.sksamuel.hoplite.transformer.PathNormalizer import com.sksamuel.hoplite.parsers.ParserRegistry import com.sksamuel.hoplite.sources.ConfigFilePropertySource +import com.sksamuel.hoplite.transform /** * Loads [Node]s from [PropertySource]s, [ConfigSource]s, files and classpath resources. */ class PropertySourceLoader( - private val prefix: String?, + prefix: String?, + private val nodeTransformers: List<NodeTransformer>, private val classpathResourceLoader: ClasspathResourceLoader, private val parserRegistry: ParserRegistry, private val allowEmptyPropertySources: Boolean ) { + private val normalizedPrefix = if (prefix == null) null else PathNormalizer.normalizePathElement(prefix) fun loadNodes( propertySources: List<PropertySource>, @@ -47,6 +52,11 @@ class PropertySourceLoader( private fun loadSources(sources: List<PropertySource>): ConfigResult<NonEmptyList<Node>> { return sources .map { it.node(PropertySourceContext(parserRegistry, allowEmptyPropertySources)) } + .map { configResult -> + configResult.flatMap { node -> + nodeTransformers.fold(node) { acc, normalizer -> acc.transform { normalizer.transform(it) } }.valid() + } + } .sequence() .mapInvalid { ConfigFailure.MultipleFailures(it) } .flatMap { @@ -59,5 +69,5 @@ class PropertySourceLoader( } private fun parentNodeOrMatchesPrefix(node: Node) = - prefix == null || node.path.keys.isEmpty() || node.path.keys[0] == prefix + normalizedPrefix == null || node.path.keys.isEmpty() || node.path.keys[0] == normalizedPrefix } diff --git a/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/nodes.kt b/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/nodes.kt index b68b769f..6c2d4e8e 100644 --- a/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/nodes.kt +++ b/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/nodes.kt @@ -107,6 +107,18 @@ fun Node.paths(): Set<Pair<DotPath, Pos>> = setOf(this.path to this.pos) + when else -> emptySet() } +/** + * Return all nodes in this tree, recursively transformed per the given transformer function. + */ +fun Node.transform(transformer: (Node) -> Node): Node = when (val transformed = transformer(this)) { + is ArrayNode -> transformed.copy(elements = transformed.elements.map { it.transform(transformer) }) + is MapNode -> transformed.copy( + map = transformed.map.mapValues { it.value.transform(transformer) }, + value = transformed.value.transform(transformer) + ) + else -> transformed +} + /** * Return all nodes in this tree, recursively filtering per the given filter function. */ diff --git a/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/sources/EnvironmentVariableOverridePropertySource.kt b/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/sources/EnvironmentVariableOverridePropertySource.kt index 0bcf13dd..520a25df 100644 --- a/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/sources/EnvironmentVariableOverridePropertySource.kt +++ b/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/sources/EnvironmentVariableOverridePropertySource.kt @@ -25,13 +25,13 @@ class EnvironmentVariableOverridePropertySource( override fun source(): String = "Env Var Overrides" override fun node(context: PropertySourceContext): ConfigResult<Node> { - val props = Properties() val vars = environmentVariableMap() .mapKeys { if (useUnderscoresAsSeparator) it.key.replace("__", ".") else it.key } .filter { it.key.startsWith(Prefix) } return if (vars.isEmpty()) Undefined.valid() else { - vars.forEach { props[it.key.removePrefix(Prefix)] = it.value } - props.toNode("Env Var Overrides").valid() + vars.toNode("Env Var Overrides") { + it.removePrefix(Prefix) + }.valid() } } } diff --git a/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/transformer/NodeTransformer.kt b/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/transformer/NodeTransformer.kt new file mode 100644 index 00000000..b31c7de4 --- /dev/null +++ b/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/transformer/NodeTransformer.kt @@ -0,0 +1,11 @@ +package com.sksamuel.hoplite.transformer + +import com.sksamuel.hoplite.* + +/** + * A [NodeTransformer] is a function that transforms a node into another node. Any type of node transformation can + * be applied at configuration loading time. + */ +interface NodeTransformer { + fun transform(node: Node): Node +} diff --git a/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/transformer/PathNormalizer.kt b/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/transformer/PathNormalizer.kt new file mode 100644 index 00000000..b7059658 --- /dev/null +++ b/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/transformer/PathNormalizer.kt @@ -0,0 +1,35 @@ +package com.sksamuel.hoplite.transformer + +import com.sksamuel.hoplite.* + +/** + * To support loading configuration from a tree based on multiple sources with different idiomatic conventions, such + * as HOCON which prefers kebab case, and environment variables which are upper-case, the path normalizer normalizes + * all paths so that the cascade happens correctly. For example, a `foo.conf` containing the HOCON standard naming + * of `abc.foo-bar` and an env var `ABC_FOOBAR` would both get mapped to data class `Foo { val fooBar: String }` + * assuming there is a Lowercase parameter mapper present. + * + * Note that with path normalization, parameters with the same name but different case will be considered the same, + * and assigned the same value. This should generally be a situation one should avoid, but if it does happen, please + * consider the use of the @[ConfigAlias] annotation to disambiguate the properties. + * + * Path normalization does the following for all node keys and each element of each node's path: + * * Removes dashes + * * Converts to lower-case + */ +object PathNormalizer : NodeTransformer { + fun normalizePathElement(element: String): String = element.replace("-", "").lowercase() + + override fun transform(node: Node): Node = node + .transform { + val normalizedPathNode = it.withPath( + it.path.copy(keys = it.path.keys.map { key -> + normalizePathElement(key) + }) + ) + when (normalizedPathNode){ + is MapNode -> normalizedPathNode.copy(map = normalizedPathNode.map.mapKeys { (key, _) -> normalizePathElement(key) }) + else -> normalizedPathNode + } + } +} diff --git a/hoplite-core/src/test/kotlin/com/sksamuel/hoplite/EmptyDecoderRegistryTest.kt b/hoplite-core/src/test/kotlin/com/sksamuel/hoplite/EmptyDecoderRegistryTest.kt index 33d055ad..eb525e8f 100644 --- a/hoplite-core/src/test/kotlin/com/sksamuel/hoplite/EmptyDecoderRegistryTest.kt +++ b/hoplite-core/src/test/kotlin/com/sksamuel/hoplite/EmptyDecoderRegistryTest.kt @@ -14,12 +14,14 @@ class EmptyDecoderRegistryTest : FunSpec() { val parsers = defaultParserRegistry() val sources = defaultPropertySources() val preprocessors = defaultPreprocessors() + val nodeTransformers = defaultNodeTransformers() val mappers = defaultParamMappers() val e = ConfigLoader( DecoderRegistry.zero, sources, parsers, preprocessors, + nodeTransformers, mappers, allowEmptyTree = false, allowUnresolvedSubstitutions = false, diff --git a/hoplite-core/src/test/kotlin/com/sksamuel/hoplite/transformer/PathNormalizerTest.kt b/hoplite-core/src/test/kotlin/com/sksamuel/hoplite/transformer/PathNormalizerTest.kt new file mode 100644 index 00000000..fc2c10cf --- /dev/null +++ b/hoplite-core/src/test/kotlin/com/sksamuel/hoplite/transformer/PathNormalizerTest.kt @@ -0,0 +1,44 @@ +package com.sksamuel.hoplite.transformer + +import com.sksamuel.hoplite.MapNode +import com.sksamuel.hoplite.Pos +import com.sksamuel.hoplite.PropertySourceContext +import com.sksamuel.hoplite.StringNode +import com.sksamuel.hoplite.decoder.DotPath +import com.sksamuel.hoplite.sources.EnvironmentVariablesPropertySource +import io.kotest.core.spec.style.FunSpec +import io.kotest.matchers.shouldBe + +class PathNormalizerTest : FunSpec({ + test("normalizes paths") { + val node = EnvironmentVariablesPropertySource( + useUnderscoresAsSeparator = false, + allowUppercaseNames = false, + environmentVariableMap = { mapOf("A" to "a", "A.B" to "ab", "A.B.CD" to "abcd") }, + ).node(PropertySourceContext.empty).getUnsafe() + + PathNormalizer.transform(node) shouldBe MapNode( + map = mapOf( + "a" to MapNode( + map = mapOf( + "b" to MapNode( + map = mapOf( + "cd" to StringNode("abcd", Pos.env, DotPath("a", "b", "cd"), sourceKey = "A.B.CD"), + ), + Pos.env, + DotPath("a", "b"), + value = StringNode("ab", Pos.env, DotPath("a", "b"), sourceKey = "A.B"), + sourceKey = "A.B" + ), + ), + Pos.env, + DotPath("a"), + value = StringNode("a", Pos.env, DotPath("a"), sourceKey = "A"), + sourceKey = "A" + ), + ), + Pos.env, + DotPath.root, + ) + } +})