diff --git a/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/ConfigLoader.kt b/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/ConfigLoader.kt index 4c655bb1..581bac1c 100644 --- a/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/ConfigLoader.kt +++ b/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/ConfigLoader.kt @@ -9,6 +9,7 @@ import com.sksamuel.hoplite.fp.getOrElse import com.sksamuel.hoplite.internal.CascadeMode import com.sksamuel.hoplite.internal.ConfigParser import com.sksamuel.hoplite.internal.DecodeMode +import com.sksamuel.hoplite.transformer.NodeTransformer import com.sksamuel.hoplite.parsers.ParserRegistry import com.sksamuel.hoplite.preprocessor.Preprocessor import com.sksamuel.hoplite.report.Print @@ -27,6 +28,7 @@ class ConfigLoader( val propertySources: List, val parserRegistry: ParserRegistry, val preprocessors: List, + val nodeTransformers: List, val paramMappers: List, val onFailure: List<(Throwable) -> Unit> = emptyList(), val decodeMode: DecodeMode = DecodeMode.Lenient, @@ -172,6 +174,7 @@ class ConfigLoader( cascadeMode = cascadeMode, preprocessors = preprocessors, preprocessingIterations = preprocessingIterations, + nodeTransformers = nodeTransformers, prefix = prefix, resolvers = resolvers, decoderRegistry = decoderRegistry, @@ -228,6 +231,7 @@ class ConfigLoader( cascadeMode = cascadeMode, preprocessors = preprocessors, preprocessingIterations = preprocessingIterations, + nodeTransformers = nodeTransformers, prefix = null, resolvers = resolvers, decoderRegistry = decoderRegistry, diff --git a/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/ConfigLoaderBuilder.kt b/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/ConfigLoaderBuilder.kt index 38a8f425..1fe8e723 100644 --- a/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/ConfigLoaderBuilder.kt +++ b/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/ConfigLoaderBuilder.kt @@ -5,6 +5,7 @@ import com.sksamuel.hoplite.decoder.DefaultDecoderRegistry import com.sksamuel.hoplite.env.Environment import com.sksamuel.hoplite.internal.CascadeMode import com.sksamuel.hoplite.internal.DecodeMode +import com.sksamuel.hoplite.transformer.NodeTransformer import com.sksamuel.hoplite.parsers.DefaultParserRegistry import com.sksamuel.hoplite.parsers.Parser import com.sksamuel.hoplite.preprocessor.EnvOrSystemPropertyPreprocessor @@ -51,6 +52,7 @@ class ConfigLoaderBuilder private constructor() { private val propertySources = mutableListOf() private val preprocessors = mutableListOf() + private val nodeTransformers = mutableListOf() private val resolvers = mutableListOf() private val paramMappers = mutableListOf() private val parsers = mutableMapOf() @@ -70,8 +72,8 @@ class ConfigLoaderBuilder private constructor() { /** * Returns a [ConfigLoaderBuilder] with all defaults applied. * - * This means that the default [Decoder]s, [Preprocessor]s, [ParameterMapper]s, [PropertySource]s, - * and [Parser]s are all registered. + * This means that the default [Decoder]s, [Preprocessor]s, [NodeTransformer]s, [ParameterMapper]s, + * [PropertySource]s, and [Parser]s are all registered. * * If you wish to avoid adding defaults, for example to avoid certain decoders or sources, then * use [empty] to obtain an empty ConfigLoaderBuilder and call the various addDefault methods manually. @@ -80,6 +82,7 @@ class ConfigLoaderBuilder private constructor() { return empty() .addDefaultDecoders() .addDefaultPreprocessors() + .addDefaultNodeTransformers() .addDefaultParamMappers() .addDefaultPropertySources() .addDefaultParsers() @@ -88,7 +91,7 @@ class ConfigLoaderBuilder private constructor() { /** * Returns a [ConfigLoaderBuilder] with all defaults applied, using resolvers in place of preprocessors. * - * This means that the default [Decoder]s, [Resolver]s, [ParameterMapper]s, [PropertySource]s, + * This means that the default [Decoder]s, [Resolver]s, [NodeTransformer]s, [ParameterMapper]s, [PropertySource]s, * and [Parser]s are all registered. * * If you wish to avoid adding defaults, for example to avoid certain decoders or sources, then @@ -102,6 +105,7 @@ class ConfigLoaderBuilder private constructor() { return empty() .addDefaultDecoders() .addDefaultResolvers() + .addDefaultNodeTransformers() .addDefaultParamMappers() .addDefaultPropertySources() .addDefaultParsers() @@ -205,6 +209,16 @@ class ConfigLoaderBuilder private constructor() { fun addDefaultPreprocessors() = addPreprocessors(defaultPreprocessors()) + fun addNodeTransformer(nodeTransformer: NodeTransformer): ConfigLoaderBuilder = apply { + this.nodeTransformers.add(nodeTransformer) + } + + fun addNodeTransformers(nodeTransformers: Iterable): ConfigLoaderBuilder = apply { + this.nodeTransformers.addAll(nodeTransformers) + } + + fun addDefaultNodeTransformers() = addNodeTransformers(defaultNodeTransformers()) + fun addParser(ext: String, parser: Parser) = addFileExtensionMapping(ext, parser) fun addParsers(map: Map) = addFileExtensionMappings(map) @@ -249,6 +263,7 @@ class ConfigLoaderBuilder private constructor() { return addDefaultDecoders() .addDefaultParsers() .addDefaultPreprocessors() + .addDefaultNodeTransformers() .addDefaultParamMappers() .addDefaultPropertySources() } @@ -372,6 +387,7 @@ class ConfigLoaderBuilder private constructor() { propertySources = propertySources.toList(), parserRegistry = DefaultParserRegistry(parsers), preprocessors = preprocessors.toList(), + nodeTransformers = nodeTransformers.toList(), paramMappers = paramMappers.toList(), onFailure = failureCallbacks.toList(), resolvers = resolvers, @@ -407,6 +423,8 @@ fun defaultPreprocessors(): List = listOf( LookupPreprocessor, ) +fun defaultNodeTransformers(): List = emptyList() + fun defaultResolvers(): List = listOf( EnvVarContextResolver, SystemPropertyContextResolver, @@ -419,6 +437,7 @@ fun defaultResolvers(): List = listOf( fun defaultParamMappers(): List = listOf( DefaultParamMapper, + LowercaseParamMapper, SnakeCaseParamMapper, KebabCaseParamMapper, AliasAnnotationParamMapper, diff --git a/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/ParameterMapper.kt b/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/ParameterMapper.kt index 6a45dfe0..5ce721d7 100644 --- a/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/ParameterMapper.kt +++ b/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/ParameterMapper.kt @@ -34,6 +34,11 @@ object DefaultParamMapper : ParameterMapper { setOfNotNull(param.name) } +object LowercaseParamMapper : ParameterMapper { + override fun map(param: KParameter, constructor: KFunction, kclass: KClass<*>): Set = + setOfNotNull(param.name?.lowercase()) +} + /** * Disabled by default so that common ENVVAR PARAMS don't override your lower case * names unexpectedly. diff --git a/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/internal/ConfigParser.kt b/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/internal/ConfigParser.kt index 3b52b78a..bbf45f13 100644 --- a/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/internal/ConfigParser.kt +++ b/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/internal/ConfigParser.kt @@ -14,6 +14,7 @@ import com.sksamuel.hoplite.env.Environment import com.sksamuel.hoplite.fp.flatMap import com.sksamuel.hoplite.fp.invalid import com.sksamuel.hoplite.fp.valid +import com.sksamuel.hoplite.transformer.NodeTransformer import com.sksamuel.hoplite.parsers.ParserRegistry import com.sksamuel.hoplite.preprocessor.Preprocessor import com.sksamuel.hoplite.report.Print @@ -26,31 +27,32 @@ import com.sksamuel.hoplite.secrets.SecretsPolicy import kotlin.reflect.KClass class ConfigParser( - classpathResourceLoader: ClasspathResourceLoader, - parserRegistry: ParserRegistry, - allowEmptyTree: Boolean, - allowNullOverride: Boolean, - cascadeMode: CascadeMode, - preprocessors: List, - preprocessingIterations: Int, - private val prefix: String?, - private val resolvers: List, - private val decoderRegistry: DecoderRegistry, - private val paramMappers: List, - private val flattenArraysToString: Boolean, - private val resolveTypesCaseInsensitive: Boolean, - private val allowUnresolvedSubstitutions: Boolean, - private val secretsPolicy: SecretsPolicy?, - private val decodeMode: DecodeMode, - private val useReport: Boolean, - private val obfuscator: Obfuscator, - private val reportPrintFn: Print, - private val environment: Environment?, - private val sealedTypeDiscriminatorField: String?, - private val contextResolverMode: ContextResolverMode, + classpathResourceLoader: ClasspathResourceLoader, + parserRegistry: ParserRegistry, + allowEmptyTree: Boolean, + allowNullOverride: Boolean, + cascadeMode: CascadeMode, + preprocessors: List, + preprocessingIterations: Int, + nodeTransformers: List, + private val prefix: String?, + private val resolvers: List, + private val decoderRegistry: DecoderRegistry, + private val paramMappers: List, + private val flattenArraysToString: Boolean, + private val resolveTypesCaseInsensitive: Boolean, + private val allowUnresolvedSubstitutions: Boolean, + private val secretsPolicy: SecretsPolicy?, + private val decodeMode: DecodeMode, + private val useReport: Boolean, + private val obfuscator: Obfuscator, + private val reportPrintFn: Print, + private val environment: Environment?, + private val sealedTypeDiscriminatorField: String?, + private val contextResolverMode: ContextResolverMode, ) { - private val loader = PropertySourceLoader(prefix, classpathResourceLoader, parserRegistry, allowEmptyTree) + private val loader = PropertySourceLoader(prefix, nodeTransformers, classpathResourceLoader, parserRegistry, allowEmptyTree) private val cascader = Cascader(cascadeMode, allowEmptyTree, allowNullOverride) private val preprocessing = Preprocessing(preprocessors, preprocessingIterations) private val decoding = Decoding(decoderRegistry, secretsPolicy) diff --git a/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/internal/PropertySourceLoader.kt b/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/internal/PropertySourceLoader.kt index 21a95efb..68cb70b4 100644 --- a/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/internal/PropertySourceLoader.kt +++ b/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/internal/PropertySourceLoader.kt @@ -1,6 +1,5 @@ package com.sksamuel.hoplite.internal -import com.sksamuel.hoplite.filter import com.sksamuel.hoplite.ClasspathResourceLoader import com.sksamuel.hoplite.ConfigFailure import com.sksamuel.hoplite.ConfigResult @@ -8,23 +7,29 @@ import com.sksamuel.hoplite.ConfigSource import com.sksamuel.hoplite.Node import com.sksamuel.hoplite.PropertySource import com.sksamuel.hoplite.PropertySourceContext +import com.sksamuel.hoplite.filter import com.sksamuel.hoplite.fp.NonEmptyList import com.sksamuel.hoplite.fp.flatMap import com.sksamuel.hoplite.fp.invalid import com.sksamuel.hoplite.fp.sequence import com.sksamuel.hoplite.fp.valid +import com.sksamuel.hoplite.transformer.NodeTransformer +import com.sksamuel.hoplite.transformer.PathNormalizer import com.sksamuel.hoplite.parsers.ParserRegistry import com.sksamuel.hoplite.sources.ConfigFilePropertySource +import com.sksamuel.hoplite.transform /** * Loads [Node]s from [PropertySource]s, [ConfigSource]s, files and classpath resources. */ class PropertySourceLoader( - private val prefix: String?, + prefix: String?, + private val nodeTransformers: List, private val classpathResourceLoader: ClasspathResourceLoader, private val parserRegistry: ParserRegistry, private val allowEmptyPropertySources: Boolean ) { + private val normalizedPrefix = if (prefix == null) null else PathNormalizer.normalizePathElement(prefix) fun loadNodes( propertySources: List, @@ -47,6 +52,11 @@ class PropertySourceLoader( private fun loadSources(sources: List): ConfigResult> { return sources .map { it.node(PropertySourceContext(parserRegistry, allowEmptyPropertySources)) } + .map { configResult -> + configResult.flatMap { node -> + nodeTransformers.fold(node) { acc, normalizer -> acc.transform { normalizer.transform(it) } }.valid() + } + } .sequence() .mapInvalid { ConfigFailure.MultipleFailures(it) } .flatMap { @@ -59,5 +69,5 @@ class PropertySourceLoader( } private fun parentNodeOrMatchesPrefix(node: Node) = - prefix == null || node.path.keys.isEmpty() || node.path.keys[0] == prefix + normalizedPrefix == null || node.path.keys.isEmpty() || node.path.keys[0] == normalizedPrefix } diff --git a/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/nodes.kt b/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/nodes.kt index b68b769f..6c2d4e8e 100644 --- a/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/nodes.kt +++ b/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/nodes.kt @@ -107,6 +107,18 @@ fun Node.paths(): Set> = setOf(this.path to this.pos) + when else -> emptySet() } +/** + * Return all nodes in this tree, recursively transformed per the given transformer function. + */ +fun Node.transform(transformer: (Node) -> Node): Node = when (val transformed = transformer(this)) { + is ArrayNode -> transformed.copy(elements = transformed.elements.map { it.transform(transformer) }) + is MapNode -> transformed.copy( + map = transformed.map.mapValues { it.value.transform(transformer) }, + value = transformed.value.transform(transformer) + ) + else -> transformed +} + /** * Return all nodes in this tree, recursively filtering per the given filter function. */ diff --git a/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/sources/EnvironmentVariableOverridePropertySource.kt b/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/sources/EnvironmentVariableOverridePropertySource.kt index 0bcf13dd..520a25df 100644 --- a/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/sources/EnvironmentVariableOverridePropertySource.kt +++ b/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/sources/EnvironmentVariableOverridePropertySource.kt @@ -25,13 +25,13 @@ class EnvironmentVariableOverridePropertySource( override fun source(): String = "Env Var Overrides" override fun node(context: PropertySourceContext): ConfigResult { - val props = Properties() val vars = environmentVariableMap() .mapKeys { if (useUnderscoresAsSeparator) it.key.replace("__", ".") else it.key } .filter { it.key.startsWith(Prefix) } return if (vars.isEmpty()) Undefined.valid() else { - vars.forEach { props[it.key.removePrefix(Prefix)] = it.value } - props.toNode("Env Var Overrides").valid() + vars.toNode("Env Var Overrides") { + it.removePrefix(Prefix) + }.valid() } } } diff --git a/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/transformer/NodeTransformer.kt b/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/transformer/NodeTransformer.kt new file mode 100644 index 00000000..b31c7de4 --- /dev/null +++ b/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/transformer/NodeTransformer.kt @@ -0,0 +1,11 @@ +package com.sksamuel.hoplite.transformer + +import com.sksamuel.hoplite.* + +/** + * A [NodeTransformer] is a function that transforms a node into another node. Any type of node transformation can + * be applied at configuration loading time. + */ +interface NodeTransformer { + fun transform(node: Node): Node +} diff --git a/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/transformer/PathNormalizer.kt b/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/transformer/PathNormalizer.kt new file mode 100644 index 00000000..b7059658 --- /dev/null +++ b/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/transformer/PathNormalizer.kt @@ -0,0 +1,35 @@ +package com.sksamuel.hoplite.transformer + +import com.sksamuel.hoplite.* + +/** + * To support loading configuration from a tree based on multiple sources with different idiomatic conventions, such + * as HOCON which prefers kebab case, and environment variables which are upper-case, the path normalizer normalizes + * all paths so that the cascade happens correctly. For example, a `foo.conf` containing the HOCON standard naming + * of `abc.foo-bar` and an env var `ABC_FOOBAR` would both get mapped to data class `Foo { val fooBar: String }` + * assuming there is a Lowercase parameter mapper present. + * + * Note that with path normalization, parameters with the same name but different case will be considered the same, + * and assigned the same value. This should generally be a situation one should avoid, but if it does happen, please + * consider the use of the @[ConfigAlias] annotation to disambiguate the properties. + * + * Path normalization does the following for all node keys and each element of each node's path: + * * Removes dashes + * * Converts to lower-case + */ +object PathNormalizer : NodeTransformer { + fun normalizePathElement(element: String): String = element.replace("-", "").lowercase() + + override fun transform(node: Node): Node = node + .transform { + val normalizedPathNode = it.withPath( + it.path.copy(keys = it.path.keys.map { key -> + normalizePathElement(key) + }) + ) + when (normalizedPathNode){ + is MapNode -> normalizedPathNode.copy(map = normalizedPathNode.map.mapKeys { (key, _) -> normalizePathElement(key) }) + else -> normalizedPathNode + } + } +} diff --git a/hoplite-core/src/test/kotlin/com/sksamuel/hoplite/EmptyDecoderRegistryTest.kt b/hoplite-core/src/test/kotlin/com/sksamuel/hoplite/EmptyDecoderRegistryTest.kt index 33d055ad..eb525e8f 100644 --- a/hoplite-core/src/test/kotlin/com/sksamuel/hoplite/EmptyDecoderRegistryTest.kt +++ b/hoplite-core/src/test/kotlin/com/sksamuel/hoplite/EmptyDecoderRegistryTest.kt @@ -14,12 +14,14 @@ class EmptyDecoderRegistryTest : FunSpec() { val parsers = defaultParserRegistry() val sources = defaultPropertySources() val preprocessors = defaultPreprocessors() + val nodeTransformers = defaultNodeTransformers() val mappers = defaultParamMappers() val e = ConfigLoader( DecoderRegistry.zero, sources, parsers, preprocessors, + nodeTransformers, mappers, allowEmptyTree = false, allowUnresolvedSubstitutions = false, diff --git a/hoplite-core/src/test/kotlin/com/sksamuel/hoplite/transformer/PathNormalizerTest.kt b/hoplite-core/src/test/kotlin/com/sksamuel/hoplite/transformer/PathNormalizerTest.kt new file mode 100644 index 00000000..fc2c10cf --- /dev/null +++ b/hoplite-core/src/test/kotlin/com/sksamuel/hoplite/transformer/PathNormalizerTest.kt @@ -0,0 +1,44 @@ +package com.sksamuel.hoplite.transformer + +import com.sksamuel.hoplite.MapNode +import com.sksamuel.hoplite.Pos +import com.sksamuel.hoplite.PropertySourceContext +import com.sksamuel.hoplite.StringNode +import com.sksamuel.hoplite.decoder.DotPath +import com.sksamuel.hoplite.sources.EnvironmentVariablesPropertySource +import io.kotest.core.spec.style.FunSpec +import io.kotest.matchers.shouldBe + +class PathNormalizerTest : FunSpec({ + test("normalizes paths") { + val node = EnvironmentVariablesPropertySource( + useUnderscoresAsSeparator = false, + allowUppercaseNames = false, + environmentVariableMap = { mapOf("A" to "a", "A.B" to "ab", "A.B.CD" to "abcd") }, + ).node(PropertySourceContext.empty).getUnsafe() + + PathNormalizer.transform(node) shouldBe MapNode( + map = mapOf( + "a" to MapNode( + map = mapOf( + "b" to MapNode( + map = mapOf( + "cd" to StringNode("abcd", Pos.env, DotPath("a", "b", "cd"), sourceKey = "A.B.CD"), + ), + Pos.env, + DotPath("a", "b"), + value = StringNode("ab", Pos.env, DotPath("a", "b"), sourceKey = "A.B"), + sourceKey = "A.B" + ), + ), + Pos.env, + DotPath("a"), + value = StringNode("a", Pos.env, DotPath("a"), sourceKey = "A"), + sourceKey = "A" + ), + ), + Pos.env, + DotPath.root, + ) + } +})