Support node normalization, add a path normalizer

The purpose of the path normalizer is to normalize all inbound paths
by making them lower case, and removing any "-" characters. Normalizing
paths means sources with different idiomatic approaches to defining
key values will all map correctly to the defined config classes.

The only downside to this is multiple config attributes in the same
class that differ only by case can no longer be disambiguated. This
should be a rare case and the advantages are more than worth losing
this "feature".

We also add a LowercaseParameterMapper by default which can handle
the normalized paths.

hoplite-core/src/main/kotlin/com/sksamuel/hoplite/ConfigLoader.kt

@@ -9,6 +9,7 @@ import com.sksamuel.hoplite.fp.getOrElse
 import com.sksamuel.hoplite.internal.CascadeMode
 import com.sksamuel.hoplite.internal.ConfigParser
 import com.sksamuel.hoplite.internal.DecodeMode
+import com.sksamuel.hoplite.transformer.NodeTransformer
 import com.sksamuel.hoplite.parsers.ParserRegistry
 import com.sksamuel.hoplite.preprocessor.Preprocessor
 import com.sksamuel.hoplite.report.Print
@@ -27,6 +28,7 @@ class ConfigLoader(
   val propertySources: List<PropertySource>,
   val parserRegistry: ParserRegistry,
   val preprocessors: List<Preprocessor>,
+  val nodeTransformers: List<NodeTransformer>,
   val paramMappers: List<ParameterMapper>,
   val onFailure: List<(Throwable) -> Unit> = emptyList(),
   val decodeMode: DecodeMode = DecodeMode.Lenient,
@@ -172,6 +174,7 @@ class ConfigLoader(
       cascadeMode = cascadeMode,
       preprocessors = preprocessors,
       preprocessingIterations = preprocessingIterations,
+      nodeTransformers = nodeTransformers,
       prefix = prefix,
       resolvers = resolvers,
       decoderRegistry = decoderRegistry,
@@ -228,6 +231,7 @@ class ConfigLoader(
       cascadeMode = cascadeMode,
       preprocessors = preprocessors,
       preprocessingIterations = preprocessingIterations,
+      nodeTransformers = nodeTransformers,
       prefix = null,
       resolvers = resolvers,
       decoderRegistry = decoderRegistry,

hoplite-core/src/main/kotlin/com/sksamuel/hoplite/ConfigLoaderBuilder.kt

@@ -5,6 +5,7 @@ import com.sksamuel.hoplite.decoder.DefaultDecoderRegistry
 import com.sksamuel.hoplite.env.Environment
 import com.sksamuel.hoplite.internal.CascadeMode
 import com.sksamuel.hoplite.internal.DecodeMode
+import com.sksamuel.hoplite.transformer.NodeTransformer
 import com.sksamuel.hoplite.parsers.DefaultParserRegistry
 import com.sksamuel.hoplite.parsers.Parser
 import com.sksamuel.hoplite.preprocessor.EnvOrSystemPropertyPreprocessor
@@ -51,6 +52,7 @@ class ConfigLoaderBuilder private constructor() {
 
   private val propertySources = mutableListOf<PropertySource>()
   private val preprocessors = mutableListOf<Preprocessor>()
+  private val nodeTransformers = mutableListOf<NodeTransformer>()
   private val resolvers = mutableListOf<Resolver>()
   private val paramMappers = mutableListOf<ParameterMapper>()
   private val parsers = mutableMapOf<String, Parser>()
@@ -70,8 +72,8 @@ class ConfigLoaderBuilder private constructor() {
     /**
      * Returns a [ConfigLoaderBuilder] with all defaults applied.
      *
-     * This means that the default [Decoder]s, [Preprocessor]s, [ParameterMapper]s, [PropertySource]s,
-     * and [Parser]s are all registered.
+     * This means that the default [Decoder]s, [Preprocessor]s, [NodeTransformer]s, [ParameterMapper]s,
+     * [PropertySource]s, and [Parser]s are all registered.
      *
      * If you wish to avoid adding defaults, for example to avoid certain decoders or sources, then
      * use [empty] to obtain an empty ConfigLoaderBuilder and call the various addDefault methods manually.
@@ -80,6 +82,7 @@ class ConfigLoaderBuilder private constructor() {
       return empty()
         .addDefaultDecoders()
         .addDefaultPreprocessors()
+        .addDefaultNodeTransformers()
         .addDefaultParamMappers()
         .addDefaultPropertySources()
         .addDefaultParsers()
@@ -88,7 +91,7 @@ class ConfigLoaderBuilder private constructor() {
     /**
      * Returns a [ConfigLoaderBuilder] with all defaults applied, using resolvers in place of preprocessors.
      *
-     * This means that the default [Decoder]s, [Resolver]s, [ParameterMapper]s, [PropertySource]s,
+     * This means that the default [Decoder]s, [Resolver]s, [NodeTransformer]s, [ParameterMapper]s, [PropertySource]s,
      * and [Parser]s are all registered.
      *
      * If you wish to avoid adding defaults, for example to avoid certain decoders or sources, then
@@ -102,6 +105,7 @@ class ConfigLoaderBuilder private constructor() {
       return empty()
         .addDefaultDecoders()
         .addDefaultResolvers()
+        .addDefaultNodeTransformers()
         .addDefaultParamMappers()
         .addDefaultPropertySources()
         .addDefaultParsers()
@@ -205,6 +209,16 @@ class ConfigLoaderBuilder private constructor() {
 
   fun addDefaultPreprocessors() = addPreprocessors(defaultPreprocessors())
 
+  fun addNodeTransformer(nodeTransformer: NodeTransformer): ConfigLoaderBuilder = apply {
+    this.nodeTransformers.add(nodeTransformer)
+  }
+
+  fun addNodeTransformers(nodeTransformers: Iterable<NodeTransformer>): ConfigLoaderBuilder = apply {
+    this.nodeTransformers.addAll(nodeTransformers)
+  }
+
+  fun addDefaultNodeTransformers() = addNodeTransformers(defaultNodeTransformers())
+
   fun addParser(ext: String, parser: Parser) = addFileExtensionMapping(ext, parser)
   fun addParsers(map: Map<String, Parser>) = addFileExtensionMappings(map)
 
@@ -249,6 +263,7 @@ class ConfigLoaderBuilder private constructor() {
     return addDefaultDecoders()
       .addDefaultParsers()
       .addDefaultPreprocessors()
+      .addDefaultNodeTransformers()
       .addDefaultParamMappers()
       .addDefaultPropertySources()
   }
@@ -372,6 +387,7 @@ class ConfigLoaderBuilder private constructor() {
       propertySources = propertySources.toList(),
       parserRegistry = DefaultParserRegistry(parsers),
       preprocessors = preprocessors.toList(),
+      nodeTransformers = nodeTransformers.toList(),
       paramMappers = paramMappers.toList(),
       onFailure = failureCallbacks.toList(),
       resolvers = resolvers,
@@ -407,6 +423,8 @@ fun defaultPreprocessors(): List<Preprocessor> = listOf(
   LookupPreprocessor,
 )
 
+fun defaultNodeTransformers(): List<NodeTransformer> = emptyList()
+
 fun defaultResolvers(): List<Resolver> = listOf(
   EnvVarContextResolver,
   SystemPropertyContextResolver,
@@ -419,6 +437,7 @@ fun defaultResolvers(): List<Resolver> = listOf(
 
 fun defaultParamMappers(): List<ParameterMapper> = listOf(
   DefaultParamMapper,
+  LowercaseParamMapper,
   SnakeCaseParamMapper,
   KebabCaseParamMapper,
   AliasAnnotationParamMapper,

hoplite-core/src/main/kotlin/com/sksamuel/hoplite/ParameterMapper.kt

@@ -34,6 +34,11 @@ object DefaultParamMapper : ParameterMapper {
     setOfNotNull(param.name)
 }
 
+object LowercaseParamMapper : ParameterMapper {
+  override fun map(param: KParameter, constructor: KFunction<Any>, kclass: KClass<*>): Set<String> =
+    setOfNotNull(param.name?.lowercase())
+}
+
 /**
  * Disabled by default so that common ENVVAR PARAMS don't override your lower case
  * names unexpectedly.

hoplite-core/src/main/kotlin/com/sksamuel/hoplite/internal/ConfigParser.kt

@@ -14,6 +14,7 @@ import com.sksamuel.hoplite.env.Environment
 import com.sksamuel.hoplite.fp.flatMap
 import com.sksamuel.hoplite.fp.invalid
 import com.sksamuel.hoplite.fp.valid
+import com.sksamuel.hoplite.transformer.NodeTransformer
 import com.sksamuel.hoplite.parsers.ParserRegistry
 import com.sksamuel.hoplite.preprocessor.Preprocessor
 import com.sksamuel.hoplite.report.Print
@@ -26,31 +27,32 @@ import com.sksamuel.hoplite.secrets.SecretsPolicy
 import kotlin.reflect.KClass
 
 class ConfigParser(
-   classpathResourceLoader: ClasspathResourceLoader,
-   parserRegistry: ParserRegistry,
-   allowEmptyTree: Boolean,
-   allowNullOverride: Boolean,
-   cascadeMode: CascadeMode,
-   preprocessors: List<Preprocessor>,
-   preprocessingIterations: Int,
-   private val prefix: String?,
-   private val resolvers: List<Resolver>,
-   private val decoderRegistry: DecoderRegistry,
-   private val paramMappers: List<ParameterMapper>,
-   private val flattenArraysToString: Boolean,
-   private val resolveTypesCaseInsensitive: Boolean,
-   private val allowUnresolvedSubstitutions: Boolean,
-   private val secretsPolicy: SecretsPolicy?,
-   private val decodeMode: DecodeMode,
-   private val useReport: Boolean,
-   private val obfuscator: Obfuscator,
-   private val reportPrintFn: Print,
-   private val environment: Environment?,
-   private val sealedTypeDiscriminatorField: String?,
-   private val contextResolverMode: ContextResolverMode,
+  classpathResourceLoader: ClasspathResourceLoader,
+  parserRegistry: ParserRegistry,
+  allowEmptyTree: Boolean,
+  allowNullOverride: Boolean,
+  cascadeMode: CascadeMode,
+  preprocessors: List<Preprocessor>,
+  preprocessingIterations: Int,
+  nodeTransformers: List<NodeTransformer>,
+  private val prefix: String?,
+  private val resolvers: List<Resolver>,
+  private val decoderRegistry: DecoderRegistry,
+  private val paramMappers: List<ParameterMapper>,
+  private val flattenArraysToString: Boolean,
+  private val resolveTypesCaseInsensitive: Boolean,
+  private val allowUnresolvedSubstitutions: Boolean,
+  private val secretsPolicy: SecretsPolicy?,
+  private val decodeMode: DecodeMode,
+  private val useReport: Boolean,
+  private val obfuscator: Obfuscator,
+  private val reportPrintFn: Print,
+  private val environment: Environment?,
+  private val sealedTypeDiscriminatorField: String?,
+  private val contextResolverMode: ContextResolverMode,
 ) {
 
-  private val loader = PropertySourceLoader(prefix, classpathResourceLoader, parserRegistry, allowEmptyTree)
+  private val loader = PropertySourceLoader(prefix, nodeTransformers, classpathResourceLoader, parserRegistry, allowEmptyTree)
   private val cascader = Cascader(cascadeMode, allowEmptyTree, allowNullOverride)
   private val preprocessing = Preprocessing(preprocessors, preprocessingIterations)
   private val decoding = Decoding(decoderRegistry, secretsPolicy)

hoplite-core/src/main/kotlin/com/sksamuel/hoplite/internal/PropertySourceLoader.kt

@@ -1,30 +1,35 @@
 package com.sksamuel.hoplite.internal
 
-import com.sksamuel.hoplite.filter
 import com.sksamuel.hoplite.ClasspathResourceLoader
 import com.sksamuel.hoplite.ConfigFailure
 import com.sksamuel.hoplite.ConfigResult
 import com.sksamuel.hoplite.ConfigSource
 import com.sksamuel.hoplite.Node
 import com.sksamuel.hoplite.PropertySource
 import com.sksamuel.hoplite.PropertySourceContext
+import com.sksamuel.hoplite.filter
 import com.sksamuel.hoplite.fp.NonEmptyList
 import com.sksamuel.hoplite.fp.flatMap
 import com.sksamuel.hoplite.fp.invalid
 import com.sksamuel.hoplite.fp.sequence
 import com.sksamuel.hoplite.fp.valid
+import com.sksamuel.hoplite.transformer.NodeTransformer
+import com.sksamuel.hoplite.transformer.PathNormalizer
 import com.sksamuel.hoplite.parsers.ParserRegistry
 import com.sksamuel.hoplite.sources.ConfigFilePropertySource
+import com.sksamuel.hoplite.transform
 
 /**
  * Loads [Node]s from [PropertySource]s, [ConfigSource]s, files and classpath resources.
  */
 class PropertySourceLoader(
-  private val prefix: String?,
+  prefix: String?,
+  private val nodeTransformers: List<NodeTransformer>,
   private val classpathResourceLoader: ClasspathResourceLoader,
   private val parserRegistry: ParserRegistry,
   private val allowEmptyPropertySources: Boolean
 ) {
+  private val normalizedPrefix = if (prefix == null) null else PathNormalizer.normalizePathElement(prefix)
 
   fun loadNodes(
     propertySources: List<PropertySource>,
@@ -47,6 +52,11 @@ class PropertySourceLoader(
   private fun loadSources(sources: List<PropertySource>): ConfigResult<NonEmptyList<Node>> {
     return sources
       .map { it.node(PropertySourceContext(parserRegistry, allowEmptyPropertySources)) }
+      .map { configResult ->
+        configResult.flatMap { node ->
+          nodeTransformers.fold(node) { acc, normalizer -> acc.transform { normalizer.transform(it) } }.valid()
+        }
+      }
       .sequence()
       .mapInvalid { ConfigFailure.MultipleFailures(it) }
       .flatMap {
@@ -59,5 +69,5 @@ class PropertySourceLoader(
   }
 
   private fun parentNodeOrMatchesPrefix(node: Node) =
-    prefix == null || node.path.keys.isEmpty() || node.path.keys[0] == prefix
+    normalizedPrefix == null || node.path.keys.isEmpty() || node.path.keys[0] == normalizedPrefix
 }

hoplite-core/src/main/kotlin/com/sksamuel/hoplite/nodes.kt

@@ -107,6 +107,18 @@ fun Node.paths(): Set<Pair<DotPath, Pos>> = setOf(this.path to this.pos) + when
   else -> emptySet()
 }
 
+/**
+ * Return all nodes in this tree, recursively transformed per the given transformer function.
+ */
+fun Node.transform(transformer: (Node) -> Node): Node = when (val transformed = transformer(this)) {
+  is ArrayNode -> transformed.copy(elements = transformed.elements.map { it.transform(transformer) })
+  is MapNode -> transformed.copy(
+    map = transformed.map.mapValues { it.value.transform(transformer) },
+    value = transformed.value.transform(transformer)
+  )
+  else -> transformed
+}
+
 /**
  * Return all nodes in this tree, recursively filtering per the given filter function.
  */

hoplite-core/src/main/kotlin/com/sksamuel/hoplite/sources/EnvironmentVariableOverridePropertySource.kt

@@ -25,13 +25,13 @@ class EnvironmentVariableOverridePropertySource(
   override fun source(): String = "Env Var Overrides"
 
   override fun node(context: PropertySourceContext): ConfigResult<Node> {
-    val props = Properties()
     val vars = environmentVariableMap()
       .mapKeys { if (useUnderscoresAsSeparator) it.key.replace("__", ".") else it.key }
       .filter { it.key.startsWith(Prefix) }
     return if (vars.isEmpty()) Undefined.valid() else {
-      vars.forEach { props[it.key.removePrefix(Prefix)] = it.value }
-      props.toNode("Env Var Overrides").valid()
+      vars.toNode("Env Var Overrides") {
+        it.removePrefix(Prefix)
+      }.valid()
     }
   }
 }

hoplite-core/src/main/kotlin/com/sksamuel/hoplite/transformer/NodeTransformer.kt

@@ -0,0 +1,11 @@
+package com.sksamuel.hoplite.transformer
+
+import com.sksamuel.hoplite.*
+
+/**
+ * A [NodeTransformer] is a function that transforms a node into another node. Any type of node transformation can
+ * be applied at configuration loading time.
+ */
+interface NodeTransformer {
+  fun transform(node: Node): Node
+}

hoplite-core/src/main/kotlin/com/sksamuel/hoplite/transformer/PathNormalizer.kt

@@ -0,0 +1,35 @@
+package com.sksamuel.hoplite.transformer
+
+import com.sksamuel.hoplite.*
+
+/**
+ * To support loading configuration from a tree based on multiple sources with different idiomatic conventions, such
+ * as HOCON which prefers kebab case, and environment variables which are upper-case, the path normalizer normalizes
+ * all paths so that the cascade happens correctly. For example, a `foo.conf` containing the HOCON standard naming
+ * of `abc.foo-bar` and an env var `ABC_FOOBAR` would both get mapped to data class `Foo { val fooBar: String }`
+ * assuming there is a Lowercase parameter mapper present.
+ *
+ * Note that with path normalization, parameters with the same name but different case will be considered the same,
+ * and assigned the same value. This should generally be a situation one should avoid, but if it does happen, please
+ * consider the use of the @[ConfigAlias] annotation to disambiguate the properties.
+ *
+ * Path normalization does the following for all node keys and each element of each node's path:
+ * * Removes dashes
+ * * Converts to lower-case
+ */
+object PathNormalizer : NodeTransformer {
+  fun normalizePathElement(element: String): String = element.replace("-", "").lowercase()
+
+  override fun transform(node: Node): Node = node
+    .transform {
+      val normalizedPathNode = it.withPath(
+        it.path.copy(keys = it.path.keys.map { key ->
+          normalizePathElement(key)
+        })
+      )
+      when (normalizedPathNode){
+        is MapNode -> normalizedPathNode.copy(map = normalizedPathNode.map.mapKeys { (key, _) -> normalizePathElement(key) })
+        else -> normalizedPathNode
+      }
+    }
+}

hoplite-core/src/test/kotlin/com/sksamuel/hoplite/EmptyDecoderRegistryTest.kt

@@ -14,12 +14,14 @@ class EmptyDecoderRegistryTest : FunSpec() {
       val parsers = defaultParserRegistry()
       val sources = defaultPropertySources()
       val preprocessors = defaultPreprocessors()
+      val nodeTransformers = defaultNodeTransformers()
       val mappers = defaultParamMappers()
       val e = ConfigLoader(
         DecoderRegistry.zero,
         sources,
         parsers,
         preprocessors,
+        nodeTransformers,
         mappers,
         allowEmptyTree = false,
         allowUnresolvedSubstitutions = false,