diff --git a/hoplite-aws/src/main/kotlin/com/sksamuel/hoplite/aws/ParameterStorePathPropertySource.kt b/hoplite-aws/src/main/kotlin/com/sksamuel/hoplite/aws/ParameterStorePathPropertySource.kt index 81c14158..ed09ceb5 100644 --- a/hoplite-aws/src/main/kotlin/com/sksamuel/hoplite/aws/ParameterStorePathPropertySource.kt +++ b/hoplite-aws/src/main/kotlin/com/sksamuel/hoplite/aws/ParameterStorePathPropertySource.kt @@ -10,7 +10,6 @@ import com.sksamuel.hoplite.PropertySource import com.sksamuel.hoplite.PropertySourceContext import com.sksamuel.hoplite.decoder.toValidated import com.sksamuel.hoplite.parsers.toNode -import java.util.Properties /** * Provides all keys under a prefix path as config values. @@ -47,12 +46,9 @@ class ParameterStorePathPropertySource( override fun node(context: PropertySourceContext): ConfigResult { return fetchParameterStoreValues().map { params -> - val props = Properties() - params.forEach { - val name = if (stripPath) it.name.removePrefix(prefix) else it.name - props[name.removePrefix("/")] = it.value + params.associate { it.name to it.value }.toNode("aws_parameter_store at $prefix", "/") { + (if (stripPath) it.removePrefix(prefix) else it).removePrefix("/") } - props.toNode("aws_parameter_store at $prefix", "/") }.toValidated { ConfigFailure.PropertySourceFailure("Could not fetch data from AWS parameter store: ${it.message}", it) } diff --git a/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/nodes.kt b/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/nodes.kt index cbaa60f0..fcda9f5d 100644 --- a/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/nodes.kt +++ b/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/nodes.kt @@ -14,6 +14,12 @@ sealed interface Node { */ val path: DotPath + /** + * The original source key of this node without any normalization. + * Useful for reporting. + */ + val sourceKey: String? + /** * Returns the [PrimitiveNode] at the given key. * If this node is not a [MapNode] or the node contained at the @@ -126,7 +132,8 @@ data class MapNode( override val pos: Pos, override val path: DotPath, val value: Node = Undefined, - override val meta: Map = emptyMap() + override val meta: Map = emptyMap(), + override val sourceKey: String? = if (path == DotPath.root) null else path.flatten(), ) : ContainerNode() { override val simpleName: String = "Map" override fun atKey(key: String): Node = map[key] ?: Undefined @@ -138,7 +145,8 @@ data class ArrayNode( val elements: List, override val pos: Pos, override val path: DotPath, - override val meta: Map = emptyMap() + override val meta: Map = emptyMap(), + override val sourceKey: String? = if (path == DotPath.root) null else path.flatten(), ) : ContainerNode() { override val simpleName: String = "List" override fun atKey(key: String): Node = Undefined @@ -157,7 +165,8 @@ data class StringNode( override val value: String, override val pos: Pos, override val path: DotPath, - override val meta: Map = emptyMap() + override val meta: Map = emptyMap(), + override val sourceKey: String? = if (path == DotPath.root) null else path.flatten(), ) : PrimitiveNode() { override val simpleName: String = "String" } @@ -166,7 +175,8 @@ data class BooleanNode( override val value: Boolean, override val pos: Pos, override val path: DotPath, - override val meta: Map = emptyMap() + override val meta: Map = emptyMap(), + override val sourceKey: String? = if (path == DotPath.root) null else path.flatten(), ) : PrimitiveNode() { override val simpleName: String = "Boolean" } @@ -177,7 +187,8 @@ data class LongNode( override val value: Long, override val pos: Pos, override val path: DotPath, - override val meta: Map = emptyMap() + override val meta: Map = emptyMap(), + override val sourceKey: String? = if (path == DotPath.root) null else path.flatten(), ) : NumberNode() { override val simpleName: String = "Long" } @@ -186,7 +197,8 @@ data class DoubleNode( override val value: Double, override val pos: Pos, override val path: DotPath, - override val meta: Map = emptyMap() + override val meta: Map = emptyMap(), + override val sourceKey: String? = if (path == DotPath.root) null else path.flatten(), ) : NumberNode() { override val simpleName: String = "Double" } @@ -194,7 +206,8 @@ data class DoubleNode( data class NullNode( override val pos: Pos, override val path: DotPath, - override val meta: Map = emptyMap() + override val meta: Map = emptyMap(), + override val sourceKey: String? = if (path == DotPath.root) null else path.flatten(), ) : PrimitiveNode() { override val simpleName: String = "null" override val value: Any? = null @@ -204,6 +217,7 @@ object Undefined : Node { override val simpleName: String = "Undefined" override val pos: Pos = Pos.NoPos override val path = DotPath.root + override val sourceKey: String? = null override fun atKey(key: String): Node = this override fun atIndex(index: Int): Node = this override val size: Int = 0 diff --git a/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/parsers/loadProps.kt b/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/parsers/loadProps.kt index b343df06..f12b8ad3 100644 --- a/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/parsers/loadProps.kt +++ b/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/parsers/loadProps.kt @@ -9,75 +9,95 @@ import com.sksamuel.hoplite.Undefined import com.sksamuel.hoplite.decoder.DotPath import java.util.Properties -fun Properties.toNode(source: String, delimiter: String = ".") = asIterable().toNode( +fun Properties.toNode( + source: String, + delimiter: String = ".", + keyExtractor: (Any) -> String = { it.toString() }, +) = asIterable().toNode( source = source, - keyExtractor = { it.key.toString() }, + sourceKeyExtractor = { it.key }, + keyExtractor = keyExtractor, valueExtractor = { it.value }, delimiter = delimiter ) -fun Map.toNode(source: String, delimiter: String = ".") = entries.toNode( +fun Map.toNode( + source: String, + delimiter: String = ".", + keyExtractor: (String) -> String = { it }, +) = entries.toNode( source = source, - keyExtractor = { it.key }, + sourceKeyExtractor = { it.key }, + keyExtractor = keyExtractor, valueExtractor = { it.value }, delimiter = delimiter ) data class Element( val values: MutableMap = hashMapOf(), - var value: Any? = null + var value: Any? = null, + var sourceKey: String? = null, ) -private fun Iterable.toNode( +private fun Iterable.toNode( source: String, - keyExtractor: (T) -> String, + sourceKeyExtractor: (T) -> K, + keyExtractor: (K) -> String, valueExtractor: (T) -> Any?, delimiter: String = "." ): Node { val map = Element() forEach { item -> - val key = keyExtractor(item) + val sourceKey = sourceKeyExtractor(item) + val key = keyExtractor(sourceKey) val value = valueExtractor(item) val segments = key.split(delimiter) segments.foldIndexed(map) { index, element, segment -> element.values.getOrPut(segment) { Element() }.also { - if (index == segments.size - 1) it.value = value + if (index == segments.size - 1) { + it.value = value + it.sourceKey = sourceKey.toString() + } } } } val pos = Pos.SourcePos(source) - fun Any.transform(path: DotPath): Node = when (this) { + fun Any.transform(path: DotPath, parentSourceKey: String? = null): Node = when (this) { is Element -> when { - value != null && values.isEmpty() -> value?.transform(path) ?: Undefined + value != null && values.isEmpty() -> value?.transform(path, sourceKey) ?: Undefined else -> MapNode( - map = values.takeUnless { it.isEmpty() }?.mapValues { it.value.transform(path.with(it.key)) }.orEmpty(), + map = values.takeUnless { it.isEmpty() }?.mapValues { it.value.transform(path.with(it.key), sourceKey) }.orEmpty(), pos = pos, path = path, - value = value?.transform(path) ?: Undefined + value = value?.transform(path, sourceKey) ?: Undefined, + sourceKey = this.sourceKey, ) } is Array<*> -> ArrayNode( - elements = mapNotNull { it?.transform(path) }, + elements = mapNotNull { it?.transform(path, parentSourceKey) }, pos = pos, - path = path + path = path, + sourceKey = parentSourceKey, ) is Collection<*> -> ArrayNode( - elements = mapNotNull { it?.transform(path) }, + elements = mapNotNull { it?.transform(path, parentSourceKey) }, pos = pos, - path = path + path = path, + sourceKey = parentSourceKey, ) is Map<*, *> -> MapNode( map = takeUnless { it.isEmpty() }?.mapNotNull { entry -> - entry.value?.let { entry.key.toString() to it.transform(path.with(entry.key.toString())) } + entry.value?.let { entry.key.toString() to it.transform(path.with(entry.key.toString()), parentSourceKey) } }?.toMap().orEmpty(), pos = pos, - path = path + path = path, + sourceKey = parentSourceKey, ) - else -> StringNode(this.toString(), pos, path = path, emptyMap()) + else -> StringNode(this.toString(), pos, path = path, emptyMap(), parentSourceKey) } return map.transform(DotPath.root) diff --git a/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/report/Reporter.kt b/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/report/Reporter.kt index 89e84c9b..4cfa1e0c 100644 --- a/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/report/Reporter.kt +++ b/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/report/Reporter.kt @@ -44,6 +44,7 @@ class Reporter( object Titles { const val Key = "Key" const val Source = "Source" + const val SourceKey = "Source Key" const val Value = "Value" } @@ -107,19 +108,22 @@ class Reporter( value = value ?: "", pos = state.node.pos, path = state.node.path, - meta = state.node.meta + meta = state.node.meta, + sourceKey = state.node.sourceKey, ) ) } val keyPadded = max(Titles.Key.length, nodes.maxOf { it.node.path.flatten().length }) val sourcePadded = nodes.maxOf { max(it.node.pos.source()?.length ?: 0, Titles.Source.length) } + val sourceKeyPadded = max(Titles.SourceKey.length, nodes.maxOf { it.node.sourceKey.orEmpty().length }) val valuePadded = max(Titles.Value.length, obfuscated.maxOf { (it.node as StringNode).value.length }) val rows = obfuscated.map { listOfNotNull( it.node.path.flatten().padEnd(keyPadded, ' '), (it.node.pos.source() ?: "").padEnd(sourcePadded, ' '), + it.node.sourceKey.orEmpty().padEnd(sourceKeyPadded, ' '), (it.node as StringNode).value.padEnd(valuePadded, ' ') ).joinToString(" | ", "| ", " |") } @@ -129,12 +133,14 @@ class Reporter( val bar = listOfNotNull( "".padEnd(keyPadded + 2, '-'), "".padEnd(sourcePadded + 2, '-'), + "".padEnd(sourceKeyPadded + 2, '-'), "".padEnd(valuePadded + 2, '-') ).joinToString("+", "+", "+") val titles = listOfNotNull( Titles.Key.padEnd(keyPadded, ' '), Titles.Source.padEnd(sourcePadded, ' '), + Titles.SourceKey.padEnd(sourceKeyPadded, ' '), Titles.Value.padEnd(valuePadded, ' ') ).joinToString(" | ", "| ", " |") diff --git a/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/sources/EnvironmentVariablesPropertySource.kt b/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/sources/EnvironmentVariablesPropertySource.kt index 48e674c4..2dad1ce1 100644 --- a/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/sources/EnvironmentVariablesPropertySource.kt +++ b/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/sources/EnvironmentVariablesPropertySource.kt @@ -6,7 +6,6 @@ import com.sksamuel.hoplite.PropertySource import com.sksamuel.hoplite.PropertySourceContext import com.sksamuel.hoplite.fp.valid import com.sksamuel.hoplite.parsers.toNode -import java.util.Properties class EnvironmentVariablesPropertySource( private val useUnderscoresAsSeparator: Boolean, @@ -18,29 +17,28 @@ class EnvironmentVariablesPropertySource( override fun source(): String = "Env Var" override fun node(context: PropertySourceContext): ConfigResult { - val props = Properties() - environmentVariableMap() + val map = environmentVariableMap() .mapKeys { if (prefix == null) it.key else it.key.removePrefix(prefix) } - .forEach { - val key = it.key - .let { key -> if (useUnderscoresAsSeparator) key.replace("__", ".") else key } - .let { key -> - if (allowUppercaseNames && Character.isUpperCase(key.codePointAt(0))) { - key.split(".").joinToString(separator = ".") { value -> - value.fold("") { acc, char -> - when { - acc.isEmpty() -> acc + char.lowercaseChar() - acc.last() == '_' -> acc.dropLast(1) + char.uppercaseChar() - else -> acc + char.lowercaseChar() - } + + return map.toNode("env") { key -> + key + .let { if (prefix == null) it else it.removePrefix(prefix) } + .let { if (useUnderscoresAsSeparator) it.replace("__", ".") else it } + .let { + if (allowUppercaseNames && Character.isUpperCase(it.codePointAt(0))) { + it.split(".").joinToString(separator = ".") { value -> + value.fold("") { acc, char -> + when { + acc.isEmpty() -> acc + char.lowercaseChar() + acc.last() == '_' -> acc.dropLast(1) + char.uppercaseChar() + else -> acc + char.lowercaseChar() } } - } else { - key } + } else { + it } - props[key] = it.value - } - return props.toNode("env").valid() + } + }.valid() } } diff --git a/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/sources/SystemPropertiesPropertySource.kt b/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/sources/SystemPropertiesPropertySource.kt index 97fc0820..34acd4c4 100644 --- a/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/sources/SystemPropertiesPropertySource.kt +++ b/hoplite-core/src/main/kotlin/com/sksamuel/hoplite/sources/SystemPropertiesPropertySource.kt @@ -23,13 +23,10 @@ open class SystemPropertiesPropertySource( override fun source(): String = "System Properties" override fun node(context: PropertySourceContext): ConfigResult { - val props = Properties() - systemPropertiesMap().let { systemPropertiesMap -> - systemPropertiesMap.keys - .filter { it.startsWith(prefix) } - .forEach { props[it.removePrefix(prefix)] = systemPropertiesMap[it] } - } - return if (props.isEmpty) Undefined.valid() else props.toNode("sysprops").valid() + val map = systemPropertiesMap().filter { it.key.startsWith(prefix) } + return if (map.isEmpty()) Undefined.valid() else map.toNode("sysprops") { + it.removePrefix(prefix) + }.valid() } companion object : SystemPropertiesPropertySource() { diff --git a/hoplite-core/src/test/kotlin/com/sksamuel/hoplite/EnvironmentVariablesPropertySourceTest.kt b/hoplite-core/src/test/kotlin/com/sksamuel/hoplite/EnvironmentVariablesPropertySourceTest.kt index 877e4c3d..fd9bfb5d 100644 --- a/hoplite-core/src/test/kotlin/com/sksamuel/hoplite/EnvironmentVariablesPropertySourceTest.kt +++ b/hoplite-core/src/test/kotlin/com/sksamuel/hoplite/EnvironmentVariablesPropertySourceTest.kt @@ -17,12 +17,13 @@ class EnvironmentVariablesPropertySourceTest : FunSpec({ ).getUnsafe() shouldBe MapNode( mapOf( "a" to MapNode( - value = StringNode("foo", Pos.env, DotPath("a")), - map = mapOf("b" to StringNode("bar", Pos.env, DotPath("a", "b"))), + value = StringNode("foo", Pos.env, DotPath("a"), sourceKey = "a"), + map = mapOf("b" to StringNode("bar", Pos.env, DotPath("a", "b"), sourceKey = "a.b")), pos = Pos.SourcePos("env"), - path = DotPath("a") + path = DotPath("a"), + sourceKey = "a" ), - "c" to StringNode("baz", Pos.env, DotPath("c")) + "c" to StringNode("baz", Pos.env, DotPath("c"), sourceKey = "c"), ), pos = Pos.env, DotPath.root diff --git a/hoplite-core/src/test/kotlin/com/sksamuel/hoplite/LoadPropsTest.kt b/hoplite-core/src/test/kotlin/com/sksamuel/hoplite/LoadPropsTest.kt index 249e3d60..6e81f3c5 100644 --- a/hoplite-core/src/test/kotlin/com/sksamuel/hoplite/LoadPropsTest.kt +++ b/hoplite-core/src/test/kotlin/com/sksamuel/hoplite/LoadPropsTest.kt @@ -21,33 +21,37 @@ class LoadPropsTest : FunSpec({ mapOf( "b" to MapNode( mapOf( - "c" to StringNode("wibble", pos = Pos.SourcePos(source = "source"), DotPath("a", "b", "c")), - "d" to StringNode("123", pos = Pos.SourcePos(source = "source"), DotPath("a", "b", "d")) + "c" to StringNode("wibble", pos = Pos.SourcePos(source = "source"), DotPath("a", "b", "c"), sourceKey = "a.b.c"), + "d" to StringNode("123", pos = Pos.SourcePos(source = "source"), DotPath("a", "b", "d"), sourceKey = "a.b.d") ), pos = Pos.SourcePos(source = "source"), DotPath("a", "b"), - value = Undefined + value = Undefined, + sourceKey = null ), - "d" to StringNode("true", pos = Pos.SourcePos(source = "source"), DotPath("a", "d")) + "d" to StringNode("true", pos = Pos.SourcePos(source = "source"), DotPath("a", "d"), sourceKey = "a.d") ), pos = Pos.SourcePos(source = "source"), DotPath("a"), - value = StringNode("foo", Pos.SourcePos(source = "source"), DotPath("a")) + value = StringNode("foo", Pos.SourcePos(source = "source"), DotPath("a"), sourceKey = "a"), + sourceKey = "a" ), "e" to MapNode( mapOf( "f" to MapNode( mapOf( - "g" to StringNode("goo", pos = Pos.SourcePos(source = "source"), DotPath("e", "f", "g")) + "g" to StringNode("goo", pos = Pos.SourcePos(source = "source"), DotPath("e", "f", "g"), sourceKey = "e.f.g") ), pos = Pos.SourcePos(source = "source"), DotPath("e", "f"), - value = StringNode("6", Pos.SourcePos(source = "source"), DotPath("e", "f")) + value = StringNode("6", Pos.SourcePos(source = "source"), DotPath("e", "f"), sourceKey = "e.f"), + sourceKey = "e.f" ) ), pos = Pos.SourcePos(source = "source"), DotPath("e"), - value = StringNode("5.5", Pos.SourcePos(source = "source"), DotPath("e")) + value = StringNode("5.5", Pos.SourcePos(source = "source"), DotPath("e"), sourceKey = "e"), + sourceKey = "e" ) ), pos = Pos.SourcePos(source = "source"), diff --git a/hoplite-core/src/test/kotlin/com/sksamuel/hoplite/PropsParserTest.kt b/hoplite-core/src/test/kotlin/com/sksamuel/hoplite/PropsParserTest.kt index eead5b3d..f402ea65 100644 --- a/hoplite-core/src/test/kotlin/com/sksamuel/hoplite/PropsParserTest.kt +++ b/hoplite-core/src/test/kotlin/com/sksamuel/hoplite/PropsParserTest.kt @@ -18,20 +18,23 @@ class PropsParserTest : StringSpec() { "c" to StringNode( value = "wibble", pos = Pos.SourcePos(source = "a.props"), - DotPath("a", "b", "c") + DotPath("a", "b", "c"), + sourceKey = "a.b.c" ), - "d" to StringNode(value = "123", pos = Pos.SourcePos(source = "a.props"), DotPath("a", "b", "d")) + "d" to StringNode(value = "123", pos = Pos.SourcePos(source = "a.props"), DotPath("a", "b", "d"), sourceKey = "a.b.d") ), pos = Pos.SourcePos(source = "a.props"), DotPath("a", "b"), - value = StringNode("qqq", pos = Pos.SourcePos(source = "a.props"), DotPath("a", "b")) + value = StringNode("qqq", pos = Pos.SourcePos(source = "a.props"), DotPath("a", "b"), sourceKey = "a.b"), + sourceKey = "a.b" ), - "g" to StringNode(value = "true", pos = Pos.SourcePos(source = "a.props"), DotPath("a", "g")) + "g" to StringNode(value = "true", pos = Pos.SourcePos(source = "a.props"), DotPath("a", "g"), sourceKey = "a.g") ), pos = Pos.SourcePos(source = "a.props"), - DotPath("a") + DotPath("a"), + sourceKey = null ), - "e" to StringNode(value = "5.5", pos = Pos.SourcePos(source = "a.props"), DotPath("e")) + "e" to StringNode(value = "5.5", pos = Pos.SourcePos(source = "a.props"), DotPath("e"), sourceKey = "e") ), pos = Pos.SourcePos(source = "a.props"), DotPath.root diff --git a/hoplite-core/src/test/kotlin/com/sksamuel/hoplite/ReporterTest.kt b/hoplite-core/src/test/kotlin/com/sksamuel/hoplite/ReporterTest.kt index 7edde058..e13d6a9c 100644 --- a/hoplite-core/src/test/kotlin/com/sksamuel/hoplite/ReporterTest.kt +++ b/hoplite-core/src/test/kotlin/com/sksamuel/hoplite/ReporterTest.kt @@ -40,14 +40,14 @@ class ReporterTest : FunSpec({ }.shouldContain( """ Used keys: 4 -+----------+---------------------+----------+ -| Key | Source | Value | -+----------+---------------------+----------+ -| host | props string source | loc***** | -| name | props string source | my ***** | -| password | props string source | ssm***** | -| port | props string source | 3306 | -+----------+---------------------+----------+ ++----------+---------------------+------------+----------+ +| Key | Source | Source Key | Value | ++----------+---------------------+------------+----------+ +| host | props string source | host | loc***** | +| name | props string source | name | my ***** | +| password | props string source | password | ssm***** | +| port | props string source | port | 3306 | ++----------+---------------------+------------+----------+ """ ) @@ -98,14 +98,14 @@ Property sources (highest to lowest priority): }.shouldContain( """ Used keys: 4 -+----------+---------------------+----------+ -| Key | Source | Value | -+----------+---------------------+----------+ -| host | props string source | lr***** | -| name | props string source | my ***** | -| password | props string source | ssm***** | -| port | props string source | 3306 | -+----------+---------------------+----------+ ++----------+---------------------+------------+----------+ +| Key | Source | Source Key | Value | ++----------+---------------------+------------+----------+ +| host | props string source | host | lr***** | +| name | props string source | name | my ***** | +| password | props string source | password | ssm***** | +| port | props string source | port | 3306 | ++----------+---------------------+------------+----------+ """ ) @@ -131,12 +131,12 @@ Used keys: 4 } out shouldContain """ -+---------------+---------------------+----------+ -| Key | Source | Value | -+---------------+---------------------+----------+ -| database.name | props string source | my ***** | -| database.port | props string source | 3306 | -+---------------+---------------------+----------+ ++---------------+---------------------+---------------+----------+ +| Key | Source | Source Key | Value | ++---------------+---------------------+---------------+----------+ +| database.name | props string source | database.name | my ***** | +| database.port | props string source | database.port | 3306 | ++---------------+---------------------+---------------+----------+ """.trim() } @@ -177,14 +177,14 @@ Used keys: 4 }.shouldContain( """ Used keys: 4 -+----------+---------------------+-------------+ -| Key | Source | Value | -+----------+---------------------+-------------+ -| host | props string source | localhost | -| name | props string source | my database | -| password | props string source | gcp***** | -| port | props string source | 3306 | -+----------+---------------------+-------------+ ++----------+---------------------+------------+-------------+ +| Key | Source | Source Key | Value | ++----------+---------------------+------------+-------------+ +| host | props string source | host | localhost | +| name | props string source | name | my database | +| password | props string source | password | gcp***** | +| port | props string source | port | 3306 | ++----------+---------------------+------------+-------------+ """ )