From 9979460cc20eae33d73ece15fb7dc2fafdc65a22 Mon Sep 17 00:00:00 2001 From: Jakub Amanowicz Date: Thu, 10 Oct 2024 14:06:00 +0200 Subject: [PATCH 1/8] CASL-561 tags querying Signed-off-by: Jakub Amanowicz --- .../executors/query/WhereClauseBuilder.kt | 87 ++++++++++++++++--- .../naksha/psql/ReadFeaturesByTagsTest.kt | 83 ++++++++++++++++++ 2 files changed, 157 insertions(+), 13 deletions(-) create mode 100644 here-naksha-lib-psql/src/commonTest/kotlin/naksha/psql/ReadFeaturesByTagsTest.kt diff --git a/here-naksha-lib-psql/src/commonMain/kotlin/naksha/psql/executors/query/WhereClauseBuilder.kt b/here-naksha-lib-psql/src/commonMain/kotlin/naksha/psql/executors/query/WhereClauseBuilder.kt index d3ac140a2..760bf133b 100644 --- a/here-naksha-lib-psql/src/commonMain/kotlin/naksha/psql/executors/query/WhereClauseBuilder.kt +++ b/here-naksha-lib-psql/src/commonMain/kotlin/naksha/psql/executors/query/WhereClauseBuilder.kt @@ -21,6 +21,7 @@ class WhereClauseBuilder(private val request: ReadFeatures) { whereVersion() whereMetadata() whereSpatial() + whereTags() return if (where.isBlank()) { null } else { @@ -143,8 +144,8 @@ class WhereClauseBuilder(private val request: ReadFeatures) { ) val placeholder = placeholderForArg(metaQuery.value, pgColumn.type) val resolvedQuery = when (val op = metaQuery.op) { - is StringOp -> resolveStringOp(op, pgColumn, placeholder) - is DoubleOp -> resolveDoubleOp(op, pgColumn, placeholder) + is StringOp -> resolveStringOp(op, pgColumn.name, placeholder) + is DoubleOp -> resolveDoubleOp(op, pgColumn.name, placeholder) else -> throw NakshaException( NakshaError.ILLEGAL_ARGUMENT, "Unknown op type: ${op::class.simpleName}" @@ -160,6 +161,62 @@ class WhereClauseBuilder(private val request: ReadFeatures) { } } + private fun whereTags() { + request.query.tags?.let { tagQuery -> + if (where.isNotEmpty()) { + where.append(" AND (") + } else { + where.append(" (") + } + whereNestedTags(tagQuery) + where.append(")") + } + } + + private tailrec fun whereNestedTags(tagQuery: ITagQuery){ + when(tagQuery){ + is TagNot -> not(tagQuery.query, this::whereNestedTags) + is TagOr -> or(tagQuery.filterNotNull(), this::whereNestedTags) + is TagAnd -> and(tagQuery.filterNotNull(), this::whereNestedTags) + is TagQuery -> resolveSingleTagQuery(tagQuery) + } + } + + private fun resolveSingleTagQuery(tagQuery: TagQuery){ + when(tagQuery){ + is TagExists -> { + where.append("$tagsAsJsonb ? ${tagQuery.name}") + } + is TagValueIsNull -> { + where.append("${tagValue(tagQuery)} = null") + } + is TagValueIsBool -> { + if(tagQuery.value){ + where.append(tagValue(tagQuery)) + } else { + where.append("not(${tagValue(tagQuery)}})") + } + } + is TagValueIsDouble -> { + val valuePlaceholder = placeholderForArg(tagQuery.value, PgType.DOUBLE) + val doubleOp = resolveDoubleOp(tagQuery.op, tagValue(tagQuery), valuePlaceholder) + where.append(doubleOp) + } + is TagValueIsString -> { + val valuePlaceholder = placeholderForArg(tagQuery.value, PgType.STRING) + val stringEquals = resolveStringOp(StringOp.EQUALS, tagValue(tagQuery), valuePlaceholder) + where.append(stringEquals) + } + is TagValueMatches -> { + val regexPlaceholder = placeholderForArg(tagQuery.regex, PgType.STRING) + where.append("$tagsAsJsonb @? '$[?(@.${tagQuery.name}=~/$regexPlaceholder/)]") + } + } + } + + private fun tagValue(tagQuery: TagQuery): String = + "$tagsAsJsonb->>${tagQuery.name}" + private fun not(subClause: T, subClauseResolver: (T) -> Unit) { where.append(" NOT (") subClauseResolver(subClause) @@ -195,12 +252,12 @@ class WhereClauseBuilder(private val request: ReadFeatures) { private fun resolveStringOp( stringOp: StringOp, - column: PgColumn, - valuePlaceholder: String + leftOperand: String, + rightOperand: String ): String { return when (stringOp) { - StringOp.EQUALS -> "${column.name} = $valuePlaceholder" - StringOp.STARTS_WITH -> "starts_with(${column.name}, $valuePlaceholder)" + StringOp.EQUALS -> "$leftOperand = $rightOperand" + StringOp.STARTS_WITH -> "starts_with($leftOperand, $rightOperand)" else -> throw NakshaException( NakshaError.ILLEGAL_ARGUMENT, "Unknown StringOp: $stringOp" @@ -210,19 +267,23 @@ class WhereClauseBuilder(private val request: ReadFeatures) { private fun resolveDoubleOp( doubleOp: DoubleOp, - column: PgColumn, - valuePlaceholder: String + leftOperand: String, + rightOperand: String ): String { return when (doubleOp) { - DoubleOp.EQ -> "${column.name} = $valuePlaceholder" - DoubleOp.GT -> "${column.name} > $valuePlaceholder" - DoubleOp.GTE -> "${column.name} >= $valuePlaceholder" - DoubleOp.LT -> "${column.name} < $valuePlaceholder" - DoubleOp.LTE -> "${column.name} <= $valuePlaceholder" + DoubleOp.EQ -> "$leftOperand = $rightOperand" + DoubleOp.GT -> "$leftOperand > $rightOperand" + DoubleOp.GTE -> "$leftOperand >= $rightOperand" + DoubleOp.LT -> "$leftOperand < $rightOperand" + DoubleOp.LTE -> "$leftOperand <= $rightOperand" else -> throw NakshaException( NakshaError.ILLEGAL_ARGUMENT, "Unknown DoubleOp: $doubleOp" ) } } + + companion object { + private val tagsAsJsonb = "naksha_tags(${PgColumn.tags}, ${PgColumn.flags})::jsonb" + } } diff --git a/here-naksha-lib-psql/src/commonTest/kotlin/naksha/psql/ReadFeaturesByTagsTest.kt b/here-naksha-lib-psql/src/commonTest/kotlin/naksha/psql/ReadFeaturesByTagsTest.kt new file mode 100644 index 000000000..d927db3cd --- /dev/null +++ b/here-naksha-lib-psql/src/commonTest/kotlin/naksha/psql/ReadFeaturesByTagsTest.kt @@ -0,0 +1,83 @@ +package naksha.psql + +import naksha.model.TagList +import naksha.model.objects.NakshaCollection +import naksha.model.objects.NakshaFeature +import naksha.model.request.ReadFeatures +import naksha.model.request.SuccessResponse +import naksha.model.request.query.TagExists +import naksha.model.request.query.TagQuery +import naksha.model.request.query.TagValueIsString +import naksha.psql.base.PgTestBase +import naksha.psql.util.ProxyFeatureGenerator +import kotlin.test.Test +import kotlin.test.assertEquals + +class ReadFeaturesByTagsTest : PgTestBase(NakshaCollection("read_by_tags_test")) { + + @Test + fun shouldReturnFeaturesWithExistingTag() { + // Given: + val inputFeature = randomFeatureWithTags("foo=bar") + + // When: + insertFeature(feature = inputFeature) + + // And: + val featuresWithFooTag = executeTagsQuery( + TagExists("foo") + ).features + + // Then: + assertEquals(1, featuresWithFooTag.size) + assertEquals(inputFeature.id, featuresWithFooTag[0]!!.id) + } + + @Test + fun shouldNotReturnFeaturesWithMissingTag() { + // Given: + val inputFeature = randomFeatureWithTags("foo") + + // When: + insertFeature(feature = inputFeature) + + // And: + val featuresWithFooTag = executeTagsQuery( + TagExists("bar") + ).features + + // Then: + assertEquals(0, featuresWithFooTag.size) + } + + @Test + fun shouldReturnFeaturesWithTagValue() { + // Given: + val inputFeature = randomFeatureWithTags("foo=bar") + + // When: + insertFeature(feature = inputFeature) + + // And: + val featuresWithFooTag = executeTagsQuery( + TagValueIsString(name = "foo", value = "bar") + ).features + + // Then: + assertEquals(1, featuresWithFooTag.size) + assertEquals(inputFeature.id, featuresWithFooTag[0]!!.id) + } + + private fun randomFeatureWithTags(vararg tags: String): NakshaFeature { + return ProxyFeatureGenerator.generateRandomFeature().apply { + properties.xyz.tags = TagList(*tags) + } + } + + private fun executeTagsQuery(tagQuery: TagQuery): SuccessResponse { + return executeRead(ReadFeatures().apply { + collectionIds += collection!!.id + query.tags = tagQuery + }) + } +} \ No newline at end of file From 575e106125930904d953f73235a1f2f8d1ebf26d Mon Sep 17 00:00:00 2001 From: Jakub Amanowicz Date: Fri, 18 Oct 2024 14:21:06 +0200 Subject: [PATCH 2/8] CASL-561 tag normalization Signed-off-by: Jakub Amanowicz --- .../kotlin/naksha/base/NormalizerForm.kt | 2 +- .../src/commonMain/kotlin/naksha/model/Tag.kt | 52 ++---- .../commonMain/kotlin/naksha/model/TagList.kt | 54 ++++-- .../commonMain/kotlin/naksha/model/TagMap.kt | 18 +- .../kotlin/naksha/model/TagNormalizer.kt | 110 ++++++++++++ .../commonMain/kotlin/naksha/model/XyzNs.kt | 60 +------ .../executors/query/WhereClauseBuilder.kt | 58 +++++-- .../naksha/psql/ReadFeaturesByTagsTest.kt | 161 +++++++++++++++++- .../kotlin/naksha/psql/TagNormalizerTest.kt | 92 ++++++++++ .../kotlin/naksha/psql/base/PgTestBase.kt | 3 + .../jvmMain/kotlin/naksha/psql/PsqlPlan.kt | 7 + .../here/naksha/storage/http/HttpStorage.java | 6 +- .../storage/http/HttpStorageProperties.java | 4 +- .../storage/http/HttpStorageReadExecute.java | 10 +- .../storage/http/HttpStorageReadSession.java | 7 +- .../storage/http/POpToQueryConverter.java | 4 +- .../naksha/storage/http/PrepareResult.java | 3 +- 17 files changed, 484 insertions(+), 167 deletions(-) create mode 100644 here-naksha-lib-model/src/commonMain/kotlin/naksha/model/TagNormalizer.kt create mode 100644 here-naksha-lib-psql/src/commonTest/kotlin/naksha/psql/TagNormalizerTest.kt diff --git a/here-naksha-lib-base/src/commonMain/kotlin/naksha/base/NormalizerForm.kt b/here-naksha-lib-base/src/commonMain/kotlin/naksha/base/NormalizerForm.kt index 04ded0e6b..852ba575a 100644 --- a/here-naksha-lib-base/src/commonMain/kotlin/naksha/base/NormalizerForm.kt +++ b/here-naksha-lib-base/src/commonMain/kotlin/naksha/base/NormalizerForm.kt @@ -4,7 +4,7 @@ import kotlin.js.ExperimentalJsExport import kotlin.js.JsExport /** - * Please refer []Unicode Normalization Forms](https://www.unicode.org/reports/tr15/#Norm_Forms) + * Please refer [Unicode Normalization Forms](https://www.unicode.org/reports/tr15/#Norm_Forms) */ @OptIn(ExperimentalJsExport::class) @JsExport diff --git a/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/Tag.kt b/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/Tag.kt index 1537e6b78..4174e7121 100644 --- a/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/Tag.kt +++ b/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/Tag.kt @@ -3,6 +3,8 @@ package naksha.model import naksha.base.* +import naksha.base.NormalizerForm.NFD +import naksha.base.NormalizerForm.NFKC import naksha.model.NakshaError.NakshaErrorCompanion.ILLEGAL_ARGUMENT import kotlin.js.JsExport import kotlin.js.JsName @@ -16,10 +18,10 @@ import kotlin.jvm.JvmStatic * @property value the value of the tag; _null_, Boolean, String or Double. */ @JsExport -class Tag(): AnyObject() { +class Tag() : AnyObject() { @JsName("of") - constructor(tag: String, key: String, value: Any?): this() { + constructor(tag: String, key: String, value: Any?) : this() { this.tag = tag this.key = key this.value = value @@ -32,42 +34,19 @@ class Tag(): AnyObject() { @JvmStatic @JsStatic - fun parse(tag: String): Tag { - val i = tag.indexOf('=') - val key: String - val value: Any? - if (i > 1) { - if (tag[i-1] == ':') { // := - key = tag.substring(0, i-1).trim() - val raw = tag.substring(i + 1).trim() - value = if ("true".equals(raw, ignoreCase = true)) { - true - } else if ("false".equals(raw, ignoreCase = true)) { - false - } else { - raw. toDouble() - } - } else { - key = tag.substring(0, i).trim() - value = tag.substring(i + 1).trim() - } - } else { - key = tag - value = null + fun of(normalizedKey: String, normalizedValue: Any?): Tag = when (normalizedValue) { + null -> Tag(normalizedKey, normalizedKey, null) + is String -> Tag("$normalizedKey=$normalizedValue", normalizedKey, normalizedValue) + is Boolean -> Tag("$normalizedKey:=$normalizedValue", normalizedKey, normalizedValue) + is Number -> { + val doubleValue = normalizedValue.toDouble() + Tag("$normalizedKey:=$doubleValue", normalizedKey, doubleValue) } - return Tag(tag, key, value) - } - @JvmStatic - @JsStatic - fun of(key: String, value: Any?): Tag = when(value) { - // TODO: Fix normalization! - null -> Tag(key, key, null) - is String -> Tag("$key=$value", key, value) - is Boolean, Double -> Tag("$key:=$value", key, value) - is Number -> of(key, value.toDouble()) - is Int64 -> of(key, value.toDouble()) - else -> throw NakshaException(ILLEGAL_ARGUMENT, "Tag values can only be String, Boolean or Double") + else -> throw NakshaException( + ILLEGAL_ARGUMENT, + "Tag values can only be String, Boolean or Number" + ) } } @@ -81,6 +60,7 @@ class Tag(): AnyObject() { if (other is Tag) return tag == other.tag return false } + override fun hashCode(): Int = tag.hashCode() override fun toString(): String = tag } \ No newline at end of file diff --git a/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/TagList.kt b/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/TagList.kt index f77d0e299..e53c55143 100644 --- a/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/TagList.kt +++ b/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/TagList.kt @@ -3,7 +3,9 @@ package naksha.model import naksha.base.ListProxy -import naksha.model.XyzNs.XyzNsCompanion.normalizeTag +import naksha.base.NormalizerForm +import naksha.base.Platform +import naksha.model.TagNormalizer.normalizeTag import kotlin.js.JsExport import kotlin.js.JsName import kotlin.js.JsStatic @@ -24,21 +26,6 @@ open class TagList() : ListProxy(String::class) { addTags(listOf(*tags), false) } - companion object TagList_C { - /** - * Create a tag list from the given array; the tags are normalized. - * @param tags the tags. - * @return the tag-list. - */ - @JvmStatic - @JsStatic - fun fromArray(tags: Array): TagList { - val list = TagList() - list.addAndNormalizeTags(*tags) - return list - } - } - /** * Returns 'true' if the tag was removed, 'false' if it was not present. * @@ -161,9 +148,44 @@ open class TagList() : ListProxy(String::class) { return this } + /** * Convert this tag-list into a tag-map. * @return this tag-list as tag-map. */ fun toTagMap(): TagMap = TagMap(this) + + companion object TagList_C { + /** + * Create a tag list from the given array; the tags are normalized. + * @param tags the tags. + * @return the tag-list. + */ + @JvmStatic + @JsStatic + fun fromArray(tags: Array): TagList { + val list = TagList() + list.addAndNormalizeTags(*tags) + return list + } + + /** + * A method to normalize a list of tags. + * + * @param tags a list of tags. + * @return the same list, just that the content is normalized. + */ + @JvmStatic + @JsStatic + fun normalizeTags(tags: TagList?): TagList? { + if (!tags.isNullOrEmpty()) { + for ((idx, tag) in tags.withIndex()) { + if (tag != null) { + tags[idx] = normalizeTag(tag) + } + } + } + return tags + } + } } diff --git a/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/TagMap.kt b/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/TagMap.kt index dbb734a05..eed811048 100644 --- a/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/TagMap.kt +++ b/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/TagMap.kt @@ -3,7 +3,6 @@ package naksha.model import naksha.base.MapProxy -import naksha.model.request.query.* import kotlin.js.JsExport import kotlin.js.JsName @@ -11,16 +10,14 @@ import kotlin.js.JsName // Improve me! @JsExport -open class TagMap() : MapProxy(String::class, Tag::class) { +open class TagMap() : MapProxy(String::class, Any::class) { @Suppress("LeakingThis") @JsName("of") - constructor(tagList: TagList) : this(){ - for (s in tagList) { - if (s == null) continue - val tag = Tag.parse(s) - put(tag.key, tag) - } + constructor(tagList: TagList) : this() { + tagList.filterNotNull() + .map { TagNormalizer.splitNormalizedTag(it) } + .forEach { tag -> put(tag.key, tag.value) } } /** @@ -29,9 +26,8 @@ open class TagMap() : MapProxy(String::class, Tag::class) { */ fun toTagList(): TagList { val list = TagList() - for (e in this) { - val tag = e.value?.tag - if (tag != null) list.add(tag) + forEach { (key, value) -> + list.add(Tag.of(key, value).tag) } return list } diff --git a/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/TagNormalizer.kt b/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/TagNormalizer.kt new file mode 100644 index 000000000..90e217767 --- /dev/null +++ b/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/TagNormalizer.kt @@ -0,0 +1,110 @@ +package naksha.model + +import naksha.base.NormalizerForm +import naksha.base.NormalizerForm.NFD +import naksha.base.NormalizerForm.NFKC +import naksha.base.Platform +import naksha.model.TagNormalizer.normalizeTag +import naksha.model.TagNormalizer.splitNormalizedTag + +/** + * An object used for Tag normalization and splitting. + * + * Process of normalization happens in [normalizeTag] method and includes following steps: + * 1) Always: apply normalization form (see [NormalizerForm]) + * 2) Conditional: lowercase the whole tag + * 3) Conditional: remove all non-ASCII characters + * + * Normalization form used in step #1 and subsequent conditional steps depend on tag prefix. + * + * + * Process of splitting happens in [splitNormalizedTag] method. + * It is about splitting the normalized tag to [key, value] pair in form of [Tag] + * Note that not all tags can be split, it depends on their prefix. + * + * Summarised per-prefix behavior: + * +----------+------------+-----------+----------+-------+ + * | prefix | norm. form | lowercase | no ASCII | split | + * +----------+------------+-----------+----------+-------+ + * | @ | NFKC | false | false | true | + * | ref_ | NFKC | false | false | false | + * | ~ | NFD | false | true | true | + * | # | NFD | false | true | true | + * | sourceID | NFKC | false | false | false | + * | < ELSE > | NFD | true | true | true | + * +----------+------------+-----------+----------+-------+ + * + * By default, (if no special prefix is found) tag is normalized with NFD, lowercased, cleaned of non-ASCII and splittable. + */ +object TagNormalizer { + private data class TagProcessingPolicy( + val normalizerForm: NormalizerForm, + val removeNonAscii: Boolean, + val lowercase: Boolean, + val split: Boolean + ) + + private val DEFAULT_POLICY = TagProcessingPolicy(NFD, removeNonAscii = true, lowercase = true, split = true) + private val PREFIX_TO_POLICY = mapOf( + "@" to TagProcessingPolicy(NFKC, removeNonAscii = false, lowercase = false, split = true), + "ref_" to TagProcessingPolicy(NFKC, removeNonAscii = false, lowercase = false, split = false), + "sourceID" to TagProcessingPolicy(NFKC, removeNonAscii = false, lowercase = false, split = false), + "~" to TagProcessingPolicy(NFD, removeNonAscii = true, lowercase = false, split = true), + "#" to TagProcessingPolicy(NFD, removeNonAscii = true, lowercase = false, split = true) + ) + + private val PRINTABLE_ASCII_CODES = 32..128 + + /** + * Main method for raw tag normalization. See[TagNormalizer] doc for more + */ + fun normalizeTag(tag: String): String { + val policy = policyFor(tag) + var normalized = Platform.normalize(tag, policy.normalizerForm) + normalized = if (policy.lowercase) normalized.lowercase() else normalized + normalized = if (policy.removeNonAscii) removeNonAscii(normalized) else normalized + return normalized + } + + /** + * Main method for normalized tag splitting. See[TagNormalizer] doc for more + */ + fun splitNormalizedTag(normalizedTag: String): Tag { + if (!policyFor(normalizedTag).split) { + return Tag.of(normalizedKey = normalizedTag, normalizedValue = null) + } + val i = normalizedTag.indexOf('=') + val key: String + val value: Any? + if (i > 1) { + if (normalizedTag[i - 1] == ':') { // := + key = normalizedTag.substring(0, i - 1).trim() + val raw = normalizedTag.substring(i + 1).trim() + value = if ("true".equals(raw, ignoreCase = true)) { + true + } else if ("false".equals(raw, ignoreCase = true)) { + false + } else { + raw.toDouble() + } + } else { + key = normalizedTag.substring(0, i).trim() + value = normalizedTag.substring(i + 1).trim() + } + } else { + key = normalizedTag + value = null + } + return Tag(normalizedTag, key, value) + } + + private fun removeNonAscii(text: String) = + text.filter { it.code in PRINTABLE_ASCII_CODES } + + private fun policyFor(tag: String): TagProcessingPolicy { + return PREFIX_TO_POLICY.entries + .firstOrNull { (prefix, _) -> tag.startsWith(prefix, ignoreCase = true) } + ?.value + ?: DEFAULT_POLICY + } +} \ No newline at end of file diff --git a/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/XyzNs.kt b/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/XyzNs.kt index 1a3a9b256..acd8c14b7 100644 --- a/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/XyzNs.kt +++ b/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/XyzNs.kt @@ -3,6 +3,7 @@ package naksha.model import naksha.base.* +import naksha.model.TagNormalizer.normalizeTag import kotlin.DeprecationLevel.WARNING import kotlin.js.JsExport import kotlin.js.JsStatic @@ -29,66 +30,7 @@ class XyzNs : AnyObject() { private val INT64_NULL = NullableProperty(Int64::class) private val TAGS = NullableProperty(TagList::class) - private var AS_IS: CharArray = CharArray(128 - 32) { (it + 32).toChar() } - private var TO_LOWER: CharArray = CharArray(128 - 32) { (it + 32).toChar().lowercaseChar() } - - /** - * A method to normalize a list of tags. - * - * @param tags a list of tags. - * @return the same list, just that the content is normalized. - */ - @JvmStatic - @JsStatic - fun normalizeTags(tags: TagList?): TagList? { - if (!tags.isNullOrEmpty()) { - for ((idx, tag) in tags.withIndex()) { - if (tag != null) { - tags[idx] = normalizeTag(tag) - } - } - } - return tags - } - /** - * A method to normalize and lower case a tag. - * - * @param tag the tag. - * @return the normalized and lower cased version of it. - */ - @JvmStatic - @JsStatic - fun normalizeTag(tag: String): String { - if (tag.isEmpty()) { - return tag - } - val first = tag[0] - // All tags starting with an at-sign, will not be modified in any way. - if (first == '@') { - return tag - } - - // Normalize the tag. - val normalized: String = Platform.normalize(tag, NormalizerForm.NFD) - - // All tags starting with a tilde, sharp, or the deprecated "ref_" / "sourceID_" prefix will not - // be lower cased. - val MAP: CharArray = - if (first == '~' || first == '#' || normalized.startsWith("ref_") || normalized.startsWith("sourceID_")) - AS_IS - else - TO_LOWER - val sb = StringBuilder(normalized.length) - for (element in normalized) { - // Note: This saves one branch, and the array-size check, because 0 - 32 will become 65504. - val c = (element.code - 32).toChar() - if (c.code < MAP.size) { - sb.append(MAP[c.code]) - } - } - return sb.toString() - } } /** diff --git a/here-naksha-lib-psql/src/commonMain/kotlin/naksha/psql/executors/query/WhereClauseBuilder.kt b/here-naksha-lib-psql/src/commonMain/kotlin/naksha/psql/executors/query/WhereClauseBuilder.kt index 760bf133b..55a346d1a 100644 --- a/here-naksha-lib-psql/src/commonMain/kotlin/naksha/psql/executors/query/WhereClauseBuilder.kt +++ b/here-naksha-lib-psql/src/commonMain/kotlin/naksha/psql/executors/query/WhereClauseBuilder.kt @@ -173,8 +173,8 @@ class WhereClauseBuilder(private val request: ReadFeatures) { } } - private tailrec fun whereNestedTags(tagQuery: ITagQuery){ - when(tagQuery){ + private tailrec fun whereNestedTags(tagQuery: ITagQuery) { + when (tagQuery) { is TagNot -> not(tagQuery.query, this::whereNestedTags) is TagOr -> or(tagQuery.filterNotNull(), this::whereNestedTags) is TagAnd -> and(tagQuery.filterNotNull(), this::whereNestedTags) @@ -182,40 +182,64 @@ class WhereClauseBuilder(private val request: ReadFeatures) { } } - private fun resolveSingleTagQuery(tagQuery: TagQuery){ - when(tagQuery){ + private fun resolveSingleTagQuery(tagQuery: TagQuery) { + when (tagQuery) { is TagExists -> { - where.append("$tagsAsJsonb ? ${tagQuery.name}") + where.append("$tagsAsJsonb ?? '${tagQuery.name}'") } + is TagValueIsNull -> { where.append("${tagValue(tagQuery)} = null") } + is TagValueIsBool -> { - if(tagQuery.value){ - where.append(tagValue(tagQuery)) + if (tagQuery.value) { + where.append(tagValue(tagQuery, PgType.BOOLEAN)) } else { - where.append("not(${tagValue(tagQuery)}})") + where.append("not(${tagValue(tagQuery, PgType.BOOLEAN)}})") } } + is TagValueIsDouble -> { val valuePlaceholder = placeholderForArg(tagQuery.value, PgType.DOUBLE) - val doubleOp = resolveDoubleOp(tagQuery.op, tagValue(tagQuery), valuePlaceholder) + val doubleOp = resolveDoubleOp( + tagQuery.op, + tagValue(tagQuery, PgType.DOUBLE), + valuePlaceholder + ) where.append(doubleOp) } + is TagValueIsString -> { val valuePlaceholder = placeholderForArg(tagQuery.value, PgType.STRING) - val stringEquals = resolveStringOp(StringOp.EQUALS, tagValue(tagQuery), valuePlaceholder) - where.append(stringEquals) + val stringEquals = resolveStringOp( + StringOp.EQUALS, + tagValue(tagQuery, PgType.STRING), + valuePlaceholder + ) + where.append(stringEquals) // naksha_tags(tags, flags)::jsonb->>foo = $1 } + is TagValueMatches -> { - val regexPlaceholder = placeholderForArg(tagQuery.regex, PgType.STRING) - where.append("$tagsAsJsonb @? '$[?(@.${tagQuery.name}=~/$regexPlaceholder/)]") + /* + SELECT * + FROM read_by_tags_test + WHERE naksha_tags(tags, flags) @? '$.year ? (@ like_regex "^202\\d$")' + */ +// val regex = Regex.escape(tagQuery.regex) + val regex = tagQuery.regex + where.append("$tagsAsJsonb @?? '\$.${tagQuery.name} ? (@ like_regex \"${regex}\")'") } } } - private fun tagValue(tagQuery: TagQuery): String = - "$tagsAsJsonb->>${tagQuery.name}" + private fun tagValue(tagQuery: TagQuery, castTo: PgType? = null): String { + return when (castTo) { + null -> "$tagsAsJsonb->'${tagQuery.name}'" + PgType.STRING -> "$tagsAsJsonb->>'${tagQuery.name}'" + else -> "($tagsAsJsonb->'${tagQuery.name}')::${castTo.value}" + } + } private fun not(subClause: T, subClauseResolver: (T) -> Unit) { where.append(" NOT (") @@ -282,8 +306,8 @@ class WhereClauseBuilder(private val request: ReadFeatures) { ) } } - + companion object { - private val tagsAsJsonb = "naksha_tags(${PgColumn.tags}, ${PgColumn.flags})::jsonb" + private val tagsAsJsonb = "naksha_tags(${PgColumn.tags}, ${PgColumn.flags})" } } diff --git a/here-naksha-lib-psql/src/commonTest/kotlin/naksha/psql/ReadFeaturesByTagsTest.kt b/here-naksha-lib-psql/src/commonTest/kotlin/naksha/psql/ReadFeaturesByTagsTest.kt index d927db3cd..3c72c10e8 100644 --- a/here-naksha-lib-psql/src/commonTest/kotlin/naksha/psql/ReadFeaturesByTagsTest.kt +++ b/here-naksha-lib-psql/src/commonTest/kotlin/naksha/psql/ReadFeaturesByTagsTest.kt @@ -5,27 +5,26 @@ import naksha.model.objects.NakshaCollection import naksha.model.objects.NakshaFeature import naksha.model.request.ReadFeatures import naksha.model.request.SuccessResponse -import naksha.model.request.query.TagExists -import naksha.model.request.query.TagQuery -import naksha.model.request.query.TagValueIsString +import naksha.model.request.query.* import naksha.psql.base.PgTestBase import naksha.psql.util.ProxyFeatureGenerator import kotlin.test.Test import kotlin.test.assertEquals +import kotlin.test.assertTrue class ReadFeaturesByTagsTest : PgTestBase(NakshaCollection("read_by_tags_test")) { @Test fun shouldReturnFeaturesWithExistingTag() { // Given: - val inputFeature = randomFeatureWithTags("foo=bar") + val inputFeature = randomFeatureWithTags("sample") // When: insertFeature(feature = inputFeature) // And: val featuresWithFooTag = executeTagsQuery( - TagExists("foo") + TagExists("sample") ).features // Then: @@ -36,14 +35,14 @@ class ReadFeaturesByTagsTest : PgTestBase(NakshaCollection("read_by_tags_test")) @Test fun shouldNotReturnFeaturesWithMissingTag() { // Given: - val inputFeature = randomFeatureWithTags("foo") + val inputFeature = randomFeatureWithTags("existing") // When: insertFeature(feature = inputFeature) // And: val featuresWithFooTag = executeTagsQuery( - TagExists("bar") + TagExists("non-existing") ).features // Then: @@ -51,7 +50,7 @@ class ReadFeaturesByTagsTest : PgTestBase(NakshaCollection("read_by_tags_test")) } @Test - fun shouldReturnFeaturesWithTagValue() { + fun shouldReturnFeaturesWithStringValue() { // Given: val inputFeature = randomFeatureWithTags("foo=bar") @@ -68,13 +67,157 @@ class ReadFeaturesByTagsTest : PgTestBase(NakshaCollection("read_by_tags_test")) assertEquals(inputFeature.id, featuresWithFooTag[0]!!.id) } + @Test + fun shouldReturnFeaturesByTagRegex() { + // Given: + val featureFrom2024 = randomFeatureWithTags("year=2024") + val featureFrom2030 = randomFeatureWithTags("year=2030") + + // When: + insertFeatures(featureFrom2024, featureFrom2030) + + // And: + val featuresFromThisDecade = executeTagsQuery( + TagValueMatches(name = "year", regex = "202[0-9]") + ).features + + // Then: + assertEquals(1, featuresFromThisDecade.size) + assertEquals(featureFrom2024.id, featuresFromThisDecade[0]!!.id) + } + + @Test + fun shouldReturnFeaturesWithDoubleValue() { + // Given: + val inputFeatures = listOf( + randomFeatureWithTags("some_number:=1").apply { id = "one" }, + randomFeatureWithTags("some_number:=5").apply { id = "five" }, + ) + + // When: + insertFeatures(inputFeatures) + + // And: + val featuresGt2 = executeTagsQuery( + TagValueIsDouble("some_number", DoubleOp.GT, 1.0) + ).features + + // Then: + assertEquals(1, featuresGt2.size) + assertEquals("five", featuresGt2[0]!!.id) + + // When + val featuresLte5 = executeTagsQuery( + TagValueIsDouble("some_number", DoubleOp.LTE, 5.0) + ).features + + // Then: + assertEquals(2, featuresLte5.size) + val lte5ids = featuresLte5.map { it!!.id } + assertTrue(lte5ids.containsAll(listOf("one", "five"))) + + // When: + val featuresEq6 = executeTagsQuery( + TagValueIsDouble("some_number", DoubleOp.EQ, 6.0) + ).features + + // Then: + assertTrue(featuresEq6.isEmpty()) + } + + @Test + fun shouldReturnFeaturesForComposedTagQuery() { + // Given: + val activeJohn = randomFeatureWithTags( + "username=john_doe", + "is_active:=true", + ) + val activeNick = randomFeatureWithTags( + "username=nick_foo", + "is_active:=true", + ) + val inactiveJohn = randomFeatureWithTags( + "username=john_bar", + "is_active:=false", + ) + val oldAdmin = randomFeatureWithTags( + "username=some_admin", + "role=admin" + ) + val invalidUserWithoutId = randomFeatureWithTags("is_active:=true") + + // And: + insertFeatures(activeJohn, activeNick, inactiveJohn, oldAdmin, invalidUserWithoutId) + + + // When: + val activeJohnsOrAdmin = TagOr( + TagAnd( + TagValueMatches(name = "username", regex = "john.+"), + TagValueIsBool(name = "is_active", value = true) + ), + TagValueIsString(name = "role", value = "admin") + ) + val features = executeTagsQuery(activeJohnsOrAdmin).features + + // Then: + assertEquals(2, features.size) + val featureIds = features.map { it!!.id } + featureIds.containsAll( + listOf( + activeJohn.id, + oldAdmin.id + ) + ) + } + + @Test + fun shouldTreatRefAsValueless() { + // Given: + val feature = randomFeatureWithTags("ref_lorem=ipsum") + insertFeatures(feature) + + // When + val byTagName = executeTagsQuery(TagExists("ref_lorem")).features + + // Then + assertTrue(byTagName.isEmpty()) + + // When + val byFullTag = executeTagsQuery(TagExists("ref_lorem=ipsum")).features + + // Then + assertEquals(1, byFullTag.size) + assertEquals(feature.id, byFullTag[0]!!.id) + } + + @Test + fun shouldTreatSourceIDAsValueless() { + // Given: + val feature = randomFeatureWithTags("sourceID:=123") + insertFeatures(feature) + + // When + val byTagName = executeTagsQuery(TagExists("sourceID")).features + + // Then + assertTrue(byTagName.isEmpty()) + + // When + val byFullTag = executeTagsQuery(TagExists("sourceID:=123")).features + + // Then + assertEquals(1, byFullTag.size) + assertEquals(feature.id, byFullTag[0]!!.id) + } + private fun randomFeatureWithTags(vararg tags: String): NakshaFeature { return ProxyFeatureGenerator.generateRandomFeature().apply { properties.xyz.tags = TagList(*tags) } } - private fun executeTagsQuery(tagQuery: TagQuery): SuccessResponse { + private fun executeTagsQuery(tagQuery: ITagQuery): SuccessResponse { return executeRead(ReadFeatures().apply { collectionIds += collection!!.id query.tags = tagQuery diff --git a/here-naksha-lib-psql/src/commonTest/kotlin/naksha/psql/TagNormalizerTest.kt b/here-naksha-lib-psql/src/commonTest/kotlin/naksha/psql/TagNormalizerTest.kt new file mode 100644 index 000000000..f28b1437c --- /dev/null +++ b/here-naksha-lib-psql/src/commonTest/kotlin/naksha/psql/TagNormalizerTest.kt @@ -0,0 +1,92 @@ +package naksha.psql + +import naksha.model.TagNormalizer +import kotlin.test.Test +import kotlin.test.assertEquals + +class TagNormalizerTest { + + @Test + fun shouldRemoveNonAscii(){ + val tagsToBeClearedFromAscii = mapOf( + "p®¡©e=100£" to "pe=100", // regular tag + "~twice_¼=single_½" to "~twice_=single_", // starting with '~' + "#some=tµ¶ag" to "#some=tag", // starting with '#' + ) + + tagsToBeClearedFromAscii.forEach { (before, after) -> + assertEquals(after, TagNormalizer.normalizeTag(before)) + } + } + + @Test + fun shouldLeaveNonAsciiAsIs(){ + val tagsWithAsciiToBePreserved = listOf( + "@p®¡©e=100£", // starting with '@' + "ref_p®¡©e=100£", // starting with 'ref_' + "sourceIDp®¡©e=100£", // starting with 'sourceID' + ) + + tagsWithAsciiToBePreserved.forEach { tag -> + assertEquals(tag, TagNormalizer.normalizeTag(tag)) + } + } + + @Test + fun shouldLowercase(){ + val tag = "Some_Tag:=1235" + assertEquals(tag.lowercase(), TagNormalizer.normalizeTag(tag)) + } + + @Test + fun shouldNotLowercase(){ + val tagsNotToBeLowercased = listOf( + "@Some_Tag:=1235", + "ref_Some_Tag:=1235", + "~Some_Tag:=1235", + "#Some_Tag:=1235", + "sourceIDSome_Tag:=1235" + ) + + tagsNotToBeLowercased.forEach { tag -> + assertEquals(tag, TagNormalizer.normalizeTag(tag)) + } + } + + @Test + fun shouldSplit(){ + val tagsToBeSplit = listOf( + "@some_tag:=1235", + "~some_tag:=1235", + "#some_tag:=1235", + "some_tag:=1235" + ) + + tagsToBeSplit.forEach { rawTag -> + val expectedKey = rawTag.split(":")[0] + val normalized = TagNormalizer.normalizeTag(rawTag) + val tag = TagNormalizer.splitNormalizedTag(normalized) + + assertEquals(expectedKey, tag.key) + assertEquals(1235.0, tag.value) + assertEquals(rawTag, tag.tag) + } + } + + @Test + fun shouldNotSplit(){ + val tagsNotToBeSplit = listOf( + "ref_some_tag:=1235", + "sourceIDsome_tag:=1235" + ) + + tagsNotToBeSplit.forEach { rawTag -> + val normalized = TagNormalizer.normalizeTag(rawTag) + val tag = TagNormalizer.splitNormalizedTag(normalized) + + assertEquals(rawTag, tag.key) + assertEquals(null, tag.value) + assertEquals(rawTag, tag.tag) + } + } +} \ No newline at end of file diff --git a/here-naksha-lib-psql/src/commonTest/kotlin/naksha/psql/base/PgTestBase.kt b/here-naksha-lib-psql/src/commonTest/kotlin/naksha/psql/base/PgTestBase.kt index 23fef9fc2..ed7b1e6d1 100644 --- a/here-naksha-lib-psql/src/commonTest/kotlin/naksha/psql/base/PgTestBase.kt +++ b/here-naksha-lib-psql/src/commonTest/kotlin/naksha/psql/base/PgTestBase.kt @@ -34,6 +34,9 @@ abstract class PgTestBase(val collection: NakshaCollection? = null) { protected fun insertFeature(feature: NakshaFeature, sessionOptions: SessionOptions? = null) = insertFeatures(listOf(feature), sessionOptions) + protected fun insertFeatures(vararg features: NakshaFeature) = + insertFeatures(listOf(*features)) + protected fun insertFeatures( features: List, sessionOptions: SessionOptions? = null diff --git a/here-naksha-lib-psql/src/jvmMain/kotlin/naksha/psql/PsqlPlan.kt b/here-naksha-lib-psql/src/jvmMain/kotlin/naksha/psql/PsqlPlan.kt index 5def357be..5f678d545 100644 --- a/here-naksha-lib-psql/src/jvmMain/kotlin/naksha/psql/PsqlPlan.kt +++ b/here-naksha-lib-psql/src/jvmMain/kotlin/naksha/psql/PsqlPlan.kt @@ -21,6 +21,13 @@ class PsqlPlan(internal val query: PsqlQuery, conn: Connection) : PgPlan { stmt.execute() return PsqlCursor(stmt, false) } + /** + * TODO: remove + * + * SELECT gzip(bytea_agg(tuple_number)) AS rs FROM (SELECT tuple_number FROM ( + * (SELECT tuple_number, id FROM read_by_tags_test WHERE (naksha_tags(tags, flags)::jsonb->>foo = '''bar''')) + * ) ORDER BY id, tuple_number) LIMIT 1000000 + */ /** * Adds the prepared statement with the given arguments into batch-execution queue. This requires a mutation query like UPDATE or diff --git a/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/HttpStorage.java b/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/HttpStorage.java index e8ab40084..7b48db5ad 100644 --- a/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/HttpStorage.java +++ b/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/HttpStorage.java @@ -20,15 +20,15 @@ import static com.here.naksha.storage.http.RequestSender.KeyProperties; -import naksha.model.NakshaContext; import com.here.naksha.lib.core.lambdas.Fe1; import com.here.naksha.lib.core.models.naksha.Storage; -import naksha.model.IReadSession; -import naksha.model.IStorage; import com.here.naksha.lib.core.util.json.JsonSerializable; import com.here.naksha.storage.http.cache.RequestSenderCache; import java.util.concurrent.Future; import java.util.concurrent.FutureTask; +import naksha.model.IReadSession; +import naksha.model.IStorage; +import naksha.model.NakshaContext; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.slf4j.Logger; diff --git a/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/HttpStorageProperties.java b/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/HttpStorageProperties.java index 9e58f1d87..cc5d6cda6 100644 --- a/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/HttpStorageProperties.java +++ b/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/HttpStorageProperties.java @@ -20,9 +20,9 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import naksha.model.NakshaVersion; -import naksha.geo.XyzProperties; import java.util.Map; +import naksha.geo.XyzProperties; +import naksha.model.NakshaVersion; import org.jetbrains.annotations.ApiStatus.AvailableSince; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; diff --git a/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/HttpStorageReadExecute.java b/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/HttpStorageReadExecute.java index d200a3a4c..929a0c359 100644 --- a/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/HttpStorageReadExecute.java +++ b/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/HttpStorageReadExecute.java @@ -22,12 +22,7 @@ import static com.here.naksha.storage.http.PrepareResult.prepareResult; import static java.lang.String.format; -import naksha.model.NakshaContext; import com.here.naksha.lib.core.models.XyzError; -import naksha.model.XyzFeature; -import naksha.model.XyzFeatureCollection; -import naksha.model.ErrorResult; -import naksha.model.POp; import com.here.naksha.lib.core.models.storage.ReadFeaturesProxyWrapper; import com.here.naksha.lib.core.models.storage.Result; import java.net.HttpURLConnection; @@ -37,6 +32,11 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import naksha.model.ErrorResult; +import naksha.model.NakshaContext; +import naksha.model.POp; +import naksha.model.XyzFeature; +import naksha.model.XyzFeatureCollection; import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/HttpStorageReadSession.java b/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/HttpStorageReadSession.java index e77a8b943..e87840105 100644 --- a/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/HttpStorageReadSession.java +++ b/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/HttpStorageReadSession.java @@ -18,14 +18,13 @@ */ package com.here.naksha.storage.http; -import naksha.model.NakshaContext; import com.here.naksha.lib.core.models.XyzError; import com.here.naksha.lib.core.models.storage.*; -import naksha.model.IReadSession; import java.util.concurrent.TimeUnit; - -import naksha.model.ReadRequest; import naksha.model.ErrorResult; +import naksha.model.IReadSession; +import naksha.model.NakshaContext; +import naksha.model.ReadRequest; import org.apache.commons.lang3.NotImplementedException; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; diff --git a/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/POpToQueryConverter.java b/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/POpToQueryConverter.java index c31ece3f9..0530899c8 100644 --- a/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/POpToQueryConverter.java +++ b/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/POpToQueryConverter.java @@ -21,14 +21,14 @@ import static com.here.naksha.lib.core.exceptions.UncheckedException.unchecked; import static naksha.model.POpType.*; -import naksha.model.POp; -import naksha.model.POpType; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; +import naksha.model.POp; +import naksha.model.POpType; import org.apache.commons.lang3.ArrayUtils; import org.jetbrains.annotations.NotNull; diff --git a/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/PrepareResult.java b/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/PrepareResult.java index 5083a44fa..3b4a87658 100644 --- a/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/PrepareResult.java +++ b/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/PrepareResult.java @@ -22,7 +22,6 @@ import com.here.naksha.lib.core.models.Typed; import com.here.naksha.lib.core.models.XyzError; -import naksha.model.XyzFeature; import com.here.naksha.lib.core.models.storage.*; import com.here.naksha.lib.core.util.json.JsonSerializable; import java.io.ByteArrayInputStream; @@ -35,8 +34,8 @@ import java.util.List; import java.util.function.Function; import java.util.zip.GZIPInputStream; - import naksha.model.ErrorResult; +import naksha.model.XyzFeature; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; From 0e50eddcfe084435538ed1e8acc79f80992ee2ad Mon Sep 17 00:00:00 2001 From: Jakub Amanowicz Date: Fri, 18 Oct 2024 14:40:25 +0200 Subject: [PATCH 3/8] CASL-561 minor cleanup Signed-off-by: Jakub Amanowicz --- .../src/commonMain/kotlin/naksha/model/TagMap.kt | 16 +++++++++++++--- .../psql/executors/query/WhereClauseBuilder.kt | 8 +------- .../src/jvmMain/kotlin/naksha/psql/PsqlPlan.kt | 7 ------- .../here/naksha/storage/http/HttpStorage.java | 6 +++--- .../storage/http/HttpStorageProperties.java | 4 ++-- .../storage/http/HttpStorageReadExecute.java | 10 +++++----- .../storage/http/HttpStorageReadSession.java | 7 ++++--- .../naksha/storage/http/POpToQueryConverter.java | 4 ++-- .../here/naksha/storage/http/PrepareResult.java | 3 ++- 9 files changed, 32 insertions(+), 33 deletions(-) diff --git a/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/TagMap.kt b/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/TagMap.kt index eed811048..290eca744 100644 --- a/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/TagMap.kt +++ b/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/TagMap.kt @@ -3,12 +3,22 @@ package naksha.model import naksha.base.MapProxy +import naksha.model.objects.NakshaFeature import kotlin.js.JsExport import kotlin.js.JsName -// TODO: Document me! -// Improve me! - +/** + * Map of tags persisted as (key, value) pairs where values are nullable. + * This class represents the persisted form of [TagList] / [Tag]. + * It is stored as byte_array and can be accessed in PG via `naksha_tags` function. + * + * It is advised to only construct it in one of two ways: + * 1) Via [TagList]-based constructor + * 2) By deserializing byte array fetched from DB + * + * If for some reason, one would like to use it otherwise, it is advised to properly prepare tags upfront + * with use of [TagNormalizer] (that is used for example by [TagList]) + */ @JsExport open class TagMap() : MapProxy(String::class, Any::class) { diff --git a/here-naksha-lib-psql/src/commonMain/kotlin/naksha/psql/executors/query/WhereClauseBuilder.kt b/here-naksha-lib-psql/src/commonMain/kotlin/naksha/psql/executors/query/WhereClauseBuilder.kt index 55a346d1a..e4f418a0a 100644 --- a/here-naksha-lib-psql/src/commonMain/kotlin/naksha/psql/executors/query/WhereClauseBuilder.kt +++ b/here-naksha-lib-psql/src/commonMain/kotlin/naksha/psql/executors/query/WhereClauseBuilder.kt @@ -217,16 +217,10 @@ class WhereClauseBuilder(private val request: ReadFeatures) { tagValue(tagQuery, PgType.STRING), valuePlaceholder ) - where.append(stringEquals) // naksha_tags(tags, flags)::jsonb->>foo = $1 + where.append(stringEquals) } is TagValueMatches -> { - /* - SELECT * - FROM read_by_tags_test - WHERE naksha_tags(tags, flags) @? '$.year ? (@ like_regex "^202\\d$")' - */ -// val regex = Regex.escape(tagQuery.regex) val regex = tagQuery.regex where.append("$tagsAsJsonb @?? '\$.${tagQuery.name} ? (@ like_regex \"${regex}\")'") } diff --git a/here-naksha-lib-psql/src/jvmMain/kotlin/naksha/psql/PsqlPlan.kt b/here-naksha-lib-psql/src/jvmMain/kotlin/naksha/psql/PsqlPlan.kt index 5f678d545..5def357be 100644 --- a/here-naksha-lib-psql/src/jvmMain/kotlin/naksha/psql/PsqlPlan.kt +++ b/here-naksha-lib-psql/src/jvmMain/kotlin/naksha/psql/PsqlPlan.kt @@ -21,13 +21,6 @@ class PsqlPlan(internal val query: PsqlQuery, conn: Connection) : PgPlan { stmt.execute() return PsqlCursor(stmt, false) } - /** - * TODO: remove - * - * SELECT gzip(bytea_agg(tuple_number)) AS rs FROM (SELECT tuple_number FROM ( - * (SELECT tuple_number, id FROM read_by_tags_test WHERE (naksha_tags(tags, flags)::jsonb->>foo = '''bar''')) - * ) ORDER BY id, tuple_number) LIMIT 1000000 - */ /** * Adds the prepared statement with the given arguments into batch-execution queue. This requires a mutation query like UPDATE or diff --git a/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/HttpStorage.java b/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/HttpStorage.java index 7b48db5ad..e8ab40084 100644 --- a/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/HttpStorage.java +++ b/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/HttpStorage.java @@ -20,15 +20,15 @@ import static com.here.naksha.storage.http.RequestSender.KeyProperties; +import naksha.model.NakshaContext; import com.here.naksha.lib.core.lambdas.Fe1; import com.here.naksha.lib.core.models.naksha.Storage; +import naksha.model.IReadSession; +import naksha.model.IStorage; import com.here.naksha.lib.core.util.json.JsonSerializable; import com.here.naksha.storage.http.cache.RequestSenderCache; import java.util.concurrent.Future; import java.util.concurrent.FutureTask; -import naksha.model.IReadSession; -import naksha.model.IStorage; -import naksha.model.NakshaContext; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.slf4j.Logger; diff --git a/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/HttpStorageProperties.java b/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/HttpStorageProperties.java index cc5d6cda6..9e58f1d87 100644 --- a/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/HttpStorageProperties.java +++ b/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/HttpStorageProperties.java @@ -20,9 +20,9 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import java.util.Map; -import naksha.geo.XyzProperties; import naksha.model.NakshaVersion; +import naksha.geo.XyzProperties; +import java.util.Map; import org.jetbrains.annotations.ApiStatus.AvailableSince; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; diff --git a/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/HttpStorageReadExecute.java b/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/HttpStorageReadExecute.java index 929a0c359..d200a3a4c 100644 --- a/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/HttpStorageReadExecute.java +++ b/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/HttpStorageReadExecute.java @@ -22,7 +22,12 @@ import static com.here.naksha.storage.http.PrepareResult.prepareResult; import static java.lang.String.format; +import naksha.model.NakshaContext; import com.here.naksha.lib.core.models.XyzError; +import naksha.model.XyzFeature; +import naksha.model.XyzFeatureCollection; +import naksha.model.ErrorResult; +import naksha.model.POp; import com.here.naksha.lib.core.models.storage.ReadFeaturesProxyWrapper; import com.here.naksha.lib.core.models.storage.Result; import java.net.HttpURLConnection; @@ -32,11 +37,6 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; -import naksha.model.ErrorResult; -import naksha.model.NakshaContext; -import naksha.model.POp; -import naksha.model.XyzFeature; -import naksha.model.XyzFeatureCollection; import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/HttpStorageReadSession.java b/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/HttpStorageReadSession.java index e87840105..e77a8b943 100644 --- a/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/HttpStorageReadSession.java +++ b/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/HttpStorageReadSession.java @@ -18,13 +18,14 @@ */ package com.here.naksha.storage.http; +import naksha.model.NakshaContext; import com.here.naksha.lib.core.models.XyzError; import com.here.naksha.lib.core.models.storage.*; -import java.util.concurrent.TimeUnit; -import naksha.model.ErrorResult; import naksha.model.IReadSession; -import naksha.model.NakshaContext; +import java.util.concurrent.TimeUnit; + import naksha.model.ReadRequest; +import naksha.model.ErrorResult; import org.apache.commons.lang3.NotImplementedException; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; diff --git a/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/POpToQueryConverter.java b/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/POpToQueryConverter.java index 0530899c8..c31ece3f9 100644 --- a/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/POpToQueryConverter.java +++ b/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/POpToQueryConverter.java @@ -21,14 +21,14 @@ import static com.here.naksha.lib.core.exceptions.UncheckedException.unchecked; import static naksha.model.POpType.*; +import naksha.model.POp; +import naksha.model.POpType; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; -import naksha.model.POp; -import naksha.model.POpType; import org.apache.commons.lang3.ArrayUtils; import org.jetbrains.annotations.NotNull; diff --git a/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/PrepareResult.java b/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/PrepareResult.java index 3b4a87658..5083a44fa 100644 --- a/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/PrepareResult.java +++ b/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/PrepareResult.java @@ -22,6 +22,7 @@ import com.here.naksha.lib.core.models.Typed; import com.here.naksha.lib.core.models.XyzError; +import naksha.model.XyzFeature; import com.here.naksha.lib.core.models.storage.*; import com.here.naksha.lib.core.util.json.JsonSerializable; import java.io.ByteArrayInputStream; @@ -34,8 +35,8 @@ import java.util.List; import java.util.function.Function; import java.util.zip.GZIPInputStream; + import naksha.model.ErrorResult; -import naksha.model.XyzFeature; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; From 62131b9058d5ea0c7b3332c3e4c78fd71654740e Mon Sep 17 00:00:00 2001 From: Jakub Amanowicz Date: Mon, 21 Oct 2024 05:55:57 +0200 Subject: [PATCH 4/8] CASL-561 review remarks Signed-off-by: Jakub Amanowicz --- .../src/commonMain/kotlin/naksha/model/Tag.kt | 66 ------------------- .../commonMain/kotlin/naksha/model/TagMap.kt | 26 ++++++-- .../kotlin/naksha/model/TagNormalizer.kt | 8 +-- .../executors/query/WhereClauseBuilder.kt | 9 ++- .../kotlin/naksha/psql/TagNormalizerTest.kt | 14 ++-- 5 files changed, 37 insertions(+), 86 deletions(-) delete mode 100644 here-naksha-lib-model/src/commonMain/kotlin/naksha/model/Tag.kt diff --git a/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/Tag.kt b/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/Tag.kt deleted file mode 100644 index 4174e7121..000000000 --- a/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/Tag.kt +++ /dev/null @@ -1,66 +0,0 @@ -@file:Suppress("OPT_IN_USAGE") - -package naksha.model - -import naksha.base.* -import naksha.base.NormalizerForm.NFD -import naksha.base.NormalizerForm.NFKC -import naksha.model.NakshaError.NakshaErrorCompanion.ILLEGAL_ARGUMENT -import kotlin.js.JsExport -import kotlin.js.JsName -import kotlin.js.JsStatic -import kotlin.jvm.JvmStatic - -/** - * An immutable mapper for a split tag. - * @property tag the full tag. - * @property key the key of the tag. - * @property value the value of the tag; _null_, Boolean, String or Double. - */ -@JsExport -class Tag() : AnyObject() { - - @JsName("of") - constructor(tag: String, key: String, value: Any?) : this() { - this.tag = tag - this.key = key - this.value = value - } - - companion object Tag_C { - private val TAG = NotNullProperty(String::class) - private val KEY = NotNullProperty(String::class) - private val VALUE = NullableProperty(Any::class) - - @JvmStatic - @JsStatic - fun of(normalizedKey: String, normalizedValue: Any?): Tag = when (normalizedValue) { - null -> Tag(normalizedKey, normalizedKey, null) - is String -> Tag("$normalizedKey=$normalizedValue", normalizedKey, normalizedValue) - is Boolean -> Tag("$normalizedKey:=$normalizedValue", normalizedKey, normalizedValue) - is Number -> { - val doubleValue = normalizedValue.toDouble() - Tag("$normalizedKey:=$doubleValue", normalizedKey, doubleValue) - } - - else -> throw NakshaException( - ILLEGAL_ARGUMENT, - "Tag values can only be String, Boolean or Number" - ) - } - } - - var tag by TAG - var key by KEY - var value by VALUE - - override fun equals(other: Any?): Boolean { - if (this === other) return true - if (other is String) return tag == other - if (other is Tag) return tag == other.tag - return false - } - - override fun hashCode(): Int = tag.hashCode() - override fun toString(): String = tag -} \ No newline at end of file diff --git a/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/TagMap.kt b/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/TagMap.kt index 290eca744..31e387760 100644 --- a/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/TagMap.kt +++ b/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/TagMap.kt @@ -3,13 +3,12 @@ package naksha.model import naksha.base.MapProxy -import naksha.model.objects.NakshaFeature import kotlin.js.JsExport import kotlin.js.JsName /** * Map of tags persisted as (key, value) pairs where values are nullable. - * This class represents the persisted form of [TagList] / [Tag]. + * This class represents the persisted form of [TagList]. * It is stored as byte_array and can be accessed in PG via `naksha_tags` function. * * It is advised to only construct it in one of two ways: @@ -27,7 +26,7 @@ open class TagMap() : MapProxy(String::class, Any::class) { constructor(tagList: TagList) : this() { tagList.filterNotNull() .map { TagNormalizer.splitNormalizedTag(it) } - .forEach { tag -> put(tag.key, tag.value) } + .forEach { (key, value) -> put(key, value) } } /** @@ -37,8 +36,27 @@ open class TagMap() : MapProxy(String::class, Any::class) { fun toTagList(): TagList { val list = TagList() forEach { (key, value) -> - list.add(Tag.of(key, value).tag) + list.add(flattenTag(key, value)) } return list } + + /** + * Converts (key, value) pair to String, so it can be part of [TagList]. + * The result depends on the value: + * - Null value is omitted: ('foo', null) -> 'foo' + * - String value is separated with simple '=': ('foo', 'bar') -> 'foo=bar' + * - Numbers and booleans are separated with ':=' -> 'foo:=true', 'foo:=12.34' + */ + private fun flattenTag(key: String, value: Any?): String = + when (value) { + null -> key + is String -> "$key=$value" + is Boolean -> "$key:=$value" + is Number -> "$key:=${value.toDouble()}" + else -> throw NakshaException( + NakshaError.ILLEGAL_ARGUMENT, + "Tag values can only be String, Boolean or Number" + ) + } } \ No newline at end of file diff --git a/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/TagNormalizer.kt b/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/TagNormalizer.kt index 90e217767..8de992d07 100644 --- a/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/TagNormalizer.kt +++ b/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/TagNormalizer.kt @@ -17,9 +17,7 @@ import naksha.model.TagNormalizer.splitNormalizedTag * * Normalization form used in step #1 and subsequent conditional steps depend on tag prefix. * - * * Process of splitting happens in [splitNormalizedTag] method. - * It is about splitting the normalized tag to [key, value] pair in form of [Tag] * Note that not all tags can be split, it depends on their prefix. * * Summarised per-prefix behavior: @@ -69,9 +67,9 @@ object TagNormalizer { /** * Main method for normalized tag splitting. See[TagNormalizer] doc for more */ - fun splitNormalizedTag(normalizedTag: String): Tag { + fun splitNormalizedTag(normalizedTag: String): Pair { if (!policyFor(normalizedTag).split) { - return Tag.of(normalizedKey = normalizedTag, normalizedValue = null) + return normalizedTag to null } val i = normalizedTag.indexOf('=') val key: String @@ -95,7 +93,7 @@ object TagNormalizer { key = normalizedTag value = null } - return Tag(normalizedTag, key, value) + return key to value } private fun removeNonAscii(text: String) = diff --git a/here-naksha-lib-psql/src/commonMain/kotlin/naksha/psql/executors/query/WhereClauseBuilder.kt b/here-naksha-lib-psql/src/commonMain/kotlin/naksha/psql/executors/query/WhereClauseBuilder.kt index e4f418a0a..33636dd9e 100644 --- a/here-naksha-lib-psql/src/commonMain/kotlin/naksha/psql/executors/query/WhereClauseBuilder.kt +++ b/here-naksha-lib-psql/src/commonMain/kotlin/naksha/psql/executors/query/WhereClauseBuilder.kt @@ -63,7 +63,8 @@ class WhereClauseBuilder(private val request: ReadFeatures) { } private fun whereSpatial() { - request.query.spatial?.let { spatialQuery -> + val spatialQuery = request.query.spatial + if (spatialQuery != null) { if (where.isNotEmpty()) { where.append(" AND (") } else { @@ -109,7 +110,8 @@ class WhereClauseBuilder(private val request: ReadFeatures) { } private fun whereMetadata() { - request.query.metadata?.let { metaQuery -> + val metaQuery = request.query.metadata + if(metaQuery != null){ if (where.isNotEmpty()) { where.append(" AND (") } else { @@ -162,7 +164,8 @@ class WhereClauseBuilder(private val request: ReadFeatures) { } private fun whereTags() { - request.query.tags?.let { tagQuery -> + val tagQuery = request.query.tags + if(tagQuery != null){ if (where.isNotEmpty()) { where.append(" AND (") } else { diff --git a/here-naksha-lib-psql/src/commonTest/kotlin/naksha/psql/TagNormalizerTest.kt b/here-naksha-lib-psql/src/commonTest/kotlin/naksha/psql/TagNormalizerTest.kt index f28b1437c..c575f67a2 100644 --- a/here-naksha-lib-psql/src/commonTest/kotlin/naksha/psql/TagNormalizerTest.kt +++ b/here-naksha-lib-psql/src/commonTest/kotlin/naksha/psql/TagNormalizerTest.kt @@ -65,11 +65,10 @@ class TagNormalizerTest { tagsToBeSplit.forEach { rawTag -> val expectedKey = rawTag.split(":")[0] val normalized = TagNormalizer.normalizeTag(rawTag) - val tag = TagNormalizer.splitNormalizedTag(normalized) + val (tagKey, tagValue) = TagNormalizer.splitNormalizedTag(normalized) - assertEquals(expectedKey, tag.key) - assertEquals(1235.0, tag.value) - assertEquals(rawTag, tag.tag) + assertEquals(expectedKey, tagKey) + assertEquals(1235.0, tagValue) } } @@ -82,11 +81,10 @@ class TagNormalizerTest { tagsNotToBeSplit.forEach { rawTag -> val normalized = TagNormalizer.normalizeTag(rawTag) - val tag = TagNormalizer.splitNormalizedTag(normalized) + val (tagKey, tagValue) = TagNormalizer.splitNormalizedTag(normalized) - assertEquals(rawTag, tag.key) - assertEquals(null, tag.value) - assertEquals(rawTag, tag.tag) + assertEquals(rawTag, tagKey) + assertEquals(null, tagValue) } } } \ No newline at end of file From c971dee4dc227923ed31c0109589bd053c0a6a30 Mon Sep 17 00:00:00 2001 From: Jakub Amanowicz Date: Mon, 21 Oct 2024 17:52:14 +0200 Subject: [PATCH 5/8] CASL-561 further review remarks Signed-off-by: Jakub Amanowicz --- .../commonMain/kotlin/naksha/model/TagList.kt | 2 +- .../commonMain/kotlin/naksha/model/TagMap.kt | 3 +- .../kotlin/naksha/model/TagNormalizer.kt | 131 ++++++++++-------- .../commonMain/kotlin/naksha/model/XyzNs.kt | 2 +- .../kotlin/naksha/model/TagMapTest.kt | 58 ++++++++ .../kotlin/naksha/model}/TagNormalizerTest.kt | 3 +- .../executors/query/WhereClauseBuilder.kt | 6 +- 7 files changed, 142 insertions(+), 63 deletions(-) create mode 100644 here-naksha-lib-model/src/commonTest/kotlin/naksha/model/TagMapTest.kt rename {here-naksha-lib-psql/src/commonTest/kotlin/naksha/psql => here-naksha-lib-model/src/commonTest/kotlin/naksha/model}/TagNormalizerTest.kt (97%) diff --git a/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/TagList.kt b/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/TagList.kt index e53c55143..5a3117751 100644 --- a/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/TagList.kt +++ b/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/TagList.kt @@ -5,7 +5,7 @@ package naksha.model import naksha.base.ListProxy import naksha.base.NormalizerForm import naksha.base.Platform -import naksha.model.TagNormalizer.normalizeTag +import naksha.model.TagNormalizer.TagNormalizer_C.normalizeTag import kotlin.js.JsExport import kotlin.js.JsName import kotlin.js.JsStatic diff --git a/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/TagMap.kt b/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/TagMap.kt index 31e387760..27e2d7e2d 100644 --- a/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/TagMap.kt +++ b/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/TagMap.kt @@ -2,6 +2,7 @@ package naksha.model +import naksha.base.Int64 import naksha.base.MapProxy import kotlin.js.JsExport import kotlin.js.JsName @@ -52,7 +53,7 @@ open class TagMap() : MapProxy(String::class, Any::class) { when (value) { null -> key is String -> "$key=$value" - is Boolean -> "$key:=$value" + is Boolean, is Long, is Int64 -> "$key:=$value" is Number -> "$key:=${value.toDouble()}" else -> throw NakshaException( NakshaError.ILLEGAL_ARGUMENT, diff --git a/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/TagNormalizer.kt b/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/TagNormalizer.kt index 8de992d07..039173923 100644 --- a/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/TagNormalizer.kt +++ b/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/TagNormalizer.kt @@ -4,8 +4,8 @@ import naksha.base.NormalizerForm import naksha.base.NormalizerForm.NFD import naksha.base.NormalizerForm.NFKC import naksha.base.Platform -import naksha.model.TagNormalizer.normalizeTag -import naksha.model.TagNormalizer.splitNormalizedTag +import naksha.model.TagNormalizer.TagNormalizer_C.normalizeTag +import naksha.model.TagNormalizer.TagNormalizer_C.splitNormalizedTag /** * An object used for Tag normalization and splitting. @@ -34,7 +34,7 @@ import naksha.model.TagNormalizer.splitNormalizedTag * * By default, (if no special prefix is found) tag is normalized with NFD, lowercased, cleaned of non-ASCII and splittable. */ -object TagNormalizer { +class TagNormalizer private constructor() { private data class TagProcessingPolicy( val normalizerForm: NormalizerForm, val removeNonAscii: Boolean, @@ -42,67 +42,88 @@ object TagNormalizer { val split: Boolean ) - private val DEFAULT_POLICY = TagProcessingPolicy(NFD, removeNonAscii = true, lowercase = true, split = true) - private val PREFIX_TO_POLICY = mapOf( - "@" to TagProcessingPolicy(NFKC, removeNonAscii = false, lowercase = false, split = true), - "ref_" to TagProcessingPolicy(NFKC, removeNonAscii = false, lowercase = false, split = false), - "sourceID" to TagProcessingPolicy(NFKC, removeNonAscii = false, lowercase = false, split = false), - "~" to TagProcessingPolicy(NFD, removeNonAscii = true, lowercase = false, split = true), - "#" to TagProcessingPolicy(NFD, removeNonAscii = true, lowercase = false, split = true) - ) + companion object TagNormalizer_C { + private val DEFAULT_POLICY = + TagProcessingPolicy(NFD, removeNonAscii = true, lowercase = true, split = true) + private val PREFIX_TO_POLICY = mapOf( + "@" to TagProcessingPolicy( NFKC, removeNonAscii = false, lowercase = false, split = true), + "ref_" to TagProcessingPolicy( NFKC, removeNonAscii = false, lowercase = false, split = false), + "sourceID" to TagProcessingPolicy( NFKC, removeNonAscii = false, lowercase = false, split = false), + "~" to TagProcessingPolicy(NFD, removeNonAscii = true, lowercase = false, split = true), + "#" to TagProcessingPolicy(NFD, removeNonAscii = true, lowercase = false, split = true) + ) - private val PRINTABLE_ASCII_CODES = 32..128 + private val AS_IS: CharArray = CharArray(128 - 32) { (it + 32).toChar() } + private val TO_LOWER: CharArray = CharArray(128 - 32) { (it + 32).toChar().lowercaseChar() } - /** - * Main method for raw tag normalization. See[TagNormalizer] doc for more - */ - fun normalizeTag(tag: String): String { - val policy = policyFor(tag) - var normalized = Platform.normalize(tag, policy.normalizerForm) - normalized = if (policy.lowercase) normalized.lowercase() else normalized - normalized = if (policy.removeNonAscii) removeNonAscii(normalized) else normalized - return normalized - } + /** + * Main method for raw tag normalization. See[TagNormalizer] doc for more + */ + fun normalizeTag(tag: String): String { + val policy = policyFor(tag) + val normalized = Platform.normalize(tag, policy.normalizerForm) + return if (policy.lowercase) { + if (policy.removeNonAscii) { + removeNonAscii(normalized, TO_LOWER) + } else { + normalized.lowercase() + } + } else if (policy.removeNonAscii){ + removeNonAscii(normalized, AS_IS) + } else { + normalized + } + } - /** - * Main method for normalized tag splitting. See[TagNormalizer] doc for more - */ - fun splitNormalizedTag(normalizedTag: String): Pair { - if (!policyFor(normalizedTag).split) { - return normalizedTag to null + private fun removeNonAscii(input: String, outputCharacterSet: CharArray): String { + val sb = StringBuilder() + for (element in input) { + val c = (element.code - 32).toChar() + if (c.code < outputCharacterSet.size) { + sb.append(outputCharacterSet[c.code]) + } + } + return sb.toString() } - val i = normalizedTag.indexOf('=') - val key: String - val value: Any? - if (i > 1) { - if (normalizedTag[i - 1] == ':') { // := - key = normalizedTag.substring(0, i - 1).trim() - val raw = normalizedTag.substring(i + 1).trim() - value = if ("true".equals(raw, ignoreCase = true)) { - true - } else if ("false".equals(raw, ignoreCase = true)) { - false + + + /** + * Main method for normalized tag splitting. See[TagNormalizer] doc for more + */ + fun splitNormalizedTag(normalizedTag: String): Pair { + if (!policyFor(normalizedTag).split) { + return normalizedTag to null + } + val i = normalizedTag.indexOf('=') + val key: String + val value: Any? + if (i > 1) { + if (normalizedTag[i - 1] == ':') { // := + key = normalizedTag.substring(0, i - 1).trim() + val raw = normalizedTag.substring(i + 1).trim() + value = if ("true".equals(raw, ignoreCase = true)) { + true + } else if ("false".equals(raw, ignoreCase = true)) { + false + } else { + raw.toDouble() + } } else { - raw.toDouble() + key = normalizedTag.substring(0, i).trim() + value = normalizedTag.substring(i + 1).trim() } } else { - key = normalizedTag.substring(0, i).trim() - value = normalizedTag.substring(i + 1).trim() + key = normalizedTag + value = null } - } else { - key = normalizedTag - value = null + return key to value } - return key to value - } - - private fun removeNonAscii(text: String) = - text.filter { it.code in PRINTABLE_ASCII_CODES } - private fun policyFor(tag: String): TagProcessingPolicy { - return PREFIX_TO_POLICY.entries - .firstOrNull { (prefix, _) -> tag.startsWith(prefix, ignoreCase = true) } - ?.value - ?: DEFAULT_POLICY + private fun policyFor(tag: String): TagProcessingPolicy { + for ((prefix, policy) in PREFIX_TO_POLICY) { + if (tag.startsWith(prefix)) return policy + } + return DEFAULT_POLICY + } } } \ No newline at end of file diff --git a/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/XyzNs.kt b/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/XyzNs.kt index acd8c14b7..6b8a28591 100644 --- a/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/XyzNs.kt +++ b/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/XyzNs.kt @@ -3,7 +3,7 @@ package naksha.model import naksha.base.* -import naksha.model.TagNormalizer.normalizeTag +import naksha.model.TagNormalizer.TagNormalizer_C.normalizeTag import kotlin.DeprecationLevel.WARNING import kotlin.js.JsExport import kotlin.js.JsStatic diff --git a/here-naksha-lib-model/src/commonTest/kotlin/naksha/model/TagMapTest.kt b/here-naksha-lib-model/src/commonTest/kotlin/naksha/model/TagMapTest.kt new file mode 100644 index 000000000..1085a0101 --- /dev/null +++ b/here-naksha-lib-model/src/commonTest/kotlin/naksha/model/TagMapTest.kt @@ -0,0 +1,58 @@ +package naksha.model + +import kotlin.test.* + +class TagMapTest { + + @Test + fun shouldBeConvertedFromTagList(){ + // Given: + val tagList = TagList("foo=bar", "no-value", "flag:=true") + + // When: + val tagMap = TagMap(tagList) + + // Then + assertEquals(3, tagList.size) + assertEquals("bar", tagMap["foo"]) + assertTrue(tagMap.contains("no-value")) + assertNull(tagMap["no-value"]) + assertEquals(true, tagMap["flag"]) + } + + @Test + fun shouldBeConvertedToTagList(){ + // Given: + val tagMap = TagMap().apply { + put("foo", "bar") + put("no-value", null) + put("flag", true) + } + + // When: + val tagList = tagMap.toTagList() + + // Then + assertTrue(tagList.containsAll(listOf("foo=bar", "no-value", "flag:=true"))) + } + + @Test + fun shouldFailWhenConvertingToListWithUnsupportedType(){ + // Given: + val tagMap = TagMap().apply { + put("foo", "bar") + put("failure-reason", NotSupportedType) + } + + // When: + val failure = assertFails { + tagMap.toTagList() + } + + // Then: + assertIs(failure) + assertEquals("Tag values can only be String, Boolean or Number", failure.message) + } + + object NotSupportedType +} \ No newline at end of file diff --git a/here-naksha-lib-psql/src/commonTest/kotlin/naksha/psql/TagNormalizerTest.kt b/here-naksha-lib-model/src/commonTest/kotlin/naksha/model/TagNormalizerTest.kt similarity index 97% rename from here-naksha-lib-psql/src/commonTest/kotlin/naksha/psql/TagNormalizerTest.kt rename to here-naksha-lib-model/src/commonTest/kotlin/naksha/model/TagNormalizerTest.kt index c575f67a2..f9e543484 100644 --- a/here-naksha-lib-psql/src/commonTest/kotlin/naksha/psql/TagNormalizerTest.kt +++ b/here-naksha-lib-model/src/commonTest/kotlin/naksha/model/TagNormalizerTest.kt @@ -1,6 +1,5 @@ -package naksha.psql +package naksha.model -import naksha.model.TagNormalizer import kotlin.test.Test import kotlin.test.assertEquals diff --git a/here-naksha-lib-psql/src/commonMain/kotlin/naksha/psql/executors/query/WhereClauseBuilder.kt b/here-naksha-lib-psql/src/commonMain/kotlin/naksha/psql/executors/query/WhereClauseBuilder.kt index 33636dd9e..9a6efdd05 100644 --- a/here-naksha-lib-psql/src/commonMain/kotlin/naksha/psql/executors/query/WhereClauseBuilder.kt +++ b/here-naksha-lib-psql/src/commonMain/kotlin/naksha/psql/executors/query/WhereClauseBuilder.kt @@ -75,7 +75,7 @@ class WhereClauseBuilder(private val request: ReadFeatures) { } } - private tailrec fun whereNestedSpatial(spatial: ISpatialQuery) { + private fun whereNestedSpatial(spatial: ISpatialQuery) { when (spatial) { is SpNot -> not( subClause = spatial.query, @@ -122,7 +122,7 @@ class WhereClauseBuilder(private val request: ReadFeatures) { } } - private tailrec fun whereNestedMetadata(metaQuery: IMetaQuery) { + private fun whereNestedMetadata(metaQuery: IMetaQuery) { when (metaQuery) { is MetaNot -> not( subClause = metaQuery.query, @@ -176,7 +176,7 @@ class WhereClauseBuilder(private val request: ReadFeatures) { } } - private tailrec fun whereNestedTags(tagQuery: ITagQuery) { + private fun whereNestedTags(tagQuery: ITagQuery) { when (tagQuery) { is TagNot -> not(tagQuery.query, this::whereNestedTags) is TagOr -> or(tagQuery.filterNotNull(), this::whereNestedTags) From 516c8e5caed387669b4f89ce6eda6bfbbd4e1a4a Mon Sep 17 00:00:00 2001 From: Jakub Amanowicz Date: Fri, 25 Oct 2024 14:42:02 +0200 Subject: [PATCH 6/8] CASL-561 placeholders instead of inlined query values Signed-off-by: Jakub Amanowicz --- .../kotlin/naksha/model/TagNormalizer.kt | 2 +- .../kotlin/naksha/model/TagNormalizerTest.kt | 30 +++++++++++++---- .../executors/query/WhereClauseBuilder.kt | 32 ++++++++++--------- .../naksha/psql/ReadFeaturesByTagsTest.kt | 22 ++++++++++++- .../jvmMain/kotlin/naksha/psql/PsqlQuery.kt | 1 + 5 files changed, 63 insertions(+), 24 deletions(-) diff --git a/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/TagNormalizer.kt b/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/TagNormalizer.kt index 039173923..2631b1413 100644 --- a/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/TagNormalizer.kt +++ b/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/TagNormalizer.kt @@ -97,7 +97,7 @@ class TagNormalizer private constructor() { val i = normalizedTag.indexOf('=') val key: String val value: Any? - if (i > 1) { + if (i >= 1) { if (normalizedTag[i - 1] == ':') { // := key = normalizedTag.substring(0, i - 1).trim() val raw = normalizedTag.substring(i + 1).trim() diff --git a/here-naksha-lib-model/src/commonTest/kotlin/naksha/model/TagNormalizerTest.kt b/here-naksha-lib-model/src/commonTest/kotlin/naksha/model/TagNormalizerTest.kt index f9e543484..503d7524b 100644 --- a/here-naksha-lib-model/src/commonTest/kotlin/naksha/model/TagNormalizerTest.kt +++ b/here-naksha-lib-model/src/commonTest/kotlin/naksha/model/TagNormalizerTest.kt @@ -6,7 +6,7 @@ import kotlin.test.assertEquals class TagNormalizerTest { @Test - fun shouldRemoveNonAscii(){ + fun shouldRemoveNonAscii() { val tagsToBeClearedFromAscii = mapOf( "p®¡©e=100£" to "pe=100", // regular tag "~twice_¼=single_½" to "~twice_=single_", // starting with '~' @@ -19,7 +19,7 @@ class TagNormalizerTest { } @Test - fun shouldLeaveNonAsciiAsIs(){ + fun shouldLeaveNonAsciiAsIs() { val tagsWithAsciiToBePreserved = listOf( "@p®¡©e=100£", // starting with '@' "ref_p®¡©e=100£", // starting with 'ref_' @@ -32,13 +32,13 @@ class TagNormalizerTest { } @Test - fun shouldLowercase(){ + fun shouldLowercase() { val tag = "Some_Tag:=1235" assertEquals(tag.lowercase(), TagNormalizer.normalizeTag(tag)) } @Test - fun shouldNotLowercase(){ + fun shouldNotLowercase() { val tagsNotToBeLowercased = listOf( "@Some_Tag:=1235", "ref_Some_Tag:=1235", @@ -53,7 +53,7 @@ class TagNormalizerTest { } @Test - fun shouldSplit(){ + fun shouldSplit() { val tagsToBeSplit = listOf( "@some_tag:=1235", "~some_tag:=1235", @@ -72,7 +72,7 @@ class TagNormalizerTest { } @Test - fun shouldNotSplit(){ + fun shouldNotSplit() { val tagsNotToBeSplit = listOf( "ref_some_tag:=1235", "sourceIDsome_tag:=1235" @@ -86,4 +86,20 @@ class TagNormalizerTest { assertEquals(null, tagValue) } } -} \ No newline at end of file + + @Test + fun shouldSplitSingleCharTags() { + // Given + val tagToBeSplit = "a=b" + + // When: + val normalized = TagNormalizer.normalizeTag(tagToBeSplit) + + // And: + val (key, value) = TagNormalizer.splitNormalizedTag(normalized) + + // Then + assertEquals("a", key) + assertEquals("b", value) + } +} diff --git a/here-naksha-lib-psql/src/commonMain/kotlin/naksha/psql/executors/query/WhereClauseBuilder.kt b/here-naksha-lib-psql/src/commonMain/kotlin/naksha/psql/executors/query/WhereClauseBuilder.kt index 9a6efdd05..096809ae9 100644 --- a/here-naksha-lib-psql/src/commonMain/kotlin/naksha/psql/executors/query/WhereClauseBuilder.kt +++ b/here-naksha-lib-psql/src/commonMain/kotlin/naksha/psql/executors/query/WhereClauseBuilder.kt @@ -111,7 +111,7 @@ class WhereClauseBuilder(private val request: ReadFeatures) { private fun whereMetadata() { val metaQuery = request.query.metadata - if(metaQuery != null){ + if (metaQuery != null) { if (where.isNotEmpty()) { where.append(" AND (") } else { @@ -165,7 +165,7 @@ class WhereClauseBuilder(private val request: ReadFeatures) { private fun whereTags() { val tagQuery = request.query.tags - if(tagQuery != null){ + if (tagQuery != null) { if (where.isNotEmpty()) { where.append(" AND (") } else { @@ -188,49 +188,51 @@ class WhereClauseBuilder(private val request: ReadFeatures) { private fun resolveSingleTagQuery(tagQuery: TagQuery) { when (tagQuery) { is TagExists -> { - where.append("$tagsAsJsonb ?? '${tagQuery.name}'") + val tagNamePlaceholder = placeholderForArg(tagQuery.name, PgType.STRING) + where.append("$tagsAsJsonb ?? $tagNamePlaceholder") } is TagValueIsNull -> { - where.append("${tagValue(tagQuery)} = null") + val tagValuePlaceholder = placeholderForArg(selectTagValue(tagQuery), PgType.STRING) + where.append("$tagValuePlaceholder IS NULL") } is TagValueIsBool -> { if (tagQuery.value) { - where.append(tagValue(tagQuery, PgType.BOOLEAN)) + where.append(selectTagValue(tagQuery, PgType.BOOLEAN)) } else { - where.append("not(${tagValue(tagQuery, PgType.BOOLEAN)}})") + where.append("not(${selectTagValue(tagQuery, PgType.BOOLEAN)})") } } is TagValueIsDouble -> { - val valuePlaceholder = placeholderForArg(tagQuery.value, PgType.DOUBLE) + val queryValuePlaceholder = placeholderForArg(tagQuery.value, PgType.DOUBLE) val doubleOp = resolveDoubleOp( tagQuery.op, - tagValue(tagQuery, PgType.DOUBLE), - valuePlaceholder + selectTagValue(tagQuery, PgType.DOUBLE), + queryValuePlaceholder ) where.append(doubleOp) } is TagValueIsString -> { - val valuePlaceholder = placeholderForArg(tagQuery.value, PgType.STRING) + val queryValuePlaceholder = placeholderForArg(tagQuery.value, PgType.STRING) val stringEquals = resolveStringOp( StringOp.EQUALS, - tagValue(tagQuery, PgType.STRING), - valuePlaceholder + selectTagValue(tagQuery, PgType.STRING), + queryValuePlaceholder ) where.append(stringEquals) } is TagValueMatches -> { - val regex = tagQuery.regex - where.append("$tagsAsJsonb @?? '\$.${tagQuery.name} ? (@ like_regex \"${regex}\")'") + val jsonPathPlaceholder = placeholderForArg("\$.${tagQuery.name} ? (@ like_regex \"${tagQuery.regex}\")", PgType.STRING) + where.append("$tagsAsJsonb @?? $jsonPathPlaceholder::jsonpath") } } } - private fun tagValue(tagQuery: TagQuery, castTo: PgType? = null): String { + private fun selectTagValue(tagQuery: TagQuery, castTo: PgType? = null): String { return when (castTo) { null -> "$tagsAsJsonb->'${tagQuery.name}'" PgType.STRING -> "$tagsAsJsonb->>'${tagQuery.name}'" diff --git a/here-naksha-lib-psql/src/commonTest/kotlin/naksha/psql/ReadFeaturesByTagsTest.kt b/here-naksha-lib-psql/src/commonTest/kotlin/naksha/psql/ReadFeaturesByTagsTest.kt index 3c72c10e8..4db871732 100644 --- a/here-naksha-lib-psql/src/commonTest/kotlin/naksha/psql/ReadFeaturesByTagsTest.kt +++ b/here-naksha-lib-psql/src/commonTest/kotlin/naksha/psql/ReadFeaturesByTagsTest.kt @@ -67,6 +67,27 @@ class ReadFeaturesByTagsTest : PgTestBase(NakshaCollection("read_by_tags_test")) assertEquals(inputFeature.id, featuresWithFooTag[0]!!.id) } + @Test + fun shouldReturnFeaturesByBoolean(){ + // Given: + val enabledFeatureA = randomFeatureWithTags("flag:=true") + val enabledFeatureB = randomFeatureWithTags("flag:=true") + val disabledFeature = randomFeatureWithTags("flag:=false") + + // When: + insertFeatures(enabledFeatureA, enabledFeatureB, disabledFeature) + + // And: + val enabledFeatures = executeTagsQuery( + TagValueIsBool(name = "flag", value = true) + ).features + + // Then: + assertEquals(2, enabledFeatures.size) + val fetchedIds = enabledFeatures.map { it!!.id } + assertTrue(fetchedIds.containsAll(listOf(enabledFeatureA.id, enabledFeatureB.id))) + } + @Test fun shouldReturnFeaturesByTagRegex() { // Given: @@ -149,7 +170,6 @@ class ReadFeaturesByTagsTest : PgTestBase(NakshaCollection("read_by_tags_test")) // And: insertFeatures(activeJohn, activeNick, inactiveJohn, oldAdmin, invalidUserWithoutId) - // When: val activeJohnsOrAdmin = TagOr( TagAnd( diff --git a/here-naksha-lib-psql/src/jvmMain/kotlin/naksha/psql/PsqlQuery.kt b/here-naksha-lib-psql/src/jvmMain/kotlin/naksha/psql/PsqlQuery.kt index fa3737cb8..0d07b0aad 100644 --- a/here-naksha-lib-psql/src/jvmMain/kotlin/naksha/psql/PsqlQuery.kt +++ b/here-naksha-lib-psql/src/jvmMain/kotlin/naksha/psql/PsqlQuery.kt @@ -3,6 +3,7 @@ package naksha.psql import naksha.base.Int64 import java.sql.Connection import java.sql.PreparedStatement +import java.sql.SQLType import java.util.ArrayList import java.util.HashMap From 2d92fc5f91a9b2bf98b4b6af32ce4bac3c738b58 Mon Sep 17 00:00:00 2001 From: Jakub Amanowicz Date: Fri, 25 Oct 2024 14:49:08 +0200 Subject: [PATCH 7/8] CASL-561 wrapping tags selection in placeholder Signed-off-by: Jakub Amanowicz --- .../naksha/psql/executors/query/WhereClauseBuilder.kt | 7 ++++--- .../src/jvmMain/kotlin/naksha/psql/PsqlQuery.kt | 1 - 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/here-naksha-lib-psql/src/commonMain/kotlin/naksha/psql/executors/query/WhereClauseBuilder.kt b/here-naksha-lib-psql/src/commonMain/kotlin/naksha/psql/executors/query/WhereClauseBuilder.kt index 096809ae9..59e5710ee 100644 --- a/here-naksha-lib-psql/src/commonMain/kotlin/naksha/psql/executors/query/WhereClauseBuilder.kt +++ b/here-naksha-lib-psql/src/commonMain/kotlin/naksha/psql/executors/query/WhereClauseBuilder.kt @@ -233,10 +233,11 @@ class WhereClauseBuilder(private val request: ReadFeatures) { } private fun selectTagValue(tagQuery: TagQuery, castTo: PgType? = null): String { + val tagKeyPlaceholder = placeholderForArg(tagQuery.name, PgType.STRING) return when (castTo) { - null -> "$tagsAsJsonb->'${tagQuery.name}'" - PgType.STRING -> "$tagsAsJsonb->>'${tagQuery.name}'" - else -> "($tagsAsJsonb->'${tagQuery.name}')::${castTo.value}" + null -> "$tagsAsJsonb->$tagKeyPlaceholder" + PgType.STRING -> "$tagsAsJsonb->>$tagKeyPlaceholder" + else -> "($tagsAsJsonb->$tagKeyPlaceholder)::${castTo.value}" } } diff --git a/here-naksha-lib-psql/src/jvmMain/kotlin/naksha/psql/PsqlQuery.kt b/here-naksha-lib-psql/src/jvmMain/kotlin/naksha/psql/PsqlQuery.kt index 0d07b0aad..fa3737cb8 100644 --- a/here-naksha-lib-psql/src/jvmMain/kotlin/naksha/psql/PsqlQuery.kt +++ b/here-naksha-lib-psql/src/jvmMain/kotlin/naksha/psql/PsqlQuery.kt @@ -3,7 +3,6 @@ package naksha.psql import naksha.base.Int64 import java.sql.Connection import java.sql.PreparedStatement -import java.sql.SQLType import java.util.ArrayList import java.util.HashMap From 53f53c4198ce16768b8c2e56e6ad221fbb5a0bfd Mon Sep 17 00:00:00 2001 From: Jakub Amanowicz Date: Fri, 25 Oct 2024 16:43:01 +0200 Subject: [PATCH 8/8] CASL-561 splitting internal & JsExport annotation on TagNormalizer Signed-off-by: Jakub Amanowicz --- .../kotlin/naksha/model/TagNormalizer.kt | 28 ++++++++++--------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/TagNormalizer.kt b/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/TagNormalizer.kt index 2631b1413..b8ea76022 100644 --- a/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/TagNormalizer.kt +++ b/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/TagNormalizer.kt @@ -6,6 +6,7 @@ import naksha.base.NormalizerForm.NFKC import naksha.base.Platform import naksha.model.TagNormalizer.TagNormalizer_C.normalizeTag import naksha.model.TagNormalizer.TagNormalizer_C.splitNormalizedTag +import kotlin.js.JsExport /** * An object used for Tag normalization and splitting. @@ -34,21 +35,15 @@ import naksha.model.TagNormalizer.TagNormalizer_C.splitNormalizedTag * * By default, (if no special prefix is found) tag is normalized with NFD, lowercased, cleaned of non-ASCII and splittable. */ +@JsExport class TagNormalizer private constructor() { - private data class TagProcessingPolicy( - val normalizerForm: NormalizerForm, - val removeNonAscii: Boolean, - val lowercase: Boolean, - val split: Boolean - ) companion object TagNormalizer_C { - private val DEFAULT_POLICY = - TagProcessingPolicy(NFD, removeNonAscii = true, lowercase = true, split = true) + private val DEFAULT_POLICY = TagProcessingPolicy(NFD, removeNonAscii = true, lowercase = true, split = true) private val PREFIX_TO_POLICY = mapOf( - "@" to TagProcessingPolicy( NFKC, removeNonAscii = false, lowercase = false, split = true), - "ref_" to TagProcessingPolicy( NFKC, removeNonAscii = false, lowercase = false, split = false), - "sourceID" to TagProcessingPolicy( NFKC, removeNonAscii = false, lowercase = false, split = false), + "@" to TagProcessingPolicy(NFKC, removeNonAscii = false, lowercase = false, split = true), + "ref_" to TagProcessingPolicy(NFKC, removeNonAscii = false, lowercase = false, split = false), + "sourceID" to TagProcessingPolicy(NFKC, removeNonAscii = false, lowercase = false, split = false), "~" to TagProcessingPolicy(NFD, removeNonAscii = true, lowercase = false, split = true), "#" to TagProcessingPolicy(NFD, removeNonAscii = true, lowercase = false, split = true) ) @@ -90,7 +85,7 @@ class TagNormalizer private constructor() { /** * Main method for normalized tag splitting. See[TagNormalizer] doc for more */ - fun splitNormalizedTag(normalizedTag: String): Pair { + internal fun splitNormalizedTag(normalizedTag: String): Pair { if (!policyFor(normalizedTag).split) { return normalizedTag to null } @@ -126,4 +121,11 @@ class TagNormalizer private constructor() { return DEFAULT_POLICY } } -} \ No newline at end of file +} + +private data class TagProcessingPolicy( + val normalizerForm: NormalizerForm, + val removeNonAscii: Boolean, + val lowercase: Boolean, + val split: Boolean +)