From 007686a44eb43ec1d3cff16804f4492a4ca70fa7 Mon Sep 17 00:00:00 2001 From: "angelo.andreussi" Date: Thu, 29 Feb 2024 15:19:10 +0100 Subject: [PATCH 1/4] fix: added validation of indexes in method "DatastoreUtils::convertToDataIndexes" returned list of indexes contains only indexes with a valid format --- .../internal/mediator/DatastoreUtils.java | 36 ++++++++++++++++--- .../datastore/steps/DatastoreSteps.java | 2 +- .../DatastoreUtilsIndexCalculatorTest.java | 4 +-- 3 files changed, 34 insertions(+), 8 deletions(-) diff --git a/service/datastore/internal/src/main/java/org/eclipse/kapua/service/datastore/internal/mediator/DatastoreUtils.java b/service/datastore/internal/src/main/java/org/eclipse/kapua/service/datastore/internal/mediator/DatastoreUtils.java index f029eff9b7b..2a50e4be207 100644 --- a/service/datastore/internal/src/main/java/org/eclipse/kapua/service/datastore/internal/mediator/DatastoreUtils.java +++ b/service/datastore/internal/src/main/java/org/eclipse/kapua/service/datastore/internal/mediator/DatastoreUtils.java @@ -32,6 +32,7 @@ import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatterBuilder; +import java.time.format.DateTimeParseException; import java.time.format.ResolverStyle; import java.time.temporal.ChronoField; import java.time.temporal.ChronoUnit; @@ -343,23 +344,34 @@ public String normalizedIndexName(String index) { * Return the list of the data indexes between start and windowEnd instant by scope id. * Only the indexes that will be *FULLY* included in the list (i.e. with a starting date ON OR AFTER the window start AND * the end date ON OR BEFORE the window end will be returned + * Only indexes matching Kapua data index name pattern will be inserted inside the result list, namely this format: + * + * [-]-data-message-YYYY-ww + * or + * [-]-data-message-YYYY-ww-ee + * or + * [-]-data-message-YYYY-ww-ee-HH * * @param indexes * @param windowStart * @param windowEnd + * @param scopeId * @return The list of the data indexes between start and end * @throws DatastoreException */ - public String[] convertToDataIndexes(@NotNull String[] indexes, Instant windowStart, Instant windowEnd) throws DatastoreException { + public String[] convertToDataIndexes(@NotNull String[] indexes, Instant windowStart, Instant windowEnd, KapuaId scopeId) throws DatastoreException { + boolean skipTemporalValidation = false; if (windowStart == null && windowEnd == null) { - return indexes; + skipTemporalValidation = true; } - List result = new ArrayList<>(); for (String index : indexes) { if (index == null) { continue; } + if (scopeId != null && (!validatePrefixIndex(index, scopeId))) { + continue; + } String strippedIndex = stripPrefixAndAccount(index); int fragments = strippedIndex.split("-").length; DateTimeFormatter formatter; @@ -386,9 +398,18 @@ public String[] convertToDataIndexes(@NotNull String[] indexes, Instant windowSt indexWidth = 1; break; } - + TemporalAccessor temporalAccessor; + try { + temporalAccessor = formatter.parse(strippedIndex); + } catch (DateTimeParseException e) { + //unable to parse...the format is not right so don't add this index into result-set + continue; + } + if (skipTemporalValidation) { //this index passed format validation...proceed to next index + result.add(index); + continue; + } try { - TemporalAccessor temporalAccessor = formatter.parse(strippedIndex); Instant indexStart = LocalDateTime.of( temporalAccessor.get(ChronoField.YEAR), temporalAccessor.get(ChronoField.MONTH_OF_YEAR), @@ -411,6 +432,11 @@ public String[] convertToDataIndexes(@NotNull String[] indexes, Instant windowSt return result.toArray(new String[0]); } + private boolean validatePrefixIndex(@NotNull String index,@NotNull KapuaId scopeId) { + String genericIndexFormat = getDataIndexName(scopeId); + return index.startsWith(genericIndexFormat.substring(0, genericIndexFormat.length() - 1)); + } + private boolean isIndexFullyAfterInstant(@NotNull Instant indexStart, @NotNull Instant indexEnd, @NotNull Instant checkpoint) { return !indexStart.isBefore(checkpoint) && !indexEnd.isBefore(checkpoint); } diff --git a/service/datastore/test-steps/src/main/java/org/eclipse/kapua/service/datastore/steps/DatastoreSteps.java b/service/datastore/test-steps/src/main/java/org/eclipse/kapua/service/datastore/steps/DatastoreSteps.java index fa66f3ba2ae..bb7d646fa52 100755 --- a/service/datastore/test-steps/src/main/java/org/eclipse/kapua/service/datastore/steps/DatastoreSteps.java +++ b/service/datastore/test-steps/src/main/java/org/eclipse/kapua/service/datastore/steps/DatastoreSteps.java @@ -1593,7 +1593,7 @@ public void deleteIndexesBetweenDates(String fromDate, String toDate) throws Exc primeException(); try { String[] indexes = KapuaLocator.getInstance().getComponent(DatastoreUtils.class).convertToDataIndexes(getDataIndexesByAccount(getCurrentScopeId()), KapuaDateUtils.parseDate(fromDate).toInstant(), - KapuaDateUtils.parseDate(toDate).toInstant()); + KapuaDateUtils.parseDate(toDate).toInstant(), null); elasticsearchClient.deleteIndexes(indexes); } catch (Exception ex) { verifyException(ex); diff --git a/service/datastore/test/src/test/java/org/eclipse/kapua/service/datastore/test/junit/DatastoreUtilsIndexCalculatorTest.java b/service/datastore/test/src/test/java/org/eclipse/kapua/service/datastore/test/junit/DatastoreUtilsIndexCalculatorTest.java index 7338e97f441..5c663f2d1e6 100644 --- a/service/datastore/test/src/test/java/org/eclipse/kapua/service/datastore/test/junit/DatastoreUtilsIndexCalculatorTest.java +++ b/service/datastore/test/src/test/java/org/eclipse/kapua/service/datastore/test/junit/DatastoreUtilsIndexCalculatorTest.java @@ -159,7 +159,7 @@ private void performTest(Date startDate, Date endDate, String[] expectedIndexes) calEndDate != null ? calEndDate.get(Calendar.WEEK_OF_YEAR) : "Infinity", calEndDate != null ? calEndDate.get(Calendar.DAY_OF_WEEK) : "Infinity"); - String[] index = datastoreUtils.convertToDataIndexes(getDataIndexesByAccount(KapuaEid.ONE), startDate != null ? startDate.toInstant() : null, endDate != null ? endDate.toInstant() : null); + String[] index = datastoreUtils.convertToDataIndexes(getDataIndexesByAccount(KapuaEid.ONE), startDate != null ? startDate.toInstant() : null, endDate != null ? endDate.toInstant() : null, null); compareResult(expectedIndexes, index); } @@ -181,7 +181,7 @@ private void performNullIndexTest(Date startDate, Date endDate) throws Datastore calEndDate != null ? calEndDate.get(Calendar.WEEK_OF_YEAR) : "Infinity", calEndDate != null ? calEndDate.get(Calendar.DAY_OF_WEEK) : "Infinity"); - String[] index = datastoreUtils.convertToDataIndexes(new String[]{null, null}, startDate != null ? startDate.toInstant() : null, endDate != null ? endDate.toInstant() : null); + String[] index = datastoreUtils.convertToDataIndexes(new String[]{null, null}, startDate != null ? startDate.toInstant() : null, endDate != null ? endDate.toInstant() : null, null); compareResult(null, index); } From 856cf63793fb6c803131ec8cad88353c634c6094 Mon Sep 17 00:00:00 2001 From: "angelo.andreussi" Date: Thu, 29 Feb 2024 16:09:00 +0100 Subject: [PATCH 2/4] test: added for DatastoreUtils::convertToDataIndexes when window is null --- .../DatastoreUtilsIndexCalculatorTest.java | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/service/datastore/test/src/test/java/org/eclipse/kapua/service/datastore/test/junit/DatastoreUtilsIndexCalculatorTest.java b/service/datastore/test/src/test/java/org/eclipse/kapua/service/datastore/test/junit/DatastoreUtilsIndexCalculatorTest.java index 5c663f2d1e6..fbaaf3b4f11 100644 --- a/service/datastore/test/src/test/java/org/eclipse/kapua/service/datastore/test/junit/DatastoreUtilsIndexCalculatorTest.java +++ b/service/datastore/test/src/test/java/org/eclipse/kapua/service/datastore/test/junit/DatastoreUtilsIndexCalculatorTest.java @@ -103,6 +103,8 @@ public void testIndex() throws KapuaException, ParseException { performTest(sdf.parse("17/12/2018 13:12 +0100"), null, buildExpectedResult("1", 52, 2018, 52, 2018, null)); performNullIndexTest(sdf.parse("02/01/2017 13:12 +0100"), sdf.parse("09/01/2017 13:12 +0100")); + + performFormatValidationTest(null, null); } @Test @@ -184,6 +186,32 @@ private void performNullIndexTest(Date startDate, Date endDate) throws Datastore String[] index = datastoreUtils.convertToDataIndexes(new String[]{null, null}, startDate != null ? startDate.toInstant() : null, endDate != null ? endDate.toInstant() : null, null); compareResult(null, index); } +// // YYYY-ww-ee-HH + private void performFormatValidationTest(Date startDate, Date endDate) throws DatastoreException { + String[] indexes = { + "1-data-message-2017-01-02", + "1-data-message-2017-01-03", + "2-data-message-2017-01-02", + "asdasdasdssd", + "1-data-message-2021-01-01-01-01", + "1-data-message-2021-01", + "1-data-message-2021-99", + "1-data-message-21-01", + "1-data-message-2024-08-08-12", + "1-data-message-2024-08-07-12", + "1-data-message-2024-23-07-17", + "1-data-message-2024-23-07-17_migrated" + }; + String[] expectedIndexesRes = { + "1-data-message-2017-01-02", + "1-data-message-2017-01-03", + "1-data-message-2021-01", + "1-data-message-2024-08-07-12", + "1-data-message-2024-23-07-17" + }; + String[] index = datastoreUtils.convertToDataIndexes(indexes, startDate != null ? startDate.toInstant() : null, endDate != null ? endDate.toInstant() : null, KapuaId.ONE); + compareResult(index, expectedIndexesRes); + } private String[] buildExpectedResult(String scopeId, int startWeek, int startYear, int endWeek, int endYear, int[] weekCountByYear) { List result = new ArrayList<>(); From 3d5b3a59f179e3cd90fc16ee29e695daf42c95cc Mon Sep 17 00:00:00 2001 From: "angelo.andreussi" Date: Fri, 1 Mar 2024 10:33:34 +0100 Subject: [PATCH 3/4] test: tests added for DatastoreUtils::convertToDataIndexes using different time windows --- .../DatastoreUtilsIndexCalculatorTest.java | 59 +++++++++++-------- 1 file changed, 34 insertions(+), 25 deletions(-) diff --git a/service/datastore/test/src/test/java/org/eclipse/kapua/service/datastore/test/junit/DatastoreUtilsIndexCalculatorTest.java b/service/datastore/test/src/test/java/org/eclipse/kapua/service/datastore/test/junit/DatastoreUtilsIndexCalculatorTest.java index fbaaf3b4f11..183be606a26 100644 --- a/service/datastore/test/src/test/java/org/eclipse/kapua/service/datastore/test/junit/DatastoreUtilsIndexCalculatorTest.java +++ b/service/datastore/test/src/test/java/org/eclipse/kapua/service/datastore/test/junit/DatastoreUtilsIndexCalculatorTest.java @@ -103,8 +103,38 @@ public void testIndex() throws KapuaException, ParseException { performTest(sdf.parse("17/12/2018 13:12 +0100"), null, buildExpectedResult("1", 52, 2018, 52, 2018, null)); performNullIndexTest(sdf.parse("02/01/2017 13:12 +0100"), sdf.parse("09/01/2017 13:12 +0100")); + } + + @Test + public void indexesFormatValidation() throws DatastoreException, ParseException { + String[] indexes = { + "1-data-message-2017-01-02", + "1-data-message-2017-01-03", + "2-data-message-2017-01-02", + "asdasdasdssd", + "1-data-message-2021-01-01-01-01", + "1-data-message-2021-01", + "1-data-message-2021-99", + "1-data-message-21-01", + "1-data-message-2024-08-08-12", + "1-data-log-2024-08-08-12", + "1-data-message-2024-08-07-12", + "1-data-message-2024-23-07-17", + "1-data-message-2024-23-07-17_migrated" + }; + String[] correctFormatIndexesWhenScopeid1 = { + "1-data-message-2017-01-02", //second day of first week in 2017 + "1-data-message-2017-01-03", + "1-data-message-2021-01", + "1-data-message-2024-08-07-12", + "1-data-message-2024-23-07-17" + }; - performFormatValidationTest(null, null); + performFormatValidationTest(null, null, indexes, correctFormatIndexesWhenScopeid1); + performFormatValidationTest(sdf.parse("01/01/2024 13:12 +0100"), null , indexes, new String[]{"1-data-message-2024-08-07-12", "1-data-message-2024-23-07-17"}); + performFormatValidationTest(null, sdf.parse("04/01/2017 13:12 +0100"),indexes, new String[]{"1-data-message-2017-01-02","1-data-message-2017-01-03"}); + performFormatValidationTest(sdf.parse("01/01/2017 13:12 +0100"), sdf.parse("01/01/2018 13:12 +0100"),indexes, new String[]{"1-data-message-2017-01-02", "1-data-message-2017-01-03"}); + performFormatValidationTest(sdf.parse("01/01/2018 13:12 +0100"), sdf.parse("01/01/2022 13:12 +0100"),indexes, new String[]{"1-data-message-2021-01"}); } @Test @@ -187,30 +217,9 @@ private void performNullIndexTest(Date startDate, Date endDate) throws Datastore compareResult(null, index); } // // YYYY-ww-ee-HH - private void performFormatValidationTest(Date startDate, Date endDate) throws DatastoreException { - String[] indexes = { - "1-data-message-2017-01-02", - "1-data-message-2017-01-03", - "2-data-message-2017-01-02", - "asdasdasdssd", - "1-data-message-2021-01-01-01-01", - "1-data-message-2021-01", - "1-data-message-2021-99", - "1-data-message-21-01", - "1-data-message-2024-08-08-12", - "1-data-message-2024-08-07-12", - "1-data-message-2024-23-07-17", - "1-data-message-2024-23-07-17_migrated" - }; - String[] expectedIndexesRes = { - "1-data-message-2017-01-02", - "1-data-message-2017-01-03", - "1-data-message-2021-01", - "1-data-message-2024-08-07-12", - "1-data-message-2024-23-07-17" - }; - String[] index = datastoreUtils.convertToDataIndexes(indexes, startDate != null ? startDate.toInstant() : null, endDate != null ? endDate.toInstant() : null, KapuaId.ONE); - compareResult(index, expectedIndexesRes); + private void performFormatValidationTest(Date startDate, Date endDate, String[] inputIndexes, String[] expectedIndexes) throws DatastoreException { + String[] index = datastoreUtils.convertToDataIndexes(inputIndexes, startDate != null ? startDate.toInstant() : null, endDate != null ? endDate.toInstant() : null, KapuaId.ONE); + compareResult(index,expectedIndexes); } private String[] buildExpectedResult(String scopeId, int startWeek, int startYear, int endWeek, int endYear, int[] weekCountByYear) { From 4d7bc77c648071dd6a4cc1fb8fc7eb82b7506d2c Mon Sep 17 00:00:00 2001 From: "angelo.andreussi" Date: Fri, 1 Mar 2024 11:50:49 +0100 Subject: [PATCH 4/4] fix: typo --- .../service/datastore/internal/mediator/DatastoreUtils.java | 6 +++--- .../test/junit/DatastoreUtilsIndexCalculatorTest.java | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/service/datastore/internal/src/main/java/org/eclipse/kapua/service/datastore/internal/mediator/DatastoreUtils.java b/service/datastore/internal/src/main/java/org/eclipse/kapua/service/datastore/internal/mediator/DatastoreUtils.java index 2a50e4be207..91bc86c02a1 100644 --- a/service/datastore/internal/src/main/java/org/eclipse/kapua/service/datastore/internal/mediator/DatastoreUtils.java +++ b/service/datastore/internal/src/main/java/org/eclipse/kapua/service/datastore/internal/mediator/DatastoreUtils.java @@ -346,11 +346,11 @@ public String normalizedIndexName(String index) { * the end date ON OR BEFORE the window end will be returned * Only indexes matching Kapua data index name pattern will be inserted inside the result list, namely this format: * - * [-]-data-message-YYYY-ww + * "scopeID-data-message-YYYY-ww" * or - * [-]-data-message-YYYY-ww-ee + * "scopeID-data-message-YYYY-ww-ee" * or - * [-]-data-message-YYYY-ww-ee-HH + * "scopeID-data-message-YYYY-ww-ee-HH" * * @param indexes * @param windowStart diff --git a/service/datastore/test/src/test/java/org/eclipse/kapua/service/datastore/test/junit/DatastoreUtilsIndexCalculatorTest.java b/service/datastore/test/src/test/java/org/eclipse/kapua/service/datastore/test/junit/DatastoreUtilsIndexCalculatorTest.java index 183be606a26..e8235bfa3ea 100644 --- a/service/datastore/test/src/test/java/org/eclipse/kapua/service/datastore/test/junit/DatastoreUtilsIndexCalculatorTest.java +++ b/service/datastore/test/src/test/java/org/eclipse/kapua/service/datastore/test/junit/DatastoreUtilsIndexCalculatorTest.java @@ -216,7 +216,8 @@ private void performNullIndexTest(Date startDate, Date endDate) throws Datastore String[] index = datastoreUtils.convertToDataIndexes(new String[]{null, null}, startDate != null ? startDate.toInstant() : null, endDate != null ? endDate.toInstant() : null, null); compareResult(null, index); } -// // YYYY-ww-ee-HH + + private void performFormatValidationTest(Date startDate, Date endDate, String[] inputIndexes, String[] expectedIndexes) throws DatastoreException { String[] index = datastoreUtils.convertToDataIndexes(inputIndexes, startDate != null ? startDate.toInstant() : null, endDate != null ? endDate.toInstant() : null, KapuaId.ONE); compareResult(index,expectedIndexes);