Skip to content

Commit

Permalink
Merge pull request #3981 from Agnul97/fix-convert_to_data_indexes
Browse files Browse the repository at this point in the history 
FIX - Function "convertToDataIndexes" throwing error when one bad format index is present
  • Loading branch information
@Coduz
Coduz authored Mar 7, 2024
2 parents ddd356d + 4d7bc77 commit af7e915
Show file tree
Hide file tree
Showing 3 changed files with 72 additions and 8 deletions.
36 changes: 31 additions & 5 deletions ...l/src/main/java/org/eclipse/kapua/service/datastore/internal/mediator/DatastoreUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeFormatterBuilder;
import java.time.format.DateTimeParseException;
import java.time.format.ResolverStyle;
import java.time.temporal.ChronoField;
import java.time.temporal.ChronoUnit;
Expand Down Expand Up @@ -343,23 +344,34 @@ public String normalizedIndexName(String index) {
* Return the list of the data indexes between start and windowEnd instant by scope id.
* Only the indexes that will be *FULLY* included in the list (i.e. with a starting date ON OR AFTER the window start AND
* the end date ON OR BEFORE the window end will be returned
* Only indexes matching Kapua data index name pattern will be inserted inside the result list, namely this format:
*
* "scopeID-data-message-YYYY-ww"
* or
* "scopeID-data-message-YYYY-ww-ee"
* or
* "scopeID-data-message-YYYY-ww-ee-HH"
*
* @param indexes
* @param windowStart
* @param windowEnd
* @param scopeId
* @return The list of the data indexes between start and end
* @throws DatastoreException
*/
public String[] convertToDataIndexes(@NotNull String[] indexes, Instant windowStart, Instant windowEnd) throws DatastoreException {
public String[] convertToDataIndexes(@NotNull String[] indexes, Instant windowStart, Instant windowEnd, KapuaId scopeId) throws DatastoreException {
boolean skipTemporalValidation = false;
if (windowStart == null && windowEnd == null) {
return indexes;
skipTemporalValidation = true;
}

List<String> result = new ArrayList<>();
for (String index : indexes) {
if (index == null) {
continue;
}
if (scopeId != null && (!validatePrefixIndex(index, scopeId))) {
continue;
}
String strippedIndex = stripPrefixAndAccount(index);
int fragments = strippedIndex.split("-").length;
DateTimeFormatter formatter;
Expand All @@ -386,9 +398,18 @@ public String[] convertToDataIndexes(@NotNull String[] indexes, Instant windowSt
indexWidth = 1;
break;
}

TemporalAccessor temporalAccessor;
try {
temporalAccessor = formatter.parse(strippedIndex);
} catch (DateTimeParseException e) {
//unable to parse...the format is not right so don't add this index into result-set
continue;
}
if (skipTemporalValidation) { //this index passed format validation...proceed to next index
result.add(index);
continue;
}
try {
TemporalAccessor temporalAccessor = formatter.parse(strippedIndex);
Instant indexStart = LocalDateTime.of(
temporalAccessor.get(ChronoField.YEAR),
temporalAccessor.get(ChronoField.MONTH_OF_YEAR),
Expand All @@ -411,6 +432,11 @@ public String[] convertToDataIndexes(@NotNull String[] indexes, Instant windowSt
return result.toArray(new String[0]);
}

private boolean validatePrefixIndex(@NotNull String index,@NotNull KapuaId scopeId) {
String genericIndexFormat = getDataIndexName(scopeId);
return index.startsWith(genericIndexFormat.substring(0, genericIndexFormat.length() - 1));
}

private boolean isIndexFullyAfterInstant(@NotNull Instant indexStart, @NotNull Instant indexEnd, @NotNull Instant checkpoint) {
return !indexStart.isBefore(checkpoint) && !indexEnd.isBefore(checkpoint);
}
Expand Down
2 changes: 1 addition & 1 deletion ...re/test-steps/src/main/java/org/eclipse/kapua/service/datastore/steps/DatastoreSteps.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1593,7 +1593,7 @@ public void deleteIndexesBetweenDates(String fromDate, String toDate) throws Exc
primeException();
try {
String[] indexes = KapuaLocator.getInstance().getComponent(DatastoreUtils.class).convertToDataIndexes(getDataIndexesByAccount(getCurrentScopeId()), KapuaDateUtils.parseDate(fromDate).toInstant(),
KapuaDateUtils.parseDate(toDate).toInstant());
KapuaDateUtils.parseDate(toDate).toInstant(), null);
elasticsearchClient.deleteIndexes(indexes);
} catch (Exception ex) {
verifyException(ex);
Expand Down
42 changes: 40 additions & 2 deletions ...ava/org/eclipse/kapua/service/datastore/test/junit/DatastoreUtilsIndexCalculatorTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,38 @@ public void testIndex() throws KapuaException, ParseException {
performNullIndexTest(sdf.parse("02/01/2017 13:12 +0100"), sdf.parse("09/01/2017 13:12 +0100"));
}

@Test
public void indexesFormatValidation() throws DatastoreException, ParseException {
String[] indexes = {
"1-data-message-2017-01-02",
"1-data-message-2017-01-03",
"2-data-message-2017-01-02",
"asdasdasdssd",
"1-data-message-2021-01-01-01-01",
"1-data-message-2021-01",
"1-data-message-2021-99",
"1-data-message-21-01",
"1-data-message-2024-08-08-12",
"1-data-log-2024-08-08-12",
"1-data-message-2024-08-07-12",
"1-data-message-2024-23-07-17",
"1-data-message-2024-23-07-17_migrated"
};
String[] correctFormatIndexesWhenScopeid1 = {
"1-data-message-2017-01-02", //second day of first week in 2017
"1-data-message-2017-01-03",
"1-data-message-2021-01",
"1-data-message-2024-08-07-12",
"1-data-message-2024-23-07-17"
};

performFormatValidationTest(null, null, indexes, correctFormatIndexesWhenScopeid1);
performFormatValidationTest(sdf.parse("01/01/2024 13:12 +0100"), null , indexes, new String[]{"1-data-message-2024-08-07-12", "1-data-message-2024-23-07-17"});
performFormatValidationTest(null, sdf.parse("04/01/2017 13:12 +0100"),indexes, new String[]{"1-data-message-2017-01-02","1-data-message-2017-01-03"});
performFormatValidationTest(sdf.parse("01/01/2017 13:12 +0100"), sdf.parse("01/01/2018 13:12 +0100"),indexes, new String[]{"1-data-message-2017-01-02", "1-data-message-2017-01-03"});
performFormatValidationTest(sdf.parse("01/01/2018 13:12 +0100"), sdf.parse("01/01/2022 13:12 +0100"),indexes, new String[]{"1-data-message-2021-01"});
}

@Test
public void dataIndexNameByScopeId() {
Assert.assertEquals("1-data-message-*", datastoreUtils.getDataIndexName(KapuaId.ONE));
Expand Down Expand Up @@ -159,7 +191,7 @@ private void performTest(Date startDate, Date endDate, String[] expectedIndexes)
calEndDate != null ? calEndDate.get(Calendar.WEEK_OF_YEAR) : "Infinity",
calEndDate != null ? calEndDate.get(Calendar.DAY_OF_WEEK) : "Infinity");

String[] index = datastoreUtils.convertToDataIndexes(getDataIndexesByAccount(KapuaEid.ONE), startDate != null ? startDate.toInstant() : null, endDate != null ? endDate.toInstant() : null);
String[] index = datastoreUtils.convertToDataIndexes(getDataIndexesByAccount(KapuaEid.ONE), startDate != null ? startDate.toInstant() : null, endDate != null ? endDate.toInstant() : null, null);
compareResult(expectedIndexes, index);
}

Expand All @@ -181,10 +213,16 @@ private void performNullIndexTest(Date startDate, Date endDate) throws Datastore
calEndDate != null ? calEndDate.get(Calendar.WEEK_OF_YEAR) : "Infinity",
calEndDate != null ? calEndDate.get(Calendar.DAY_OF_WEEK) : "Infinity");

String[] index = datastoreUtils.convertToDataIndexes(new String[]{null, null}, startDate != null ? startDate.toInstant() : null, endDate != null ? endDate.toInstant() : null);
String[] index = datastoreUtils.convertToDataIndexes(new String[]{null, null}, startDate != null ? startDate.toInstant() : null, endDate != null ? endDate.toInstant() : null, null);
compareResult(null, index);
}


private void performFormatValidationTest(Date startDate, Date endDate, String[] inputIndexes, String[] expectedIndexes) throws DatastoreException {
String[] index = datastoreUtils.convertToDataIndexes(inputIndexes, startDate != null ? startDate.toInstant() : null, endDate != null ? endDate.toInstant() : null, KapuaId.ONE);
compareResult(index,expectedIndexes);
}

private String[] buildExpectedResult(String scopeId, int startWeek, int startYear, int endWeek, int endYear, int[] weekCountByYear) {
List<String> result = new ArrayList<>();
for (int i = startYear; i <= endYear; i++) {
Expand Down

0 comments on commit af7e915

Please sign in to comment.