diff --git a/indexer/src/main/java/au/org/aodn/esindexer/configuration/AppConstants.java b/indexer/src/main/java/au/org/aodn/esindexer/configuration/AppConstants.java index 3dd0b72c..df00f348 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/configuration/AppConstants.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/configuration/AppConstants.java @@ -2,7 +2,7 @@ public interface AppConstants { String PORTAL_RECORDS_MAPPING_JSON_FILE = "portal_records_index_schema.json"; - String DTASET_INDEX_MAPPING_JSON_FILE = "dataset_index_schema.json"; + String DATASET_INDEX_MAPPING_JSON_FILE = "dataset_index_schema.json"; String FORMAT_XML = "xml"; String FORMAT_ISO19115_3_2018 = "iso19115-3.2018"; diff --git a/indexer/src/main/java/au/org/aodn/esindexer/configuration/DatasetAccessConfig.java b/indexer/src/main/java/au/org/aodn/esindexer/configuration/DatasetAccessConfig.java index 80c3181f..b2f9e46f 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/configuration/DatasetAccessConfig.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/configuration/DatasetAccessConfig.java @@ -1,6 +1,6 @@ package au.org.aodn.esindexer.configuration; -import au.org.aodn.esindexer.service.DatasetAccessServiceImpl; +import au.org.aodn.esindexer.service.DataAccessServiceImpl; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -9,9 +9,9 @@ public class DatasetAccessConfig { @Bean(name = "DatasetAccessService") - public DatasetAccessServiceImpl createDatasetAccessService( + public DataAccessServiceImpl createDatasetAccessService( @Value("${dataaccess.host:defaulthost}") String serverUrl ){ - return new DatasetAccessServiceImpl(serverUrl); + return new DataAccessServiceImpl(serverUrl); } } diff --git a/indexer/src/main/java/au/org/aodn/esindexer/controller/IndexerController.java b/indexer/src/main/java/au/org/aodn/esindexer/controller/IndexerController.java index 6aa05c47..5ac921dd 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/controller/IndexerController.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/controller/IndexerController.java @@ -1,7 +1,7 @@ package au.org.aodn.esindexer.controller; import au.org.aodn.esindexer.model.Dataset; -import au.org.aodn.esindexer.service.DatasetAccessService; +import au.org.aodn.esindexer.service.DataAccessService; import au.org.aodn.esindexer.service.GeoNetworkService; import au.org.aodn.esindexer.service.IndexerService; import co.elastic.clients.elasticsearch.core.BulkResponse; @@ -39,7 +39,7 @@ public class IndexerController { GeoNetworkService geonetworkResourceService; @Autowired - DatasetAccessService datasetAccessService; + DataAccessService dataAccessService; @GetMapping(path="/records/{uuid}", produces = "application/json") @Operation(description = "Get a document from GeoNetwork by UUID directly - JSON format response") @@ -172,7 +172,7 @@ public ResponseEntity> addDatasetByUUID(@PathVariable("uuid") Strin while (startDate.isBefore(maxDate)) { // For speed optimizing, check whether data is existing in this year. If no data, skip to next year var endDate = startDate.plusYears(1).minusDays(1); - var hasData = datasetAccessService.doesDataExist(uuid, startDate, endDate); + var hasData = dataAccessService.doesDataExist(uuid, startDate, endDate); if (!hasData) { log.info("No data found for dataset {} from {} to {}", uuid, startDate, endDate); startDate = startDate.plusYears(1); @@ -207,7 +207,7 @@ private List>> indexDatasetMonthly( while (startDateToLoop.isBefore(maxDate)) { var endDate = startDateToLoop.plusMonths(1).minusDays(1); - Dataset dataset = datasetAccessService.getIndexingDatasetBy(uuid, startDateToLoop, endDate); + Dataset dataset = dataAccessService.getIndexingDatasetBy(uuid, startDateToLoop, endDate); if (dataset != null && dataset.data() != null && !dataset.data().isEmpty()) { CompletableFuture> future = indexerService.indexDataset(dataset); futures.add(future); diff --git a/indexer/src/main/java/au/org/aodn/esindexer/model/Dataset.java b/indexer/src/main/java/au/org/aodn/esindexer/model/Dataset.java index 2506a552..7c9a5949 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/model/Dataset.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/model/Dataset.java @@ -6,5 +6,5 @@ public record Dataset( String uuid, YearMonth yearMonth, - List data + List data ) {} diff --git a/indexer/src/main/java/au/org/aodn/esindexer/model/DataRecord.java b/indexer/src/main/java/au/org/aodn/esindexer/model/Datum.java similarity index 79% rename from indexer/src/main/java/au/org/aodn/esindexer/model/DataRecord.java rename to indexer/src/main/java/au/org/aodn/esindexer/model/Datum.java index d86ff55c..81360e8c 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/model/DataRecord.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/model/Datum.java @@ -5,10 +5,11 @@ import java.time.LocalDate; -// TODO: if more fields are needed to be filtered, please add more columns here +// If more fields are needed to be filtered, please add more columns here +// and don't forget updating the override equals() method @Getter @Setter -public class DataRecord{ +public class Datum { private final LocalDate time; @@ -18,7 +19,7 @@ public class DataRecord{ private long count = 1; - public DataRecord(LocalDate time, double longitude, double latitude, double depth) { + public Datum(LocalDate time, double longitude, double latitude, double depth) { this.time = time; this.longitude = longitude; this.latitude = latitude; @@ -39,7 +40,7 @@ public boolean equals(Object obj) { if (obj == null || getClass() != obj.getClass()) { return false; } - DataRecord that = (DataRecord) obj; + Datum that = (Datum) obj; return Double.compare(that.longitude, longitude) == 0 && Double.compare(that.latitude, latitude) == 0 && Double.compare(that.depth, depth) == 0 && diff --git a/indexer/src/main/java/au/org/aodn/esindexer/service/DatasetAccessService.java b/indexer/src/main/java/au/org/aodn/esindexer/service/DataAccessService.java similarity index 89% rename from indexer/src/main/java/au/org/aodn/esindexer/service/DatasetAccessService.java rename to indexer/src/main/java/au/org/aodn/esindexer/service/DataAccessService.java index af0a2f8a..82654d48 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/service/DatasetAccessService.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/service/DataAccessService.java @@ -4,7 +4,7 @@ import java.time.LocalDate; -public interface DatasetAccessService { +public interface DataAccessService { Dataset getIndexingDatasetBy(String uuid, LocalDate startDate, LocalDate endDate); boolean doesDataExist(String uuid, LocalDate startDate, LocalDate endDate); String getServiceUrl(); diff --git a/indexer/src/main/java/au/org/aodn/esindexer/service/DatasetAccessServiceImpl.java b/indexer/src/main/java/au/org/aodn/esindexer/service/DataAccessServiceImpl.java similarity index 77% rename from indexer/src/main/java/au/org/aodn/esindexer/service/DatasetAccessServiceImpl.java rename to indexer/src/main/java/au/org/aodn/esindexer/service/DataAccessServiceImpl.java index 227a371f..466a9354 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/service/DatasetAccessServiceImpl.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/service/DataAccessServiceImpl.java @@ -1,7 +1,7 @@ package au.org.aodn.esindexer.service; import au.org.aodn.esindexer.exception.MetadataNotFoundException; -import au.org.aodn.esindexer.model.DataRecord; +import au.org.aodn.esindexer.model.Datum; import au.org.aodn.esindexer.model.Dataset; import org.springframework.http.*; import org.springframework.web.client.HttpClientErrorException; @@ -15,7 +15,7 @@ import java.util.List; import java.util.Map; -public class DatasetAccessServiceImpl implements DatasetAccessService { +public class DataAccessServiceImpl implements DataAccessService { private String serverUrl; @@ -29,7 +29,7 @@ public void setServiceUrl(String url) { this.serverUrl = url; } - public DatasetAccessServiceImpl(String serverUrl) { + public DataAccessServiceImpl(String serverUrl) { setServiceUrl(serverUrl); } @@ -48,33 +48,33 @@ public Dataset getIndexingDatasetBy(String uuid, LocalDate startDate, LocalDate Map params = new HashMap<>(); params.put("uuid", uuid); - String url = UriComponentsBuilder.fromHttpUrl(getDatasetAccessEndpoint() + "/data/{uuid}") + String url = UriComponentsBuilder.fromHttpUrl(getDataAccessEndpoint() + "/data/{uuid}") .queryParam("is_to_index", "true") .queryParam("start_date", startDate) .queryParam("end_date", endDate) .buildAndExpand(uuid) .toUriString(); - ResponseEntity responseEntity = restTemplate.exchange( + ResponseEntity responseEntity = restTemplate.exchange( url, HttpMethod.GET, request, - DataRecord[].class, + Datum[].class, params ); if (responseEntity.getStatusCode().is2xxSuccessful()) { - List records = new ArrayList<>(); + List data = new ArrayList<>(); if (responseEntity.getBody() != null) { - records = List.of(responseEntity.getBody()); + data = List.of(responseEntity.getBody()); } - var recordsToIndex = summarizeRecords(records); + var dataToIndex = aggregateData(data); return new Dataset( uuid, YearMonth.of(startDate.getYear(), startDate.getMonth()), - recordsToIndex + dataToIndex ); } throw new RuntimeException("Unable to retrieve dataset with UUID: " + uuid ); @@ -94,7 +94,7 @@ public boolean doesDataExist(String uuid, LocalDate startDate, LocalDate endDate Map params = new HashMap<>(); params.put("uuid", uuid); - String url = UriComponentsBuilder.fromHttpUrl(getDatasetAccessEndpoint() + "/data/{uuid}/has_data") + String url = UriComponentsBuilder.fromHttpUrl(getDataAccessEndpoint() + "/data/{uuid}/has_data") .queryParam("start_date", startDate) .queryParam("end_date", endDate) .buildAndExpand(uuid) @@ -118,24 +118,24 @@ public boolean doesDataExist(String uuid, LocalDate startDate, LocalDate endDate } /** - * Summarize the records by counting the number if all the concerned fields are the same - * @param records the records to summarize - * @return the summarized records + * Summarize the data by counting the number if all the concerned fields are the same + * @param data the data to summarize + * @return the summarized data */ - private List summarizeRecords(List records) { - var summarizedRecords = new ArrayList(); - for (var record: records) { - if (summarizedRecords.contains(record)) { - var existingRecord = summarizedRecords.get(summarizedRecords.indexOf(record)); - existingRecord.incrementCount(); + private List aggregateData(List data) { + var aggregatedData = new ArrayList(); + for (var datum: data) { + if (aggregatedData.contains(datum)) { + var existingDatum = aggregatedData.get(aggregatedData.indexOf(datum)); + existingDatum.incrementCount(); } else { - summarizedRecords.add(record); + aggregatedData.add(datum); } } - return summarizedRecords; + return aggregatedData; } - private String getDatasetAccessEndpoint() { + private String getDataAccessEndpoint() { return getServiceUrl() + "/api/v1/das/"; }