Skip to content

Commit

Permalink
Merge pull request #153 from aodn/bugs/5916-fix-bug-non-noded
Browse files Browse the repository at this point in the history 
Bugs/5916 fix bug non noded
  • Loading branch information
@vietnguyengit
vietnguyengit authored Oct 18, 2024
2 parents 6a5bded + 2dcec06 commit 7e87e70
Show file tree
Hide file tree
Showing 22 changed files with 5,877 additions and 972 deletions.
22 changes: 22 additions & 0 deletions indexer/src/main/java/au/org/aodn/esindexer/configuration/ForkJoinConfig.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
package au.org.aodn.esindexer.configuration;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.annotation.Configuration;

@Configuration
public class ForkJoinConfig {

protected static Logger logger = LoggerFactory.getLogger(ForkJoinConfig.class);

static {
// By default, ForkJoinPool set parallelism to core - 1, therefore 2 core still use 1 thread, we want to change
// this default utilize the resource better. Affect stream().parallel() or parllelStream()
String coreNum = String.valueOf(Runtime.getRuntime().availableProcessors());
System.setProperty(
"java.util.concurrent.ForkJoinPool.common.parallelism",
coreNum
);
logger.info("Set ForkJoin use number of cores = {}", coreNum);
}
}
57 changes: 31 additions & 26 deletions indexer/src/main/java/au/org/aodn/esindexer/service/StacCollectionMapperService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,7 @@
import jakarta.xml.bind.JAXBElement;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.mapstruct.Mapper;
import org.mapstruct.Mapping;
import org.mapstruct.Named;
import org.mapstruct.*;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
Expand All @@ -39,11 +37,13 @@
@Service
@Mapper(componentModel = "spring")
public abstract class StacCollectionMapperService {

public static final String REAL_TIME = "real-time";
@BeforeMapping
void beforeMapping(MDMetadataType source) {
logger.info("Processing uuid: {}", CommonUtils.getUUID(source));
}

@Mapping(target="uuid", source = "source", qualifiedByName = "mapUUID")
@Mapping(target="title", source = "source", qualifiedByName = "mapTitle" )
@Mapping(target="title", source = "source", qualifiedByName = "mapTitle")
@Mapping(target="description", source = "source", qualifiedByName = "mapDescription")
@Mapping(target="extent.bbox", source = "source", qualifiedByName = "mapExtentBbox")
@Mapping(target="extent.temporal", source = "source", qualifiedByName = "mapExtentTemporal")
Expand All @@ -68,7 +68,7 @@ public abstract class StacCollectionMapperService {
@Mapping(target="summaries.revision", source = "source", qualifiedByName = "mapSummaries.revision")
public abstract StacCollectionModel mapToSTACCollection(MDMetadataType source);

private static final Logger logger = LogManager.getLogger(StacCollectionMapperService.class);
protected static final Logger logger = LogManager.getLogger(StacCollectionMapperService.class);

@Value("${spring.jpa.properties.hibernate.jdbc.time_zone}")
private String timeZoneId;
Expand Down Expand Up @@ -104,7 +104,7 @@ List<String[]> createExtentTemporal(MDMetadataType source) {

List<MDDataIdentificationType> items = MapperUtils.findMDDataIdentificationType(source);
if (items.isEmpty()) {
logger.warn("Unable to find extent temporal information for metadata record: " + this.mapUUID(source));
logger.warn("Unable to find extent temporal information for metadata record: {}", CommonUtils.getUUID(source));
return null;
}

Expand All @@ -128,13 +128,13 @@ List<String[]> createExtentTemporal(MDMetadataType source) {
if (pair0.isEmpty()) {
pair0 = safeGet(() -> timePeriodType.getBeginPosition().getValue().get(0));
}
pair0.ifPresent(pair -> temporalPair[0] = convertDateToZonedDateTime(this.mapUUID(source), pair, true));
pair0.ifPresent(pair -> temporalPair[0] = convertDateToZonedDateTime(CommonUtils.getUUID(source), pair, true));

var pair1 = safeGet(() -> timePeriodType.getEnd().getTimeInstant().getTimePosition().getValue().get(0));
if (pair1.isEmpty()) {
pair1 = safeGet(() -> timePeriodType.getEndPosition().getValue().get(0));
}
pair1.ifPresent(pair -> temporalPair[1] = convertDateToZonedDateTime(this.mapUUID(source), pair, false));
pair1.ifPresent(pair -> temporalPair[1] = convertDateToZonedDateTime(CommonUtils.getUUID(source), pair, false));
}

result.add(temporalPair);
Expand Down Expand Up @@ -381,7 +381,7 @@ Map<String, String> createSummariesScope(MDMetadataType source) {
}
}

logger.warn("Unable to find scope metadata record: " + this.mapUUID(source));
logger.warn("Unable to find scope metadata record: {}", CommonUtils.getUUID(source));
return null;
}
/**
Expand Down Expand Up @@ -434,13 +434,15 @@ String mapUpdateFrequency(MDMetadataType source) {
@Named("mapSummaries.datasetProvider")
String mapDatasetOwner(MDMetadataType source) {
List<ProviderModel> providers = mapProviders(source);
return providers.stream().anyMatch(p -> p.getName().contains("IMOS")) ? "IMOS" : null;
return providers.stream()
.filter(p -> p.getName() != null)
.anyMatch(p -> p.getName().contains("IMOS")) ? "IMOS" : null;
}

@Named("mapSummaries.datasetGroup")
String mapGeoNetworkGroup(MDMetadataType source) {
try {
String group = geoNetworkService.findGroupById(mapUUID(source));
String group = geoNetworkService.findGroupById(CommonUtils.getUUID(source));
return group != null ? group.toLowerCase() : null;
}
catch (IOException e) {
Expand Down Expand Up @@ -559,7 +561,7 @@ List<ThemesModel> mapThemes(MDMetadataType source) {
for (MDDataIdentificationType i : items) {
i.getDescriptiveKeywords().forEach(descriptiveKeyword -> {
ThemesModel themesModel = ThemesModel.builder().build();
String uuid = this.mapUUID(source);
String uuid = CommonUtils.getUUID(source);

themesModel.setConcepts(mapThemesConcepts(descriptiveKeyword));
themesModel.setTitle(mapThemesTitle(descriptiveKeyword, uuid));
Expand Down Expand Up @@ -615,11 +617,11 @@ else if (LinkUtils.isWfs(protocol)) {
}

// Now add links for logos
geoNetworkService.getLogo(this.mapUUID(source))
geoNetworkService.getLogo(CommonUtils.getUUID(source))
.ifPresent(results::add);

// Thumbnail link
geoNetworkService.getThumbnail(this.mapUUID(source))
geoNetworkService.getThumbnail(CommonUtils.getUUID(source))
.ifPresent(results::add);

// full metadata link
Expand Down Expand Up @@ -650,7 +652,7 @@ else if (LinkUtils.isWfs(protocol)) {
}

private List<LinkModel> getAssociatedRecords(MDMetadataType source) {
var associatedRecordsData = geoNetworkService.getAssociatedRecords(mapUUID(source));
var associatedRecordsData = geoNetworkService.getAssociatedRecords(CommonUtils.getUUID(source));
return AssociatedRecordsUtil.generateAssociatedRecords(associatedRecordsData);
}

Expand Down Expand Up @@ -742,7 +744,11 @@ List<ProviderModel> mapProviders(MDMetadataType source) {
.ifPresent(p -> p.forEach(party -> {
if(party.getAbstractCIParty().getValue() instanceof CIOrganisationType2 organisationType2) {
ProviderModel providerModel = ProviderModel.builder().build();
providerModel.setRoles(Collections.singletonList(ciResponsibility.getRole().getCIRoleCode().getCodeListValue()));
providerModel.setRoles(Collections.singletonList(
ciResponsibility.getRole().getCIRoleCode() != null ?
ciResponsibility.getRole().getCIRoleCode().getCodeListValue() :
null
));
providerModel.setName(organisationType2.getName() != null ? organisationType2.getName().getCharacterString().getValue().toString() : "");

organisationType2.getIndividual().forEach(individual -> individual.getCIIndividual().getContactInfo().forEach(contactInfo -> {
Expand All @@ -763,12 +769,12 @@ else if(party.getAbstractCIParty().getValue() instanceof CIIndividualType2 indiv
results.add(providerModel);
}
else {
logger.error("Unable to cast getAbstractCIParty().getValue() to CIOrganisationType2 or CIIndividualType2 for metadata record: {}", mapUUID(source));
logger.error("Unable to cast getAbstractCIParty().getValue() to CIOrganisationType2 or CIIndividualType2 for metadata record: {}", CommonUtils.getUUID(source));
}
}));
}
else {
logger.warn("getContact().getAbstractResponsibility() in mapProviders is not of type CIResponsibilityType2 for UUID {}", mapUUID(source));
logger.warn("getContact().getAbstractResponsibility() in mapProviders is not of type CIResponsibilityType2 for UUID {}", CommonUtils.getUUID(source));
}
}));
return results;
Expand Down Expand Up @@ -814,7 +820,7 @@ String mapLicense(MDMetadataType source) {
if (!licenses.isEmpty()) {
return String.join(" | ", licenses);
} else {
logger.debug("Unable to find license information for metadata record: " + this.mapUUID(source));
logger.debug("Unable to find license information for metadata record: {}", CommonUtils.getUUID(source));
return "";
}
}
Expand Down Expand Up @@ -859,7 +865,7 @@ List<ContactsModel> mapContacts(MDMetadataType source) {
if (responsibilityType instanceof final CIResponsibilityType2 ciResponsibility) {

if (ciResponsibility.getParty().isEmpty()) {
logger.warn("Unable to find contact info for metadata record: {}", this.mapUUID(source));
logger.warn("Unable to find contact info for metadata record: {}", CommonUtils.getUUID(source));
}
else {
ciResponsibility.getParty().forEach(party -> {
Expand All @@ -872,7 +878,7 @@ List<ContactsModel> mapContacts(MDMetadataType source) {
}
}
else {
logger.warn("getAbstractResponsibility() is null in mapContact for metadata record: {}", mapUUID(source));
logger.warn("getAbstractResponsibility() is null in mapContact for metadata record: {}", CommonUtils.getUUID(source));
}
});
}
Expand Down Expand Up @@ -918,7 +924,7 @@ List<ContactsModel> mapContacts(MDMetadataType source) {
safeGet(() -> property.getCIResponsibility().getParty())
.ifPresent(parties -> {
if (parties.isEmpty()) {
logger.warn("Unable to find citation contact info for metadata record: {}", this.mapUUID(source));
logger.warn("Unable to find citation contact info for metadata record: {}", CommonUtils.getUUID(source));
} else {
parties.forEach(party -> {
var mappedContacts = MapperUtils.mapOrgContacts(ciResponsibility, party);
Expand Down Expand Up @@ -950,7 +956,7 @@ protected List<LanguageModel> mapLanguages(MDMetadataType source) {
case "eng" -> languageModel.setName("English");
case "fra" -> languageModel.setName("French");
default -> {
logger.warn("Unable to find language for metadata record: {}, default to eng", this.mapUUID(source));
logger.warn("Unable to find language for metadata record: {}, default to eng", CommonUtils.getUUID(source));
languageModel.setCode("eng");
languageModel.setName("English");
}
Expand All @@ -962,7 +968,6 @@ protected List<LanguageModel> mapLanguages(MDMetadataType source) {

return results;
}

/**
* Special handle for MimeFileType object.
* @param onlineResource
Expand Down
30 changes: 19 additions & 11 deletions indexer/src/main/java/au/org/aodn/esindexer/utils/GeometryBase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ else if(!r.isEmpty() && r.get(0) instanceof EXGeographicBoundingBoxType) {
}

protected static List<Geometry> findPolygonsFromEXBoundingPolygonType(String rawCRS, List<AbstractEXGeographicExtentType> rawInput) {
List<Geometry> polygons = new ArrayList<>();
final List<Geometry> polygons = new ArrayList<>();

if(COORDINATE_SYSTEM_CRS84.equals(rawCRS)) {
List<List<GMObjectPropertyType>> input = rawInput
Expand Down Expand Up @@ -145,11 +145,15 @@ protected static List<Geometry> findPolygonsFromEXBoundingPolygonType(String raw
items.add(coordinate);
}
}

// We need to store it so that we can create the multi-array as told by spec
Polygon polygon = geoJsonFactory.createPolygon(items.toArray(new Coordinate[0]));
polygons.add(polygon);
logger.debug("MultiSurfaceType 2D Polygon added {}", polygon);
try {
// We need to store it so that we can create the multi-array as told by spec
Polygon polygon = geoJsonFactory.createPolygon(items.toArray(new Coordinate[0]));
polygons.add(polygon);
logger.debug("MultiSurfaceType 2D added (findPolygonsFromEXBoundingPolygonType) {}", polygon);
}
catch(IllegalArgumentException iae) {
logger.warn("Invalid geometry point for polygon", iae);
}
}
});
}
Expand Down Expand Up @@ -195,12 +199,16 @@ protected static List<Geometry> findPolygonsFromEXBoundingPolygonType(String raw
items.add(coordinate);
}
}
try {
// We need to store it so that we can create the multi-array as told by spec
Polygon polygon = geoJsonFactory.createPolygon(items.toArray(new Coordinate[0]));
polygons.add(polygon);

// We need to store it so that we can create the multi-array as told by spec
Polygon polygon = geoJsonFactory.createPolygon(items.toArray(new Coordinate[0]));
polygons.add(polygon);

logger.debug("PolygonType 2D Polygon added {}", polygon);
logger.debug("LinearRingType added {}", polygon);
}
catch(IllegalArgumentException iae) {
logger.warn("Invalid LinearRingType", iae);
}
}
});
}
Expand Down
Loading

0 comments on commit 7e87e70

Please sign in to comment.