From ac2142c9fa51b82a6de9998ae4085f30a599265f Mon Sep 17 00:00:00 2001 From: utas-raymondng Date: Fri, 1 Nov 2024 15:32:47 +1100 Subject: [PATCH 1/2] Bug fix on grid not generate due to size too small --- .../configuration/IndexerConfig.java | 5 -- .../service/StacCollectionMapperService.java | 9 ++- .../org/aodn/esindexer/utils/BBoxUtils.java | 2 +- .../aodn/esindexer/utils/GeometryUtils.java | 69 ++++++++++-------- .../esindexer/utils/GeometryUtilsTest.java | 50 +++++++++++-- .../resources/canned/sample7_stac_no_es.json | 47 ++++++++++++- .../test/resources/canned/sample9_stac.json | 23 ++++-- .../sample_multiple_temporal1_stac.json | 70 ++++++++++++++++--- .../sample_multiple_temporal2_stac.json | 70 ++++++++++++++++--- 9 files changed, 278 insertions(+), 67 deletions(-) diff --git a/indexer/src/main/java/au/org/aodn/esindexer/configuration/IndexerConfig.java b/indexer/src/main/java/au/org/aodn/esindexer/configuration/IndexerConfig.java index 88f1fa10..0d956782 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/configuration/IndexerConfig.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/configuration/IndexerConfig.java @@ -20,8 +20,6 @@ @EnableRetry @EnableAsync public class IndexerConfig { - @Value("${app.geometry.gridLandSize:10.0}") - protected double gridSize; @Value("${app.geometry.enableGridSpatialExtents:false}") protected boolean girdSpatialExtents; @@ -31,11 +29,8 @@ public class IndexerConfig { @PostConstruct public void init() { - GeometryUtils.setGridSize(gridSize); GeometryUtils.setGridSpatialExtents(girdSpatialExtents); GeometryUtils.setCoastalPrecision(coastalPrecision); - GeometryUtils.setExecutorService(Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors())); - GeometryUtils.init(); } diff --git a/indexer/src/main/java/au/org/aodn/esindexer/service/StacCollectionMapperService.java b/indexer/src/main/java/au/org/aodn/esindexer/service/StacCollectionMapperService.java index 3028a417..61496e8b 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/service/StacCollectionMapperService.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/service/StacCollectionMapperService.java @@ -91,6 +91,7 @@ String mapUUID(MDMetadataType source) { List> mapExtentBbox(MDMetadataType source) { return GeometryUtils.createGeometryItems( source, + null, BBoxUtils::createBBoxFrom ); } @@ -343,11 +344,16 @@ private HashMap getMetadataDateInfoFrom(List> mapGeometryCentroid(MDMetadataType source) { return GeometryUtils.createGeometryItems( source, + 10, GeometryUtils::createCentroidFrom ); } @@ -356,6 +362,7 @@ List> mapGeometryCentroid(MDMetadataType source) { Map mapSummariesGeometry(MDMetadataType source) { return GeometryUtils.createGeometryItems( source, + 10, // This is useful in testing/edge only. GeometryUtils::createGeometryFrom ); } diff --git a/indexer/src/main/java/au/org/aodn/esindexer/utils/BBoxUtils.java b/indexer/src/main/java/au/org/aodn/esindexer/utils/BBoxUtils.java index 0ddb33fb..2777f941 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/utils/BBoxUtils.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/utils/BBoxUtils.java @@ -12,7 +12,7 @@ public class BBoxUtils { protected static Logger logger = LogManager.getLogger(BBoxUtils.class); - public static List> createBBoxFrom(List> rawInput) { + public static List> createBBoxFrom(List> rawInput, final Integer noUse) { //TODO: avoid hardcode CRS, get it from document List> polygons = GeometryBase.findPolygonsFrom(GeometryBase.COORDINATE_SYSTEM_CRS84, rawInput); return StacUtils.createStacBBox(polygons); diff --git a/indexer/src/main/java/au/org/aodn/esindexer/utils/GeometryUtils.java b/indexer/src/main/java/au/org/aodn/esindexer/utils/GeometryUtils.java index 068292df..b6ed825b 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/utils/GeometryUtils.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/utils/GeometryUtils.java @@ -13,7 +13,6 @@ import org.geotools.data.simple.SimpleFeatureSource; import org.geotools.geojson.geom.GeometryJSON; import org.locationtech.jts.algorithm.Area; -import org.locationtech.jts.algorithm.Orientation; import org.locationtech.jts.geom.*; import org.locationtech.jts.operation.union.UnaryUnionOp; import org.locationtech.jts.simplify.DouglasPeuckerSimplifier; @@ -26,6 +25,7 @@ import java.net.URL; import java.util.*; import java.util.concurrent.*; +import java.util.function.BiFunction; import java.util.function.Function; public class GeometryUtils { @@ -43,7 +43,7 @@ public enum PointOrientation { // Create an ExecutorService with a fixed thread pool size @Getter @Setter - protected static ExecutorService executorService; + protected static ExecutorService executorService = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); @Getter @Setter @@ -53,10 +53,6 @@ public enum PointOrientation { @Setter protected static int centroidScale = 3; - @Getter - @Setter - protected static double gridSize = 10.0; - @Getter @Setter protected static double coastalPrecision = 0.03; @@ -243,28 +239,40 @@ protected static Coordinate[] reverseCoordinates(Coordinate[] coords) { */ protected static List createGridPolygons(Envelope envelope, double cellSize) { List gridPolygons = new ArrayList<>(); - GeometryFactory geometryFactory = new GeometryFactory(); double minX = envelope.getMinX(); double minY = envelope.getMinY(); double maxX = envelope.getMaxX(); double maxY = envelope.getMaxY(); - // Loop to create grid cells - for (double x = minX; x < maxX; x += cellSize) { - for (double y = minY; y < maxY; y += cellSize) { - // Create a polygon for each grid cell - Polygon gridCell = geometryFactory.createPolygon(new Coordinate[]{ - new Coordinate(x, y), - new Coordinate(x + cellSize, y), - new Coordinate(x + cellSize, y + cellSize), - new Coordinate(x, y + cellSize), - new Coordinate(x, y) // Closing the polygon - }); - gridPolygons.add(gridCell); + // Check if the cellSize is too small on both side, that is we need to split even if the + // envelope have long width but short height etc + if (cellSize <= 0 || (cellSize > (maxX - minX) && cellSize > (maxY - minY))) { + Polygon itself = factory.createPolygon(new Coordinate[]{ + new Coordinate(minX, minY), + new Coordinate(maxX, minY), + new Coordinate(maxX, maxY), + new Coordinate(minX, maxY), + new Coordinate(minX, minY) // Closing the polygon + }); + gridPolygons.add(itself); + } + else { + // Loop to create grid cells + for (double x = minX; x < maxX; x += cellSize) { + for (double y = minY; y < maxY; y += cellSize) { + // Create a polygon for each grid cell + Polygon gridCell = factory.createPolygon(new Coordinate[]{ + new Coordinate(x, y), + new Coordinate(x + cellSize, y), + new Coordinate(x + cellSize, y + cellSize), + new Coordinate(x, y + cellSize), + new Coordinate(x, y) // Closing the polygon + }); + gridPolygons.add(gridCell); + } } } - return gridPolygons; } /** @@ -294,14 +302,14 @@ protected static List convertToListGeometry(Geometry multipolygon) { * @param large - A Polygon to break into grid * @return - A polygon the break into grid. */ - protected static List breakLargeGeometryToGrid(final Geometry large) { + protected static List breakLargeGeometryToGrid(final Geometry large, int gridSize) { logger.debug("Break down large geometry to grid {}", large); // Get the bounding box (extent) of the large polygon Envelope envelope = large.getEnvelopeInternal(); // Hard code cell size, we can adjust the break grid size. 10.0 result in 3x3 grid // cover Australia - List gridPolygons = createGridPolygons(envelope, getGridSize()); + List gridPolygons = createGridPolygons(envelope, gridSize); // List to store Future objects representing the results of the tasks List> futureResults = new ArrayList<>(); @@ -334,10 +342,10 @@ protected static List breakLargeGeometryToGrid(final Geometry large) { return intersectedPolygons; } - protected static List> splitAreaToGrid(List> geoList) { + protected static List> splitAreaToGrid(List> geoList, final int gridSize) { return geoList.stream() .flatMap(Collection::stream) - .map(GeometryUtils::breakLargeGeometryToGrid) + .map(i -> GeometryUtils.breakLargeGeometryToGrid(i, gridSize)) .toList(); } /** @@ -445,7 +453,8 @@ protected static List> createCentroid(List> geom */ public static R createGeometryItems( MDMetadataType source, - Function>, R> handler) { + Integer gridSize, + BiFunction>, Integer, R> handler) { List items = MapperUtils.findMDDataIdentificationType(source); if(!items.isEmpty()) { @@ -492,7 +501,7 @@ public static R createGeometryItems( .toList() ) .toList(); - return handler.apply(rawInput); + return handler.apply(rawInput, gridSize); } return null; } @@ -510,8 +519,8 @@ protected static List> createGeometryWithoutLand(List> createCentroidFrom(List> rawInput) { - List> grid = splitAreaToGrid(createGeometryWithoutLand(rawInput)); + public static List> createCentroidFrom(List> rawInput, Integer gridSize) { + List> grid = splitAreaToGrid(createGeometryWithoutLand(rawInput), gridSize); return (grid != null && !grid.isEmpty()) ? createCentroid(grid) : null; } /** @@ -521,7 +530,7 @@ public static List> createCentroidFrom(List createGeometryFrom(List> rawInput) { + public static Map createGeometryFrom(List> rawInput, Integer gridSize) { // The return polygon is in EPSG:4326, so we can call createGeoJson directly // This line will cause the spatial extents to break into grid, it may help to debug but will make production @@ -530,7 +539,7 @@ public static List> createCentroidFrom(List> polygon = GeometryBase.findPolygonsFrom(GeometryBase.COORDINATE_SYSTEM_CRS84, rawInput); // This is helpful when debug, it help to visualize the grid size on say edge env. - polygon = isGridSpatialExtents() ? splitAreaToGrid(polygon) : polygon; + polygon = isGridSpatialExtents() ? splitAreaToGrid(polygon, gridSize) : polygon; return (polygon != null && !polygon.isEmpty()) ? createGeoJson(polygon) : null; } diff --git a/indexer/src/test/java/au/org/aodn/esindexer/utils/GeometryUtilsTest.java b/indexer/src/test/java/au/org/aodn/esindexer/utils/GeometryUtilsTest.java index e32a8472..9ede56f9 100644 --- a/indexer/src/test/java/au/org/aodn/esindexer/utils/GeometryUtilsTest.java +++ b/indexer/src/test/java/au/org/aodn/esindexer/utils/GeometryUtilsTest.java @@ -5,6 +5,7 @@ import org.geotools.feature.FeatureCollection; import org.geotools.feature.FeatureIterator; import org.geotools.geojson.feature.FeatureJSON; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.locationtech.jts.geom.*; @@ -18,7 +19,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Objects; -import java.util.concurrent.Executors; import static au.org.aodn.esindexer.BaseTestClass.readResourceFile; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -34,9 +34,7 @@ public GeometryUtilsTest() throws JAXBException { @BeforeEach public void init() { GeometryUtils.setCoastalPrecision(0.03); - GeometryUtils.setGridSize(10.0); GeometryUtils.setGridSpatialExtents(false); - GeometryUtils.setExecutorService(Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors())); GeometryUtils.init(); } @@ -48,7 +46,8 @@ public void verifyLandStrippedFromSpatialExtents() throws IOException, JAXBExcep // Whole spatial extends List> withLand = GeometryUtils.createGeometryItems( source, - (rawInput) -> GeometryBase.findPolygonsFrom(GeometryBase.COORDINATE_SYSTEM_CRS84, rawInput) + null, + (rawInput, size) -> GeometryBase.findPolygonsFrom(GeometryBase.COORDINATE_SYSTEM_CRS84, rawInput) ); List> l = Objects.requireNonNull(withLand); @@ -73,7 +72,8 @@ public void verifyLandStrippedFromSpatialExtents() throws IOException, JAXBExcep // Strip the land away. List> noLand = GeometryUtils.createGeometryItems( source, - GeometryUtils::createGeometryWithoutLand + null, + (rawInput, s) -> GeometryUtils.createGeometryWithoutLand(rawInput) ); List> nl = Objects.requireNonNull(noLand); @@ -139,4 +139,44 @@ protected GeometryCollection convertToGeometryCollection( // Create and return a GeometryCollection from the array of geometries return geometryFactory.createGeometryCollection(geometryArray); } + + @Test + public void verifyCreateGirdPolygonWithValidCellSize() { + Envelope envelope = new Envelope(0, 10, 0, 10); // 10x10 envelope + double cellSize = 2.0; // Valid cell size + + List gridPolygons = GeometryUtils.createGridPolygons(envelope, cellSize); + + // Check that the grid has been divided into cells + Assertions.assertFalse(gridPolygons.isEmpty(), "Expected non-empty grid polygons list"); + + // Check the number of cells created (should be 5x5 = 25 for a 10x10 grid with cell size 2) + Assertions.assertEquals(25, gridPolygons.size(), "Expected 25 grid cells"); + + // Check that each cell is a valid polygon and has the expected cell size + gridPolygons.forEach(polygon -> { + Assertions.assertNotNull(polygon, "Expected each grid cell to be a valid polygon"); + Assertions.assertTrue(polygon.getArea() <= (cellSize * cellSize), "Expected each cell to have an area <= cellSize^2"); + }); + } + + @Test + public void verifyCreateGirdPolygonWithTooBigCellSize() { + Envelope envelope = new Envelope(0, 10, 0, 10); // 10x10 envelope + Polygon polygon = GeometryUtils.factory.createPolygon(new Coordinate[]{ + new Coordinate(envelope.getMinX(), envelope.getMinY()), + new Coordinate(envelope.getMaxX(), envelope.getMinY()), + new Coordinate(envelope.getMaxX(), envelope.getMaxY()), + new Coordinate(envelope.getMinX(), envelope.getMaxY()), + new Coordinate(envelope.getMinX(), envelope.getMinY()) // Closing point + }); + double cellSize = 20; // Too small cell size + + // Verify that an exception is thrown for too small cell size + List gridPolygons = GeometryUtils.createGridPolygons(envelope, cellSize); + + // Check that the exception message is as expected + Assertions.assertEquals(1, gridPolygons.size(), "Get 1 back"); + Assertions.assertTrue(gridPolygons.get(0).equalsExact(polygon), "Get back itself"); + } } diff --git a/indexer/src/test/resources/canned/sample7_stac_no_es.json b/indexer/src/test/resources/canned/sample7_stac_no_es.json index 55c80865..35871eee 100644 --- a/indexer/src/test/resources/canned/sample7_stac_no_es.json +++ b/indexer/src/test/resources/canned/sample7_stac_no_es.json @@ -190,7 +190,52 @@ "end": "2022-12-31T12:59:59Z" } ], - "centroid": [] + "centroid": [ + [ + 142.942, + -9.999 + ], + [ + 143.397, + -9.735 + ], + [ + 143.397, + -9.847 + ], + [ + 145.443, + -14.685 + ], + [ + 146.228, + -16.863 + ], + [ + 146.5, + -18.572 + ], + [ + 146.52, + -18.644 + ], + [ + 147.647, + -18.779 + ], + [ + 150.97, + -23.197 + ], + [ + 151.925, + -23.446 + ], + [ + 152.392, + -23.893 + ] + ] }, "contacts": [ { diff --git a/indexer/src/test/resources/canned/sample9_stac.json b/indexer/src/test/resources/canned/sample9_stac.json index e406b29e..0fe6da96 100644 --- a/indexer/src/test/resources/canned/sample9_stac.json +++ b/indexer/src/test/resources/canned/sample9_stac.json @@ -50,7 +50,7 @@ "statement": "", "creation": "2022-02-08T00:00:00", "revision": "2024-03-15T00:00:00", - "update_frequency":"completed", + "update_frequency": "completed", "proj:geometry": { "geometries": [ { @@ -70,11 +70,22 @@ ], "type": "GeometryCollection" }, - "temporal" : [ { - "start" : "2018-10-28T13:00:00Z", - "end" : "2019-11-20T12:59:59Z" - } ], - "centroid" : [ ] + "temporal": [ + { + "start": "2018-10-28T13:00:00Z", + "end": "2019-11-20T12:59:59Z" + } + ], + "centroid": [ + [ + 146.862, + -19.104 + ], + [ + 146.85, + -19.168 + ] + ] }, "contacts": [ { diff --git a/indexer/src/test/resources/canned/sample_multiple_temporal1_stac.json b/indexer/src/test/resources/canned/sample_multiple_temporal1_stac.json index 966cd211..1b65af7f 100644 --- a/indexer/src/test/resources/canned/sample_multiple_temporal1_stac.json +++ b/indexer/src/test/resources/canned/sample_multiple_temporal1_stac.json @@ -141,7 +141,7 @@ "statement": "", "creation": "2009-11-17T00:00:00", "revision": "2024-08-02T04:43:08.277909Z", - "update_frequency":"other", + "update_frequency": "other", "proj:geometry": { "geometries": [ { @@ -259,14 +259,66 @@ ], "type": "GeometryCollection" }, - "temporal" : [ { - "start" : "1980-02-27T13:00:00Z", - "end" : "1982-02-28T12:59:59Z" - }, { - "start" : "1984-02-27T13:00:00Z", - "end" : null - } ], - "centroid" : [ ] + "temporal": [ + { + "start": "1980-02-27T13:00:00Z", + "end": "1982-02-28T12:59:59Z" + }, + { + "start": "1984-02-27T13:00:00Z", + "end": null + } + ], + "centroid": [ + [ + 146.483, + -18.613 + ], + [ + 146.879, + -18.478 + ], + [ + 147.067, + -18.633 + ], + [ + 147.9, + -18.8 + ], + [ + 147.4, + -19.3 + ], + [ + 145.823, + -15.962 + ], + [ + 149.181, + -19.74 + ], + [ + 147.383, + -18.275 + ], + [ + 146.881, + -19.156 + ], + [ + 150.939, + -23.154 + ], + [ + 147.015, + -19.185 + ], + [ + 143.069, + -10.051 + ] + ] }, "contacts": [ { diff --git a/indexer/src/test/resources/canned/sample_multiple_temporal2_stac.json b/indexer/src/test/resources/canned/sample_multiple_temporal2_stac.json index 32462237..68dd6f16 100644 --- a/indexer/src/test/resources/canned/sample_multiple_temporal2_stac.json +++ b/indexer/src/test/resources/canned/sample_multiple_temporal2_stac.json @@ -141,7 +141,7 @@ "statement": "", "creation": "2009-11-17T00:00:00", "revision": "2024-08-02T04:43:08.277909Z", - "update_frequency":"other", + "update_frequency": "other", "proj:geometry": { "geometries": [ { @@ -259,14 +259,66 @@ ], "type": "GeometryCollection" }, - "temporal" : [ { - "start" : "1980-02-27T13:00:00Z", - "end" : "1982-02-28T12:59:59Z" - }, { - "start" : "1984-02-27T13:00:00Z", - "end" : "1985-02-28T12:59:59Z" - } ], - "centroid" : [ ] + "temporal": [ + { + "start": "1980-02-27T13:00:00Z", + "end": "1982-02-28T12:59:59Z" + }, + { + "start": "1984-02-27T13:00:00Z", + "end": "1985-02-28T12:59:59Z" + } + ], + "centroid": [ + [ + 146.483, + -18.613 + ], + [ + 146.879, + -18.478 + ], + [ + 147.067, + -18.633 + ], + [ + 147.9, + -18.8 + ], + [ + 147.4, + -19.3 + ], + [ + 145.823, + -15.962 + ], + [ + 149.181, + -19.74 + ], + [ + 147.383, + -18.275 + ], + [ + 146.881, + -19.156 + ], + [ + 150.939, + -23.154 + ], + [ + 147.015, + -19.185 + ], + [ + 143.069, + -10.051 + ] + ] }, "contacts": [ { From c7cc58abf7439077ee52eabf4f200102d5695b7a Mon Sep 17 00:00:00 2001 From: utas-raymondng Date: Fri, 1 Nov 2024 15:40:03 +1100 Subject: [PATCH 2/2] Fix test case --- .../test/resources/canned/sample7_stac.json | 47 ++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/indexer/src/test/resources/canned/sample7_stac.json b/indexer/src/test/resources/canned/sample7_stac.json index db235d54..ad707c3f 100644 --- a/indexer/src/test/resources/canned/sample7_stac.json +++ b/indexer/src/test/resources/canned/sample7_stac.json @@ -191,7 +191,52 @@ "end": "2022-12-31T12:59:59Z" } ], - "centroid": [] + "centroid": [ + [ + 142.942, + -9.999 + ], + [ + 143.397, + -9.735 + ], + [ + 143.397, + -9.847 + ], + [ + 145.443, + -14.685 + ], + [ + 146.228, + -16.863 + ], + [ + 146.5, + -18.572 + ], + [ + 146.52, + -18.644 + ], + [ + 147.647, + -18.779 + ], + [ + 150.97, + -23.197 + ], + [ + 151.925, + -23.446 + ], + [ + 152.392, + -23.893 + ] + ] }, "contacts": [ {