From 873e6073652e2921dd8d6a8a45764f426175967a Mon Sep 17 00:00:00 2001 From: utas-raymondng Date: Thu, 21 Nov 2024 09:05:54 +1100 Subject: [PATCH 1/4] Add reducer support --- .../configuration/IndexerConfig.java | 4 ++++ .../aodn/esindexer/utils/GeometryUtils.java | 22 +++++++++++++++---- .../src/main/resources/application-dev.yaml | 1 + .../src/main/resources/application-edge.yaml | 1 + .../resources/application-production.yaml | 4 ++++ .../main/resources/application-staging.yaml | 4 ++++ 6 files changed, 32 insertions(+), 4 deletions(-) diff --git a/indexer/src/main/java/au/org/aodn/esindexer/configuration/IndexerConfig.java b/indexer/src/main/java/au/org/aodn/esindexer/configuration/IndexerConfig.java index 89e9bb57..e8bf7247 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/configuration/IndexerConfig.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/configuration/IndexerConfig.java @@ -26,9 +26,13 @@ public class IndexerConfig { @Value("${app.geometry.coastalPrecision:0.5}") protected double coastalPrecision; + @Value("${app.geometry.reducerPrecision:#{null}}") + protected Double reducerPrecision; + @PostConstruct public void init() { GeometryUtils.setCoastalPrecision(coastalPrecision); + GeometryUtils.setReducerPrecision(reducerPrecision); GeometryUtils.init(); } diff --git a/indexer/src/main/java/au/org/aodn/esindexer/utils/GeometryUtils.java b/indexer/src/main/java/au/org/aodn/esindexer/utils/GeometryUtils.java index 2888f5e9..b45bc9e0 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/utils/GeometryUtils.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/utils/GeometryUtils.java @@ -8,6 +8,7 @@ import org.apache.logging.log4j.Logger; import org.geotools.data.shapefile.ShapefileDataStore; import org.geotools.data.shapefile.ShapefileDataStoreFactory; +import org.locationtech.jts.precision.GeometryPrecisionReducer; import org.geotools.data.simple.SimpleFeatureCollection; import org.geotools.data.simple.SimpleFeatureIterator; import org.geotools.data.simple.SimpleFeatureSource; @@ -45,6 +46,12 @@ public enum PointOrientation { @Getter @Setter protected static double coastalPrecision = 0.1; + // A value based on trial and error, to simplify the polygon to avoid complex geojson that takes time to transfer + // By default is set to null to disable it so test case do not need to change, but each env have this value set + // to reduce processing time. + @Getter + @Setter + protected static Double reducerPrecision = null; // Load a coastline shape file so that we can get a spatial extents that cover sea only public static void init() { @@ -63,6 +70,12 @@ public static void init() { SimpleFeatureCollection featureCollection = featureSource.getFeatures(); List geometries = new ArrayList<>(); + GeometryPrecisionReducer reducer = null; + if(getReducerPrecision() != null) { + PrecisionModel pm = new PrecisionModel(getReducerPrecision()); // 1 / 1000 meters ~= 1km + reducer = new GeometryPrecisionReducer(pm); + } + try (SimpleFeatureIterator iterator = featureCollection.features()) { while (iterator.hasNext()) { SimpleFeature feature = iterator.next(); @@ -75,7 +88,7 @@ public static void init() { Geometry simplifiedGeometry = DouglasPeuckerSimplifier .simplify(landFeatureGeometry, getCoastalPrecision()); // Adjust tolerance - geometries.add(simplifiedGeometry); + geometries.add(reducer != null ? reducer.reduce(simplifiedGeometry) : simplifiedGeometry); } } // Faster to use union list rather than union by geometry one by one. @@ -347,9 +360,10 @@ protected static List> createGeometryWithoutLand(List createGeometryFrom(List> rawInput, Integer gridSize) { // The return polygon is in EPSG:4326, so we can call createGeoJson directly - // This line will cause the spatial extents to break into grid, it may help to debug but will make production - // slow and sometimes cause polygon break. - // List> polygonNoLand = splitAreaToGrid(createGeometryWithoutLand(rawInput)); + // Un-remark this line and remark the line above if you want to visualize the polygon on map, change this + // line will cause the spatial extends draw on map with land removed. + // List> polygon = createGeometryWithoutLand(rawInput); + List> polygon = GeometryBase.findPolygonsFrom(GeometryBase.COORDINATE_SYSTEM_CRS84, rawInput); return (polygon != null && !polygon.isEmpty()) ? createGeoJson(polygon) : null; } diff --git a/indexer/src/main/resources/application-dev.yaml b/indexer/src/main/resources/application-dev.yaml index 15c0b4be..da40d7ea 100644 --- a/indexer/src/main/resources/application-dev.yaml +++ b/indexer/src/main/resources/application-dev.yaml @@ -4,6 +4,7 @@ server: app: geometry: enableGridSpatialExtents: true + reducerPrecision: 2.0 elasticsearch: index: diff --git a/indexer/src/main/resources/application-edge.yaml b/indexer/src/main/resources/application-edge.yaml index e00d5acd..b72a4220 100644 --- a/indexer/src/main/resources/application-edge.yaml +++ b/indexer/src/main/resources/application-edge.yaml @@ -7,6 +7,7 @@ spring: app: geometry: enableGridSpatialExtents: true + reducerPrecision: 2.0 management: endpoints: diff --git a/indexer/src/main/resources/application-production.yaml b/indexer/src/main/resources/application-production.yaml index f25750bb..fa654c6d 100644 --- a/indexer/src/main/resources/application-production.yaml +++ b/indexer/src/main/resources/application-production.yaml @@ -1,3 +1,7 @@ +app: + geometry: + reducerPrecision: 2.0 + management: endpoints: web: diff --git a/indexer/src/main/resources/application-staging.yaml b/indexer/src/main/resources/application-staging.yaml index f25750bb..fa654c6d 100644 --- a/indexer/src/main/resources/application-staging.yaml +++ b/indexer/src/main/resources/application-staging.yaml @@ -1,3 +1,7 @@ +app: + geometry: + reducerPrecision: 2.0 + management: endpoints: web: From 0f4a08463c137f63b6349b670aef55f00218e9fd Mon Sep 17 00:00:00 2001 From: utas-raymondng Date: Fri, 22 Nov 2024 09:21:01 +1100 Subject: [PATCH 2/4] Refactor to move method out of GeometryUtils --- .../org/aodn/esindexer/utils/FileUtils.java | 32 +++++++++++++++++++ .../aodn/esindexer/utils/GeometryUtils.java | 27 ++-------------- 2 files changed, 34 insertions(+), 25 deletions(-) create mode 100644 indexer/src/main/java/au/org/aodn/esindexer/utils/FileUtils.java diff --git a/indexer/src/main/java/au/org/aodn/esindexer/utils/FileUtils.java b/indexer/src/main/java/au/org/aodn/esindexer/utils/FileUtils.java new file mode 100644 index 00000000..556cccb4 --- /dev/null +++ b/indexer/src/main/java/au/org/aodn/esindexer/utils/FileUtils.java @@ -0,0 +1,32 @@ +package au.org.aodn.esindexer.utils; + +import org.springframework.core.io.ClassPathResource; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; + +public class FileUtils { + public static File saveResourceToTemp(String resourceName, String filename) { + String tempDir = System.getProperty("java.io.tmpdir"); + ClassPathResource resource = new ClassPathResource(resourceName); + + File tempFile = new File(tempDir, filename); + try(InputStream input = resource.getInputStream()) { + tempFile.deleteOnExit(); // Ensure the file is deleted when the JVM exits + + // Write the InputStream to the temporary file + try (FileOutputStream outputStream = new FileOutputStream(tempFile)) { + byte[] buffer = new byte[1024]; + int bytesRead; + while ((bytesRead = input.read(buffer)) != -1) { + outputStream.write(buffer, 0, bytesRead); + } + } + } catch (IOException e) { + throw new RuntimeException(e); + } + return tempFile; + } +} diff --git a/indexer/src/main/java/au/org/aodn/esindexer/utils/GeometryUtils.java b/indexer/src/main/java/au/org/aodn/esindexer/utils/GeometryUtils.java index b45bc9e0..f3ed1243 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/utils/GeometryUtils.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/utils/GeometryUtils.java @@ -18,7 +18,6 @@ import org.locationtech.jts.operation.union.UnaryUnionOp; import org.locationtech.jts.simplify.DouglasPeuckerSimplifier; import org.opengis.feature.simple.SimpleFeature; -import org.springframework.core.io.ClassPathResource; import java.io.*; import java.net.URL; @@ -57,8 +56,8 @@ public enum PointOrientation { public static void init() { try { // shp file depends on shx, so need to have shx appear in temp folder. - saveResourceToTemp("land/ne_10m_land.shx", "shapefile.shx"); - File tempFile = saveResourceToTemp("land/ne_10m_land.shp", "shapefile.shp"); + FileUtils.saveResourceToTemp("land/ne_10m_land.shx", "shapefile.shx"); + File tempFile = FileUtils.saveResourceToTemp("land/ne_10m_land.shp", "shapefile.shp"); // Load the shapefile from the temporary file using ShapefileDataStore URL tempFileUrl = tempFile.toURI().toURL(); @@ -98,28 +97,6 @@ public static void init() { throw new RuntimeException(ioe); } } - - protected static File saveResourceToTemp(String resourceName, String filename) { - String tempDir = System.getProperty("java.io.tmpdir"); - ClassPathResource resource = new ClassPathResource(resourceName); - - File tempFile = new File(tempDir, filename); - try(InputStream input = resource.getInputStream()) { - tempFile.deleteOnExit(); // Ensure the file is deleted when the JVM exits - - // Write the InputStream to the temporary file - try (FileOutputStream outputStream = new FileOutputStream(tempFile)) { - byte[] buffer = new byte[1024]; - int bytesRead; - while ((bytesRead = input.read(buffer)) != -1) { - outputStream.write(buffer, 0, bytesRead); - } - } - } catch (IOException e) { - throw new RuntimeException(e); - } - return tempFile; - } /** * @param polygons - Assume to be EPSG:4326, as GeoJson always use this encoding. * @return From 5eb1801703d8fd061ecf9d962a117b7d2a3d1fa5 Mon Sep 17 00:00:00 2001 From: utas-raymondng Date: Fri, 22 Nov 2024 10:21:18 +1100 Subject: [PATCH 3/4] Increase the default reducer precision and add test --- .../aodn/esindexer/utils/GeometryUtils.java | 6 ++- .../src/main/resources/application-dev.yaml | 2 +- .../src/main/resources/application-edge.yaml | 2 +- .../resources/application-production.yaml | 2 +- .../main/resources/application-staging.yaml | 2 +- .../esindexer/utils/GeometryUtilsTest.java | 47 +++++++++++++++++++ 6 files changed, 55 insertions(+), 6 deletions(-) diff --git a/indexer/src/main/java/au/org/aodn/esindexer/utils/GeometryUtils.java b/indexer/src/main/java/au/org/aodn/esindexer/utils/GeometryUtils.java index f3ed1243..69f772a9 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/utils/GeometryUtils.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/utils/GeometryUtils.java @@ -52,6 +52,8 @@ public enum PointOrientation { @Setter protected static Double reducerPrecision = null; + protected static GeometryPrecisionReducer reducer = null; + // Load a coastline shape file so that we can get a spatial extents that cover sea only public static void init() { try { @@ -69,7 +71,6 @@ public static void init() { SimpleFeatureCollection featureCollection = featureSource.getFeatures(); List geometries = new ArrayList<>(); - GeometryPrecisionReducer reducer = null; if(getReducerPrecision() != null) { PrecisionModel pm = new PrecisionModel(getReducerPrecision()); // 1 / 1000 meters ~= 1km reducer = new GeometryPrecisionReducer(pm); @@ -242,6 +243,7 @@ protected static List> removeLandAreaFromGeometry(List geometry.isValid() ? geometry : geometry.buffer(0)) .map(geometry -> geometry.difference(landGeometry)) + .map(geometry -> reducer != null ? reducer.reduce(geometry) : geometry) .map(GeometryUtils::convertToListGeometry) .flatMap(Collection::stream) .toList() @@ -337,7 +339,7 @@ protected static List> createGeometryWithoutLand(List createGeometryFrom(List> rawInput, Integer gridSize) { // The return polygon is in EPSG:4326, so we can call createGeoJson directly - // Un-remark this line and remark the line above if you want to visualize the polygon on map, change this + // Un-remark this line and remark the line below if you want to visualize the polygon on map, change this // line will cause the spatial extends draw on map with land removed. // List> polygon = createGeometryWithoutLand(rawInput); diff --git a/indexer/src/main/resources/application-dev.yaml b/indexer/src/main/resources/application-dev.yaml index da40d7ea..10891bab 100644 --- a/indexer/src/main/resources/application-dev.yaml +++ b/indexer/src/main/resources/application-dev.yaml @@ -4,7 +4,7 @@ server: app: geometry: enableGridSpatialExtents: true - reducerPrecision: 2.0 + reducerPrecision: 4.0 elasticsearch: index: diff --git a/indexer/src/main/resources/application-edge.yaml b/indexer/src/main/resources/application-edge.yaml index b72a4220..ae48bcf4 100644 --- a/indexer/src/main/resources/application-edge.yaml +++ b/indexer/src/main/resources/application-edge.yaml @@ -7,7 +7,7 @@ spring: app: geometry: enableGridSpatialExtents: true - reducerPrecision: 2.0 + reducerPrecision: 4.0 management: endpoints: diff --git a/indexer/src/main/resources/application-production.yaml b/indexer/src/main/resources/application-production.yaml index fa654c6d..4f1f365d 100644 --- a/indexer/src/main/resources/application-production.yaml +++ b/indexer/src/main/resources/application-production.yaml @@ -1,6 +1,6 @@ app: geometry: - reducerPrecision: 2.0 + reducerPrecision: 4.0 management: endpoints: diff --git a/indexer/src/main/resources/application-staging.yaml b/indexer/src/main/resources/application-staging.yaml index fa654c6d..4f1f365d 100644 --- a/indexer/src/main/resources/application-staging.yaml +++ b/indexer/src/main/resources/application-staging.yaml @@ -1,6 +1,6 @@ app: geometry: - reducerPrecision: 2.0 + reducerPrecision: 4.0 management: endpoints: diff --git a/indexer/src/test/java/au/org/aodn/esindexer/utils/GeometryUtilsTest.java b/indexer/src/test/java/au/org/aodn/esindexer/utils/GeometryUtilsTest.java index e913b192..f651a962 100644 --- a/indexer/src/test/java/au/org/aodn/esindexer/utils/GeometryUtilsTest.java +++ b/indexer/src/test/java/au/org/aodn/esindexer/utils/GeometryUtilsTest.java @@ -87,4 +87,51 @@ public void verifyLandStrippedFromSpatialExtents() throws IOException, JAXBExcep assertEquals(126.0, ncoors[3].getX(), 0.01); assertEquals(-35.9999, ncoors[3].getY(), 0.01); } + /** + * This test turn on the reducer to further reduce the complexity of the land area after + * DouglasPeuckerSimplifier simplifier, this further reduce the number of digit to get a smaller geojson + * which should improve transfer speed. + * + * @throws IOException - Not expect to throw + * @throws JAXBException - Not expect to throw + */ + @Test + public void verifyLandStrippedFromSpatialExtentsWithReducerOn() throws IOException, JAXBException { + GeometryUtils.setReducerPrecision(4.0); + GeometryUtils.init(); + + String xml = readResourceFile("classpath:canned/sample_complex_area.xml"); + MDMetadataType source = jaxb.unmarshal(xml); + + // Strip the land away. + List> noLand = GeometryUtils.createGeometryItems( + source, + (rawInput, s) -> GeometryUtils.createGeometryWithoutLand(rawInput), + null + ); + + List> nl = Objects.requireNonNull(noLand); + + assertEquals(nl.size(),1, "No Land have 1 polygon array"); + assertEquals(16, nl.get(0).size(), "Size 16 with land"); + + Geometry nle = nl.get(0).get(0).getEnvelope(); + Coordinate[] ncoors = nle.getCoordinates(); + + // The envelope of the two polygon should match given one is the original and the other just strip the land + assertEquals(118.0, ncoors[0].getX(), 0.00); + assertEquals(-36.0, ncoors[0].getY(), 0.00); + + assertEquals(118.0, ncoors[1].getX(), 0.00); + assertEquals(-32.25, ncoors[1].getY(), 0.00); + + assertEquals(126, ncoors[2].getX(), 0.00); + assertEquals(-32.25, ncoors[2].getY(), 0.00); + + assertEquals(126.0, ncoors[3].getX(), 0.00); + assertEquals(-36.0, ncoors[3].getY(), 0.00); + + assertEquals(118.0, ncoors[4].getX(), 0.00); + assertEquals(-36.0, ncoors[4].getY(), 0.00); + } } From 770c0f6b52ff64e120d295ff52a72bcfb357b216 Mon Sep 17 00:00:00 2001 From: utas-raymondng Date: Fri, 22 Nov 2024 10:35:12 +1100 Subject: [PATCH 4/4] Fix test --- .../main/java/au/org/aodn/esindexer/utils/GeometryUtils.java | 5 ++++- .../java/au/org/aodn/esindexer/utils/GeometryUtilsTest.java | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/indexer/src/main/java/au/org/aodn/esindexer/utils/GeometryUtils.java b/indexer/src/main/java/au/org/aodn/esindexer/utils/GeometryUtils.java index 69f772a9..b20e9800 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/utils/GeometryUtils.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/utils/GeometryUtils.java @@ -55,8 +55,11 @@ public enum PointOrientation { protected static GeometryPrecisionReducer reducer = null; // Load a coastline shape file so that we can get a spatial extents that cover sea only - public static void init() { + public static synchronized void init() { try { + // Reset reducer + reducer = null; + // shp file depends on shx, so need to have shx appear in temp folder. FileUtils.saveResourceToTemp("land/ne_10m_land.shx", "shapefile.shx"); File tempFile = FileUtils.saveResourceToTemp("land/ne_10m_land.shp", "shapefile.shp"); diff --git a/indexer/src/test/java/au/org/aodn/esindexer/utils/GeometryUtilsTest.java b/indexer/src/test/java/au/org/aodn/esindexer/utils/GeometryUtilsTest.java index f651a962..ebeda929 100644 --- a/indexer/src/test/java/au/org/aodn/esindexer/utils/GeometryUtilsTest.java +++ b/indexer/src/test/java/au/org/aodn/esindexer/utils/GeometryUtilsTest.java @@ -26,11 +26,13 @@ public GeometryUtilsTest() throws JAXBException { @BeforeEach public void init() { GeometryUtils.setCoastalPrecision(0.03); - GeometryUtils.init(); } @Test public void verifyLandStrippedFromSpatialExtents() throws IOException, JAXBException { + GeometryUtils.setReducerPrecision(null); + GeometryUtils.init(); + String xml = readResourceFile("classpath:canned/sample_complex_area.xml"); MDMetadataType source = jaxb.unmarshal(xml); // Whole spatial extends