diff --git a/indexer/src/main/java/au/org/aodn/esindexer/configuration/IndexerConfig.java b/indexer/src/main/java/au/org/aodn/esindexer/configuration/IndexerConfig.java index 89e9bb57..e8bf7247 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/configuration/IndexerConfig.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/configuration/IndexerConfig.java @@ -26,9 +26,13 @@ public class IndexerConfig { @Value("${app.geometry.coastalPrecision:0.5}") protected double coastalPrecision; + @Value("${app.geometry.reducerPrecision:#{null}}") + protected Double reducerPrecision; + @PostConstruct public void init() { GeometryUtils.setCoastalPrecision(coastalPrecision); + GeometryUtils.setReducerPrecision(reducerPrecision); GeometryUtils.init(); } diff --git a/indexer/src/main/java/au/org/aodn/esindexer/utils/FileUtils.java b/indexer/src/main/java/au/org/aodn/esindexer/utils/FileUtils.java new file mode 100644 index 00000000..556cccb4 --- /dev/null +++ b/indexer/src/main/java/au/org/aodn/esindexer/utils/FileUtils.java @@ -0,0 +1,32 @@ +package au.org.aodn.esindexer.utils; + +import org.springframework.core.io.ClassPathResource; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; + +public class FileUtils { + public static File saveResourceToTemp(String resourceName, String filename) { + String tempDir = System.getProperty("java.io.tmpdir"); + ClassPathResource resource = new ClassPathResource(resourceName); + + File tempFile = new File(tempDir, filename); + try(InputStream input = resource.getInputStream()) { + tempFile.deleteOnExit(); // Ensure the file is deleted when the JVM exits + + // Write the InputStream to the temporary file + try (FileOutputStream outputStream = new FileOutputStream(tempFile)) { + byte[] buffer = new byte[1024]; + int bytesRead; + while ((bytesRead = input.read(buffer)) != -1) { + outputStream.write(buffer, 0, bytesRead); + } + } + } catch (IOException e) { + throw new RuntimeException(e); + } + return tempFile; + } +} diff --git a/indexer/src/main/java/au/org/aodn/esindexer/utils/GeometryUtils.java b/indexer/src/main/java/au/org/aodn/esindexer/utils/GeometryUtils.java index 2888f5e9..b20e9800 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/utils/GeometryUtils.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/utils/GeometryUtils.java @@ -8,6 +8,7 @@ import org.apache.logging.log4j.Logger; import org.geotools.data.shapefile.ShapefileDataStore; import org.geotools.data.shapefile.ShapefileDataStoreFactory; +import org.locationtech.jts.precision.GeometryPrecisionReducer; import org.geotools.data.simple.SimpleFeatureCollection; import org.geotools.data.simple.SimpleFeatureIterator; import org.geotools.data.simple.SimpleFeatureSource; @@ -17,7 +18,6 @@ import org.locationtech.jts.operation.union.UnaryUnionOp; import org.locationtech.jts.simplify.DouglasPeuckerSimplifier; import org.opengis.feature.simple.SimpleFeature; -import org.springframework.core.io.ClassPathResource; import java.io.*; import java.net.URL; @@ -45,13 +45,24 @@ public enum PointOrientation { @Getter @Setter protected static double coastalPrecision = 0.1; + // A value based on trial and error, to simplify the polygon to avoid complex geojson that takes time to transfer + // By default is set to null to disable it so test case do not need to change, but each env have this value set + // to reduce processing time. + @Getter + @Setter + protected static Double reducerPrecision = null; + + protected static GeometryPrecisionReducer reducer = null; // Load a coastline shape file so that we can get a spatial extents that cover sea only - public static void init() { + public static synchronized void init() { try { + // Reset reducer + reducer = null; + // shp file depends on shx, so need to have shx appear in temp folder. - saveResourceToTemp("land/ne_10m_land.shx", "shapefile.shx"); - File tempFile = saveResourceToTemp("land/ne_10m_land.shp", "shapefile.shp"); + FileUtils.saveResourceToTemp("land/ne_10m_land.shx", "shapefile.shx"); + File tempFile = FileUtils.saveResourceToTemp("land/ne_10m_land.shp", "shapefile.shp"); // Load the shapefile from the temporary file using ShapefileDataStore URL tempFileUrl = tempFile.toURI().toURL(); @@ -63,6 +74,11 @@ public static void init() { SimpleFeatureCollection featureCollection = featureSource.getFeatures(); List geometries = new ArrayList<>(); + if(getReducerPrecision() != null) { + PrecisionModel pm = new PrecisionModel(getReducerPrecision()); // 1 / 1000 meters ~= 1km + reducer = new GeometryPrecisionReducer(pm); + } + try (SimpleFeatureIterator iterator = featureCollection.features()) { while (iterator.hasNext()) { SimpleFeature feature = iterator.next(); @@ -75,7 +91,7 @@ public static void init() { Geometry simplifiedGeometry = DouglasPeuckerSimplifier .simplify(landFeatureGeometry, getCoastalPrecision()); // Adjust tolerance - geometries.add(simplifiedGeometry); + geometries.add(reducer != null ? reducer.reduce(simplifiedGeometry) : simplifiedGeometry); } } // Faster to use union list rather than union by geometry one by one. @@ -85,28 +101,6 @@ public static void init() { throw new RuntimeException(ioe); } } - - protected static File saveResourceToTemp(String resourceName, String filename) { - String tempDir = System.getProperty("java.io.tmpdir"); - ClassPathResource resource = new ClassPathResource(resourceName); - - File tempFile = new File(tempDir, filename); - try(InputStream input = resource.getInputStream()) { - tempFile.deleteOnExit(); // Ensure the file is deleted when the JVM exits - - // Write the InputStream to the temporary file - try (FileOutputStream outputStream = new FileOutputStream(tempFile)) { - byte[] buffer = new byte[1024]; - int bytesRead; - while ((bytesRead = input.read(buffer)) != -1) { - outputStream.write(buffer, 0, bytesRead); - } - } - } catch (IOException e) { - throw new RuntimeException(e); - } - return tempFile; - } /** * @param polygons - Assume to be EPSG:4326, as GeoJson always use this encoding. * @return @@ -252,6 +246,7 @@ protected static List> removeLandAreaFromGeometry(List geometry.isValid() ? geometry : geometry.buffer(0)) .map(geometry -> geometry.difference(landGeometry)) + .map(geometry -> reducer != null ? reducer.reduce(geometry) : geometry) .map(GeometryUtils::convertToListGeometry) .flatMap(Collection::stream) .toList() @@ -347,9 +342,10 @@ protected static List> createGeometryWithoutLand(List createGeometryFrom(List> rawInput, Integer gridSize) { // The return polygon is in EPSG:4326, so we can call createGeoJson directly - // This line will cause the spatial extents to break into grid, it may help to debug but will make production - // slow and sometimes cause polygon break. - // List> polygonNoLand = splitAreaToGrid(createGeometryWithoutLand(rawInput)); + // Un-remark this line and remark the line below if you want to visualize the polygon on map, change this + // line will cause the spatial extends draw on map with land removed. + // List> polygon = createGeometryWithoutLand(rawInput); + List> polygon = GeometryBase.findPolygonsFrom(GeometryBase.COORDINATE_SYSTEM_CRS84, rawInput); return (polygon != null && !polygon.isEmpty()) ? createGeoJson(polygon) : null; } diff --git a/indexer/src/main/resources/application-dev.yaml b/indexer/src/main/resources/application-dev.yaml index 15c0b4be..10891bab 100644 --- a/indexer/src/main/resources/application-dev.yaml +++ b/indexer/src/main/resources/application-dev.yaml @@ -4,6 +4,7 @@ server: app: geometry: enableGridSpatialExtents: true + reducerPrecision: 4.0 elasticsearch: index: diff --git a/indexer/src/main/resources/application-edge.yaml b/indexer/src/main/resources/application-edge.yaml index e00d5acd..ae48bcf4 100644 --- a/indexer/src/main/resources/application-edge.yaml +++ b/indexer/src/main/resources/application-edge.yaml @@ -7,6 +7,7 @@ spring: app: geometry: enableGridSpatialExtents: true + reducerPrecision: 4.0 management: endpoints: diff --git a/indexer/src/main/resources/application-production.yaml b/indexer/src/main/resources/application-production.yaml index f25750bb..4f1f365d 100644 --- a/indexer/src/main/resources/application-production.yaml +++ b/indexer/src/main/resources/application-production.yaml @@ -1,3 +1,7 @@ +app: + geometry: + reducerPrecision: 4.0 + management: endpoints: web: diff --git a/indexer/src/main/resources/application-staging.yaml b/indexer/src/main/resources/application-staging.yaml index f25750bb..4f1f365d 100644 --- a/indexer/src/main/resources/application-staging.yaml +++ b/indexer/src/main/resources/application-staging.yaml @@ -1,3 +1,7 @@ +app: + geometry: + reducerPrecision: 4.0 + management: endpoints: web: diff --git a/indexer/src/test/java/au/org/aodn/esindexer/utils/GeometryUtilsTest.java b/indexer/src/test/java/au/org/aodn/esindexer/utils/GeometryUtilsTest.java index e913b192..ebeda929 100644 --- a/indexer/src/test/java/au/org/aodn/esindexer/utils/GeometryUtilsTest.java +++ b/indexer/src/test/java/au/org/aodn/esindexer/utils/GeometryUtilsTest.java @@ -26,11 +26,13 @@ public GeometryUtilsTest() throws JAXBException { @BeforeEach public void init() { GeometryUtils.setCoastalPrecision(0.03); - GeometryUtils.init(); } @Test public void verifyLandStrippedFromSpatialExtents() throws IOException, JAXBException { + GeometryUtils.setReducerPrecision(null); + GeometryUtils.init(); + String xml = readResourceFile("classpath:canned/sample_complex_area.xml"); MDMetadataType source = jaxb.unmarshal(xml); // Whole spatial extends @@ -87,4 +89,51 @@ public void verifyLandStrippedFromSpatialExtents() throws IOException, JAXBExcep assertEquals(126.0, ncoors[3].getX(), 0.01); assertEquals(-35.9999, ncoors[3].getY(), 0.01); } + /** + * This test turn on the reducer to further reduce the complexity of the land area after + * DouglasPeuckerSimplifier simplifier, this further reduce the number of digit to get a smaller geojson + * which should improve transfer speed. + * + * @throws IOException - Not expect to throw + * @throws JAXBException - Not expect to throw + */ + @Test + public void verifyLandStrippedFromSpatialExtentsWithReducerOn() throws IOException, JAXBException { + GeometryUtils.setReducerPrecision(4.0); + GeometryUtils.init(); + + String xml = readResourceFile("classpath:canned/sample_complex_area.xml"); + MDMetadataType source = jaxb.unmarshal(xml); + + // Strip the land away. + List> noLand = GeometryUtils.createGeometryItems( + source, + (rawInput, s) -> GeometryUtils.createGeometryWithoutLand(rawInput), + null + ); + + List> nl = Objects.requireNonNull(noLand); + + assertEquals(nl.size(),1, "No Land have 1 polygon array"); + assertEquals(16, nl.get(0).size(), "Size 16 with land"); + + Geometry nle = nl.get(0).get(0).getEnvelope(); + Coordinate[] ncoors = nle.getCoordinates(); + + // The envelope of the two polygon should match given one is the original and the other just strip the land + assertEquals(118.0, ncoors[0].getX(), 0.00); + assertEquals(-36.0, ncoors[0].getY(), 0.00); + + assertEquals(118.0, ncoors[1].getX(), 0.00); + assertEquals(-32.25, ncoors[1].getY(), 0.00); + + assertEquals(126, ncoors[2].getX(), 0.00); + assertEquals(-32.25, ncoors[2].getY(), 0.00); + + assertEquals(126.0, ncoors[3].getX(), 0.00); + assertEquals(-36.0, ncoors[3].getY(), 0.00); + + assertEquals(118.0, ncoors[4].getX(), 0.00); + assertEquals(-36.0, ncoors[4].getY(), 0.00); + } }