diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index d35b840d3..16a956d9f 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -18,6 +18,9 @@ jobs: uses: actions/setup-java@v1 with: java-version: '17' + - name: Add hosts to /etc/hosts + run: | + sudo echo "127.0.0.1 dynamodb postgres.local postgres postgres.replica" | sudo tee -a /etc/hosts - name: Start the XYZ Hub stack run: mvn clean install -Pdocker -DskipTests=true -DdockerComposeFile=docker-compose-dynamodb.yml - name: Deploy StepLambda for Job API diff --git a/xyz-jobs/xyz-job-service/src/main/java/com/here/xyz/jobs/datasets/DatasetDescription.java b/xyz-jobs/xyz-job-service/src/main/java/com/here/xyz/jobs/datasets/DatasetDescription.java index a069ada11..b43f7430a 100644 --- a/xyz-jobs/xyz-job-service/src/main/java/com/here/xyz/jobs/datasets/DatasetDescription.java +++ b/xyz-jobs/xyz-job-service/src/main/java/com/here/xyz/jobs/datasets/DatasetDescription.java @@ -77,9 +77,9 @@ public Map withVersionRef(Ref versionRef) { @JsonIgnoreProperties(ignoreUnknown = true) public static class Space extends Identifiable implements FilteringSource, VersionedSource { - @JsonView({Public.class}) + @JsonView({Public.class, Static.class}) private Filters filters; - @JsonView({Public.class}) + @JsonView({Public.class, Static.class}) private Ref versionRef; @Override diff --git a/xyz-jobs/xyz-job-service/src/main/java/com/here/xyz/jobs/datasets/Spaces.java b/xyz-jobs/xyz-job-service/src/main/java/com/here/xyz/jobs/datasets/Spaces.java index 64683425f..eb29e99f3 100644 --- a/xyz-jobs/xyz-job-service/src/main/java/com/here/xyz/jobs/datasets/Spaces.java +++ b/xyz-jobs/xyz-job-service/src/main/java/com/here/xyz/jobs/datasets/Spaces.java @@ -19,6 +19,7 @@ package com.here.xyz.jobs.datasets; +import com.fasterxml.jackson.annotation.JsonView; import com.here.xyz.jobs.datasets.DatasetDescription.Space; import com.here.xyz.jobs.datasets.filters.FilteringSource; import com.here.xyz.jobs.datasets.filters.Filters; @@ -28,6 +29,8 @@ public class Spaces extends DatasetDescription implements FilteringSource, CombinedDatasetDescription { private List spaceIds; + + @JsonView({Public.class, Static.class}) private Filters filters; public List getSpaceIds() { diff --git a/xyz-jobs/xyz-job-service/src/main/java/com/here/xyz/jobs/datasets/filters/Filters.java b/xyz-jobs/xyz-job-service/src/main/java/com/here/xyz/jobs/datasets/filters/Filters.java index eaff9237e..66c0a68a4 100644 --- a/xyz-jobs/xyz-job-service/src/main/java/com/here/xyz/jobs/datasets/filters/Filters.java +++ b/xyz-jobs/xyz-job-service/src/main/java/com/here/xyz/jobs/datasets/filters/Filters.java @@ -40,7 +40,7 @@ public class Filters { @JsonView({Internal.class, Static.class}) private String propertyFilterAsString; - @JsonView({Public.class}) + @JsonView({Public.class, Static.class}) private SpatialFilter spatialFilter; @JsonView({Public.class, Static.class}) diff --git a/xyz-jobs/xyz-job-steps/src/main/java/com/here/xyz/jobs/steps/impl/transport/TransportTools.java b/xyz-jobs/xyz-job-steps/src/main/java/com/here/xyz/jobs/steps/impl/transport/TransportTools.java index 6df3c1617..e0cd34774 100644 --- a/xyz-jobs/xyz-job-steps/src/main/java/com/here/xyz/jobs/steps/impl/transport/TransportTools.java +++ b/xyz-jobs/xyz-job-steps/src/main/java/com/here/xyz/jobs/steps/impl/transport/TransportTools.java @@ -164,7 +164,7 @@ protected static Map createQueryContext(String stepId, String sc "stepId", stepId, "schema", schema, "table", table, - "context", superTable != null ? "'DEFAULT'" : "NULL", + "context", superTable != null ? "DEFAULT" : "NULL", "historyEnabled", historyEnabled )); diff --git a/xyz-jobs/xyz-job-steps/src/main/java/com/here/xyz/jobs/util/test/StepTestBase.java b/xyz-jobs/xyz-job-steps/src/main/java/com/here/xyz/jobs/util/test/StepTestBase.java index cd4ed4c40..447c49285 100644 --- a/xyz-jobs/xyz-job-steps/src/main/java/com/here/xyz/jobs/util/test/StepTestBase.java +++ b/xyz-jobs/xyz-job-steps/src/main/java/com/here/xyz/jobs/util/test/StepTestBase.java @@ -33,6 +33,7 @@ import com.google.common.net.MediaType; import com.here.xyz.XyzSerializable; import com.here.xyz.events.ContextAwareEvent; +import com.here.xyz.events.ContextAwareEvent.SpaceContext; import com.here.xyz.jobs.steps.Config; import com.here.xyz.jobs.steps.execution.LambdaBasedStep; import com.here.xyz.jobs.steps.impl.transport.ImportFilesToSpace; @@ -177,16 +178,25 @@ protected void deleteSpace(String spaceId) { protected StatisticsResponse getStatistics(String spaceId) { try { - return hubWebClient.loadSpaceStatistics(spaceId, ContextAwareEvent.SpaceContext.EXTENSION); + return hubWebClient.loadSpaceStatistics(spaceId, SpaceContext.EXTENSION); } catch (XyzWebClient.WebClientException e) { System.out.println("Hub Error: " + e.getMessage()); } return null; } - protected FeatureCollection getFeaturesFromSmallSpace(String spaceId, String propertyFilter, boolean force2D) { + protected FeatureCollection getFeaturesFromSmallSpace(String spaceId ,String propertyFilter, boolean force2D) { try { - return hubWebClient.getFeaturesFromSmallSpace(spaceId, ContextAwareEvent.SpaceContext.EXTENSION, propertyFilter, force2D); + return hubWebClient.getFeaturesFromSmallSpace(spaceId, SpaceContext.EXTENSION, propertyFilter, force2D); + } catch (XyzWebClient.WebClientException e) { + System.out.println("Hub Error: " + e.getMessage()); + } + return null; + } + + protected FeatureCollection getFeaturesFromSmallSpace(String spaceId, ContextAwareEvent.SpaceContext context, String propertyFilter, boolean force2D) { + try { + return hubWebClient.getFeaturesFromSmallSpace(spaceId, context, propertyFilter, force2D); } catch (XyzWebClient.WebClientException e) { System.out.println("Hub Error: " + e.getMessage()); } @@ -210,6 +220,14 @@ protected void putFeatureCollectionToSpace(String spaceId, FeatureCollection fc) } } + protected void deleteFeaturesInSpace(String spaceId, List ids) { + try { + hubWebClient.deleteFeatures(spaceId, ids); + } catch (XyzWebClient.WebClientException e) { + System.out.println("Hub Error: " + e.getMessage()); + } + } + protected void putRandomFeatureCollectionToSpace(String spaceId, int featureCount) { try { hubWebClient.putFeaturesWithoutResponse(spaceId, ContentCreator.generateRandomFeatureCollection(featureCount)); diff --git a/xyz-jobs/xyz-job-steps/src/test/java/com/here/xyz/jobs/steps/impl/CreateIndexStepTest.java b/xyz-jobs/xyz-job-steps/src/test/java/com/here/xyz/jobs/steps/impl/CreateIndexStepTest.java index 7bf96b6bb..e424dde8e 100644 --- a/xyz-jobs/xyz-job-steps/src/test/java/com/here/xyz/jobs/steps/impl/CreateIndexStepTest.java +++ b/xyz-jobs/xyz-job-steps/src/test/java/com/here/xyz/jobs/steps/impl/CreateIndexStepTest.java @@ -39,7 +39,7 @@ public void testCreateIndex() throws Exception { LambdaBasedStep step = new CreateIndex().withSpaceId(SPACE_ID).withIndex(GEO); - sendLambdaStepRequest(step, START_EXECUTION, false); + sendLambdaStepRequest(step, START_EXECUTION, true); //Index Creation takes time sleep(1000); diff --git a/xyz-jobs/xyz-job-steps/src/test/java/com/here/xyz/jobs/steps/impl/DropIndexStepTest.java b/xyz-jobs/xyz-job-steps/src/test/java/com/here/xyz/jobs/steps/impl/DropIndexStepTest.java index 4f4917f42..3bc7d0403 100644 --- a/xyz-jobs/xyz-job-steps/src/test/java/com/here/xyz/jobs/steps/impl/DropIndexStepTest.java +++ b/xyz-jobs/xyz-job-steps/src/test/java/com/here/xyz/jobs/steps/impl/DropIndexStepTest.java @@ -33,7 +33,7 @@ public void testDropIndexesStep() throws Exception { Assertions.assertTrue(listExistingIndexes(SPACE_ID).size() > 0); LambdaBasedStep step = new DropIndexes().withSpaceId(SPACE_ID); - sendLambdaStepRequest(step, START_EXECUTION, false); + sendLambdaStepRequest(step, START_EXECUTION, true); sleep(1000); Assertions.assertEquals(0, listExistingIndexes(SPACE_ID).size()); diff --git a/xyz-jobs/xyz-job-steps/src/test/java/com/here/xyz/jobs/steps/impl/ExportStepTest.java b/xyz-jobs/xyz-job-steps/src/test/java/com/here/xyz/jobs/steps/impl/ExportStepTest.java deleted file mode 100644 index b05da2c4f..000000000 --- a/xyz-jobs/xyz-job-steps/src/test/java/com/here/xyz/jobs/steps/impl/ExportStepTest.java +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Copyright (C) 2017-2024 HERE Europe B.V. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * License-Filename: LICENSE - */ - -package com.here.xyz.jobs.steps.impl; - -import com.here.xyz.events.PropertiesQuery; -import com.here.xyz.jobs.datasets.filters.SpatialFilter; -import com.here.xyz.jobs.steps.execution.LambdaBasedStep; -import com.here.xyz.jobs.steps.impl.transport.ExportSpaceToFiles; -import com.here.xyz.jobs.steps.outputs.DownloadUrl; -import com.here.xyz.jobs.steps.outputs.FileStatistics; -import com.here.xyz.jobs.steps.outputs.Output; -import com.here.xyz.models.geojson.coordinates.PointCoordinates; -import com.here.xyz.models.geojson.implementation.Feature; -import com.here.xyz.models.geojson.implementation.FeatureCollection; -import com.here.xyz.models.geojson.implementation.Point; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -public class ExportStepTest extends StepTest { - /** - * fcWithMixedGeometryTypes.geojson: - * 11 features: - * - * 3 Points - * 1 MultiPoint (2 Points) - * 2 Lines - * 2 MultiLine (2 Lines each) - * 1 Polygon with hole - * 1 Polygon without hole - * 1 MultiPolygon (2 Polygons) - * - */ - - - @BeforeEach - public void setUp() throws Exception { - putFeatureCollectionToSpace(SPACE_ID, readTestFeatureCollection("/testFeatureCollections/fcWithMixedGeometryTypes.geojson")); - } - - @Test - public void testExportSpaceToFilesStepUnfiltered() throws Exception { - FeatureCollection allExistingFeatures = getFeaturesFromSmallSpace(SPACE_ID, null,false); - - LambdaBasedStep step = new ExportSpaceToFiles() - .withSpaceId(SPACE_ID) - .withJobId(JOB_ID); - - sendLambdaStepRequest(step, LambdaBasedStep.LambdaStepRequest.RequestType.START_EXECUTION, false); - Thread.sleep(2000); - //TODO: switch back to simulation if test issue is fixed - //sendLambdaStepRequestBlock(step); - checkOutputs(allExistingFeatures, step.loadOutputs(true)); - } - - @Test - public void testExportSpaceToFilesStepWithPropertyFilter() throws Exception { - String propertyFilterString = "p.description=\"Point\""; - - FeatureCollection allExistingFeatures = getFeaturesFromSmallSpace(SPACE_ID, propertyFilterString, false); - - LambdaBasedStep step = new ExportSpaceToFiles() - .withPropertyFilter(PropertiesQuery.fromString(propertyFilterString)) - .withSpaceId(SPACE_ID) - .withJobId(JOB_ID); - - sendLambdaStepRequest(step, LambdaBasedStep.LambdaStepRequest.RequestType.START_EXECUTION, false); - Thread.sleep(2000); - - //TODO: switch back to simulation if test issue is fixed - //sendLambdaStepRequestBlock(step); - checkOutputs(allExistingFeatures, step.loadOutputs(true)); - } - - @Test - public void testExportSpaceToFilesStepWithSpatialFilter() throws Exception { - String spatialFilterString = "spatial?lat=50.102964&lon=8.6709594&clip=true&radius=5500"; - - SpatialFilter spatialFilter = new SpatialFilter() - .withGeometry( - new Point().withCoordinates(new PointCoordinates(8.6709594,50.102964)) - ) - .withRadius(5500) - .withClip(true); - - FeatureCollection allExistingFeatures = customReadFeaturesQuery(SPACE_ID, spatialFilterString); - - LambdaBasedStep step = new ExportSpaceToFiles() - .withSpatialFilter(spatialFilter) - .withSpaceId(SPACE_ID) - .withJobId(JOB_ID); - - sendLambdaStepRequest(step, LambdaBasedStep.LambdaStepRequest.RequestType.START_EXECUTION, false); - Thread.sleep(2000); - //TODO: switch back to simulation if test issue is fixed - //sendLambdaStepRequestBlock(step); - checkOutputs(allExistingFeatures, step.loadOutputs(true)); - } - - private void checkOutputs(FeatureCollection expectedFeatures, List outputs) throws IOException { - Assertions.assertNotEquals(0, outputs.size()); - - List exportedFeatures = new ArrayList<>(); - - for (Object output : outputs) { - if(output instanceof DownloadUrl) { - exportedFeatures.addAll(downloadFileAndSerializeFeatures((DownloadUrl) output)); - }else if(output instanceof FileStatistics statistics) { - Assertions.assertEquals(expectedFeatures.getFeatures().size(), statistics.getExportedFeatures()); - Assertions.assertTrue(statistics.getExportedFiles() > 0); - } - } - - List existingFeaturesIdList = expectedFeatures.getFeatures().stream().map(Feature::getId).collect(Collectors.toList()); - List exportedFeaturesFeaturesIdList = exportedFeatures.stream().map(Feature::getId).collect(Collectors.toList()); - - Assertions.assertTrue(exportedFeaturesFeaturesIdList.containsAll(existingFeaturesIdList)); - } -} diff --git a/xyz-jobs/xyz-job-steps/src/test/java/com/here/xyz/jobs/steps/impl/ImportStepTest.java b/xyz-jobs/xyz-job-steps/src/test/java/com/here/xyz/jobs/steps/impl/ImportStepTest.java index a21114506..2c0b96486 100644 --- a/xyz-jobs/xyz-job-steps/src/test/java/com/here/xyz/jobs/steps/impl/ImportStepTest.java +++ b/xyz-jobs/xyz-job-steps/src/test/java/com/here/xyz/jobs/steps/impl/ImportStepTest.java @@ -26,6 +26,8 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.io.IOException; + import static com.here.xyz.jobs.datasets.space.UpdateStrategy.DEFAULT_UPDATE_STRATEGY; public class ImportStepTest extends StepTest { @@ -102,10 +104,42 @@ public void testImportFilesToSpaceStep() throws Exception { .withUpdateStrategy(DEFAULT_UPDATE_STRATEGY) .withSpaceId(SPACE_ID); - sendLambdaStepRequest(step, LambdaBasedStep.LambdaStepRequest.RequestType.START_EXECUTION, false); - Thread.sleep(2000); - //TODO: switch back to simulation if test issue is fixed -// sendLambdaStepRequestBlock(step); + sendLambdaStepRequestBlock(step); + + StatisticsResponse statsAfter = getStatistics(SPACE_ID); + Assertions.assertEquals(Long.valueOf(FILE_COUNT * FEATURE_COUNT), statsAfter.getCount().getValue()); + } + + @Test + public void testImportFilesToSpaceStepWithGEOJson() throws Exception { + //Gets executed SYNC + executeImportStep(Format.GEOJSON); + } + + @Test + public void testImportFilesToSpaceStepWithCSV_Json_WKB() throws Exception { + //Gets executed ASYNC + executeImportStep(Format.CSV_JSON_WKB); + } + + @Test + public void testImportFilesToSpaceStepWithCSV_GeoJson() throws Exception { + //Gets executed ASYNC + executeImportStep(Format.CSV_GEOJSON); + } + + + private void executeImportStep(Format format) throws IOException, InterruptedException { + StatisticsResponse statsBefore = getStatistics(SPACE_ID); + Assertions.assertEquals(0L, (Object) statsBefore.getCount().getValue()); + + uploadFiles(JOB_ID, FILE_COUNT, FEATURE_COUNT, format); + LambdaBasedStep step = new ImportFilesToSpace() + .withFormat(format) + .withUpdateStrategy(DEFAULT_UPDATE_STRATEGY) + .withSpaceId(SPACE_ID); + + sendLambdaStepRequestBlock(step); StatisticsResponse statsAfter = getStatistics(SPACE_ID); Assertions.assertEquals(Long.valueOf(FILE_COUNT * FEATURE_COUNT), statsAfter.getCount().getValue()); diff --git a/xyz-jobs/xyz-job-steps/src/test/java/com/here/xyz/jobs/steps/impl/export/CompositeExportStepTest.java b/xyz-jobs/xyz-job-steps/src/test/java/com/here/xyz/jobs/steps/impl/export/CompositeExportStepTest.java new file mode 100644 index 000000000..81011bf58 --- /dev/null +++ b/xyz-jobs/xyz-job-steps/src/test/java/com/here/xyz/jobs/steps/impl/export/CompositeExportStepTest.java @@ -0,0 +1,120 @@ +/* + * Copyright (C) 2017-2024 HERE Europe B.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package com.here.xyz.jobs.steps.impl.export; + +import com.here.xyz.XyzSerializable; +import com.here.xyz.events.ContextAwareEvent.SpaceContext; +import com.here.xyz.models.geojson.implementation.FeatureCollection; +import com.here.xyz.models.hub.Space; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.sql.SQLException; +import java.util.List; + +public class CompositeExportStepTest extends ExportTestBase { + /** + * fcWithMixedGeometryTypes.geojson: + * 11 features: + * + * 3 Points + * 1 MultiPoint (2 Points) + * 2 Lines + * 2 MultiLine (2 Lines each) + * 1 Polygon with hole + * 1 Polygon without hole + * 1 MultiPolygon (2 Polygons) + * + */ + private final String SPACE_ID_EXT = SPACE_ID + "_ext"; + + @BeforeEach + public void setUp() throws Exception { + putFeatureCollectionToSpace(SPACE_ID, readTestFeatureCollection("/testFeatureCollections/fcWithMixedGeometryTypes.geojson")); + + //Create Composite Space + createSpace(new Space().withId(SPACE_ID_EXT).withExtension(new Space.Extension().withSpaceId(SPACE_ID)), false); + + //Delete Feature with id foo_polygon + deleteFeaturesInSpace(SPACE_ID_EXT, List.of("foo_polygon")); + + //Add two new Features + FeatureCollection fc2 = XyzSerializable.deserialize(""" + { + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "id": "new_point1", + "properties": {}, + "geometry": { + "coordinates": [ + 8.43, + 50.06 + ], + "type": "Point" + } + }, + { + "type": "Feature", + "id": "new_point2", + "properties": {}, + "geometry": { + "coordinates": [ + 8.49, + 50.07 + ], + "type": "Point" + } + } + ] + } + """, FeatureCollection.class); + putFeatureCollectionToSpace(SPACE_ID, fc2); + } + + @AfterEach + public void cleanup() throws SQLException { + super.cleanup(); + deleteSpace(SPACE_ID_EXT); + } + + //TODO: activate after context export is fixed +// @Test + public void exportWithContextSuper() throws IOException, InterruptedException { + executeExportStepAndCheckResults(SPACE_ID_EXT, SpaceContext.SUPER, null, null, + null, "/search?context=SUPER"); + } + + //TODO: activate after context export is fixed +// @Test + public void exportWithContextDefault() throws IOException, InterruptedException { + executeExportStepAndCheckResults(SPACE_ID_EXT, SpaceContext.DEFAULT, null, null, + null, "/search?context=DEFAULT"); + } + + @Test + public void exportWithContextExtension() throws IOException, InterruptedException { + executeExportStepAndCheckResults(SPACE_ID_EXT, SpaceContext.EXTENSION, null, null, + null, "/search?context=EXTENSION"); + } +} diff --git a/xyz-jobs/xyz-job-steps/src/test/java/com/here/xyz/jobs/steps/impl/export/ExportStepTest.java b/xyz-jobs/xyz-job-steps/src/test/java/com/here/xyz/jobs/steps/impl/export/ExportStepTest.java new file mode 100644 index 000000000..6bd63bcf2 --- /dev/null +++ b/xyz-jobs/xyz-job-steps/src/test/java/com/here/xyz/jobs/steps/impl/export/ExportStepTest.java @@ -0,0 +1,95 @@ +/* + * Copyright (C) 2017-2024 HERE Europe B.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package com.here.xyz.jobs.steps.impl.export; + +import com.here.xyz.events.PropertiesQuery; +import com.here.xyz.jobs.datasets.filters.SpatialFilter; +import com.here.xyz.models.geojson.coordinates.PointCoordinates; +import com.here.xyz.models.geojson.implementation.Point; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; + +public class ExportStepTest extends ExportTestBase { + /** + * fcWithMixedGeometryTypes.geojson: + * 11 features: + * + * 3 Points + * 1 MultiPoint (2 Points) + * 2 Lines + * 2 MultiLine (2 Lines each) + * 1 Polygon with hole + * 1 Polygon without hole + * 1 MultiPolygon (2 Polygons) + * + */ + + + @BeforeEach + public void setUp() throws Exception { + putFeatureCollectionToSpace(SPACE_ID, readTestFeatureCollection("/testFeatureCollections/fcWithMixedGeometryTypes.geojson")); + } + + @Test + public void testExportSpaceToFilesStepUnfiltered() throws Exception { + executeExportStepAndCheckResults(SPACE_ID, null, null, null, null, + "/search"); + } + + @Test + public void testExportSpaceToFilesStepWithPropertyFilter() throws Exception { + String propertiesQuery = URLEncoder.encode("p.description=\"Point\"", StandardCharsets.UTF_8); + executeExportStepAndCheckResults(SPACE_ID, null, null, + PropertiesQuery.fromString(propertiesQuery), null, "/search?" + propertiesQuery); + } + + @Test + public void testExportSpaceToFilesStepWithSpatialFilter() throws Exception { + SpatialFilter spatialFilter = new SpatialFilter() + .withGeometry( + new Point().withCoordinates(new PointCoordinates(8.6709594,50.102964)) + ) + .withRadius(5500) + .withClip(true); + + executeExportStepAndCheckResults(SPACE_ID, null, spatialFilter, null, null, + "spatial?lat=50.102964&lon=8.6709594&clip=true&radius=5500"); + } + + @Test + public void testExportSpaceToFilesStepWithSpatialAndPropertyFilter() throws Exception { + String propertiesQuery = URLEncoder.encode("p.description=\"Point\"", StandardCharsets.UTF_8); + String hubQuery = "spatial?lat=50.102964&lon=8.6709594&clip=true&radius=55000&" + + propertiesQuery; + + SpatialFilter spatialFilter = new SpatialFilter() + .withGeometry( + new Point().withCoordinates(new PointCoordinates(8.6709594,50.102964)) + ) + .withRadius(55000) + .withClip(true); + + executeExportStepAndCheckResults(SPACE_ID, null, spatialFilter, + PropertiesQuery.fromString(propertiesQuery), null, hubQuery); + } +} diff --git a/xyz-jobs/xyz-job-steps/src/test/java/com/here/xyz/jobs/steps/impl/ExportStepValidationTest.java b/xyz-jobs/xyz-job-steps/src/test/java/com/here/xyz/jobs/steps/impl/export/ExportStepValidationTest.java similarity index 97% rename from xyz-jobs/xyz-job-steps/src/test/java/com/here/xyz/jobs/steps/impl/ExportStepValidationTest.java rename to xyz-jobs/xyz-job-steps/src/test/java/com/here/xyz/jobs/steps/impl/export/ExportStepValidationTest.java index f68e0d5a7..8287e22bf 100644 --- a/xyz-jobs/xyz-job-steps/src/test/java/com/here/xyz/jobs/steps/impl/ExportStepValidationTest.java +++ b/xyz-jobs/xyz-job-steps/src/test/java/com/here/xyz/jobs/steps/impl/export/ExportStepValidationTest.java @@ -17,10 +17,11 @@ * License-Filename: LICENSE */ -package com.here.xyz.jobs.steps.impl; +package com.here.xyz.jobs.steps.impl.export; import com.here.xyz.jobs.datasets.filters.SpatialFilter; import com.here.xyz.jobs.steps.execution.LambdaBasedStep; +import com.here.xyz.jobs.steps.impl.StepTest; import com.here.xyz.jobs.steps.impl.transport.ExportSpaceToFiles; import com.here.xyz.jobs.steps.outputs.DownloadUrl; import com.here.xyz.jobs.steps.outputs.FileStatistics; diff --git a/xyz-jobs/xyz-job-steps/src/test/java/com/here/xyz/jobs/steps/impl/export/ExportTestBase.java b/xyz-jobs/xyz-job-steps/src/test/java/com/here/xyz/jobs/steps/impl/export/ExportTestBase.java new file mode 100644 index 000000000..697722630 --- /dev/null +++ b/xyz-jobs/xyz-job-steps/src/test/java/com/here/xyz/jobs/steps/impl/export/ExportTestBase.java @@ -0,0 +1,69 @@ +package com.here.xyz.jobs.steps.impl.export; + +import com.here.xyz.events.ContextAwareEvent; +import com.here.xyz.events.PropertiesQuery; +import com.here.xyz.jobs.datasets.filters.SpatialFilter; +import com.here.xyz.jobs.steps.impl.StepTest; +import com.here.xyz.jobs.steps.impl.transport.ExportSpaceToFiles; +import com.here.xyz.jobs.steps.outputs.DownloadUrl; +import com.here.xyz.jobs.steps.outputs.FileStatistics; +import com.here.xyz.jobs.steps.outputs.Output; +import com.here.xyz.models.geojson.implementation.Feature; +import com.here.xyz.models.geojson.implementation.FeatureCollection; +import com.here.xyz.models.hub.Ref; +import org.junit.jupiter.api.Assertions; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +public class ExportTestBase extends StepTest { + + protected void executeExportStepAndCheckResults(String spaceId, ContextAwareEvent.SpaceContext context, + SpatialFilter spatialFilter, PropertiesQuery propertiesQuery, + Ref versionRef, + String hubPathAndQuery) + throws IOException, InterruptedException { + //Retrieve all Features from Space + FeatureCollection allExpectedFeatures = customReadFeaturesQuery(spaceId, hubPathAndQuery); + + //Create Step definition + ExportSpaceToFiles step = new ExportSpaceToFiles() + .withSpaceId(spaceId) + .withJobId(JOB_ID); + + if(context != null) + step.withContext(context); + if(propertiesQuery != null) + step.withPropertyFilter(propertiesQuery); + if(spaceId != null) + step.withSpatialFilter(spatialFilter); + if(versionRef != null) + step.withVersionRef(versionRef); + + //Send Lambda Requests + sendLambdaStepRequestBlock(step); + checkOutputs(allExpectedFeatures, step.loadOutputs(true)); + } + + protected void checkOutputs(FeatureCollection expectedFeatures, List outputs) throws IOException { + Assertions.assertNotEquals(0, outputs.size()); + + List exportedFeatures = new ArrayList<>(); + + for (Object output : outputs) { + if(output instanceof DownloadUrl) { + exportedFeatures.addAll(downloadFileAndSerializeFeatures((DownloadUrl) output)); + }else if(output instanceof FileStatistics statistics) { + Assertions.assertEquals(expectedFeatures.getFeatures().size(), statistics.getExportedFeatures()); + Assertions.assertTrue(statistics.getExportedFiles() > 0); + } + } + + List existingFeaturesIdList = expectedFeatures.getFeatures().stream().map(Feature::getId).collect(Collectors.toList()); + List exportedFeaturesFeaturesIdList = exportedFeatures.stream().map(Feature::getId).collect(Collectors.toList()); + + Assertions.assertTrue(exportedFeaturesFeaturesIdList.containsAll(existingFeaturesIdList)); + } +} diff --git a/xyz-jobs/xyz-job-steps/src/test/java/com/here/xyz/jobs/steps/impl/export/VersionRefExportStepTest.java b/xyz-jobs/xyz-job-steps/src/test/java/com/here/xyz/jobs/steps/impl/export/VersionRefExportStepTest.java new file mode 100644 index 000000000..f35f6ebc5 --- /dev/null +++ b/xyz-jobs/xyz-job-steps/src/test/java/com/here/xyz/jobs/steps/impl/export/VersionRefExportStepTest.java @@ -0,0 +1,134 @@ +/* + * Copyright (C) 2017-2024 HERE Europe B.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package com.here.xyz.jobs.steps.impl.export; + +import com.here.xyz.XyzSerializable; +import com.here.xyz.jobs.steps.impl.transport.ExportSpaceToFiles; +import com.here.xyz.models.geojson.implementation.FeatureCollection; +import com.here.xyz.models.hub.Ref; +import com.here.xyz.models.hub.Space; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.List; + +public class VersionRefExportStepTest extends ExportTestBase { + + @BeforeEach + public void setUp() throws Exception { + createSpace(new Space().withId(SPACE_ID).withVersionsToKeep(10) , false); + //Add two new Features + FeatureCollection fc1 = XyzSerializable.deserialize(""" + { + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "id": "point1", + "properties": { + "value" : "1" + }, + "geometry": { + "coordinates": [ + 8.43, + 50.06 + ], + "type": "Point" + } + }, + { + "type": "Feature", + "id": "point2", + "properties": { + "value" : "2" + }, + "geometry": { + "coordinates": [ + 8.49, + 50.07 + ], + "type": "Point" + } + } + ] + } + """, FeatureCollection.class); + + FeatureCollection fc2 = XyzSerializable.deserialize(""" + { + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "id": "point1", + "properties": { + "value" : "new" + }, + "geometry": { + "coordinates": [ + 8.43, + 50.06 + ], + "type": "Point" + } + }, + { + "type": "Feature", + "id": "point3", + "properties": { + "value" : "3" + }, + "geometry": { + "coordinates": [ + 8.49, + 50.07 + ], + "type": "Point" + } + } + ] + } + """, FeatureCollection.class); + + putFeatureCollectionToSpace(SPACE_ID, fc1); + deleteFeaturesInSpace(SPACE_ID, List.of("point2")); + putFeatureCollectionToSpace(SPACE_ID, fc2); + } + + @Test + public void exportWithSingleVersion() throws IOException, InterruptedException { + Ref versionRef = new Ref("3"); + + executeExportStepAndCheckResults(SPACE_ID, null, null, null, + versionRef,"search?versionRef=" + versionRef); + } + +// @Test + //TODO: implement + public void exportWithVersionRange() throws IOException, InterruptedException { + Ref versionRef = new Ref("1..3"); + + ExportSpaceToFiles step = new ExportSpaceToFiles() + .withSpaceId(SPACE_ID) + .withJobId(JOB_ID) + .withVersionRef(versionRef); + } +} diff --git a/xyz-util/src/main/java/com/here/xyz/util/web/HubWebClient.java b/xyz-util/src/main/java/com/here/xyz/util/web/HubWebClient.java index 67b5a6eff..d053a2c16 100644 --- a/xyz-util/src/main/java/com/here/xyz/util/web/HubWebClient.java +++ b/xyz-util/src/main/java/com/here/xyz/util/web/HubWebClient.java @@ -130,6 +130,14 @@ public void putFeaturesWithoutResponse(String spaceId, FeatureCollection fc) thr .method("PUT", BodyPublishers.ofByteArray(XyzSerializable.serialize(fc).getBytes()))); } + public void deleteFeatures(String spaceId, List featureIds) throws WebClientException { + String idList = "?id="+String.join(",id=", featureIds); + request(HttpRequest.newBuilder() + .DELETE() + .uri(uri("/spaces/" + spaceId + "/features" + idList)) + .header(CONTENT_TYPE, JSON_UTF_8.toString())); + } + public XyzResponse postFeatures(String spaceId, FeatureCollection fc, Map queryParams) throws WebClientException { try { return deserialize(request(HttpRequest.newBuilder()