From e5367f4f6309b9c6077b0550d9e296c8abbccc79 Mon Sep 17 00:00:00 2001 From: mchrza Date: Sat, 9 Nov 2024 13:20:52 +0100 Subject: [PATCH] Finalize export, on composite spaces, by using context. Signed-off-by: Maximilian Chrzan Signed-off-by: mchrza --- .../impl/transport/ExportSpaceToFiles.java | 17 +++++-- .../impl/export/CompositeExportStepTest.java | 46 +++++++++++++++++-- .../steps/impl/export/ExportTestBase.java | 12 +++-- .../query/GetFeaturesByGeometryBuilder.java | 4 +- 4 files changed, 62 insertions(+), 17 deletions(-) diff --git a/xyz-jobs/xyz-job-steps/src/main/java/com/here/xyz/jobs/steps/impl/transport/ExportSpaceToFiles.java b/xyz-jobs/xyz-job-steps/src/main/java/com/here/xyz/jobs/steps/impl/transport/ExportSpaceToFiles.java index 5b8643dd4..1ec8f4682 100644 --- a/xyz-jobs/xyz-job-steps/src/main/java/com/here/xyz/jobs/steps/impl/transport/ExportSpaceToFiles.java +++ b/xyz-jobs/xyz-job-steps/src/main/java/com/here/xyz/jobs/steps/impl/transport/ExportSpaceToFiles.java @@ -19,7 +19,8 @@ package com.here.xyz.jobs.steps.impl.transport; -import static com.here.xyz.events.ContextAwareEvent.SpaceContext.EXTENSION; +import static com.here.xyz.events.ContextAwareEvent.SpaceContext.DEFAULT; +import static com.here.xyz.events.ContextAwareEvent.SpaceContext.SUPER; import static com.here.xyz.jobs.steps.impl.transport.TransportTools.Phase.JOB_EXECUTOR; import static com.here.xyz.jobs.steps.impl.transport.TransportTools.Phase.STEP_EXECUTE; import static com.here.xyz.jobs.steps.impl.transport.TransportTools.Phase.STEP_ON_ASYNC_SUCCESS; @@ -84,10 +85,13 @@ public class ExportSpaceToFiles extends SpaceBasedStep { private Format format = Format.GEOJSON; + @JsonView({Internal.class, Static.class}) private SpatialFilter spatialFilter; + @JsonView({Internal.class, Static.class}) private PropertiesQuery propertyFilter; + @JsonView({Internal.class, Static.class}) private SpaceContext context; - + @JsonView({Internal.class, Static.class}) private Ref versionRef; /** @@ -143,7 +147,7 @@ public ExportSpaceToFiles withPropertyFilter(PropertiesQuery propertyFilter){ } public SpaceContext getContext() { - return context == null ? EXTENSION :context; + return this.context; } public void setContext(SpaceContext context) { @@ -372,11 +376,14 @@ private void createAndFillTemporaryJobTable(List s3FileNames) throws private String generateFilteredExportQuery(int threadNumber) throws WebClientException, TooManyResourcesClaimed, QueryBuildingException { GetFeaturesByGeometryBuilder queryBuilder = new GetFeaturesByGeometryBuilder() .withDataSourceProvider(requestResource(db(), 0)); + if(context == SUPER) + space().switchToSuper(superSpace().getId()); GetFeaturesByGeometryInput input = new GetFeaturesByGeometryInput( - getSpaceId(), + space().getId(), hubWebClient().loadConnector(space().getStorage().getId()).params, - context == null ? EXTENSION : context, + space().getExtension() != null ? space().resolveCompositeParams(superSpace()) : null, + context == null ? DEFAULT : context, space().getVersionsToKeep(), versionRef, spatialFilter != null ? spatialFilter.getGeometry() : null, diff --git a/xyz-jobs/xyz-job-steps/src/test/java/com/here/xyz/jobs/steps/impl/export/CompositeExportStepTest.java b/xyz-jobs/xyz-job-steps/src/test/java/com/here/xyz/jobs/steps/impl/export/CompositeExportStepTest.java index 81011bf58..6ec68c67a 100644 --- a/xyz-jobs/xyz-job-steps/src/test/java/com/here/xyz/jobs/steps/impl/export/CompositeExportStepTest.java +++ b/xyz-jobs/xyz-job-steps/src/test/java/com/here/xyz/jobs/steps/impl/export/CompositeExportStepTest.java @@ -21,6 +21,7 @@ import com.here.xyz.XyzSerializable; import com.here.xyz.events.ContextAwareEvent.SpaceContext; +import com.here.xyz.events.PropertiesQuery; import com.here.xyz.models.geojson.implementation.FeatureCollection; import com.here.xyz.models.hub.Space; import org.junit.jupiter.api.AfterEach; @@ -28,6 +29,8 @@ import org.junit.jupiter.api.Test; import java.io.IOException; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.sql.SQLException; import java.util.List; @@ -89,7 +92,7 @@ public void setUp() throws Exception { ] } """, FeatureCollection.class); - putFeatureCollectionToSpace(SPACE_ID, fc2); + putFeatureCollectionToSpace(SPACE_ID_EXT, fc2); } @AfterEach @@ -98,23 +101,56 @@ public void cleanup() throws SQLException { deleteSpace(SPACE_ID_EXT); } - //TODO: activate after context export is fixed -// @Test + @Test public void exportWithContextSuper() throws IOException, InterruptedException { executeExportStepAndCheckResults(SPACE_ID_EXT, SpaceContext.SUPER, null, null, null, "/search?context=SUPER"); } - //TODO: activate after context export is fixed -// @Test + @Test + public void exportWithContextSuperAndWithPropertyFilter() throws Exception { + //new_point does not exist in base + exportWithContextAndWithPropertyFilter(SpaceContext.SUPER, + URLEncoder.encode("f.id=\"new_point1\"", StandardCharsets.UTF_8)); + } + + @Test public void exportWithContextDefault() throws IOException, InterruptedException { executeExportStepAndCheckResults(SPACE_ID_EXT, SpaceContext.DEFAULT, null, null, null, "/search?context=DEFAULT"); } + @Test + public void exportWithoutContext() throws IOException, InterruptedException { + //Default context should get used + executeExportStepAndCheckResults(SPACE_ID_EXT, null, null, null, + null, "/search?context=DEFAULT"); + } + + @Test + public void exportWithContextDefaultAndWithPropertyFilter() throws Exception { + //new_point exists in composite compound + exportWithContextAndWithPropertyFilter(SpaceContext.DEFAULT, + URLEncoder.encode("f.id=\"new_point1\"", StandardCharsets.UTF_8)); + } + @Test public void exportWithContextExtension() throws IOException, InterruptedException { executeExportStepAndCheckResults(SPACE_ID_EXT, SpaceContext.EXTENSION, null, null, null, "/search?context=EXTENSION"); } + + @Test + public void exportWithContextExtensionAndWithPropertyFilter() throws Exception { + //new_point exists in extension + exportWithContextAndWithPropertyFilter(SpaceContext.EXTENSION, + URLEncoder.encode("f.id=\"new_point1\"", StandardCharsets.UTF_8)); + } + + private void exportWithContextAndWithPropertyFilter(SpaceContext context, String propertiesQuery) throws Exception { + String hubQuery = "/search?context="+context+"&"+ propertiesQuery; + + executeExportStepAndCheckResults(SPACE_ID_EXT, context, null, + PropertiesQuery.fromString(propertiesQuery), null, hubQuery); + } } diff --git a/xyz-jobs/xyz-job-steps/src/test/java/com/here/xyz/jobs/steps/impl/export/ExportTestBase.java b/xyz-jobs/xyz-job-steps/src/test/java/com/here/xyz/jobs/steps/impl/export/ExportTestBase.java index 268d01e30..e1e09bdeb 100644 --- a/xyz-jobs/xyz-job-steps/src/test/java/com/here/xyz/jobs/steps/impl/export/ExportTestBase.java +++ b/xyz-jobs/xyz-job-steps/src/test/java/com/here/xyz/jobs/steps/impl/export/ExportTestBase.java @@ -34,13 +34,13 @@ protected void executeExportStepAndCheckResults(String spaceId, ContextAwareEven .withJobId(JOB_ID); if(context != null) - step.withContext(context); + step.setContext(context); if(propertiesQuery != null) - step.withPropertyFilter(propertiesQuery); + step.setPropertyFilter(propertiesQuery); if(spaceId != null) - step.withSpatialFilter(spatialFilter); + step.setSpatialFilter(spatialFilter); if(versionRef != null) - step.withVersionRef(versionRef); + step.setVersionRef(versionRef); //Send Lambda Requests sendLambdaStepRequestBlock(step, true); @@ -57,7 +57,9 @@ protected void checkOutputs(FeatureCollection expectedFeatures, List out exportedFeatures.addAll(downloadFileAndSerializeFeatures((DownloadUrl) output)); }else if(output instanceof FileStatistics statistics) { Assertions.assertEquals(expectedFeatures.getFeatures().size(), statistics.getExportedFeatures()); - Assertions.assertTrue(statistics.getExportedFiles() > 0); + //if we have one Feature - we expect at least one file + if(expectedFeatures.getFeatures().size() > 1) + Assertions.assertTrue(statistics.getExportedFiles() > 0); } } diff --git a/xyz-psql-connector/src/main/java/com/here/xyz/psql/query/GetFeaturesByGeometryBuilder.java b/xyz-psql-connector/src/main/java/com/here/xyz/psql/query/GetFeaturesByGeometryBuilder.java index 80a9e7b1c..1e27036b9 100644 --- a/xyz-psql-connector/src/main/java/com/here/xyz/psql/query/GetFeaturesByGeometryBuilder.java +++ b/xyz-psql-connector/src/main/java/com/here/xyz/psql/query/GetFeaturesByGeometryBuilder.java @@ -21,7 +21,6 @@ import static com.here.xyz.models.hub.Ref.HEAD; -import com.here.xyz.XyzSerializable; import com.here.xyz.connectors.ErrorResponseException; import com.here.xyz.events.ContextAwareEvent.SpaceContext; import com.here.xyz.events.GetFeaturesByGeometryEvent; @@ -43,7 +42,7 @@ public SQLQuery buildQuery(GetFeaturesByGeometryInput input) throws QueryBuildin GetFeaturesByGeometryEvent event = new GetFeaturesByGeometryEvent() .withSpace(input.spaceId) .withConnectorParams(input.connectorParams) - .withParams(XyzSerializable.toMap(getConnectorParameters())) + .withParams(input.spaceParams) .withVersionsToKeep(input.versionsToKeep) .withContext(input.context) .withRef(input.ref) @@ -69,6 +68,7 @@ public GetFeaturesByGeometryBuilder withAdditionalFilterFragment(SQLQuery additi public record GetFeaturesByGeometryInput( String spaceId, Map connectorParams, + Map spaceParams, SpaceContext context, int versionsToKeep, Ref ref,