New job tests

.github/workflows/test.yaml

@@ -18,6 +18,9 @@ jobs:
         uses: actions/setup-java@v1
         with:
           java-version: '17'
+      - name: Add hosts to /etc/hosts
+        run: |
+          sudo echo "127.0.0.1 dynamodb postgres.local postgres postgres.replica" | sudo tee -a /etc/hosts
       - name: Start the XYZ Hub stack
         run: mvn clean install -Pdocker -DskipTests=true -DdockerComposeFile=docker-compose-dynamodb.yml
       - name: Deploy StepLambda for Job API

xyz-jobs/xyz-job-service/src/main/java/com/here/xyz/jobs/datasets/DatasetDescription.java

@@ -77,9 +77,9 @@ public Map withVersionRef(Ref versionRef) {
 
   @JsonIgnoreProperties(ignoreUnknown = true)
   public static class Space<T extends Space> extends Identifiable<T> implements FilteringSource<T>, VersionedSource<T> {
-    @JsonView({Public.class})
+    @JsonView({Public.class, Static.class})
     private Filters filters;
-    @JsonView({Public.class})
+    @JsonView({Public.class, Static.class})
     private Ref versionRef;
 
     @Override

xyz-jobs/xyz-job-service/src/main/java/com/here/xyz/jobs/datasets/Spaces.java

@@ -19,6 +19,7 @@
 
 package com.here.xyz.jobs.datasets;
 
+import com.fasterxml.jackson.annotation.JsonView;
 import com.here.xyz.jobs.datasets.DatasetDescription.Space;
 import com.here.xyz.jobs.datasets.filters.FilteringSource;
 import com.here.xyz.jobs.datasets.filters.Filters;
@@ -28,6 +29,8 @@
 public class Spaces<T extends Spaces> extends DatasetDescription implements FilteringSource<T>, CombinedDatasetDescription<Space> {
 
   private List<String> spaceIds;
+
+  @JsonView({Public.class, Static.class})
   private Filters filters;
 
   public List<String> getSpaceIds() {

xyz-jobs/xyz-job-service/src/main/java/com/here/xyz/jobs/datasets/filters/Filters.java

@@ -40,7 +40,7 @@ public class Filters {
   @JsonView({Internal.class, Static.class})
   private String propertyFilterAsString;
 
-  @JsonView({Public.class})
+  @JsonView({Public.class, Static.class})
   private SpatialFilter spatialFilter;
 
   @JsonView({Public.class, Static.class})

xyz-jobs/xyz-job-steps/src/main/java/com/here/xyz/jobs/steps/impl/transport/TransportTools.java

@@ -164,7 +164,7 @@ protected static Map<String, Object> createQueryContext(String stepId, String sc
             "stepId", stepId,
             "schema", schema,
             "table", table,
-            "context", superTable != null ? "'DEFAULT'" : "NULL",
+            "context", superTable != null ? "DEFAULT" : "NULL",
             "historyEnabled", historyEnabled
     ));
 

xyz-jobs/xyz-job-steps/src/main/java/com/here/xyz/jobs/util/test/StepTestBase.java

@@ -33,6 +33,7 @@
 import com.google.common.net.MediaType;
 import com.here.xyz.XyzSerializable;
 import com.here.xyz.events.ContextAwareEvent;
+import com.here.xyz.events.ContextAwareEvent.SpaceContext;
 import com.here.xyz.jobs.steps.Config;
 import com.here.xyz.jobs.steps.execution.LambdaBasedStep;
 import com.here.xyz.jobs.steps.impl.transport.ImportFilesToSpace;
@@ -177,16 +178,25 @@ protected void deleteSpace(String spaceId) {
 
   protected StatisticsResponse getStatistics(String spaceId) {
     try {
-      return hubWebClient.loadSpaceStatistics(spaceId, ContextAwareEvent.SpaceContext.EXTENSION);
+      return hubWebClient.loadSpaceStatistics(spaceId, SpaceContext.EXTENSION);
     } catch (XyzWebClient.WebClientException e) {
       System.out.println("Hub Error: " + e.getMessage());
     }
     return null;
   }
 
-  protected FeatureCollection getFeaturesFromSmallSpace(String spaceId, String propertyFilter, boolean force2D) {
+  protected FeatureCollection getFeaturesFromSmallSpace(String spaceId ,String propertyFilter, boolean force2D) {
     try {
-      return hubWebClient.getFeaturesFromSmallSpace(spaceId, ContextAwareEvent.SpaceContext.EXTENSION, propertyFilter, force2D);
+      return hubWebClient.getFeaturesFromSmallSpace(spaceId, SpaceContext.EXTENSION, propertyFilter, force2D);
+    } catch (XyzWebClient.WebClientException e) {
+      System.out.println("Hub Error: " + e.getMessage());
+    }
+    return null;
+  }
+
+  protected FeatureCollection getFeaturesFromSmallSpace(String spaceId, ContextAwareEvent.SpaceContext context, String propertyFilter, boolean force2D) {
+    try {
+      return hubWebClient.getFeaturesFromSmallSpace(spaceId, context, propertyFilter, force2D);
     } catch (XyzWebClient.WebClientException e) {
       System.out.println("Hub Error: " + e.getMessage());
     }
@@ -210,6 +220,14 @@ protected void putFeatureCollectionToSpace(String spaceId, FeatureCollection fc)
     }
   }
 
+  protected void deleteFeaturesInSpace(String spaceId, List<String> ids) {
+    try {
+      hubWebClient.deleteFeatures(spaceId, ids);
+    } catch (XyzWebClient.WebClientException e) {
+      System.out.println("Hub Error: " + e.getMessage());
+    }
+  }
+
   protected void putRandomFeatureCollectionToSpace(String spaceId, int featureCount) {
     try {
       hubWebClient.putFeaturesWithoutResponse(spaceId, ContentCreator.generateRandomFeatureCollection(featureCount));

xyz-jobs/xyz-job-steps/src/test/java/com/here/xyz/jobs/steps/impl/CreateIndexStepTest.java

@@ -39,7 +39,7 @@ public void testCreateIndex() throws Exception {
 
         LambdaBasedStep step = new CreateIndex().withSpaceId(SPACE_ID).withIndex(GEO);
 
-        sendLambdaStepRequest(step, START_EXECUTION, false);
+        sendLambdaStepRequest(step, START_EXECUTION, true);
         //Index Creation takes time
         sleep(1000);
 

xyz-jobs/xyz-job-steps/src/test/java/com/here/xyz/jobs/steps/impl/DropIndexStepTest.java

@@ -33,7 +33,7 @@ public void testDropIndexesStep() throws Exception {
     Assertions.assertTrue(listExistingIndexes(SPACE_ID).size() > 0);
 
     LambdaBasedStep step = new DropIndexes().withSpaceId(SPACE_ID);
-    sendLambdaStepRequest(step, START_EXECUTION, false);
+    sendLambdaStepRequest(step, START_EXECUTION, true);
     sleep(1000);
 
     Assertions.assertEquals(0, listExistingIndexes(SPACE_ID).size());

xyz-jobs/xyz-job-steps/src/test/java/com/here/xyz/jobs/steps/impl/ImportStepTest.java

@@ -102,10 +102,7 @@ public void testImportFilesToSpaceStep() throws Exception {
             .withUpdateStrategy(DEFAULT_UPDATE_STRATEGY)
             .withSpaceId(SPACE_ID);
 
-    sendLambdaStepRequest(step, LambdaBasedStep.LambdaStepRequest.RequestType.START_EXECUTION, false);
-    Thread.sleep(2000);
-    //TODO: switch back to simulation if test issue is fixed
-//    sendLambdaStepRequestBlock(step);
+    sendLambdaStepRequestBlock(step);
 
     StatisticsResponse statsAfter = getStatistics(SPACE_ID);
     Assertions.assertEquals(Long.valueOf(FILE_COUNT * FEATURE_COUNT), statsAfter.getCount().getValue());