Merge branch 'master' into version-params-optional-changeset-collection

.run/jobServiceSetup.run.xml

@@ -19,17 +19,17 @@
 
 <component name="ProjectRunConfigurationManager">
   <configuration default="false" name="jobServiceSetup" type="ShConfigurationType">
-    <option name="SCRIPT_TEXT" value="" />
+    <option name="SCRIPT_TEXT" value="docker run --rm -it --entrypoint '' -v ./localSetup.sh:/aws/localSetup.sh --add-host host.docker.internal=host-gateway amazon/aws-cli ./localSetup.sh true" />
     <option name="INDEPENDENT_SCRIPT_PATH" value="true" />
-    <option name="SCRIPT_PATH" value="$PROJECT_DIR$/xyz-jobs/xyz-job-service/src/main/bash/localSetup.sh" />
+    <option name="SCRIPT_PATH" value="" />
     <option name="SCRIPT_OPTIONS" value="" />
     <option name="INDEPENDENT_SCRIPT_WORKING_DIRECTORY" value="true" />
     <option name="SCRIPT_WORKING_DIRECTORY" value="$PROJECT_DIR$/xyz-jobs/xyz-job-service/src/main/bash" />
     <option name="INDEPENDENT_INTERPRETER_PATH" value="true" />
     <option name="INTERPRETER_PATH" value="/bin/bash" />
     <option name="INTERPRETER_OPTIONS" value="" />
     <option name="EXECUTE_IN_TERMINAL" value="true" />
-    <option name="EXECUTE_SCRIPT_FILE" value="true" />
+    <option name="EXECUTE_SCRIPT_FILE" value="false" />
     <envs />
     <method v="2" />
   </configuration>

README.md

@@ -19,8 +19,6 @@ Some of the features of XYZ Hub are:
 * Build a real-time geodata pipeline with processors
 * Attach listeners to react on events
 
-You can find more information in the [XYZ Documentation](https://www.here.xyz/api) and in the [OpenAPI specification](https://xyz.api.here.com/hub/static/redoc/index.html).
-
 XYZ Hub uses [GeoJSON](https://tools.ietf.org/html/rfc79460) as the main geospatial data exchange format. Tiled data can also be provided as [MVT](https://github.com/mapbox/vector-tile-spec/blob/master/2.1/README.md).
 
 # Prerequisites

docker-compose-dynamodb.yml

@@ -89,6 +89,8 @@ services:
         volumes:
             - "/var/run/docker.sock:/var/run/docker.sock"
             - "./xyz-hub-test/src/test/resources/mock-servers/localstack/docker-entrypoint-initaws.d/01-create-bucket.sh:/etc/localstack/init/ready.d/init-aws.sh"
+            - "./xyz-jobs/xyz-job-steps/src/main/bash:/home/localstack/job-steps-scripts"
+            - "./xyz-jobs/xyz-job-steps:/home/localstack/job-steps"
     s3-explorer:
         image: aws-js-s3-explorer
         build:

docker-compose.yml

@@ -66,6 +66,8 @@ services:
         volumes:
             - "/var/run/docker.sock:/var/run/docker.sock"
             - "./xyz-hub-test/src/test/resources/mock-servers/localstack/docker-entrypoint-initaws.d/01-create-bucket.sh:/etc/localstack/init/ready.d/init-aws.sh"
+            - "./xyz-jobs/xyz-job-steps/src/main/bash:/home/localstack/job-steps-scripts"
+            - "./xyz-jobs/xyz-job-steps:/home/localstack/job-steps"
     s3-explorer:
         image: aws-js-s3-explorer
         build:

pom.xml

@@ -41,7 +41,7 @@
 
   <groupId>com.here.xyz</groupId>
   <artifactId>xyz-hub</artifactId>
-  <version>3.10.12-SNAPSHOT</version>
+  <version>3.10.17-SNAPSHOT</version>
   <packaging>pom</packaging>
 
   <modules>

xyz-connectors/pom.xml

@@ -25,7 +25,7 @@
     <groupId>com.here.xyz</groupId>
     <artifactId>xyz-hub</artifactId>
     <relativePath>../pom.xml</relativePath>
-    <version>3.10.12-SNAPSHOT</version>
+    <version>3.10.17-SNAPSHOT</version>
   </parent>
 
   <licenses>

xyz-hub-service/pom.xml

@@ -25,7 +25,7 @@
     <groupId>com.here.xyz</groupId>
     <artifactId>xyz-hub</artifactId>
     <relativePath>../pom.xml</relativePath>
-    <version>3.10.12-SNAPSHOT</version>
+    <version>3.10.17-SNAPSHOT</version>
   </parent>
 
   <licenses>

xyz-hub-service/src/main/java/com/here/xyz/hub/config/dynamo/DynamoSpaceConfigClient.java

@@ -390,7 +390,7 @@ private Future<List<Space>> getSpacesWithFullAccess(SpaceSelectionCondition sele
       if (propsQuery != null) {
         valueMap.put(":typeValue", "SPACE");
         valueMap.put(":contentUpdatedAtValue", propsQuery.get(0).get(0).getValues().get(0));
-        String operator = QueryOperation.getOperation(propsQuery.get(0).get(0).getOperation());
+        String operator = QueryOperation.getOutputRepresentation(propsQuery.get(0).get(0).getOperation());
 
         spaces.getIndex("type-contentUpdatedAt-index")
             .query(new QuerySpec()
@@ -580,7 +580,7 @@ private Future<Void> filterByContentUpdatedAt(PropertiesQuery propsQuery, Set<St
       Map<String, Object> valueMap = new HashMap<>();
       valueMap.put(":typeValue", "SPACE");
       valueMap.put(":contentUpdatedAtValue", propsQuery.get(0).get(0).getValues().get(0));
-      String operator = QueryOperation.getOperation(propsQuery.get(0).get(0).getOperation());
+      String operator = QueryOperation.getOutputRepresentation(propsQuery.get(0).get(0).getOperation());
       var contentUpdatedAtSpaceIds = new HashSet<String>();
       spaces.getIndex("type-contentUpdatedAt-index").query(new QuerySpec()
               .withKeyConditionExpression("#type = :typeValue and contentUpdatedAt " +  operator + " :contentUpdatedAtValue")

xyz-hub-service/src/main/java/com/here/xyz/hub/config/jdbc/JDBCSpaceConfigClient.java

@@ -128,7 +128,7 @@ protected Future<List<Space>> getSelectedSpaces(Marker marker, SpaceAuthorizatio
         List<String> contentUpdatedAtConjunctions = new ArrayList<>();
         conjunctions.forEach(conj -> {
             conj.getValues().forEach(v -> {
-              contentUpdatedAtConjunctions.add("(cast(config->>'contentUpdatedAt' AS TEXT) "+ QueryOperation.getOperation(conj.getOperation()) + "'" +v + "' )");
+              contentUpdatedAtConjunctions.add("(cast(config->>'contentUpdatedAt' AS TEXT) "+ QueryOperation.getOutputRepresentation(conj.getOperation()) + "'" +v + "' )");
             });
         });
         whereConjunctions.add(String.join(" OR ", contentUpdatedAtConjunctions));

xyz-hub-service/src/main/java/com/here/xyz/hub/config/memory/InMemSpaceConfigClient.java

@@ -91,7 +91,7 @@ protected Future<List<Space>> getSelectedSpaces(Marker marker, SpaceAuthorizatio
       propsQuery.forEach(conjunctions -> {
         conjunctions.forEach(conj -> {
             conj.getValues().forEach(v -> {
-              String operator = QueryOperation.getOperation(conj.getOperation());
+              String operator = QueryOperation.getOutputRepresentation(conj.getOperation());
               contentUpdatedAtList.add(operator);
               contentUpdatedAtList.add(v.toString());
             });

xyz-hub-service/src/main/java/com/here/xyz/hub/connectors/RemoteFunctionClient.java

@@ -21,7 +21,8 @@
 
 import static com.here.xyz.hub.util.AtomicUtils.compareAndDecrement;
 import static com.here.xyz.hub.util.AtomicUtils.compareAndIncrementUpTo;
-import static io.netty.handler.codec.http.HttpResponseStatus.TOO_MANY_REQUESTS;
+import static com.here.xyz.util.service.rest.TooManyRequestsException.ThrottlingReason.QUOTA;
+import static com.here.xyz.util.service.rest.TooManyRequestsException.ThrottlingReason.STORAGE_QUEUE_FULL;
 
 import com.google.common.io.ByteStreams;
 import com.here.xyz.Payload;
@@ -31,7 +32,7 @@
 import com.here.xyz.hub.util.ByteSizeAware;
 import com.here.xyz.hub.util.LimitedQueue;
 import com.here.xyz.util.service.Core;
-import com.here.xyz.util.service.HttpException;
+import com.here.xyz.util.service.rest.TooManyRequestsException;
 import io.vertx.core.AsyncResult;
 import io.vertx.core.Context;
 import io.vertx.core.Future;
@@ -186,8 +187,8 @@ private static boolean checkRequesterThrottling(Marker marker, Handler<AsyncResu
       if (!compareAndIncrementUpTo(context.getConnector().getMaxConnectionsPerRequester(), connectionCount)) {
         logger.warn(marker, "Sending to many concurrent requests for user {}. Number of active connections: {}, Maximum allowed per node: {}",
             context.getRequesterId(), connectionCount.get(), context.getConnector().getMaxConnectionsPerRequester());
-        callback.handle(Future.failedFuture(new HttpException(TOO_MANY_REQUESTS, "Maximum number of concurrent requests. "
-                + "Max concurrent connections: " + Math.round(context.getConnector().connectionSettings.maxConnectionsPerRequester * 0.6))));
+        callback.handle(Future.failedFuture(new TooManyRequestsException("Maximum number of concurrent requests. "
+                + "Max concurrent connections: " + Math.round(context.getConnector().connectionSettings.maxConnectionsPerRequester * 0.6), QUOTA)));
         return true;
       }
     }
@@ -471,7 +472,7 @@ private void enqueue(final FunctionCall fc) {
         //Send timeout for discarded (old) calls
         .forEach(timeoutFc ->
             timeoutFc.callback
-                .handle(Future.failedFuture(new HttpException(TOO_MANY_REQUESTS, "Remote function is busy or cannot be invoked."))));
+                .handle(Future.failedFuture(new TooManyRequestsException("Remote function is busy or cannot be invoked.", STORAGE_QUEUE_FULL))));
   }
 
   public class FunctionCall implements ByteSizeAware {

xyz-hub-service/src/main/java/com/here/xyz/hub/connectors/RpcClient.java

@@ -21,14 +21,14 @@
 
 import static com.here.xyz.events.GetFeaturesByTileEvent.ResponseType.MVT;
 import static com.here.xyz.events.GetFeaturesByTileEvent.ResponseType.MVT_FLATTENED;
+import static com.here.xyz.util.service.rest.TooManyRequestsException.ThrottlingReason.CONNECTOR;
 import static io.netty.handler.codec.http.HttpHeaderValues.APPLICATION_JSON;
 import static io.netty.handler.codec.http.HttpResponseStatus.BAD_GATEWAY;
 import static io.netty.handler.codec.http.HttpResponseStatus.BAD_REQUEST;
 import static io.netty.handler.codec.http.HttpResponseStatus.CONFLICT;
 import static io.netty.handler.codec.http.HttpResponseStatus.FORBIDDEN;
 import static io.netty.handler.codec.http.HttpResponseStatus.GATEWAY_TIMEOUT;
 import static io.netty.handler.codec.http.HttpResponseStatus.NOT_IMPLEMENTED;
-import static io.netty.handler.codec.http.HttpResponseStatus.TOO_MANY_REQUESTS;
 import static io.netty.handler.codec.rtsp.RtspResponseStatuses.REQUEST_ENTITY_TOO_LARGE;
 
 import com.fasterxml.jackson.core.JsonParseException;
@@ -57,6 +57,7 @@
 import com.here.xyz.responses.XyzResponse;
 import com.here.xyz.util.service.Core;
 import com.here.xyz.util.service.HttpException;
+import com.here.xyz.util.service.rest.TooManyRequestsException;
 import io.vertx.core.AsyncResult;
 import io.vertx.core.Future;
 import io.vertx.core.Handler;
@@ -400,8 +401,8 @@ private void validateResponsePayload(Marker marker, final Typed payload) throws
         case FORBIDDEN:
           throw new HttpException(FORBIDDEN, "The user is not authorized.", errorResponse.getErrorDetails());
         case TOO_MANY_REQUESTS:
-          throw new HttpException(TOO_MANY_REQUESTS,
-              "The connector cannot process the message due to a limitation in an upstream service or a database.", errorResponse.getErrorDetails());
+          throw new TooManyRequestsException("The connector cannot process the message due to a limitation in an upstream service or a database.",
+              CONNECTOR, errorResponse.getErrorDetails());
         case ILLEGAL_ARGUMENT:
           throw new HttpException(BAD_REQUEST, errorResponse.getErrorMessage(), errorResponse.getErrorDetails());
         case TIMEOUT:

xyz-hub-service/src/main/java/com/here/xyz/hub/rest/Api.java

@@ -34,6 +34,7 @@
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.here.xyz.XyzSerializable.Public;
 import com.here.xyz.hub.Service;
+import com.here.xyz.hub.XYZHubRESTVerticle;
 import com.here.xyz.hub.connectors.models.Space.CacheProfile;
 import com.here.xyz.hub.rest.ApiParam.Query;
 import com.here.xyz.hub.task.FeatureTask;
@@ -52,6 +53,7 @@
 import com.here.xyz.responses.changesets.ChangesetCollection;
 import com.here.xyz.util.service.HttpException;
 import com.here.xyz.util.service.logging.LogUtil;
+import com.here.xyz.util.service.rest.TooManyRequestsException;
 import io.netty.handler.codec.http.HttpResponseStatus;
 import io.vertx.core.MultiMap;
 import io.vertx.core.buffer.Buffer;
@@ -305,6 +307,9 @@ void sendResponse(final SpaceTask<?> task) throws JsonProcessingException {
    * @param e the exception that should be used to generate an {@link ErrorResponse}, if null an internal server error is returned.
    */
   void sendErrorResponse(final Task task, final Throwable e) {
+    if (e instanceof TooManyRequestsException throttleException)
+      XYZHubRESTVerticle.addStreamInfo(task.context, "THR", throttleException.reason); //Set the throttling reason at the stream-info header
+
     sendErrorResponse(task.context, e);
   }