Simplify script installation / make it fully transparent to users of …

…DatabaseSettings

Signed-off-by: Benjamin Rögner <[email protected]>

xyz-hub-service/src/main/java/com/here/xyz/httpconnector/config/JDBCMaintainer.java

@@ -34,9 +34,9 @@
 import com.here.xyz.responses.maintenance.SpaceStatus;
 import com.here.xyz.util.Hasher;
 import com.here.xyz.util.db.ConnectorParameters;
-import com.here.xyz.util.db.DatabaseSettings;
 import com.here.xyz.util.db.JdbcClient;
 import com.here.xyz.util.db.SQLQuery;
+import com.here.xyz.util.db.datasource.DatabaseSettings;
 import com.here.xyz.util.service.Core;
 import io.vertx.core.Future;
 import java.io.IOException;

xyz-hub-service/src/main/java/com/here/xyz/httpconnector/task/JdbcBasedHandler.java

@@ -22,9 +22,9 @@
 import com.here.xyz.httpconnector.CService;
 import com.here.xyz.httpconnector.util.web.LegacyHubWebClient;
 import com.here.xyz.util.db.ConnectorParameters;
-import com.here.xyz.util.db.DatabaseSettings;
 import com.here.xyz.util.db.ECPSTool;
 import com.here.xyz.util.db.JdbcClient;
+import com.here.xyz.util.db.datasource.DatabaseSettings;
 import com.here.xyz.util.db.datasource.PooledDataSources;
 import com.here.xyz.util.service.Core;
 import io.vertx.core.Future;

xyz-hub-service/src/main/java/com/here/xyz/httpconnector/task/StatusHandler.java

@@ -27,9 +27,9 @@
 import com.here.xyz.httpconnector.util.status.RDSStatus;
 import com.here.xyz.httpconnector.util.status.RunningQueryStatistic;
 import com.here.xyz.httpconnector.util.status.RunningQueryStatistics;
-import com.here.xyz.util.db.DatabaseSettings;
 import com.here.xyz.util.db.JdbcClient;
 import com.here.xyz.util.db.SQLQuery;
+import com.here.xyz.util.db.datasource.DatabaseSettings;
 import io.vertx.core.Future;
 import java.sql.ResultSet;
 import java.sql.SQLException;

xyz-hub-service/src/main/java/com/here/xyz/hub/config/jdbc/JDBCConfigClient.java

@@ -21,9 +21,9 @@
 
 import com.here.xyz.httpconnector.CService;
 import com.here.xyz.hub.Service;
-import com.here.xyz.util.db.DatabaseSettings;
 import com.here.xyz.util.db.JdbcClient;
 import com.here.xyz.util.db.SQLQuery;
+import com.here.xyz.util.db.datasource.DatabaseSettings;
 import com.here.xyz.util.db.datasource.PooledDataSources;
 import io.vertx.core.Future;
 import io.vertx.core.json.Json;

xyz-jobs/xyz-job-steps/src/main/java/com/here/xyz/jobs/steps/execution/LambdaBasedStep.java

@@ -117,12 +117,6 @@ public abstract class LambdaBasedStep<T extends LambdaBasedStep> extends Step<T>
   @JsonIgnore
   public abstract AsyncExecutionState getExecutionState() throws UnknownStateException;
 
-  /**
-   * This method can be overridden by subclasses.
-   * It is getting called at the beginning of each Lambda execution and can be used for initialization.
-   */
-  public void init() throws Exception {};
-
   private void startExecution() throws Exception {
     updateState(RUNNING);
 
@@ -493,12 +487,6 @@ public void handleRequest(InputStream inputStream, OutputStream outputStream, Co
 
         new LambdaFunctionRuntime(context, request.getStep().getGlobalStepId());
 
-        try {
-            request.getStep().init();
-        } catch (Exception e) {
-            throw new RuntimeException("Error during initialization",e);
-        }
-
         if (request.getStep() == null)
           throw new NullPointerException("Malformed step request, missing step definition.");
 

xyz-jobs/xyz-job-steps/src/main/java/com/here/xyz/jobs/steps/execution/db/Database.java

@@ -21,8 +21,8 @@
 
 import static com.here.xyz.jobs.steps.execution.db.Database.DatabaseRole.READER;
 import static com.here.xyz.jobs.steps.execution.db.Database.DatabaseRole.WRITER;
-import static com.here.xyz.util.db.DatabaseSettings.PSQL_HOST;
-import static com.here.xyz.util.db.DatabaseSettings.PSQL_REPLICA_HOST;
+import static com.here.xyz.util.db.datasource.DatabaseSettings.PSQL_HOST;
+import static com.here.xyz.util.db.datasource.DatabaseSettings.PSQL_REPLICA_HOST;
 
 import com.google.common.cache.Cache;
 import com.google.common.cache.CacheBuilder;
@@ -32,9 +32,9 @@
 import com.here.xyz.models.hub.Connector;
 import com.here.xyz.util.Hasher;
 import com.here.xyz.util.db.ConnectorParameters;
-import com.here.xyz.util.db.DatabaseSettings;
 import com.here.xyz.util.db.ECPSTool;
 import com.here.xyz.util.db.datasource.DataSourceProvider;
+import com.here.xyz.util.db.datasource.DatabaseSettings;
 import com.here.xyz.util.db.datasource.PooledDataSources;
 import com.here.xyz.util.web.HubWebClient;
 import com.here.xyz.util.web.HubWebClientAsync;
@@ -113,8 +113,9 @@ static void closeAll() {
 
   DatabaseSettings getDatabaseSettings() {
     if (dbSettings == null)
-      dbSettings = new RestrictedDatabaseSettings(getName(), connectorDbSettingsMap, SCRIPT_RESOURCE_PATH)
-          .withApplicationName("JobFramework");
+      dbSettings = new RestrictedDatabaseSettings(getName(), connectorDbSettingsMap)
+          .withApplicationName("JobFramework")
+          .withScriptResourcePaths(List.of(SCRIPT_RESOURCE_PATH));
     dbSettings.setStatementTimeoutSeconds(600);
     return dbSettings;
   }
@@ -208,7 +209,8 @@ private static List<Database> loadDatabasesForConnector(Connector connector) {
             connectorParameters.getEcps());
         fixLocalDbHosts(connectorDbSettingsMap);
 
-        DatabaseSettings connectorDbSettings = new DatabaseSettings(connector.id, connectorDbSettingsMap, SCRIPT_RESOURCE_PATH );
+        DatabaseSettings connectorDbSettings = new DatabaseSettings(connector.id, connectorDbSettingsMap)
+            .withScriptResourcePaths(List.of(SCRIPT_RESOURCE_PATH));
 
         String rdsClusterId = getClusterIdFromHostname(connectorDbSettings.getHost());
 
@@ -341,8 +343,8 @@ public enum DatabaseRole {
    * depending on the role of the parent database instance.
    */
   private class RestrictedDatabaseSettings extends DatabaseSettings {
-    public RestrictedDatabaseSettings(String id, Map<String, Object> databaseSettings, String resourcePath) {
-      super(id, databaseSettings, resourcePath);
+    public RestrictedDatabaseSettings(String id, Map<String, Object> databaseSettings) {
+      super(id, databaseSettings);
     }
 
     @Override

xyz-jobs/xyz-job-steps/src/main/java/com/here/xyz/jobs/steps/execution/db/DatabaseBasedStep.java

@@ -281,13 +281,6 @@ protected final DataSourceProvider requestResource(Database db, double estimated
     return db.getDataSources();
   }
 
-  /**
-   * Checks whether the latest version of all SQL scripts is installed on the DB.
-   */
-  protected synchronized void checkScripts(Database db) {
-    db.getDatabaseSettings().checkScripts();
-  }
-
   private record RunningQuery(@JsonProperty("queryId") String queryId, @JsonProperty("dbName") String dbName,
                               @JsonProperty("dbId") String dbId) implements XyzSerializable {}
 }

xyz-jobs/xyz-job-steps/src/main/java/com/here/xyz/jobs/steps/execution/db/SingleDatabaseSettings.java

@@ -20,7 +20,7 @@
 package com.here.xyz.jobs.steps.execution.db;
 
 import com.here.xyz.jobs.steps.execution.db.Database.DatabaseRole;
-import com.here.xyz.util.db.DatabaseSettings;
+import com.here.xyz.util.db.datasource.DatabaseSettings;
 import java.util.Map;
 
 public class SingleDatabaseSettings extends DatabaseSettings {

xyz-jobs/xyz-job-steps/src/main/java/com/here/xyz/jobs/steps/impl/SpaceBasedStep.java

@@ -80,11 +80,6 @@ public T withSpaceId(String spaceId) {
     return (T) this;
   }
 
-  @Override
-  public void init() throws WebClientException {
-    checkScripts(db());
-  }
-
   protected final String getRootTableName(Space space) throws WebClientException {
     return getTableNameFromSpaceParamsOrSpaceId(space.getStorage().getParams(), space.getId(),
         ConnectorParameters.fromMap(hubWebClient().loadConnector(space.getStorage().getId()).params).isEnableHashedSpaceId());

xyz-jobs/xyz-job-steps/src/main/java/com/here/xyz/jobs/util/test/StepTestBase.java

@@ -19,6 +19,15 @@
 
 package com.here.xyz.jobs.util.test;
 
+import static com.google.common.net.HttpHeaders.CONTENT_TYPE;
+import static com.here.xyz.jobs.steps.execution.LambdaBasedStep.LambdaStepRequest.RequestType.START_EXECUTION;
+import static com.here.xyz.jobs.steps.execution.LambdaBasedStep.LambdaStepRequest.RequestType.SUCCESS_CALLBACK;
+import static com.here.xyz.jobs.steps.inputs.Input.inputS3Prefix;
+import static com.here.xyz.util.Random.randomAlpha;
+import static com.here.xyz.util.db.pg.XyzSpaceTableHelper.buildSpaceTableDropIndexQueries;
+import static java.lang.Thread.sleep;
+import static java.net.http.HttpClient.Redirect.NORMAL;
+
 import com.amazonaws.services.lambda.runtime.Context;
 import com.google.common.io.ByteStreams;
 import com.google.common.net.MediaType;
@@ -35,22 +44,13 @@
 import com.here.xyz.models.hub.Space;
 import com.here.xyz.models.hub.Tag;
 import com.here.xyz.responses.StatisticsResponse;
-import com.here.xyz.util.db.DatabaseSettings;
 import com.here.xyz.util.db.SQLQuery;
 import com.here.xyz.util.db.datasource.DataSourceProvider;
+import com.here.xyz.util.db.datasource.DatabaseSettings;
 import com.here.xyz.util.db.datasource.PooledDataSources;
 import com.here.xyz.util.service.aws.SimulatedContext;
 import com.here.xyz.util.web.HubWebClient;
 import com.here.xyz.util.web.XyzWebClient;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
-import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
-import software.amazon.awssdk.core.SdkBytes;
-import software.amazon.awssdk.regions.Region;
-import software.amazon.awssdk.services.lambda.LambdaClient;
-import software.amazon.awssdk.services.lambda.model.InvokeRequest;
-
 import java.io.BufferedReader;
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
@@ -70,15 +70,14 @@
 import java.util.Map;
 import java.util.UUID;
 import java.util.zip.GZIPInputStream;
-
-import static com.google.common.net.HttpHeaders.CONTENT_TYPE;
-import static com.here.xyz.jobs.steps.execution.LambdaBasedStep.LambdaStepRequest.RequestType.START_EXECUTION;
-import static com.here.xyz.jobs.steps.execution.LambdaBasedStep.LambdaStepRequest.RequestType.SUCCESS_CALLBACK;
-import static com.here.xyz.jobs.steps.inputs.Input.inputS3Prefix;
-import static com.here.xyz.util.Random.randomAlpha;
-import static com.here.xyz.util.db.pg.XyzSpaceTableHelper.buildSpaceTableDropIndexQueries;
-import static java.lang.Thread.sleep;
-import static java.net.http.HttpClient.Redirect.NORMAL;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
+import software.amazon.awssdk.core.SdkBytes;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.lambda.LambdaClient;
+import software.amazon.awssdk.services.lambda.model.InvokeRequest;
 
 public class StepTestBase {
   private static final Logger logger = LogManager.getLogger();

xyz-psql-connector/src/main/java/com/here/xyz/psql/DatabaseHandler.java

@@ -30,7 +30,6 @@
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.here.xyz.connectors.ErrorResponseException;
 import com.here.xyz.connectors.StorageConnector;
-import com.here.xyz.util.runtime.FunctionRuntime;
 import com.here.xyz.events.Event;
 import com.here.xyz.events.GetFeaturesByIdEvent;
 import com.here.xyz.events.ModifyFeaturesEvent;
@@ -47,15 +46,12 @@
 import com.here.xyz.responses.XyzError;
 import com.here.xyz.util.Random;
 import com.here.xyz.util.db.ConnectorParameters;
-import com.here.xyz.util.db.DatabaseSettings;
 import com.here.xyz.util.db.ECPSTool;
 import com.here.xyz.util.db.SQLQuery;
 import com.here.xyz.util.db.datasource.CachedPooledDataSources;
 import com.here.xyz.util.db.datasource.DataSourceProvider;
-import com.here.xyz.util.db.datasource.StaticDataSources;
-import com.here.xyz.util.db.pg.Script;
-import java.io.IOException;
-import java.net.URISyntaxException;
+import com.here.xyz.util.db.datasource.DatabaseSettings;
+import com.here.xyz.util.runtime.FunctionRuntime;
 import java.sql.BatchUpdateException;
 import java.sql.Connection;
 import java.sql.SQLException;
@@ -67,7 +63,6 @@
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
-import java.util.concurrent.ConcurrentHashMap;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 import org.apache.logging.log4j.LogManager;
@@ -114,11 +109,10 @@ protected void initialize(Event event) {
 
         //Decrypt the ECPS into an instance of DatabaseSettings
         dbSettings = new DatabaseSettings(connectorId,
-            ECPSTool.decryptToMap(FunctionRuntime.getInstance().getEnvironmentVariable(ECPS_PHRASE), connectorParams.getEcps()), SCRIPT_RESOURCE_PATH)
-            .withApplicationName(FunctionRuntime.getInstance().getApplicationName());
-
+            ECPSTool.decryptToMap(FunctionRuntime.getInstance().getEnvironmentVariable(ECPS_PHRASE), connectorParams.getEcps()))
+            .withApplicationName(FunctionRuntime.getInstance().getApplicationName())
+            .withScriptResourcePaths(List.of(SCRIPT_RESOURCE_PATH));
         //TODO - set scriptResourcePath if ext & h3 functions should get installed here.
-        dbSettings.checkScripts();
 
         dataSourceProvider = new CachedPooledDataSources(dbSettings);
         retryAttempted = false;

xyz-psql-connector/src/main/java/com/here/xyz/psql/DatabaseMaintainer.java

@@ -24,8 +24,8 @@
 import com.here.xyz.connectors.AbstractConnectorHandler.TraceItem;
 import com.here.xyz.psql.factory.MaintenanceSQL;
 import com.here.xyz.util.db.ConnectorParameters;
-import com.here.xyz.util.db.DatabaseSettings;
 import com.here.xyz.util.db.datasource.DataSourceProvider;
+import com.here.xyz.util.db.datasource.DatabaseSettings;
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStream;

xyz-psql-connector/src/main/java/com/here/xyz/psql/DatabaseWriter.java

@@ -31,16 +31,16 @@
 
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.here.xyz.XyzSerializable.Static;
-import com.here.xyz.util.runtime.FunctionRuntime;
 import com.here.xyz.events.ModifyFeaturesEvent;
 import com.here.xyz.models.geojson.implementation.Feature;
 import com.here.xyz.models.geojson.implementation.FeatureCollection;
 import com.here.xyz.models.geojson.implementation.Geometry;
 import com.here.xyz.models.geojson.implementation.Properties;
 import com.here.xyz.models.geojson.implementation.XyzNamespace;
 import com.here.xyz.psql.query.XyzEventBasedQueryRunner;
-import com.here.xyz.util.db.DatabaseSettings;
 import com.here.xyz.util.db.SQLQuery;
+import com.here.xyz.util.db.datasource.DatabaseSettings;
+import com.here.xyz.util.runtime.FunctionRuntime;
 import java.sql.Connection;
 import java.sql.PreparedStatement;
 import java.sql.SQLException;

xyz-psql-connector/src/main/java/com/here/xyz/psql/QueryRunner.java

@@ -20,10 +20,10 @@
 package com.here.xyz.psql;
 
 import com.here.xyz.connectors.ErrorResponseException;
-import com.here.xyz.util.runtime.FunctionRuntime;
-import com.here.xyz.util.db.DatabaseSettings;
 import com.here.xyz.util.db.SQLQuery;
 import com.here.xyz.util.db.datasource.DataSourceProvider;
+import com.here.xyz.util.db.datasource.DatabaseSettings;
+import com.here.xyz.util.runtime.FunctionRuntime;
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import org.apache.commons.dbutils.ResultSetHandler;

xyz-psql-connector/src/test/java/com/here/xyz/psql/PSQLAbstractIT.java

@@ -20,9 +20,9 @@
 package com.here.xyz.psql;
 
 import static com.here.xyz.psql.DatabaseHandler.ECPS_PHRASE;
-import static com.here.xyz.util.db.DatabaseSettings.PSQL_HOST;
-import static com.here.xyz.util.db.DatabaseSettings.PSQL_PASSWORD;
-import static com.here.xyz.util.db.DatabaseSettings.PSQL_USER;
+import static com.here.xyz.util.db.datasource.DatabaseSettings.PSQL_HOST;
+import static com.here.xyz.util.db.datasource.DatabaseSettings.PSQL_PASSWORD;
+import static com.here.xyz.util.db.datasource.DatabaseSettings.PSQL_USER;
 import static io.restassured.path.json.JsonPath.with;
 import static org.junit.Assert.assertEquals;
 

xyz-psql-connector/src/test/java/com/here/xyz/psql/PSQLXyzConnectorIT.java

@@ -24,11 +24,11 @@
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.here.xyz.XyzSerializable;
-import com.here.xyz.util.runtime.FunctionRuntime;
 import com.here.xyz.events.HealthCheckEvent;
 import com.here.xyz.util.db.ConnectorParameters;
-import com.here.xyz.util.db.DatabaseSettings;
 import com.here.xyz.util.db.ECPSTool;
+import com.here.xyz.util.db.datasource.DatabaseSettings;
+import com.here.xyz.util.runtime.FunctionRuntime;
 import java.util.HashMap;
 import java.util.Map;
 import org.junit.Test;

xyz-util/src/main/java/com/here/xyz/psql/query/QueryBuilder.java

@@ -19,9 +19,9 @@
 
 package com.here.xyz.psql.query;
 
-import com.here.xyz.util.db.DatabaseSettings;
 import com.here.xyz.util.db.SQLQuery;
 import com.here.xyz.util.db.datasource.DataSourceProvider;
+import com.here.xyz.util.db.datasource.DatabaseSettings;
 
 public abstract class QueryBuilder<I> {
   private DataSourceProvider dataSourceProvider;

xyz-util/src/main/java/com/here/xyz/util/db/datasource/CachedPooledDataSources.java

@@ -19,7 +19,6 @@
 
 package com.here.xyz.util.db.datasource;
 
-import com.here.xyz.util.db.DatabaseSettings;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;

xyz-util/src/main/java/com/here/xyz/util/db/datasource/DataSourceProvider.java

@@ -19,7 +19,6 @@
 
 package com.here.xyz.util.db.datasource;
 
-import com.here.xyz.util.db.DatabaseSettings;
 import javax.sql.DataSource;
 
 public abstract class DataSourceProvider implements AutoCloseable {
@@ -28,6 +27,8 @@ public abstract class DataSourceProvider implements AutoCloseable {
 
   public DataSourceProvider(DatabaseSettings dbSettings) {
     this.dbSettings = dbSettings;
+    if (dbSettings != null)
+      dbSettings.init();
   }
 
   public abstract DataSource getReader();