From 6f3575a5b3faebfce777ef183e8476b40a67d2fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20R=C3=B6gner?= Date: Fri, 18 Oct 2024 15:34:22 +0200 Subject: [PATCH] Simplify script installation / make it fully transparent to users of DatabaseSettings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Benjamin Rögner --- .../httpconnector/config/JDBCMaintainer.java | 2 +- .../httpconnector/task/JdbcBasedHandler.java | 2 +- .../xyz/httpconnector/task/StatusHandler.java | 2 +- .../xyz/hub/config/jdbc/JDBCConfigClient.java | 2 +- .../jobs/steps/execution/LambdaBasedStep.java | 12 ---- .../xyz/jobs/steps/execution/db/Database.java | 18 +++--- .../steps/execution/db/DatabaseBasedStep.java | 7 --- .../execution/db/SingleDatabaseSettings.java | 2 +- .../xyz/jobs/steps/impl/SpaceBasedStep.java | 5 -- .../here/xyz/jobs/util/test/StepTestBase.java | 37 ++++++----- .../com/here/xyz/psql/DatabaseHandler.java | 16 ++--- .../com/here/xyz/psql/DatabaseMaintainer.java | 2 +- .../com/here/xyz/psql/DatabaseWriter.java | 4 +- .../java/com/here/xyz/psql/QueryRunner.java | 4 +- .../com/here/xyz/psql/PSQLAbstractIT.java | 6 +- .../com/here/xyz/psql/PSQLXyzConnectorIT.java | 4 +- .../com/here/xyz/psql/query/QueryBuilder.java | 2 +- .../datasource/CachedPooledDataSources.java | 1 - .../db/datasource/DataSourceProvider.java | 3 +- .../db/{ => datasource}/DatabaseSettings.java | 62 ++++++++++++------- .../util/db/datasource/PooledDataSources.java | 1 - .../util/db/datasource/StaticDataSources.java | 1 - .../java/com/here/xyz/test/SQLITBase.java | 2 +- .../java/com/here/xyz/test/SQLScriptsIT.java | 2 +- 24 files changed, 91 insertions(+), 108 deletions(-) rename xyz-util/src/main/java/com/here/xyz/util/db/{ => datasource}/DatabaseSettings.java (91%) diff --git a/xyz-hub-service/src/main/java/com/here/xyz/httpconnector/config/JDBCMaintainer.java b/xyz-hub-service/src/main/java/com/here/xyz/httpconnector/config/JDBCMaintainer.java index 5076e62e39..31b5c7839d 100644 --- a/xyz-hub-service/src/main/java/com/here/xyz/httpconnector/config/JDBCMaintainer.java +++ b/xyz-hub-service/src/main/java/com/here/xyz/httpconnector/config/JDBCMaintainer.java @@ -34,9 +34,9 @@ import com.here.xyz.responses.maintenance.SpaceStatus; import com.here.xyz.util.Hasher; import com.here.xyz.util.db.ConnectorParameters; -import com.here.xyz.util.db.DatabaseSettings; import com.here.xyz.util.db.JdbcClient; import com.here.xyz.util.db.SQLQuery; +import com.here.xyz.util.db.datasource.DatabaseSettings; import com.here.xyz.util.service.Core; import io.vertx.core.Future; import java.io.IOException; diff --git a/xyz-hub-service/src/main/java/com/here/xyz/httpconnector/task/JdbcBasedHandler.java b/xyz-hub-service/src/main/java/com/here/xyz/httpconnector/task/JdbcBasedHandler.java index 4d4b831f58..8921a5cadf 100644 --- a/xyz-hub-service/src/main/java/com/here/xyz/httpconnector/task/JdbcBasedHandler.java +++ b/xyz-hub-service/src/main/java/com/here/xyz/httpconnector/task/JdbcBasedHandler.java @@ -22,9 +22,9 @@ import com.here.xyz.httpconnector.CService; import com.here.xyz.httpconnector.util.web.LegacyHubWebClient; import com.here.xyz.util.db.ConnectorParameters; -import com.here.xyz.util.db.DatabaseSettings; import com.here.xyz.util.db.ECPSTool; import com.here.xyz.util.db.JdbcClient; +import com.here.xyz.util.db.datasource.DatabaseSettings; import com.here.xyz.util.db.datasource.PooledDataSources; import com.here.xyz.util.service.Core; import io.vertx.core.Future; diff --git a/xyz-hub-service/src/main/java/com/here/xyz/httpconnector/task/StatusHandler.java b/xyz-hub-service/src/main/java/com/here/xyz/httpconnector/task/StatusHandler.java index 1dee8f3d37..a2850ad02a 100644 --- a/xyz-hub-service/src/main/java/com/here/xyz/httpconnector/task/StatusHandler.java +++ b/xyz-hub-service/src/main/java/com/here/xyz/httpconnector/task/StatusHandler.java @@ -27,9 +27,9 @@ import com.here.xyz.httpconnector.util.status.RDSStatus; import com.here.xyz.httpconnector.util.status.RunningQueryStatistic; import com.here.xyz.httpconnector.util.status.RunningQueryStatistics; -import com.here.xyz.util.db.DatabaseSettings; import com.here.xyz.util.db.JdbcClient; import com.here.xyz.util.db.SQLQuery; +import com.here.xyz.util.db.datasource.DatabaseSettings; import io.vertx.core.Future; import java.sql.ResultSet; import java.sql.SQLException; diff --git a/xyz-hub-service/src/main/java/com/here/xyz/hub/config/jdbc/JDBCConfigClient.java b/xyz-hub-service/src/main/java/com/here/xyz/hub/config/jdbc/JDBCConfigClient.java index 49c1986941..2f2504a556 100644 --- a/xyz-hub-service/src/main/java/com/here/xyz/hub/config/jdbc/JDBCConfigClient.java +++ b/xyz-hub-service/src/main/java/com/here/xyz/hub/config/jdbc/JDBCConfigClient.java @@ -21,9 +21,9 @@ import com.here.xyz.httpconnector.CService; import com.here.xyz.hub.Service; -import com.here.xyz.util.db.DatabaseSettings; import com.here.xyz.util.db.JdbcClient; import com.here.xyz.util.db.SQLQuery; +import com.here.xyz.util.db.datasource.DatabaseSettings; import com.here.xyz.util.db.datasource.PooledDataSources; import io.vertx.core.Future; import io.vertx.core.json.Json; diff --git a/xyz-jobs/xyz-job-steps/src/main/java/com/here/xyz/jobs/steps/execution/LambdaBasedStep.java b/xyz-jobs/xyz-job-steps/src/main/java/com/here/xyz/jobs/steps/execution/LambdaBasedStep.java index bf3b62976c..d2c167ded8 100644 --- a/xyz-jobs/xyz-job-steps/src/main/java/com/here/xyz/jobs/steps/execution/LambdaBasedStep.java +++ b/xyz-jobs/xyz-job-steps/src/main/java/com/here/xyz/jobs/steps/execution/LambdaBasedStep.java @@ -117,12 +117,6 @@ public abstract class LambdaBasedStep extends Step @JsonIgnore public abstract AsyncExecutionState getExecutionState() throws UnknownStateException; - /** - * This method can be overridden by subclasses. - * It is getting called at the beginning of each Lambda execution and can be used for initialization. - */ - public void init() throws Exception {}; - private void startExecution() throws Exception { updateState(RUNNING); @@ -493,12 +487,6 @@ public void handleRequest(InputStream inputStream, OutputStream outputStream, Co new LambdaFunctionRuntime(context, request.getStep().getGlobalStepId()); - try { - request.getStep().init(); - } catch (Exception e) { - throw new RuntimeException("Error during initialization",e); - } - if (request.getStep() == null) throw new NullPointerException("Malformed step request, missing step definition."); diff --git a/xyz-jobs/xyz-job-steps/src/main/java/com/here/xyz/jobs/steps/execution/db/Database.java b/xyz-jobs/xyz-job-steps/src/main/java/com/here/xyz/jobs/steps/execution/db/Database.java index 4ab1074dab..c97412785c 100644 --- a/xyz-jobs/xyz-job-steps/src/main/java/com/here/xyz/jobs/steps/execution/db/Database.java +++ b/xyz-jobs/xyz-job-steps/src/main/java/com/here/xyz/jobs/steps/execution/db/Database.java @@ -21,8 +21,8 @@ import static com.here.xyz.jobs.steps.execution.db.Database.DatabaseRole.READER; import static com.here.xyz.jobs.steps.execution.db.Database.DatabaseRole.WRITER; -import static com.here.xyz.util.db.DatabaseSettings.PSQL_HOST; -import static com.here.xyz.util.db.DatabaseSettings.PSQL_REPLICA_HOST; +import static com.here.xyz.util.db.datasource.DatabaseSettings.PSQL_HOST; +import static com.here.xyz.util.db.datasource.DatabaseSettings.PSQL_REPLICA_HOST; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; @@ -32,9 +32,9 @@ import com.here.xyz.models.hub.Connector; import com.here.xyz.util.Hasher; import com.here.xyz.util.db.ConnectorParameters; -import com.here.xyz.util.db.DatabaseSettings; import com.here.xyz.util.db.ECPSTool; import com.here.xyz.util.db.datasource.DataSourceProvider; +import com.here.xyz.util.db.datasource.DatabaseSettings; import com.here.xyz.util.db.datasource.PooledDataSources; import com.here.xyz.util.web.HubWebClient; import com.here.xyz.util.web.HubWebClientAsync; @@ -113,8 +113,9 @@ static void closeAll() { DatabaseSettings getDatabaseSettings() { if (dbSettings == null) - dbSettings = new RestrictedDatabaseSettings(getName(), connectorDbSettingsMap, SCRIPT_RESOURCE_PATH) - .withApplicationName("JobFramework"); + dbSettings = new RestrictedDatabaseSettings(getName(), connectorDbSettingsMap) + .withApplicationName("JobFramework") + .withScriptResourcePaths(List.of(SCRIPT_RESOURCE_PATH)); dbSettings.setStatementTimeoutSeconds(600); return dbSettings; } @@ -208,7 +209,8 @@ private static List loadDatabasesForConnector(Connector connector) { connectorParameters.getEcps()); fixLocalDbHosts(connectorDbSettingsMap); - DatabaseSettings connectorDbSettings = new DatabaseSettings(connector.id, connectorDbSettingsMap, SCRIPT_RESOURCE_PATH ); + DatabaseSettings connectorDbSettings = new DatabaseSettings(connector.id, connectorDbSettingsMap) + .withScriptResourcePaths(List.of(SCRIPT_RESOURCE_PATH)); String rdsClusterId = getClusterIdFromHostname(connectorDbSettings.getHost()); @@ -341,8 +343,8 @@ public enum DatabaseRole { * depending on the role of the parent database instance. */ private class RestrictedDatabaseSettings extends DatabaseSettings { - public RestrictedDatabaseSettings(String id, Map databaseSettings, String resourcePath) { - super(id, databaseSettings, resourcePath); + public RestrictedDatabaseSettings(String id, Map databaseSettings) { + super(id, databaseSettings); } @Override diff --git a/xyz-jobs/xyz-job-steps/src/main/java/com/here/xyz/jobs/steps/execution/db/DatabaseBasedStep.java b/xyz-jobs/xyz-job-steps/src/main/java/com/here/xyz/jobs/steps/execution/db/DatabaseBasedStep.java index 6c94d0925a..3cc1904793 100644 --- a/xyz-jobs/xyz-job-steps/src/main/java/com/here/xyz/jobs/steps/execution/db/DatabaseBasedStep.java +++ b/xyz-jobs/xyz-job-steps/src/main/java/com/here/xyz/jobs/steps/execution/db/DatabaseBasedStep.java @@ -281,13 +281,6 @@ protected final DataSourceProvider requestResource(Database db, double estimated return db.getDataSources(); } - /** - * Checks whether the latest version of all SQL scripts is installed on the DB. - */ - protected synchronized void checkScripts(Database db) { - db.getDatabaseSettings().checkScripts(); - } - private record RunningQuery(@JsonProperty("queryId") String queryId, @JsonProperty("dbName") String dbName, @JsonProperty("dbId") String dbId) implements XyzSerializable {} } diff --git a/xyz-jobs/xyz-job-steps/src/main/java/com/here/xyz/jobs/steps/execution/db/SingleDatabaseSettings.java b/xyz-jobs/xyz-job-steps/src/main/java/com/here/xyz/jobs/steps/execution/db/SingleDatabaseSettings.java index d696f029b3..52eb5b4e13 100644 --- a/xyz-jobs/xyz-job-steps/src/main/java/com/here/xyz/jobs/steps/execution/db/SingleDatabaseSettings.java +++ b/xyz-jobs/xyz-job-steps/src/main/java/com/here/xyz/jobs/steps/execution/db/SingleDatabaseSettings.java @@ -20,7 +20,7 @@ package com.here.xyz.jobs.steps.execution.db; import com.here.xyz.jobs.steps.execution.db.Database.DatabaseRole; -import com.here.xyz.util.db.DatabaseSettings; +import com.here.xyz.util.db.datasource.DatabaseSettings; import java.util.Map; public class SingleDatabaseSettings extends DatabaseSettings { diff --git a/xyz-jobs/xyz-job-steps/src/main/java/com/here/xyz/jobs/steps/impl/SpaceBasedStep.java b/xyz-jobs/xyz-job-steps/src/main/java/com/here/xyz/jobs/steps/impl/SpaceBasedStep.java index c19135f8db..040a20a328 100644 --- a/xyz-jobs/xyz-job-steps/src/main/java/com/here/xyz/jobs/steps/impl/SpaceBasedStep.java +++ b/xyz-jobs/xyz-job-steps/src/main/java/com/here/xyz/jobs/steps/impl/SpaceBasedStep.java @@ -80,11 +80,6 @@ public T withSpaceId(String spaceId) { return (T) this; } - @Override - public void init() throws WebClientException { - checkScripts(db()); - } - protected final String getRootTableName(Space space) throws WebClientException { return getTableNameFromSpaceParamsOrSpaceId(space.getStorage().getParams(), space.getId(), ConnectorParameters.fromMap(hubWebClient().loadConnector(space.getStorage().getId()).params).isEnableHashedSpaceId()); diff --git a/xyz-jobs/xyz-job-steps/src/main/java/com/here/xyz/jobs/util/test/StepTestBase.java b/xyz-jobs/xyz-job-steps/src/main/java/com/here/xyz/jobs/util/test/StepTestBase.java index 66c656d8a4..cd4ed4c40a 100644 --- a/xyz-jobs/xyz-job-steps/src/main/java/com/here/xyz/jobs/util/test/StepTestBase.java +++ b/xyz-jobs/xyz-job-steps/src/main/java/com/here/xyz/jobs/util/test/StepTestBase.java @@ -19,6 +19,15 @@ package com.here.xyz.jobs.util.test; +import static com.google.common.net.HttpHeaders.CONTENT_TYPE; +import static com.here.xyz.jobs.steps.execution.LambdaBasedStep.LambdaStepRequest.RequestType.START_EXECUTION; +import static com.here.xyz.jobs.steps.execution.LambdaBasedStep.LambdaStepRequest.RequestType.SUCCESS_CALLBACK; +import static com.here.xyz.jobs.steps.inputs.Input.inputS3Prefix; +import static com.here.xyz.util.Random.randomAlpha; +import static com.here.xyz.util.db.pg.XyzSpaceTableHelper.buildSpaceTableDropIndexQueries; +import static java.lang.Thread.sleep; +import static java.net.http.HttpClient.Redirect.NORMAL; + import com.amazonaws.services.lambda.runtime.Context; import com.google.common.io.ByteStreams; import com.google.common.net.MediaType; @@ -35,22 +44,13 @@ import com.here.xyz.models.hub.Space; import com.here.xyz.models.hub.Tag; import com.here.xyz.responses.StatisticsResponse; -import com.here.xyz.util.db.DatabaseSettings; import com.here.xyz.util.db.SQLQuery; import com.here.xyz.util.db.datasource.DataSourceProvider; +import com.here.xyz.util.db.datasource.DatabaseSettings; import com.here.xyz.util.db.datasource.PooledDataSources; import com.here.xyz.util.service.aws.SimulatedContext; import com.here.xyz.util.web.HubWebClient; import com.here.xyz.util.web.XyzWebClient; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; -import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; -import software.amazon.awssdk.core.SdkBytes; -import software.amazon.awssdk.regions.Region; -import software.amazon.awssdk.services.lambda.LambdaClient; -import software.amazon.awssdk.services.lambda.model.InvokeRequest; - import java.io.BufferedReader; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -70,15 +70,14 @@ import java.util.Map; import java.util.UUID; import java.util.zip.GZIPInputStream; - -import static com.google.common.net.HttpHeaders.CONTENT_TYPE; -import static com.here.xyz.jobs.steps.execution.LambdaBasedStep.LambdaStepRequest.RequestType.START_EXECUTION; -import static com.here.xyz.jobs.steps.execution.LambdaBasedStep.LambdaStepRequest.RequestType.SUCCESS_CALLBACK; -import static com.here.xyz.jobs.steps.inputs.Input.inputS3Prefix; -import static com.here.xyz.util.Random.randomAlpha; -import static com.here.xyz.util.db.pg.XyzSpaceTableHelper.buildSpaceTableDropIndexQueries; -import static java.lang.Thread.sleep; -import static java.net.http.HttpClient.Redirect.NORMAL; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.core.SdkBytes; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.lambda.LambdaClient; +import software.amazon.awssdk.services.lambda.model.InvokeRequest; public class StepTestBase { private static final Logger logger = LogManager.getLogger(); diff --git a/xyz-psql-connector/src/main/java/com/here/xyz/psql/DatabaseHandler.java b/xyz-psql-connector/src/main/java/com/here/xyz/psql/DatabaseHandler.java index 86f34b81e1..9940f7f073 100644 --- a/xyz-psql-connector/src/main/java/com/here/xyz/psql/DatabaseHandler.java +++ b/xyz-psql-connector/src/main/java/com/here/xyz/psql/DatabaseHandler.java @@ -30,7 +30,6 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.here.xyz.connectors.ErrorResponseException; import com.here.xyz.connectors.StorageConnector; -import com.here.xyz.util.runtime.FunctionRuntime; import com.here.xyz.events.Event; import com.here.xyz.events.GetFeaturesByIdEvent; import com.here.xyz.events.ModifyFeaturesEvent; @@ -47,15 +46,12 @@ import com.here.xyz.responses.XyzError; import com.here.xyz.util.Random; import com.here.xyz.util.db.ConnectorParameters; -import com.here.xyz.util.db.DatabaseSettings; import com.here.xyz.util.db.ECPSTool; import com.here.xyz.util.db.SQLQuery; import com.here.xyz.util.db.datasource.CachedPooledDataSources; import com.here.xyz.util.db.datasource.DataSourceProvider; -import com.here.xyz.util.db.datasource.StaticDataSources; -import com.here.xyz.util.db.pg.Script; -import java.io.IOException; -import java.net.URISyntaxException; +import com.here.xyz.util.db.datasource.DatabaseSettings; +import com.here.xyz.util.runtime.FunctionRuntime; import java.sql.BatchUpdateException; import java.sql.Connection; import java.sql.SQLException; @@ -67,7 +63,6 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.logging.log4j.LogManager; @@ -114,11 +109,10 @@ protected void initialize(Event event) { //Decrypt the ECPS into an instance of DatabaseSettings dbSettings = new DatabaseSettings(connectorId, - ECPSTool.decryptToMap(FunctionRuntime.getInstance().getEnvironmentVariable(ECPS_PHRASE), connectorParams.getEcps()), SCRIPT_RESOURCE_PATH) - .withApplicationName(FunctionRuntime.getInstance().getApplicationName()); - + ECPSTool.decryptToMap(FunctionRuntime.getInstance().getEnvironmentVariable(ECPS_PHRASE), connectorParams.getEcps())) + .withApplicationName(FunctionRuntime.getInstance().getApplicationName()) + .withScriptResourcePaths(List.of(SCRIPT_RESOURCE_PATH)); //TODO - set scriptResourcePath if ext & h3 functions should get installed here. - dbSettings.checkScripts(); dataSourceProvider = new CachedPooledDataSources(dbSettings); retryAttempted = false; diff --git a/xyz-psql-connector/src/main/java/com/here/xyz/psql/DatabaseMaintainer.java b/xyz-psql-connector/src/main/java/com/here/xyz/psql/DatabaseMaintainer.java index ee0ddd519f..300918ec77 100644 --- a/xyz-psql-connector/src/main/java/com/here/xyz/psql/DatabaseMaintainer.java +++ b/xyz-psql-connector/src/main/java/com/here/xyz/psql/DatabaseMaintainer.java @@ -24,8 +24,8 @@ import com.here.xyz.connectors.AbstractConnectorHandler.TraceItem; import com.here.xyz.psql.factory.MaintenanceSQL; import com.here.xyz.util.db.ConnectorParameters; -import com.here.xyz.util.db.DatabaseSettings; import com.here.xyz.util.db.datasource.DataSourceProvider; +import com.here.xyz.util.db.datasource.DatabaseSettings; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; diff --git a/xyz-psql-connector/src/main/java/com/here/xyz/psql/DatabaseWriter.java b/xyz-psql-connector/src/main/java/com/here/xyz/psql/DatabaseWriter.java index 036bb90050..5fde99ed24 100644 --- a/xyz-psql-connector/src/main/java/com/here/xyz/psql/DatabaseWriter.java +++ b/xyz-psql-connector/src/main/java/com/here/xyz/psql/DatabaseWriter.java @@ -31,7 +31,6 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.here.xyz.XyzSerializable.Static; -import com.here.xyz.util.runtime.FunctionRuntime; import com.here.xyz.events.ModifyFeaturesEvent; import com.here.xyz.models.geojson.implementation.Feature; import com.here.xyz.models.geojson.implementation.FeatureCollection; @@ -39,8 +38,9 @@ import com.here.xyz.models.geojson.implementation.Properties; import com.here.xyz.models.geojson.implementation.XyzNamespace; import com.here.xyz.psql.query.XyzEventBasedQueryRunner; -import com.here.xyz.util.db.DatabaseSettings; import com.here.xyz.util.db.SQLQuery; +import com.here.xyz.util.db.datasource.DatabaseSettings; +import com.here.xyz.util.runtime.FunctionRuntime; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.SQLException; diff --git a/xyz-psql-connector/src/main/java/com/here/xyz/psql/QueryRunner.java b/xyz-psql-connector/src/main/java/com/here/xyz/psql/QueryRunner.java index 5b2685800f..6b8fede01d 100644 --- a/xyz-psql-connector/src/main/java/com/here/xyz/psql/QueryRunner.java +++ b/xyz-psql-connector/src/main/java/com/here/xyz/psql/QueryRunner.java @@ -20,10 +20,10 @@ package com.here.xyz.psql; import com.here.xyz.connectors.ErrorResponseException; -import com.here.xyz.util.runtime.FunctionRuntime; -import com.here.xyz.util.db.DatabaseSettings; import com.here.xyz.util.db.SQLQuery; import com.here.xyz.util.db.datasource.DataSourceProvider; +import com.here.xyz.util.db.datasource.DatabaseSettings; +import com.here.xyz.util.runtime.FunctionRuntime; import java.sql.ResultSet; import java.sql.SQLException; import org.apache.commons.dbutils.ResultSetHandler; diff --git a/xyz-psql-connector/src/test/java/com/here/xyz/psql/PSQLAbstractIT.java b/xyz-psql-connector/src/test/java/com/here/xyz/psql/PSQLAbstractIT.java index 929e597186..db0b931b99 100644 --- a/xyz-psql-connector/src/test/java/com/here/xyz/psql/PSQLAbstractIT.java +++ b/xyz-psql-connector/src/test/java/com/here/xyz/psql/PSQLAbstractIT.java @@ -20,9 +20,9 @@ package com.here.xyz.psql; import static com.here.xyz.psql.DatabaseHandler.ECPS_PHRASE; -import static com.here.xyz.util.db.DatabaseSettings.PSQL_HOST; -import static com.here.xyz.util.db.DatabaseSettings.PSQL_PASSWORD; -import static com.here.xyz.util.db.DatabaseSettings.PSQL_USER; +import static com.here.xyz.util.db.datasource.DatabaseSettings.PSQL_HOST; +import static com.here.xyz.util.db.datasource.DatabaseSettings.PSQL_PASSWORD; +import static com.here.xyz.util.db.datasource.DatabaseSettings.PSQL_USER; import static io.restassured.path.json.JsonPath.with; import static org.junit.Assert.assertEquals; diff --git a/xyz-psql-connector/src/test/java/com/here/xyz/psql/PSQLXyzConnectorIT.java b/xyz-psql-connector/src/test/java/com/here/xyz/psql/PSQLXyzConnectorIT.java index 60068604e5..c907e1b2e4 100644 --- a/xyz-psql-connector/src/test/java/com/here/xyz/psql/PSQLXyzConnectorIT.java +++ b/xyz-psql-connector/src/test/java/com/here/xyz/psql/PSQLXyzConnectorIT.java @@ -24,11 +24,11 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.here.xyz.XyzSerializable; -import com.here.xyz.util.runtime.FunctionRuntime; import com.here.xyz.events.HealthCheckEvent; import com.here.xyz.util.db.ConnectorParameters; -import com.here.xyz.util.db.DatabaseSettings; import com.here.xyz.util.db.ECPSTool; +import com.here.xyz.util.db.datasource.DatabaseSettings; +import com.here.xyz.util.runtime.FunctionRuntime; import java.util.HashMap; import java.util.Map; import org.junit.Test; diff --git a/xyz-util/src/main/java/com/here/xyz/psql/query/QueryBuilder.java b/xyz-util/src/main/java/com/here/xyz/psql/query/QueryBuilder.java index d7e671bb32..b1221b31c9 100644 --- a/xyz-util/src/main/java/com/here/xyz/psql/query/QueryBuilder.java +++ b/xyz-util/src/main/java/com/here/xyz/psql/query/QueryBuilder.java @@ -19,9 +19,9 @@ package com.here.xyz.psql.query; -import com.here.xyz.util.db.DatabaseSettings; import com.here.xyz.util.db.SQLQuery; import com.here.xyz.util.db.datasource.DataSourceProvider; +import com.here.xyz.util.db.datasource.DatabaseSettings; public abstract class QueryBuilder { private DataSourceProvider dataSourceProvider; diff --git a/xyz-util/src/main/java/com/here/xyz/util/db/datasource/CachedPooledDataSources.java b/xyz-util/src/main/java/com/here/xyz/util/db/datasource/CachedPooledDataSources.java index defaf2057a..afb6d6dcaa 100644 --- a/xyz-util/src/main/java/com/here/xyz/util/db/datasource/CachedPooledDataSources.java +++ b/xyz-util/src/main/java/com/here/xyz/util/db/datasource/CachedPooledDataSources.java @@ -19,7 +19,6 @@ package com.here.xyz.util.db.datasource; -import com.here.xyz.util.db.DatabaseSettings; import java.util.Collections; import java.util.HashMap; import java.util.Map; diff --git a/xyz-util/src/main/java/com/here/xyz/util/db/datasource/DataSourceProvider.java b/xyz-util/src/main/java/com/here/xyz/util/db/datasource/DataSourceProvider.java index 9d2f076803..085a81c465 100644 --- a/xyz-util/src/main/java/com/here/xyz/util/db/datasource/DataSourceProvider.java +++ b/xyz-util/src/main/java/com/here/xyz/util/db/datasource/DataSourceProvider.java @@ -19,7 +19,6 @@ package com.here.xyz.util.db.datasource; -import com.here.xyz.util.db.DatabaseSettings; import javax.sql.DataSource; public abstract class DataSourceProvider implements AutoCloseable { @@ -28,6 +27,8 @@ public abstract class DataSourceProvider implements AutoCloseable { public DataSourceProvider(DatabaseSettings dbSettings) { this.dbSettings = dbSettings; + if (dbSettings != null) + dbSettings.init(); } public abstract DataSource getReader(); diff --git a/xyz-util/src/main/java/com/here/xyz/util/db/DatabaseSettings.java b/xyz-util/src/main/java/com/here/xyz/util/db/datasource/DatabaseSettings.java similarity index 91% rename from xyz-util/src/main/java/com/here/xyz/util/db/DatabaseSettings.java rename to xyz-util/src/main/java/com/here/xyz/util/db/datasource/DatabaseSettings.java index a6bd1177f3..f1ce9fb9c3 100644 --- a/xyz-util/src/main/java/com/here/xyz/util/db/DatabaseSettings.java +++ b/xyz-util/src/main/java/com/here/xyz/util/db/datasource/DatabaseSettings.java @@ -17,30 +17,27 @@ * License-Filename: LICENSE */ -package com.here.xyz.util.db; +package com.here.xyz.util.db.datasource; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.core.JsonProcessingException; import com.here.xyz.Payload; import com.here.xyz.util.Hasher; -import com.here.xyz.util.db.datasource.DataSourceProvider; -import com.here.xyz.util.db.datasource.StaticDataSources; import com.here.xyz.util.db.pg.Script; import com.here.xyz.util.runtime.FunctionRuntime; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - import java.io.IOException; import java.net.URISyntaxException; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; public class DatabaseSettings extends Payload { private static final Logger logger = LogManager.getLogger(); private static final int SCRIPT_VERSIONS_TO_KEEP = 5; private Map> sqlScripts = new ConcurrentHashMap<>(); - private String scriptResourcePath = null; + private volatile boolean initialized = false; /** * A constant that is normally used as environment variable name for the host. @@ -110,6 +107,7 @@ public class DatabaseSettings extends Payload { private int port = 5432; private String applicationName; private List searchPath; + private List scriptResourcePaths; /** * Connection Pool settings @@ -126,24 +124,15 @@ public class DatabaseSettings extends Payload { private DatabaseSettings() {} - //Load only defaults public DatabaseSettings(String id) { this.id = id; } - //No scriptResourcePath is set -> checkScripts gets skipped public DatabaseSettings(String id, Map databaseSettings) { this(id); setValuesFromMap(databaseSettings); } - //ScriptResourcePath is set -> checkScripts will install related scripts - public DatabaseSettings(String id, Map databaseSettings, String scriptResourcePath) { - this(id); - setValuesFromMap(databaseSettings); - this.scriptResourcePath = scriptResourcePath; - } - public String getId() { return id; } @@ -312,6 +301,19 @@ public DatabaseSettings withSearchPath(List searchPath) { return this; } + public List getScriptResourcePaths() { + return scriptResourcePaths; + } + + public void setScriptResourcePaths(List scriptResourcePaths) { + this.scriptResourcePaths = scriptResourcePaths; + } + + public DatabaseSettings withScriptResourcePaths(List scriptResourcePaths) { + setScriptResourcePaths(scriptResourcePaths); + return this; + } + public int getDbInitialPoolSize() { return dbInitialPoolSize; } @@ -472,20 +474,22 @@ public String toString() { * Checks whether the latest version of all SQL scripts is installed on the DB and set all script schemas for * the use in the search path. */ - public synchronized void checkScripts() { - if(scriptResourcePath == null) + private synchronized void checkScripts() { + if (scriptResourcePaths == null || scriptResourcePaths.isEmpty()) return; String softwareVersion = FunctionRuntime.getInstance().getSoftwareVersion(); if (!sqlScripts.containsKey(getId())) { logger.info("Checking scripts for connector {} ...", getId()); try (DataSourceProvider dataSourceProvider = new StaticDataSources(this)) { - List