From 56d4b620a668b09ddf6f3ed24d434d286489ad65 Mon Sep 17 00:00:00 2001 From: anighanta Date: Thu, 26 May 2022 13:45:50 -0500 Subject: [PATCH 01/13] initial commit to add support for GCP Signed-off-by: anighanta --- pom.xml | 5 ++ .../java/com/hedera/exchange/ERTParams.java | 36 +++++++--- .../java/com/hedera/exchange/Environment.java | 5 ++ .../com/hedera/exchange/ExchangeRateTool.java | 34 ++++++++- .../hedera/exchange/api/ExchangeRateAPI.java | 4 +- .../exchange/api/ExchangeRateHistoryAPI.java | 4 +- .../{AWSDBParams.java => DBParams.java} | 55 ++++++++++++-- .../exchange/database/ExchangeRateAWSRD.java | 35 ++++----- .../exchange/database/GCPExchangeRateDB.java | 71 +++++++++++++++++-- 9 files changed, 202 insertions(+), 47 deletions(-) create mode 100644 src/main/java/com/hedera/exchange/Environment.java rename src/main/java/com/hedera/exchange/database/{AWSDBParams.java => DBParams.java} (65%) diff --git a/pom.xml b/pom.xml index a2a88ad..ca41641 100644 --- a/pom.xml +++ b/pom.xml @@ -212,6 +212,11 @@ postgresql 42.3.3 + + com.google.cloud.functions + functions-framework-api + 1.0.4 + diff --git a/src/main/java/com/hedera/exchange/ERTParams.java b/src/main/java/com/hedera/exchange/ERTParams.java index 7e2ad40..d7b4e9e 100644 --- a/src/main/java/com/hedera/exchange/ERTParams.java +++ b/src/main/java/com/hedera/exchange/ERTParams.java @@ -66,8 +66,9 @@ import com.google.cloud.storage.BlobId; import com.google.cloud.storage.Storage; import com.google.cloud.storage.StorageOptions; +import com.hedera.exchange.database.GCPExchangeRateDB; import com.hedera.hashgraph.sdk.AccountId; -import com.hedera.exchange.database.AWSDBParams; +import com.hedera.exchange.database.DBParams; import com.hedera.exchange.database.ExchangeDB; import com.hedera.exchange.database.ExchangeRateAWSRD; import com.hedera.exchange.exchanges.Exchange; @@ -151,12 +152,12 @@ public class ERTParams { */ public static ERTParams readConfig(final String[] args) throws IOException { if (args == null || args.length == 0) { - return readDefaultConfig(); + return readDefaultConfigFromAWS(); } final String configurationPath = args[0]; if (configurationPath == null || configurationPath.trim().length() < 1) { - return readDefaultConfig(); + return readDefaultConfigFromAWS(); } LOGGER.debug("Using configuration file: {}", configurationPath); @@ -170,19 +171,28 @@ public static ERTParams readConfig(final String[] args) throws IOException { } if (configurationPath.contains("TO_DECIDE:AWS_NodeAddressFormat")){ - return readDefaultConfig(configurationPath); + return readDefaultConfigFromAWS(configurationPath); } return readConfig(configurationPath); } + + public static ERTParams readConfig(Environment env) throws IOException { + if (env == Environment.AWS) { + return readDefaultConfigFromAWS(); + } else { + return readDefaultConfigFromGCP(); + } + } + /** * Reads the AWS instance address from the arguments and replaces the node address in the * default configuration * @param awsInstanceAddress * @return ERTParams object */ - private static ERTParams readDefaultConfig(String awsInstanceAddress) throws IOException { + private static ERTParams readDefaultConfigFromAWS(String awsInstanceAddress) throws IOException { final String defaultConfigUri = ERTUtils.getDecryptedEnvironmentVariableFromAWS("DEFAULT_CONFIG_URI"); ERTParams ertParams = readConfigFromAWSS3(defaultConfigUri); @@ -198,11 +208,16 @@ private static ERTParams readDefaultConfig(String awsInstanceAddress) throws IOE * @return ERTParams object * @throws IOException */ - private static ERTParams readDefaultConfig() throws IOException { + private static ERTParams readDefaultConfigFromAWS() throws IOException { final String defaultConfigUri = ERTUtils.getDecryptedEnvironmentVariableFromAWS("DEFAULT_CONFIG_URI"); return readConfigFromAWSS3(defaultConfigUri); } + private static ERTParams readDefaultConfigFromGCP() throws IOException { + // TODO + return null; + } + /** * Read Config file from the Amazon S3 bucket * @param endpoint @@ -427,8 +442,13 @@ public long getValidationDelayInMilliseconds() { * Get the Database class to read and write the Exchange Rate Files. * @return ExchangeRateDb object as we are configured with AWS POSTGRESQL for now. */ - public ExchangeDB getExchangeDB() { - return new ExchangeRateAWSRD(new AWSDBParams()); + public ExchangeDB getExchangeDB(Environment env) { + final var dbParams = new DBParams(env); + if (env == Environment.AWS) + return new ExchangeRateAWSRD(dbParams); + else { + return new GCPExchangeRateDB(dbParams); + } } /** diff --git a/src/main/java/com/hedera/exchange/Environment.java b/src/main/java/com/hedera/exchange/Environment.java new file mode 100644 index 0000000..0c540a2 --- /dev/null +++ b/src/main/java/com/hedera/exchange/Environment.java @@ -0,0 +1,5 @@ +package com.hedera.exchange; + +public enum Environment { + AWS, GCP +} diff --git a/src/main/java/com/hedera/exchange/ExchangeRateTool.java b/src/main/java/com/hedera/exchange/ExchangeRateTool.java index 39f3461..83f5af8 100644 --- a/src/main/java/com/hedera/exchange/ExchangeRateTool.java +++ b/src/main/java/com/hedera/exchange/ExchangeRateTool.java @@ -52,6 +52,9 @@ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +import com.google.cloud.functions.HttpFunction; +import com.google.cloud.functions.HttpRequest; +import com.google.cloud.functions.HttpResponse; import com.hedera.hashgraph.sdk.AccountId; import com.hedera.hashgraph.sdk.Client; import com.hedera.exchange.database.ExchangeDB; @@ -81,12 +84,15 @@ * * @author Anirudh, Cesar */ -public class ExchangeRateTool { +public class ExchangeRateTool implements HttpFunction{ private static final Logger LOGGER = LogManager.getLogger(ExchangeRateTool.class); + private static final String AWS_TAG = "aws"; + private static final String GCP_TAG = "gcp"; private static final Map emptyMap = Collections.emptyMap(); static final int DEFAULT_RETRIES = 4; + private Environment env; private ERTParams ertParams; private ExchangeDB exchangeDB; @@ -103,8 +109,9 @@ public static void main(final String ... args) { protected void run(final String ... args) { LOGGER.debug(Exchange.EXCHANGE_FILTER, "Starting ExchangeRateTool"); try { - ertParams = ERTParams.readConfig(args); - exchangeDB = ertParams.getExchangeDB(); + env = validateTag(args); + ertParams = ERTParams.readConfig(env); + exchangeDB = ertParams.getExchangeDB(env); execute(); } catch (Exception ex) { final var subject = "FAILED : ERT Run Failed"; @@ -114,6 +121,21 @@ protected void run(final String ... args) { } } + private Environment validateTag(final String[] args) { + if (args == null || args.length != 1 || !(args[0].matches(AWS_TAG) && args[0].matches(GCP_TAG))) { + LOGGER.error(Exchange.EXCHANGE_FILTER, "invalid tag provided. Has to be one of {}, {}", GCP_TAG, AWS_TAG); + throw new IllegalArgumentException("Invalid tag provided. Has to be one of " + GCP_TAG + ", " + AWS_TAG); + } + + if (args[0].matches(AWS_TAG)) { + LOGGER.info(Exchange.EXCHANGE_FILTER, "Running on AWS Lambda"); + return Environment.AWS; + } else { + LOGGER.info(Exchange.EXCHANGE_FILTER, "Running on GCP Cloud Functions"); + return Environment.GCP; + } + } + /** * This method encapsulates all the execution logic * - Execute ERTProc @@ -282,4 +304,10 @@ private Rate getCurrentRate(final ExchangeDB exchangeDb, final ERTParams params) LOGGER.info(Exchange.EXCHANGE_FILTER, "Using default exchange rate as current exchange rate"); return params.getDefaultRate(); } + + @Override + public void service(final HttpRequest httpRequest, final HttpResponse httpResponse) throws Exception { + // this will be called from GCP cloud function. + // just call main with GCP tag + } } \ No newline at end of file diff --git a/src/main/java/com/hedera/exchange/api/ExchangeRateAPI.java b/src/main/java/com/hedera/exchange/api/ExchangeRateAPI.java index 78a24fd..46a7a33 100644 --- a/src/main/java/com/hedera/exchange/api/ExchangeRateAPI.java +++ b/src/main/java/com/hedera/exchange/api/ExchangeRateAPI.java @@ -53,7 +53,7 @@ */ import com.hedera.exchange.ExchangeRate; -import com.hedera.exchange.database.AWSDBParams; +import com.hedera.exchange.database.DBParams; import com.hedera.exchange.database.ExchangeDB; import com.hedera.exchange.database.ExchangeRateAWSRD; @@ -75,7 +75,7 @@ public class ExchangeRateAPI { } public static LambdaResponse getLatest() throws Exception { - final ExchangeDB exchangeDb = new ExchangeRateAWSRD(new AWSDBParams()); + final ExchangeDB exchangeDb = new ExchangeRateAWSRD(new DBParams()); final ExchangeRate latestExchangeRate = exchangeDb.getLatestExchangeRate(); if (latestExchangeRate == null) { return new LambdaResponse(200, "No exchange rate available yet"); diff --git a/src/main/java/com/hedera/exchange/api/ExchangeRateHistoryAPI.java b/src/main/java/com/hedera/exchange/api/ExchangeRateHistoryAPI.java index 51142e3..aafe071 100644 --- a/src/main/java/com/hedera/exchange/api/ExchangeRateHistoryAPI.java +++ b/src/main/java/com/hedera/exchange/api/ExchangeRateHistoryAPI.java @@ -61,7 +61,7 @@ import com.google.gson.JsonParser; import com.hedera.exchange.ExchangeRate; import com.hedera.exchange.Rate; -import com.hedera.exchange.database.AWSDBParams; +import com.hedera.exchange.database.DBParams; import com.hedera.exchange.database.ExchangeDB; import com.hedera.exchange.database.ExchangeRateAWSRD; import com.hedera.exchange.exchanges.Exchange; @@ -129,7 +129,7 @@ public void handleRequest(InputStream inputStream, OutputStream outputStream, Co } } - final ExchangeDB exchangeDb = new ExchangeRateAWSRD(new AWSDBParams()); + final ExchangeDB exchangeDb = new ExchangeRateAWSRD(new DBParams()); LOGGER.info(Exchange.EXCHANGE_FILTER, "params received : {}", no_of_records); NO_OF_RECORDS = no_of_records; ExchangeRate midnightRate = exchangeDb.getLatestMidnightExchangeRate(); diff --git a/src/main/java/com/hedera/exchange/database/AWSDBParams.java b/src/main/java/com/hedera/exchange/database/DBParams.java similarity index 65% rename from src/main/java/com/hedera/exchange/database/AWSDBParams.java rename to src/main/java/com/hedera/exchange/database/DBParams.java index affa2a4..acdf154 100644 --- a/src/main/java/com/hedera/exchange/database/AWSDBParams.java +++ b/src/main/java/com/hedera/exchange/database/DBParams.java @@ -53,21 +53,66 @@ */ import com.hedera.exchange.ERTUtils; +import com.hedera.exchange.Environment; +import com.hedera.exchange.exchanges.Exchange; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; + +public class DBParams { + private static final Logger LOGGER = LogManager.getLogger(DBParams.class); + + private final Environment env; + public DBParams(Environment env) { + this.env = env; + } -public class AWSDBParams { public String getEndpoint() { - return ERTUtils.getDecryptedEnvironmentVariableFromAWS("ENDPOINT") + getDatabaseName(); + if (env == Environment.AWS) { + return ERTUtils.getDecryptedEnvironmentVariableFromAWS("ENDPOINT") + getDatabaseName(); + } + else { + // TODO + return null; + } } public String getUsername() { - return ERTUtils.getDecryptedEnvironmentVariableFromAWS("USERNAME"); + if (env == Environment.AWS) { + return ERTUtils.getDecryptedEnvironmentVariableFromAWS("USERNAME"); + } + else { + // TODO + return null; + } } public String getPassword() { - return ERTUtils.getDecryptedEnvironmentVariableFromAWS("PASSWORD"); + if (env == Environment.AWS) { + return ERTUtils.getDecryptedEnvironmentVariableFromAWS("PASSWORD"); + } + else { + // TODO + return null; + } } public String getDatabaseName() { - return ERTUtils.getDecryptedEnvironmentVariableFromAWS("DATABASE"); + if (env == Environment.AWS) { + return ERTUtils.getDecryptedEnvironmentVariableFromAWS("DATABASE"); + } + else { + // TODO + return null; + } + } + + public Connection getConnection() throws SQLException { + final String endpoint = getEndpoint(); + LOGGER.info(Exchange.EXCHANGE_FILTER, "Connecting to endpoint: {}", endpoint); + return DriverManager.getConnection(endpoint, getUsername(), getPassword()); } } diff --git a/src/main/java/com/hedera/exchange/database/ExchangeRateAWSRD.java b/src/main/java/com/hedera/exchange/database/ExchangeRateAWSRD.java index 9d4cd65..0c20b8b 100644 --- a/src/main/java/com/hedera/exchange/database/ExchangeRateAWSRD.java +++ b/src/main/java/com/hedera/exchange/database/ExchangeRateAWSRD.java @@ -61,7 +61,6 @@ import java.io.IOException; import java.sql.Connection; -import java.sql.DriverManager; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; @@ -86,9 +85,9 @@ public class ExchangeRateAWSRD implements ExchangeDB { private static final String LATEST_QUERIED_QUERY = "SELECT e1.expirationTime, e1.queriedrates FROM queried_rate AS e1 INNER JOIN (SELECT MAX(expirationTime) expirationTime FROM queried_rate) AS e2 ON e1.expirationTime = e2.expirationTime LIMIT 1"; - private final AWSDBParams params; + private final DBParams params; - public ExchangeRateAWSRD(final AWSDBParams params) { + public ExchangeRateAWSRD(final DBParams params) { this.params = params; this.migrate(); } @@ -107,7 +106,7 @@ private void migrate() { public ExchangeRate getMidnightExchangeRate(long expirationTime) throws SQLException, IOException { LOGGER.info(Exchange.EXCHANGE_FILTER, "query to get the midnight exchange rate from midnight rate table " + "with nextExpiration time :{}", expirationTime); - try (final Connection conn = getConnection(); + try (final Connection conn = params.getConnection(); final PreparedStatement prepStatement = conn.prepareStatement( "SELECT expirationTime, exchangeRateFile FROM midnight_rate where expirationTime = ?")){ prepStatement.setLong(1, expirationTime); @@ -125,7 +124,7 @@ public ExchangeRate getMidnightExchangeRate(long expirationTime) throws SQLExcep @Override public ERTAddressBook getLatestERTAddressBook(String networkName) throws SQLException, IOException { LOGGER.info(Exchange.EXCHANGE_FILTER, "query to get latest ERTAddressBook from address_book table"); - try (final Connection conn = getConnection(); + try (final Connection conn = params.getConnection(); final PreparedStatement statement = conn.prepareStatement(LATEST_ADDRESSBOOK_QUERY)) { statement.setString(1, networkName); LOGGER.info(Exchange.EXCHANGE_FILTER,"final query for addressbook : {}", @@ -147,7 +146,7 @@ public void pushERTAddressBook(long expirationTime, ERTAddressBook ertAddressBoo String addressBook = ertAddressBook.toJson(); LOGGER.info(Exchange.EXCHANGE_FILTER, "push latest addressBook to address_book table : {}", addressBook); - try (final Connection conn = getConnection(); + try (final Connection conn = params.getConnection(); final PreparedStatement statement = conn.prepareStatement( "INSERT INTO address_book (expirationTime,addressBook,networkName) VALUES(?,?::JSON,?)")) { statement.setLong(1, expirationTime); @@ -160,7 +159,7 @@ public void pushERTAddressBook(long expirationTime, ERTAddressBook ertAddressBoo @Override public ExchangeRate getLatestExchangeRate() throws SQLException, IOException { LOGGER.info(Exchange.EXCHANGE_FILTER, "query to get latest exchange rate from exchange rate table"); - try (final Connection conn = getConnection(); + try (final Connection conn = params.getConnection(); final Statement statement = conn.createStatement(); final ResultSet result = statement.executeQuery(LATEST_EXCHANGE_QUERY)) { if (result.next()) { @@ -176,7 +175,7 @@ public ExchangeRate getLatestExchangeRate() throws SQLException, IOException { public ExchangeRate getExchangeRate(long expirationTime) throws SQLException, IOException { LOGGER.info(Exchange.EXCHANGE_FILTER, "query to get the exchange rate from exchange rate table " + "with nextExpiration time :{}", expirationTime); - try (final Connection conn = getConnection(); + try (final Connection conn = params.getConnection(); final PreparedStatement prepStatement = conn.prepareStatement( "SELECT expirationTime, exchangeRateFile FROM exchange_rate where expirationTime = ?")){ prepStatement.setLong(1, expirationTime); @@ -195,7 +194,7 @@ public ExchangeRate getExchangeRate(long expirationTime) throws SQLException, IO public String getQueriedRate(long expirationTime) throws SQLException { LOGGER.info(Exchange.EXCHANGE_FILTER, "query to get the queried rate from queried rate table " + "with nextExpiration time :{}", expirationTime); - try (final Connection conn = getConnection(); + try (final Connection conn = params.getConnection(); final PreparedStatement prepStatement = conn.prepareStatement( "SELECT expirationTime, queriedrates FROM queried_rate where expirationTime = ?")){ prepStatement.setLong(1, expirationTime); @@ -213,7 +212,7 @@ public String getQueriedRate(long expirationTime) throws SQLException { @Override public ExchangeRate getLatestMidnightExchangeRate() throws SQLException, IOException { LOGGER.info(Exchange.EXCHANGE_FILTER, "query to get midnight exchange rate from midnight rate table"); - try (final Connection conn = getConnection(); + try (final Connection conn = params.getConnection(); final Statement statement = conn.createStatement(); final ResultSet result = statement.executeQuery(MIDNIGHT_EXCHANGE_QUERY)) { if (result.next()) { @@ -228,7 +227,7 @@ public ExchangeRate getLatestMidnightExchangeRate() throws SQLException, IOExcep @Override public String getLatestQueriedRate() throws SQLException { LOGGER.info(Exchange.EXCHANGE_FILTER, "query to get the latest queried rate"); - try (final Connection conn = getConnection(); + try (final Connection conn = params.getConnection(); final Statement statement = conn.createStatement(); final ResultSet result = statement.executeQuery(LATEST_QUERIED_QUERY)) { if (result.next()) { @@ -244,7 +243,7 @@ public String getLatestQueriedRate() throws SQLException { public void pushExchangeRate(ExchangeRate exchangeRate) throws SQLException, IOException { LOGGER.info(Exchange.EXCHANGE_FILTER, "push latest exchange rate to exchange rate table : {}", exchangeRate.toJson()); - try (final Connection conn = getConnection(); + try (final Connection conn = params.getConnection(); final PreparedStatement statement = conn.prepareStatement( "INSERT INTO exchange_rate (expirationTime,exchangeRateFile) VALUES(?,?::JSON)")) { statement.setLong(1, exchangeRate.getNextExpirationTimeInSeconds()); @@ -257,7 +256,7 @@ public void pushExchangeRate(ExchangeRate exchangeRate) throws SQLException, IOE public void pushMidnightRate(ExchangeRate exchangeRate) throws SQLException, IOException { LOGGER.info(Exchange.EXCHANGE_FILTER, "push the midnight exchange rate to midnight rate table : {}", exchangeRate.toJson()); - try (final Connection conn = getConnection(); + try (final Connection conn = params.getConnection(); final PreparedStatement statement = conn.prepareStatement( "INSERT INTO midnight_rate (expirationTime,exchangeRateFile) VALUES(?,?::JSON)")) { statement.setLong(1, exchangeRate.getNextExpirationTimeInSeconds()); @@ -270,7 +269,7 @@ public void pushMidnightRate(ExchangeRate exchangeRate) throws SQLException, IOE public void pushQueriedRate(long expirationTime, String queriedRate) throws SQLException { LOGGER.info(Exchange.EXCHANGE_FILTER, "push the queried exchanges to queried rate table : {}:{}", expirationTime, queriedRate); - try (final Connection conn = getConnection(); + try (final Connection conn = params.getConnection(); final PreparedStatement statement = conn.prepareStatement( "INSERT INTO queried_rate (expirationTime,queriedrates) VALUES(?,?::JSON)")) { statement.setLong(1, expirationTime); @@ -279,12 +278,4 @@ public void pushQueriedRate(long expirationTime, String queriedRate) throws SQLE } } - private Connection getConnection() throws SQLException { - final String endpoint = this.params.getEndpoint(); - LOGGER.info(Exchange.EXCHANGE_FILTER, "Connecting to endpoint: {}", endpoint); - return DriverManager.getConnection(endpoint, - this.params.getUsername(), - this.params.getPassword()); - } - } diff --git a/src/main/java/com/hedera/exchange/database/GCPExchangeRateDB.java b/src/main/java/com/hedera/exchange/database/GCPExchangeRateDB.java index 1f1a5fe..808fdb0 100644 --- a/src/main/java/com/hedera/exchange/database/GCPExchangeRateDB.java +++ b/src/main/java/com/hedera/exchange/database/GCPExchangeRateDB.java @@ -54,16 +54,51 @@ import com.google.cloud.spanner.*; +import com.hedera.exchange.ERTAddressBook; import com.hedera.exchange.ExchangeRate; -public class GCPExchangeRateDB { +import java.io.IOException; +import java.sql.SQLException; + +public class GCPExchangeRateDB implements ExchangeDB{ // copy instance and db id from spanner website private static final String projectId = "ert-public-test-1"; private static final String instanceId = "ert-spanner-instance"; private static final String databaseId = "ert_data"; - public static void pushExchangeRate(ExchangeRate exchangeRate){ + private final DBParams params; + + public GCPExchangeRateDB(DBParams params) { + this.params = params; + } + + @Override + public ExchangeRate getLatestExchangeRate() throws SQLException, IOException { + return null; + } + + @Override + public ExchangeRate getExchangeRate(final long expirationTime) throws SQLException, IOException { + return null; + } + + @Override + public ExchangeRate getLatestMidnightExchangeRate() throws SQLException, IOException { + return null; + } + + @Override + public String getQueriedRate(final long expirationTime) throws SQLException { + return null; + } + + @Override + public String getLatestQueriedRate() throws SQLException { + return null; + } + + public void pushExchangeRate(ExchangeRate exchangeRate){ // Instantiates a client SpannerOptions options = SpannerOptions.newBuilder().setProjectId(projectId).build(); Spanner spanner = options.getService(); @@ -94,7 +129,33 @@ public Void run(TransactionContext transaction) throws Exception { } } - public static void pushMidnightExchangeRate(ExchangeRate exchangeRate){ + @Override + public void pushMidnightRate(final ExchangeRate exchangeRate) throws SQLException, IOException { + + } + + @Override + public void pushQueriedRate(final long expirationTime, final String queriedRate) throws SQLException { + + } + + @Override + public ExchangeRate getMidnightExchangeRate(final long expirationTime) throws SQLException, IOException { + return null; + } + + @Override + public ERTAddressBook getLatestERTAddressBook(final String networkName) throws SQLException, IOException { + return null; + } + + @Override + public void pushERTAddressBook(final long expirationTime, final ERTAddressBook ertAddressBook, + final String networkName) throws SQLException, IOException { + + } + + public void pushMidnightExchangeRate(ExchangeRate exchangeRate){ // Instantiates a client SpannerOptions options = SpannerOptions.newBuilder().setProjectId(projectId).build(); Spanner spanner = options.getService(); @@ -126,7 +187,7 @@ public Void run(TransactionContext transaction) throws Exception { } // TODO build a DS to hold all the queries to exchanges and data they sent back - public static void pushQueriedExchangesData(){ + public void pushQueriedExchangesData(){ // Instantiates a client SpannerOptions options = SpannerOptions.newBuilder().setProjectId(projectId).build(); Spanner spanner = options.getService(); @@ -157,7 +218,7 @@ public Void run(TransactionContext transaction) throws Exception { } } - public static void getMidnightExchange(long expirationTime){ + public void getMidnightExchange(long expirationTime){ // Instantiates a client SpannerOptions options = SpannerOptions.newBuilder().setProjectId(projectId).build(); Spanner spanner = options.getService(); From b096a208ff3559dc9088c184f0700060d02c71f5 Mon Sep 17 00:00:00 2001 From: anighanta Date: Fri, 10 Jun 2022 09:05:41 -0500 Subject: [PATCH 02/13] WIP : gcp support Signed-off-by: anighanta --- src/main/java/com/hedera/exchange/ERTParams.java | 4 ++-- .../com/hedera/exchange/api/ExchangeRateAPI.java | 4 ++-- .../hedera/exchange/api/ExchangeRateHistoryAPI.java | 4 ++-- .../java/com/hedera/exchange/database/DBParams.java | 12 ++++++++---- .../{ExchangeRateAWSRD.java => QueryHelper.java} | 6 +++--- 5 files changed, 17 insertions(+), 13 deletions(-) rename src/main/java/com/hedera/exchange/database/{ExchangeRateAWSRD.java => QueryHelper.java} (98%) diff --git a/src/main/java/com/hedera/exchange/ERTParams.java b/src/main/java/com/hedera/exchange/ERTParams.java index d7b4e9e..2ad6e03 100644 --- a/src/main/java/com/hedera/exchange/ERTParams.java +++ b/src/main/java/com/hedera/exchange/ERTParams.java @@ -70,7 +70,7 @@ import com.hedera.hashgraph.sdk.AccountId; import com.hedera.exchange.database.DBParams; import com.hedera.exchange.database.ExchangeDB; -import com.hedera.exchange.database.ExchangeRateAWSRD; +import com.hedera.exchange.database.QueryHelper; import com.hedera.exchange.exchanges.Exchange; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -445,7 +445,7 @@ public long getValidationDelayInMilliseconds() { public ExchangeDB getExchangeDB(Environment env) { final var dbParams = new DBParams(env); if (env == Environment.AWS) - return new ExchangeRateAWSRD(dbParams); + return new QueryHelper(dbParams); else { return new GCPExchangeRateDB(dbParams); } diff --git a/src/main/java/com/hedera/exchange/api/ExchangeRateAPI.java b/src/main/java/com/hedera/exchange/api/ExchangeRateAPI.java index 46a7a33..b651581 100644 --- a/src/main/java/com/hedera/exchange/api/ExchangeRateAPI.java +++ b/src/main/java/com/hedera/exchange/api/ExchangeRateAPI.java @@ -55,7 +55,7 @@ import com.hedera.exchange.ExchangeRate; import com.hedera.exchange.database.DBParams; import com.hedera.exchange.database.ExchangeDB; -import com.hedera.exchange.database.ExchangeRateAWSRD; +import com.hedera.exchange.database.QueryHelper; import java.util.HashMap; import java.util.Map; @@ -75,7 +75,7 @@ public class ExchangeRateAPI { } public static LambdaResponse getLatest() throws Exception { - final ExchangeDB exchangeDb = new ExchangeRateAWSRD(new DBParams()); + final ExchangeDB exchangeDb = new QueryHelper(new DBParams()); final ExchangeRate latestExchangeRate = exchangeDb.getLatestExchangeRate(); if (latestExchangeRate == null) { return new LambdaResponse(200, "No exchange rate available yet"); diff --git a/src/main/java/com/hedera/exchange/api/ExchangeRateHistoryAPI.java b/src/main/java/com/hedera/exchange/api/ExchangeRateHistoryAPI.java index aafe071..ba4c7e1 100644 --- a/src/main/java/com/hedera/exchange/api/ExchangeRateHistoryAPI.java +++ b/src/main/java/com/hedera/exchange/api/ExchangeRateHistoryAPI.java @@ -63,7 +63,7 @@ import com.hedera.exchange.Rate; import com.hedera.exchange.database.DBParams; import com.hedera.exchange.database.ExchangeDB; -import com.hedera.exchange.database.ExchangeRateAWSRD; +import com.hedera.exchange.database.QueryHelper; import com.hedera.exchange.exchanges.Exchange; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -129,7 +129,7 @@ public void handleRequest(InputStream inputStream, OutputStream outputStream, Co } } - final ExchangeDB exchangeDb = new ExchangeRateAWSRD(new DBParams()); + final ExchangeDB exchangeDb = new QueryHelper(new DBParams()); LOGGER.info(Exchange.EXCHANGE_FILTER, "params received : {}", no_of_records); NO_OF_RECORDS = no_of_records; ExchangeRate midnightRate = exchangeDb.getLatestMidnightExchangeRate(); diff --git a/src/main/java/com/hedera/exchange/database/DBParams.java b/src/main/java/com/hedera/exchange/database/DBParams.java index acdf154..3265f1f 100644 --- a/src/main/java/com/hedera/exchange/database/DBParams.java +++ b/src/main/java/com/hedera/exchange/database/DBParams.java @@ -76,7 +76,8 @@ public String getEndpoint() { } else { // TODO - return null; + // jdbc:mysql://35.225.57.43:3306/ + return "jdbc:mysql://35.225.57.43:3306/"; } } @@ -86,7 +87,8 @@ public String getUsername() { } else { // TODO - return null; + // exchange + return "exchange"; } } @@ -96,7 +98,8 @@ public String getPassword() { } else { // TODO - return null; + // hedera + return "hedera"; } } @@ -106,7 +109,8 @@ public String getDatabaseName() { } else { // TODO - return null; + // exchangeRate + return "exchangeRate"; } } diff --git a/src/main/java/com/hedera/exchange/database/ExchangeRateAWSRD.java b/src/main/java/com/hedera/exchange/database/QueryHelper.java similarity index 98% rename from src/main/java/com/hedera/exchange/database/ExchangeRateAWSRD.java rename to src/main/java/com/hedera/exchange/database/QueryHelper.java index 0c20b8b..ab0caec 100644 --- a/src/main/java/com/hedera/exchange/database/ExchangeRateAWSRD.java +++ b/src/main/java/com/hedera/exchange/database/QueryHelper.java @@ -73,9 +73,9 @@ * If you foresee doing more of this mapping, I would recommend moving to JPA/Hibernate. * */ -public class ExchangeRateAWSRD implements ExchangeDB { +public class QueryHelper implements ExchangeDB { - private static final Logger LOGGER = LogManager.getLogger(ExchangeRateAWSRD.class); + private static final Logger LOGGER = LogManager.getLogger(QueryHelper.class); private static final String LATEST_EXCHANGE_QUERY = "SELECT e1.expirationTime, e1.exchangeRateFile FROM exchange_rate AS e1 INNER JOIN (SELECT MAX(expirationTime) expirationTime FROM exchange_rate) AS e2 ON e1.expirationTime = e2.expirationTime LIMIT 1"; @@ -87,7 +87,7 @@ public class ExchangeRateAWSRD implements ExchangeDB { private final DBParams params; - public ExchangeRateAWSRD(final DBParams params) { + public QueryHelper(final DBParams params) { this.params = params; this.migrate(); } From 1766e7a79940103673cf67d7cc75f38b37ecba24 Mon Sep 17 00:00:00 2001 From: anighanta Date: Thu, 16 Jun 2022 09:42:54 -0500 Subject: [PATCH 03/13] working on gcp Signed-off-by: anighanta --- pom.xml | 46 ++++++++++- .../exchange/ERTNotificationHelper.java | 77 +++++++++++++++++++ .../java/com/hedera/exchange/ERTParams.java | 21 ++--- .../java/com/hedera/exchange/ERTUtils.java | 4 +- .../com/hedera/exchange/ExchangeRateTool.java | 23 ++++-- .../exchange/HederaNetworkCommunicator.java | 6 +- .../hedera/exchange/api/ExchangeRateAPI.java | 8 +- .../exchange/api/ExchangeRateHistoryAPI.java | 9 ++- .../hedera/exchange/database/DBParams.java | 37 ++++----- .../hedera/exchange/database/QueryHelper.java | 43 +++++------ 10 files changed, 200 insertions(+), 74 deletions(-) diff --git a/pom.xml b/pom.xml index ca41641..b5b49cc 100644 --- a/pom.xml +++ b/pom.xml @@ -217,6 +217,11 @@ functions-framework-api 1.0.4 + + com.google.cloud + google-cloud-pubsub + 1.119.1 + @@ -243,7 +248,23 @@ ${project.artifactId} + + + com.google.cloud.functions + function-maven-plugin + 0.10.0 + + com.hedera.exchange.ExchangeRateTool + + @@ -302,7 +323,30 @@ - + + org.apache.maven.plugins + maven-shade-plugin + + + package + shade + + ${project.build.directory}/deployment/${build.finalName}.jar + + + + .SF + .DSA + .RSA + + + + + + + + org.apache.maven.plugins maven-shade-plugin diff --git a/src/main/java/com/hedera/exchange/ERTNotificationHelper.java b/src/main/java/com/hedera/exchange/ERTNotificationHelper.java index 335a12e..3533859 100644 --- a/src/main/java/com/hedera/exchange/ERTNotificationHelper.java +++ b/src/main/java/com/hedera/exchange/ERTNotificationHelper.java @@ -57,10 +57,23 @@ import com.amazonaws.services.sns.AmazonSNSClient; import com.amazonaws.services.sns.AmazonSNSClientBuilder; import com.amazonaws.services.sns.model.AmazonSNSException; +import com.google.api.core.ApiFuture; +import com.google.api.core.ApiFutureCallback; +import com.google.api.core.ApiFutures; +import com.google.api.gax.rpc.ApiException; +import com.google.cloud.pubsub.v1.Publisher; +import com.google.common.util.concurrent.MoreExecutors; +import com.google.protobuf.ByteString; +import com.google.pubsub.v1.PubsubMessage; +import com.google.pubsub.v1.TopicName; import com.hedera.exchange.exchanges.Exchange; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.io.IOException; +import java.util.List; +import java.util.concurrent.TimeUnit; + import static com.hedera.exchange.ERTUtils.getDecryptedEnvironmentVariableFromAWS; public final class ERTNotificationHelper { @@ -78,6 +91,70 @@ private ERTNotificationHelper() { * Content of the Email */ public static void publishMessage(final String subject, final String message, final String region) { + if (ExchangeRateTool.env == Environment.AWS) { + publishMessageUsingSNS(subject, message, region); + } else { + try { + publishMessageUsingPubSub(List.of(subject, message)); + } catch (InterruptedException ex) { + LOGGER.error(Exchange.EXCHANGE_FILTER, "failed to terminate PubSub publisher"); + } + } + } + + private static void publishMessageUsingPubSub(List messages) throws InterruptedException { + final var projectId = System.getenv("PROJECT_ID"); + final var topicId = System.getenv("TOPIC_ID"); + LOGGER.info(Exchange.EXCHANGE_FILTER, "projectId is : {}, topicId is {}", projectId, topicId); + TopicName topicName = TopicName.of(projectId, topicId); + Publisher publisher = null; + + try { + publisher = Publisher.newBuilder(topicName).build(); + + for (var message : messages) { + ByteString data = ByteString.copyFromUtf8(message); + PubsubMessage pubsubMessage = PubsubMessage.newBuilder().setData(data).build(); + + // Once published, returns a server-assigned message id (unique within the topic) + ApiFuture future = publisher.publish(pubsubMessage); + + // Add an asynchronous callback to handle success / failure + ApiFutures.addCallback( + future, + new ApiFutureCallback<>() { + + @Override + public void onFailure(Throwable throwable) { + if (throwable instanceof ApiException) { + ApiException apiException = ((ApiException) throwable); + // details on the API exception + System.out.println(apiException.getStatusCode().getCode()); + System.out.println(apiException.isRetryable()); + } + System.out.println("Error publishing message : " + message); + } + + @Override + public void onSuccess(String messageId) { + // Once published, returns server-assigned message ids (unique within the topic) + System.out.println("Published message ID: " + messageId); + } + }, + MoreExecutors.directExecutor()); + } + } catch (IOException ex) { + LOGGER.error(Exchange.EXCHANGE_FILTER, "Failed to publish message"); + } finally { + if (publisher != null) { + // When finished with the publisher, shutdown to free up resources. + publisher.shutdown(); + publisher.awaitTermination(1, TimeUnit.MINUTES); + } + } + } + + private static void publishMessageUsingSNS(final String subject, final String message, final String region) { try { final AmazonSNSClient SNS_CLIENT = (AmazonSNSClient) AmazonSNSClientBuilder.standard() .withRegion(getValidRegion(region)) diff --git a/src/main/java/com/hedera/exchange/ERTParams.java b/src/main/java/com/hedera/exchange/ERTParams.java index 2ad6e03..4bdd8c2 100644 --- a/src/main/java/com/hedera/exchange/ERTParams.java +++ b/src/main/java/com/hedera/exchange/ERTParams.java @@ -83,6 +83,8 @@ import java.util.Map; import java.util.Set; +import static com.hedera.exchange.ExchangeRateTool.env; + /** * This class reads the parameters from the config file and provides get methods to fetch the configuration parameters. * @@ -178,7 +180,7 @@ public static ERTParams readConfig(final String[] args) throws IOException { } - public static ERTParams readConfig(Environment env) throws IOException { + public static ERTParams readConfig() throws IOException { if (env == Environment.AWS) { return readDefaultConfigFromAWS(); } else { @@ -215,7 +217,7 @@ private static ERTParams readDefaultConfigFromAWS() throws IOException { private static ERTParams readDefaultConfigFromGCP() throws IOException { // TODO - return null; + return readConfig("src/main/resources/config.json"); } /** @@ -419,7 +421,11 @@ public String getOperatorId() { @JsonIgnore public String getOperatorKey(String networkName) { - return ERTUtils.getDecryptedEnvironmentVariableFromAWS("OPERATOR_KEY_" + networkName); + if (env == Environment.AWS) { + return ERTUtils.getDecryptedEnvironmentVariableFromAWS("OPERATOR_KEY_" + networkName); + } else { + return System.getenv("OPERATOR_KEY_" + networkName); + } } /** @@ -442,13 +448,8 @@ public long getValidationDelayInMilliseconds() { * Get the Database class to read and write the Exchange Rate Files. * @return ExchangeRateDb object as we are configured with AWS POSTGRESQL for now. */ - public ExchangeDB getExchangeDB(Environment env) { - final var dbParams = new DBParams(env); - if (env == Environment.AWS) - return new QueryHelper(dbParams); - else { - return new GCPExchangeRateDB(dbParams); - } + public ExchangeDB getExchangeDB() { + return new QueryHelper(); } /** diff --git a/src/main/java/com/hedera/exchange/ERTUtils.java b/src/main/java/com/hedera/exchange/ERTUtils.java index 3b56623..1bbc40c 100644 --- a/src/main/java/com/hedera/exchange/ERTUtils.java +++ b/src/main/java/com/hedera/exchange/ERTUtils.java @@ -88,7 +88,7 @@ import java.util.List; import java.util.Map; -import static com.hedera.exchange.ExchangeRateTool.LAMBDA_FUNCTION_NAME; +import static com.hedera.exchange.ExchangeRateTool.FUNCTION_NAME; /** * This class implements helper functions of ERT @@ -132,7 +132,7 @@ private ERTUtils() { */ public static String getDecryptedEnvironmentVariableFromAWS(final String environmentVariable) { final String environmentValue = System.getenv(environmentVariable); - return getDecryptedValueFromAWS(environmentValue, LAMBDA_FUNCTION_NAME); + return getDecryptedValueFromAWS(environmentValue, FUNCTION_NAME); } static String getDecryptedValueFromAWS(final String value, final String lambdaFunctionName) { diff --git a/src/main/java/com/hedera/exchange/ExchangeRateTool.java b/src/main/java/com/hedera/exchange/ExchangeRateTool.java index cb3f3b7..4f8ffc6 100644 --- a/src/main/java/com/hedera/exchange/ExchangeRateTool.java +++ b/src/main/java/com/hedera/exchange/ExchangeRateTool.java @@ -63,10 +63,10 @@ import com.hedera.hashgraph.sdk.PrecheckStatusException; import com.hedera.hashgraph.sdk.PrivateKey; import com.hedera.hashgraph.sdk.ReceiptStatusException; -import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.io.BufferedWriter; import java.io.IOException; import java.sql.SQLException; import java.util.Collections; @@ -91,9 +91,10 @@ public class ExchangeRateTool implements HttpFunction{ private static final Map EMPTY_MAP = Collections.emptyMap(); static final int DEFAULT_RETRIES = 4; - static final String LAMBDA_FUNCTION_NAME = System.getenv("AWS_LAMBDA_FUNCTION_NAME"); - private Environment env; + public static Environment env; + static String FUNCTION_NAME; + private ERTParams ertParams; private ExchangeDB exchangeDB; @@ -111,19 +112,22 @@ protected void run(final String ... args) { LOGGER.debug(Exchange.EXCHANGE_FILTER, "Starting ExchangeRateTool"); try { env = validateTag(args); - ertParams = ERTParams.readConfig(env); - exchangeDB = ertParams.getExchangeDB(env); + FUNCTION_NAME = env == Environment.AWS ? + System.getenv("AWS_LAMBDA_FUNCTION_NAME") : + System.getenv("K_SERVICE"); + ertParams = ERTParams.readConfig(); + exchangeDB = ertParams.getExchangeDB(); execute(); } catch (Exception ex) { - final var subject = "FAILED : ERT Run Failed on " + LAMBDA_FUNCTION_NAME; + final var subject = "FAILED : ERT Run Failed on " + FUNCTION_NAME; final var message = ex.getMessage() + "\n"; LOGGER.error(Exchange.EXCHANGE_FILTER, subject, ex); - ERTNotificationHelper.publishMessage(subject, message + ExceptionUtils.getStackTrace(ex), ertParams.getRegion()); + ERTNotificationHelper.publishMessage(subject, message + ex.getMessage(), ertParams.getRegion()); } } private Environment validateTag(final String[] args) { - if (args == null || args.length != 1 || !(args[0].matches(AWS_TAG) && args[0].matches(GCP_TAG))) { + if (args == null || args.length != 1 || !(args[0].matches(AWS_TAG) || args[0].matches(GCP_TAG))) { LOGGER.error(Exchange.EXCHANGE_FILTER, "invalid tag provided. Has to be one of {}, {}", GCP_TAG, AWS_TAG); throw new IllegalArgumentException("Invalid tag provided. Has to be one of " + GCP_TAG + ", " + AWS_TAG); } @@ -310,5 +314,8 @@ private Rate getCurrentRate(final ExchangeDB exchangeDb, final ERTParams params) public void service(final HttpRequest httpRequest, final HttpResponse httpResponse) throws Exception { // this will be called from GCP cloud function. // just call main with GCP tag + BufferedWriter writer = httpResponse.getWriter(); + writer.write("Hello World!"); + main("gcp"); } } \ No newline at end of file diff --git a/src/main/java/com/hedera/exchange/HederaNetworkCommunicator.java b/src/main/java/com/hedera/exchange/HederaNetworkCommunicator.java index ec6609d..f06fc53 100644 --- a/src/main/java/com/hedera/exchange/HederaNetworkCommunicator.java +++ b/src/main/java/com/hedera/exchange/HederaNetworkCommunicator.java @@ -68,9 +68,7 @@ import com.hedera.hashgraph.sdk.ReceiptStatusException; import com.hedera.hashgraph.sdk.TransactionReceipt; import com.hedera.hashgraph.sdk.TransactionResponse; -import com.hedera.hashgraph.sdk.proto.FileServiceGrpc; import com.hedera.hashgraph.sdk.proto.NodeAddressBook; -import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -257,8 +255,8 @@ private void updateExchangeRateFileTxn( } } catch (PrecheckStatusException ex) { var subject = String.format("ERROR : %s : PreCheckStatusException : %s", networkName, ex.status); - LOGGER.error(Exchange.EXCHANGE_FILTER, subject); - ERTNotificationHelper.publishMessage(subject, ExceptionUtils.getStackTrace(ex), region); + LOGGER.error(Exchange.EXCHANGE_FILTER, subject, ex); + ERTNotificationHelper.publishMessage(subject, ex.getMessage(), region); if( retryCount++ == DEFAULT_RETRIES ) { throw ex; } diff --git a/src/main/java/com/hedera/exchange/api/ExchangeRateAPI.java b/src/main/java/com/hedera/exchange/api/ExchangeRateAPI.java index b651581..a2d6415 100644 --- a/src/main/java/com/hedera/exchange/api/ExchangeRateAPI.java +++ b/src/main/java/com/hedera/exchange/api/ExchangeRateAPI.java @@ -52,7 +52,9 @@ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +import com.hedera.exchange.Environment; import com.hedera.exchange.ExchangeRate; +import com.hedera.exchange.ExchangeRateTool; import com.hedera.exchange.database.DBParams; import com.hedera.exchange.database.ExchangeDB; import com.hedera.exchange.database.QueryHelper; @@ -60,6 +62,8 @@ import java.util.HashMap; import java.util.Map; +import static com.hedera.exchange.Environment.AWS; + /** * This class implements an API that one can trigger using an AWS lambda for example and get the latest Exchange rate file * from the database. @@ -75,7 +79,9 @@ public class ExchangeRateAPI { } public static LambdaResponse getLatest() throws Exception { - final ExchangeDB exchangeDb = new QueryHelper(new DBParams()); + // currently only supported on AWS + ExchangeRateTool.env = AWS; + final ExchangeDB exchangeDb = new QueryHelper(); final ExchangeRate latestExchangeRate = exchangeDb.getLatestExchangeRate(); if (latestExchangeRate == null) { return new LambdaResponse(200, "No exchange rate available yet"); diff --git a/src/main/java/com/hedera/exchange/api/ExchangeRateHistoryAPI.java b/src/main/java/com/hedera/exchange/api/ExchangeRateHistoryAPI.java index ba4c7e1..e8d09c4 100644 --- a/src/main/java/com/hedera/exchange/api/ExchangeRateHistoryAPI.java +++ b/src/main/java/com/hedera/exchange/api/ExchangeRateHistoryAPI.java @@ -59,7 +59,9 @@ import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonParser; +import com.hedera.exchange.Environment; import com.hedera.exchange.ExchangeRate; +import com.hedera.exchange.ExchangeRateTool; import com.hedera.exchange.Rate; import com.hedera.exchange.database.DBParams; import com.hedera.exchange.database.ExchangeDB; @@ -88,6 +90,8 @@ import java.util.Map; import java.util.TimeZone; +import static com.hedera.exchange.Environment.AWS; + /** * This class implements an API which returns the data from the last 'n'[defaulted to 5] successful runs of ERT * @@ -128,8 +132,9 @@ public void handleRequest(InputStream inputStream, OutputStream outputStream, Co no_of_records = Integer.parseInt((String) queryStringParameters.get("no_of_records")); } } - - final ExchangeDB exchangeDb = new QueryHelper(new DBParams()); + // currently only supported on AWS + ExchangeRateTool.env = AWS; + final ExchangeDB exchangeDb = new QueryHelper(); LOGGER.info(Exchange.EXCHANGE_FILTER, "params received : {}", no_of_records); NO_OF_RECORDS = no_of_records; ExchangeRate midnightRate = exchangeDb.getLatestMidnightExchangeRate(); diff --git a/src/main/java/com/hedera/exchange/database/DBParams.java b/src/main/java/com/hedera/exchange/database/DBParams.java index 3265f1f..bb7c087 100644 --- a/src/main/java/com/hedera/exchange/database/DBParams.java +++ b/src/main/java/com/hedera/exchange/database/DBParams.java @@ -62,61 +62,54 @@ import java.sql.DriverManager; import java.sql.SQLException; -public class DBParams { +import static com.hedera.exchange.ExchangeRateTool.env; + +public final class DBParams { private static final Logger LOGGER = LogManager.getLogger(DBParams.class); - private final Environment env; - public DBParams(Environment env) { - this.env = env; + public DBParams() { + throw new UnsupportedOperationException("Utility Class"); } - public String getEndpoint() { + public static String getEndpoint() { if (env == Environment.AWS) { return ERTUtils.getDecryptedEnvironmentVariableFromAWS("ENDPOINT") + getDatabaseName(); } else { - // TODO - // jdbc:mysql://35.225.57.43:3306/ - return "jdbc:mysql://35.225.57.43:3306/"; + LOGGER.info(Exchange.EXCHANGE_FILTER, "endpoint: {}", System.getenv("ENDPOINT") + System.getenv("DATABASE")); + return System.getenv("ENDPOINT") + System.getenv("DATABASE"); } } - public String getUsername() { + public static String getUsername() { if (env == Environment.AWS) { return ERTUtils.getDecryptedEnvironmentVariableFromAWS("USERNAME"); } else { - // TODO - // exchange - return "exchange"; + return System.getenv("USERNAME"); } } - public String getPassword() { + public static String getPassword() { if (env == Environment.AWS) { return ERTUtils.getDecryptedEnvironmentVariableFromAWS("PASSWORD"); } else { - // TODO - // hedera - return "hedera"; + return System.getenv("PASSWORD"); } } - public String getDatabaseName() { + public static String getDatabaseName() { if (env == Environment.AWS) { return ERTUtils.getDecryptedEnvironmentVariableFromAWS("DATABASE"); } else { - // TODO - // exchangeRate - return "exchangeRate"; + return System.getenv("DATABASE"); } } - public Connection getConnection() throws SQLException { + public static Connection getConnection() throws SQLException { final String endpoint = getEndpoint(); - LOGGER.info(Exchange.EXCHANGE_FILTER, "Connecting to endpoint: {}", endpoint); return DriverManager.getConnection(endpoint, getUsername(), getPassword()); } } diff --git a/src/main/java/com/hedera/exchange/database/QueryHelper.java b/src/main/java/com/hedera/exchange/database/QueryHelper.java index ab0caec..80dbb61 100644 --- a/src/main/java/com/hedera/exchange/database/QueryHelper.java +++ b/src/main/java/com/hedera/exchange/database/QueryHelper.java @@ -67,7 +67,7 @@ import java.sql.Statement; /** - * This class implements the ExchangeDB interface using AWS RDS + * This class implements the ExchangeDB interface * which species what APIs that we need to fetch/push data into the Database. * * If you foresee doing more of this mapping, I would recommend moving to JPA/Hibernate. @@ -77,26 +77,21 @@ public class QueryHelper implements ExchangeDB { private static final Logger LOGGER = LogManager.getLogger(QueryHelper.class); - private static final String LATEST_EXCHANGE_QUERY = "SELECT e1.expirationTime, e1.exchangeRateFile FROM exchange_rate AS e1 INNER JOIN (SELECT MAX(expirationTime) expirationTime FROM exchange_rate) AS e2 ON e1.expirationTime = e2.expirationTime LIMIT 1"; + private static final String LATEST_EXCHANGE_QUERY = "SELECT e1.expiration_time, e1.exchange_rate_file FROM exchange_rate AS e1 INNER JOIN (SELECT MAX(expiration_time) expiration_time FROM exchange_rate) AS e2 ON e1.expiration_time = e2.expiration_time LIMIT 1"; - private static final String LATEST_ADDRESSBOOK_QUERY = "SELECT e1.expirationTime, e1.addressBook FROM address_book AS e1 INNER JOIN (SELECT MAX(expirationTime) expirationTime FROM address_book) AS e2 ON e1.expirationTime = e2.expirationTime and networkName = ? LIMIT 1"; + private static final String LATEST_ADDRESS_BOOK_QUERY = "SELECT e1.expiration_time, e1.address_book FROM address_book AS e1 INNER JOIN (SELECT MAX(expiration_time) expiration_time FROM address_book) AS e2 ON e1.expiration_time = e2.expiration_time and network_name = ? LIMIT 1"; - private static final String MIDNIGHT_EXCHANGE_QUERY = "SELECT e1.expirationTime, e1.exchangeRateFile FROM midnight_rate AS e1 INNER JOIN (SELECT MAX(expirationTime) expirationTime FROM midnight_rate) AS e2 ON e1.expirationTime = e2.expirationTime LIMIT 1"; + private static final String MIDNIGHT_EXCHANGE_QUERY = "SELECT e1.expiration_time, e1.exchange_rate_file FROM midnight_rate AS e1 INNER JOIN (SELECT MAX(expiration_time) expiration_time FROM midnight_rate) AS e2 ON e1.expiration_time = e2.expiration_time LIMIT 1"; - private static final String LATEST_QUERIED_QUERY = "SELECT e1.expirationTime, e1.queriedrates FROM queried_rate AS e1 INNER JOIN (SELECT MAX(expirationTime) expirationTime FROM queried_rate) AS e2 ON e1.expirationTime = e2.expirationTime LIMIT 1"; + private static final String LATEST_QUERIED_QUERY = "SELECT e1.expiration_time, e1.queried_rates FROM queried_rates AS e1 INNER JOIN (SELECT MAX(expiration_time) expiration_time FROM queried_rates) AS e2 ON e1.expiration_time = e2.expiration_time LIMIT 1"; - private final DBParams params; - - public QueryHelper(final DBParams params) { - this.params = params; + public QueryHelper() { this.migrate(); } private void migrate() { final Flyway flyway = Flyway.configure() - .dataSource(this.params.getEndpoint(), - this.params.getUsername(), - this.params.getPassword()) + .dataSource(DBParams.getEndpoint(), DBParams.getUsername(), DBParams.getPassword()) .baselineOnMigrate(true) .load(); flyway.migrate(); @@ -106,7 +101,7 @@ private void migrate() { public ExchangeRate getMidnightExchangeRate(long expirationTime) throws SQLException, IOException { LOGGER.info(Exchange.EXCHANGE_FILTER, "query to get the midnight exchange rate from midnight rate table " + "with nextExpiration time :{}", expirationTime); - try (final Connection conn = params.getConnection(); + try (final Connection conn = DBParams.getConnection(); final PreparedStatement prepStatement = conn.prepareStatement( "SELECT expirationTime, exchangeRateFile FROM midnight_rate where expirationTime = ?")){ prepStatement.setLong(1, expirationTime); @@ -124,8 +119,8 @@ public ExchangeRate getMidnightExchangeRate(long expirationTime) throws SQLExcep @Override public ERTAddressBook getLatestERTAddressBook(String networkName) throws SQLException, IOException { LOGGER.info(Exchange.EXCHANGE_FILTER, "query to get latest ERTAddressBook from address_book table"); - try (final Connection conn = params.getConnection(); - final PreparedStatement statement = conn.prepareStatement(LATEST_ADDRESSBOOK_QUERY)) { + try (final Connection conn = DBParams.getConnection(); + final PreparedStatement statement = conn.prepareStatement(LATEST_ADDRESS_BOOK_QUERY)) { statement.setString(1, networkName); LOGGER.info(Exchange.EXCHANGE_FILTER,"final query for addressbook : {}", statement.toString()); @@ -146,7 +141,7 @@ public void pushERTAddressBook(long expirationTime, ERTAddressBook ertAddressBoo String addressBook = ertAddressBook.toJson(); LOGGER.info(Exchange.EXCHANGE_FILTER, "push latest addressBook to address_book table : {}", addressBook); - try (final Connection conn = params.getConnection(); + try (final Connection conn = DBParams.getConnection(); final PreparedStatement statement = conn.prepareStatement( "INSERT INTO address_book (expirationTime,addressBook,networkName) VALUES(?,?::JSON,?)")) { statement.setLong(1, expirationTime); @@ -159,7 +154,7 @@ public void pushERTAddressBook(long expirationTime, ERTAddressBook ertAddressBoo @Override public ExchangeRate getLatestExchangeRate() throws SQLException, IOException { LOGGER.info(Exchange.EXCHANGE_FILTER, "query to get latest exchange rate from exchange rate table"); - try (final Connection conn = params.getConnection(); + try (final Connection conn = DBParams.getConnection(); final Statement statement = conn.createStatement(); final ResultSet result = statement.executeQuery(LATEST_EXCHANGE_QUERY)) { if (result.next()) { @@ -175,7 +170,7 @@ public ExchangeRate getLatestExchangeRate() throws SQLException, IOException { public ExchangeRate getExchangeRate(long expirationTime) throws SQLException, IOException { LOGGER.info(Exchange.EXCHANGE_FILTER, "query to get the exchange rate from exchange rate table " + "with nextExpiration time :{}", expirationTime); - try (final Connection conn = params.getConnection(); + try (final Connection conn = DBParams.getConnection(); final PreparedStatement prepStatement = conn.prepareStatement( "SELECT expirationTime, exchangeRateFile FROM exchange_rate where expirationTime = ?")){ prepStatement.setLong(1, expirationTime); @@ -194,7 +189,7 @@ public ExchangeRate getExchangeRate(long expirationTime) throws SQLException, IO public String getQueriedRate(long expirationTime) throws SQLException { LOGGER.info(Exchange.EXCHANGE_FILTER, "query to get the queried rate from queried rate table " + "with nextExpiration time :{}", expirationTime); - try (final Connection conn = params.getConnection(); + try (final Connection conn = DBParams.getConnection(); final PreparedStatement prepStatement = conn.prepareStatement( "SELECT expirationTime, queriedrates FROM queried_rate where expirationTime = ?")){ prepStatement.setLong(1, expirationTime); @@ -212,7 +207,7 @@ public String getQueriedRate(long expirationTime) throws SQLException { @Override public ExchangeRate getLatestMidnightExchangeRate() throws SQLException, IOException { LOGGER.info(Exchange.EXCHANGE_FILTER, "query to get midnight exchange rate from midnight rate table"); - try (final Connection conn = params.getConnection(); + try (final Connection conn = DBParams.getConnection(); final Statement statement = conn.createStatement(); final ResultSet result = statement.executeQuery(MIDNIGHT_EXCHANGE_QUERY)) { if (result.next()) { @@ -227,7 +222,7 @@ public ExchangeRate getLatestMidnightExchangeRate() throws SQLException, IOExcep @Override public String getLatestQueriedRate() throws SQLException { LOGGER.info(Exchange.EXCHANGE_FILTER, "query to get the latest queried rate"); - try (final Connection conn = params.getConnection(); + try (final Connection conn = DBParams.getConnection(); final Statement statement = conn.createStatement(); final ResultSet result = statement.executeQuery(LATEST_QUERIED_QUERY)) { if (result.next()) { @@ -243,7 +238,7 @@ public String getLatestQueriedRate() throws SQLException { public void pushExchangeRate(ExchangeRate exchangeRate) throws SQLException, IOException { LOGGER.info(Exchange.EXCHANGE_FILTER, "push latest exchange rate to exchange rate table : {}", exchangeRate.toJson()); - try (final Connection conn = params.getConnection(); + try (final Connection conn = DBParams.getConnection(); final PreparedStatement statement = conn.prepareStatement( "INSERT INTO exchange_rate (expirationTime,exchangeRateFile) VALUES(?,?::JSON)")) { statement.setLong(1, exchangeRate.getNextExpirationTimeInSeconds()); @@ -256,7 +251,7 @@ public void pushExchangeRate(ExchangeRate exchangeRate) throws SQLException, IOE public void pushMidnightRate(ExchangeRate exchangeRate) throws SQLException, IOException { LOGGER.info(Exchange.EXCHANGE_FILTER, "push the midnight exchange rate to midnight rate table : {}", exchangeRate.toJson()); - try (final Connection conn = params.getConnection(); + try (final Connection conn = DBParams.getConnection(); final PreparedStatement statement = conn.prepareStatement( "INSERT INTO midnight_rate (expirationTime,exchangeRateFile) VALUES(?,?::JSON)")) { statement.setLong(1, exchangeRate.getNextExpirationTimeInSeconds()); @@ -269,7 +264,7 @@ public void pushMidnightRate(ExchangeRate exchangeRate) throws SQLException, IOE public void pushQueriedRate(long expirationTime, String queriedRate) throws SQLException { LOGGER.info(Exchange.EXCHANGE_FILTER, "push the queried exchanges to queried rate table : {}:{}", expirationTime, queriedRate); - try (final Connection conn = params.getConnection(); + try (final Connection conn = DBParams.getConnection(); final PreparedStatement statement = conn.prepareStatement( "INSERT INTO queried_rate (expirationTime,queriedrates) VALUES(?,?::JSON)")) { statement.setLong(1, expirationTime); From fed91c31461d5218b274fb90ab360b6c40fcd6b8 Mon Sep 17 00:00:00 2001 From: anighanta Date: Tue, 21 Jun 2022 12:30:37 -0500 Subject: [PATCH 04/13] WIP : set up gcp support with pub sub notifications Signed-off-by: anighanta --- pom.xml | 70 +++-- .../java/com/hedera/exchange/ERTParams.java | 14 +- .../com/hedera/exchange/ExchangeRateTool.java | 14 +- .../exchange/HederaNetworkCommunicator.java | 5 +- .../hedera/exchange/database/DBParams.java | 26 +- .../exchange/database/GCPExchangeRateDB.java | 281 ------------------ .../hedera/exchange/database/QueryHelper.java | 23 +- 7 files changed, 101 insertions(+), 332 deletions(-) delete mode 100644 src/main/java/com/hedera/exchange/database/GCPExchangeRateDB.java diff --git a/pom.xml b/pom.xml index b5b49cc..3fb5b75 100644 --- a/pom.xml +++ b/pom.xml @@ -40,18 +40,18 @@ 1.3 - 3.12.4 - 5.7.2 - 1.8.0-RC1 - 5.8.0-RC1 - 1.48 + 4.5.1 + 5.8.2 + 1.8.2 + 5.8.2 + 1.49 UTF-8 - 2.17.1 - 1.12.73 + 2.17.2 + 1.12.237 1.2.1 - 1.2.0 - 1.12.73 - 2.14.0-beta.1 + 1.5.1 + 1.12.237 + 2.15.0 @@ -71,7 +71,7 @@ com.fasterxml.jackson.core jackson-databind - 2.12.6.1 + 2.13.3 org.jmockit @@ -93,7 +93,7 @@ org.apache.commons commons-math3 - 3.5 + 3.6.1 @@ -159,6 +159,12 @@ com.hedera.hashgraph sdk-jdk7 ${hedera-java-sdk.version} + + + com.google.protobuf + * + + com.amazonaws @@ -183,45 +189,49 @@ javax.xml.bind jaxb-api - 2.3.0 - - - - com.google.cloud - google-cloud-spanner - 1.33.0 - - - com.google.cloud - google-cloud-storage - 1.88.0 + 2.3.1 org.flywaydb flyway-core - 7.4.0 + 8.5.11 org.bouncycastle bcprov-jdk15on - 1.67 + 1.70 org.postgresql postgresql - 42.3.3 + 42.3.4 - com.google.cloud.functions - functions-framework-api - 1.0.4 + com.google.cloud + google-cloud-storage + 2.8.1 com.google.cloud google-cloud-pubsub 1.119.1 + + com.google.cloud.functions + functions-framework-api + 1.0.4 + + + com.google.cloud.sql + postgres-socket-factory + 1.6.1 + + + com.zaxxer + HikariCP + 5.0.1 + diff --git a/src/main/java/com/hedera/exchange/ERTParams.java b/src/main/java/com/hedera/exchange/ERTParams.java index 4bdd8c2..c538dff 100644 --- a/src/main/java/com/hedera/exchange/ERTParams.java +++ b/src/main/java/com/hedera/exchange/ERTParams.java @@ -66,9 +66,7 @@ import com.google.cloud.storage.BlobId; import com.google.cloud.storage.Storage; import com.google.cloud.storage.StorageOptions; -import com.hedera.exchange.database.GCPExchangeRateDB; import com.hedera.hashgraph.sdk.AccountId; -import com.hedera.exchange.database.DBParams; import com.hedera.exchange.database.ExchangeDB; import com.hedera.exchange.database.QueryHelper; import com.hedera.exchange.exchanges.Exchange; @@ -216,8 +214,16 @@ private static ERTParams readDefaultConfigFromAWS() throws IOException { } private static ERTParams readDefaultConfigFromGCP() throws IOException { - // TODO - return readConfig("src/main/resources/config.json"); + final var projectId = System.getenv("PROJECT_ID"); + final var bucket = System.getenv("BUCKET_NAME"); + final var config = System.getenv("CONFIG_PATH"); + LOGGER.info("Reading configuration file from GCP: {}, {}, {}", projectId, bucket, config); + var options = StorageOptions.newBuilder().setProjectId(projectId).build(); + var storage = options.getService(); + var blob = storage.get(bucket, config); + + return OBJECT_MAPPER.readValue(blob.getContent(), ERTParams.class); +// return readConfig("src/main/resources/config.json"); } /** diff --git a/src/main/java/com/hedera/exchange/ExchangeRateTool.java b/src/main/java/com/hedera/exchange/ExchangeRateTool.java index 3f61015..5ebf6de 100644 --- a/src/main/java/com/hedera/exchange/ExchangeRateTool.java +++ b/src/main/java/com/hedera/exchange/ExchangeRateTool.java @@ -100,7 +100,7 @@ public class ExchangeRateTool implements HttpFunction{ public static void main(final String ... args) { ExchangeRateTool ert = new ExchangeRateTool(); - ert.run(args); + ert.run("gcp"); } /** @@ -127,17 +127,17 @@ protected void run(final String ... args) { } private Environment validateTag(final String[] args) { - if (args == null || args.length != 1 || !(args[0].matches(AWS_TAG) || args[0].matches(GCP_TAG))) { + if (args == null) { LOGGER.error(Exchange.EXCHANGE_FILTER, "invalid tag provided. Has to be one of {}, {}", GCP_TAG, AWS_TAG); throw new IllegalArgumentException("Invalid tag provided. Has to be one of " + GCP_TAG + ", " + AWS_TAG); } - if (args[0].matches(AWS_TAG)) { - LOGGER.info(Exchange.EXCHANGE_FILTER, "Running on AWS Lambda"); - return Environment.AWS; - } else { + if (args[0].matches(GCP_TAG)) { LOGGER.info(Exchange.EXCHANGE_FILTER, "Running on GCP Cloud Functions"); return Environment.GCP; + } else { + LOGGER.info(Exchange.EXCHANGE_FILTER, "Running on AWS Lambda"); + return Environment.AWS; } } @@ -220,7 +220,7 @@ protected Status fileUpdateTransactionForNetwork( final var nodesFromPrevRun = ertAddressBookFromPreviousRun != null ? getNodesForClient(ertAddressBookFromPreviousRun.getNodes()) : EMPTY_MAP; - final Map nodesForClient = nodesFromPrevRun.isEmpty() ? nodesFromPrevRun : nodesFromConfig; + final Map nodesForClient = !nodesFromPrevRun.isEmpty() ? nodesFromPrevRun : nodesFromConfig; LOGGER.info(Exchange.EXCHANGE_FILTER, "Building a Hedera Client with nodes {} \n Account {}", nodesForClient, diff --git a/src/main/java/com/hedera/exchange/HederaNetworkCommunicator.java b/src/main/java/com/hedera/exchange/HederaNetworkCommunicator.java index 91e9dff..54859c4 100644 --- a/src/main/java/com/hedera/exchange/HederaNetworkCommunicator.java +++ b/src/main/java/com/hedera/exchange/HederaNetworkCommunicator.java @@ -148,6 +148,7 @@ public ERTAddressBook updateExchangeRateFile( LOGGER.info(Exchange.EXCHANGE_FILTER, "Balance before updating the Exchange Rate file: {}", currentBalance.hbars.toString()); +// ERTAddressBook newAddressBook = new ERTAddressBook(); ERTAddressBook newAddressBook = fetchAddressBook(client); updateExchangeRateFileTxn(exchangeRate, exchangeRateFileId, exchangeRateAsBytes, client, memo, ertParams.getRegion()); @@ -352,8 +353,8 @@ private byte[] getFileContentsQuery(final Client client, final FileId fileId) th final Hbar getContentsQueryFee = new FileContentsQuery() .setFileId(fileId) .getCost(client); - LOGGER.debug(Exchange.EXCHANGE_FILTER, "Cost to get file {} contents is : {}", fileId, getContentsQueryFee); - client.setDefaultMaxQueryPayment(getContentsQueryFee); + LOGGER.info(Exchange.EXCHANGE_FILTER, "Cost to get file {} contents is : {}", fileId, getContentsQueryFee); + client.setDefaultMaxQueryPayment(Hbar.from(1L)); final ByteString contentsResponse = new FileContentsQuery() .setFileId(fileId) diff --git a/src/main/java/com/hedera/exchange/database/DBParams.java b/src/main/java/com/hedera/exchange/database/DBParams.java index bb7c087..bf25cc4 100644 --- a/src/main/java/com/hedera/exchange/database/DBParams.java +++ b/src/main/java/com/hedera/exchange/database/DBParams.java @@ -55,9 +55,12 @@ import com.hedera.exchange.ERTUtils; import com.hedera.exchange.Environment; import com.hedera.exchange.exchanges.Exchange; +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import javax.sql.DataSource; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; @@ -76,8 +79,8 @@ public static String getEndpoint() { return ERTUtils.getDecryptedEnvironmentVariableFromAWS("ENDPOINT") + getDatabaseName(); } else { - LOGGER.info(Exchange.EXCHANGE_FILTER, "endpoint: {}", System.getenv("ENDPOINT") + System.getenv("DATABASE")); - return System.getenv("ENDPOINT") + System.getenv("DATABASE"); + LOGGER.error(Exchange.EXCHANGE_FILTER, "should not be possible"); + return ""; } } @@ -109,7 +112,22 @@ public static String getDatabaseName() { } public static Connection getConnection() throws SQLException { - final String endpoint = getEndpoint(); - return DriverManager.getConnection(endpoint, getUsername(), getPassword()); + if (env == Environment.AWS) { + return DriverManager.getConnection(getEndpoint(), getUsername(), getPassword()); + } else { + DataSource pool = getGcpDataSource(); + return pool.getConnection(); + } + } + + public static DataSource getGcpDataSource() { + HikariConfig config = new HikariConfig(); + config.setJdbcUrl(String.format("jdbc:postgresql:///%s", getDatabaseName())); + config.setUsername(getUsername()); + config.setPassword(getPassword()); + config.addDataSourceProperty("socketFactory", "com.google.cloud.sql.postgres.SocketFactory"); + config.addDataSourceProperty("cloudSqlInstance", System.getenv("INSTANCE_CONNECTION_NAME")); + config.addDataSourceProperty("ipTypes", "PUBLIC,PRIVATE"); + return new HikariDataSource(config); } } diff --git a/src/main/java/com/hedera/exchange/database/GCPExchangeRateDB.java b/src/main/java/com/hedera/exchange/database/GCPExchangeRateDB.java deleted file mode 100644 index 808fdb0..0000000 --- a/src/main/java/com/hedera/exchange/database/GCPExchangeRateDB.java +++ /dev/null @@ -1,281 +0,0 @@ -package com.hedera.exchange.database; - -/*- - * ‌ - * Hedera Exchange Rate Tool - * ​ - * Copyright (C) 2019 - 2020 Hedera Hashgraph, LLC - * ​ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ‍ - * - * Copyright (c) 2007-present, Stephen Colebourne & Michael Nascimento Santos. - * - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * * Neither the name of JSR-310 nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - - -import com.google.cloud.spanner.*; -import com.hedera.exchange.ERTAddressBook; -import com.hedera.exchange.ExchangeRate; - -import java.io.IOException; -import java.sql.SQLException; - -public class GCPExchangeRateDB implements ExchangeDB{ - - // copy instance and db id from spanner website - private static final String projectId = "ert-public-test-1"; - private static final String instanceId = "ert-spanner-instance"; - private static final String databaseId = "ert_data"; - - private final DBParams params; - - public GCPExchangeRateDB(DBParams params) { - this.params = params; - } - - @Override - public ExchangeRate getLatestExchangeRate() throws SQLException, IOException { - return null; - } - - @Override - public ExchangeRate getExchangeRate(final long expirationTime) throws SQLException, IOException { - return null; - } - - @Override - public ExchangeRate getLatestMidnightExchangeRate() throws SQLException, IOException { - return null; - } - - @Override - public String getQueriedRate(final long expirationTime) throws SQLException { - return null; - } - - @Override - public String getLatestQueriedRate() throws SQLException { - return null; - } - - public void pushExchangeRate(ExchangeRate exchangeRate){ - // Instantiates a client - SpannerOptions options = SpannerOptions.newBuilder().setProjectId(projectId).build(); - Spanner spanner = options.getService(); - try{ - // Creates a database client - DatabaseClient dbClient = spanner.getDatabaseClient(DatabaseId.of(projectId, instanceId, databaseId)); - - String sql = String.format("INSERT INTO ExchangeRate (ExpirationTime, ExchagneRateData) values ('{}', {})", - exchangeRate.getNextExpirationTimeInSeconds(), exchangeRate.toString()); - - dbClient.readWriteTransaction().run( - new TransactionRunner.TransactionCallable() { - @Override - public Void run(TransactionContext transaction) throws Exception { - long rowCount = transaction.executeUpdate(Statement.of(sql)); - System.out.printf("%d record inserted.\n", rowCount); - return null; - } - } - ); - - } - catch( Exception e ){ - System.out.println("Writing to DB failed : " + e.getMessage()); - } - finally { - spanner.close(); - } - } - - @Override - public void pushMidnightRate(final ExchangeRate exchangeRate) throws SQLException, IOException { - - } - - @Override - public void pushQueriedRate(final long expirationTime, final String queriedRate) throws SQLException { - - } - - @Override - public ExchangeRate getMidnightExchangeRate(final long expirationTime) throws SQLException, IOException { - return null; - } - - @Override - public ERTAddressBook getLatestERTAddressBook(final String networkName) throws SQLException, IOException { - return null; - } - - @Override - public void pushERTAddressBook(final long expirationTime, final ERTAddressBook ertAddressBook, - final String networkName) throws SQLException, IOException { - - } - - public void pushMidnightExchangeRate(ExchangeRate exchangeRate){ - // Instantiates a client - SpannerOptions options = SpannerOptions.newBuilder().setProjectId(projectId).build(); - Spanner spanner = options.getService(); - try{ - // Creates a database client - DatabaseClient dbClient = spanner.getDatabaseClient(DatabaseId.of(projectId, instanceId, databaseId)); - - String sql = String.format("INSERT INTO MidnightExchangeRate (ExpirationTime, ExchangeRateData) values ('{}', {})", - exchangeRate.getNextExpirationTimeInSeconds(), exchangeRate.toJson()); - - dbClient.readWriteTransaction().run( - new TransactionRunner.TransactionCallable() { - @Override - public Void run(TransactionContext transaction) throws Exception { - long rowCount = transaction.executeUpdate(Statement.of(sql)); - System.out.printf("%d record inserted.\n", rowCount); - return null; - } - } - ); - - } - catch( Exception e ){ - System.out.println("Writing to DB failed : " + e.getMessage()); - } - finally { - spanner.close(); - } - } - - // TODO build a DS to hold all the queries to exchanges and data they sent back - public void pushQueriedExchangesData(){ - // Instantiates a client - SpannerOptions options = SpannerOptions.newBuilder().setProjectId(projectId).build(); - Spanner spanner = options.getService(); - try{ - // Creates a database client - DatabaseClient dbClient = spanner.getDatabaseClient(DatabaseId.of(projectId, instanceId, databaseId)); - - String sql = String.format("INSERT INTO ExcahngesData (ExpirationTime, ExchangesData) values ('{}', {})", - "" , ""); - - dbClient.readWriteTransaction().run( - new TransactionRunner.TransactionCallable() { - @Override - public Void run(TransactionContext transaction) throws Exception { - long rowCount = transaction.executeUpdate(Statement.of(sql)); - System.out.printf("%d record inserted.\n", rowCount); - return null; - } - } - ); - - } - catch( Exception e ){ - System.out.println("Writing to DB failed : " + e.getMessage()); - } - finally { - spanner.close(); - } - } - - public void getMidnightExchange(long expirationTime){ - // Instantiates a client - SpannerOptions options = SpannerOptions.newBuilder().setProjectId(projectId).build(); - Spanner spanner = options.getService(); - try{ - // Creates a database client - DatabaseClient dbClient = spanner.getDatabaseClient(DatabaseId.of(projectId, instanceId, databaseId)); - - String sql = String.format("SELECT ExchangeRateData FROM MidnightExchangeRate where ExpirationTime = {}", - expirationTime); - - ResultSet resultSet = dbClient.singleUse().executeQuery(Statement.of(sql)); - - // TODO map the result set to exchange rate. - - } - catch( Exception e ){ - System.out.println("Writing to DB failed : " + e.getMessage()); - } - finally { - spanner.close(); - } - } - - - public static void main(){ - // copy instance and db id from spanner website - String projectId = "ert-public-test-1"; - String instanceId = "ert-spanner-instance"; - String databaseId = "ert_data"; - // Instantiates a client - SpannerOptions options = SpannerOptions.newBuilder().setProjectId(projectId).build(); - Spanner spanner = options.getService(); - - - try{ - // Creates a database client - DatabaseClient dbClient = spanner.getDatabaseClient(DatabaseId.of(projectId, instanceId, databaseId)); - - dbClient.readWriteTransaction().run( - new TransactionRunner.TransactionCallable() { - @Override - public Void run(TransactionContext transaction) throws Exception { - String sql = "INSERT INTO ExchangeRate (ExpirationTime, ExchagneRateData) " + - "values ('1567022400', '1,15,1567018800;1,14,1567022400')"; - long rowCount = transaction.executeUpdate(Statement.of(sql)); - System.out.printf("%d record inserted.\n", rowCount); - return null; - } - } - ); - - } - catch( Exception e ){ - System.out.println("Writing to DB failed : " + e.getMessage()); - } - finally { - spanner.close(); - } - } -} diff --git a/src/main/java/com/hedera/exchange/database/QueryHelper.java b/src/main/java/com/hedera/exchange/database/QueryHelper.java index 80dbb61..f781dc5 100644 --- a/src/main/java/com/hedera/exchange/database/QueryHelper.java +++ b/src/main/java/com/hedera/exchange/database/QueryHelper.java @@ -53,6 +53,7 @@ */ import com.hedera.exchange.ERTAddressBook; +import com.hedera.exchange.Environment; import com.hedera.exchange.ExchangeRate; import com.hedera.exchange.exchanges.Exchange; import org.apache.logging.log4j.LogManager; @@ -66,6 +67,12 @@ import java.sql.SQLException; import java.sql.Statement; +import static com.hedera.exchange.ExchangeRateTool.env; +import static com.hedera.exchange.database.DBParams.getEndpoint; +import static com.hedera.exchange.database.DBParams.getGcpDataSource; +import static com.hedera.exchange.database.DBParams.getPassword; +import static com.hedera.exchange.database.DBParams.getUsername; + /** * This class implements the ExchangeDB interface * which species what APIs that we need to fetch/push data into the Database. @@ -90,10 +97,18 @@ public QueryHelper() { } private void migrate() { - final Flyway flyway = Flyway.configure() - .dataSource(DBParams.getEndpoint(), DBParams.getUsername(), DBParams.getPassword()) - .baselineOnMigrate(true) - .load(); + final Flyway flyway; + if (env == Environment.AWS) { + flyway = Flyway.configure() + .dataSource(getEndpoint(), getUsername(), getPassword()) + .baselineOnMigrate(true) + .load(); + } else { + flyway = Flyway.configure() + .dataSource(getGcpDataSource()) + .baselineOnMigrate(true) + .load(); + } flyway.migrate(); } From 321e9702bac24b5dbe7f53d108d01080880b1706 Mon Sep 17 00:00:00 2001 From: anighanta Date: Tue, 21 Jun 2022 16:05:22 -0500 Subject: [PATCH 05/13] WIP pom fixes Signed-off-by: anighanta --- pom.xml | 114 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) diff --git a/pom.xml b/pom.xml index 3fb5b75..2879403 100644 --- a/pom.xml +++ b/pom.xml @@ -166,6 +166,106 @@ + + + io.grpc + grpc-netty-shaded + 1.40.0 + + + io.grpc + grpc-core + + + io.grpc + grpc-protobuf-lite + + + + + io.grpc + grpc-api + 1.40.0 + + + io.grpc + grpc-protobuf + 1.40.0 + + + io.grpc + grpc-alts + 1.40.0 + + + io.grpc + grpc-grpclb + 1.40.0 + + + io.grpc + grpc-core + + + + + io.grpc + grpc-auth + 1.40.0 + + + io.grpc + grpc-api + + + + + io.grpc + grpc-googleapis + 1.46.0 + + + io.grpc + grpc-netty-shaded + + + io.grpc + grpc-core + + + io.grpc + grpc-api + + + io.grpc + grpc-alts + + + io.grpc + grpc-xds + + + io.grpc + grpc-context + + + + + io.grpc + grpc-xds + 1.40.0 + + + io.grpc + grpc-services + 1.40.0 + + + io.grpc + grpc-core + + + com.amazonaws aws-lambda-java-core @@ -216,6 +316,20 @@ com.google.cloud google-cloud-pubsub 1.119.1 + + + io.grpc + grpc-netty-shaded + + + io.grpc + grpc-core + + + io.grpc + grpc-api + + com.google.cloud.functions From 52f25e7887c330379724ed73817e14dcf716d2b7 Mon Sep 17 00:00:00 2001 From: anighanta Date: Tue, 21 Jun 2022 17:45:11 -0500 Subject: [PATCH 06/13] use new entry points for aws and gcp Signed-off-by: anighanta --- pom.xml | 118 +----------------- src/main/java/com/hedera/exchange/ERTAws.java | 62 +++++++++ src/main/java/com/hedera/exchange/ERTGcp.java | 67 ++++++++++ .../java/com/hedera/exchange/ERTParams.java | 2 +- .../com/hedera/exchange/ExchangeRateTool.java | 22 ++-- .../hedera/exchange/database/DBParams.java | 2 +- 6 files changed, 140 insertions(+), 133 deletions(-) create mode 100644 src/main/java/com/hedera/exchange/ERTAws.java create mode 100644 src/main/java/com/hedera/exchange/ERTGcp.java diff --git a/pom.xml b/pom.xml index 2879403..ce07c3b 100644 --- a/pom.xml +++ b/pom.xml @@ -51,7 +51,7 @@ 1.2.1 1.5.1 1.12.237 - 2.15.0 + 2.16.0-beta.1 @@ -159,112 +159,12 @@ com.hedera.hashgraph sdk-jdk7 ${hedera-java-sdk.version} - - - com.google.protobuf - * - - io.grpc grpc-netty-shaded - 1.40.0 - - - io.grpc - grpc-core - - - io.grpc - grpc-protobuf-lite - - - - - io.grpc - grpc-api - 1.40.0 - - - io.grpc - grpc-protobuf - 1.40.0 - - - io.grpc - grpc-alts - 1.40.0 - - - io.grpc - grpc-grpclb - 1.40.0 - - - io.grpc - grpc-core - - - - - io.grpc - grpc-auth - 1.40.0 - - - io.grpc - grpc-api - - - - - io.grpc - grpc-googleapis 1.46.0 - - - io.grpc - grpc-netty-shaded - - - io.grpc - grpc-core - - - io.grpc - grpc-api - - - io.grpc - grpc-alts - - - io.grpc - grpc-xds - - - io.grpc - grpc-context - - - - - io.grpc - grpc-xds - 1.40.0 - - - io.grpc - grpc-services - 1.40.0 - - - io.grpc - grpc-core - - com.amazonaws @@ -316,20 +216,6 @@ com.google.cloud google-cloud-pubsub 1.119.1 - - - io.grpc - grpc-netty-shaded - - - io.grpc - grpc-core - - - io.grpc - grpc-api - - com.google.cloud.functions @@ -386,7 +272,7 @@ function-maven-plugin 0.10.0 - com.hedera.exchange.ExchangeRateTool + com.hedera.exchange.ERTGcp diff --git a/src/main/java/com/hedera/exchange/ERTAws.java b/src/main/java/com/hedera/exchange/ERTAws.java new file mode 100644 index 0000000..dbee211 --- /dev/null +++ b/src/main/java/com/hedera/exchange/ERTAws.java @@ -0,0 +1,62 @@ +package com.hedera.exchange; + +/*- + * ‌ + * Hedera Exchange Rate Tool + * ​ + * Copyright (C) 2019 - 2020 Hedera Hashgraph, LLC + * ​ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ‍ + * + * Copyright (c) 2007-present, Stephen Colebourne & Michael Nascimento Santos. + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the name of JSR-310 nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import static com.hedera.exchange.ExchangeRateTool.AWS_TAG; + +public class ERTAws { + public static void main(final String ... args) { + ExchangeRateTool ert = new ExchangeRateTool(); + ert.run(AWS_TAG); + } +} diff --git a/src/main/java/com/hedera/exchange/ERTGcp.java b/src/main/java/com/hedera/exchange/ERTGcp.java new file mode 100644 index 0000000..fe571f0 --- /dev/null +++ b/src/main/java/com/hedera/exchange/ERTGcp.java @@ -0,0 +1,67 @@ +package com.hedera.exchange; + +/*- + * ‌ + * Hedera Exchange Rate Tool + * ​ + * Copyright (C) 2019 - 2020 Hedera Hashgraph, LLC + * ​ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ‍ + * + * Copyright (c) 2007-present, Stephen Colebourne & Michael Nascimento Santos. + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the name of JSR-310 nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import com.google.cloud.functions.HttpFunction; +import com.google.cloud.functions.HttpRequest; +import com.google.cloud.functions.HttpResponse; + +import static com.hedera.exchange.ExchangeRateTool.GCP_TAG; + +public class ERTGcp implements HttpFunction { + @Override + public void service(final HttpRequest httpRequest, final HttpResponse httpResponse) throws Exception { + ExchangeRateTool ert = new ExchangeRateTool(); + ert.run(GCP_TAG); + } +} diff --git a/src/main/java/com/hedera/exchange/ERTParams.java b/src/main/java/com/hedera/exchange/ERTParams.java index c538dff..3ae7c1e 100644 --- a/src/main/java/com/hedera/exchange/ERTParams.java +++ b/src/main/java/com/hedera/exchange/ERTParams.java @@ -223,7 +223,7 @@ private static ERTParams readDefaultConfigFromGCP() throws IOException { var blob = storage.get(bucket, config); return OBJECT_MAPPER.readValue(blob.getContent(), ERTParams.class); -// return readConfig("src/main/resources/config.json"); +// return readConfig("src/main/resources/configIntegration.json"); } /** diff --git a/src/main/java/com/hedera/exchange/ExchangeRateTool.java b/src/main/java/com/hedera/exchange/ExchangeRateTool.java index 5ebf6de..b1f6736 100644 --- a/src/main/java/com/hedera/exchange/ExchangeRateTool.java +++ b/src/main/java/com/hedera/exchange/ExchangeRateTool.java @@ -84,12 +84,13 @@ * * @author Anirudh, Cesar */ -public class ExchangeRateTool implements HttpFunction{ +public class ExchangeRateTool{ private static final Logger LOGGER = LogManager.getLogger(ExchangeRateTool.class); - private static final String AWS_TAG = "aws"; - private static final String GCP_TAG = "gcp"; private static final Map EMPTY_MAP = Collections.emptyMap(); + protected static final String AWS_TAG = "aws"; + protected static final String GCP_TAG = "gcp"; + static final int DEFAULT_RETRIES = 4; public static Environment env; @@ -98,11 +99,6 @@ public class ExchangeRateTool implements HttpFunction{ private ERTParams ertParams; private ExchangeDB exchangeDB; - public static void main(final String ... args) { - ExchangeRateTool ert = new ExchangeRateTool(); - ert.run("gcp"); - } - /** * This method executes the ERT logic and if an execution fails, it retries for the a fixed number of times * mentioned in DEFAULT_RETRIES. @@ -310,12 +306,8 @@ private Rate getCurrentRate(final ExchangeDB exchangeDb, final ERTParams params) return params.getDefaultRate(); } - @Override - public void service(final HttpRequest httpRequest, final HttpResponse httpResponse) throws Exception { - // this will be called from GCP cloud function. - // just call main with GCP tag - BufferedWriter writer = httpResponse.getWriter(); - writer.write("Hello World!"); - main("gcp"); + public static void main(final String ... args) { + ExchangeRateTool ert = new ExchangeRateTool(); + ert.run("gcp"); } } \ No newline at end of file diff --git a/src/main/java/com/hedera/exchange/database/DBParams.java b/src/main/java/com/hedera/exchange/database/DBParams.java index bf25cc4..3e59df2 100644 --- a/src/main/java/com/hedera/exchange/database/DBParams.java +++ b/src/main/java/com/hedera/exchange/database/DBParams.java @@ -80,7 +80,7 @@ public static String getEndpoint() { } else { LOGGER.error(Exchange.EXCHANGE_FILTER, "should not be possible"); - return ""; + return System.getenv("ENDPOINT") + getDatabaseName(); } } From bec302125b92868cba70f5438f260fbe6cb57975 Mon Sep 17 00:00:00 2001 From: anighanta Date: Wed, 22 Jun 2022 14:50:39 -0500 Subject: [PATCH 07/13] use sdk-full instead of sdk-jdk7 artifact from hedera Signed-off-by: anighanta --- pom.xml | 9 +++------ src/main/java/com/hedera/exchange/ExchangeRateTool.java | 4 ---- src/main/resources/configIntegration.json | 6 ++++++ 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/pom.xml b/pom.xml index ce07c3b..4975333 100644 --- a/pom.xml +++ b/pom.xml @@ -152,13 +152,10 @@ ${mockito.version} test - - - com.hedera.hashgraph - sdk-jdk7 - ${hedera-java-sdk.version} + sdk-full + 2.16.1 @@ -341,7 +338,7 @@ package shade - ${project.build.directory}/deployment/${build.finalName}.jar + ${project.build.directory}/deployment/${project.build.finalName}.jar diff --git a/src/main/java/com/hedera/exchange/ExchangeRateTool.java b/src/main/java/com/hedera/exchange/ExchangeRateTool.java index b1f6736..64e307c 100644 --- a/src/main/java/com/hedera/exchange/ExchangeRateTool.java +++ b/src/main/java/com/hedera/exchange/ExchangeRateTool.java @@ -52,9 +52,6 @@ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -import com.google.cloud.functions.HttpFunction; -import com.google.cloud.functions.HttpRequest; -import com.google.cloud.functions.HttpResponse; import com.hedera.hashgraph.sdk.AccountId; import com.hedera.hashgraph.sdk.Client; import com.hedera.exchange.database.ExchangeDB; @@ -66,7 +63,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import java.io.BufferedWriter; import java.io.IOException; import java.sql.SQLException; import java.util.Collections; diff --git a/src/main/resources/configIntegration.json b/src/main/resources/configIntegration.json index 101b4ab..5662a46 100644 --- a/src/main/resources/configIntegration.json +++ b/src/main/resources/configIntegration.json @@ -18,6 +18,12 @@ "Nodes": { "0.0.3": "35.196.146.2:50211" }, + "Networks": { + "integrationNet": { + "0.0.4": "35.245.150.69:50211", + "0.0.5": "34.70.193.123:50211" + } + }, "payerAccount": "0.0.57" } From 9408bf6e704c1d4bcf900b686657f3980456c377 Mon Sep 17 00:00:00 2001 From: Neeharika-Sompalli Date: Tue, 30 Aug 2022 22:43:24 -0500 Subject: [PATCH 08/13] update to latest sdk Signed-off-by: Neeharika-Sompalli --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 4975333..4e19041 100644 --- a/pom.xml +++ b/pom.xml @@ -51,7 +51,7 @@ 1.2.1 1.5.1 1.12.237 - 2.16.0-beta.1 + 2.17.3 From 8faa308e21553c194d4fb3f1a6447bb65b58ec50 Mon Sep 17 00:00:00 2001 From: avetswirldslabs <119908649+avetswirldslabs@users.noreply.github.com> Date: Wed, 1 Mar 2023 22:46:32 -0500 Subject: [PATCH 09/13] GCP-support-SQL-matching-AWS Signed-off-by: avetswirldslabs <119908649+avetswirldslabs@users.noreply.github.com> --- .../hedera/exchange/database/QueryHelper.java | 77 ++++++++++--------- 1 file changed, 41 insertions(+), 36 deletions(-) diff --git a/src/main/java/com/hedera/exchange/database/QueryHelper.java b/src/main/java/com/hedera/exchange/database/QueryHelper.java index f781dc5..c55316c 100644 --- a/src/main/java/com/hedera/exchange/database/QueryHelper.java +++ b/src/main/java/com/hedera/exchange/database/QueryHelper.java @@ -77,20 +77,21 @@ * This class implements the ExchangeDB interface * which species what APIs that we need to fetch/push data into the Database. * - * If you foresee doing more of this mapping, I would recommend moving to JPA/Hibernate. + * If you foresee doing more of this mapping, I would recommend moving to + * JPA/Hibernate. * */ public class QueryHelper implements ExchangeDB { private static final Logger LOGGER = LogManager.getLogger(QueryHelper.class); - private static final String LATEST_EXCHANGE_QUERY = "SELECT e1.expiration_time, e1.exchange_rate_file FROM exchange_rate AS e1 INNER JOIN (SELECT MAX(expiration_time) expiration_time FROM exchange_rate) AS e2 ON e1.expiration_time = e2.expiration_time LIMIT 1"; + private static final String LATEST_EXCHANGE_QUERY = "SELECT e1.expirationtime, e1.exchangeratefile FROM exchange_rate AS e1 INNER JOIN (SELECT MAX(expirationtime) expirationtime FROM exchange_rate) AS e2 ON e1.expirationtime = e2.expirationtime LIMIT 1"; - private static final String LATEST_ADDRESS_BOOK_QUERY = "SELECT e1.expiration_time, e1.address_book FROM address_book AS e1 INNER JOIN (SELECT MAX(expiration_time) expiration_time FROM address_book) AS e2 ON e1.expiration_time = e2.expiration_time and network_name = ? LIMIT 1"; + private static final String LATEST_ADDRESS_BOOK_QUERY = "SELECT e1.expirationtime, e1.addressbook FROM address_book AS e1 INNER JOIN (SELECT MAX(expirationtime) expirationtime FROM address_book) AS e2 ON e1.expirationtime = e2.expirationtime and network_name = ? LIMIT 1"; - private static final String MIDNIGHT_EXCHANGE_QUERY = "SELECT e1.expiration_time, e1.exchange_rate_file FROM midnight_rate AS e1 INNER JOIN (SELECT MAX(expiration_time) expiration_time FROM midnight_rate) AS e2 ON e1.expiration_time = e2.expiration_time LIMIT 1"; + private static final String MIDNIGHT_EXCHANGE_QUERY = "SELECT e1.expirationtime, e1.exchangerate_file FROM midnight_rate AS e1 INNER JOIN (SELECT MAX(expirationtime) expirationtime FROM midnight_rate) AS e2 ON e1.expirationtime = e2.expirationtime LIMIT 1"; - private static final String LATEST_QUERIED_QUERY = "SELECT e1.expiration_time, e1.queried_rates FROM queried_rates AS e1 INNER JOIN (SELECT MAX(expiration_time) expiration_time FROM queried_rates) AS e2 ON e1.expiration_time = e2.expiration_time LIMIT 1"; + private static final String LATEST_QUERIED_QUERY = "SELECT e1.expirationtime, e1.queriedrates FROM queried_rates AS e1 INNER JOIN (SELECT MAX(expirationtime) expirationtime FROM queried_rates) AS e2 ON e1.expirationtime = e2.expirationtime LIMIT 1"; public QueryHelper() { this.migrate(); @@ -117,12 +118,13 @@ public ExchangeRate getMidnightExchangeRate(long expirationTime) throws SQLExcep LOGGER.info(Exchange.EXCHANGE_FILTER, "query to get the midnight exchange rate from midnight rate table " + "with nextExpiration time :{}", expirationTime); try (final Connection conn = DBParams.getConnection(); - final PreparedStatement prepStatement = conn.prepareStatement( - "SELECT expirationTime, exchangeRateFile FROM midnight_rate where expirationTime = ?")){ + final PreparedStatement prepStatement = conn.prepareStatement( + "SELECT expirationTime, exchangeRateFile FROM midnight_rate where expirationTime = ?")) { prepStatement.setLong(1, expirationTime); final ResultSet result = prepStatement.executeQuery(); if (result.next()) { - LOGGER.info(Exchange.EXCHANGE_FILTER, "the midnight rate at {} is {}", expirationTime, result.getString(2)); + LOGGER.info(Exchange.EXCHANGE_FILTER, "the midnight rate at {} is {}", expirationTime, + result.getString(2)); return ExchangeRate.fromJson(result.getString(2)); } LOGGER.warn(Exchange.EXCHANGE_FILTER, "failed to midnight exchange rate from midnight rate table " + @@ -135,13 +137,14 @@ public ExchangeRate getMidnightExchangeRate(long expirationTime) throws SQLExcep public ERTAddressBook getLatestERTAddressBook(String networkName) throws SQLException, IOException { LOGGER.info(Exchange.EXCHANGE_FILTER, "query to get latest ERTAddressBook from address_book table"); try (final Connection conn = DBParams.getConnection(); - final PreparedStatement statement = conn.prepareStatement(LATEST_ADDRESS_BOOK_QUERY)) { + final PreparedStatement statement = conn.prepareStatement(LATEST_ADDRESS_BOOK_QUERY)) { statement.setString(1, networkName); - LOGGER.info(Exchange.EXCHANGE_FILTER,"final query for addressbook : {}", + LOGGER.info(Exchange.EXCHANGE_FILTER, "final query for addressbook : {}", statement.toString()); final ResultSet result = statement.executeQuery(); if (result.next()) { - LOGGER.info(Exchange.EXCHANGE_FILTER, "the latest ERTAddressBook from address_book table {} for network {}", + LOGGER.info(Exchange.EXCHANGE_FILTER, + "the latest ERTAddressBook from address_book table {} for network {}", result.getString(2), networkName); return ERTAddressBook.fromJson(result.getString(2)); } @@ -157,11 +160,11 @@ public void pushERTAddressBook(long expirationTime, ERTAddressBook ertAddressBoo LOGGER.info(Exchange.EXCHANGE_FILTER, "push latest addressBook to address_book table : {}", addressBook); try (final Connection conn = DBParams.getConnection(); - final PreparedStatement statement = conn.prepareStatement( - "INSERT INTO address_book (expirationTime,addressBook,networkName) VALUES(?,?::JSON,?)")) { + final PreparedStatement statement = conn.prepareStatement( + "INSERT INTO address_book (expirationTime,addressBook,networkName) VALUES(?,?::JSON,?)")) { statement.setLong(1, expirationTime); statement.setObject(2, addressBook); - statement.setString(3,networkName); + statement.setString(3, networkName); statement.executeUpdate(); } } @@ -170,12 +173,12 @@ public void pushERTAddressBook(long expirationTime, ERTAddressBook ertAddressBoo public ExchangeRate getLatestExchangeRate() throws SQLException, IOException { LOGGER.info(Exchange.EXCHANGE_FILTER, "query to get latest exchange rate from exchange rate table"); try (final Connection conn = DBParams.getConnection(); - final Statement statement = conn.createStatement(); - final ResultSet result = statement.executeQuery(LATEST_EXCHANGE_QUERY)) { - if (result.next()) { - LOGGER.info(Exchange.EXCHANGE_FILTER, "the latest exchange rate : {}", result.getString(2)); - return ExchangeRate.fromJson(result.getString(2)); - } + final Statement statement = conn.createStatement(); + final ResultSet result = statement.executeQuery(LATEST_EXCHANGE_QUERY)) { + if (result.next()) { + LOGGER.info(Exchange.EXCHANGE_FILTER, "the latest exchange rate : {}", result.getString(2)); + return ExchangeRate.fromJson(result.getString(2)); + } LOGGER.warn(Exchange.EXCHANGE_FILTER, "failed to get latest exchange rate from exchange rate table "); return null; } @@ -186,12 +189,13 @@ public ExchangeRate getExchangeRate(long expirationTime) throws SQLException, IO LOGGER.info(Exchange.EXCHANGE_FILTER, "query to get the exchange rate from exchange rate table " + "with nextExpiration time :{}", expirationTime); try (final Connection conn = DBParams.getConnection(); - final PreparedStatement prepStatement = conn.prepareStatement( - "SELECT expirationTime, exchangeRateFile FROM exchange_rate where expirationTime = ?")){ + final PreparedStatement prepStatement = conn.prepareStatement( + "SELECT expirationTime, exchangeRateFile FROM exchange_rate where expirationTime = ?")) { prepStatement.setLong(1, expirationTime); final ResultSet result = prepStatement.executeQuery(); if (result.next()) { - LOGGER.info(Exchange.EXCHANGE_FILTER, "the exchange rate at {} is {}", expirationTime, result.getString(2)); + LOGGER.info(Exchange.EXCHANGE_FILTER, "the exchange rate at {} is {}", expirationTime, + result.getString(2)); return ExchangeRate.fromJson(result.getString(2)); } LOGGER.warn(Exchange.EXCHANGE_FILTER, "failed to exchange rate from exchange rate table " + @@ -205,12 +209,13 @@ public String getQueriedRate(long expirationTime) throws SQLException { LOGGER.info(Exchange.EXCHANGE_FILTER, "query to get the queried rate from queried rate table " + "with nextExpiration time :{}", expirationTime); try (final Connection conn = DBParams.getConnection(); - final PreparedStatement prepStatement = conn.prepareStatement( - "SELECT expirationTime, queriedrates FROM queried_rate where expirationTime = ?")){ + final PreparedStatement prepStatement = conn.prepareStatement( + "SELECT expirationTime, queriedrates FROM queried_rate where expirationTime = ?")) { prepStatement.setLong(1, expirationTime); final ResultSet result = prepStatement.executeQuery(); if (result.next()) { - LOGGER.info(Exchange.EXCHANGE_FILTER, "the queried rate at {} is {}", expirationTime, result.getString(2)); + LOGGER.info(Exchange.EXCHANGE_FILTER, "the queried rate at {} is {}", expirationTime, + result.getString(2)); return result.getString(2); } LOGGER.warn(Exchange.EXCHANGE_FILTER, "failed to get the queried rate from queried rate table " + @@ -223,8 +228,8 @@ public String getQueriedRate(long expirationTime) throws SQLException { public ExchangeRate getLatestMidnightExchangeRate() throws SQLException, IOException { LOGGER.info(Exchange.EXCHANGE_FILTER, "query to get midnight exchange rate from midnight rate table"); try (final Connection conn = DBParams.getConnection(); - final Statement statement = conn.createStatement(); - final ResultSet result = statement.executeQuery(MIDNIGHT_EXCHANGE_QUERY)) { + final Statement statement = conn.createStatement(); + final ResultSet result = statement.executeQuery(MIDNIGHT_EXCHANGE_QUERY)) { if (result.next()) { LOGGER.info(Exchange.EXCHANGE_FILTER, "the midnight exchange rate : {}", result.getString(2)); return ExchangeRate.fromJson(result.getString(2)); @@ -238,8 +243,8 @@ public ExchangeRate getLatestMidnightExchangeRate() throws SQLException, IOExcep public String getLatestQueriedRate() throws SQLException { LOGGER.info(Exchange.EXCHANGE_FILTER, "query to get the latest queried rate"); try (final Connection conn = DBParams.getConnection(); - final Statement statement = conn.createStatement(); - final ResultSet result = statement.executeQuery(LATEST_QUERIED_QUERY)) { + final Statement statement = conn.createStatement(); + final ResultSet result = statement.executeQuery(LATEST_QUERIED_QUERY)) { if (result.next()) { LOGGER.info(Exchange.EXCHANGE_FILTER, "the queried rate : {}", result.getString(2)); return result.getString(2); @@ -254,8 +259,8 @@ public void pushExchangeRate(ExchangeRate exchangeRate) throws SQLException, IOE LOGGER.info(Exchange.EXCHANGE_FILTER, "push latest exchange rate to exchange rate table : {}", exchangeRate.toJson()); try (final Connection conn = DBParams.getConnection(); - final PreparedStatement statement = conn.prepareStatement( - "INSERT INTO exchange_rate (expirationTime,exchangeRateFile) VALUES(?,?::JSON)")) { + final PreparedStatement statement = conn.prepareStatement( + "INSERT INTO exchange_rate (expirationTime,exchangeRateFile) VALUES(?,?::JSON)")) { statement.setLong(1, exchangeRate.getNextExpirationTimeInSeconds()); statement.setObject(2, exchangeRate.toJson()); statement.executeUpdate(); @@ -267,8 +272,8 @@ public void pushMidnightRate(ExchangeRate exchangeRate) throws SQLException, IOE LOGGER.info(Exchange.EXCHANGE_FILTER, "push the midnight exchange rate to midnight rate table : {}", exchangeRate.toJson()); try (final Connection conn = DBParams.getConnection(); - final PreparedStatement statement = conn.prepareStatement( - "INSERT INTO midnight_rate (expirationTime,exchangeRateFile) VALUES(?,?::JSON)")) { + final PreparedStatement statement = conn.prepareStatement( + "INSERT INTO midnight_rate (expirationTime,exchangeRateFile) VALUES(?,?::JSON)")) { statement.setLong(1, exchangeRate.getNextExpirationTimeInSeconds()); statement.setObject(2, exchangeRate.toJson()); statement.executeUpdate(); @@ -280,8 +285,8 @@ public void pushQueriedRate(long expirationTime, String queriedRate) throws SQLE LOGGER.info(Exchange.EXCHANGE_FILTER, "push the queried exchanges to queried rate table : {}:{}", expirationTime, queriedRate); try (final Connection conn = DBParams.getConnection(); - final PreparedStatement statement = conn.prepareStatement( - "INSERT INTO queried_rate (expirationTime,queriedrates) VALUES(?,?::JSON)")) { + final PreparedStatement statement = conn.prepareStatement( + "INSERT INTO queried_rate (expirationTime,queriedrates) VALUES(?,?::JSON)")) { statement.setLong(1, expirationTime); statement.setObject(2, queriedRate); statement.executeUpdate(); From c59328238d5b5a55f543fb2d87092373f2419717 Mon Sep 17 00:00:00 2001 From: avetswirldslabs <119908649+avetswirldslabs@users.noreply.github.com> Date: Sun, 19 Mar 2023 10:28:54 -0400 Subject: [PATCH 10/13] Update V1__Create_ERT_table.sql create address_book table --- src/main/resources/db/migration/V1__Create_ERT_table.sql | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/main/resources/db/migration/V1__Create_ERT_table.sql b/src/main/resources/db/migration/V1__Create_ERT_table.sql index 0f9f567..c5a09bf 100644 --- a/src/main/resources/db/migration/V1__Create_ERT_table.sql +++ b/src/main/resources/db/migration/V1__Create_ERT_table.sql @@ -1,3 +1,12 @@ +CREATE TABLE "address_book" +( + id SERIAL NOT NULL, + expirationTime BIGINT, + exchangeRateFile JSON +); + +CREATE INDEX ON address_book (expirationTime); + CREATE TABLE "exchange_rate" ( id SERIAL NOT NULL, From c116e0986dcbb065f8597e4a0568b39ca42a1cf4 Mon Sep 17 00:00:00 2001 From: avetswirldslabs <119908649+avetswirldslabs@users.noreply.github.com> Date: Sun, 19 Mar 2023 18:37:27 -0400 Subject: [PATCH 11/13] Update V1__Create_ERT_table.sql --- src/main/resources/db/migration/V1__Create_ERT_table.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/resources/db/migration/V1__Create_ERT_table.sql b/src/main/resources/db/migration/V1__Create_ERT_table.sql index c5a09bf..96a9423 100644 --- a/src/main/resources/db/migration/V1__Create_ERT_table.sql +++ b/src/main/resources/db/migration/V1__Create_ERT_table.sql @@ -2,7 +2,8 @@ CREATE TABLE "address_book" ( id SERIAL NOT NULL, expirationTime BIGINT, - exchangeRateFile JSON + exchangeRateFile JSON, + networkname VARCHAR ); CREATE INDEX ON address_book (expirationTime); From de6307694f15354c1fc44a71ef2038d11ff7dca2 Mon Sep 17 00:00:00 2001 From: avetswirldslabs <119908649+avetswirldslabs@users.noreply.github.com> Date: Sun, 19 Mar 2023 20:04:38 -0400 Subject: [PATCH 12/13] Update V1__Create_ERT_table.sql --- src/main/resources/db/migration/V1__Create_ERT_table.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/db/migration/V1__Create_ERT_table.sql b/src/main/resources/db/migration/V1__Create_ERT_table.sql index 96a9423..429f817 100644 --- a/src/main/resources/db/migration/V1__Create_ERT_table.sql +++ b/src/main/resources/db/migration/V1__Create_ERT_table.sql @@ -2,7 +2,7 @@ CREATE TABLE "address_book" ( id SERIAL NOT NULL, expirationTime BIGINT, - exchangeRateFile JSON, + addressbook JSON, networkname VARCHAR ); From fcc62b2427f1d4cea85cc85124835684645a460c Mon Sep 17 00:00:00 2001 From: avetswirldslabs <119908649+avetswirldslabs@users.noreply.github.com> Date: Sun, 19 Mar 2023 21:27:28 -0400 Subject: [PATCH 13/13] Update QueryHelper.java --- src/main/java/com/hedera/exchange/database/QueryHelper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/hedera/exchange/database/QueryHelper.java b/src/main/java/com/hedera/exchange/database/QueryHelper.java index c55316c..3d799b9 100644 --- a/src/main/java/com/hedera/exchange/database/QueryHelper.java +++ b/src/main/java/com/hedera/exchange/database/QueryHelper.java @@ -91,7 +91,7 @@ public class QueryHelper implements ExchangeDB { private static final String MIDNIGHT_EXCHANGE_QUERY = "SELECT e1.expirationtime, e1.exchangerate_file FROM midnight_rate AS e1 INNER JOIN (SELECT MAX(expirationtime) expirationtime FROM midnight_rate) AS e2 ON e1.expirationtime = e2.expirationtime LIMIT 1"; - private static final String LATEST_QUERIED_QUERY = "SELECT e1.expirationtime, e1.queriedrates FROM queried_rates AS e1 INNER JOIN (SELECT MAX(expirationtime) expirationtime FROM queried_rates) AS e2 ON e1.expirationtime = e2.expirationtime LIMIT 1"; + private static final String LATEST_QUERIED_QUERY = "SELECT e1.expirationtime, e1.queriedrates FROM queried_rate AS e1 INNER JOIN (SELECT MAX(expirationtime) expirationtime FROM queried_rate) AS e2 ON e1.expirationtime = e2.expirationtime LIMIT 1"; public QueryHelper() { this.migrate();