Cndit 1648 outbound rdb (#23)

* rdb model classes * deleted the model classes * Update DataPullService.java * poll service for the rdb * Updated sync Rdb Data * polling completed for three tables. * Renamed the table. * updated config table * Create insert.sql created insert sql * Updated * Updated * Updated for the RDB tables and log4j * Updated * Updated * Updated. This can be a base version for the rdb polling. tested with exchange api. * Added new table entries. * Updated for the rdb poll service * Update DataPullService.java * Updated env variables * Renamed the file name * updated for unit testing. * Update .DS_Store * renamed the file and used the the content from main. * updated for sonarscan * Updated for sonarscan * updated for test cases
CDCgov · Sep 9, 2024 · e24e0b4 · e24e0b4
1 parent 72db5df
commit e24e0b4
Show file tree

Hide file tree

Showing 28 changed files with 1,035 additions and 39 deletions.
diff --git a/.DS_Store b/.DS_Store
diff --git a/nnd-data-exchange-service/.DS_Store b/nnd-data-exchange-service/.DS_Store
diff --git a/...exchange-service/src/main/java/gov/cdc/nnddataexchangeservice/security/OpenAPIConfig.java b/...exchange-service/src/main/java/gov/cdc/nnddataexchangeservice/security/OpenAPIConfig.java
@@ -40,14 +40,14 @@ public OpenAPI myOpenAPI() throws Exception{
         server.setDescription("Server URL");
 
         Contact contact = new Contact();
-        contact.setEmail("dataingestionservice@cdc.com");
-        contact.setName("Data Ingestion Service");
+        contact.setEmail("nndservice@cdc.com");
+        contact.setName("NND Service");
 
         Info info = new Info()
-                .title("Data Ingestion API")
+                .title("NND Service API")
                 .version("1.0")
                 .contact(contact)
-                .description("This API exposes endpoints to manage Data Ingestion Service.");
+                .description("This API exposes endpoints to manage NND Service.");
 
         Components components=new Components().
                 addSecuritySchemes("bearer-key",

diff --git a/nnd-data-poll-service/.DS_Store b/nnd-data-poll-service/.DS_Store
diff --git a/nnd-data-poll-service/build.gradle b/nnd-data-poll-service/build.gradle
@@ -32,7 +32,7 @@ dependencies {
     testImplementation 'org.springframework.boot:spring-boot-starter-test'
     testRuntimeOnly 'org.junit.platform:junit-platform-launcher'
     implementation 'com.microsoft.sqlserver:mssql-jdbc:12.2.0.jre11'
-
+    implementation 'net.logstash.logback:logstash-logback-encoder:7.4'
     testImplementation(platform('org.junit:junit-bom'))
     testImplementation('org.junit.jupiter:junit-jupiter')
     testImplementation 'org.junit.jupiter:junit-jupiter-engine'

diff --git a/nnd-data-poll-service/execute_script/nnd_data_poll_service.cmd b/nnd-data-poll-service/execute_script/nnd_data_poll_service.cmd
@@ -1,17 +1,21 @@
 @echo off
-set NBS_NND_CRON=* * * * * *
+set NBS_NND_CRON=0 */1 * * * *
 set NBS_NND_CRON_TIME_ZONE=UTC
 set NND_FULL_LOAD=false
 set NND_PULL_LIMIT=0
 set NND_INSERT_LIMIT=1000
 set NND_DE_CLIENT_ID=nnd-keycloak-client
-set NND_DE_DE=/api/nnd/data-exchange
 set NND_DE_SECRET=bpVTppDam4sxXt4hfgm5hZ6Rteponjb9
-set NND_DE_TOKEN=/api/auth/token
+@REM set NND_DE_TOKEN=/api/auth/token
+@REM set NND_DE_DE=/api/nnd/data-exchange
+@REM set NND_DE_GENERIC=/api/data-exchange-generic
 set NND_DE_URL=http://localhost:8081
 set NND_FILE_LOCATION=C:\Users\DucNguyen\Desktop\LOG\Poll
+set NND_POLL_ENABLED=true
+set RDB_POLL_ENABLED=true
 set OP_DBPASSWORD=fake.fake.fake.1234
 set OP_DBSERVER=localhost:1433
 set OP_DBUSER=sa
 set OP_DBNAME=MSGOUTE
-java -jar nnd-data-poll-service.jar
+set RDB_DBNAME=RDB
+java -jar nnd-data-poll-service.jar
diff --git a/nnd-data-poll-service/execute_script/nnd_data_poll_service.sh b/nnd-data-poll-service/execute_script/nnd_data_poll_service.sh
@@ -5,13 +5,17 @@ export NND_FULL_LOAD="false"
 export NND_PULL_LIMIT="0"
 export NND_INSERT_LIMIT="1000"
 export NND_DE_CLIENT_ID="nnd-keycloak-client"
-export NND_DE_DE="/api/nnd/data-exchange"
 export NND_DE_SECRET="bpVTppDam4sxXt4hfgm5hZ6Rteponjb9"
-export NND_DE_TOKEN="/api/auth/token"
+#export NND_DE_TOKEN="/api/auth/token"
+#export NND_DE_DE="/api/nnd/data-exchange"
+#export NND_DE_GENERIC="/api/data-exchange-generic"
 export NND_DE_URL="http://localhost:8081"
 export NND_FILE_LOCATION="/Users/DucNguyen/Desktop/LOG/Poll"
+export NND_POLL_ENABLED="true"
+export RDB_POLL_ENABLED="true"
 export OP_DBPASSWORD="fake.fake.fake.1234"
 export OP_DBSERVER="localhost:1433"
 export OP_DBUSER="sa"
 export OP_DBNAME="MSGOUTE"
+export RDB_DBNAME="RDB"
 java -jar nnd-data-poll-service.jar
diff --git a/nnd-data-poll-service/readme.md b/nnd-data-poll-service/readme.md
@@ -25,3 +25,9 @@ export SPRING_PROFILES_ACTIVE=dev
 export CUSTOM_PROPERTY=myValue
 java -jar build/libs/nnd-data-poll.jar
 
+Ex: Setting up the scheduler time.
+"*/1 * * * * ?" => For 1 second
+"0 */1 * * * ?" => For 1 minute
+"0 0 */1 * * ?" => For 1 hour
+"0 0 0 */1 * ?" => For every day
+"0 0 0 1 1 *" => It scheduled as 1 date of 1st month(january) every year
diff --git a/nnd-data-poll-service/src/.DS_Store b/nnd-data-poll-service/src/.DS_Store
diff --git a/nnd-data-poll-service/src/main/.DS_Store b/nnd-data-poll-service/src/main/.DS_Store
diff --git a/...l-service/src/main/java/gov/cdc/nnddatapollservice/configuration/RdbDataSourceConfig.java b/...l-service/src/main/java/gov/cdc/nnddatapollservice/configuration/RdbDataSourceConfig.java
@@ -0,0 +1,41 @@
+package gov.cdc.nnddatapollservice.configuration;
+
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.jdbc.DataSourceBuilder;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.jdbc.core.JdbcTemplate;
+
+import javax.sql.DataSource;
+
+@Configuration
+public class RdbDataSourceConfig {
+    @Value("${spring.datasource.driverClassName}")
+    private String driverClassName;
+
+    @Value("${spring.datasource.rdb.url}")
+    private String dbUrl;
+
+    @Value("${spring.datasource.username}")
+    private String dbUserName;
+
+    @Value("${spring.datasource.password}")
+    private String dbUserPassword;
+
+    @Bean(name = "rdbDataSource")
+    public DataSource rdbDataSource() {
+        DataSourceBuilder dataSourceBuilder = DataSourceBuilder.create();
+
+        dataSourceBuilder.driverClassName(driverClassName);
+        dataSourceBuilder.url(dbUrl);
+        dataSourceBuilder.username(dbUserName);
+        dataSourceBuilder.password(dbUserPassword);
+
+        return dataSourceBuilder.build();
+    }
+    @Bean(name = "rdbJdbcTemplate")
+    public JdbcTemplate rdbJdbcTemplate(@Qualifier("rdbDataSource") DataSource dataSource) {
+        return new JdbcTemplate(dataSource);
+    }
+}
diff --git a/...a-poll-service/src/main/java/gov/cdc/nnddatapollservice/rdb/dao/RdbDataPersistentDAO.java b/...a-poll-service/src/main/java/gov/cdc/nnddatapollservice/rdb/dao/RdbDataPersistentDAO.java
@@ -0,0 +1,269 @@
+package gov.cdc.nnddatapollservice.rdb.dao;
+
+import com.google.gson.FieldNamingPolicy;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.reflect.TypeToken;
+import gov.cdc.nnddatapollservice.rdb.dto.Condition;
+import gov.cdc.nnddatapollservice.rdb.dto.ConfirmationMethod;
+import gov.cdc.nnddatapollservice.rdb.dto.PollDataSyncConfig;
+import gov.cdc.nnddatapollservice.rdb.dto.RdbDate;
+import lombok.extern.slf4j.Slf4j;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.jdbc.core.BeanPropertyRowMapper;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.jdbc.core.namedparam.SqlParameterSourceUtils;
+import org.springframework.jdbc.core.simple.SimpleJdbcInsert;
+import org.springframework.stereotype.Service;
+
+import java.lang.reflect.Type;
+import java.sql.Timestamp;
+import java.text.SimpleDateFormat;
+import java.util.List;
+import java.util.Map;
+
+@Service
+@Slf4j
+public class RdbDataPersistentDAO {
+    private static Logger logger = LoggerFactory.getLogger(RdbDataPersistentDAO.class);
+    private JdbcTemplate jdbcTemplate;
+    private static final String TIMESTAMP_FORMAT = "yyyy-MM-dd HH:mm:ss.SSS";
+
+    @Autowired
+    public RdbDataPersistentDAO(@Qualifier("rdbJdbcTemplate") JdbcTemplate jdbcTemplate) {
+        this.jdbcTemplate = jdbcTemplate;
+    }
+
+    @SuppressWarnings("java:S3776")
+    public int saveRDBData(String tableName, String jsonData) {
+        logger.info("saveRDBData tableName: {}", tableName);
+        int noOfRecordsSaved = 0;
+        if ("CONFIRMATION_METHOD".equalsIgnoreCase(tableName)) {
+            Gson gson = new GsonBuilder()
+                    .setFieldNamingPolicy(FieldNamingPolicy.UPPER_CASE_WITH_UNDERSCORES)
+                    .create();
+            Type resultType = new TypeToken<List<ConfirmationMethod>>() {
+            }.getType();
+            List<ConfirmationMethod> list = gson.fromJson(jsonData, resultType);
+            for (ConfirmationMethod confirmationMethod : list) {
+                noOfRecordsSaved=upsertConfirmationMethod(confirmationMethod);
+            }
+        } else if ("CONDITION".equalsIgnoreCase(tableName)) {
+            Gson gson = new GsonBuilder()
+                    .setFieldNamingPolicy(FieldNamingPolicy.UPPER_CASE_WITH_UNDERSCORES)
+                    .create();
+            Type resultType = new TypeToken<List<Condition>>() {
+            }.getType();
+            List<Condition> list = gson.fromJson(jsonData, resultType);
+            for (Condition condition : list) {
+                noOfRecordsSaved=upsertCondition(condition);
+            }
+        } else if ("RDB_DATE".equalsIgnoreCase(tableName)) {
+            Gson gson = new GsonBuilder()
+                    .setFieldNamingPolicy(FieldNamingPolicy.UPPER_CASE_WITH_UNDERSCORES)
+                    .create();
+            Type resultType = new TypeToken<List<RdbDate>>() {
+            }.getType();
+            List<RdbDate> list = gson.fromJson(jsonData, resultType);
+            for (RdbDate rdbDate : list) {
+                noOfRecordsSaved=upsertRdbDate(rdbDate);
+            }
+        } else {
+            try {
+                SimpleJdbcInsert simpleJdbcInsert =
+                        new SimpleJdbcInsert(jdbcTemplate);
+                if(tableName!=null && !tableName.isEmpty()) {
+                    simpleJdbcInsert = simpleJdbcInsert.withTableName(tableName);
+                    List<Map<String, Object>> records = jsonToListOfMap(jsonData);
+                    if (records != null && !records.isEmpty()) {
+                        logger.info("Inside generic code before executeBatch tableName: {} Records size:{}", tableName, records.size());
+                        int[] noOfInserts =simpleJdbcInsert.executeBatch(SqlParameterSourceUtils.createBatch(records));
+                        noOfRecordsSaved=noOfInserts.length;
+                        logger.info("executeBatch completed. tableName: {}", tableName);
+                    } else {
+                        logger.info("Inside generic code tableName: {} Records size:0", tableName);
+                    }
+                }
+            } catch (Exception e) {
+                logger.error("Error executeBatch. tableName: {}, Error:{}", tableName, e.getMessage());
+            }
+        }
+        logger.info("saveRDBData tableName: {} Records size:{}",tableName, noOfRecordsSaved);
+        return noOfRecordsSaved;
+    }
+
+    private List<Map<String, Object>> jsonToListOfMap(String jsonData) {
+        List<Map<String, Object>> list = null;
+        if (jsonData != null && !jsonData.isEmpty()) {
+            Gson gson = new GsonBuilder().serializeNulls().create();
+            Type resultType = new TypeToken<List<Map<String, Object>>>() {
+            }.getType();
+            list = gson.fromJson(jsonData, resultType);
+        }
+        return list;
+    }
+
+    private int upsertConfirmationMethod(ConfirmationMethod confirmationMethod) {
+        int noOfRecordsUpdated = 0;
+        String sql = "MERGE INTO CONFIRMATION_METHOD AS target USING " +
+                "(select " + confirmationMethod.getConfirmationMethodKey() + " as id) AS source " +
+                "ON (target.CONFIRMATION_METHOD_KEY = source.id) " +
+                "WHEN MATCHED THEN " +
+                " UPDATE SET target.CONFIRMATION_METHOD_CD = " + getSqlString(confirmationMethod.getConfirmationMethodCd()) +
+                ",target.CONFIRMATION_METHOD_DESC = " + getSqlString(confirmationMethod.getConfirmationMethodDesc()) +
+                " WHEN NOT MATCHED THEN " +
+                " INSERT (CONFIRMATION_METHOD_KEY, CONFIRMATION_METHOD_CD,CONFIRMATION_METHOD_DESC) " +
+                "VALUES (" + confirmationMethod.getConfirmationMethodKey() +
+                "," + getSqlString(confirmationMethod.getConfirmationMethodCd()) +
+                "," + getSqlString(confirmationMethod.getConfirmationMethodDesc()) + ");";
+        try {
+            noOfRecordsUpdated= jdbcTemplate.update(sql);
+        } catch (Exception e) {
+            logger.error("Error in upsert for CONFIRMATION_METHOD table:{}",e.getMessage());
+        }
+        return noOfRecordsUpdated;
+    }
+
+    private int upsertCondition(Condition condition) {
+        int noOfRecordsUpdated = 0;
+        String sql = "MERGE INTO CONDITION AS target " +
+                "USING (select " + condition.getConditionKey() + " as id) AS source ON" +
+                "(target.CONDITION_KEY = source.id)" +
+                "WHEN MATCHED THEN " +
+                "UPDATE " +
+                "SET " +
+                " target.CONDITION_CD = " + getSqlString(condition.getConditionCd()) + "," +
+                " target.CONDITION_DESC = " + getSqlString(condition.getConditionDesc()) + "," +
+                " target.CONDITION_SHORT_NM = " + getSqlString(condition.getConditionShortNm()) + "," +
+                " target.CONDITION_CD_EFF_DT = " + getSqlString(condition.getConditionCdEffDt()) + "," +
+                " target.CONDITION_CD_END_DT = " + getSqlString(condition.getConditionCdEndDt()) + "," +
+                " target.NND_IND = " + getSqlString(condition.getNndInd()) + "," +
+                " target.DISEASE_GRP_CD = " + getSqlString(condition.getDiseaseGrpCd()) + "," +
+                " target.DISEASE_GRP_DESC = " + getSqlString(condition.getDiseaseGrpDesc()) + "," +
+                " target.PROGRAM_AREA_CD = " + getSqlString(condition.getProgramAreaCd()) + "," +
+                " target.PROGRAM_AREA_DESC = " + getSqlString(condition.getProgramAreaDesc()) + "," +
+                " target.CONDITION_CD_SYS_CD_NM = " + getSqlString(condition.getConditionCdSysCdNm()) + "," +
+                " target.ASSIGNING_AUTHORITY_CD = " + getSqlString(condition.getAssigningAuthorityCd()) + "," +
+                " target.ASSIGNING_AUTHORITY_DESC = " + getSqlString(condition.getAssigningAuthorityDesc()) + "," +
+                " target.CONDITION_CD_SYS_CD = " + getSqlString(condition.getConditionCdSysCd()) +
+                " WHEN NOT MATCHED THEN " +
+                "INSERT " +
+                " (CONDITION_KEY," +
+                " CONDITION_CD," +
+                " CONDITION_DESC,CONDITION_SHORT_NM,CONDITION_CD_EFF_DT,CONDITION_CD_END_DT,NND_IND," +
+                " DISEASE_GRP_CD,DISEASE_GRP_DESC,PROGRAM_AREA_CD,PROGRAM_AREA_DESC," +
+                " CONDITION_CD_SYS_CD_NM,ASSIGNING_AUTHORITY_CD,ASSIGNING_AUTHORITY_DESC,CONDITION_CD_SYS_CD)" +
+                "VALUES ("
+                + condition.getConditionKey() + "," +
+                getSqlString(condition.getConditionCd()) + "," +
+                getSqlString(condition.getConditionDesc()) + "," +
+                getSqlString(condition.getConditionShortNm()) + "," +
+                getSqlString(condition.getConditionCdEffDt()) + "," +
+                getSqlString(condition.getConditionCdEndDt()) + "," +
+                getSqlString(condition.getNndInd()) + "," +
+                getSqlString(condition.getDiseaseGrpCd()) + "," +
+                getSqlString(condition.getDiseaseGrpDesc()) + "," +
+                getSqlString(condition.getProgramAreaCd()) + "," +
+                getSqlString(condition.getProgramAreaDesc()) + "," +
+                getSqlString(condition.getConditionCdSysCdNm()) + "," +
+                getSqlString(condition.getAssigningAuthorityCd()) + "," +
+                getSqlString(condition.getAssigningAuthorityDesc()) + "," +
+                getSqlString(condition.getConditionCdSysCd()) +
+                ");";
+        try {
+            noOfRecordsUpdated= jdbcTemplate.update(sql);
+        } catch (Exception e) {
+            logger.error("Error in upsert for CONDITION table:{}",e.getMessage());
+        }
+        return noOfRecordsUpdated;
+    }
+
+    private int upsertRdbDate(RdbDate rdbDate) {
+        int noOfRecordsUpdated = 0;
+        String sql = "MERGE INTO RDB_DATE AS target " +
+                "USING (select " + rdbDate.getDateKey() + " as id) AS source ON" +
+                "(target.DATE_KEY = source.id)" +
+                "WHEN MATCHED THEN " +
+                "UPDATE " +
+                "SET " +
+                " target.DATE_MM_DD_YYYY = " + getSqlString(rdbDate.getDateMmDdYyyy()) + "," +
+                " target.DAY_OF_WEEK = " + getSqlString(rdbDate.getDayOfWeek()) + "," +
+                " target.DAY_NBR_IN_CLNDR_MON = " + rdbDate.getDayNbrInClndrMon() + "," +
+                " target.DAY_NBR_IN_CLNDR_YR = " + rdbDate.getDayNbrInClndrYr() + "," +
+                " target.WK_NBR_IN_CLNDR_MON = " + rdbDate.getWkNbrInClndrMon() + "," +
+                " target.WK_NBR_IN_CLNDR_YR = " + rdbDate.getWkNbrInClndrYr() + "," +
+                " target.CLNDR_MON_NAME = " + getSqlString(rdbDate.getClndrMonName()) + "," +
+                " target.CLNDR_MON_IN_YR = " + rdbDate.getClndrMonInYr() + "," +
+                " target.CLNDR_QRTR = " + rdbDate.getClndrQrtr() + "," +
+                " target.CLNDR_YR = " + rdbDate.getClndrYr() +
+                " WHEN NOT MATCHED THEN " +
+                "INSERT " +
+                " (DATE_KEY," +
+                " DATE_MM_DD_YYYY," +
+                " DAY_OF_WEEK,DAY_NBR_IN_CLNDR_MON,DAY_NBR_IN_CLNDR_YR,WK_NBR_IN_CLNDR_MON,WK_NBR_IN_CLNDR_YR," +
+                " CLNDR_MON_NAME,CLNDR_MON_IN_YR,CLNDR_QRTR,CLNDR_YR)" +
+                " VALUES ("
+                + rdbDate.getDateKey() + "," +
+                getSqlString(rdbDate.getDateMmDdYyyy()) + "," +
+                getSqlString(rdbDate.getDayOfWeek()) + "," +
+                rdbDate.getDayNbrInClndrMon() + "," +
+                rdbDate.getDayNbrInClndrYr() + "," +
+                rdbDate.getWkNbrInClndrMon() + "," +
+                rdbDate.getWkNbrInClndrYr() + "," +
+                getSqlString(rdbDate.getClndrMonName()) + "," +
+                rdbDate.getClndrMonInYr() + "," +
+                rdbDate.getClndrQrtr() + "," +
+                rdbDate.getClndrYr() +
+                ");";
+
+        try {
+            noOfRecordsUpdated= jdbcTemplate.update(sql);
+        } catch (Exception e) {
+            logger.error("Error in upsert for RDB_DATE table:{}",e.getMessage());
+        }
+        return noOfRecordsUpdated;
+    }
+
+    private static String getSqlString(String val) {
+        if (val != null && !val.isEmpty()) {
+            return "'" + val + "'";
+        }
+        return val;
+    }
+
+    public String getLastUpdatedTime(String tableName) {
+        String sql = "select last_update_time from POLL_DATA_SYNC_CONFIG where table_name=?";
+        String updatedTime = "";
+        Timestamp lastTime = this.jdbcTemplate.queryForObject(
+                sql,
+                Timestamp.class, tableName);
+        if (lastTime != null) {
+            SimpleDateFormat formatter = new SimpleDateFormat(TIMESTAMP_FORMAT);
+            updatedTime = formatter.format(lastTime);
+        }
+        logger.info("getLastUpdatedTime from config. tableName: {} lastUpdatedTime:{}", tableName, lastTime);
+        return updatedTime;
+    }
+
+    public void updateLastUpdatedTime(String tableName, Timestamp timestamp) {
+        String updateSql = "update RDB.dbo.POLL_DATA_SYNC_CONFIG set last_update_time =? where table_name=?;";
+        jdbcTemplate.update(updateSql, timestamp, tableName);
+    }
+
+    public List<PollDataSyncConfig> getTableListFromConfig() {
+        String sql = "select * from RDB.dbo.poll_data_sync_config pdsc order by table_order";
+        List<PollDataSyncConfig> tableList = jdbcTemplate.query(
+                sql,
+                new BeanPropertyRowMapper<>(PollDataSyncConfig.class));
+        logger.info("getTableListFromConfig size:{}", tableList.size());
+        return tableList;
+    }
+
+    public void deleteTable(String tableName) {
+        String deleteSql = "delete " + tableName;
+        jdbcTemplate.execute(deleteSql);
+    }
+}