generated from CDCgov/template
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* rdb model classes * deleted the model classes * Update DataPullService.java * poll service for the rdb * Updated sync Rdb Data * polling completed for three tables. * Renamed the table. * updated config table * Create insert.sql created insert sql * Updated * Updated * Updated for the RDB tables and log4j * Updated * Updated * Updated. This can be a base version for the rdb polling. tested with exchange api. * Added new table entries. * Updated for the rdb poll service * Update DataPullService.java * Updated env variables * Renamed the file name * updated for unit testing. * Update .DS_Store * renamed the file and used the the content from main. * updated for sonarscan * Updated for sonarscan * updated for test cases
- Loading branch information
Showing
28 changed files
with
1,035 additions
and
39 deletions.
There are no files selected for viewing
Binary file not shown.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Binary file not shown.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
12 changes: 8 additions & 4 deletions
12
nnd-data-poll-service/execute_script/nnd_data_poll_service.cmd
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,17 +1,21 @@ | ||
@echo off | ||
set NBS_NND_CRON=* * * * * * | ||
set NBS_NND_CRON=0 */1 * * * * | ||
set NBS_NND_CRON_TIME_ZONE=UTC | ||
set NND_FULL_LOAD=false | ||
set NND_PULL_LIMIT=0 | ||
set NND_INSERT_LIMIT=1000 | ||
set NND_DE_CLIENT_ID=nnd-keycloak-client | ||
set NND_DE_DE=/api/nnd/data-exchange | ||
set NND_DE_SECRET=bpVTppDam4sxXt4hfgm5hZ6Rteponjb9 | ||
set NND_DE_TOKEN=/api/auth/token | ||
@REM set NND_DE_TOKEN=/api/auth/token | ||
@REM set NND_DE_DE=/api/nnd/data-exchange | ||
@REM set NND_DE_GENERIC=/api/data-exchange-generic | ||
set NND_DE_URL=http://localhost:8081 | ||
set NND_FILE_LOCATION=C:\Users\DucNguyen\Desktop\LOG\Poll | ||
set NND_POLL_ENABLED=true | ||
set RDB_POLL_ENABLED=true | ||
set OP_DBPASSWORD=fake.fake.fake.1234 | ||
set OP_DBSERVER=localhost:1433 | ||
set OP_DBUSER=sa | ||
set OP_DBNAME=MSGOUTE | ||
java -jar nnd-data-poll-service.jar | ||
set RDB_DBNAME=RDB | ||
java -jar nnd-data-poll-service.jar |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Binary file not shown.
Binary file not shown.
41 changes: 41 additions & 0 deletions
41
...l-service/src/main/java/gov/cdc/nnddatapollservice/configuration/RdbDataSourceConfig.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,41 @@ | ||
package gov.cdc.nnddatapollservice.configuration; | ||
|
||
import org.springframework.beans.factory.annotation.Qualifier; | ||
import org.springframework.beans.factory.annotation.Value; | ||
import org.springframework.boot.jdbc.DataSourceBuilder; | ||
import org.springframework.context.annotation.Bean; | ||
import org.springframework.context.annotation.Configuration; | ||
import org.springframework.jdbc.core.JdbcTemplate; | ||
|
||
import javax.sql.DataSource; | ||
|
||
@Configuration | ||
public class RdbDataSourceConfig { | ||
@Value("${spring.datasource.driverClassName}") | ||
private String driverClassName; | ||
|
||
@Value("${spring.datasource.rdb.url}") | ||
private String dbUrl; | ||
|
||
@Value("${spring.datasource.username}") | ||
private String dbUserName; | ||
|
||
@Value("${spring.datasource.password}") | ||
private String dbUserPassword; | ||
|
||
@Bean(name = "rdbDataSource") | ||
public DataSource rdbDataSource() { | ||
DataSourceBuilder dataSourceBuilder = DataSourceBuilder.create(); | ||
|
||
dataSourceBuilder.driverClassName(driverClassName); | ||
dataSourceBuilder.url(dbUrl); | ||
dataSourceBuilder.username(dbUserName); | ||
dataSourceBuilder.password(dbUserPassword); | ||
|
||
return dataSourceBuilder.build(); | ||
} | ||
@Bean(name = "rdbJdbcTemplate") | ||
public JdbcTemplate rdbJdbcTemplate(@Qualifier("rdbDataSource") DataSource dataSource) { | ||
return new JdbcTemplate(dataSource); | ||
} | ||
} |
269 changes: 269 additions & 0 deletions
269
...a-poll-service/src/main/java/gov/cdc/nnddatapollservice/rdb/dao/RdbDataPersistentDAO.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,269 @@ | ||
package gov.cdc.nnddatapollservice.rdb.dao; | ||
|
||
import com.google.gson.FieldNamingPolicy; | ||
import com.google.gson.Gson; | ||
import com.google.gson.GsonBuilder; | ||
import com.google.gson.reflect.TypeToken; | ||
import gov.cdc.nnddatapollservice.rdb.dto.Condition; | ||
import gov.cdc.nnddatapollservice.rdb.dto.ConfirmationMethod; | ||
import gov.cdc.nnddatapollservice.rdb.dto.PollDataSyncConfig; | ||
import gov.cdc.nnddatapollservice.rdb.dto.RdbDate; | ||
import lombok.extern.slf4j.Slf4j; | ||
import org.slf4j.Logger; | ||
import org.slf4j.LoggerFactory; | ||
import org.springframework.beans.factory.annotation.Autowired; | ||
import org.springframework.beans.factory.annotation.Qualifier; | ||
import org.springframework.jdbc.core.BeanPropertyRowMapper; | ||
import org.springframework.jdbc.core.JdbcTemplate; | ||
import org.springframework.jdbc.core.namedparam.SqlParameterSourceUtils; | ||
import org.springframework.jdbc.core.simple.SimpleJdbcInsert; | ||
import org.springframework.stereotype.Service; | ||
|
||
import java.lang.reflect.Type; | ||
import java.sql.Timestamp; | ||
import java.text.SimpleDateFormat; | ||
import java.util.List; | ||
import java.util.Map; | ||
|
||
@Service | ||
@Slf4j | ||
public class RdbDataPersistentDAO { | ||
private static Logger logger = LoggerFactory.getLogger(RdbDataPersistentDAO.class); | ||
private JdbcTemplate jdbcTemplate; | ||
private static final String TIMESTAMP_FORMAT = "yyyy-MM-dd HH:mm:ss.SSS"; | ||
|
||
@Autowired | ||
public RdbDataPersistentDAO(@Qualifier("rdbJdbcTemplate") JdbcTemplate jdbcTemplate) { | ||
this.jdbcTemplate = jdbcTemplate; | ||
} | ||
|
||
@SuppressWarnings("java:S3776") | ||
public int saveRDBData(String tableName, String jsonData) { | ||
logger.info("saveRDBData tableName: {}", tableName); | ||
int noOfRecordsSaved = 0; | ||
if ("CONFIRMATION_METHOD".equalsIgnoreCase(tableName)) { | ||
Gson gson = new GsonBuilder() | ||
.setFieldNamingPolicy(FieldNamingPolicy.UPPER_CASE_WITH_UNDERSCORES) | ||
.create(); | ||
Type resultType = new TypeToken<List<ConfirmationMethod>>() { | ||
}.getType(); | ||
List<ConfirmationMethod> list = gson.fromJson(jsonData, resultType); | ||
for (ConfirmationMethod confirmationMethod : list) { | ||
noOfRecordsSaved=upsertConfirmationMethod(confirmationMethod); | ||
} | ||
} else if ("CONDITION".equalsIgnoreCase(tableName)) { | ||
Gson gson = new GsonBuilder() | ||
.setFieldNamingPolicy(FieldNamingPolicy.UPPER_CASE_WITH_UNDERSCORES) | ||
.create(); | ||
Type resultType = new TypeToken<List<Condition>>() { | ||
}.getType(); | ||
List<Condition> list = gson.fromJson(jsonData, resultType); | ||
for (Condition condition : list) { | ||
noOfRecordsSaved=upsertCondition(condition); | ||
} | ||
} else if ("RDB_DATE".equalsIgnoreCase(tableName)) { | ||
Gson gson = new GsonBuilder() | ||
.setFieldNamingPolicy(FieldNamingPolicy.UPPER_CASE_WITH_UNDERSCORES) | ||
.create(); | ||
Type resultType = new TypeToken<List<RdbDate>>() { | ||
}.getType(); | ||
List<RdbDate> list = gson.fromJson(jsonData, resultType); | ||
for (RdbDate rdbDate : list) { | ||
noOfRecordsSaved=upsertRdbDate(rdbDate); | ||
} | ||
} else { | ||
try { | ||
SimpleJdbcInsert simpleJdbcInsert = | ||
new SimpleJdbcInsert(jdbcTemplate); | ||
if(tableName!=null && !tableName.isEmpty()) { | ||
simpleJdbcInsert = simpleJdbcInsert.withTableName(tableName); | ||
List<Map<String, Object>> records = jsonToListOfMap(jsonData); | ||
if (records != null && !records.isEmpty()) { | ||
logger.info("Inside generic code before executeBatch tableName: {} Records size:{}", tableName, records.size()); | ||
int[] noOfInserts =simpleJdbcInsert.executeBatch(SqlParameterSourceUtils.createBatch(records)); | ||
noOfRecordsSaved=noOfInserts.length; | ||
logger.info("executeBatch completed. tableName: {}", tableName); | ||
} else { | ||
logger.info("Inside generic code tableName: {} Records size:0", tableName); | ||
} | ||
} | ||
} catch (Exception e) { | ||
logger.error("Error executeBatch. tableName: {}, Error:{}", tableName, e.getMessage()); | ||
} | ||
} | ||
logger.info("saveRDBData tableName: {} Records size:{}",tableName, noOfRecordsSaved); | ||
return noOfRecordsSaved; | ||
} | ||
|
||
private List<Map<String, Object>> jsonToListOfMap(String jsonData) { | ||
List<Map<String, Object>> list = null; | ||
if (jsonData != null && !jsonData.isEmpty()) { | ||
Gson gson = new GsonBuilder().serializeNulls().create(); | ||
Type resultType = new TypeToken<List<Map<String, Object>>>() { | ||
}.getType(); | ||
list = gson.fromJson(jsonData, resultType); | ||
} | ||
return list; | ||
} | ||
|
||
private int upsertConfirmationMethod(ConfirmationMethod confirmationMethod) { | ||
int noOfRecordsUpdated = 0; | ||
String sql = "MERGE INTO CONFIRMATION_METHOD AS target USING " + | ||
"(select " + confirmationMethod.getConfirmationMethodKey() + " as id) AS source " + | ||
"ON (target.CONFIRMATION_METHOD_KEY = source.id) " + | ||
"WHEN MATCHED THEN " + | ||
" UPDATE SET target.CONFIRMATION_METHOD_CD = " + getSqlString(confirmationMethod.getConfirmationMethodCd()) + | ||
",target.CONFIRMATION_METHOD_DESC = " + getSqlString(confirmationMethod.getConfirmationMethodDesc()) + | ||
" WHEN NOT MATCHED THEN " + | ||
" INSERT (CONFIRMATION_METHOD_KEY, CONFIRMATION_METHOD_CD,CONFIRMATION_METHOD_DESC) " + | ||
"VALUES (" + confirmationMethod.getConfirmationMethodKey() + | ||
"," + getSqlString(confirmationMethod.getConfirmationMethodCd()) + | ||
"," + getSqlString(confirmationMethod.getConfirmationMethodDesc()) + ");"; | ||
try { | ||
noOfRecordsUpdated= jdbcTemplate.update(sql); | ||
} catch (Exception e) { | ||
logger.error("Error in upsert for CONFIRMATION_METHOD table:{}",e.getMessage()); | ||
} | ||
return noOfRecordsUpdated; | ||
} | ||
|
||
private int upsertCondition(Condition condition) { | ||
int noOfRecordsUpdated = 0; | ||
String sql = "MERGE INTO CONDITION AS target " + | ||
"USING (select " + condition.getConditionKey() + " as id) AS source ON" + | ||
"(target.CONDITION_KEY = source.id)" + | ||
"WHEN MATCHED THEN " + | ||
"UPDATE " + | ||
"SET " + | ||
" target.CONDITION_CD = " + getSqlString(condition.getConditionCd()) + "," + | ||
" target.CONDITION_DESC = " + getSqlString(condition.getConditionDesc()) + "," + | ||
" target.CONDITION_SHORT_NM = " + getSqlString(condition.getConditionShortNm()) + "," + | ||
" target.CONDITION_CD_EFF_DT = " + getSqlString(condition.getConditionCdEffDt()) + "," + | ||
" target.CONDITION_CD_END_DT = " + getSqlString(condition.getConditionCdEndDt()) + "," + | ||
" target.NND_IND = " + getSqlString(condition.getNndInd()) + "," + | ||
" target.DISEASE_GRP_CD = " + getSqlString(condition.getDiseaseGrpCd()) + "," + | ||
" target.DISEASE_GRP_DESC = " + getSqlString(condition.getDiseaseGrpDesc()) + "," + | ||
" target.PROGRAM_AREA_CD = " + getSqlString(condition.getProgramAreaCd()) + "," + | ||
" target.PROGRAM_AREA_DESC = " + getSqlString(condition.getProgramAreaDesc()) + "," + | ||
" target.CONDITION_CD_SYS_CD_NM = " + getSqlString(condition.getConditionCdSysCdNm()) + "," + | ||
" target.ASSIGNING_AUTHORITY_CD = " + getSqlString(condition.getAssigningAuthorityCd()) + "," + | ||
" target.ASSIGNING_AUTHORITY_DESC = " + getSqlString(condition.getAssigningAuthorityDesc()) + "," + | ||
" target.CONDITION_CD_SYS_CD = " + getSqlString(condition.getConditionCdSysCd()) + | ||
" WHEN NOT MATCHED THEN " + | ||
"INSERT " + | ||
" (CONDITION_KEY," + | ||
" CONDITION_CD," + | ||
" CONDITION_DESC,CONDITION_SHORT_NM,CONDITION_CD_EFF_DT,CONDITION_CD_END_DT,NND_IND," + | ||
" DISEASE_GRP_CD,DISEASE_GRP_DESC,PROGRAM_AREA_CD,PROGRAM_AREA_DESC," + | ||
" CONDITION_CD_SYS_CD_NM,ASSIGNING_AUTHORITY_CD,ASSIGNING_AUTHORITY_DESC,CONDITION_CD_SYS_CD)" + | ||
"VALUES (" | ||
+ condition.getConditionKey() + "," + | ||
getSqlString(condition.getConditionCd()) + "," + | ||
getSqlString(condition.getConditionDesc()) + "," + | ||
getSqlString(condition.getConditionShortNm()) + "," + | ||
getSqlString(condition.getConditionCdEffDt()) + "," + | ||
getSqlString(condition.getConditionCdEndDt()) + "," + | ||
getSqlString(condition.getNndInd()) + "," + | ||
getSqlString(condition.getDiseaseGrpCd()) + "," + | ||
getSqlString(condition.getDiseaseGrpDesc()) + "," + | ||
getSqlString(condition.getProgramAreaCd()) + "," + | ||
getSqlString(condition.getProgramAreaDesc()) + "," + | ||
getSqlString(condition.getConditionCdSysCdNm()) + "," + | ||
getSqlString(condition.getAssigningAuthorityCd()) + "," + | ||
getSqlString(condition.getAssigningAuthorityDesc()) + "," + | ||
getSqlString(condition.getConditionCdSysCd()) + | ||
");"; | ||
try { | ||
noOfRecordsUpdated= jdbcTemplate.update(sql); | ||
} catch (Exception e) { | ||
logger.error("Error in upsert for CONDITION table:{}",e.getMessage()); | ||
} | ||
return noOfRecordsUpdated; | ||
} | ||
|
||
private int upsertRdbDate(RdbDate rdbDate) { | ||
int noOfRecordsUpdated = 0; | ||
String sql = "MERGE INTO RDB_DATE AS target " + | ||
"USING (select " + rdbDate.getDateKey() + " as id) AS source ON" + | ||
"(target.DATE_KEY = source.id)" + | ||
"WHEN MATCHED THEN " + | ||
"UPDATE " + | ||
"SET " + | ||
" target.DATE_MM_DD_YYYY = " + getSqlString(rdbDate.getDateMmDdYyyy()) + "," + | ||
" target.DAY_OF_WEEK = " + getSqlString(rdbDate.getDayOfWeek()) + "," + | ||
" target.DAY_NBR_IN_CLNDR_MON = " + rdbDate.getDayNbrInClndrMon() + "," + | ||
" target.DAY_NBR_IN_CLNDR_YR = " + rdbDate.getDayNbrInClndrYr() + "," + | ||
" target.WK_NBR_IN_CLNDR_MON = " + rdbDate.getWkNbrInClndrMon() + "," + | ||
" target.WK_NBR_IN_CLNDR_YR = " + rdbDate.getWkNbrInClndrYr() + "," + | ||
" target.CLNDR_MON_NAME = " + getSqlString(rdbDate.getClndrMonName()) + "," + | ||
" target.CLNDR_MON_IN_YR = " + rdbDate.getClndrMonInYr() + "," + | ||
" target.CLNDR_QRTR = " + rdbDate.getClndrQrtr() + "," + | ||
" target.CLNDR_YR = " + rdbDate.getClndrYr() + | ||
" WHEN NOT MATCHED THEN " + | ||
"INSERT " + | ||
" (DATE_KEY," + | ||
" DATE_MM_DD_YYYY," + | ||
" DAY_OF_WEEK,DAY_NBR_IN_CLNDR_MON,DAY_NBR_IN_CLNDR_YR,WK_NBR_IN_CLNDR_MON,WK_NBR_IN_CLNDR_YR," + | ||
" CLNDR_MON_NAME,CLNDR_MON_IN_YR,CLNDR_QRTR,CLNDR_YR)" + | ||
" VALUES (" | ||
+ rdbDate.getDateKey() + "," + | ||
getSqlString(rdbDate.getDateMmDdYyyy()) + "," + | ||
getSqlString(rdbDate.getDayOfWeek()) + "," + | ||
rdbDate.getDayNbrInClndrMon() + "," + | ||
rdbDate.getDayNbrInClndrYr() + "," + | ||
rdbDate.getWkNbrInClndrMon() + "," + | ||
rdbDate.getWkNbrInClndrYr() + "," + | ||
getSqlString(rdbDate.getClndrMonName()) + "," + | ||
rdbDate.getClndrMonInYr() + "," + | ||
rdbDate.getClndrQrtr() + "," + | ||
rdbDate.getClndrYr() + | ||
");"; | ||
|
||
try { | ||
noOfRecordsUpdated= jdbcTemplate.update(sql); | ||
} catch (Exception e) { | ||
logger.error("Error in upsert for RDB_DATE table:{}",e.getMessage()); | ||
} | ||
return noOfRecordsUpdated; | ||
} | ||
|
||
private static String getSqlString(String val) { | ||
if (val != null && !val.isEmpty()) { | ||
return "'" + val + "'"; | ||
} | ||
return val; | ||
} | ||
|
||
public String getLastUpdatedTime(String tableName) { | ||
String sql = "select last_update_time from POLL_DATA_SYNC_CONFIG where table_name=?"; | ||
String updatedTime = ""; | ||
Timestamp lastTime = this.jdbcTemplate.queryForObject( | ||
sql, | ||
Timestamp.class, tableName); | ||
if (lastTime != null) { | ||
SimpleDateFormat formatter = new SimpleDateFormat(TIMESTAMP_FORMAT); | ||
updatedTime = formatter.format(lastTime); | ||
} | ||
logger.info("getLastUpdatedTime from config. tableName: {} lastUpdatedTime:{}", tableName, lastTime); | ||
return updatedTime; | ||
} | ||
|
||
public void updateLastUpdatedTime(String tableName, Timestamp timestamp) { | ||
String updateSql = "update RDB.dbo.POLL_DATA_SYNC_CONFIG set last_update_time =? where table_name=?;"; | ||
jdbcTemplate.update(updateSql, timestamp, tableName); | ||
} | ||
|
||
public List<PollDataSyncConfig> getTableListFromConfig() { | ||
String sql = "select * from RDB.dbo.poll_data_sync_config pdsc order by table_order"; | ||
List<PollDataSyncConfig> tableList = jdbcTemplate.query( | ||
sql, | ||
new BeanPropertyRowMapper<>(PollDataSyncConfig.class)); | ||
logger.info("getTableListFromConfig size:{}", tableList.size()); | ||
return tableList; | ||
} | ||
|
||
public void deleteTable(String tableName) { | ||
String deleteSql = "delete " + tableName; | ||
jdbcTemplate.execute(deleteSql); | ||
} | ||
} |
Oops, something went wrong.