From f25cd3f37b32bf7e432c8d2bd97856b95bb5f47f Mon Sep 17 00:00:00 2001 From: Duc Nguyen <117302958+ndduc01@users.noreply.github.com> Date: Wed, 28 Aug 2024 11:59:37 -0700 Subject: [PATCH] query for notification (#19) * query for notification * ignore list * swapping to parent build * update * update * added support for ODSE DB * moved compress and decode and encode to share class * added compressing and encoding for nnd * added ignore * clean up * clean up * clean up * update sonar scan * support for compressed data in poll servie * update POll service * update * update * update * clean up * update adding null query and support for srte --- .github/workflows/sonar.yaml | 2 +- build.gradle | 10 ++ .../NndMessageProcessorApplication.java | 1 - .../service/MessageProcessingService.java | 2 - .../service/NetssCaseService.java | 6 +- .../service/MessageProcessingServiceTest.java | 2 + .../configuration/OdseDataSourceConfig.java | 6 + .../configuration/SrteDataSourceConfig.java | 86 ++++++++++ .../constant/DataSyncConstant.java | 10 ++ .../controller/DataExchangeController.java | 25 ++- .../msg/NETSSTransportQOutRepository.java | 4 +- .../msg/TransportQOutRepository.java | 4 +- .../odse/CNTransportQOutRepository.java | 4 +- .../repository/rdb/model/DataSyncConfig.java | 2 + .../srte/SrteDataSyncRepository.java | 4 + .../service/DataExchangeGenericService.java | 77 ++++----- .../service/DataExchangeService.java | 22 ++- .../IDataExchangeGenericService.java | 4 +- .../interfaces/IDataExchangeService.java | 7 +- .../shared/DataSimplification.java | 49 ++++++ .../resources/query/create/data_config.sql | 1 + .../main/resources/query/select/insert.sql | 148 +++++++++++++----- .../main/resources/query/select/select.sql | 9 +- .../DataExchangeControllerTest.java | 44 ++++-- .../DataExchangeGenericServiceTest.java | 37 ++++- .../service/DataExchangeServiceTest.java | 42 ++++- .../execute_script/nnd_data_poll_service.cmd | 3 + .../execute_script/nnd_data_poll_service.sh | 3 + .../service/CNTransportQOutService.java | 10 +- .../service/DataHandlingService.java | 28 ++-- .../service/NetsstTransportService.java | 8 +- .../service/TransportQOutService.java | 8 +- .../share/DataSimplification.java | 40 +++++ .../src/main/resources/application.yaml | 3 + .../service/DataHandlingServiceTest.java | 101 ++++++++++-- .../service/NetsstTransportServiceTest.java | 2 +- .../service/TransportQOutServiceTest.java | 2 +- .../share/DataSimplificationTest.java | 46 ++++++ 38 files changed, 694 insertions(+), 168 deletions(-) create mode 100644 nnd-data-exchange-service/src/main/java/gov/cdc/nnddataexchangeservice/configuration/SrteDataSourceConfig.java create mode 100644 nnd-data-exchange-service/src/main/java/gov/cdc/nnddataexchangeservice/constant/DataSyncConstant.java create mode 100644 nnd-data-exchange-service/src/main/java/gov/cdc/nnddataexchangeservice/repository/srte/SrteDataSyncRepository.java create mode 100644 nnd-data-exchange-service/src/main/java/gov/cdc/nnddataexchangeservice/shared/DataSimplification.java create mode 100644 nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/share/DataSimplification.java create mode 100644 nnd-data-poll-service/src/test/java/gov/cdc/nnddatapollservice/share/DataSimplificationTest.java diff --git a/.github/workflows/sonar.yaml b/.github/workflows/sonar.yaml index ca613e0..3bc950b 100644 --- a/.github/workflows/sonar.yaml +++ b/.github/workflows/sonar.yaml @@ -15,7 +15,7 @@ on: paths: - "nnd-data-exchange-service/**" - "nnd-data-poll-service/**" - - "nnd-message-processor/**" + - "netss-message-processor/**" - ".github/workflows/sonar.yaml" env: deployment_env: dev diff --git a/build.gradle b/build.gradle index c312cbe..efddba3 100644 --- a/build.gradle +++ b/build.gradle @@ -47,5 +47,15 @@ sonarqube { property "sonar.projectKey", "CDCgov_NEDSS-NNDSS" property "sonar.organization", "cdcgov" property "sonar.host.url", "https://sonarcloud.io" + property "sonar.exclusions", "**/configuration/**," + + "**/exception/**, " + + "**/security/**," + + "**/repository/**, " + + "**/constant/**, " + + "**/service/model/** ," + + "**/json_config/**, " + + "**/NndMessageProcessorApplication.java, " + + "**/NndDataExchangeServiceApplication.java, " + + "**/NndDataPollServiceApplication.java" } } \ No newline at end of file diff --git a/netss-message-processor/src/main/java/gov/cdc/nndmessageprocessor/NndMessageProcessorApplication.java b/netss-message-processor/src/main/java/gov/cdc/nndmessageprocessor/NndMessageProcessorApplication.java index fbba114..ae57d23 100644 --- a/netss-message-processor/src/main/java/gov/cdc/nndmessageprocessor/NndMessageProcessorApplication.java +++ b/netss-message-processor/src/main/java/gov/cdc/nndmessageprocessor/NndMessageProcessorApplication.java @@ -2,7 +2,6 @@ import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.scheduling.annotation.EnableScheduling; @SpringBootApplication //@EnableScheduling diff --git a/netss-message-processor/src/main/java/gov/cdc/nndmessageprocessor/service/MessageProcessingService.java b/netss-message-processor/src/main/java/gov/cdc/nndmessageprocessor/service/MessageProcessingService.java index 1209ed6..9bca88c 100644 --- a/netss-message-processor/src/main/java/gov/cdc/nndmessageprocessor/service/MessageProcessingService.java +++ b/netss-message-processor/src/main/java/gov/cdc/nndmessageprocessor/service/MessageProcessingService.java @@ -6,12 +6,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; -import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import java.text.ParseException; import java.text.SimpleDateFormat; -import java.time.ZoneId; import java.util.Calendar; import java.util.Date; import java.util.HashMap; diff --git a/netss-message-processor/src/main/java/gov/cdc/nndmessageprocessor/service/NetssCaseService.java b/netss-message-processor/src/main/java/gov/cdc/nndmessageprocessor/service/NetssCaseService.java index b25a370..4631a3a 100644 --- a/netss-message-processor/src/main/java/gov/cdc/nndmessageprocessor/service/NetssCaseService.java +++ b/netss-message-processor/src/main/java/gov/cdc/nndmessageprocessor/service/NetssCaseService.java @@ -13,8 +13,10 @@ import java.io.FileWriter; import java.io.IOException; import java.io.PrintWriter; -import java.util.*; -import java.util.stream.Collectors; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; @Service public class NetssCaseService implements INetssCaseService { diff --git a/netss-message-processor/src/test/java/gov/cdc/nndmessageprocessor/service/MessageProcessingServiceTest.java b/netss-message-processor/src/test/java/gov/cdc/nndmessageprocessor/service/MessageProcessingServiceTest.java index 8536795..4fc5ea8 100644 --- a/netss-message-processor/src/test/java/gov/cdc/nndmessageprocessor/service/MessageProcessingServiceTest.java +++ b/netss-message-processor/src/test/java/gov/cdc/nndmessageprocessor/service/MessageProcessingServiceTest.java @@ -34,6 +34,8 @@ void setUp() { ReflectionTestUtils.setField(messageProcessingService, "prior", true); } + + @Test void testScheduleDataFetch_Success() throws DataProcessorException { messageProcessingService.scheduleDataFetch(); diff --git a/nnd-data-exchange-service/src/main/java/gov/cdc/nnddataexchangeservice/configuration/OdseDataSourceConfig.java b/nnd-data-exchange-service/src/main/java/gov/cdc/nnddataexchangeservice/configuration/OdseDataSourceConfig.java index dfd9efd..2cb9299 100644 --- a/nnd-data-exchange-service/src/main/java/gov/cdc/nnddataexchangeservice/configuration/OdseDataSourceConfig.java +++ b/nnd-data-exchange-service/src/main/java/gov/cdc/nnddataexchangeservice/configuration/OdseDataSourceConfig.java @@ -9,6 +9,7 @@ import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Primary; import org.springframework.data.jpa.repository.config.EnableJpaRepositories; +import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.orm.jpa.JpaTransactionManager; import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean; import org.springframework.orm.jpa.vendor.HibernateJpaVendorAdapter; @@ -75,4 +76,9 @@ public PlatformTransactionManager odseTransactionManager( @Qualifier("odseEntityManagerFactory") EntityManagerFactory entityManagerFactory) { return new JpaTransactionManager(entityManagerFactory); } + + @Bean(name = "odseJdbcTemplate") + public JdbcTemplate odseJdbcTemplate(@Qualifier("odseDataSource") DataSource dataSource) { + return new JdbcTemplate(dataSource); + } } diff --git a/nnd-data-exchange-service/src/main/java/gov/cdc/nnddataexchangeservice/configuration/SrteDataSourceConfig.java b/nnd-data-exchange-service/src/main/java/gov/cdc/nnddataexchangeservice/configuration/SrteDataSourceConfig.java new file mode 100644 index 0000000..f132454 --- /dev/null +++ b/nnd-data-exchange-service/src/main/java/gov/cdc/nnddataexchangeservice/configuration/SrteDataSourceConfig.java @@ -0,0 +1,86 @@ +package gov.cdc.nnddataexchangeservice.configuration; + +import jakarta.persistence.EntityManagerFactory; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.jdbc.DataSourceBuilder; +import org.springframework.boot.orm.jpa.EntityManagerFactoryBuilder; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; +import org.springframework.data.jpa.repository.config.EnableJpaRepositories; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.orm.jpa.JpaTransactionManager; +import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean; +import org.springframework.orm.jpa.vendor.HibernateJpaVendorAdapter; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.annotation.EnableTransactionManagement; + +import javax.sql.DataSource; +import java.util.HashMap; + + +@Configuration +@EnableTransactionManagement +@EnableJpaRepositories( + entityManagerFactoryRef = "srteEntityManagerFactory", + transactionManagerRef = "srteTransactionManager", + basePackages = { + "gov.cdc.nnddataexchangeservice.repository.srte" + } +) +public class SrteDataSourceConfig { + @Value("${spring.datasource.driverClassName}") + private String driverClassName; + + @Value("${spring.datasource.srte.url}") + private String dbUrl; + + @Value("${spring.datasource.username}") + private String dbUserName; + + @Value("${spring.datasource.password}") + private String dbUserPassword; + + @Bean(name = "srteDataSource") + public DataSource srteDataSource() { + DataSourceBuilder dataSourceBuilder = DataSourceBuilder.create(); + + dataSourceBuilder.driverClassName(driverClassName); + dataSourceBuilder.url(dbUrl); + dataSourceBuilder.username(dbUserName); + dataSourceBuilder.password(dbUserPassword); + + return dataSourceBuilder.build(); + } + + @Bean(name = "srteEntityManagerFactoryBuilder") + public EntityManagerFactoryBuilder srteEntityManagerFactoryBuilder() { + return new EntityManagerFactoryBuilder(new HibernateJpaVendorAdapter(), new HashMap<>(), null); + } + + @Primary + @Bean(name = "srteEntityManagerFactory") + public LocalContainerEntityManagerFactoryBean srteEntityManagerFactory( + @Qualifier("srteEntityManagerFactoryBuilder") EntityManagerFactoryBuilder builder, + @Qualifier("srteDataSource") DataSource dataSource) { + return builder + .dataSource(dataSource) + .packages("gov.cdc.nnddataexchangeservice.repository.srte.model") + .persistenceUnit("srte") + .build(); + } + + @Primary + @Bean(name = "srteTransactionManager") + public PlatformTransactionManager srteTransactionManager( + @Qualifier("srteEntityManagerFactory") EntityManagerFactory entityManagerFactory) { + return new JpaTransactionManager(entityManagerFactory); + } + + + @Bean(name = "srteJdbcTemplate") + public JdbcTemplate srteJdbcTemplate(@Qualifier("srteDataSource") DataSource dataSource) { + return new JdbcTemplate(dataSource); + } +} diff --git a/nnd-data-exchange-service/src/main/java/gov/cdc/nnddataexchangeservice/constant/DataSyncConstant.java b/nnd-data-exchange-service/src/main/java/gov/cdc/nnddataexchangeservice/constant/DataSyncConstant.java new file mode 100644 index 0000000..763fdaf --- /dev/null +++ b/nnd-data-exchange-service/src/main/java/gov/cdc/nnddataexchangeservice/constant/DataSyncConstant.java @@ -0,0 +1,10 @@ +package gov.cdc.nnddataexchangeservice.constant; + +public class DataSyncConstant { + public static final String DEFAULT_TIME_STAMP = "1753-01-01"; + public static final String TIME_STAMP_PARAM = ":timestamp"; + public static final String LIMIT_PARAM = ":limit"; + public static final Integer BYTE_SIZE = 1024; + public static final String DB_RDB = "RDB"; + public static final String DB_SRTE = "SRTE"; +} diff --git a/nnd-data-exchange-service/src/main/java/gov/cdc/nnddataexchangeservice/controller/DataExchangeController.java b/nnd-data-exchange-service/src/main/java/gov/cdc/nnddataexchangeservice/controller/DataExchangeController.java index 7cdfe0e..2e9f1f2 100644 --- a/nnd-data-exchange-service/src/main/java/gov/cdc/nnddataexchangeservice/controller/DataExchangeController.java +++ b/nnd-data-exchange-service/src/main/java/gov/cdc/nnddataexchangeservice/controller/DataExchangeController.java @@ -4,7 +4,6 @@ import gov.cdc.nnddataexchangeservice.exception.DataExchangeException; import gov.cdc.nnddataexchangeservice.service.interfaces.IDataExchangeGenericService; import gov.cdc.nnddataexchangeservice.service.interfaces.IDataExchangeService; -import gov.cdc.nnddataexchangeservice.service.model.DataExchangeModel; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.enums.ParameterIn; @@ -14,6 +13,8 @@ import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.*; +import java.io.IOException; + @RestController @SecurityRequirement(name = "bearer-key") public class DataExchangeController { @@ -43,17 +44,24 @@ public DataExchangeController(IDataExchangeService dataExchangeService, schema = @Schema(type = "string"))} ) @GetMapping(path = "/api/nnd/data-exchange") - public ResponseEntity exchangingData(@RequestParam("cnStatusTime") String cnStatusTime, + public ResponseEntity exchangingData(@RequestParam("cnStatusTime") String cnStatusTime, @RequestParam("transportStatusTime") String transportStatusTime, @RequestParam("netssTime") String netssTime, @RequestParam("statusCd") String statusCd, - @RequestHeader(name = "limit", defaultValue = "0") String limit) throws DataExchangeException { + @RequestHeader(name = "limit", defaultValue = "0", required = false) String limit, + @RequestHeader(name = "compress", defaultValue = "false", required = false) String compress) throws DataExchangeException, IOException { if (statusCd.isEmpty()) { throw new DataExchangeException("Status Code is Missing"); } + + boolean compressCheck = false; + if (compress.equalsIgnoreCase("true") ) { + compressCheck = true; + } + int intLimit = Integer.parseInt(limit); - return ResponseEntity.ok(dataExchangeService.getDataForOnPremExchanging(cnStatusTime, transportStatusTime,netssTime, statusCd, intLimit)); + return ResponseEntity.ok(dataExchangeService.getDataForOnPremExchanging(cnStatusTime, transportStatusTime,netssTime, statusCd, intLimit, compressCheck)); } @Operation( @@ -73,14 +81,17 @@ public ResponseEntity exchangingData(@RequestParam("cnStatusT ) @GetMapping(path = "/api/data-exchange-generic/{tableName}") public ResponseEntity exchangingData(@PathVariable String tableName, @RequestParam(required = false) String timestamp, - @RequestHeader(name = "limit", defaultValue = "0") String limit) throws DataExchangeException { + @RequestHeader(name = "limit", defaultValue = "0") String limit, + @RequestHeader(name = "null_allow", defaultValue = "false", required = false) String nulLAllow) throws DataExchangeException { int intLimit = Integer.parseInt(limit); - var base64CompressedData = dataExchangeGenericService.getGenericDataExchange(tableName, timestamp, intLimit); + + boolean nullApplied = nulLAllow.equalsIgnoreCase("true"); + var base64CompressedData = dataExchangeGenericService.getGenericDataExchange(tableName, timestamp, intLimit, nullApplied); return new ResponseEntity<>(base64CompressedData, HttpStatus.OK); } @PostMapping(path = "/api/data-exchange-generic") - public ResponseEntity decodeAndDecompress(@RequestBody String tableName) { + public ResponseEntity decodeAndDecompress(@RequestBody String tableName) throws DataExchangeException { var val = dataExchangeGenericService.decodeAndDecompress(tableName); return new ResponseEntity<>(val, HttpStatus.OK); } diff --git a/nnd-data-exchange-service/src/main/java/gov/cdc/nnddataexchangeservice/repository/msg/NETSSTransportQOutRepository.java b/nnd-data-exchange-service/src/main/java/gov/cdc/nnddataexchangeservice/repository/msg/NETSSTransportQOutRepository.java index 0e56559..7ae3096 100644 --- a/nnd-data-exchange-service/src/main/java/gov/cdc/nnddataexchangeservice/repository/msg/NETSSTransportQOutRepository.java +++ b/nnd-data-exchange-service/src/main/java/gov/cdc/nnddataexchangeservice/repository/msg/NETSSTransportQOutRepository.java @@ -12,10 +12,10 @@ @Repository public interface NETSSTransportQOutRepository extends JpaRepository { - @Query("SELECT a FROM NETSSTransportQOut a") + @Query("SELECT a FROM NETSSTransportQOut a ORDER BY a.addTime ASC") Optional> findNetssTransport (); - @Query("SELECT a FROM NETSSTransportQOut a WHERE a.addTime > :recordStatusTime") + @Query("SELECT a FROM NETSSTransportQOut a WHERE a.addTime > :recordStatusTime ORDER BY a.addTime ASC") Optional> findNetssTransportByCreationTime (@Param("recordStatusTime") Timestamp recordStatusTime); @Query(value = "SELECT TOP (:limit) a.* FROM NETSS_TransportQ_out a ORDER BY a.add_time ASC ", nativeQuery = true) diff --git a/nnd-data-exchange-service/src/main/java/gov/cdc/nnddataexchangeservice/repository/msg/TransportQOutRepository.java b/nnd-data-exchange-service/src/main/java/gov/cdc/nnddataexchangeservice/repository/msg/TransportQOutRepository.java index 565b8e4..077844f 100644 --- a/nnd-data-exchange-service/src/main/java/gov/cdc/nnddataexchangeservice/repository/msg/TransportQOutRepository.java +++ b/nnd-data-exchange-service/src/main/java/gov/cdc/nnddataexchangeservice/repository/msg/TransportQOutRepository.java @@ -12,10 +12,10 @@ @Repository public interface TransportQOutRepository extends JpaRepository { - @Query("SELECT a FROM TransportQOut a WHERE a.messageCreationTime > :recordStatusTime") + @Query("SELECT a FROM TransportQOut a WHERE a.messageCreationTime > :recordStatusTime ORDER BY a.messageCreationTime ASC") Optional> findTransportByCreationTime(@Param("recordStatusTime") String recordStatusTime); - @Query("SELECT a FROM TransportQOut a") + @Query("SELECT a FROM TransportQOut a ORDER BY a.messageCreationTime ASC") Optional> findTransportByWithoutCreationTime(); diff --git a/nnd-data-exchange-service/src/main/java/gov/cdc/nnddataexchangeservice/repository/odse/CNTransportQOutRepository.java b/nnd-data-exchange-service/src/main/java/gov/cdc/nnddataexchangeservice/repository/odse/CNTransportQOutRepository.java index e2f64a9..f3cbf47 100644 --- a/nnd-data-exchange-service/src/main/java/gov/cdc/nnddataexchangeservice/repository/odse/CNTransportQOutRepository.java +++ b/nnd-data-exchange-service/src/main/java/gov/cdc/nnddataexchangeservice/repository/odse/CNTransportQOutRepository.java @@ -12,10 +12,10 @@ @Repository public interface CNTransportQOutRepository extends JpaRepository { - @Query("SELECT a FROM CNTransportQOut a WHERE a.recordStatusCd = :statusCd") + @Query("SELECT a FROM CNTransportQOut a WHERE a.recordStatusCd = :statusCd ORDER BY a.recordStatusTime ASC") Optional> findTransportByStatusCd (@Param("statusCd") String statusCd); - @Query("SELECT a FROM CNTransportQOut a WHERE a.recordStatusTime > :recordStatusTime AND a.recordStatusCd = :recordStatusCd") + @Query("SELECT a FROM CNTransportQOut a WHERE a.recordStatusTime > :recordStatusTime AND a.recordStatusCd = :recordStatusCd ORDER BY a.recordStatusTime ASC") Optional> findTransportByCreationTimeAndStatus (@Param("recordStatusTime") Timestamp recordStatusTime, @Param("recordStatusCd") String recordStatusCd); diff --git a/nnd-data-exchange-service/src/main/java/gov/cdc/nnddataexchangeservice/repository/rdb/model/DataSyncConfig.java b/nnd-data-exchange-service/src/main/java/gov/cdc/nnddataexchangeservice/repository/rdb/model/DataSyncConfig.java index 9d7aa9a..3cdac08 100644 --- a/nnd-data-exchange-service/src/main/java/gov/cdc/nnddataexchangeservice/repository/rdb/model/DataSyncConfig.java +++ b/nnd-data-exchange-service/src/main/java/gov/cdc/nnddataexchangeservice/repository/rdb/model/DataSyncConfig.java @@ -31,6 +31,8 @@ public class DataSyncConfig { @Column(name = "query_with_limit") private String queryWithLimit; + @Column(name = "query_with_null_timestamp") + private String queryWithNullTimeStamp; @Column(name = "created_at") private Timestamp createdAt; diff --git a/nnd-data-exchange-service/src/main/java/gov/cdc/nnddataexchangeservice/repository/srte/SrteDataSyncRepository.java b/nnd-data-exchange-service/src/main/java/gov/cdc/nnddataexchangeservice/repository/srte/SrteDataSyncRepository.java new file mode 100644 index 0000000..8664a45 --- /dev/null +++ b/nnd-data-exchange-service/src/main/java/gov/cdc/nnddataexchangeservice/repository/srte/SrteDataSyncRepository.java @@ -0,0 +1,4 @@ +package gov.cdc.nnddataexchangeservice.repository.srte; + +public interface SrteDataSyncRepository { +} diff --git a/nnd-data-exchange-service/src/main/java/gov/cdc/nnddataexchangeservice/service/DataExchangeGenericService.java b/nnd-data-exchange-service/src/main/java/gov/cdc/nnddataexchangeservice/service/DataExchangeGenericService.java index 2d17a71..13f8fce 100644 --- a/nnd-data-exchange-service/src/main/java/gov/cdc/nnddataexchangeservice/service/DataExchangeGenericService.java +++ b/nnd-data-exchange-service/src/main/java/gov/cdc/nnddataexchangeservice/service/DataExchangeGenericService.java @@ -6,30 +6,31 @@ import gov.cdc.nnddataexchangeservice.exception.DataExchangeException; import gov.cdc.nnddataexchangeservice.repository.rdb.DataSyncConfigRepository; import gov.cdc.nnddataexchangeservice.service.interfaces.IDataExchangeGenericService; +import gov.cdc.nnddataexchangeservice.shared.DataSimplification; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Service; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.sql.Timestamp; -import java.util.Base64; import java.util.List; import java.util.Map; -import java.util.zip.GZIPInputStream; -import java.util.zip.GZIPOutputStream; + +import static gov.cdc.nnddataexchangeservice.constant.DataSyncConstant.*; @Service public class DataExchangeGenericService implements IDataExchangeGenericService { private final DataSyncConfigRepository dataSyncConfigRepository; private final JdbcTemplate jdbcTemplate; + private final JdbcTemplate srteJdbcTemplate; private final Gson gson; public DataExchangeGenericService(DataSyncConfigRepository dataSyncConfigRepository, - @Qualifier("rdbJdbcTemplate") JdbcTemplate jdbcTemplate) { + @Qualifier("rdbJdbcTemplate") JdbcTemplate jdbcTemplate, + @Qualifier("srteJdbcTemplate") JdbcTemplate srteJdbcTemplate) { this.dataSyncConfigRepository = dataSyncConfigRepository; this.jdbcTemplate = jdbcTemplate; + this.srteJdbcTemplate = srteJdbcTemplate; this.gson = new GsonBuilder() .registerTypeAdapter(Timestamp.class, TimestampAdapter.getTimestampSerializer()) @@ -39,7 +40,7 @@ public DataExchangeGenericService(DataSyncConfigRepository dataSyncConfigReposit } @SuppressWarnings("javasecurity:S3649") - public String getGenericDataExchange(String tableName, String timeStamp, Integer limit) throws DataExchangeException { + public String getGenericDataExchange(String tableName, String timeStamp, Integer limit, boolean nullAllow) throws DataExchangeException { // Retrieve configuration based on table name var dataConfig = dataSyncConfigRepository.findById(tableName).orElseThrow(() -> new DataExchangeException("Selected Table Not Found")); @@ -48,33 +49,36 @@ public String getGenericDataExchange(String tableName, String timeStamp, Integer } try { // Execute the query and retrieve the dataset - String baseQuery = (limit > 0 && dataConfig.getQueryWithLimit() != null && !dataConfig.getQueryWithLimit().isEmpty()) - ? dataConfig.getQueryWithLimit() - : dataConfig.getQuery(); + String baseQuery = ""; + + if (nullAllow && dataConfig.getQueryWithNullTimeStamp() != null && !dataConfig.getQueryWithNullTimeStamp().isEmpty()) { + baseQuery = dataConfig.getQueryWithNullTimeStamp(); + } else { + baseQuery = (limit > 0 && dataConfig.getQueryWithLimit() != null && !dataConfig.getQueryWithLimit().isEmpty()) + ? dataConfig.getQueryWithLimit() + : dataConfig.getQuery(); + } + + - String effectiveTimestamp = timeStamp.isEmpty() ? "'1753-01-01'" : "'" + timeStamp + "'"; - String query = baseQuery.replace(":timestamp", effectiveTimestamp); + String effectiveTimestamp = timeStamp.isEmpty() ? "'" + DEFAULT_TIME_STAMP +"'" : "'" + timeStamp + "'"; + String query = baseQuery.replace(TIME_STAMP_PARAM, effectiveTimestamp); - if (baseQuery.contains(":limit")) { - query = query.replace(":limit", limit.toString()); + if (baseQuery.contains(LIMIT_PARAM)) { + query = query.replace(LIMIT_PARAM, limit.toString()); } + List> data; - List> data = jdbcTemplate.queryForList(query); + if (dataConfig.getSourceDb().equalsIgnoreCase(DB_SRTE)) { + data = srteJdbcTemplate.queryForList(query); + } else { + data = jdbcTemplate.queryForList(query); + } // Serialize the data to JSON using Gson String jsonData = gson.toJson(data); - - // Compress the JSON data using GZIP and return the Base64 encoded result - try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); - GZIPOutputStream gzipOutputStream = new GZIPOutputStream(byteArrayOutputStream)) { - - gzipOutputStream.write(jsonData.getBytes()); - gzipOutputStream.finish(); - - byte[] compressedData = byteArrayOutputStream.toByteArray(); - return Base64.getEncoder().encodeToString(compressedData); - } + return DataSimplification.dataCompressionAndEncode(jsonData); } catch (IOException e) { // Catch IOException and throw DataExchangeException @@ -83,24 +87,7 @@ public String getGenericDataExchange(String tableName, String timeStamp, Integer } // DECODE TEST METHOD - public String decodeAndDecompress(String base64EncodedData) { - byte[] compressedData = Base64.getDecoder().decode(base64EncodedData); - - try (ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(compressedData); - GZIPInputStream gzipInputStream = new GZIPInputStream(byteArrayInputStream); - ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) { - - byte[] buffer = new byte[1024]; - int len; - while ((len = gzipInputStream.read(buffer)) > 0) { - byteArrayOutputStream.write(buffer, 0, len); - } - - String decompressedJson = byteArrayOutputStream.toString("UTF-8"); - - return decompressedJson; - } catch (IOException e) { - throw new RuntimeException(e); - } + public String decodeAndDecompress(String base64EncodedData) throws DataExchangeException { + return DataSimplification.decodeAndDecompress(base64EncodedData); } } diff --git a/nnd-data-exchange-service/src/main/java/gov/cdc/nnddataexchangeservice/service/DataExchangeService.java b/nnd-data-exchange-service/src/main/java/gov/cdc/nnddataexchangeservice/service/DataExchangeService.java index 3891479..07134d1 100644 --- a/nnd-data-exchange-service/src/main/java/gov/cdc/nnddataexchangeservice/service/DataExchangeService.java +++ b/nnd-data-exchange-service/src/main/java/gov/cdc/nnddataexchangeservice/service/DataExchangeService.java @@ -1,30 +1,37 @@ package gov.cdc.nnddataexchangeservice.service; +import com.google.gson.Gson; import gov.cdc.nnddataexchangeservice.exception.DataExchangeException; import gov.cdc.nnddataexchangeservice.service.interfaces.ICNTransportQOutService; import gov.cdc.nnddataexchangeservice.service.interfaces.IDataExchangeService; import gov.cdc.nnddataexchangeservice.service.interfaces.INetsstTransportService; import gov.cdc.nnddataexchangeservice.service.interfaces.ITransportQOutService; import gov.cdc.nnddataexchangeservice.service.model.DataExchangeModel; +import gov.cdc.nnddataexchangeservice.shared.DataSimplification; import org.springframework.stereotype.Service; +import java.io.IOException; + @Service public class DataExchangeService implements IDataExchangeService { private final INetsstTransportService netsstTransportService; private final ITransportQOutService transportQOutService; private final ICNTransportQOutService cnTransportQOutService; + private final Gson gson; public DataExchangeService(INetsstTransportService netsstTransportService, ITransportQOutService transportQOutService, - ICNTransportQOutService icnTransportQOutService) { + ICNTransportQOutService icnTransportQOutService, Gson gson) { this.netsstTransportService = netsstTransportService; this.transportQOutService = transportQOutService; this.cnTransportQOutService = icnTransportQOutService; + this.gson = gson; } - public DataExchangeModel getDataForOnPremExchanging(String cnStatusTime, String transportTime,String netssTime, String statusCd, - Integer limit) throws DataExchangeException { + public String getDataForOnPremExchanging(String cnStatusTime, String transportTime, + String netssTime, String statusCd, + Integer limit, boolean compressionApplied) throws DataExchangeException, IOException { var dataExchange = new DataExchangeModel(); var cnTransportDatas = cnTransportQOutService.getTransportData(statusCd, cnStatusTime, limit); var transportDatas = transportQOutService.getTransportData(transportTime, limit); @@ -37,6 +44,13 @@ public DataExchangeModel getDataForOnPremExchanging(String cnStatusTime, String dataExchange.setCountCnTransport(cnTransportDatas.size()); dataExchange.setCountTransport(transportDatas.size()); dataExchange.setCountNetssTransport(netssDatas.size()); - return dataExchange; + String strData = gson.toJson(dataExchange); + String finalData; + if (compressionApplied) { + finalData = DataSimplification.dataCompressionAndEncode(strData); + } else { + finalData = strData; + } + return finalData; } } diff --git a/nnd-data-exchange-service/src/main/java/gov/cdc/nnddataexchangeservice/service/interfaces/IDataExchangeGenericService.java b/nnd-data-exchange-service/src/main/java/gov/cdc/nnddataexchangeservice/service/interfaces/IDataExchangeGenericService.java index 2f30ce3..4026321 100644 --- a/nnd-data-exchange-service/src/main/java/gov/cdc/nnddataexchangeservice/service/interfaces/IDataExchangeGenericService.java +++ b/nnd-data-exchange-service/src/main/java/gov/cdc/nnddataexchangeservice/service/interfaces/IDataExchangeGenericService.java @@ -3,6 +3,6 @@ import gov.cdc.nnddataexchangeservice.exception.DataExchangeException; public interface IDataExchangeGenericService { - String getGenericDataExchange(String tableName, String timeStamp, Integer limit) throws DataExchangeException; - String decodeAndDecompress(String base64EncodedData); + String getGenericDataExchange(String tableName, String timeStamp, Integer limit, boolean nullAllow) throws DataExchangeException; + String decodeAndDecompress(String base64EncodedData) throws DataExchangeException; } diff --git a/nnd-data-exchange-service/src/main/java/gov/cdc/nnddataexchangeservice/service/interfaces/IDataExchangeService.java b/nnd-data-exchange-service/src/main/java/gov/cdc/nnddataexchangeservice/service/interfaces/IDataExchangeService.java index fc63e08..41bf083 100644 --- a/nnd-data-exchange-service/src/main/java/gov/cdc/nnddataexchangeservice/service/interfaces/IDataExchangeService.java +++ b/nnd-data-exchange-service/src/main/java/gov/cdc/nnddataexchangeservice/service/interfaces/IDataExchangeService.java @@ -1,8 +1,11 @@ package gov.cdc.nnddataexchangeservice.service.interfaces; import gov.cdc.nnddataexchangeservice.exception.DataExchangeException; -import gov.cdc.nnddataexchangeservice.service.model.DataExchangeModel; + +import java.io.IOException; public interface IDataExchangeService { - DataExchangeModel getDataForOnPremExchanging(String cnStatusTime, String transportStatusTime, String netssTime , String statusCd, Integer limit) throws DataExchangeException; + String getDataForOnPremExchanging(String cnStatusTime, String transportStatusTime, + String netssTime , String statusCd, + Integer limit, boolean compressionApplied) throws DataExchangeException, IOException; } diff --git a/nnd-data-exchange-service/src/main/java/gov/cdc/nnddataexchangeservice/shared/DataSimplification.java b/nnd-data-exchange-service/src/main/java/gov/cdc/nnddataexchangeservice/shared/DataSimplification.java new file mode 100644 index 0000000..0d0f6d1 --- /dev/null +++ b/nnd-data-exchange-service/src/main/java/gov/cdc/nnddataexchangeservice/shared/DataSimplification.java @@ -0,0 +1,49 @@ +package gov.cdc.nnddataexchangeservice.shared; + +import gov.cdc.nnddataexchangeservice.exception.DataExchangeException; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; + +import static gov.cdc.nnddataexchangeservice.constant.DataSyncConstant.BYTE_SIZE; + +public class DataSimplification { + private DataSimplification() { + } + public static String dataCompressionAndEncode(String jsonData) throws IOException { + // Compress the JSON data using GZIP and return the Base64 encoded result + try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + GZIPOutputStream gzipOutputStream = new GZIPOutputStream(byteArrayOutputStream)) { + + gzipOutputStream.write(jsonData.getBytes(StandardCharsets.UTF_8)); + gzipOutputStream.finish(); + + byte[] compressedData = byteArrayOutputStream.toByteArray(); + return Base64.getEncoder().encodeToString(compressedData); + } + } + // DECODE TEST METHOD + public static String decodeAndDecompress(String base64EncodedData) throws DataExchangeException { + byte[] compressedData = Base64.getDecoder().decode(base64EncodedData); + + try (ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(compressedData); + GZIPInputStream gzipInputStream = new GZIPInputStream(byteArrayInputStream); + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) { + + byte[] buffer = new byte[BYTE_SIZE]; + int len; + while ((len = gzipInputStream.read(buffer)) > 0) { + byteArrayOutputStream.write(buffer, 0, len); + } + + return byteArrayOutputStream.toString(StandardCharsets.UTF_8); + } catch (Exception e) { + throw new DataExchangeException(e.getMessage()); + } + } +} diff --git a/nnd-data-exchange-service/src/main/resources/query/create/data_config.sql b/nnd-data-exchange-service/src/main/resources/query/create/data_config.sql index 4acdc85..7d0c455 100644 --- a/nnd-data-exchange-service/src/main/resources/query/create/data_config.sql +++ b/nnd-data-exchange-service/src/main/resources/query/create/data_config.sql @@ -20,6 +20,7 @@ CREATE TABLE data_sync_config source_db NVARCHAR(255) NOT NULL, query NVARCHAR(MAX) NOT NULL, query_with_limit NVARCHAR(MAX) NULL, + query_with_null_timestamp NVARCHAR(MAX) NULL, created_at DATETIME2 DEFAULT GETDATE(), updated_at DATETIME2 DEFAULT GETDATE() ); diff --git a/nnd-data-exchange-service/src/main/resources/query/select/insert.sql b/nnd-data-exchange-service/src/main/resources/query/select/insert.sql index 98f326f..6910267 100644 --- a/nnd-data-exchange-service/src/main/resources/query/select/insert.sql +++ b/nnd-data-exchange-service/src/main/resources/query/select/insert.sql @@ -1,41 +1,115 @@ -INSERT INTO [RDB].[dbo].[data_sync_config] (table_name, source_db, query, query_with_limit) VALUES - ('CONDITION', 'RDB', 'SELECT * FROM CONDITION;', NULL), - ('Rdb_Date', 'RDB', 'SELECT * FROM Rdb_Date;', NULL), - ('D_PATIENT', 'RDB', 'SELECT * FROM D_PATIENT WHERE D_PATIENT.PATIENT_LAST_CHANGE_TIME >= :timestamp;', 'SELECT TOP(:limit) * FROM D_PATIENT WHERE D_PATIENT.PATIENT_LAST_CHANGE_TIME >= :timestamp ORDER BY D_PATIENT.PATIENT_LAST_CHANGE_TIME ASC;'), - ('INVESTIGATION', 'RDB', 'SELECT * FROM INVESTIGATION WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp;', 'SELECT TOP(:limit) * FROM INVESTIGATION WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp ORDER BY INVESTIGATION.LAST_CHG_TIME ASC;'), - ('CASE_COUNT', 'RDB', 'SELECT A.* FROM CASE_COUNT as A INNER JOIN INVESTIGATION as B ON A.INVESTIGATION_KEY = B.INVESTIGATION_KEY WHERE B.ADD_TIME >= :timestamp OR B.LAST_CHG_TIME >= :timestamp;', 'SELECT TOP(:limit) A.* FROM CASE_COUNT as A INNER JOIN INVESTIGATION as B ON A.INVESTIGATION_KEY = B.INVESTIGATION_KEY WHERE B.ADD_TIME >= :timestamp OR B.LAST_CHG_TIME >= :timestamp ORDER BY CASE WHEN B.ADD_TIME >= :timestamp THEN B.ADD_TIME ELSE B.LAST_CHG_TIME END ASC;'), - ('BMIRD_CASE', 'RDB', 'SELECT BMIRD_CASE.* FROM BMIRD_CASE JOIN INVESTIGATION ON BMIRD_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp;', 'SELECT TOP(:limit) BMIRD_CASE.* FROM BMIRD_CASE JOIN INVESTIGATION ON BMIRD_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp ORDER BY INVESTIGATION.LAST_CHG_TIME ASC;'), - ('RUBELLA_CASE', 'RDB', 'SELECT RUBELLA_CASE.* FROM RUBELLA_CASE JOIN INVESTIGATION ON RUBELLA_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp;', 'SELECT TOP(:limit) RUBELLA_CASE.* FROM RUBELLA_CASE JOIN INVESTIGATION ON RUBELLA_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp ORDER BY INVESTIGATION.LAST_CHG_TIME ASC;'), - ('PERTUSSIS_CASE', 'RDB', 'SELECT PERTUSSIS_CASE.* FROM PERTUSSIS_CASE JOIN INVESTIGATION ON PERTUSSIS_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp;', 'SELECT TOP(:limit) PERTUSSIS_CASE.* FROM PERTUSSIS_CASE JOIN INVESTIGATION ON PERTUSSIS_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp ORDER BY INVESTIGATION.LAST_CHG_TIME ASC;'), - ('HEPATITIS_CASE', 'RDB', 'SELECT HEPATITIS_CASE.* FROM HEPATITIS_CASE JOIN INVESTIGATION ON HEPATITIS_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp;', 'SELECT TOP(:limit) HEPATITIS_CASE.* FROM HEPATITIS_CASE JOIN INVESTIGATION ON HEPATITIS_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp ORDER BY INVESTIGATION.LAST_CHG_TIME ASC;'), - ('GENERIC_CASE', 'RDB', 'SELECT GENERIC_CASE.* FROM GENERIC_CASE JOIN INVESTIGATION ON GENERIC_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp;', 'SELECT TOP(:limit) GENERIC_CASE.* FROM GENERIC_CASE JOIN INVESTIGATION ON GENERIC_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp ORDER BY INVESTIGATION.LAST_CHG_TIME ASC;'), - ('MEASLES_CASE', 'RDB', 'SELECT MEASLES_CASE.* FROM MEASLES_CASE JOIN INVESTIGATION ON MEASLES_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp;', 'SELECT TOP(:limit) MEASLES_CASE.* FROM MEASLES_CASE JOIN INVESTIGATION ON MEASLES_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp ORDER BY INVESTIGATION.LAST_CHG_TIME ASC;'), - ('HEPATITIS_DATAMART', 'RDB', 'SELECT * FROM HEPATITIS_DATAMART WHERE REFRESH_DATETIME >= :timestamp;', 'SELECT TOP(:limit) * FROM HEPATITIS_DATAMART WHERE REFRESH_DATETIME >= :timestamp ORDER BY REFRESH_DATETIME ASC;'), - ('CONFIRMATION_METHOD', 'RDB', 'SELECT * FROM CONFIRMATION_METHOD;', NULL), - ('CONFIRMATION_METHOD_GROUP', 'RDB', 'SELECT CONFIRMATION_METHOD_GROUP.* FROM CONFIRMATION_METHOD_GROUP JOIN INVESTIGATION ON CONFIRMATION_METHOD_GROUP.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp;', 'SELECT TOP(:limit) CONFIRMATION_METHOD_GROUP.* FROM CONFIRMATION_METHOD_GROUP JOIN INVESTIGATION ON CONFIRMATION_METHOD_GROUP.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp ORDER BY INVESTIGATION.LAST_CHG_TIME ASC;'), - ('D_INTERVIEW', 'RDB', 'SELECT D_INTERVIEW.* FROM D_INTERVIEW WHERE D_INTERVIEW.ADD_TIME >= :timestamp OR D_INTERVIEW.LAST_CHG_TIME >= :timestamp;', 'SELECT TOP(:limit) D_INTERVIEW.* FROM D_INTERVIEW WHERE D_INTERVIEW.ADD_TIME >= :timestamp OR D_INTERVIEW.LAST_CHG_TIME >= :timestamp ORDER BY CASE WHEN D_INTERVIEW.ADD_TIME >= :timestamp THEN D_INTERVIEW.ADD_TIME ELSE D_INTERVIEW.LAST_CHG_TIME END ASC;'), - ('D_CASE_MANAGEMENT', 'RDB', 'SELECT D_CASE_MANAGEMENT.* FROM D_CASE_MANAGEMENT JOIN INVESTIGATION ON D_CASE_MANAGEMENT.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp;', 'SELECT TOP(:limit) D_CASE_MANAGEMENT.* FROM D_CASE_MANAGEMENT JOIN INVESTIGATION ON D_CASE_MANAGEMENT.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp ORDER BY INVESTIGATION.LAST_CHG_TIME ASC;'), - ('VAR_PAM_LDF', 'RDB', 'SELECT VAR_PAM_LDF.* FROM VAR_PAM_LDF JOIN INVESTIGATION ON VAR_PAM_LDF.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE VAR_PAM_LDF.add_time >= :timestamp OR INVESTIGATION.LAST_CHG_TIME >= :timestamp;', 'SELECT TOP(:limit) VAR_PAM_LDF.* FROM VAR_PAM_LDF JOIN INVESTIGATION ON VAR_PAM_LDF.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE VAR_PAM_LDF.add_time >= :timestamp OR INVESTIGATION.LAST_CHG_TIME >= :timestamp ORDER BY CASE WHEN VAR_PAM_LDF.add_time >= :timestamp THEN VAR_PAM_LDF.add_time ELSE INVESTIGATION.LAST_CHG_TIME END ASC;'), - ('F_VAR_PAM', 'RDB', 'SELECT F_VAR_PAM.* FROM F_VAR_PAM JOIN INVESTIGATION ON F_VAR_PAM.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.ADD_TIME >= :timestamp OR INVESTIGATION.LAST_CHG_TIME >= :timestamp;', 'SELECT TOP(:limit) F_VAR_PAM.* FROM F_VAR_PAM JOIN INVESTIGATION ON F_VAR_PAM.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.ADD_TIME >= :timestamp OR INVESTIGATION.LAST_CHG_TIME >= :timestamp ORDER BY CASE WHEN INVESTIGATION.ADD_TIME >= :timestamp THEN INVESTIGATION.ADD_TIME ELSE INVESTIGATION.LAST_CHG_TIME END ASC;'), - ('LDF_DATA', 'RDB', 'SELECT LDF_DATA.* FROM LDF_DATA JOIN LDF_GROUP ON LDF_DATA.LDF_GROUP_KEY = LDF_GROUP.LDF_GROUP_KEY JOIN INVESTIGATION ON LDF_GROUP.BUSINESS_OBJECT_UID = INVESTIGATION.CASE_UID WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp;', 'SELECT TOP(:limit) LDF_DATA.* FROM LDF_DATA JOIN LDF_GROUP ON LDF_DATA.LDF_GROUP_KEY = LDF_GROUP.LDF_GROUP_KEY JOIN INVESTIGATION ON LDF_GROUP.BUSINESS_OBJECT_UID = INVESTIGATION.CASE_UID WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp ORDER BY INVESTIGATION.LAST_CHG_TIME ASC;'), - ('LDF_FOODBORNE', 'RDB', 'SELECT LDF_FOODBORNE.* FROM LDF_FOODBORNE JOIN INVESTIGATION ON LDF_FOODBORNE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp;', 'SELECT TOP(:limit) LDF_FOODBORNE.* FROM LDF_FOODBORNE JOIN INVESTIGATION ON LDF_FOODBORNE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp ORDER BY INVESTIGATION.LAST_CHG_TIME ASC;'), - ('NOTIFICATION_EVENT', 'RDB', 'SELECT NOTIFICATION_EVENT.* FROM NOTIFICATION_EVENT JOIN INVESTIGATION ON NOTIFICATION_EVENT.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp;', 'SELECT TOP(:limit) NOTIFICATION_EVENT.* FROM NOTIFICATION_EVENT JOIN INVESTIGATION ON NOTIFICATION_EVENT.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp ORDER BY INVESTIGATION.LAST_CHG_TIME ASC;'), - ('D_INV_ADMINISTRATIVE', 'RDB', 'SELECT D_INV_ADMINISTRATIVE.* FROM D_INV_ADMINISTRATIVE JOIN F_PAGE_CASE ON D_INV_ADMINISTRATIVE.D_INV_ADMINISTRATIVE_KEY = F_PAGE_CASE.INVESTIGATION_KEY JOIN INVESTIGATION ON F_PAGE_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp;', 'SELECT TOP(:limit) D_INV_ADMINISTRATIVE.* FROM D_INV_ADMINISTRATIVE JOIN F_PAGE_CASE ON D_INV_ADMINISTRATIVE.D_INV_ADMINISTRATIVE_KEY = F_PAGE_CASE.INVESTIGATION_KEY JOIN INVESTIGATION ON F_PAGE_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp ORDER BY INVESTIGATION.LAST_CHG_TIME ASC;'), - ('D_INV_EPIDEMIOLOGY', 'RDB', 'SELECT D_INV_EPIDEMIOLOGY.* FROM D_INV_EPIDEMIOLOGY JOIN F_PAGE_CASE ON D_INV_EPIDEMIOLOGY.D_INV_EPIDEMIOLOGY_KEY = F_PAGE_CASE.INVESTIGATION_KEY JOIN INVESTIGATION ON F_PAGE_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp;', 'SELECT TOP(:limit) D_INV_EPIDEMIOLOGY.* FROM D_INV_EPIDEMIOLOGY JOIN F_PAGE_CASE ON D_INV_EPIDEMIOLOGY.D_INV_EPIDEMIOLOGY_KEY = F_PAGE_CASE.INVESTIGATION_KEY JOIN INVESTIGATION ON F_PAGE_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp ORDER BY INVESTIGATION.LAST_CHG_TIME ASC;'), - ('D_INV_HIV', 'RDB', 'SELECT D_INV_HIV.* FROM D_INV_HIV JOIN F_PAGE_CASE ON D_INV_HIV.D_INV_HIV_KEY = F_PAGE_CASE.INVESTIGATION_KEY JOIN INVESTIGATION ON F_PAGE_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp;', 'SELECT TOP(:limit) D_INV_HIV.* FROM D_INV_HIV JOIN F_PAGE_CASE ON D_INV_HIV.D_INV_HIV_KEY = F_PAGE_CASE.INVESTIGATION_KEY JOIN INVESTIGATION ON F_PAGE_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp ORDER BY INVESTIGATION.LAST_CHG_TIME ASC;'), - ('D_INV_LAB_FINDING', 'RDB', 'SELECT D_INV_LAB_FINDING.* FROM D_INV_LAB_FINDING JOIN F_PAGE_CASE ON D_INV_LAB_FINDING.D_INV_LAB_FINDING_KEY = F_PAGE_CASE.INVESTIGATION_KEY JOIN INVESTIGATION ON F_PAGE_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp;', 'SELECT TOP(:limit) D_INV_LAB_FINDING.* FROM D_INV_LAB_FINDING JOIN F_PAGE_CASE ON D_INV_LAB_FINDING.D_INV_LAB_FINDING_KEY = F_PAGE_CASE.INVESTIGATION_KEY JOIN INVESTIGATION ON F_PAGE_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp ORDER BY INVESTIGATION.LAST_CHG_TIME ASC;'), - ('D_INV_MEDICAL_HISTORY', 'RDB', 'SELECT D_INV_MEDICAL_HISTORY.* FROM D_INV_MEDICAL_HISTORY JOIN F_PAGE_CASE ON D_INV_MEDICAL_HISTORY.D_INV_MEDICAL_HISTORY_KEY = F_PAGE_CASE.INVESTIGATION_KEY JOIN INVESTIGATION ON F_PAGE_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp;', 'SELECT TOP(:limit) D_INV_MEDICAL_HISTORY.* FROM D_INV_MEDICAL_HISTORY JOIN F_PAGE_CASE ON D_INV_MEDICAL_HISTORY.D_INV_MEDICAL_HISTORY_KEY = F_PAGE_CASE.INVESTIGATION_KEY JOIN INVESTIGATION ON F_PAGE_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp ORDER BY INVESTIGATION.LAST_CHG_TIME ASC;'), - ('D_INV_RISK_FACTOR', 'RDB', 'SELECT D_INV_RISK_FACTOR.* FROM D_INV_RISK_FACTOR JOIN F_PAGE_CASE ON D_INV_RISK_FACTOR.D_INV_RISK_FACTOR_KEY = F_PAGE_CASE.INVESTIGATION_KEY JOIN INVESTIGATION ON F_PAGE_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp;', 'SELECT TOP(:limit) D_INV_RISK_FACTOR.* FROM D_INV_RISK_FACTOR JOIN F_PAGE_CASE ON D_INV_RISK_FACTOR.D_INV_RISK_FACTOR_KEY = F_PAGE_CASE.INVESTIGATION_KEY JOIN INVESTIGATION ON F_PAGE_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp ORDER BY INVESTIGATION.LAST_CHG_TIME ASC;'), - ('D_INV_TREATMENT', 'RDB', 'SELECT D_INV_TREATMENT.* FROM D_INV_TREATMENT JOIN F_PAGE_CASE ON D_INV_TREATMENT.D_INV_TREATMENT_KEY = F_PAGE_CASE.INVESTIGATION_KEY JOIN INVESTIGATION ON F_PAGE_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp;', 'SELECT TOP(:limit) D_INV_TREATMENT.* FROM D_INV_TREATMENT JOIN F_PAGE_CASE ON D_INV_TREATMENT.D_INV_TREATMENT_KEY = F_PAGE_CASE.INVESTIGATION_KEY JOIN INVESTIGATION ON F_PAGE_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp ORDER BY INVESTIGATION.LAST_CHG_TIME ASC;'), - ('D_INV_VACCINATION', 'RDB', 'SELECT D_INV_VACCINATION.* FROM D_INV_VACCINATION JOIN F_PAGE_CASE ON D_INV_VACCINATION.D_INV_VACCINATION_KEY = F_PAGE_CASE.INVESTIGATION_KEY JOIN INVESTIGATION ON F_PAGE_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp;', 'SELECT TOP(:limit) D_INV_VACCINATION.* FROM D_INV_VACCINATION JOIN F_PAGE_CASE ON D_INV_VACCINATION.D_INV_VACCINATION_KEY = F_PAGE_CASE.INVESTIGATION_KEY JOIN INVESTIGATION ON F_PAGE_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp ORDER BY INVESTIGATION.LAST_CHG_TIME ASC;'), - ('F_INTERVIEW_CASE', 'RDB', 'SELECT F_INTERVIEW_CASE.* FROM F_INTERVIEW_CASE JOIN INVESTIGATION ON F_INTERVIEW_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY JOIN D_INTERVIEW ON F_INTERVIEW_CASE.D_INTERVIEW_KEY = D_INTERVIEW.D_INTERVIEW_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp OR D_INTERVIEW.ADD_TIME >= :timestamp OR D_INTERVIEW.LAST_CHG_TIME >= :timestamp;', 'SELECT TOP(:limit) F_INTERVIEW_CASE.* FROM F_INTERVIEW_CASE JOIN INVESTIGATION ON F_INTERVIEW_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY JOIN D_INTERVIEW ON F_INTERVIEW_CASE.D_INTERVIEW_KEY = D_INTERVIEW.D_INTERVIEW_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp OR D_INTERVIEW.ADD_TIME >= :timestamp OR D_INTERVIEW.LAST_CHG_TIME >= :timestamp ORDER BY CASE WHEN D_INTERVIEW.ADD_TIME >= :timestamp THEN D_INTERVIEW.ADD_TIME ELSE D_INTERVIEW.LAST_CHG_TIME END ASC;'), - ('F_PAGE_CASE', 'RDB', 'SELECT F_PAGE_CASE.* FROM F_PAGE_CASE JOIN INVESTIGATION ON F_PAGE_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp;', 'SELECT TOP(:limit) F_PAGE_CASE.* FROM F_PAGE_CASE JOIN INVESTIGATION ON F_PAGE_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp ORDER BY INVESTIGATION.LAST_CHG_TIME ASC;'), - ('F_STD_PAGE_CASE', 'RDB', 'SELECT F_STD_PAGE_CASE.* FROM F_STD_PAGE_CASE JOIN INVESTIGATION ON F_STD_PAGE_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp;', 'SELECT TOP(:limit) F_STD_PAGE_CASE.* FROM F_STD_PAGE_CASE JOIN INVESTIGATION ON F_STD_PAGE_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp ORDER BY INVESTIGATION.LAST_CHG_TIME ASC;'), - ('TREATMENT', 'RDB', 'SELECT TREATMENT.* FROM TREATMENT JOIN TREATMENT_EVENT ON TREATMENT.TREATMENT_KEY = TREATMENT_EVENT.TREATMENT_KEY JOIN INVESTIGATION ON TREATMENT_EVENT.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp;', 'SELECT TOP(:limit) TREATMENT.* FROM TREATMENT JOIN TREATMENT_EVENT ON TREATMENT.TREATMENT_KEY = TREATMENT_EVENT.TREATMENT_KEY JOIN INVESTIGATION ON TREATMENT_EVENT.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp ORDER BY INVESTIGATION.LAST_CHG_TIME ASC;'), - ('TREATMENT_EVENT', 'RDB', 'SELECT TREATMENT_EVENT.* FROM TREATMENT_EVENT JOIN INVESTIGATION ON TREATMENT_EVENT.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp;', 'SELECT TOP(:limit) TREATMENT_EVENT.* FROM TREATMENT_EVENT JOIN INVESTIGATION ON TREATMENT_EVENT.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp ORDER BY INVESTIGATION.LAST_CHG_TIME ASC;'), +INSERT INTO [RDB].[dbo].[data_sync_config] (table_name, source_db, query, query_with_limit, query_with_null_timestamp) VALUES + ('CONDITION', 'RDB', 'SELECT * FROM CONDITION;', NULL, NULL), + ('Rdb_Date', 'RDB', 'SELECT * FROM Rdb_Date;', NULL, NULL), + ('D_PATIENT', 'RDB', 'SELECT * FROM D_PATIENT WHERE D_PATIENT.PATIENT_LAST_CHANGE_TIME >= :timestamp;', 'SELECT TOP(:limit) * FROM D_PATIENT WHERE D_PATIENT.PATIENT_LAST_CHANGE_TIME >= :timestamp ORDER BY D_PATIENT.PATIENT_LAST_CHANGE_TIME ASC;', 'SELECT * FROM D_PATIENT WHERE D_PATIENT.PATIENT_LAST_CHANGE_TIME IS NULL;'), + ('INVESTIGATION', 'RDB', 'SELECT * FROM INVESTIGATION WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp;', 'SELECT TOP(:limit) * FROM INVESTIGATION WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp ORDER BY INVESTIGATION.LAST_CHG_TIME ASC;', 'SELECT * FROM INVESTIGATION WHERE INVESTIGATION.LAST_CHG_TIME IS NULL;'), + ('CASE_COUNT', 'RDB', 'SELECT A.* FROM CASE_COUNT as A INNER JOIN INVESTIGATION as B ON A.INVESTIGATION_KEY = B.INVESTIGATION_KEY WHERE B.ADD_TIME >= :timestamp OR B.LAST_CHG_TIME >= :timestamp;', + 'SELECT TOP(:limit) A.* FROM CASE_COUNT as A INNER JOIN INVESTIGATION as B ON A.INVESTIGATION_KEY = B.INVESTIGATION_KEY WHERE B.ADD_TIME >= :timestamp OR B.LAST_CHG_TIME >= :timestamp ORDER BY CASE WHEN B.ADD_TIME >= :timestamp THEN B.ADD_TIME ELSE B.LAST_CHG_TIME END ASC;', + NULL), + ('BMIRD_CASE', 'RDB', 'SELECT BMIRD_CASE.* FROM BMIRD_CASE JOIN INVESTIGATION ON BMIRD_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp;', + 'SELECT TOP(:limit) BMIRD_CASE.* FROM BMIRD_CASE JOIN INVESTIGATION ON BMIRD_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp ORDER BY INVESTIGATION.LAST_CHG_TIME ASC;', + NULL), + ('RUBELLA_CASE', 'RDB', 'SELECT RUBELLA_CASE.* FROM RUBELLA_CASE JOIN INVESTIGATION ON RUBELLA_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp;', + 'SELECT TOP(:limit) RUBELLA_CASE.* FROM RUBELLA_CASE JOIN INVESTIGATION ON RUBELLA_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp ORDER BY INVESTIGATION.LAST_CHG_TIME ASC;', + NULL), + ('CONFIRMATION_METHOD', 'RDB', 'SELECT * FROM CONFIRMATION_METHOD;', NULL, NULL), + ('PERTUSSIS_CASE', 'RDB', 'SELECT PERTUSSIS_CASE.* FROM PERTUSSIS_CASE JOIN INVESTIGATION ON PERTUSSIS_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp;', + 'SELECT TOP(:limit) PERTUSSIS_CASE.* FROM PERTUSSIS_CASE JOIN INVESTIGATION ON PERTUSSIS_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp ORDER BY INVESTIGATION.LAST_CHG_TIME ASC;', + NULL), + ('HEPATITIS_CASE', 'RDB', 'SELECT HEPATITIS_CASE.* FROM HEPATITIS_CASE JOIN INVESTIGATION ON HEPATITIS_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp;', 'SELECT TOP(:limit) HEPATITIS_CASE.* FROM HEPATITIS_CASE JOIN INVESTIGATION ON HEPATITIS_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp ORDER BY INVESTIGATION.LAST_CHG_TIME ASC;', + NULL), + ('GENERIC_CASE', 'RDB', 'SELECT GENERIC_CASE.* FROM GENERIC_CASE JOIN INVESTIGATION ON GENERIC_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp;', 'SELECT TOP(:limit) GENERIC_CASE.* FROM GENERIC_CASE JOIN INVESTIGATION ON GENERIC_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp ORDER BY INVESTIGATION.LAST_CHG_TIME ASC;', + NULL), + ('MEASLES_CASE', 'RDB', 'SELECT MEASLES_CASE.* FROM MEASLES_CASE JOIN INVESTIGATION ON MEASLES_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp;', 'SELECT TOP(:limit) MEASLES_CASE.* FROM MEASLES_CASE JOIN INVESTIGATION ON MEASLES_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp ORDER BY INVESTIGATION.LAST_CHG_TIME ASC;', + NULL), + ('HEPATITIS_DATAMART', 'RDB', 'SELECT * FROM HEPATITIS_DATAMART WHERE REFRESH_DATETIME >= :timestamp;', 'SELECT TOP(:limit) * FROM HEPATITIS_DATAMART WHERE REFRESH_DATETIME >= :timestamp ORDER BY REFRESH_DATETIME ASC;', + NULL), + ('CONFIRMATION_METHOD_GROUP', 'RDB', 'SELECT CONFIRMATION_METHOD_GROUP.* FROM CONFIRMATION_METHOD_GROUP JOIN INVESTIGATION ON CONFIRMATION_METHOD_GROUP.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp;', 'SELECT TOP(:limit) CONFIRMATION_METHOD_GROUP.* FROM CONFIRMATION_METHOD_GROUP JOIN INVESTIGATION ON CONFIRMATION_METHOD_GROUP.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp ORDER BY INVESTIGATION.LAST_CHG_TIME ASC;', 'SELECT CONFIRMATION_METHOD_GROUP.* FROM CONFIRMATION_METHOD_GROUP JOIN INVESTIGATION ON CONFIRMATION_METHOD_GROUP.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME IS NULL;'), + ('D_INTERVIEW', 'RDB', 'SELECT D_INTERVIEW.* FROM D_INTERVIEW WHERE D_INTERVIEW.ADD_TIME >= :timestamp OR D_INTERVIEW.LAST_CHG_TIME >= :timestamp;', 'SELECT TOP(:limit) D_INTERVIEW.* FROM D_INTERVIEW WHERE D_INTERVIEW.ADD_TIME >= :timestamp OR D_INTERVIEW.LAST_CHG_TIME >= :timestamp ORDER BY CASE WHEN D_INTERVIEW.ADD_TIME >= :timestamp THEN D_INTERVIEW.ADD_TIME ELSE D_INTERVIEW.LAST_CHG_TIME END ASC;', + NULL), + ('D_CASE_MANAGEMENT', 'RDB', 'SELECT D_CASE_MANAGEMENT.* FROM D_CASE_MANAGEMENT JOIN INVESTIGATION ON D_CASE_MANAGEMENT.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp;', 'SELECT TOP(:limit) D_CASE_MANAGEMENT.* FROM D_CASE_MANAGEMENT JOIN INVESTIGATION ON D_CASE_MANAGEMENT.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp ORDER BY INVESTIGATION.LAST_CHG_TIME ASC;', + NULL), + ('VAR_PAM_LDF', 'RDB', 'SELECT VAR_PAM_LDF.* FROM VAR_PAM_LDF JOIN INVESTIGATION ON VAR_PAM_LDF.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE VAR_PAM_LDF.add_time >= :timestamp OR INVESTIGATION.LAST_CHG_TIME >= :timestamp;', 'SELECT TOP(:limit) VAR_PAM_LDF.* FROM VAR_PAM_LDF JOIN INVESTIGATION ON VAR_PAM_LDF.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE VAR_PAM_LDF.add_time >= :timestamp OR INVESTIGATION.LAST_CHG_TIME >= :timestamp ORDER BY CASE WHEN VAR_PAM_LDF.add_time >= :timestamp THEN VAR_PAM_LDF.add_time ELSE INVESTIGATION.LAST_CHG_TIME END ASC;', + NULL), + ('F_VAR_PAM', 'RDB', + 'SELECT F_VAR_PAM.* FROM F_VAR_PAM JOIN INVESTIGATION ON F_VAR_PAM.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.ADD_TIME >= :timestamp OR INVESTIGATION.LAST_CHG_TIME >= :timestamp;', + 'SELECT TOP(:limit) F_VAR_PAM.* FROM F_VAR_PAM JOIN INVESTIGATION ON F_VAR_PAM.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.ADD_TIME >= :timestamp OR INVESTIGATION.LAST_CHG_TIME >= :timestamp ORDER BY CASE WHEN INVESTIGATION.ADD_TIME >= :timestamp THEN INVESTIGATION.ADD_TIME ELSE INVESTIGATION.LAST_CHG_TIME END ASC;', + NULL), + ('LDF_DATA', 'RDB', + 'SELECT LDF_DATA.* FROM LDF_DATA JOIN LDF_GROUP ON LDF_DATA.LDF_GROUP_KEY = LDF_GROUP.LDF_GROUP_KEY JOIN INVESTIGATION ON LDF_GROUP.BUSINESS_OBJECT_UID = INVESTIGATION.CASE_UID WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp;', + 'SELECT TOP(:limit) LDF_DATA.* FROM LDF_DATA JOIN LDF_GROUP ON LDF_DATA.LDF_GROUP_KEY = LDF_GROUP.LDF_GROUP_KEY JOIN INVESTIGATION ON LDF_GROUP.BUSINESS_OBJECT_UID = INVESTIGATION.CASE_UID WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp ORDER BY INVESTIGATION.LAST_CHG_TIME ASC;', + 'SELECT LDF_DATA.* FROM LDF_DATA JOIN LDF_GROUP ON LDF_DATA.LDF_GROUP_KEY = LDF_GROUP.LDF_GROUP_KEY JOIN INVESTIGATION ON LDF_GROUP.BUSINESS_OBJECT_UID = INVESTIGATION.CASE_UID WHERE INVESTIGATION.LAST_CHG_TIME IS NULL;'), + ('LDF_FOODBORNE', 'RDB', + 'SELECT LDF_FOODBORNE.* FROM LDF_FOODBORNE JOIN INVESTIGATION ON LDF_FOODBORNE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp;', + 'SELECT TOP(:limit) LDF_FOODBORNE.* FROM LDF_FOODBORNE JOIN INVESTIGATION ON LDF_FOODBORNE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp ORDER BY INVESTIGATION.LAST_CHG_TIME ASC;', + NULL), + ('NOTIFICATION_EVENT', 'RDB', + 'SELECT NOTIFICATION_EVENT.* FROM NOTIFICATION_EVENT JOIN INVESTIGATION ON NOTIFICATION_EVENT.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp;', + 'SELECT TOP(:limit) NOTIFICATION_EVENT.* FROM NOTIFICATION_EVENT JOIN INVESTIGATION ON NOTIFICATION_EVENT.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp ORDER BY INVESTIGATION.LAST_CHG_TIME ASC;', + 'SELECT NOTIFICATION_EVENT.* FROM NOTIFICATION_EVENT JOIN INVESTIGATION ON NOTIFICATION_EVENT.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME IS NULL;'), + ('D_INV_ADMINISTRATIVE', 'RDB', + 'SELECT D_INV_ADMINISTRATIVE.* FROM D_INV_ADMINISTRATIVE JOIN F_PAGE_CASE ON D_INV_ADMINISTRATIVE.D_INV_ADMINISTRATIVE_KEY = F_PAGE_CASE.INVESTIGATION_KEY JOIN INVESTIGATION ON F_PAGE_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp;', + 'SELECT TOP(:limit) D_INV_ADMINISTRATIVE.* FROM D_INV_ADMINISTRATIVE JOIN F_PAGE_CASE ON D_INV_ADMINISTRATIVE.D_INV_ADMINISTRATIVE_KEY = F_PAGE_CASE.INVESTIGATION_KEY JOIN INVESTIGATION ON F_PAGE_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp ORDER BY INVESTIGATION.LAST_CHG_TIME ASC;', + 'SELECT D_INV_ADMINISTRATIVE.* FROM D_INV_ADMINISTRATIVE JOIN F_PAGE_CASE ON D_INV_ADMINISTRATIVE.D_INV_ADMINISTRATIVE_KEY = F_PAGE_CASE.INVESTIGATION_KEY JOIN INVESTIGATION ON F_PAGE_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME IS NULL;'), + ('D_INV_EPIDEMIOLOGY', 'RDB', + 'SELECT D_INV_EPIDEMIOLOGY.* FROM D_INV_EPIDEMIOLOGY JOIN F_PAGE_CASE ON D_INV_EPIDEMIOLOGY.D_INV_EPIDEMIOLOGY_KEY = F_PAGE_CASE.INVESTIGATION_KEY JOIN INVESTIGATION ON F_PAGE_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp;', + 'SELECT TOP(:limit) D_INV_EPIDEMIOLOGY.* FROM D_INV_EPIDEMIOLOGY JOIN F_PAGE_CASE ON D_INV_EPIDEMIOLOGY.D_INV_EPIDEMIOLOGY_KEY = F_PAGE_CASE.INVESTIGATION_KEY JOIN INVESTIGATION ON F_PAGE_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp ORDER BY INVESTIGATION.LAST_CHG_TIME ASC;', + 'SELECT D_INV_EPIDEMIOLOGY.* FROM D_INV_EPIDEMIOLOGY JOIN F_PAGE_CASE ON D_INV_EPIDEMIOLOGY.D_INV_EPIDEMIOLOGY_KEY = F_PAGE_CASE.INVESTIGATION_KEY JOIN INVESTIGATION ON F_PAGE_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME IS NULL;'), + ('D_INV_HIV', 'RDB', + 'SELECT D_INV_HIV.* FROM D_INV_HIV JOIN F_PAGE_CASE ON D_INV_HIV.D_INV_HIV_KEY = F_PAGE_CASE.INVESTIGATION_KEY JOIN INVESTIGATION ON F_PAGE_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp;', + 'SELECT TOP(:limit) D_INV_HIV.* FROM D_INV_HIV JOIN F_PAGE_CASE ON D_INV_HIV.D_INV_HIV_KEY = F_PAGE_CASE.INVESTIGATION_KEY JOIN INVESTIGATION ON F_PAGE_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp ORDER BY INVESTIGATION.LAST_CHG_TIME ASC;', + 'SELECT D_INV_HIV.* FROM D_INV_HIV JOIN F_PAGE_CASE ON D_INV_HIV.D_INV_HIV_KEY = F_PAGE_CASE.INVESTIGATION_KEY JOIN INVESTIGATION ON F_PAGE_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME IS NULL;'), + ('D_INV_LAB_FINDING', 'RDB', + 'SELECT D_INV_LAB_FINDING.* FROM D_INV_LAB_FINDING JOIN F_PAGE_CASE ON D_INV_LAB_FINDING.D_INV_LAB_FINDING_KEY = F_PAGE_CASE.INVESTIGATION_KEY JOIN INVESTIGATION ON F_PAGE_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp;', + 'SELECT TOP(:limit) D_INV_LAB_FINDING.* FROM D_INV_LAB_FINDING JOIN F_PAGE_CASE ON D_INV_LAB_FINDING.D_INV_LAB_FINDING_KEY = F_PAGE_CASE.INVESTIGATION_KEY JOIN INVESTIGATION ON F_PAGE_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp ORDER BY INVESTIGATION.LAST_CHG_TIME ASC;', + 'SELECT D_INV_LAB_FINDING.* FROM D_INV_LAB_FINDING JOIN F_PAGE_CASE ON D_INV_LAB_FINDING.D_INV_LAB_FINDING_KEY = F_PAGE_CASE.INVESTIGATION_KEY JOIN INVESTIGATION ON F_PAGE_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME IS NULL;'), + ('D_INV_MEDICAL_HISTORY', 'RDB', + 'SELECT D_INV_MEDICAL_HISTORY.* FROM D_INV_MEDICAL_HISTORY JOIN F_PAGE_CASE ON D_INV_MEDICAL_HISTORY.D_INV_MEDICAL_HISTORY_KEY = F_PAGE_CASE.INVESTIGATION_KEY JOIN INVESTIGATION ON F_PAGE_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp;', + 'SELECT TOP(:limit) D_INV_MEDICAL_HISTORY.* FROM D_INV_MEDICAL_HISTORY JOIN F_PAGE_CASE ON D_INV_MEDICAL_HISTORY.D_INV_MEDICAL_HISTORY_KEY = F_PAGE_CASE.INVESTIGATION_KEY JOIN INVESTIGATION ON F_PAGE_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp ORDER BY INVESTIGATION.LAST_CHG_TIME ASC;', + 'SELECT D_INV_MEDICAL_HISTORY.* FROM D_INV_MEDICAL_HISTORY JOIN F_PAGE_CASE ON D_INV_MEDICAL_HISTORY.D_INV_MEDICAL_HISTORY_KEY = F_PAGE_CASE.INVESTIGATION_KEY JOIN INVESTIGATION ON F_PAGE_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME IS NULL;'), + ('D_INV_RISK_FACTOR', 'RDB', + 'SELECT D_INV_RISK_FACTOR.* FROM D_INV_RISK_FACTOR JOIN F_PAGE_CASE ON D_INV_RISK_FACTOR.D_INV_RISK_FACTOR_KEY = F_PAGE_CASE.INVESTIGATION_KEY JOIN INVESTIGATION ON F_PAGE_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp;', + 'SELECT TOP(:limit) D_INV_RISK_FACTOR.* FROM D_INV_RISK_FACTOR JOIN F_PAGE_CASE ON D_INV_RISK_FACTOR.D_INV_RISK_FACTOR_KEY = F_PAGE_CASE.INVESTIGATION_KEY JOIN INVESTIGATION ON F_PAGE_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp ORDER BY INVESTIGATION.LAST_CHG_TIME ASC;', + 'SELECT D_INV_RISK_FACTOR.* FROM D_INV_RISK_FACTOR JOIN F_PAGE_CASE ON D_INV_RISK_FACTOR.D_INV_RISK_FACTOR_KEY = F_PAGE_CASE.INVESTIGATION_KEY JOIN INVESTIGATION ON F_PAGE_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME IS NULL;'), + ('D_INV_TREATMENT', 'RDB', + 'SELECT D_INV_TREATMENT.* FROM D_INV_TREATMENT JOIN F_PAGE_CASE ON D_INV_TREATMENT.D_INV_TREATMENT_KEY = F_PAGE_CASE.INVESTIGATION_KEY JOIN INVESTIGATION ON F_PAGE_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp;', + 'SELECT TOP(:limit) D_INV_TREATMENT.* FROM D_INV_TREATMENT JOIN F_PAGE_CASE ON D_INV_TREATMENT.D_INV_TREATMENT_KEY = F_PAGE_CASE.INVESTIGATION_KEY JOIN INVESTIGATION ON F_PAGE_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp ORDER BY INVESTIGATION.LAST_CHG_TIME ASC;', + 'SELECT D_INV_TREATMENT.* FROM D_INV_TREATMENT JOIN F_PAGE_CASE ON D_INV_TREATMENT.D_INV_TREATMENT_KEY = F_PAGE_CASE.INVESTIGATION_KEY JOIN INVESTIGATION ON F_PAGE_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME IS NULL;'), + ('D_INV_VACCINATION', 'RDB', + 'SELECT D_INV_VACCINATION.* FROM D_INV_VACCINATION JOIN F_PAGE_CASE ON D_INV_VACCINATION.D_INV_VACCINATION_KEY = F_PAGE_CASE.INVESTIGATION_KEY JOIN INVESTIGATION ON F_PAGE_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp;', + 'SELECT TOP(:limit) D_INV_VACCINATION.* FROM D_INV_VACCINATION JOIN F_PAGE_CASE ON D_INV_VACCINATION.D_INV_VACCINATION_KEY = F_PAGE_CASE.INVESTIGATION_KEY JOIN INVESTIGATION ON F_PAGE_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp ORDER BY INVESTIGATION.LAST_CHG_TIME ASC;', + 'SELECT D_INV_VACCINATION.* FROM D_INV_VACCINATION JOIN F_PAGE_CASE ON D_INV_VACCINATION.D_INV_VACCINATION_KEY = F_PAGE_CASE.INVESTIGATION_KEY JOIN INVESTIGATION ON F_PAGE_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME IS NULL;'), + ('F_INTERVIEW_CASE', 'RDB', + 'SELECT F_INTERVIEW_CASE.* FROM F_INTERVIEW_CASE JOIN INVESTIGATION ON F_INTERVIEW_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY JOIN D_INTERVIEW ON F_INTERVIEW_CASE.D_INTERVIEW_KEY = D_INTERVIEW.D_INTERVIEW_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp OR D_INTERVIEW.ADD_TIME >= :timestamp OR D_INTERVIEW.LAST_CHG_TIME >= :timestamp;', + 'SELECT TOP(:limit) F_INTERVIEW_CASE.* FROM F_INTERVIEW_CASE JOIN INVESTIGATION ON F_INTERVIEW_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY JOIN D_INTERVIEW ON F_INTERVIEW_CASE.D_INTERVIEW_KEY = D_INTERVIEW.D_INTERVIEW_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp OR D_INTERVIEW.ADD_TIME >= :timestamp OR D_INTERVIEW.LAST_CHG_TIME >= :timestamp ORDER BY CASE WHEN D_INTERVIEW.ADD_TIME >= :timestamp THEN D_INTERVIEW.ADD_TIME ELSE D_INTERVIEW.LAST_CHG_TIME END ASC;', + NULL), + ('F_PAGE_CASE', 'RDB', + 'SELECT F_PAGE_CASE.* FROM F_PAGE_CASE JOIN INVESTIGATION ON F_PAGE_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp;', + 'SELECT TOP(:limit) F_PAGE_CASE.* FROM F_PAGE_CASE JOIN INVESTIGATION ON F_PAGE_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp ORDER BY INVESTIGATION.LAST_CHG_TIME ASC;', + NULL), + ('F_STD_PAGE_CASE', 'RDB', + 'SELECT F_STD_PAGE_CASE.* FROM F_STD_PAGE_CASE JOIN INVESTIGATION ON F_STD_PAGE_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp;', + 'SELECT TOP(:limit) F_STD_PAGE_CASE.* FROM F_STD_PAGE_CASE JOIN INVESTIGATION ON F_STD_PAGE_CASE.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp ORDER BY INVESTIGATION.LAST_CHG_TIME ASC;', + NULL), + ('TREATMENT', 'RDB', + 'SELECT TREATMENT.* FROM TREATMENT JOIN TREATMENT_EVENT ON TREATMENT.TREATMENT_KEY = TREATMENT_EVENT.TREATMENT_KEY JOIN INVESTIGATION ON TREATMENT_EVENT.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp;', + 'SELECT TOP(:limit) TREATMENT.* FROM TREATMENT JOIN TREATMENT_EVENT ON TREATMENT.TREATMENT_KEY = TREATMENT_EVENT.TREATMENT_KEY JOIN INVESTIGATION ON TREATMENT_EVENT.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp ORDER BY INVESTIGATION.LAST_CHG_TIME ASC;', + 'SELECT TREATMENT.* FROM TREATMENT JOIN TREATMENT_EVENT ON TREATMENT.TREATMENT_KEY = TREATMENT_EVENT.TREATMENT_KEY JOIN INVESTIGATION ON TREATMENT_EVENT.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME IS NULL;'), + ('TREATMENT_EVENT', 'RDB', + 'SELECT TREATMENT_EVENT.* FROM TREATMENT_EVENT JOIN INVESTIGATION ON TREATMENT_EVENT.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp;', + 'SELECT TOP(:limit) TREATMENT_EVENT.* FROM TREATMENT_EVENT JOIN INVESTIGATION ON TREATMENT_EVENT.INVESTIGATION_KEY = INVESTIGATION.INVESTIGATION_KEY WHERE INVESTIGATION.LAST_CHG_TIME >= :timestamp ORDER BY INVESTIGATION.LAST_CHG_TIME ASC;', + NULL), ('D_ORGANIZATION', 'RDB', 'SELECT * FROM D_ORGANIZATION WHERE D_ORGANIZATION.ORGANIZATION_LAST_CHANGE_TIME >= :timestamp;', - 'SELECT TOP(:limit) * FROM D_ORGANIZATION WHERE D_ORGANIZATION.ORGANIZATION_LAST_CHANGE_TIME >= :timestamp ORDER BY D_ORGANIZATION.ORGANIZATION_LAST_CHANGE_TIME ASC;'), + 'SELECT TOP(:limit) * FROM D_ORGANIZATION WHERE D_ORGANIZATION.ORGANIZATION_LAST_CHANGE_TIME >= :timestamp ORDER BY D_ORGANIZATION.ORGANIZATION_LAST_CHANGE_TIME ASC;', + 'SELECT * FROM D_ORGANIZATION WHERE D_ORGANIZATION.ORGANIZATION_LAST_CHANGE_TIME IS NULL;'), ('D_PROVIDER', 'RDB', 'SELECT * FROM D_PROVIDER WHERE D_PROVIDER.PROVIDER_LAST_CHANGE_TIME >= :timestamp;', - 'SELECT TOP(:limit) * FROM D_PROVIDER WHERE D_PROVIDER.PROVIDER_LAST_CHANGE_TIME >= :timestamp ORDER BY D_PROVIDER.PROVIDER_LAST_CHANGE_TIME ASC;'); + 'SELECT TOP(:limit) * FROM D_PROVIDER WHERE D_PROVIDER.PROVIDER_LAST_CHANGE_TIME >= :timestamp ORDER BY D_PROVIDER.PROVIDER_LAST_CHANGE_TIME ASC;', + 'SELECT * FROM D_PROVIDER WHERE D_PROVIDER.PROVIDER_LAST_CHANGE_TIME IS NULL;'), + ('NOTIFICATION', 'RDB', + 'SELECT N.* FROM NOTIFICATION N WHERE (N.NOTIFICATION_LAST_CHANGE_TIME >= :timestamp) OR (N.NOTIFICATION_LAST_CHANGE_TIME IS NULL AND EXISTS (SELECT 1 FROM NOTIFICATION_EVENT NE JOIN INVESTIGATION I ON NE.INVESTIGATION_KEY = I.INVESTIGATION_KEY WHERE NE.NOTIFICATION_KEY = N.NOTIFICATION_KEY AND I.LAST_CHG_TIME >= :timestamp));', + 'SELECT TOP(:limit) N.* FROM NOTIFICATION N WHERE (N.NOTIFICATION_LAST_CHANGE_TIME >= :timestamp) OR (N.NOTIFICATION_LAST_CHANGE_TIME IS NULL AND EXISTS (SELECT 1 FROM NOTIFICATION_EVENT NE JOIN INVESTIGATION I ON NE.INVESTIGATION_KEY = I.INVESTIGATION_KEY WHERE NE.NOTIFICATION_KEY = N.NOTIFICATION_KEY AND I.LAST_CHG_TIME >= :timestamp)) ORDER BY N.NOTIFICATION_LAST_CHANGE_TIME ASC;', + NULL), + ('CONDITION_CODE', 'SRTE', 'SELECT * FROM CONDITION_CODE', NULL, NULL), + ('CODE_TO_CODE', 'SRTE', 'SELECT * FROM CODE_TO_CODE', NULL, NULL); diff --git a/nnd-data-exchange-service/src/main/resources/query/select/select.sql b/nnd-data-exchange-service/src/main/resources/query/select/select.sql index 5e865f5..8c3bd68 100644 --- a/nnd-data-exchange-service/src/main/resources/query/select/select.sql +++ b/nnd-data-exchange-service/src/main/resources/query/select/select.sql @@ -320,9 +320,12 @@ WHERE - - - +SELECT N.* FROM NOTIFICATION N + WHERE (N.NOTIFICATION_LAST_CHANGE_TIME >= :timestamp) + OR (N.NOTIFICATION_LAST_CHANGE_TIME IS NULL AND EXISTS + (SELECT 1 FROM NOTIFICATION_EVENT NE JOIN INVESTIGATION I + ON NE.INVESTIGATION_KEY = I.INVESTIGATION_KEY WHERE NE.NOTIFICATION_KEY = N.NOTIFICATION_KEY + AND I.LAST_CHG_TIME >= :timestamp)); diff --git a/nnd-data-exchange-service/src/test/java/gov/cdc/nnddataexchangeservice/controller/DataExchangeControllerTest.java b/nnd-data-exchange-service/src/test/java/gov/cdc/nnddataexchangeservice/controller/DataExchangeControllerTest.java index 7cac5d6..40a275e 100644 --- a/nnd-data-exchange-service/src/test/java/gov/cdc/nnddataexchangeservice/controller/DataExchangeControllerTest.java +++ b/nnd-data-exchange-service/src/test/java/gov/cdc/nnddataexchangeservice/controller/DataExchangeControllerTest.java @@ -1,5 +1,6 @@ package gov.cdc.nnddataexchangeservice.controller; +import com.google.gson.Gson; import gov.cdc.nnddataexchangeservice.exception.DataExchangeException; import gov.cdc.nnddataexchangeservice.service.interfaces.IDataExchangeGenericService; import gov.cdc.nnddataexchangeservice.service.interfaces.IDataExchangeService; @@ -12,6 +13,8 @@ import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; +import java.io.IOException; + import static org.junit.jupiter.api.Assertions.*; import static org.mockito.Mockito.*; @@ -32,7 +35,28 @@ void setUp() { } @Test - void exchangingData_WithValidParams_ReturnsDataExchangeModel() throws DataExchangeException { + void exchangingData_WithValidParams_ReturnsDataExchangeModel() throws DataExchangeException, IOException { + String cnStatusTime = "2024-07-11"; + String transportStatusTime = "2024-07-12"; + String netssTime = "2024-07-13"; + String statusCd = "COMPLETE"; + String limit = "10"; + + DataExchangeModel dataExchangeModel = new DataExchangeModel(); + Gson gson = new Gson(); + String data = gson.toJson(dataExchangeModel); + when(dataExchangeService.getDataForOnPremExchanging(anyString(), anyString(), anyString(), anyString(), anyInt(), anyBoolean())) + .thenReturn(data); + + ResponseEntity response = dataExchangeController.exchangingData(cnStatusTime, transportStatusTime, netssTime, statusCd, limit, "false"); + + assertNotNull(response); + assertEquals(HttpStatus.OK, response.getStatusCode()); + } + + + @Test + void exchangingData_WithValidParams_ReturnsDataExchangeModel_True() throws DataExchangeException, IOException { String cnStatusTime = "2024-07-11"; String transportStatusTime = "2024-07-12"; String netssTime = "2024-07-13"; @@ -40,14 +64,15 @@ void exchangingData_WithValidParams_ReturnsDataExchangeModel() throws DataExchan String limit = "10"; DataExchangeModel dataExchangeModel = new DataExchangeModel(); - when(dataExchangeService.getDataForOnPremExchanging(anyString(), anyString(), anyString(), anyString(), anyInt())) - .thenReturn(dataExchangeModel); + Gson gson = new Gson(); + String data = gson.toJson(dataExchangeModel); + when(dataExchangeService.getDataForOnPremExchanging(anyString(), anyString(), anyString(), anyString(), anyInt(), anyBoolean())) + .thenReturn(data); - ResponseEntity response = dataExchangeController.exchangingData(cnStatusTime, transportStatusTime, netssTime, statusCd, limit); + ResponseEntity response = dataExchangeController.exchangingData(cnStatusTime, transportStatusTime, netssTime, statusCd, limit, "true"); assertNotNull(response); assertEquals(HttpStatus.OK, response.getStatusCode()); - assertEquals(dataExchangeModel, response.getBody()); } @Test @@ -59,7 +84,7 @@ void exchangingData_WithMissingStatusCd_ThrowsDataExchangeException() { String limit = "10"; assertThrows(DataExchangeException.class, () -> - dataExchangeController.exchangingData(cnStatusTime, transportStatusTime, netssTime, statusCd, limit)); + dataExchangeController.exchangingData(cnStatusTime, transportStatusTime, netssTime, statusCd, limit, "false")); } @Test @@ -68,11 +93,12 @@ void exchangingGenericData_WithValidParams_ReturnsBase64CompressedData() throws String timestamp = "2024-07-11"; String limit = "10"; String base64CompressedData = "mockBase64Data"; + String nullApply = "true"; - when(dataExchangeGenericService.getGenericDataExchange(anyString(), anyString(), anyInt())) + when(dataExchangeGenericService.getGenericDataExchange(anyString(), anyString(), anyInt(), anyBoolean())) .thenReturn(base64CompressedData); - ResponseEntity response = dataExchangeController.exchangingData(tableName, timestamp, limit); + ResponseEntity response = dataExchangeController.exchangingData(tableName, timestamp, limit, nullApply); assertNotNull(response); assertEquals(HttpStatus.OK, response.getStatusCode()); @@ -82,7 +108,7 @@ void exchangingGenericData_WithValidParams_ReturnsBase64CompressedData() throws @Test - void decodeAndDecompress_WithValidTableName_ReturnsDecodedData() { + void decodeAndDecompress_WithValidTableName_ReturnsDecodedData() throws DataExchangeException { String tableName = "test_table"; String decodedData = "mockDecodedData"; diff --git a/nnd-data-exchange-service/src/test/java/gov/cdc/nnddataexchangeservice/service/DataExchangeGenericServiceTest.java b/nnd-data-exchange-service/src/test/java/gov/cdc/nnddataexchangeservice/service/DataExchangeGenericServiceTest.java index a39e29d..130a49f 100644 --- a/nnd-data-exchange-service/src/test/java/gov/cdc/nnddataexchangeservice/service/DataExchangeGenericServiceTest.java +++ b/nnd-data-exchange-service/src/test/java/gov/cdc/nnddataexchangeservice/service/DataExchangeGenericServiceTest.java @@ -51,7 +51,7 @@ void getGenericDataExchange_WithMissingTable_ThrowsDataExchangeException() { when(dataSyncConfigRepository.findById(tableName)).thenReturn(Optional.empty()); assertThrows(DataExchangeException.class, () -> - dataExchangeGenericService.getGenericDataExchange(tableName, timeStamp, limit)); + dataExchangeGenericService.getGenericDataExchange(tableName, timeStamp, limit, false)); } @Test @@ -75,7 +75,7 @@ void getGenericDataExchange_1() throws DataExchangeException { when(dataSyncConfigRepository.findById(tableName)).thenReturn(Optional.of(config)); when(jdbcTemplate.queryForList(any())).thenReturn(data); when(gson.toJson(data)).thenReturn("TEST"); - var res = dataExchangeGenericService.getGenericDataExchange(tableName, timeStamp, limit); + var res = dataExchangeGenericService.getGenericDataExchange(tableName, timeStamp, limit, false); assertNotNull(res); } @Test @@ -99,7 +99,7 @@ void getGenericDataExchange_2() throws DataExchangeException { when(dataSyncConfigRepository.findById(tableName)).thenReturn(Optional.of(config)); when(jdbcTemplate.queryForList(any())).thenReturn(data); when(gson.toJson(data)).thenReturn("TEST"); - var res = dataExchangeGenericService.getGenericDataExchange(tableName, timeStamp, limit); + var res = dataExchangeGenericService.getGenericDataExchange(tableName, timeStamp, limit, false); assertNotNull(res); } @@ -125,12 +125,39 @@ void getGenericDataExchange_3() throws DataExchangeException { when(dataSyncConfigRepository.findById(tableName)).thenReturn(Optional.of(config)); when(jdbcTemplate.queryForList(any())).thenReturn(data); when(gson.toJson(data)).thenReturn("TEST"); - var res = dataExchangeGenericService.getGenericDataExchange(tableName, timeStamp, limit); + var res = dataExchangeGenericService.getGenericDataExchange(tableName, timeStamp, limit, false); assertNotNull(res); } @Test - void decodeAndDecompress_WithValidData_ReturnsDecompressedJson() throws IOException { + void getGenericDataExchange_4() throws DataExchangeException { + String tableName = "table"; + String timeStamp = "2024-01-01"; + + DataSyncConfig config = new DataSyncConfig(); + config.setQuery("SELECT * FROM HERE :timestamp "); + config.setQueryWithLimit("SELECT * FROM HERE :timestamp :limit"); + config.setQueryWithNullTimeStamp("SELECT * FROM HERE NULL"); + config.setTableName(tableName); + config.setSourceDb("SRTE"); + int limit = 10; + + List> data = new ArrayList<>(); + Map map = new HashMap<>(); + map.put("CONDITION", "RDB"); + map.put("SELECT * FROM CONDITION;", null); + map.put("2024-08-19 15:19:49.4830000", "2024-08-19 15:19:49.4830000"); + data.add(map); + + when(dataSyncConfigRepository.findById(tableName)).thenReturn(Optional.of(config)); + when(jdbcTemplate.queryForList(any())).thenReturn(data); + when(gson.toJson(data)).thenReturn("TEST"); + var res = dataExchangeGenericService.getGenericDataExchange(tableName, timeStamp, limit, true); + assertNotNull(res); + } + + @Test + void decodeAndDecompress_WithValidData_ReturnsDecompressedJson() throws DataExchangeException, IOException { String json = "[{\"key\":\"value\"}]"; byte[] compressedData; diff --git a/nnd-data-exchange-service/src/test/java/gov/cdc/nnddataexchangeservice/service/DataExchangeServiceTest.java b/nnd-data-exchange-service/src/test/java/gov/cdc/nnddataexchangeservice/service/DataExchangeServiceTest.java index 1a8c2f0..0a30e71 100644 --- a/nnd-data-exchange-service/src/test/java/gov/cdc/nnddataexchangeservice/service/DataExchangeServiceTest.java +++ b/nnd-data-exchange-service/src/test/java/gov/cdc/nnddataexchangeservice/service/DataExchangeServiceTest.java @@ -1,6 +1,7 @@ package gov.cdc.nnddataexchangeservice.service; +import com.google.gson.Gson; import gov.cdc.nnddataexchangeservice.exception.DataExchangeException; import gov.cdc.nnddataexchangeservice.service.interfaces.ICNTransportQOutService; import gov.cdc.nnddataexchangeservice.service.interfaces.INetsstTransportService; @@ -18,7 +19,8 @@ import java.util.Collections; import java.util.List; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.Mockito.*; class DataExchangeServiceTest { @@ -32,6 +34,9 @@ class DataExchangeServiceTest { @Mock private ICNTransportQOutService cnTransportQOutService; + @Mock + private Gson gson; + @InjectMocks private DataExchangeService dataExchangeService; @@ -50,17 +55,42 @@ void testGetDataForOnPremExchanging() throws Exception { List cnTransportQOutDtos = Collections.singletonList(new CNTransportQOutDto()); List transportQOutDtos = Collections.singletonList(new TransportQOutDto()); List netssTransportQOutDtos = Collections.singletonList(new NETSSTransportQOutDto()); + when(gson.toJson(any(DataExchangeModel.class))).thenReturn("Test"); + when(cnTransportQOutService.getTransportData(statusCd, cnStatusTime, 0)).thenReturn(cnTransportQOutDtos); + when(transportQOutService.getTransportData(transportTime, 0)).thenReturn(transportQOutDtos); + when(netsstTransportService.getNetssTransportData(netssTime, 0)).thenReturn(netssTransportQOutDtos); + + + String result = dataExchangeService.getDataForOnPremExchanging(cnStatusTime, transportTime, netssTime, statusCd, 0, false); + + assertNotNull(result); + + verify(cnTransportQOutService, times(1)).getTransportData(statusCd, cnStatusTime, 0); + verify(transportQOutService, times(1)).getTransportData(transportTime, 0); + verify(netsstTransportService, times(1)).getNetssTransportData(netssTime, 0); + } + + @Test + void testGetDataForOnPremExchanging_True() throws Exception { + String cnStatusTime = "2023-07-30 10:00:00.000"; + String transportTime = "2023-07-30 10:00:00.000"; + String netssTime = "2023-07-30 10:00:00.000"; + String statusCd = "status"; + + List cnTransportQOutDtos = Collections.singletonList(new CNTransportQOutDto()); + List transportQOutDtos = Collections.singletonList(new TransportQOutDto()); + List netssTransportQOutDtos = Collections.singletonList(new NETSSTransportQOutDto()); + + when(cnTransportQOutService.getTransportData(statusCd, cnStatusTime, 0)).thenReturn(cnTransportQOutDtos); when(transportQOutService.getTransportData(transportTime, 0)).thenReturn(transportQOutDtos); when(netsstTransportService.getNetssTransportData(netssTime, 0)).thenReturn(netssTransportQOutDtos); + when(gson.toJson(any(DataExchangeModel.class))).thenReturn("Test"); - DataExchangeModel result = dataExchangeService.getDataForOnPremExchanging(cnStatusTime, transportTime, netssTime, statusCd, 0); + String result = dataExchangeService.getDataForOnPremExchanging(cnStatusTime, transportTime, netssTime, statusCd, 0, true); assertNotNull(result); - assertEquals(cnTransportQOutDtos.size(), result.getCountCnTransport()); - assertEquals(transportQOutDtos.size(), result.getCountTransport()); - assertEquals(netssTransportQOutDtos.size(), result.getCountNetssTransport()); verify(cnTransportQOutService, times(1)).getTransportData(statusCd, cnStatusTime, 0); verify(transportQOutService, times(1)).getTransportData(transportTime, 0); @@ -76,7 +106,7 @@ void testGetDataForOnPremExchanging_Exception() throws Exception { when(cnTransportQOutService.getTransportData(statusCd, cnStatusTime, 0)).thenThrow(new DataExchangeException("Exception")); - assertThrows(DataExchangeException.class, () -> dataExchangeService.getDataForOnPremExchanging(cnStatusTime, transportTime, netssTime, statusCd, 0)); + assertThrows(DataExchangeException.class, () -> dataExchangeService.getDataForOnPremExchanging(cnStatusTime, transportTime, netssTime, statusCd, 0, false)); verify(cnTransportQOutService, times(1)).getTransportData(statusCd, cnStatusTime, 0); verify(transportQOutService, times(0)).getTransportData(transportTime,0); diff --git a/nnd-data-poll-service/execute_script/nnd_data_poll_service.cmd b/nnd-data-poll-service/execute_script/nnd_data_poll_service.cmd index 20c0de2..03c7cc0 100644 --- a/nnd-data-poll-service/execute_script/nnd_data_poll_service.cmd +++ b/nnd-data-poll-service/execute_script/nnd_data_poll_service.cmd @@ -1,6 +1,9 @@ @echo off set NBS_NND_CRON=* * * * * * set NBS_NND_CRON_TIME_ZONE=UTC +set NND_FULL_LOAD=false +set NND_PULL_LIMIT=0 +set NND_INSERT_LIMIT=1000 set NND_DE_CLIENT_ID=nnd-keycloak-client set NND_DE_DE=/api/nnd/data-exchange set NND_DE_SECRET=bpVTppDam4sxXt4hfgm5hZ6Rteponjb9 diff --git a/nnd-data-poll-service/execute_script/nnd_data_poll_service.sh b/nnd-data-poll-service/execute_script/nnd_data_poll_service.sh index 3efd422..471018b 100755 --- a/nnd-data-poll-service/execute_script/nnd_data_poll_service.sh +++ b/nnd-data-poll-service/execute_script/nnd_data_poll_service.sh @@ -1,6 +1,9 @@ #!/bin/bash export NBS_NND_CRON="* * * * * *" export NBS_NND_CRON_TIME_ZONE="UTC" +export NND_FULL_LOAD="false" +export NND_PULL_LIMIT="0" +export NND_INSERT_LIMIT="1000" export NND_DE_CLIENT_ID="nnd-keycloak-client" export NND_DE_DE="/api/nnd/data-exchange" export NND_DE_SECRET="bpVTppDam4sxXt4hfgm5hZ6Rteponjb9" diff --git a/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/service/CNTransportQOutService.java b/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/service/CNTransportQOutService.java index 95f5f61..6b05b42 100644 --- a/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/service/CNTransportQOutService.java +++ b/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/service/CNTransportQOutService.java @@ -21,7 +21,10 @@ public class CNTransportQOutService implements ICNTransportQOutService { private static final String TIMESTAMP_FORMAT = "yyyy-MM-dd HH:mm:ss.SSS"; @Value("${io.finalLocation}") - private String fileLocation; + protected String fileLocation; + + @Value("${nnd.insertLimit}") + protected Integer insertLimit = 1000; public CNTransportQOutService(CNTransportQOutRepository cnTransportQOutRepository, @@ -55,9 +58,8 @@ public void saveDataExchange(List transportQOutDtoList) thro cnTransportQOutList.add(transportQOut); } - int batchSize = 10; - for (int i = 0; i < cnTransportQOutList.size(); i += batchSize) { - int end = Math.min(i + batchSize, cnTransportQOutList.size()); + for (int i = 0; i < cnTransportQOutList.size(); i += insertLimit) { + int end = Math.min(i + insertLimit, cnTransportQOutList.size()); List batch = cnTransportQOutList.subList(i, end); try { diff --git a/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/service/DataHandlingService.java b/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/service/DataHandlingService.java index a8e6967..2430320 100644 --- a/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/service/DataHandlingService.java +++ b/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/service/DataHandlingService.java @@ -1,11 +1,10 @@ package gov.cdc.nnddatapollservice.service; import com.google.gson.Gson; -import com.google.gson.GsonBuilder; import gov.cdc.nnddatapollservice.exception.DataPollException; -import gov.cdc.nnddatapollservice.json_config.ByteArrayDeserializer; import gov.cdc.nnddatapollservice.service.interfaces.*; import gov.cdc.nnddatapollservice.service.model.DataExchangeModel; +import gov.cdc.nnddatapollservice.share.DataSimplification; import org.springframework.beans.factory.annotation.Value; import org.springframework.http.HttpEntity; import org.springframework.http.HttpHeaders; @@ -25,16 +24,21 @@ public class DataHandlingService implements IDataHandlingService { @Value("${data_exchange.clientId}") - private String clientId; + private String clientId = "clientId"; @Value("${data_exchange.secret}") - private String clientSecret; + private String clientSecret = "clientSecret"; @Value("${data_exchange.endpoint_de}") - private String exchangeEndpoint; + protected String exchangeEndpoint; @Value("${nnd.fullLoad}") - protected boolean fullLoadApplied; + protected boolean fullLoadApplied = false; + + @Value("${nnd.pullLimit}") + private String pullLimit = "0"; + + private final Gson gson; private final RestTemplate restTemplate = new RestTemplate(); private final ICNTransportQOutService icnTransportQOutService; @@ -42,10 +46,11 @@ public class DataHandlingService implements IDataHandlingService { private final ITransportQOutService transportQOutService; private final ITokenService tokenService; - public DataHandlingService(ICNTransportQOutService icnTransportQOutService, + public DataHandlingService(Gson gson, ICNTransportQOutService icnTransportQOutService, INetsstTransportService netsstTransportService, ITransportQOutService transportQOutService, ITokenService tokenService) { + this.gson = gson; this.icnTransportQOutService = icnTransportQOutService; this.netsstTransportService = netsstTransportService; this.transportQOutService = transportQOutService; @@ -68,14 +73,13 @@ public void handlingExchangedData() throws DataPollException { String data = callDataExchangeEndpoint(token, param); - persistingExchangeData(data); + var deCompressedData = DataSimplification.decodeAndDecompress(data); + + persistingExchangeData(deCompressedData); } public void persistingExchangeData(String data) throws DataPollException { try { - Gson gson = new GsonBuilder() - .registerTypeAdapter(byte[].class, new ByteArrayDeserializer()) - .create(); var dataExchangeModel = gson.fromJson(data, DataExchangeModel.class); if (!dataExchangeModel.getCnTransportQOutDtoList().isEmpty()) { @@ -110,6 +114,8 @@ protected String callDataExchangeEndpoint(String token, Map para headers.setBearerAuth(token); headers.add("clientid", clientId); headers.add("clientsecret", clientSecret); + headers.add("compress", "true"); + headers.add("limit", pullLimit); HttpEntity entity = new HttpEntity<>(headers); MultiValueMap multiValueParams = new LinkedMultiValueMap<>(); for (Map.Entry entry : params.entrySet()) { diff --git a/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/service/NetsstTransportService.java b/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/service/NetsstTransportService.java index f2d9ade..7ffe28c 100644 --- a/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/service/NetsstTransportService.java +++ b/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/service/NetsstTransportService.java @@ -23,6 +23,9 @@ public class NetsstTransportService implements INetsstTransportService { @Value("${io.finalLocation}") private String fileLocation; + @Value("${nnd.insertLimit}") + private Integer insertLimit = 1000; + public NetsstTransportService(NETSSTransportQOutRepository netssTransportQOutRepository, IErrorHandlingService errorHandlingService) { this.netssTransportQOutRepository = netssTransportQOutRepository; this.errorHandlingService = errorHandlingService; @@ -53,9 +56,8 @@ public void saveDataExchange(List transportQOutDtoList) t cnTransportQOutList.add(transportQOut); } - int batchSize = 10; - for (int i = 0; i < cnTransportQOutList.size(); i += batchSize) { - int end = Math.min(i + batchSize, cnTransportQOutList.size()); + for (int i = 0; i < cnTransportQOutList.size(); i += insertLimit) { + int end = Math.min(i + insertLimit, cnTransportQOutList.size()); List batch = cnTransportQOutList.subList(i, end); try { diff --git a/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/service/TransportQOutService.java b/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/service/TransportQOutService.java index 81b03c9..45709a9 100644 --- a/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/service/TransportQOutService.java +++ b/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/service/TransportQOutService.java @@ -20,6 +20,9 @@ public class TransportQOutService implements ITransportQOutService { @Value("${io.finalLocation}") private String fileLocation; + @Value("${nnd.insertLimit}") + private Integer insertLimit = 1000; + public TransportQOutService(TransportQOutRepository transportQOutRepository, IErrorHandlingService errorHandlingService) { this.transportQOutRepository = transportQOutRepository; @@ -54,9 +57,8 @@ public void saveDataExchange(List transportQOutDtoList) throws cnTransportQOutList.add(transportQOut); } - int batchSize = 10; - for (int i = 0; i < cnTransportQOutList.size(); i += batchSize) { - int end = Math.min(i + batchSize, cnTransportQOutList.size()); + for (int i = 0; i < cnTransportQOutList.size(); i += insertLimit) { + int end = Math.min(i + insertLimit, cnTransportQOutList.size()); List batch = cnTransportQOutList.subList(i, end); try { diff --git a/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/share/DataSimplification.java b/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/share/DataSimplification.java new file mode 100644 index 0000000..fb90a02 --- /dev/null +++ b/nnd-data-poll-service/src/main/java/gov/cdc/nnddatapollservice/share/DataSimplification.java @@ -0,0 +1,40 @@ +package gov.cdc.nnddatapollservice.share; + +import gov.cdc.nnddatapollservice.exception.DataPollException; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.zip.GZIPInputStream; + +public class DataSimplification { + private DataSimplification() { + + } + + @SuppressWarnings("java:S1141") + public static String decodeAndDecompress(String base64EncodedData) { + try { + byte[] compressedData = Base64.getDecoder().decode(base64EncodedData); + + try (ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(compressedData); + GZIPInputStream gzipInputStream = new GZIPInputStream(byteArrayInputStream); + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) { + + byte[] buffer = new byte[1024]; + int len; + while ((len = gzipInputStream.read(buffer)) > 0) { + byteArrayOutputStream.write(buffer, 0, len); + } + + return byteArrayOutputStream.toString(StandardCharsets.UTF_8); + } catch (Exception e) { + throw new DataPollException(e.getMessage()); + } + } catch (Exception e) { + // Return raw string - un decoded if error occur + return base64EncodedData; + } + } +} diff --git a/nnd-data-poll-service/src/main/resources/application.yaml b/nnd-data-poll-service/src/main/resources/application.yaml index 7e83278..f89dae9 100644 --- a/nnd-data-poll-service/src/main/resources/application.yaml +++ b/nnd-data-poll-service/src/main/resources/application.yaml @@ -12,6 +12,9 @@ data_exchange: endpoint_de: ${NND_DE_URL}${NND_DE_DE} nnd: fullLoad: ${NND_FULL_LOAD:false} + pullLimit: ${NND_PULL_LIMIT:0} + insertLimit: ${NND_INSERT_LIMIT:1000} + spring: application: diff --git a/nnd-data-poll-service/src/test/java/gov/cdc/nnddatapollservice/service/DataHandlingServiceTest.java b/nnd-data-poll-service/src/test/java/gov/cdc/nnddatapollservice/service/DataHandlingServiceTest.java index e0d1be9..0914c11 100644 --- a/nnd-data-poll-service/src/test/java/gov/cdc/nnddatapollservice/service/DataHandlingServiceTest.java +++ b/nnd-data-poll-service/src/test/java/gov/cdc/nnddatapollservice/service/DataHandlingServiceTest.java @@ -1,23 +1,32 @@ package gov.cdc.nnddatapollservice.service; +import com.google.gson.Gson; import gov.cdc.nnddatapollservice.exception.DataPollException; import gov.cdc.nnddatapollservice.service.interfaces.ICNTransportQOutService; import gov.cdc.nnddatapollservice.service.interfaces.INetsstTransportService; import gov.cdc.nnddatapollservice.service.interfaces.ITokenService; import gov.cdc.nnddatapollservice.service.interfaces.ITransportQOutService; +import gov.cdc.nnddatapollservice.service.model.DataExchangeModel; +import gov.cdc.nnddatapollservice.service.model.dto.CNTransportQOutDto; +import gov.cdc.nnddatapollservice.service.model.dto.NETSSTransportQOutDto; +import gov.cdc.nnddatapollservice.service.model.dto.TransportQOutDto; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.MockitoAnnotations; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpMethod; +import org.springframework.http.ResponseEntity; import org.springframework.web.client.RestTemplate; import java.lang.reflect.Field; +import java.net.URI; +import java.util.List; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.anyList; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.*; public class DataHandlingServiceTest { @@ -36,6 +45,9 @@ public class DataHandlingServiceTest { @Mock private RestTemplate restTemplate; + @Mock + private Gson gson; + @InjectMocks private DataHandlingService dataHandlingService; @@ -50,17 +62,6 @@ void setUp() throws NoSuchFieldException, IllegalAccessException { } - @Test - void testPersistingExchangeData_Success() throws DataPollException { - String data = "{\"cnTransportQOutDtoList\":[], \"transportQOutDtoList\":[], \"netssTransportQOutDtoList\":[]}"; - - dataHandlingService.persistingExchangeData(data); - - verify(icnTransportQOutService, never()).saveDataExchange(anyList()); - verify(transportQOutService, never()).saveDataExchange(anyList()); - verify(netsstTransportService, never()).saveDataExchange(anyList()); - } - @Test void testPersistingExchangeData_Exception() throws DataPollException { String data = "invalid json"; @@ -91,4 +92,78 @@ void testTruncatingDataForFullLoadingFalse() { verify(netsstTransportService, never()).truncatingData(); } + + @Test + void testHandleData1() throws DataPollException { + dataHandlingService.exchangeEndpoint = "http://ip.jsontest.com/"; + when(icnTransportQOutService.getMaxTimestamp()).thenReturn("2020-01-01"); + when(transportQOutService.getMaxTimestamp()).thenReturn("2020-01-01"); + when(netsstTransportService.getMaxTimestamp()).thenReturn("2020-01-01"); + when(tokenService.getToken()).thenReturn("Whatever"); + + ResponseEntity mockResponse = ResponseEntity.ok("Mock Response Body"); + when(restTemplate.exchange(any(URI.class), eq(HttpMethod.GET), any(HttpEntity.class), eq(String.class))) + .thenReturn(mockResponse); + + DataExchangeModel dataModel = new DataExchangeModel(); + when(gson.fromJson(anyString(), any(Class.class))).thenReturn(dataModel); + + dataHandlingService.handlingExchangedData(); + + verify(tokenService, times(1)).getToken(); + } + + @Test + void testHandleData2() throws DataPollException { + dataHandlingService.exchangeEndpoint = "http://ip.jsontest.com/"; + when(icnTransportQOutService.getMaxTimestamp()).thenReturn("2020-01-01"); + when(transportQOutService.getMaxTimestamp()).thenReturn("2020-01-01"); + when(netsstTransportService.getMaxTimestamp()).thenReturn("2020-01-01"); + when(tokenService.getToken()).thenReturn("Whatever"); + + ResponseEntity mockResponse = ResponseEntity.ok("Mock Response Body"); + when(restTemplate.exchange(any(URI.class), eq(HttpMethod.GET), any(HttpEntity.class), eq(String.class))) + .thenReturn(mockResponse); + + DataExchangeModel dataModel = new DataExchangeModel(); + dataModel.setCnTransportQOutDtoList(List.of(new CNTransportQOutDto())); + dataModel.setTransportQOutDtoList(List.of(new TransportQOutDto())); + dataModel.setNetssTransportQOutDtoList(List.of(new NETSSTransportQOutDto())); + + when(gson.fromJson(anyString(), any(Class.class))).thenReturn(dataModel); + + dataHandlingService.handlingExchangedData(); + + verify(tokenService, times(1)).getToken(); + } + + + @Test + void testHandleData3() throws DataPollException { + // Arrange + dataHandlingService.exchangeEndpoint = "http://ip.jsontest.com/"; + when(icnTransportQOutService.getMaxTimestamp()).thenReturn("2020-01-01"); + when(transportQOutService.getMaxTimestamp()).thenReturn("2020-01-01"); + when(netsstTransportService.getMaxTimestamp()).thenReturn("2020-01-01"); + when(tokenService.getToken()).thenReturn("Whatever"); + + doThrow(new RuntimeException("Whatever")).when(transportQOutService).saveDataExchange(any()); + + ResponseEntity mockResponse = ResponseEntity.ok("Mock Response Body"); + when(restTemplate.exchange(any(URI.class), eq(HttpMethod.GET), any(HttpEntity.class), eq(String.class))) + .thenReturn(mockResponse); + + DataExchangeModel dataModel = new DataExchangeModel(); + dataModel.setCnTransportQOutDtoList(List.of(new CNTransportQOutDto())); + dataModel.setTransportQOutDtoList(List.of(new TransportQOutDto())); + dataModel.setNetssTransportQOutDtoList(List.of(new NETSSTransportQOutDto())); + + when(gson.fromJson(anyString(), any(Class.class))).thenReturn(dataModel); + + // Act & Assert + assertThrows(DataPollException.class, () -> dataHandlingService.handlingExchangedData()); + + // Verify that the tokenService was called once + verify(tokenService, times(1)).getToken(); + } } diff --git a/nnd-data-poll-service/src/test/java/gov/cdc/nnddatapollservice/service/NetsstTransportServiceTest.java b/nnd-data-poll-service/src/test/java/gov/cdc/nnddatapollservice/service/NetsstTransportServiceTest.java index 51f6ca6..092908c 100644 --- a/nnd-data-poll-service/src/test/java/gov/cdc/nnddatapollservice/service/NetsstTransportServiceTest.java +++ b/nnd-data-poll-service/src/test/java/gov/cdc/nnddatapollservice/service/NetsstTransportServiceTest.java @@ -83,7 +83,7 @@ void testSaveDataExchange_PartialFailure() throws Exception { netsstTransportService.saveDataExchange(transportQOutDtoList); - verify(netssTransportQOutRepository, times(2)).saveAll(anyList()); + verify(netssTransportQOutRepository, times(1)).saveAll(anyList()); verify(netssTransportQOutRepository, never()).flush(); verify(errorHandlingService, atLeastOnce()).dumpBatchToFile(anyList(), anyString(), anyString()); } diff --git a/nnd-data-poll-service/src/test/java/gov/cdc/nnddatapollservice/service/TransportQOutServiceTest.java b/nnd-data-poll-service/src/test/java/gov/cdc/nnddatapollservice/service/TransportQOutServiceTest.java index ef3f27d..4c1368c 100644 --- a/nnd-data-poll-service/src/test/java/gov/cdc/nnddatapollservice/service/TransportQOutServiceTest.java +++ b/nnd-data-poll-service/src/test/java/gov/cdc/nnddatapollservice/service/TransportQOutServiceTest.java @@ -90,7 +90,7 @@ void testSaveDataExchange_PartialFailure() throws DataPollException { transportQOutService.saveDataExchange(transportQOutDtoList); - verify(transportQOutRepository, times(2)).saveAll(anyList()); + verify(transportQOutRepository, times(1)).saveAll(anyList()); verify(transportQOutRepository, never()).flush(); verify(errorHandlingService, atLeastOnce()).dumpBatchToFile(anyList(), anyString(), anyString()); } diff --git a/nnd-data-poll-service/src/test/java/gov/cdc/nnddatapollservice/share/DataSimplificationTest.java b/nnd-data-poll-service/src/test/java/gov/cdc/nnddatapollservice/share/DataSimplificationTest.java new file mode 100644 index 0000000..9960603 --- /dev/null +++ b/nnd-data-poll-service/src/test/java/gov/cdc/nnddatapollservice/share/DataSimplificationTest.java @@ -0,0 +1,46 @@ +package gov.cdc.nnddatapollservice.share; + + +import gov.cdc.nnddatapollservice.exception.DataPollException; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.zip.GZIPOutputStream; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class DataSimplificationTest { + + @Test + void testDecodeAndDecompressSuccess() throws DataPollException { + String originalString = "Test string for compression"; + String encodedCompressedString = compressAndEncode(originalString); + + String result = DataSimplification.decodeAndDecompress(encodedCompressedString); + + assertEquals(originalString, result); + } + + @Test + void testDecodeAndDecompressInvalidBase64() { + String invalidBase64 = "ThisIsNotABase64EncodedString"; + String result = DataSimplification.decodeAndDecompress(invalidBase64); + assertEquals(invalidBase64, result); + } + + + + private String compressAndEncode(String str) throws DataPollException { + try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + GZIPOutputStream gzipOutputStream = new GZIPOutputStream(byteArrayOutputStream)) { + + gzipOutputStream.write(str.getBytes(StandardCharsets.UTF_8)); + gzipOutputStream.finish(); + return Base64.getEncoder().encodeToString(byteArrayOutputStream.toByteArray()); + } catch (Exception e) { + throw new DataPollException("Failed to compress and encode test string"); + } + } +} \ No newline at end of file