query for notification (#19)

* query for notification * ignore list * swapping to parent build * update * update * added support for ODSE DB * moved compress and decode and encode to share class * added compressing and encoding for nnd * added ignore * clean up * clean up * clean up * update sonar scan * support for compressed data in poll servie * update POll service * update * update * update * clean up * update adding null query and support for srte
CDCgov · Aug 28, 2024 · f25cd3f · f25cd3f
1 parent 69a07dc
commit f25cd3f
Show file tree

Hide file tree

Showing 38 changed files with 694 additions and 168 deletions.
diff --git a/.github/workflows/sonar.yaml b/.github/workflows/sonar.yaml
@@ -15,7 +15,7 @@ on:
     paths:
       - "nnd-data-exchange-service/**"
       - "nnd-data-poll-service/**"
-      -  "nnd-message-processor/**"
+      - "netss-message-processor/**"
       - ".github/workflows/sonar.yaml"
 env:
   deployment_env: dev

diff --git a/build.gradle b/build.gradle
@@ -47,5 +47,15 @@ sonarqube {
         property "sonar.projectKey", "CDCgov_NEDSS-NNDSS"
         property "sonar.organization", "cdcgov"
         property "sonar.host.url", "https://sonarcloud.io"
+        property "sonar.exclusions", "**/configuration/**," +
+                "**/exception/**, " +
+                "**/security/**," +
+                "**/repository/**, " +
+                "**/constant/**, " +
+                "**/service/model/** ," +
+                "**/json_config/**, " +
+                "**/NndMessageProcessorApplication.java, " +
+                "**/NndDataExchangeServiceApplication.java, " +
+                "**/NndDataPollServiceApplication.java"
     }
 }
diff --git a/...e-processor/src/main/java/gov/cdc/nndmessageprocessor/NndMessageProcessorApplication.java b/...e-processor/src/main/java/gov/cdc/nndmessageprocessor/NndMessageProcessorApplication.java
@@ -2,7 +2,6 @@
 
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
-import org.springframework.scheduling.annotation.EnableScheduling;
 
 @SpringBootApplication
 //@EnableScheduling

diff --git a/...processor/src/main/java/gov/cdc/nndmessageprocessor/service/MessageProcessingService.java b/...processor/src/main/java/gov/cdc/nndmessageprocessor/service/MessageProcessingService.java
@@ -6,12 +6,10 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Value;
-import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Service;
 
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
-import java.time.ZoneId;
 import java.util.Calendar;
 import java.util.Date;
 import java.util.HashMap;

diff --git a/...message-processor/src/main/java/gov/cdc/nndmessageprocessor/service/NetssCaseService.java b/...message-processor/src/main/java/gov/cdc/nndmessageprocessor/service/NetssCaseService.java
@@ -13,8 +13,10 @@
 import java.io.FileWriter;
 import java.io.IOException;
 import java.io.PrintWriter;
-import java.util.*;
-import java.util.stream.Collectors;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 
 @Service
 public class NetssCaseService implements INetssCaseService {

diff --git a/...essor/src/test/java/gov/cdc/nndmessageprocessor/service/MessageProcessingServiceTest.java b/...essor/src/test/java/gov/cdc/nndmessageprocessor/service/MessageProcessingServiceTest.java
@@ -34,6 +34,8 @@ void setUp() {
         ReflectionTestUtils.setField(messageProcessingService, "prior", true);
     }
 
+
+
     @Test
     void testScheduleDataFetch_Success() throws DataProcessorException {
         messageProcessingService.scheduleDataFetch();

diff --git a/...vice/src/main/java/gov/cdc/nnddataexchangeservice/configuration/OdseDataSourceConfig.java b/...vice/src/main/java/gov/cdc/nnddataexchangeservice/configuration/OdseDataSourceConfig.java
@@ -9,6 +9,7 @@
 import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.Primary;
 import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
+import org.springframework.jdbc.core.JdbcTemplate;
 import org.springframework.orm.jpa.JpaTransactionManager;
 import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean;
 import org.springframework.orm.jpa.vendor.HibernateJpaVendorAdapter;
@@ -75,4 +76,9 @@ public PlatformTransactionManager odseTransactionManager(
             @Qualifier("odseEntityManagerFactory") EntityManagerFactory entityManagerFactory) {
         return new JpaTransactionManager(entityManagerFactory);
     }
+
+    @Bean(name = "odseJdbcTemplate")
+    public JdbcTemplate odseJdbcTemplate(@Qualifier("odseDataSource") DataSource dataSource) {
+        return new JdbcTemplate(dataSource);
+    }
 }
diff --git a/...vice/src/main/java/gov/cdc/nnddataexchangeservice/configuration/SrteDataSourceConfig.java b/...vice/src/main/java/gov/cdc/nnddataexchangeservice/configuration/SrteDataSourceConfig.java
@@ -0,0 +1,86 @@
+package gov.cdc.nnddataexchangeservice.configuration;
+
+import jakarta.persistence.EntityManagerFactory;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.jdbc.DataSourceBuilder;
+import org.springframework.boot.orm.jpa.EntityManagerFactoryBuilder;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Primary;
+import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.orm.jpa.JpaTransactionManager;
+import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean;
+import org.springframework.orm.jpa.vendor.HibernateJpaVendorAdapter;
+import org.springframework.transaction.PlatformTransactionManager;
+import org.springframework.transaction.annotation.EnableTransactionManagement;
+
+import javax.sql.DataSource;
+import java.util.HashMap;
+
+
+@Configuration
+@EnableTransactionManagement
+@EnableJpaRepositories(
+        entityManagerFactoryRef = "srteEntityManagerFactory",
+        transactionManagerRef = "srteTransactionManager",
+        basePackages = {
+                "gov.cdc.nnddataexchangeservice.repository.srte"
+        }
+)
+public class SrteDataSourceConfig {
+    @Value("${spring.datasource.driverClassName}")
+    private String driverClassName;
+
+    @Value("${spring.datasource.srte.url}")
+    private String dbUrl;
+
+    @Value("${spring.datasource.username}")
+    private String dbUserName;
+
+    @Value("${spring.datasource.password}")
+    private String dbUserPassword;
+
+    @Bean(name = "srteDataSource")
+    public DataSource srteDataSource() {
+        DataSourceBuilder dataSourceBuilder = DataSourceBuilder.create();
+
+        dataSourceBuilder.driverClassName(driverClassName);
+        dataSourceBuilder.url(dbUrl);
+        dataSourceBuilder.username(dbUserName);
+        dataSourceBuilder.password(dbUserPassword);
+
+        return dataSourceBuilder.build();
+    }
+
+    @Bean(name = "srteEntityManagerFactoryBuilder")
+    public EntityManagerFactoryBuilder srteEntityManagerFactoryBuilder() {
+        return new EntityManagerFactoryBuilder(new HibernateJpaVendorAdapter(), new HashMap<>(), null);
+    }
+
+    @Primary
+    @Bean(name = "srteEntityManagerFactory")
+    public LocalContainerEntityManagerFactoryBean srteEntityManagerFactory(
+            @Qualifier("srteEntityManagerFactoryBuilder") EntityManagerFactoryBuilder builder,
+            @Qualifier("srteDataSource") DataSource dataSource) {
+        return builder
+                .dataSource(dataSource)
+                .packages("gov.cdc.nnddataexchangeservice.repository.srte.model")
+                .persistenceUnit("srte")
+                .build();
+    }
+
+    @Primary
+    @Bean(name = "srteTransactionManager")
+    public PlatformTransactionManager srteTransactionManager(
+            @Qualifier("srteEntityManagerFactory") EntityManagerFactory entityManagerFactory) {
+        return new JpaTransactionManager(entityManagerFactory);
+    }
+
+
+    @Bean(name = "srteJdbcTemplate")
+    public JdbcTemplate srteJdbcTemplate(@Qualifier("srteDataSource") DataSource dataSource) {
+        return new JdbcTemplate(dataSource);
+    }
+}
diff --git a/...hange-service/src/main/java/gov/cdc/nnddataexchangeservice/constant/DataSyncConstant.java b/...hange-service/src/main/java/gov/cdc/nnddataexchangeservice/constant/DataSyncConstant.java
@@ -0,0 +1,10 @@
+package gov.cdc.nnddataexchangeservice.constant;
+
+public class DataSyncConstant {
+    public static final String DEFAULT_TIME_STAMP = "1753-01-01";
+    public static final String TIME_STAMP_PARAM = ":timestamp";
+    public static final String LIMIT_PARAM = ":limit";
+    public static final Integer BYTE_SIZE = 1024;
+    public static final String DB_RDB = "RDB";
+    public static final String DB_SRTE = "SRTE";
+}
diff --git a/...rvice/src/main/java/gov/cdc/nnddataexchangeservice/controller/DataExchangeController.java b/...rvice/src/main/java/gov/cdc/nnddataexchangeservice/controller/DataExchangeController.java
@@ -4,7 +4,6 @@
 import gov.cdc.nnddataexchangeservice.exception.DataExchangeException;
 import gov.cdc.nnddataexchangeservice.service.interfaces.IDataExchangeGenericService;
 import gov.cdc.nnddataexchangeservice.service.interfaces.IDataExchangeService;
-import gov.cdc.nnddataexchangeservice.service.model.DataExchangeModel;
 import io.swagger.v3.oas.annotations.Operation;
 import io.swagger.v3.oas.annotations.Parameter;
 import io.swagger.v3.oas.annotations.enums.ParameterIn;
@@ -14,6 +13,8 @@
 import org.springframework.http.ResponseEntity;
 import org.springframework.web.bind.annotation.*;
 
+import java.io.IOException;
+
 @RestController
 @SecurityRequirement(name = "bearer-key")
 public class DataExchangeController {
@@ -43,17 +44,24 @@ public DataExchangeController(IDataExchangeService dataExchangeService,
                             schema = @Schema(type = "string"))}
     )
     @GetMapping(path = "/api/nnd/data-exchange")
-    public ResponseEntity<DataExchangeModel> exchangingData(@RequestParam("cnStatusTime") String cnStatusTime,
+    public ResponseEntity<String> exchangingData(@RequestParam("cnStatusTime") String cnStatusTime,
                                                             @RequestParam("transportStatusTime") String transportStatusTime,
                                                             @RequestParam("netssTime") String netssTime,
                                                             @RequestParam("statusCd") String statusCd,
-                                                            @RequestHeader(name = "limit", defaultValue = "0") String limit) throws DataExchangeException {
+                                                            @RequestHeader(name = "limit", defaultValue = "0", required = false) String limit,
+                                                            @RequestHeader(name = "compress", defaultValue = "false", required = false) String compress) throws DataExchangeException, IOException {
         if (statusCd.isEmpty()) {
             throw new DataExchangeException("Status Code is Missing");
         }
 
+
+        boolean compressCheck = false;
+        if (compress.equalsIgnoreCase("true") ) {
+            compressCheck = true;
+        }
+
         int intLimit = Integer.parseInt(limit);
-        return ResponseEntity.ok(dataExchangeService.getDataForOnPremExchanging(cnStatusTime, transportStatusTime,netssTime, statusCd, intLimit));
+        return ResponseEntity.ok(dataExchangeService.getDataForOnPremExchanging(cnStatusTime, transportStatusTime,netssTime, statusCd, intLimit, compressCheck));
     }
 
     @Operation(
@@ -73,14 +81,17 @@ public ResponseEntity<DataExchangeModel> exchangingData(@RequestParam("cnStatusT
     )
     @GetMapping(path = "/api/data-exchange-generic/{tableName}")
     public ResponseEntity<String> exchangingData(@PathVariable String tableName, @RequestParam(required = false) String timestamp,
-                                                 @RequestHeader(name = "limit", defaultValue = "0") String limit) throws DataExchangeException {
+                                                 @RequestHeader(name = "limit", defaultValue = "0") String limit,
+                                                 @RequestHeader(name = "null_allow", defaultValue = "false", required = false) String nulLAllow) throws DataExchangeException {
             int intLimit = Integer.parseInt(limit);
-            var base64CompressedData = dataExchangeGenericService.getGenericDataExchange(tableName, timestamp, intLimit);
+
+            boolean nullApplied = nulLAllow.equalsIgnoreCase("true");
+            var base64CompressedData = dataExchangeGenericService.getGenericDataExchange(tableName, timestamp, intLimit, nullApplied);
             return new ResponseEntity<>(base64CompressedData, HttpStatus.OK);
     }
 
     @PostMapping(path = "/api/data-exchange-generic")
-    public ResponseEntity<String> decodeAndDecompress(@RequestBody String tableName) {
+    public ResponseEntity<String> decodeAndDecompress(@RequestBody String tableName) throws DataExchangeException {
         var val = dataExchangeGenericService.decodeAndDecompress(tableName);
         return new ResponseEntity<>(val, HttpStatus.OK);
     }

diff --git a/...main/java/gov/cdc/nnddataexchangeservice/repository/msg/NETSSTransportQOutRepository.java b/...main/java/gov/cdc/nnddataexchangeservice/repository/msg/NETSSTransportQOutRepository.java
@@ -12,10 +12,10 @@
 
 @Repository
 public interface NETSSTransportQOutRepository extends JpaRepository<NETSSTransportQOut, Long> {
-    @Query("SELECT a FROM NETSSTransportQOut a")
+    @Query("SELECT a FROM NETSSTransportQOut a ORDER BY a.addTime ASC")
     Optional<Collection<NETSSTransportQOut>> findNetssTransport ();
 
-    @Query("SELECT a FROM NETSSTransportQOut a WHERE a.addTime > :recordStatusTime")
+    @Query("SELECT a FROM NETSSTransportQOut a WHERE a.addTime > :recordStatusTime ORDER BY a.addTime ASC")
     Optional<Collection<NETSSTransportQOut>> findNetssTransportByCreationTime (@Param("recordStatusTime") Timestamp recordStatusTime);
 
     @Query(value = "SELECT TOP (:limit) a.* FROM NETSS_TransportQ_out a ORDER BY a.add_time ASC ", nativeQuery = true)

diff --git a/.../src/main/java/gov/cdc/nnddataexchangeservice/repository/msg/TransportQOutRepository.java b/.../src/main/java/gov/cdc/nnddataexchangeservice/repository/msg/TransportQOutRepository.java
@@ -12,10 +12,10 @@
 @Repository
 public interface TransportQOutRepository extends JpaRepository<TransportQOut, Long> {
 
-    @Query("SELECT a FROM TransportQOut a WHERE a.messageCreationTime > :recordStatusTime")
+    @Query("SELECT a FROM TransportQOut a WHERE a.messageCreationTime > :recordStatusTime ORDER BY a.messageCreationTime ASC")
     Optional<Collection<TransportQOut>> findTransportByCreationTime(@Param("recordStatusTime") String recordStatusTime);
 
-    @Query("SELECT a FROM TransportQOut a")
+    @Query("SELECT a FROM TransportQOut a ORDER BY a.messageCreationTime ASC")
     Optional<Collection<TransportQOut>> findTransportByWithoutCreationTime();
 
 

diff --git a/...c/main/java/gov/cdc/nnddataexchangeservice/repository/odse/CNTransportQOutRepository.java b/...c/main/java/gov/cdc/nnddataexchangeservice/repository/odse/CNTransportQOutRepository.java
@@ -12,10 +12,10 @@
 
 @Repository
 public interface CNTransportQOutRepository extends JpaRepository<CNTransportQOut, Long> {
-    @Query("SELECT a FROM CNTransportQOut a WHERE a.recordStatusCd = :statusCd")
+    @Query("SELECT a FROM CNTransportQOut a WHERE a.recordStatusCd = :statusCd ORDER BY a.recordStatusTime ASC")
     Optional<Collection<CNTransportQOut>> findTransportByStatusCd (@Param("statusCd") String statusCd);
 
-    @Query("SELECT a FROM CNTransportQOut a WHERE a.recordStatusTime > :recordStatusTime AND a.recordStatusCd = :recordStatusCd")
+    @Query("SELECT a FROM CNTransportQOut a WHERE a.recordStatusTime > :recordStatusTime AND a.recordStatusCd = :recordStatusCd ORDER BY a.recordStatusTime ASC")
     Optional<Collection<CNTransportQOut>> findTransportByCreationTimeAndStatus (@Param("recordStatusTime") Timestamp recordStatusTime,
                                                                                 @Param("recordStatusCd") String recordStatusCd);
 

diff --git a/...ice/src/main/java/gov/cdc/nnddataexchangeservice/repository/rdb/model/DataSyncConfig.java b/...ice/src/main/java/gov/cdc/nnddataexchangeservice/repository/rdb/model/DataSyncConfig.java
@@ -31,6 +31,8 @@ public class DataSyncConfig {
     @Column(name = "query_with_limit")
     private String queryWithLimit;
 
+    @Column(name = "query_with_null_timestamp")
+    private String queryWithNullTimeStamp;
 
     @Column(name = "created_at")
     private Timestamp createdAt;

diff --git a/.../src/main/java/gov/cdc/nnddataexchangeservice/repository/srte/SrteDataSyncRepository.java b/.../src/main/java/gov/cdc/nnddataexchangeservice/repository/srte/SrteDataSyncRepository.java
@@ -0,0 +1,4 @@
+package gov.cdc.nnddataexchangeservice.repository.srte;
+
+public interface SrteDataSyncRepository {
+}