Merge branch 'main' into story-621-save_new_metadata

.github/workflows/dast.yml

@@ -33,7 +33,7 @@ jobs:
         run: ./health-check.sh
 
       - name: Run ZAP API Scan
-        run: docker run -v $(pwd):/zap/wrk/:rw --user root -t owasp/zap2docker-weekly zap-api-scan.py -t http://$(ip -f inet -o addr show docker0 | awk '{print $4}' | cut -d '/' -f 1):8080/openapi -f openapi -r api-scan-report.html
+        run: docker run -v $(pwd):/zap/wrk/:rw --user root -t ghcr.io/zaproxy/zaproxy:weekly zap-api-scan.py -t http://$(ip -f inet -o addr show docker0 | awk '{print $4}' | cut -d '/' -f 1):8080/openapi -f openapi -r api-scan-report.html
 
       - name: Upload Report
         uses: actions/upload-artifact@v4

e2e/src/test/groovy/gov/hhs/cdc/trustedintermediary/e2e/ConsolidatedSummaryTest.groovy

@@ -19,15 +19,18 @@ class ConsolidatedSummaryTest extends Specification {
 
         def orderClient = new EndpointClient("/v1/etor/orders")
         def labOrderJsonFileString = Files.readString(Path.of("../examples/Test/Orders/002_ORM_O01.fhir"))
-        def senderName = "flexion.simulated-hospital"
+        def senderName = "PLACE_HOLDER"  //TODO: when story #990 is implemented, update this to be the sender from the 002_ORM_O01.fhir message
+
         when:
         def orderResponse = orderClient.submit(labOrderJsonFileString, inboundSubmissionId, true)
+
         then:
         orderResponse.getCode() == expectedStatusCode
 
         when:
         def senderNameResponse = ConsolidatedSummaryClient.get(senderName, true)
         def jsonBody = JsonParsing.parseContent(senderNameResponse)
+
         then:
         jsonBody.get((jsonBody.keySet().toArray())[0]).stale != null
         jsonBody.get((jsonBody.keySet().toArray())[0]).failureReason == null

etor/src/main/java/gov/hhs/cdc/trustedintermediary/etor/RSEndpointClient.java

@@ -15,4 +15,7 @@ String requestWatersEndpoint(String body, String bearerToken)
 
     String requestHistoryEndpoint(String submissionId, String bearerToken)
             throws ReportStreamEndpointClientException;
+
+    String requestDeliveryEndpoint(String reportId, String bearerToken)
+            throws ReportStreamEndpointClientException;
 }

etor/src/main/java/gov/hhs/cdc/trustedintermediary/etor/metadata/partner/PartnerMetadataOrchestrator.java

@@ -10,6 +10,7 @@
 import java.time.Instant;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.stream.Collectors;
@@ -45,30 +46,38 @@ public void updateMetadataForReceivedMessage(
             MessageHdDataType receivingFacilityDetails,
             String placerOrderNumber)
             throws PartnerMetadataException {
-        // currently blocked by: https://github.com/CDCgov/prime-reportstream/issues/12624
-        // once we get the right receivedSubmissionId from RS, this method should work
+
         logger.logInfo(
-                "Looking up sender name and timeReceived from RS history API for receivedSubmissionId: {}",
+                "Looking up sender name and timeReceived from RS delivery API for receivedSubmissionId: {}",
                 receivedSubmissionId);
 
-        String sender;
         Instant timeReceived;
         try {
             String bearerToken = rsclient.getRsToken();
             String responseBody =
-                    rsclient.requestHistoryEndpoint(receivedSubmissionId, bearerToken);
+                    rsclient.requestDeliveryEndpoint(receivedSubmissionId, bearerToken);
             Map<String, Object> responseObject =
                     formatter.convertJsonToObject(responseBody, new TypeReference<>() {});
 
-            sender = responseObject.get("sender").toString();
-            String timestamp = responseObject.get("timestamp").toString();
+            List<Map<String, String>> originalIngestion =
+                    (List<Map<String, String>>) responseObject.get("originalIngestion");
+
+            if (originalIngestion.size() > 1) {
+                logger.logWarning(
+                        "More than 1 report ids found in originalIngestion,"
+                                + " check to make sure batching wasn't turned on for receiver in RS");
+            }
+
+            // We should only have 1 object in originalIngestion, it is a list to support other RS
+            // use cases
+            String timestamp = originalIngestion.get(0).get("ingestionTime");
             timeReceived = Instant.parse(timestamp);
 
         } catch (Exception e) {
             // write the received submission ID so that the rest of the metadata flow works even if
             // some data is missing
             logger.logWarning(
-                    "Unable to retrieve metadata from RS history API, but writing basic metadata entry anyway for received submission ID {}",
+                    "Unable to retrieve metadata from RS delivery API, but writing basic metadata entry anyway for received submission ID {}",
                     receivedSubmissionId);
             PartnerMetadata partnerMetadata =
                     new PartnerMetadata(
@@ -83,9 +92,10 @@ public void updateMetadataForReceivedMessage(
             partnerMetadataStorage.saveMetadata(partnerMetadata);
 
             throw new PartnerMetadataException(
-                    "Unable to retrieve metadata from RS history API", e);
+                    "Unable to retrieve metadata from RS delivery API", e);
         }
 
+        String sender = "PLACE_HOLDER";
         logger.logInfo(
                 "Updating metadata with sender: {}, timeReceived: {} and hash",
                 sender,

etor/src/main/java/gov/hhs/cdc/trustedintermediary/external/localfile/MockRSEndpointClient.java

@@ -60,4 +60,35 @@ public String requestHistoryEndpoint(String submissionId, String bearerToken) {
                     }]
                  }""";
     }
+
+    @Override
+    public String requestDeliveryEndpoint(String reportId, String bearerToken)
+            throws ReportStreamEndpointClientException {
+        return """
+            {
+                "deliveryId": 20,
+                "batchReadyAt": "2024-04-09T18:19:00.431Z",
+                "expires": "2024-05-09T18:19:00.431Z",
+                "receiver": "flexion.etor-service-receiver-orders",
+                "receivingOrgSvcStatus": null,
+                "reportId": "ddfeb4e2-af58-433e-9297-a4be01957225",
+                "topic": "etor-ti",
+                "reportItemCount": 2,
+                "fileName": "fhir-transform-sample.yml-ddfeb4e2-af58-433e-9297-a4be01957225-20240409181900.fhir",
+                "fileType": "FHIR",
+                "originalIngestion": [
+                    {
+                        "reportId": "2f5f17e7-2161-44d9-b091-2d53c10f6e90",
+                        "ingestionTime": "2024-04-09T18:17:56.571Z",
+                        "sendingOrg": "DogCow Associates"
+                    },
+                    {
+                        "reportId": "e18c283e-e2e4-4804-bca3-33afe32e6b69",
+                        "ingestionTime": "2024-04-09T18:18:00.553Z",
+                        "sendingOrg": "DogCow Associates"
+                    }
+                ]
+            }
+        """;
+    }
 }

etor/src/main/java/gov/hhs/cdc/trustedintermediary/external/reportstream/ReportStreamEndpointClient.java

@@ -32,6 +32,7 @@ public class ReportStreamEndpointClient implements RSEndpointClient {
     private static final String RS_AUTH_API_URL = RS_URL_PREFIX + "/api/token";
     private static final String RS_HISTORY_API_URL =
             RS_URL_PREFIX + "/api/waters/report/{id}/history";
+    private static final String RS_DELIVERY_API_URL = RS_WATERS_API_URL + "/report/{id}/delivery";
 
     private static final String OUR_PRIVATE_KEY_ID =
             "trusted-intermediary-private-key-" + ApplicationContext.getEnvironment();
@@ -123,6 +124,22 @@ public String requestHistoryEndpoint(String submissionId, String bearerToken)
         }
     }
 
+    @Override
+    public String requestDeliveryEndpoint(String reportId, String bearerToken)
+            throws ReportStreamEndpointClientException {
+        logger.logInfo("Requesting delivery API from ReportStream");
+
+        Map<String, String> headers = Map.of("Authorization", "Bearer " + bearerToken);
+
+        try {
+            String url = RS_DELIVERY_API_URL.replace("{id}", reportId);
+            return client.get(url, headers);
+        } catch (HttpClientException e) {
+            throw new ReportStreamEndpointClientException(
+                    "Error GETting deliveries from ReportStream", e);
+        }
+    }
+
     protected String requestToken() throws ReportStreamEndpointClientException {
         logger.logInfo("Requesting token from ReportStream");
 

etor/src/test/groovy/gov/hhs/cdc/trustedintermediary/etor/metadata/partner/PartnerMetadataOrchestratorTest.groovy

@@ -47,30 +47,55 @@ class PartnerMetadataOrchestratorTest extends Specification {
         TestApplicationContext.injectRegisteredImplementations()
     }
 
-    def "updateMetadataForReceivedOrder updates metadata successfully"() {
+    def "updateMetadataForReceivedMessage updates metadata successfully"() {
         given:
+
+        TestApplicationContext.register(Formatter, Jackson.getInstance())
+        TestApplicationContext.injectRegisteredImplementations()
+
         def receivedSubmissionId = "receivedSubmissionId"
-        def sender = "senderName"
         def timestamp = "2020-01-01T00:00:00.000Z"
-        def timeDelivered = "2020-01-02T00:00:00.000Z"
         def hashCode = "123"
         def bearerToken = "token"
         def messageType = PartnerMetadataMessageType.RESULT
-        def rsHistoryApiResponse = "{\"actualCompletionAt\": \"2023-10-24T19:48:26.921Z\",\"sender\": \"${sender}\", \"timestamp\": \"${timestamp}\"}"
         def deliveryStatus = PartnerMetadataStatus.PENDING
-
-        mockFormatter.convertJsonToObject(rsHistoryApiResponse, _ as TypeReference) >> [sender: sender, timestamp: timestamp, actualCompletionAt: timeDelivered]
+        def rsDeliveryApiResponse = """
+            {
+                "deliveryId": 20,
+                "batchReadyAt": "2024-04-09T18:19:00.431Z",
+                "expires": "2024-05-09T18:19:00.431Z",
+                "receiver": "flexion.etor-service-receiver-orders",
+                "receivingOrgSvcStatus": null,
+                "reportId": "ddfeb4e2-af58-433e-9297-a4be01957225",
+                "topic": "etor-ti",
+                "reportItemCount": 2,
+                "fileName": "fhir-transform-sample.yml-ddfeb4e2-af58-433e-9297-a4be01957225-20240409181900.fhir",
+                "fileType": "FHIR",
+                "originalIngestion": [
+                    {
+                        "reportId": "2f5f17e7-2161-44d9-b091-2d53c10f6e90",
+                        "ingestionTime": "${timestamp}",
+                        "sendingOrg": "Clarus Doctors"
+                    },
+                    {
+                        "reportId": "e18c283e-e2e4-4804-bca3-33afe32e6b69",
+                        "ingestionTime": "2024-04-09T18:18:00.553Z",
+                        "sendingOrg": "DogCow Associates"
+                    }
+                ]
+            }
+        """
 
         when:
         PartnerMetadataOrchestrator.getInstance().updateMetadataForReceivedMessage(receivedSubmissionId, hashCode, messageType, sendingApp, sendingFacility, receivingApp, receivingFacility, placerOrderNumber)
 
         then:
         1 * mockClient.getRsToken() >> bearerToken
-        1 * mockClient.requestHistoryEndpoint(receivedSubmissionId, bearerToken) >> rsHistoryApiResponse
-        1 * mockPartnerMetadataStorage.saveMetadata(new PartnerMetadata(receivedSubmissionId, sender, Instant.parse(timestamp), null, hashCode, deliveryStatus, messageType, sendingApp, sendingFacility, receivingApp, receivingFacility, placerOrderNumber))
+        1 * mockClient.requestDeliveryEndpoint(receivedSubmissionId, bearerToken) >> rsDeliveryApiResponse
+        1 * mockPartnerMetadataStorage.saveMetadata(new PartnerMetadata(receivedSubmissionId, "PLACE_HOLDER", Instant.parse(timestamp), null, hashCode, deliveryStatus, messageType, sendingApp, sendingFacility, receivingApp, receivingFacility, placerOrderNumber))
     }
 
-    def "updateMetadataForSentOrder test case when sentSubmissionId is null"() {
+    def "updateMetadataForSentMessage test case when sentSubmissionId is null"() {
         given:
         def receivedSubmissionId = "receivedSubmissionId"
         def sentSubmissionId = null
@@ -82,7 +107,7 @@ class PartnerMetadataOrchestratorTest extends Specification {
         0 * mockPartnerMetadataStorage.readMetadata(receivedSubmissionId)
     }
 
-    def "updateMetadataForSentOrder test case when PartnerMetadata returns no data"() {
+    def "updateMetadataForSentMessage test case when PartnerMetadata returns no data"() {
         given:
         def receivedSubmissionId = "receivedSubmissionId"
         def sentSubmissionId = "sentSubmissionId"
@@ -109,12 +134,12 @@ class PartnerMetadataOrchestratorTest extends Specification {
         1 * mockPartnerMetadataStorage.readMetadata(receivedSubmissionId) >> mockMetadata
     }
 
-    def "updateMetadataForReceivedOrder throws PartnerMetadataException on client error"() {
+    def "updateMetadataForReceivedMessage throws PartnerMetadataException on client error"() {
         given:
         def receivedSubmissionId = "receivedSubmissionId"
 
         mockClient.getRsToken() >> "token"
-        mockClient.requestHistoryEndpoint(_ as String, _ as String) >> { throw new ReportStreamEndpointClientException("Client error", new Exception()) }
+        mockClient.requestDeliveryEndpoint(_ as String, _ as String) >> { throw new ReportStreamEndpointClientException("Client error", new Exception()) }
 
         when:
         PartnerMetadataOrchestrator.getInstance().updateMetadataForReceivedMessage(receivedSubmissionId, "hash", PartnerMetadataMessageType.RESULT, sendingApp, sendingFacility, receivingApp, receivingFacility, placerOrderNumber)
@@ -126,27 +151,29 @@ class PartnerMetadataOrchestratorTest extends Specification {
         thrown(PartnerMetadataException)
     }
 
-    def "updateMetadataForReceivedOrder throws PartnerMetadataException on formatter error"() {
+    def "updateMetadataForReceivedMessage throws PartnerMetadataException on formatter error"() {
         given:
-        def rsHistoryApiResponse = "{\"sender\": \"responseName\", \"timestamp\": \"2020-01-01T00:00:00.000Z\"}"
+        def receivedSubmissionId = "receivedSubmissionId"
+        def messageType = PartnerMetadataMessageType.RESULT
+        def rsDeliveryApiResponse = "{ASDF}"
 
         mockClient.getRsToken() >> "token"
-        mockClient.requestHistoryEndpoint(_ as String, _ as String) >> rsHistoryApiResponse
-        mockFormatter.convertJsonToObject(rsHistoryApiResponse, _ as TypeReference) >> { throw new FormatterProcessingException("Formatter error", new Exception()) }
+        mockClient.requestDeliveryEndpoint(_ as String, _ as String) >> rsDeliveryApiResponse
+        mockFormatter.convertJsonToObject(rsDeliveryApiResponse, _ as TypeReference) >> { throw new FormatterProcessingException("Formatter error", new Exception()) }
 
         when:
-        PartnerMetadataOrchestrator.getInstance().updateMetadataForReceivedMessage("receivedSubmissionId", "hash", PartnerMetadataMessageType.RESULT, sendingApp, sendingFacility, receivingApp, receivingFacility, placerOrderNumber)
+        PartnerMetadataOrchestrator.getInstance().updateMetadataForReceivedMessage(receivedSubmissionId, "hash", messageType, sendingApp, sendingFacility, receivingApp, receivingFacility, placerOrderNumber)
 
         then:
         thrown(PartnerMetadataException)
     }
 
-    def "updateMetadataForReceivedOrder throws PartnerMetadataException on formatter error due to unexpected response format"() {
+    def "updateMetadataForReceivedMessage throws PartnerMetadataException on formatter error due to unexpected response format"() {
         given:
         def wrongFormatResponse = "{\"someotherkey\": \"value\"}"
 
         mockClient.getRsToken() >> "token"
-        mockClient.requestHistoryEndpoint(_ as String, _ as String) >> wrongFormatResponse
+        mockClient.requestDeliveryEndpoint(_ as String, _ as String) >> wrongFormatResponse
         mockFormatter.convertJsonToObject(wrongFormatResponse, _ as TypeReference) >> [someotherkey: "value"]
 
         when:
@@ -156,7 +183,24 @@ class PartnerMetadataOrchestratorTest extends Specification {
         thrown(PartnerMetadataException)
     }
 
-    def "updateMetadataForSentOrder updates metadata successfully"() {
+    def "updateMetadataForReceivedMessage throws PartnerMetadataException due to 0 originalIngestions"() {
+        given:
+        def receivedSubmissionId = "receivedSubmissionId"
+        def wrongFormatResponse = "{\"originalIngestion\": []}"
+        def messageType = PartnerMetadataMessageType.RESULT
+
+        mockClient.getRsToken() >> "token"
+        mockClient.requestDeliveryEndpoint(_ as String, _ as String) >> wrongFormatResponse
+        mockFormatter.convertJsonToObject(wrongFormatResponse, _ as TypeReference) >> [originalIngestion: []]
+
+        when:
+        PartnerMetadataOrchestrator.getInstance().updateMetadataForReceivedMessage(receivedSubmissionId, "hash", messageType, sendingApp, sendingFacility, receivingApp, receivingFacility, placerOrderNumber)
+
+        then:
+        thrown(PartnerMetadataException)
+    }
+
+    def "updateMetadataForSentMessage updates metadata successfully"() {
         given:
         def receivedSubmissionId = "receivedSubmissionId"
         def sentSubmissionId = "sentSubmissionId"
@@ -171,7 +215,7 @@ class PartnerMetadataOrchestratorTest extends Specification {
         1 * mockPartnerMetadataStorage.saveMetadata(updatedPartnerMetadata)
     }
 
-    def "updateMetadataForSentOrder test case when sentSubmissionId is null"() {
+    def "updateMetadataForSentMessage test case when sentSubmissionId is null"() {
         given:
         def receivedSubmissionId = "receivedSubmissionId"
         def sentSubmissionId = null
@@ -190,7 +234,7 @@ class PartnerMetadataOrchestratorTest extends Specification {
 
         mockPartnerMetadataStorage.readMetadata(receivedSubmissionId) >> Optional.of(partnerMetadata)
         mockClient.getRsToken() >> "token"
-        mockClient.requestHistoryEndpoint(_ as String, _ as String) >> { throw new ReportStreamEndpointClientException("Client error", new Exception()) }
+        mockClient.requestDeliveryEndpoint(_ as String, _ as String) >> { throw new ReportStreamEndpointClientException("Client error", new Exception()) }
 
         when:
         PartnerMetadataOrchestrator.getInstance().getMetadata(receivedSubmissionId)

etor/src/test/groovy/gov/hhs/cdc/trustedintermediary/external/database/DatabasePartnerMetadataStorageTest.groovy

@@ -4,6 +4,10 @@ import com.fasterxml.jackson.databind.ObjectMapper
 import gov.hhs.cdc.trustedintermediary.context.TestApplicationContext
 import gov.hhs.cdc.trustedintermediary.etor.messages.MessageHdDataType
 import gov.hhs.cdc.trustedintermediary.etor.metadata.partner.PartnerMetadata
+import gov.hhs.cdc.trustedintermediary.etor.metadata.partner.PartnerMetadataException
+import gov.hhs.cdc.trustedintermediary.etor.metadata.partner.PartnerMetadataMessageType
+import gov.hhs.cdc.trustedintermediary.etor.metadata.partner.PartnerMetadataStatus
+import gov.hhs.cdc.trustedintermediary.etor.metadata.partner.PartnerMetadataStorage
 import gov.hhs.cdc.trustedintermediary.external.jackson.Jackson
 import gov.hhs.cdc.trustedintermediary.wrappers.formatter.Formatter
 import gov.hhs.cdc.trustedintermediary.wrappers.formatter.FormatterProcessingException

etor/src/test/groovy/gov/hhs/cdc/trustedintermediary/external/database/PostgresDaoTest.groovy

@@ -19,6 +19,7 @@ import java.sql.PreparedStatement
 import java.sql.ResultSet
 import java.sql.SQLException
 import java.sql.Timestamp
+import java.sql.Types
 import java.time.Instant
 
 class PostgresDaoTest extends Specification {

etor/src/test/groovy/gov/hhs/cdc/trustedintermediary/external/localfile/MockRSEndpointClientTest.groovy

@@ -68,4 +68,17 @@ class MockRSEndpointClientTest extends Specification {
         cleanup:
         readonlyLocalFile.toFile().delete()
     }
+
+    def "requestDeliveryEndpoint happy path"() {
+        when:
+        def token = MockRSEndpointClient.getInstance().getRsToken()
+        def response = MockRSEndpointClient.getInstance().requestDeliveryEndpoint("delivery", token)
+        def responseObject =
+                Jackson.getInstance().convertJsonToObject(response, new TypeReference<Map<String, Object>>() {})
+
+        then:
+        responseObject.originalIngestion != null
+        responseObject.originalIngestion[0] != null
+        responseObject.originalIngestion[0].ingestionTime != null
+    }
 }