Merge branch 'main' into remove-la-obr-except-54089-8

app/src/main/java/gov/hhs/cdc/trustedintermediary/external/javalin/App.java

@@ -6,6 +6,7 @@
 import gov.hhs.cdc.trustedintermediary.domainconnector.DomainConnectorConstructionException;
 import gov.hhs.cdc.trustedintermediary.domainconnector.DomainResponseHelper;
 import gov.hhs.cdc.trustedintermediary.domainconnector.UnableToReadOpenApiSpecificationException;
+import gov.hhs.cdc.trustedintermediary.etor.utils.security.HashHelper;
 import gov.hhs.cdc.trustedintermediary.external.apache.ApacheClient;
 import gov.hhs.cdc.trustedintermediary.external.azure.AzureDatabaseCredentialsProvider;
 import gov.hhs.cdc.trustedintermediary.external.azure.AzureSecrets;
@@ -85,6 +86,7 @@ private static void registerClasses() {
         ApplicationContext.register(YamlCombiner.class, Jackson.getInstance());
         ApplicationContext.register(OpenApi.class, OpenApi.getInstance());
         ApplicationContext.register(HttpClient.class, ApacheClient.getInstance());
+        ApplicationContext.register(HashHelper.class, HashHelper.getInstance());
         ApplicationContext.register(AuthEngine.class, JjwtEngine.getInstance());
         ApplicationContext.register(Cache.class, KeyCache.getInstance());
         ApplicationContext.register(DomainResponseHelper.class, DomainResponseHelper.getInstance());

e2e/build.gradle

@@ -19,10 +19,10 @@ dependencies {
     implementation 'com.fasterxml.jackson.core:jackson-databind:2.18.1'
 
     //fhir
-    implementation 'ca.uhn.hapi.fhir:hapi-fhir-base:7.4.5'
-    implementation 'ca.uhn.hapi.fhir:hapi-fhir-structures-r4:7.4.5'
-    implementation 'ca.uhn.hapi.fhir:hapi-fhir-caching-caffeine:7.4.5'
-    implementation 'ca.uhn.hapi.fhir:hapi-fhir-validation-resources-r4:7.4.5'
+    implementation 'ca.uhn.hapi.fhir:hapi-fhir-base:7.6.0'
+    implementation 'ca.uhn.hapi.fhir:hapi-fhir-structures-r4:7.6.0'
+    implementation 'ca.uhn.hapi.fhir:hapi-fhir-caching-caffeine:7.6.0'
+    implementation 'ca.uhn.hapi.fhir:hapi-fhir-validation-resources-r4:7.6.0'
     implementation 'org.fhir:ucum:1.0.8'
 
     testImplementation 'org.apache.groovy:groovy:4.0.24'

etor/src/main/java/gov/hhs/cdc/trustedintermediary/etor/orders/SendOrderUseCase.java

@@ -6,6 +6,7 @@
 import gov.hhs.cdc.trustedintermediary.etor.metadata.partner.PartnerMetadata;
 import gov.hhs.cdc.trustedintermediary.etor.metadata.partner.PartnerMetadataMessageType;
 import gov.hhs.cdc.trustedintermediary.etor.ruleengine.transformation.TransformationRuleEngine;
+import gov.hhs.cdc.trustedintermediary.etor.utils.security.HashHelper;
 import gov.hhs.cdc.trustedintermediary.wrappers.Logger;
 import gov.hhs.cdc.trustedintermediary.wrappers.MetricMetadata;
 import javax.inject.Inject;
@@ -18,6 +19,7 @@ public class SendOrderUseCase implements SendMessageUseCase<Order<?>> {
     @Inject MetricMetadata metadata;
     @Inject SendMessageHelper sendMessageHelper;
     @Inject Logger logger;
+    @Inject HashHelper hashHelper;
 
     private SendOrderUseCase() {}
 
@@ -29,10 +31,12 @@ public static SendOrderUseCase getInstance() {
     public void convertAndSend(final Order<?> order, String receivedSubmissionId)
             throws UnableToSendMessageException {
 
+        String hashedOrder = hashHelper.generateHash(order);
+
         PartnerMetadata partnerMetadata =
                 new PartnerMetadata(
                         receivedSubmissionId,
-                        String.valueOf(order.hashCode()),
+                        hashedOrder,
                         PartnerMetadataMessageType.ORDER,
                         order.getSendingApplicationDetails(),
                         order.getSendingFacilityDetails(),

etor/src/main/java/gov/hhs/cdc/trustedintermediary/etor/results/SendResultUseCase.java

@@ -6,6 +6,7 @@
 import gov.hhs.cdc.trustedintermediary.etor.metadata.partner.PartnerMetadata;
 import gov.hhs.cdc.trustedintermediary.etor.metadata.partner.PartnerMetadataMessageType;
 import gov.hhs.cdc.trustedintermediary.etor.ruleengine.transformation.TransformationRuleEngine;
+import gov.hhs.cdc.trustedintermediary.etor.utils.security.HashHelper;
 import gov.hhs.cdc.trustedintermediary.wrappers.Logger;
 import javax.inject.Inject;
 
@@ -20,6 +21,8 @@ public class SendResultUseCase implements SendMessageUseCase<Result<?>> {
 
     @Inject Logger logger;
 
+    @Inject HashHelper hashHelper;
+
     private SendResultUseCase() {}
 
     public static SendResultUseCase getInstance() {
@@ -30,10 +33,12 @@ public static SendResultUseCase getInstance() {
     public void convertAndSend(Result<?> result, String receivedSubmissionId)
             throws UnableToSendMessageException {
 
+        String hashedResult = hashHelper.generateHash(result);
+
         PartnerMetadata partnerMetadata =
                 new PartnerMetadata(
                         receivedSubmissionId,
-                        String.valueOf(result.hashCode()),
+                        hashedResult,
                         PartnerMetadataMessageType.RESULT,
                         result.getSendingApplicationDetails(),
                         result.getSendingFacilityDetails(),

etor/src/main/java/gov/hhs/cdc/trustedintermediary/etor/utils/security/HashHelper.java

@@ -0,0 +1,26 @@
+package gov.hhs.cdc.trustedintermediary.etor.utils.security;
+
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.HexFormat;
+
+public class HashHelper {
+
+    private static final HashHelper INSTANCE = new HashHelper();
+
+    public static HashHelper getInstance() {
+        return INSTANCE;
+    }
+
+    public String generateHash(Object input) {
+        try {
+            MessageDigest digest = MessageDigest.getInstance("SHA3-512");
+            byte[] objBytes = input.toString().getBytes(StandardCharsets.UTF_8);
+            byte[] hashBytes = digest.digest(objBytes);
+            return HexFormat.of().formatHex(hashBytes);
+        } catch (NoSuchAlgorithmException e) {
+            throw new RuntimeException("Algorithm does not exist!", e);
+        }
+    }
+}

etor/src/test/groovy/gov/hhs/cdc/trustedintermediary/etor/orders/SendOrderUseCaseTest.groovy

@@ -9,6 +9,7 @@ import gov.hhs.cdc.trustedintermediary.etor.metadata.partner.PartnerMetadataExce
 import gov.hhs.cdc.trustedintermediary.etor.metadata.partner.PartnerMetadataOrchestrator
 
 import gov.hhs.cdc.trustedintermediary.etor.ruleengine.transformation.TransformationRuleEngine
+import gov.hhs.cdc.trustedintermediary.etor.utils.security.HashHelper
 import gov.hhs.cdc.trustedintermediary.wrappers.Logger
 import gov.hhs.cdc.trustedintermediary.wrappers.MetricMetadata
 import spock.lang.Specification
@@ -29,6 +30,7 @@ class SendOrderUseCaseTest extends Specification {
         TestApplicationContext.register(SendMessageHelper, SendMessageHelper.getInstance())
         TestApplicationContext.register(TransformationRuleEngine, mockEngine)
         TestApplicationContext.register(OrderSender, mockSender)
+        TestApplicationContext.register(HashHelper, HashHelper.getInstance())
         TestApplicationContext.register(Logger, mockLogger)
     }
 

etor/src/test/groovy/gov/hhs/cdc/trustedintermediary/etor/results/SendResultUseCaseTest.groovy

@@ -9,6 +9,7 @@ import gov.hhs.cdc.trustedintermediary.etor.metadata.partner.PartnerMetadata
 import gov.hhs.cdc.trustedintermediary.etor.metadata.partner.PartnerMetadataException
 import gov.hhs.cdc.trustedintermediary.etor.metadata.partner.PartnerMetadataOrchestrator
 import gov.hhs.cdc.trustedintermediary.etor.ruleengine.transformation.TransformationRuleEngine
+import gov.hhs.cdc.trustedintermediary.etor.utils.security.HashHelper
 import gov.hhs.cdc.trustedintermediary.wrappers.Logger
 import gov.hhs.cdc.trustedintermediary.wrappers.MetricMetadata
 import spock.lang.Specification
@@ -30,6 +31,7 @@ class SendResultUseCaseTest extends Specification {
         TestApplicationContext.register(TransformationRuleEngine, mockEngine)
         TestApplicationContext.register(ResultSender, mockSender)
         TestApplicationContext.register(Logger, mockLogger)
+        TestApplicationContext.register(HashHelper, HashHelper.getInstance())
         TestApplicationContext.injectRegisteredImplementations()
     }
 

etor/src/test/groovy/gov/hhs/cdc/trustedintermediary/etor/utils/security/HashHelperTest.groovy

@@ -0,0 +1,45 @@
+package gov.hhs.cdc.trustedintermediary.etor.utils.security
+import gov.hhs.cdc.trustedintermediary.context.TestApplicationContext
+import gov.hhs.cdc.trustedintermediary.etor.orders.Order
+import gov.hhs.cdc.trustedintermediary.etor.results.Result
+import gov.hhs.cdc.trustedintermediary.wrappers.Logger
+import spock.lang.Specification
+
+class HashHelperTest extends Specification {
+    def mockLogger = Mock(Logger)
+    def hashHelper = new HashHelper()
+
+    def setup() {
+        TestApplicationContext.reset()
+        TestApplicationContext.init()
+        TestApplicationContext.register(Logger, mockLogger)
+        TestApplicationContext.injectRegisteredImplementations()
+    }
+
+    def "generateHash generates hash for an order"() {
+        given:
+        def mockOrder = Mock(Order)
+
+        when:
+        String mockHash = hashHelper.generateHash(mockOrder)
+
+        then:
+        mockHash !== ""
+        0 * mockLogger.logError(_, _)
+    }
+
+    def "generateHash generates the same hash for the same object"() {
+        given:
+        def mockResult = Mock(Result)
+        def mockResult2 = mockResult
+
+        when:
+        String mockHash = hashHelper.generateHash(mockResult)
+        String mockHash2 = hashHelper.generateHash(mockResult2)
+
+        then:
+        mockHash !== ""
+        mockHash == mockHash2
+        0 * mockLogger.logError(_, _)
+    }
+}

operations/environments/dev/main.tf

@@ -2,7 +2,7 @@ terraform {
   required_providers {
     azurerm = {
       source  = "hashicorp/azurerm"
-      version = "4.10.0"
+      version = "4.11.0"
     }
   }
 

operations/environments/internal/main.tf

@@ -2,7 +2,7 @@ terraform {
   required_providers {
     azurerm = {
       source  = "hashicorp/azurerm"
-      version = "4.10.0"
+      version = "4.11.0"
     }
   }
 

operations/environments/pr/main.tf

@@ -2,7 +2,7 @@ terraform {
   required_providers {
     azurerm = {
       source  = "hashicorp/azurerm"
-      version = "4.10.0"
+      version = "4.11.0"
     }
   }
 

operations/environments/prd/main.tf

@@ -2,7 +2,7 @@ terraform {
   required_providers {
     azurerm = {
       source  = "hashicorp/azurerm"
-      version = "4.10.0"
+      version = "4.11.0"
     }
   }
 

operations/environments/stg/main.tf

@@ -2,7 +2,7 @@ terraform {
   required_providers {
     azurerm = {
       source  = "hashicorp/azurerm"
-      version = "4.10.0"
+      version = "4.11.0"
     }
   }
 

operations/template/alert.tf

@@ -217,47 +217,6 @@ resource "azurerm_monitor_metric_alert" "azure_5XX_alert" {
   }
 }
 
-resource "azurerm_monitor_metric_alert" "dynamic_memory_alert" {
-  count               = local.non_pr_environment ? 1 : 0
-  name                = "cdcti-${var.environment}-dynamic-memory-alert"
-  resource_group_name = data.azurerm_resource_group.group.name
-  scopes              = [azurerm_linux_web_app.api.id]
-  description         = "This alert checks if the backpack is starting to get heavy but does it in a way that keeps watching how much stuff is added. If it gets too full, it lets you know so you can take action before it becomes a problem."
-  severity            = 2
-  frequency           = "PT5M"
-  window_size         = "PT15M"
-
-  dynamic_criteria {
-    metric_name       = "MemoryWorkingSet"
-    metric_namespace  = "Microsoft.Web/sites"
-    aggregation       = "Average"
-    operator          = "GreaterThan"
-    alert_sensitivity = "Medium"
-  }
-
-  action {
-    action_group_id = azurerm_monitor_action_group.notify_slack_email[count.index].id
-  }
-
-  lifecycle {
-    # Ignore changes to tags because the CDC sets these automagically
-    ignore_changes = [
-      tags["business_steward"],
-      tags["center"],
-      tags["environment"],
-      tags["escid"],
-      tags["funding_source"],
-      tags["pii_data"],
-      tags["security_compliance"],
-      tags["security_steward"],
-      tags["support_group"],
-      tags["system"],
-      tags["technical_steward"],
-      tags["zone"]
-    ]
-  }
-}
-
 resource "azurerm_monitor_metric_alert" "database_memory_alert" {
   count               = local.non_pr_environment ? 1 : 0
   name                = "cdcti-${var.environment}-database-memory-alert"

rs-e2e/README.md

@@ -13,14 +13,44 @@ Information on how to set up the sample files evaluated by the tests can be foun
 
 ## Running the tests
 
-- Automatically - these are scheduled to run every weekday
-- Manually via Github
-  - Run the [automated-staging-test-submit](/.github/workflows/automated-staging-test-submit.yml) action
-  - Wait for RS and TI to finish processing files
-  - Run the [automated-staging-test-run](/.github/workflows/automated-staging-test-run.yml) action
-- Locally
-  - Set the `AZURE_STORAGE_CONNECTION_STRING` environment variable to the [value in Keybase](keybase://team/cdc_ti/service_keys/TI/staging/azure-storage-connection-string-for-automated-rs-e2e-tests.txt)
-  - Run the tests with `./gradlew rs-e2e:clean rs-e2e:automatedTest`
+### Automatically
+
+There are two scheduled tasks that run every weekday around midnight EST:
+
+- [Automated Staging Test - Submit Messages](https://github.com/CDCgov/trusted-intermediary/actions/workflows/automated-staging-test-submit.yml) submits the messages in `/examples/Test/Automated`
+- [Automated Staging Test - Run integration tests](https://github.com/CDCgov/trusted-intermediary/actions/workflows/automated-staging-test-run.yml) triggers a couple of hours later and runs the Automated Tests on the input files in `/examples/Test/Automated` and the output files in the `automated` container for the `cdctiautomatedstg` Azure storage account.
+
+### Manually from your local machine
+
+When running locally, we usually run the tests from either the command line using gradle, or from IntelliJ. Please note that even though you are running the test from your local machine, the test will need to connect to the Azure container to pull the output files to apply the assertions on. For this reason, you will need to set the `AZURE_STORAGE_CONNECTION_STRING` environment variable to authenticate the connection to Azure. You can find the value for `AZURE_STORAGE_CONNECTION_STRING` in Keybase (keybase://team/cdc_ti/service_keys/TI/staging/azure-storage-connection-string-for-automated-rs-e2e-tests.txt).
+
+From the command line:
+
+1. Set the `AZURE_STORAGE_CONNECTION_STRING` environment variable in your shell
+2. Run: `./gradlew rs-e2e:clean rs-e2e:automatedTest`
+
+From IntelliJ:
+
+1. Set `AZURE_STORAGE_CONNECTION_STRING` environment variable in the IntelliJ test run configuration (instructions on how to do that [here](https://stackoverflow.com/a/32761503))
+2. Go to `rs-e2e/src/test/groovy/gov/hhs/cdc/trustedintermediary/rse2e/AutomatedTest.groovy` and either run or debug `AutomatedTest` as you normally would from IntelliJ
+
+### Manually via Github
+
+1. Run the [Automated Staging Test - Submit Messages](https://github.com/CDCgov/trusted-intermediary/actions/workflows/automated-staging-test-submit.yml) action
+2. Wait for RS and TI to finish processing files
+3. Run the [Automated Staging Test - Run integration tests](https://github.com/CDCgov/trusted-intermediary/actions/workflows/automated-staging-test-run.yml) action
+
+#### Testing a branch
+
+If you have added new files to `/examples/Test/Automated` in your branch/PR, before running the `Automated Staging Test - Submit Messages` action in step 1, select the branch you're working on as shown in the screenshot below. This will make sure to include your new file(s) when submitting the messages.
+
+![Run Workflow from branch](../images/run_workflow_from_branch.png)
+
+If you have added new assertion rules to the [assertion_definitions.json](/rs-e2e/src/main/resources/assertion_definitions.json) file, you should do the same for step 3 and select your branch when running the `Automated Staging Test - Run integration tests` action.
+
+Instead of running the `Run integration tests` action, you could also test it from your local machine by following the steps in the previous section.
+
+**Note**: when testing a branch with new assertions, it's recommended to make sure the assertions fail as a gut check.
 
 ## Assertions Definition
 

shared/build.gradle

@@ -28,10 +28,10 @@ dependencies {
     implementation 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.18.1'
 
     // hapi fhir
-    api 'ca.uhn.hapi.fhir:hapi-fhir-base:7.4.5'
-    api 'ca.uhn.hapi.fhir:hapi-fhir-structures-r4:7.4.5'
-    implementation 'ca.uhn.hapi.fhir:hapi-fhir-caching-caffeine:7.4.5'
-    implementation 'ca.uhn.hapi.fhir:hapi-fhir-validation-resources-r4:7.4.5'
+    api 'ca.uhn.hapi.fhir:hapi-fhir-base:7.6.0'
+    api 'ca.uhn.hapi.fhir:hapi-fhir-structures-r4:7.6.0'
+    implementation 'ca.uhn.hapi.fhir:hapi-fhir-caching-caffeine:7.6.0'
+    implementation 'ca.uhn.hapi.fhir:hapi-fhir-validation-resources-r4:7.6.0'
     api 'org.fhir:ucum:1.0.8'
 
     // hapi hl7