Organize and cleanup azure blob outputs for Automated Tests

rs-e2e/README.md

@@ -6,6 +6,8 @@ Intermediary and ReportStream. It's scheduled to run daily using the
 
 Information on how to set up the sample files evaluated by the tests can be found [here](/examples/Test/Automated/README.md)
 
+**Note**: the output files generated by the framework are stored in an Azure blob storage container. The files are retained in the container for 90 days before being deleted
+
 ## Running the tests
 
 - Automatically - these are scheduled to run every weekday

rs-e2e/src/main/java/gov/hhs/cdc/trustedintermediary/rse2e/AzureBlobFileFetcher.java

@@ -4,24 +4,28 @@
 import com.azure.storage.blob.BlobContainerClient;
 import com.azure.storage.blob.BlobContainerClientBuilder;
 import com.azure.storage.blob.models.BlobItem;
-import com.azure.storage.blob.models.BlobProperties;
+import com.azure.storage.blob.models.ListBlobsOptions;
 import java.time.LocalDate;
+import java.time.ZoneId;
 import java.time.ZoneOffset;
 import java.util.ArrayList;
 import java.util.List;
 
 /**
  * The AzureBlobFileFetcher class implements the {@link FileFetcher FileFetcher} interface and
- * fetches files from an Azure Blob Storage container.
+ * retrieves files from an Azure Blob Storage container.
  */
 public class AzureBlobFileFetcher implements FileFetcher {
 
-    private static final FileFetcher INSTANCE = new AzureBlobFileFetcher();
+    private static final ZoneId TIME_ZONE = ZoneOffset.UTC;
+    private static final int RETENTION_DAYS = 90;
+    private static final String CONTAINER_NAME = "automated";
 
     private final BlobContainerClient blobContainerClient;
 
+    private static final FileFetcher INSTANCE = new AzureBlobFileFetcher();
+
     private AzureBlobFileFetcher() {
-        String azureStorageConnectionName = "automated";
         String azureStorageConnectionString = System.getenv("AZURE_STORAGE_CONNECTION_STRING");
 
         if (azureStorageConnectionString == null || azureStorageConnectionString.isEmpty()) {
@@ -31,8 +35,11 @@ private AzureBlobFileFetcher() {
         this.blobContainerClient =
                 new BlobContainerClientBuilder()
                         .connectionString(azureStorageConnectionString)
-                        .containerName(azureStorageConnectionName)
+                        .containerName(CONTAINER_NAME)
                         .buildClient();
+
+        AzureBlobOrganizer blobOrganizer = new AzureBlobOrganizer(blobContainerClient);
+        blobOrganizer.organizeAndCleanupBlobsByDate(RETENTION_DAYS, TIME_ZONE);
     }
 
     public static FileFetcher getInstance() {
@@ -41,33 +48,17 @@ public static FileFetcher getInstance() {
 
     @Override
     public List<HL7FileStream> fetchFiles() {
-        List<HL7FileStream> recentFiles = new ArrayList<>();
-        LocalDate mostRecentDay = null;
+        List<HL7FileStream> relevantFiles = new ArrayList<>();
 
-        for (BlobItem blobItem : blobContainerClient.listBlobs()) {
+        LocalDate today = LocalDate.now(TIME_ZONE);
+        String pathPrefix = AzureBlobHelper.buildDatePathPrefix(today);
+        ListBlobsOptions options = new ListBlobsOptions().setPrefix(pathPrefix);
+        for (BlobItem blobItem : blobContainerClient.listBlobs(options, null)) {
             BlobClient blobClient = blobContainerClient.getBlobClient(blobItem.getName());
-            BlobProperties properties = blobClient.getProperties();
-
-            // Currently we're doing everything in UTC. If we start uploading files manually and
-            // running
-            // this test manually, we may want to revisit this logic and/or the file structure
-            // because midnight UTC is 5pm PDT on the previous day
-            LocalDate blobCreationDate =
-                    properties.getLastModified().toInstant().atZone(ZoneOffset.UTC).toLocalDate();
-
-            if (mostRecentDay != null && blobCreationDate.isBefore(mostRecentDay)) {
-                continue;
-            }
-
-            if (mostRecentDay == null || blobCreationDate.isAfter(mostRecentDay)) {
-                mostRecentDay = blobCreationDate;
-                recentFiles.clear();
-            }
-
-            recentFiles.add(
+            relevantFiles.add(
                     new HL7FileStream(blobClient.getBlobName(), blobClient.openInputStream()));
         }
 
-        return recentFiles;
+        return relevantFiles;
     }
 }

rs-e2e/src/main/java/gov/hhs/cdc/trustedintermediary/rse2e/AzureBlobHelper.java

@@ -0,0 +1,25 @@
+package gov.hhs.cdc.trustedintermediary.rse2e;
+
+import java.time.LocalDate;
+
+/* The AzureBlobHelper is a utility class that provides helper methods for working with Azure Blob Storage. */
+public class AzureBlobHelper {
+
+    private AzureBlobHelper() {}
+
+    // Builds a path prefix for a given date in the format "YYYY/MM/DD/". This is meant to make it
+    // easier for people in the team to find files in the Azure Blob Storage
+    public static String buildDatePathPrefix(LocalDate date) {
+        return String.format(
+                "%d/%02d/%02d/", date.getYear(), date.getMonthValue(), date.getDayOfMonth());
+    }
+
+    public static String createDateBasedPath(LocalDate date, String originalName) {
+        return buildDatePathPrefix(date) + originalName;
+    }
+
+    public static boolean isInDateFolder(String blobPath, LocalDate creationDate) {
+        String expectedPath = buildDatePathPrefix(creationDate);
+        return blobPath.startsWith(expectedPath);
+    }
+}

rs-e2e/src/main/java/gov/hhs/cdc/trustedintermediary/rse2e/AzureBlobOrganizer.java

@@ -0,0 +1,69 @@
+package gov.hhs.cdc.trustedintermediary.rse2e;
+
+import com.azure.storage.blob.BlobClient;
+import com.azure.storage.blob.BlobContainerClient;
+import com.azure.storage.blob.models.BlobItem;
+import com.azure.storage.blob.models.BlobProperties;
+import com.azure.storage.blob.models.CopyStatusType;
+import gov.hhs.cdc.trustedintermediary.context.ApplicationContext;
+import gov.hhs.cdc.trustedintermediary.wrappers.Logger;
+import java.time.Duration;
+import java.time.LocalDate;
+import java.time.ZoneId;
+
+/* AzureBlobOrganizer is responsible for organizing and cleaning up blobs in an Azure container */
+public class AzureBlobOrganizer {
+
+    private final BlobContainerClient blobContainerClient;
+
+    protected final Logger logger = ApplicationContext.getImplementation(Logger.class);
+
+    public AzureBlobOrganizer(BlobContainerClient blobContainerClient) {
+        this.blobContainerClient = blobContainerClient;
+    }
+
+    public void organizeAndCleanupBlobsByDate(int retentionDays, ZoneId timeZone) {
+        for (BlobItem blobItem : blobContainerClient.listBlobs()) {
+            String sourceName = blobItem.getName();
+            try {
+                BlobClient sourceBlob = blobContainerClient.getBlobClient(sourceName);
+                BlobProperties sourceProperties = sourceBlob.getProperties();
+                LocalDate sourceCreationDate =
+                        sourceProperties
+                                .getCreationTime()
+                                .toInstant()
+                                .atZone(timeZone)
+                                .toLocalDate();
+
+                LocalDate retentionDate = LocalDate.now(timeZone).minusDays(retentionDays);
+                if (sourceCreationDate.isBefore(retentionDate)) {
+                    sourceBlob.delete();
+                    logger.logInfo("Deleted old blob: {}", sourceName);
+                    continue;
+                }
+
+                if (AzureBlobHelper.isInDateFolder(sourceName, sourceCreationDate)) {
+                    continue;
+                }
+
+                String destinationName =
+                        AzureBlobHelper.createDateBasedPath(sourceCreationDate, sourceName);
+                BlobClient destinationBlob = blobContainerClient.getBlobClient(destinationName);
+                destinationBlob
+                        .beginCopy(sourceBlob.getBlobUrl(), null)
+                        .waitForCompletion(Duration.ofSeconds(30));
+
+                CopyStatusType copyStatus = destinationBlob.getProperties().getCopyStatus();
+                if (copyStatus == CopyStatusType.SUCCESS) {
+                    sourceBlob.delete();
+                    logger.logInfo("Moved blob {} to {}", sourceName, destinationName);
+                } else {
+                    destinationBlob.delete();
+                    logger.logError("Failed to copy blob: " + sourceName);
+                }
+            } catch (Exception e) {
+                logger.logError("Error processing blob: " + sourceName, e);
+            }
+        }
+    }
+}

rs-e2e/src/main/java/gov/hhs/cdc/trustedintermediary/rse2e/LocalFileFetcher.java

@@ -17,6 +17,7 @@ public class LocalFileFetcher implements FileFetcher {
 
     private static final String FILES_PATH = "../examples/Test/Automated/";
     private static final String EXTENSION = "hl7";
+
     private static final FileFetcher INSTANCE = new LocalFileFetcher();
 
     private LocalFileFetcher() {}

rs-e2e/src/test/groovy/gov/hhs/cdc/trustedintermediary/rse2e/AutomatedTest.groovy

@@ -14,19 +14,13 @@ import spock.lang.Specification
 
 class AutomatedTest extends Specification  {
 
-    List<HL7FileStream> recentAzureFiles
-    List<HL7FileStream> recentLocalFiles
+    List<HL7FileStream> azureFiles
+    List<HL7FileStream> localFiles
     AssertionRuleEngine engine
     HapiHL7FileMatcher fileMatcher
-    def mockLogger = Mock(Logger)
+    Logger mockLogger = Mock(Logger)
 
     def setup() {
-        FileFetcher azureFileFetcher = AzureBlobFileFetcher.getInstance()
-        recentAzureFiles = azureFileFetcher.fetchFiles()
-
-        FileFetcher localFileFetcher = LocalFileFetcher.getInstance()
-        recentLocalFiles = localFileFetcher.fetchFiles()
-
         engine = AssertionRuleEngine.getInstance()
         fileMatcher =  HapiHL7FileMatcher.getInstance()
 
@@ -38,20 +32,25 @@ class AutomatedTest extends Specification  {
         TestApplicationContext.register(Formatter, Jackson.getInstance())
         TestApplicationContext.register(HapiHL7FileMatcher, fileMatcher)
         TestApplicationContext.register(HealthDataExpressionEvaluator, HapiHL7ExpressionEvaluator.getInstance())
-        TestApplicationContext.register(AzureBlobFileFetcher, azureFileFetcher)
         TestApplicationContext.register(LocalFileFetcher, LocalFileFetcher.getInstance())
         TestApplicationContext.injectRegisteredImplementations()
+
+        FileFetcher azureFileFetcher = AzureBlobFileFetcher.getInstance()
+        azureFiles = azureFileFetcher.fetchFiles()
+
+        FileFetcher localFileFetcher = LocalFileFetcher.getInstance()
+        localFiles = localFileFetcher.fetchFiles()
     }
 
     def cleanup() {
-        for (HL7FileStream fileStream : recentLocalFiles + recentAzureFiles) {
+        for (HL7FileStream fileStream : localFiles + azureFiles) {
             fileStream.inputStream().close()
         }
     }
 
     def "test defined assertions on relevant messages"() {
         given:
-        def matchedFiles = fileMatcher.matchFiles(recentAzureFiles, recentLocalFiles)
+        def matchedFiles = fileMatcher.matchFiles(azureFiles, localFiles)
 
         when:
         for (messagePair in matchedFiles) {

rs-e2e/src/test/groovy/gov/hhs/cdc/trustedintermediary/rse2e/AzureBlobHelperTest.groovy

@@ -0,0 +1,40 @@
+package gov.hhs.cdc.trustedintermediary.rse2e
+
+import spock.lang.Specification
+
+import java.time.LocalDate
+
+class AzureBlobHelperTest extends Specification {
+
+    def "buildDatePathPrefix should create correct path format"() {
+        given:
+        def date = LocalDate.of(2024, 3, 15)
+
+        when:
+        def result = AzureBlobHelper.buildDatePathPrefix(date)
+
+        then:
+        result == "2024/03/15/"
+    }
+
+    def "createDateBasedPath should combine date prefix with filename"() {
+        given:
+        def date = LocalDate.of(2024, 3, 15)
+        def fileName = "test.hl7"
+
+        when:
+        def result = AzureBlobHelper.createDateBasedPath(date, fileName)
+
+        then:
+        result == "2024/03/15/test.hl7"
+    }
+
+    def "isInDateFolder should return true for matching date folder"() {
+        given:
+        def date = LocalDate.of(2024, 3, 15)
+        def path = "2024/03/15/test.hl7"
+
+        expect:
+        AzureBlobHelper.isInDateFolder(path, date)
+    }
+}