diff --git a/rs-e2e/README.md b/rs-e2e/README.md index 242909bd0..3037887a6 100644 --- a/rs-e2e/README.md +++ b/rs-e2e/README.md @@ -6,6 +6,8 @@ Intermediary and ReportStream. It's scheduled to run daily using the Information on how to set up the sample files evaluated by the tests can be found [here](/examples/Test/Automated/README.md) +**Note**: the output files generated by the framework are stored in an Azure blob storage container. Every time the tests are run, the files are moved to a folder with the year/month/day format for better organization. The files are retained in the container for 90 days before being deleted + ## Running the tests - Automatically - these are scheduled to run every weekday diff --git a/rs-e2e/src/main/java/gov/hhs/cdc/trustedintermediary/rse2e/AzureBlobFileFetcher.java b/rs-e2e/src/main/java/gov/hhs/cdc/trustedintermediary/rse2e/AzureBlobFileFetcher.java index 3245bb6d1..ceefe5c4f 100644 --- a/rs-e2e/src/main/java/gov/hhs/cdc/trustedintermediary/rse2e/AzureBlobFileFetcher.java +++ b/rs-e2e/src/main/java/gov/hhs/cdc/trustedintermediary/rse2e/AzureBlobFileFetcher.java @@ -4,24 +4,28 @@ import com.azure.storage.blob.BlobContainerClient; import com.azure.storage.blob.BlobContainerClientBuilder; import com.azure.storage.blob.models.BlobItem; -import com.azure.storage.blob.models.BlobProperties; +import com.azure.storage.blob.models.ListBlobsOptions; import java.time.LocalDate; +import java.time.ZoneId; import java.time.ZoneOffset; import java.util.ArrayList; import java.util.List; /** * The AzureBlobFileFetcher class implements the {@link FileFetcher FileFetcher} interface and - * fetches files from an Azure Blob Storage container. + * retrieves files from an Azure Blob Storage container. */ public class AzureBlobFileFetcher implements FileFetcher { - private static final FileFetcher INSTANCE = new AzureBlobFileFetcher(); + private static final ZoneId TIME_ZONE = ZoneOffset.UTC; + private static final int RETENTION_DAYS = 90; + private static final String CONTAINER_NAME = "automated"; private final BlobContainerClient blobContainerClient; + private static final FileFetcher INSTANCE = new AzureBlobFileFetcher(); + private AzureBlobFileFetcher() { - String azureStorageConnectionName = "automated"; String azureStorageConnectionString = System.getenv("AZURE_STORAGE_CONNECTION_STRING"); if (azureStorageConnectionString == null || azureStorageConnectionString.isEmpty()) { @@ -31,8 +35,11 @@ private AzureBlobFileFetcher() { this.blobContainerClient = new BlobContainerClientBuilder() .connectionString(azureStorageConnectionString) - .containerName(azureStorageConnectionName) + .containerName(CONTAINER_NAME) .buildClient(); + + AzureBlobOrganizer blobOrganizer = new AzureBlobOrganizer(blobContainerClient); + blobOrganizer.organizeAndCleanupBlobsByDate(RETENTION_DAYS, TIME_ZONE); } public static FileFetcher getInstance() { @@ -41,33 +48,17 @@ public static FileFetcher getInstance() { @Override public List fetchFiles() { - List recentFiles = new ArrayList<>(); - LocalDate mostRecentDay = null; + List relevantFiles = new ArrayList<>(); - for (BlobItem blobItem : blobContainerClient.listBlobs()) { + LocalDate today = LocalDate.now(TIME_ZONE); + String pathPrefix = AzureBlobHelper.buildDatePathPrefix(today); + ListBlobsOptions options = new ListBlobsOptions().setPrefix(pathPrefix); + for (BlobItem blobItem : blobContainerClient.listBlobs(options, null)) { BlobClient blobClient = blobContainerClient.getBlobClient(blobItem.getName()); - BlobProperties properties = blobClient.getProperties(); - - // Currently we're doing everything in UTC. If we start uploading files manually and - // running - // this test manually, we may want to revisit this logic and/or the file structure - // because midnight UTC is 5pm PDT on the previous day - LocalDate blobCreationDate = - properties.getLastModified().toInstant().atZone(ZoneOffset.UTC).toLocalDate(); - - if (mostRecentDay != null && blobCreationDate.isBefore(mostRecentDay)) { - continue; - } - - if (mostRecentDay == null || blobCreationDate.isAfter(mostRecentDay)) { - mostRecentDay = blobCreationDate; - recentFiles.clear(); - } - - recentFiles.add( + relevantFiles.add( new HL7FileStream(blobClient.getBlobName(), blobClient.openInputStream())); } - return recentFiles; + return relevantFiles; } } diff --git a/rs-e2e/src/main/java/gov/hhs/cdc/trustedintermediary/rse2e/AzureBlobHelper.java b/rs-e2e/src/main/java/gov/hhs/cdc/trustedintermediary/rse2e/AzureBlobHelper.java new file mode 100644 index 000000000..d6548229b --- /dev/null +++ b/rs-e2e/src/main/java/gov/hhs/cdc/trustedintermediary/rse2e/AzureBlobHelper.java @@ -0,0 +1,25 @@ +package gov.hhs.cdc.trustedintermediary.rse2e; + +import java.time.LocalDate; + +/* The AzureBlobHelper is a utility class that provides helper methods for working with Azure Blob Storage. */ +public class AzureBlobHelper { + + private AzureBlobHelper() {} + + // Builds a path prefix for a given date in the format "YYYY/MM/DD/". This is meant to make it + // easier for people in the team to find files in the Azure Blob Storage + public static String buildDatePathPrefix(LocalDate date) { + return String.format( + "%d/%02d/%02d/", date.getYear(), date.getMonthValue(), date.getDayOfMonth()); + } + + public static String createDateBasedPath(LocalDate date, String originalName) { + return buildDatePathPrefix(date) + originalName; + } + + public static boolean isInDateFolder(String blobPath, LocalDate creationDate) { + String expectedPath = buildDatePathPrefix(creationDate); + return blobPath.startsWith(expectedPath); + } +} diff --git a/rs-e2e/src/main/java/gov/hhs/cdc/trustedintermediary/rse2e/AzureBlobOrganizer.java b/rs-e2e/src/main/java/gov/hhs/cdc/trustedintermediary/rse2e/AzureBlobOrganizer.java new file mode 100644 index 000000000..4f61997e0 --- /dev/null +++ b/rs-e2e/src/main/java/gov/hhs/cdc/trustedintermediary/rse2e/AzureBlobOrganizer.java @@ -0,0 +1,69 @@ +package gov.hhs.cdc.trustedintermediary.rse2e; + +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.BlobContainerClient; +import com.azure.storage.blob.models.BlobItem; +import com.azure.storage.blob.models.BlobProperties; +import com.azure.storage.blob.models.CopyStatusType; +import gov.hhs.cdc.trustedintermediary.context.ApplicationContext; +import gov.hhs.cdc.trustedintermediary.wrappers.Logger; +import java.time.Duration; +import java.time.LocalDate; +import java.time.ZoneId; + +/* AzureBlobOrganizer is responsible for organizing and cleaning up blobs in an Azure container */ +public class AzureBlobOrganizer { + + private final BlobContainerClient blobContainerClient; + + protected final Logger logger = ApplicationContext.getImplementation(Logger.class); + + public AzureBlobOrganizer(BlobContainerClient blobContainerClient) { + this.blobContainerClient = blobContainerClient; + } + + public void organizeAndCleanupBlobsByDate(int retentionDays, ZoneId timeZone) { + for (BlobItem blobItem : blobContainerClient.listBlobs()) { + String sourceName = blobItem.getName(); + try { + BlobClient sourceBlob = blobContainerClient.getBlobClient(sourceName); + BlobProperties sourceProperties = sourceBlob.getProperties(); + LocalDate sourceCreationDate = + sourceProperties + .getCreationTime() + .toInstant() + .atZone(timeZone) + .toLocalDate(); + + LocalDate retentionDate = LocalDate.now(timeZone).minusDays(retentionDays); + if (sourceCreationDate.isBefore(retentionDate)) { + sourceBlob.delete(); + logger.logInfo("Deleted old blob: {}", sourceName); + continue; + } + + if (AzureBlobHelper.isInDateFolder(sourceName, sourceCreationDate)) { + continue; + } + + String destinationName = + AzureBlobHelper.createDateBasedPath(sourceCreationDate, sourceName); + BlobClient destinationBlob = blobContainerClient.getBlobClient(destinationName); + destinationBlob + .beginCopy(sourceBlob.getBlobUrl(), null) + .waitForCompletion(Duration.ofSeconds(30)); + + CopyStatusType copyStatus = destinationBlob.getProperties().getCopyStatus(); + if (copyStatus == CopyStatusType.SUCCESS) { + sourceBlob.delete(); + logger.logInfo("Moved blob {} to {}", sourceName, destinationName); + } else { + destinationBlob.delete(); + logger.logError("Failed to copy blob: " + sourceName); + } + } catch (Exception e) { + logger.logError("Error processing blob: " + sourceName, e); + } + } + } +} diff --git a/rs-e2e/src/main/java/gov/hhs/cdc/trustedintermediary/rse2e/LocalFileFetcher.java b/rs-e2e/src/main/java/gov/hhs/cdc/trustedintermediary/rse2e/LocalFileFetcher.java index e8cb9cada..3172c5e36 100644 --- a/rs-e2e/src/main/java/gov/hhs/cdc/trustedintermediary/rse2e/LocalFileFetcher.java +++ b/rs-e2e/src/main/java/gov/hhs/cdc/trustedintermediary/rse2e/LocalFileFetcher.java @@ -17,6 +17,7 @@ public class LocalFileFetcher implements FileFetcher { private static final String FILES_PATH = "../examples/Test/Automated/"; private static final String EXTENSION = "hl7"; + private static final FileFetcher INSTANCE = new LocalFileFetcher(); private LocalFileFetcher() {} diff --git a/rs-e2e/src/test/groovy/gov/hhs/cdc/trustedintermediary/rse2e/AutomatedTest.groovy b/rs-e2e/src/test/groovy/gov/hhs/cdc/trustedintermediary/rse2e/AutomatedTest.groovy index 5cb1cd713..295f84784 100644 --- a/rs-e2e/src/test/groovy/gov/hhs/cdc/trustedintermediary/rse2e/AutomatedTest.groovy +++ b/rs-e2e/src/test/groovy/gov/hhs/cdc/trustedintermediary/rse2e/AutomatedTest.groovy @@ -14,19 +14,13 @@ import spock.lang.Specification class AutomatedTest extends Specification { - List recentAzureFiles - List recentLocalFiles + List azureFiles + List localFiles AssertionRuleEngine engine HapiHL7FileMatcher fileMatcher - def mockLogger = Mock(Logger) + Logger mockLogger = Mock(Logger) def setup() { - FileFetcher azureFileFetcher = AzureBlobFileFetcher.getInstance() - recentAzureFiles = azureFileFetcher.fetchFiles() - - FileFetcher localFileFetcher = LocalFileFetcher.getInstance() - recentLocalFiles = localFileFetcher.fetchFiles() - engine = AssertionRuleEngine.getInstance() fileMatcher = HapiHL7FileMatcher.getInstance() @@ -38,20 +32,25 @@ class AutomatedTest extends Specification { TestApplicationContext.register(Formatter, Jackson.getInstance()) TestApplicationContext.register(HapiHL7FileMatcher, fileMatcher) TestApplicationContext.register(HealthDataExpressionEvaluator, HapiHL7ExpressionEvaluator.getInstance()) - TestApplicationContext.register(AzureBlobFileFetcher, azureFileFetcher) TestApplicationContext.register(LocalFileFetcher, LocalFileFetcher.getInstance()) TestApplicationContext.injectRegisteredImplementations() + + FileFetcher azureFileFetcher = AzureBlobFileFetcher.getInstance() + azureFiles = azureFileFetcher.fetchFiles() + + FileFetcher localFileFetcher = LocalFileFetcher.getInstance() + localFiles = localFileFetcher.fetchFiles() } def cleanup() { - for (HL7FileStream fileStream : recentLocalFiles + recentAzureFiles) { + for (HL7FileStream fileStream : localFiles + azureFiles) { fileStream.inputStream().close() } } def "test defined assertions on relevant messages"() { given: - def matchedFiles = fileMatcher.matchFiles(recentAzureFiles, recentLocalFiles) + def matchedFiles = fileMatcher.matchFiles(azureFiles, localFiles) when: for (messagePair in matchedFiles) { diff --git a/rs-e2e/src/test/groovy/gov/hhs/cdc/trustedintermediary/rse2e/AzureBlobHelperTest.groovy b/rs-e2e/src/test/groovy/gov/hhs/cdc/trustedintermediary/rse2e/AzureBlobHelperTest.groovy new file mode 100644 index 000000000..29b39b674 --- /dev/null +++ b/rs-e2e/src/test/groovy/gov/hhs/cdc/trustedintermediary/rse2e/AzureBlobHelperTest.groovy @@ -0,0 +1,40 @@ +package gov.hhs.cdc.trustedintermediary.rse2e + +import spock.lang.Specification + +import java.time.LocalDate + +class AzureBlobHelperTest extends Specification { + + def "buildDatePathPrefix should create correct path format"() { + given: + def date = LocalDate.of(2024, 3, 15) + + when: + def result = AzureBlobHelper.buildDatePathPrefix(date) + + then: + result == "2024/03/15/" + } + + def "createDateBasedPath should combine date prefix with filename"() { + given: + def date = LocalDate.of(2024, 3, 15) + def fileName = "test.hl7" + + when: + def result = AzureBlobHelper.createDateBasedPath(date, fileName) + + then: + result == "2024/03/15/test.hl7" + } + + def "isInDateFolder should return true for matching date folder"() { + given: + def date = LocalDate.of(2024, 3, 15) + def path = "2024/03/15/test.hl7" + + expect: + AzureBlobHelper.isInDateFolder(path, date) + } +}