From 1a0fcae324feaa4c8b5c97af6353c636aedaa47d Mon Sep 17 00:00:00 2001 From: elmiomar Date: Wed, 11 Oct 2023 12:44:21 -0400 Subject: [PATCH 01/20] add support for uncaching --- .../distrib/service/RPACachingService.java | 7 +++++ .../distrib/web/RPADataCachingController.java | 30 +++++++++++++++++++ .../web/RPARequestHandlerController.java | 4 +-- 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/src/main/java/gov/nist/oar/distrib/service/RPACachingService.java b/src/main/java/gov/nist/oar/distrib/service/RPACachingService.java index ec937291..5e51793a 100644 --- a/src/main/java/gov/nist/oar/distrib/service/RPACachingService.java +++ b/src/main/java/gov/nist/oar/distrib/service/RPACachingService.java @@ -62,6 +62,7 @@ public String cacheAndGenerateRandomId(String datasetID, String version) logger.debug("Request to cache dataset with ID=" + datasetID); + // this is to handle ark IDs String dsid = datasetID; if (datasetID.startsWith("ark:/")) { // Split the dataset ID into components @@ -246,4 +247,10 @@ private String getDownloadUrl(String baseDownloadUrl, String randomId, String pa private String generateRandomID(int length, boolean useLetters, boolean useNumbers) { return RandomStringUtils.random(length, useLetters, useNumbers); } + + public boolean uncacheById(String randomId) throws CacheManagementException { + logger.debug("Request to cache dataset with ID=" + randomId); + this.pdrCacheManager.uncache(randomId); + return false; + } } \ No newline at end of file diff --git a/src/main/java/gov/nist/oar/distrib/web/RPADataCachingController.java b/src/main/java/gov/nist/oar/distrib/web/RPADataCachingController.java index f585194c..53e84289 100644 --- a/src/main/java/gov/nist/oar/distrib/web/RPADataCachingController.java +++ b/src/main/java/gov/nist/oar/distrib/web/RPADataCachingController.java @@ -12,6 +12,7 @@ import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.ExceptionHandler; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; @@ -112,6 +113,35 @@ public Map retrieveMetadata(@PathVariable("cacheid") String cach return metadata; } + /** + * This endpoint handles the removal of a dataset from the cache based on its randomly generated ID. + * + * @param randomId The randomly generated ID that was assigned when the dataset was cached. + * + * @return A ResponseEntity containing a status message and HTTP status code. + * + * @throws CacheManagementException If there is an issue with removing the dataset from the cache. + * @throws ResourceNotFoundException If the dataset associated with the randomId is not found in the cache. + * @throws StorageVolumeException If there is an issue with the storage volume. + */ + @PutMapping(value = "/uncache/{randomId}") + public ResponseEntity uncacheDataset( + @PathVariable("randomId") String randomId) + throws CacheManagementException { + + logger.debug("randomId to uncache=" + randomId); + + // Perform uncache operation via the service + boolean success = restrictedSrvc.uncacheById(randomId); + + if (success) { + return new ResponseEntity<>("Dataset was successfully removed from cache.", HttpStatus.OK); + } else { + return new ResponseEntity<>("Failed to uncache the dataset.", HttpStatus.INTERNAL_SERVER_ERROR); + } + } + + @ExceptionHandler(MetadataNotFoundException.class) @ResponseStatus(HttpStatus.NOT_FOUND) public ErrorInfo handleMetadataNotFoundException(MetadataNotFoundException ex) { diff --git a/src/main/java/gov/nist/oar/distrib/web/RPARequestHandlerController.java b/src/main/java/gov/nist/oar/distrib/web/RPARequestHandlerController.java index 0ddee587..183b3f3a 100644 --- a/src/main/java/gov/nist/oar/distrib/web/RPARequestHandlerController.java +++ b/src/main/java/gov/nist/oar/distrib/web/RPARequestHandlerController.java @@ -299,8 +299,8 @@ public ResponseEntity updateRecord(@PathVariable String id, @RequestBody RecordP LOGGER.debug("Missing required claim detected: " + missingClaimName); throw new InvalidRequestException("JWT token invalid"); } catch (JwtException e) { - LOGGER.debug("Token validation failed due to JwtException: " + e.getMessage()); - throw new RequestProcessingException("JWT token validation failed"); + LOGGER.debug("Token validation failed due to a JwtException: " + e.getMessage()); + throw new UnauthorizedException("JWT token validation failed"); } if (tokenDetails != null) { From 668350e8c7f41b436e2cd932e2f7426039fc809c Mon Sep 17 00:00:00 2001 From: elmiomar Date: Wed, 1 Nov 2023 11:07:07 -0400 Subject: [PATCH 02/20] add rpa- keyword to random id --- .../distrib/service/RPACachingService.java | 45 +++++++++++++++++-- 1 file changed, 41 insertions(+), 4 deletions(-) diff --git a/src/main/java/gov/nist/oar/distrib/service/RPACachingService.java b/src/main/java/gov/nist/oar/distrib/service/RPACachingService.java index 5e51793a..51e04da8 100644 --- a/src/main/java/gov/nist/oar/distrib/service/RPACachingService.java +++ b/src/main/java/gov/nist/oar/distrib/service/RPACachingService.java @@ -74,7 +74,9 @@ public String cacheAndGenerateRandomId(String datasetID, String version) } logger.debug("Caching dataset with dsid=" + dsid); - String randomID = generateRandomID(RANDOM_ID_LENGTH, true, true); + // append "rpa-" with the generated random ID + String randomID = "rpa-" + generateRandomID(RANDOM_ID_LENGTH, true, true); + int prefs = ROLE_RESTRICTED_DATA; if (!version.isEmpty()) @@ -248,9 +250,44 @@ private String generateRandomID(int length, boolean useLetters, boolean useNumbe return RandomStringUtils.random(length, useLetters, useNumbers); } + /** + * Uncache dataset objects using a specified random ID. + * + * @param randomId - The random ID used to fetch and uncache dataset objects. + * @return boolean - True if at least one dataset object was uncached successfully; otherwise, false. + * @throws CacheManagementException if an error occurs during the uncaching process. + */ public boolean uncacheById(String randomId) throws CacheManagementException { - logger.debug("Request to cache dataset with ID=" + randomId); - this.pdrCacheManager.uncache(randomId); - return false; + // Validate input + if (randomId == null || randomId.isEmpty()) { + throw new IllegalArgumentException("Random ID cannot be null or empty."); + } + + logger.debug("Request to uncache dataset with ID=" + randomId); + + // Retrieve dataset objects using the randomId + List objects = this.pdrCacheManager.selectDatasetObjects(randomId, this.pdrCacheManager.VOL_FOR_INFO); + + if (objects.isEmpty()) { + logger.debug("No objects found for ID=" + randomId); + return false; + } + + boolean isUncached = false; + + // Iterate through the retrieved objects and attempt to uncache them + for (CacheObject obj : objects) { + try { + logger.debug("Deleting file with ID=" + obj.id); + this.pdrCacheManager.uncache(obj.id); + isUncached = true; + } catch (CacheManagementException e) { + // Log the exception without throwing it to continue attempting to uncache remaining objects + logger.error("Failed to uncache object with ID=" + obj.id, e); + } + } + + return isUncached; } + } \ No newline at end of file From 878f44fd2be17c0db77a6e6c36d9a0b54e8d7cad Mon Sep 17 00:00:00 2001 From: elmiomar Date: Wed, 1 Nov 2023 11:10:36 -0400 Subject: [PATCH 03/20] update caching logic: cache first then append randomId to status --- ...URLConnectionRPARequestHandlerService.java | 48 ++++++++++++------- .../service/rpa/RecordResponseHandler.java | 2 +- .../rpa/RecordResponseHandlerImpl.java | 18 ++++--- 3 files changed, 39 insertions(+), 29 deletions(-) diff --git a/src/main/java/gov/nist/oar/distrib/service/rpa/HttpURLConnectionRPARequestHandlerService.java b/src/main/java/gov/nist/oar/distrib/service/rpa/HttpURLConnectionRPARequestHandlerService.java index 1ba31865..0de195e1 100644 --- a/src/main/java/gov/nist/oar/distrib/service/rpa/HttpURLConnectionRPARequestHandlerService.java +++ b/src/main/java/gov/nist/oar/distrib/service/rpa/HttpURLConnectionRPARequestHandlerService.java @@ -98,6 +98,7 @@ public class HttpURLConnectionRPARequestHandlerService implements IRPARequestHan */ private RecordResponseHandler recordResponseHandler; + private RPADatasetCacher rpaDatasetCacher; /** * Sets the HTTP URL connection factory. * @@ -164,6 +165,9 @@ public HttpURLConnectionRPARequestHandlerService(RPAConfiguration rpaConfigurati this.recordResponseHandler = new RecordResponseHandlerImpl(this.rpaConfiguration, this.connectionFactory, rpaCachingService); + // Set RPADatasetCacher + this.rpaDatasetCacher = new DefaultRPADatasetCacher(rpaCachingService); + // Set HttpClient this.httpClient = HttpClients.createDefault(); @@ -388,13 +392,16 @@ public RecordStatus updateRecord(String recordId, String status, String smeId) t // Initialize return object RecordStatus recordStatus; - // Get endpoint - String updateRecordUri = getConfig().getSalesforceEndpoints().get(UPDATE_RECORD_ENDPOINT_KEY); + // Cache here before updating the status in SF + LOGGER.info("Starting caching..."); + Record record = this.getRecord(recordId).getRecord(); + String datasetId = record.getUserInfo().getSubject(); + String randomId = this.rpaDatasetCacher.cache(datasetId); + if (randomId == null) + throw new RequestProcessingException("Caching process returned a null randomId"); // Create a valid approval status based on input - String approvalStatus = generateApprovalStatus(status, smeId); - - // TODO: try caching here before updating the status in SF + String approvalStatus = generateApprovalStatus(status, smeId, randomId); // PATCH request payload // Approval_Status__c is how SF service expect the key @@ -404,6 +411,9 @@ public RecordStatus updateRecord(String recordId, String status, String smeId) t // Get token JWTToken token = jwtHelper.getToken(); + // Get endpoint + String updateRecordUri = getConfig().getSalesforceEndpoints().get(UPDATE_RECORD_ENDPOINT_KEY); + // Build request URL String url; try { @@ -448,12 +458,9 @@ public RecordStatus updateRecord(String recordId, String status, String smeId) t throw new RequestProcessingException("I/O error: " + e.getMessage()); } - // Retrieve updated record from SF service - Record record = this.getRecord(recordId).getRecord(); - // Check if status is approved if (recordStatus.getApprovalStatus().toLowerCase().contains("approved")) { - this.recordResponseHandler.onRecordUpdateApproved(record); + this.recordResponseHandler.onRecordUpdateApproved(record, randomId); } else { this.recordResponseHandler.onRecordUpdateDeclined(record); } @@ -462,24 +469,29 @@ public RecordStatus updateRecord(String recordId, String status, String smeId) t } /** - * Generates an approval status string based on the given status and current date/time. - * The date is in ISO 8601 format. + * Generates an approval status string based on the given status, current date/time, and random ID. + * The date is in ISO 8601 format. If the status is "Declined", the randomId will not be appended. * - * @param status the approval status to use, either "Approved" or "Declined" - * @param email the email to append to the status - * @return the generated approval status string, in the format "[status]_[yyyy-MM-dd'T'HH:mm:ss.SSSZ]_[email]" + * @param status the approval status to use, either "Approved" or "Declined" + * @param smeId the SME ID to append to the status + * @param randomId the generated random ID to append (only if status is "Approved") + * @return the generated approval status string. + * If status is "Approved", the format is: + * "[status]_[yyyy-MM-dd'T'HH:mm:ss.SSSZ]_[smeId]_[randomId]". + * If status is "Declined", the format is: + * "[status]_[yyyy-MM-dd'T'HH:mm:ss.SSSZ]_[smeId]" * @throws InvalidRequestException if the provided status is not "Approved" or "Declined" */ - private String generateApprovalStatus(String status, String smeId) throws InvalidRequestException { + private String generateApprovalStatus(String status, String smeId, String randomId) throws InvalidRequestException { String formattedDate = Instant.now().toString(); // ISO 8601 format: 2023-05-09T15:59:03.872Z String approvalStatus; if (status != null) { switch (status.toLowerCase()) { case RECORD_APPROVED_STATUS: - approvalStatus = "Approved_"; + approvalStatus = "Approved_" + formattedDate + "_" + smeId + "_" + randomId; break; case RECORD_DECLINED_STATUS: - approvalStatus = "Declined_"; + approvalStatus = "Declined_" + formattedDate + "_" + smeId; break; default: throw new InvalidRequestException("Invalid approval status: " + status); @@ -487,7 +499,7 @@ private String generateApprovalStatus(String status, String smeId) throws Invali } else { throw new InvalidRequestException("Invalid approval status: status is null"); } - return approvalStatus + formattedDate + "_" + smeId; + return approvalStatus; } } diff --git a/src/main/java/gov/nist/oar/distrib/service/rpa/RecordResponseHandler.java b/src/main/java/gov/nist/oar/distrib/service/rpa/RecordResponseHandler.java index a1941a2e..7e29f588 100644 --- a/src/main/java/gov/nist/oar/distrib/service/rpa/RecordResponseHandler.java +++ b/src/main/java/gov/nist/oar/distrib/service/rpa/RecordResponseHandler.java @@ -23,7 +23,7 @@ public interface RecordResponseHandler { * This method is called when a record update operation is successful and user was approved. * @param record The record that was the user approved for. */ - void onRecordUpdateApproved(Record record); + void onRecordUpdateApproved(Record record, String randomId); /** * This method is called when a record update operation is successful but user was declined. diff --git a/src/main/java/gov/nist/oar/distrib/service/rpa/RecordResponseHandlerImpl.java b/src/main/java/gov/nist/oar/distrib/service/rpa/RecordResponseHandlerImpl.java index 5a456954..7a5315a0 100644 --- a/src/main/java/gov/nist/oar/distrib/service/rpa/RecordResponseHandlerImpl.java +++ b/src/main/java/gov/nist/oar/distrib/service/rpa/RecordResponseHandlerImpl.java @@ -92,20 +92,18 @@ public void onRecordCreationFailure(int statusCode) throws RequestProcessingExce } /** - * Called when a record status was updated to "Approved". - * This uses {@link RPADatasetCacher} to cache the dataset. + * Called when a record update operation has been approved. * - * @param record the record that was updated + * @param record the record that was updated and approved + * @param randomId the ID generated after caching the dataset related to the record + * @throws InvalidRequestException if there is an error in the request + * @throws RequestProcessingException if there is an error while processing the request */ @Override - public void onRecordUpdateApproved(Record record) throws InvalidRequestException, RequestProcessingException { - LOGGER.info("User was approved by SME. Starting caching..."); - String datasetId = record.getUserInfo().getSubject(); - // NEW: case dataset using the RPADatasetCacher - String randomId = this.rpaDatasetCacher.cache(datasetId); - if (randomId == null) - throw new RequestProcessingException("Caching process return a null randomId"); + public void onRecordUpdateApproved(Record record, String randomId) throws InvalidRequestException, RequestProcessingException { + LOGGER.info("Dataset was cached successfully. Sending email to user..."); + // Build Download URL String downloadUrl; try { From 02dbd568bece3f7ac42d2493f5803cd26c856db1 Mon Sep 17 00:00:00 2001 From: elmiomar Date: Wed, 29 Nov 2023 11:56:48 -0500 Subject: [PATCH 04/20] update caching logic --- .../service/rpa/DefaultRPADatasetCacher.java | 13 ++++ .../service/rpa/HttpRPADatasetCacher.java | 5 ++ ...URLConnectionRPARequestHandlerService.java | 69 +++++++++++++++---- .../distrib/service/rpa/RPADatasetCacher.java | 2 + .../rpa/RecordResponseHandlerImpl.java | 2 +- 5 files changed, 77 insertions(+), 14 deletions(-) diff --git a/src/main/java/gov/nist/oar/distrib/service/rpa/DefaultRPADatasetCacher.java b/src/main/java/gov/nist/oar/distrib/service/rpa/DefaultRPADatasetCacher.java index 3ac34daf..0e915b71 100644 --- a/src/main/java/gov/nist/oar/distrib/service/rpa/DefaultRPADatasetCacher.java +++ b/src/main/java/gov/nist/oar/distrib/service/rpa/DefaultRPADatasetCacher.java @@ -37,6 +37,19 @@ public String cache(String datasetId) throws RequestProcessingException { return randomId; } + @Override + public boolean uncache(String randomId) { + boolean uncached = false; + try { + uncached = rpaCachingService.uncacheById(randomId); + } catch (Exception e) { + this.logCachingException(e); + throw new RequestProcessingException(e.getMessage()); + } + + return uncached; + } + /** * Logs the specified exception to the debug log, along with its stack trace. * diff --git a/src/main/java/gov/nist/oar/distrib/service/rpa/HttpRPADatasetCacher.java b/src/main/java/gov/nist/oar/distrib/service/rpa/HttpRPADatasetCacher.java index 9bf5f76f..19204c88 100644 --- a/src/main/java/gov/nist/oar/distrib/service/rpa/HttpRPADatasetCacher.java +++ b/src/main/java/gov/nist/oar/distrib/service/rpa/HttpRPADatasetCacher.java @@ -50,6 +50,11 @@ public String cache(String datasetId) { return sendHttpRequest(datasetId, url); } + @Override + public boolean uncache(String randomId) { + return false; + } + /** * Builds the URL for the given dataset ID and using the given {@link RPAConfiguration} object. * diff --git a/src/main/java/gov/nist/oar/distrib/service/rpa/HttpURLConnectionRPARequestHandlerService.java b/src/main/java/gov/nist/oar/distrib/service/rpa/HttpURLConnectionRPARequestHandlerService.java index 0de195e1..6a391b9e 100644 --- a/src/main/java/gov/nist/oar/distrib/service/rpa/HttpURLConnectionRPARequestHandlerService.java +++ b/src/main/java/gov/nist/oar/distrib/service/rpa/HttpURLConnectionRPARequestHandlerService.java @@ -377,28 +377,52 @@ private String prepareRequestPayload(UserInfoWrapper userInfoWrapper) throws Jso /** - * Updates the status of a record with a given ID. + * Updates the status of a record in the database. + *

+ * This method handles the approval or decline of a record. When a record is approved, it caches the dataset, + * generates a random ID, and appends this ID to the status before updating the record in the database. + * When a record is declined, the dataset is not cached, a null random ID is used, and the record status is + * updated in the database without appending the random ID. + *

+ *

+ * In cases where a record was initially approved (and thus cached with a random ID) but is later declined, + * this method retrieves the random ID from the status, uncaches the dataset using this ID, and updates + * the record status without the random ID. + *

* * @param recordId The ID of the record to update. - * @param status The status to update the record with. - * @return The {@link RecordStatus} object representing the updated record status. + * @param status The new status to be set for the record. Can be 'Approved' or 'Declined'. + * @param smeId The SME ID associated with the record update. + * @return A {@link RecordStatus} object representing the updated record status. * @throws RecordNotFoundException If the record with the given ID is not found. - * @throws InvalidRequestException If the request is invalid. - * @throws RequestProcessingException If there is an error processing the request. + * @throws InvalidRequestException If the provided status is invalid or the request is otherwise invalid. + * @throws RequestProcessingException If there is an error in processing the update request, such as issues + * with caching or communication errors with the database. */ @Override public RecordStatus updateRecord(String recordId, String status, String smeId) throws RecordNotFoundException, InvalidRequestException, RequestProcessingException { // Initialize return object RecordStatus recordStatus; - - // Cache here before updating the status in SF - LOGGER.info("Starting caching..."); Record record = this.getRecord(recordId).getRecord(); String datasetId = record.getUserInfo().getSubject(); - String randomId = this.rpaDatasetCacher.cache(datasetId); - if (randomId == null) - throw new RequestProcessingException("Caching process returned a null randomId"); + String randomId = null; + + // If the record is being approved + if (RECORD_APPROVED_STATUS.equalsIgnoreCase(status)) { + LOGGER.info("Starting caching..."); + randomId = this.rpaDatasetCacher.cache(datasetId); + if (randomId == null) { + throw new RequestProcessingException("Caching process returned a null randomId"); + } + } + // If the record is being declined, check if it needs uncaching + else if (RECORD_DECLINED_STATUS.equalsIgnoreCase(status)) { + randomId = extractRandomIdFromCurrentStatus(record.getUserInfo().getApprovalStatus()); + if (randomId != null) { + this.rpaDatasetCacher.uncache(randomId); + } + } // Create a valid approval status based on input String approvalStatus = generateApprovalStatus(status, smeId, randomId); @@ -468,6 +492,20 @@ public RecordStatus updateRecord(String recordId, String status, String smeId) t return recordStatus; } + private String extractRandomIdFromCurrentStatus(String currentStatus) { + if (currentStatus != null && currentStatus.startsWith("Approved_")) { + String[] parts = currentStatus.split("_"); + + // Since the expected format is "[status]_[yyyy-MM-dd'T'HH:mm:ss.SSSZ]_[smeId]_[randomId]", + // the randomId should be the 4th part, if all parts are present + if (parts.length == 4) { + return parts[3]; + } + } + return null; + } + + /** * Generates an approval status string based on the given status, current date/time, and random ID. * The date is in ISO 8601 format. If the status is "Declined", the randomId will not be appended. @@ -483,12 +521,16 @@ public RecordStatus updateRecord(String recordId, String status, String smeId) t * @throws InvalidRequestException if the provided status is not "Approved" or "Declined" */ private String generateApprovalStatus(String status, String smeId, String randomId) throws InvalidRequestException { - String formattedDate = Instant.now().toString(); // ISO 8601 format: 2023-05-09T15:59:03.872Z + String formattedDate = Instant.now().toString(); String approvalStatus; + if (status != null) { switch (status.toLowerCase()) { case RECORD_APPROVED_STATUS: - approvalStatus = "Approved_" + formattedDate + "_" + smeId + "_" + randomId; + approvalStatus = "Approved_" + formattedDate + "_" + smeId; + if (randomId != null) { + approvalStatus += "_" + randomId; + } break; case RECORD_DECLINED_STATUS: approvalStatus = "Declined_" + formattedDate + "_" + smeId; @@ -502,4 +544,5 @@ private String generateApprovalStatus(String status, String smeId, String random return approvalStatus; } + } diff --git a/src/main/java/gov/nist/oar/distrib/service/rpa/RPADatasetCacher.java b/src/main/java/gov/nist/oar/distrib/service/rpa/RPADatasetCacher.java index 2a2150c3..93395cc8 100644 --- a/src/main/java/gov/nist/oar/distrib/service/rpa/RPADatasetCacher.java +++ b/src/main/java/gov/nist/oar/distrib/service/rpa/RPADatasetCacher.java @@ -14,4 +14,6 @@ public interface RPADatasetCacher { * @throws RequestProcessingException */ String cache(String datasetId) throws RequestProcessingException; + + boolean uncache(String randomId); } diff --git a/src/main/java/gov/nist/oar/distrib/service/rpa/RecordResponseHandlerImpl.java b/src/main/java/gov/nist/oar/distrib/service/rpa/RecordResponseHandlerImpl.java index 7a5315a0..ca7f4150 100644 --- a/src/main/java/gov/nist/oar/distrib/service/rpa/RecordResponseHandlerImpl.java +++ b/src/main/java/gov/nist/oar/distrib/service/rpa/RecordResponseHandlerImpl.java @@ -125,7 +125,7 @@ public void onRecordUpdateApproved(Record record, String randomId) throws Invali * @param record the record that was updated */ @Override - public void onRecordUpdateDeclined(Record record) { + public void onRecordUpdateDeclined(Record record) throws InvalidRequestException, RequestProcessingException { LOGGER.debug("User was declined by SME"); } From 830675c36539eedb75eb5bf44c65ece0673c1150 Mon Sep 17 00:00:00 2001 From: elmiomar Date: Tue, 5 Dec 2023 06:57:16 -0500 Subject: [PATCH 05/20] include aipid in Download URLs --- .../cachemgr/pdr/PDRDatasetRestorer.java | 2 +- .../distrib/service/RPACachingService.java | 43 ++++++++++++++----- 2 files changed, 33 insertions(+), 12 deletions(-) diff --git a/src/main/java/gov/nist/oar/distrib/cachemgr/pdr/PDRDatasetRestorer.java b/src/main/java/gov/nist/oar/distrib/cachemgr/pdr/PDRDatasetRestorer.java index 12b77b2d..f4da33b1 100644 --- a/src/main/java/gov/nist/oar/distrib/cachemgr/pdr/PDRDatasetRestorer.java +++ b/src/main/java/gov/nist/oar/distrib/cachemgr/pdr/PDRDatasetRestorer.java @@ -645,7 +645,7 @@ public String idForObject(String aipid, String filepath, String forVersion, Stri String id; id = aipid + "/" + filepath; if (target != null && !target.isEmpty()) - id = target + "/" + filepath; + id = target + "/" + aipid + "/" + filepath; if (forVersion != null && forVersion.length() > 0) id += "#" + forVersion; return id; diff --git a/src/main/java/gov/nist/oar/distrib/service/RPACachingService.java b/src/main/java/gov/nist/oar/distrib/service/RPACachingService.java index 51e04da8..b3abf742 100644 --- a/src/main/java/gov/nist/oar/distrib/service/RPACachingService.java +++ b/src/main/java/gov/nist/oar/distrib/service/RPACachingService.java @@ -129,6 +129,7 @@ public Map retrieveMetadata(String randomID) throws CacheManagem /** * Formats the metadata from a cache object to a JSON object with an additional field for the download URL. + * The download URL includes the random temporary ID, aipid, and the file path from the metadata. * * @param inMd the metadata from the cache object * @param randomID the random temporary ID associated with the cache object @@ -139,17 +140,27 @@ private JSONObject formatMetadata(JSONObject inMd, String randomID) throws Reque JSONObject outMd = new JSONObject(); List missingFields = new ArrayList<>(); + String aipid = ""; + if (inMd.has("aipid")) { + aipid = inMd.getString("aipid"); + outMd.put("aipid", aipid); + } else { + missingFields.add("aipid"); + } + if (inMd.has("filepath")) { String downloadURL = getDownloadUrl( rpaConfiguration.getBaseDownloadUrl(), randomID, - inMd.get("filepath").toString()); + aipid, + inMd.getString("filepath")); outMd.put("downloadURL", downloadURL); - outMd.put("filePath", inMd.get("filepath")); + outMd.put("filePath", inMd.getString("filepath")); } else { missingFields.add("filepath"); } + if (inMd.has("contentType")) { outMd.put("mediaType", inMd.get("contentType")); } else { @@ -199,12 +210,6 @@ private JSONObject formatMetadata(JSONObject inMd, String randomID) throws Reque missingFields.add("ediid"); } - if (inMd.has("aipid")) { - outMd.put("aipid", inMd.get("aipid")); - } else { - missingFields.add("aipid"); - } - if (inMd.has("sinceDate")) { outMd.put("sinceDate", inMd.get("sinceDate")); } else { @@ -220,28 +225,44 @@ private JSONObject formatMetadata(JSONObject inMd, String randomID) throws Reque /** - * Constructs a download URL using the given base download URL, random ID, and file path from the metadata. + * Constructs a download URL using the given base download URL, random ID, aipid, and file path from the metadata. * * @param baseDownloadUrl the base download URL * @param randomId the random temporary ID + * @param aipid the aipid from the metadata * @param path the file path from the metadata * @return the download URL as a string * @throws RequestProcessingException if there was an error building the download URL */ - private String getDownloadUrl(String baseDownloadUrl, String randomId, String path) throws RequestProcessingException { + + private String getDownloadUrl(String baseDownloadUrl, String randomId, String aipid, String path) throws RequestProcessingException { URL downloadUrl; try { URL url = new URL(baseDownloadUrl); + StringBuilder pathBuilder = new StringBuilder(); + + // append the randomId to the path + pathBuilder.append(randomId); + + // append the aipid if it's not empty + if (!aipid.isEmpty()) { + pathBuilder.append("/").append(aipid); + } + + // append the file path, ensuring it doesn't start with a "/" if (path.startsWith("/")) { path = path.substring(1); } - downloadUrl = new URL(url, randomId + "/" + path); + pathBuilder.append("/").append(path); + + downloadUrl = new URL(url, pathBuilder.toString()); } catch (MalformedURLException e) { throw new RequestProcessingException("Failed to build downloadUrl: " + e.getMessage()); } return downloadUrl.toString(); } + /** * Generate a random alphanumeric string for the dataset to store * This function uses the {@link RandomStringUtils} from Apache Commons. From 9bbd5847f8ca5298b3a299f2e30c0f16302202c4 Mon Sep 17 00:00:00 2001 From: elmiomar Date: Tue, 5 Dec 2023 07:41:36 -0500 Subject: [PATCH 06/20] add certificates --- docker/build-test/Dockerfile | 24 +++++++---- .../cacerts/Forward_Proxy_NIST_CA.crt | 41 +++++++++++++++++++ docker/build-test/cacerts/NISTRoot02.crt | 29 +++++++++++++ docker/build-test/cacerts/README.md | 13 ++++++ docker/cacerts/README.md | 13 ++++++ 5 files changed, 111 insertions(+), 9 deletions(-) create mode 100644 docker/build-test/cacerts/Forward_Proxy_NIST_CA.crt create mode 100644 docker/build-test/cacerts/NISTRoot02.crt create mode 100644 docker/build-test/cacerts/README.md create mode 100644 docker/cacerts/README.md diff --git a/docker/build-test/Dockerfile b/docker/build-test/Dockerfile index 431e9d1c..92068bad 100644 --- a/docker/build-test/Dockerfile +++ b/docker/build-test/Dockerfile @@ -1,14 +1,20 @@ -FROM ibmjava:8-sdk - -# a hack that gets around an installation problem with update-alternatives, openjdk-8-jdk-headless +FROM eclipse-temurin:8-jdk-focal RUN mkdir -p /usr/share/man/man1 - -RUN echo "deb http://archive.debian.org/debian stretch main contrib non-free" > /etc/apt/sources.list - -RUN apt-get update && apt-get upgrade -y && apt-get install -y netcat-openbsd zip git less \ - python2 curl maven -RUN cd /usr/bin && ln -s python2 python +RUN apt-get update && apt-get install -y netcat-openbsd zip git less \ + ca-certificates python3 curl maven gnupg +RUN cd /usr/bin && ln -s python3 python + +COPY cacerts/README.md cacerts/*.crt /usr/local/share/ca-certificates/ +RUN update-ca-certificates +RUN java_certs=$JAVA_HOME/jre/lib/security/cacerts; \ + add_certs=`ls /usr/local/share/ca-certificates/*.crt` && \ + for crt in $add_certs; do \ + name=`basename -s .crt $crt`; \ + echo -n ${name}: " "; \ + keytool -import -keystore $java_certs -trustcacerts -file $crt \ + -storepass changeit -alias $name -noprompt; \ + done; # Create the user that build/test operations should run as. Normally, # this is set to match identity information of the host user that is diff --git a/docker/build-test/cacerts/Forward_Proxy_NIST_CA.crt b/docker/build-test/cacerts/Forward_Proxy_NIST_CA.crt new file mode 100644 index 00000000..6c7a12cb --- /dev/null +++ b/docker/build-test/cacerts/Forward_Proxy_NIST_CA.crt @@ -0,0 +1,41 @@ +-----BEGIN CERTIFICATE----- +MIIG7TCCBNWgAwIBAgITGAAAAAecWWKCXTfeJAAAAAAABzANBgkqhkiG9w0BAQsF +ADAVMRMwEQYDVQQDEwpOSVNUUm9vdDAyMB4XDTIxMTIwMTE4MDU0NloXDTI2MTIw +MTE4MTU0NlowgcIxCzAJBgNVBAYTAlVTMREwDwYDVQQIEwhNYXJ5bGFuZDEVMBMG +A1UEBxMMR2FpdGhlcnNidXJnMTcwNQYDVQQKEy5OYXRpb25hbCBJbnN0aXR1dGUg +b2YgU3RhbmRhcmRzIGFuZCBUZWNobm9sb2d5MQ0wCwYDVQQLEwRPSVNNMR4wHAYD +VQQDDBVGb3J3YXJkX1Byb3h5X05JU1RfQ0ExITAfBgkqhkiG9w0BCQEWEm5ldHNl +Y3VyZUBuaXN0LmdvdjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAL/8 +PgucU6LfbThmVCiQU5zH7HRdJ0QeM8xa9Hy3BnBdD4/CxQklo7dz+AXquaOfI5Br +H8SYZCySWTveFeJW+XvhjmEVpobz8GGrEgdR5nAKg3ZJHvAMPKgGMSnXja227TVj +qqCZX9cIWQifqcM1iWTkS4BW2oZazwXYCqs5dfwy92ey5f/7AYC4dFeL//QtqQs/ +EUFApYabhKLcDLleDh4hwlhbTO9Zjt/eRujB/5f183RVb+igoy/xVZ8S82cNpxHS +2DdO58GZzvAgYMYuXXJkdINkag/fpCXEy9bGaDfydHLpTWviiGz3HfXh/Chb66BG +ZoZJmJrovVO9rSMyptMCAwEAAaOCAoYwggKCMB0GA1UdDgQWBBQquDqJ3U24XoOQ +/6y8kFgbAp9fPDAfBgNVHSMEGDAWgBQlEQPjYg4e56GOSdev1HJtWx0z+TCB/QYD +VR0fBIH1MIHyMIHvoIHsoIHphjFodHRwOi8vbmlzdHBraS5uaXN0Lmdvdi9DZXJ0 +RW5yb2xsL05JU1RSb290MDIuY3JshoGzbGRhcDovLy9DTj1OSVNUUm9vdDAyLENO +PU5JU1Ryb290Q0EwMixDTj1DRFAsQ049UHVibGljJTIwS2V5JTIwU2VydmljZXMs +Q049U2VydmljZXMsQ049Q29uZmlndXJhdGlvbixEQz1OSVNULERDPUdPVj9jZXJ0 +aWZpY2F0ZVJldm9jYXRpb25MaXN0P2Jhc2U/b2JqZWN0Q2xhc3M9Y1JMRGlzdHJp +YnV0aW9uUG9pbnQwggEFBggrBgEFBQcBAQSB+DCB9TBKBggrBgEFBQcwAoY+aHR0 +cDovL25pc3Rwa2kubmlzdC5nb3YvQ2VydEVucm9sbC9OSVNUcm9vdENBMDJfTklT +VFJvb3QwMi5jcnQwgaYGCCsGAQUFBzAChoGZbGRhcDovLy9DTj1OSVNUUm9vdDAy +LENOPUFJQSxDTj1QdWJsaWMlMjBLZXklMjBTZXJ2aWNlcyxDTj1TZXJ2aWNlcyxD +Tj1Db25maWd1cmF0aW9uLERDPU5JU1QsREM9R09WP2NBQ2VydGlmaWNhdGU/YmFz +ZT9vYmplY3RDbGFzcz1jZXJ0aWZpY2F0aW9uQXV0aG9yaXR5MBkGCSsGAQQBgjcU +AgQMHgoAUwB1AGIAQwBBMA8GA1UdEwEB/wQFMAMBAf8wCwYDVR0PBAQDAgGGMA0G +CSqGSIb3DQEBCwUAA4ICAQB3OCkcbjepVN7tbK3PlLzG5HkRBG1QSmFsRnQdUTov +/rWhdLDpHGKO4k/W2zTxNNxPW8ooD1PCy+cIlBLGcq8YcyhvWk0V2Gx1P+/f4+eq +eH4hcUQO/7INohcnh4QXiSVMa7jNaLC+/usqWbsmTvVDbl2aYbQtwizXnUW1qNhz +Bt76OoM7C95rNktNiaJ1VmFmd+Z3rRhzAZiC9XFIwIN1F+um7IG43nsoM4hnCByc +/SBb3LC8R+7vNUYedkrfNPq8SGCHuPuK8H0gJX+8/8hmaaNPtZoe0VZkTdNXitnY +HNof6w5mDoPu9lgLmNO0c36dNrmhHlPAu71EkL3afBhrdgb4Gel0WlENaur2MWf+ +yg6IQz7+aCTu2bMIkW3gm942tp7IrkXMGshUsJjLHFVrpIVkP+70QnO0wGzzQWlI +gt+/gKvj951KGagVzsFyiQtFL9uFYMiS0awLVkSLYtBzdykm8mpG1n6EO5DlEYWe +MOhVSeki05s0+6zUWU6TIhVDgCeUJYvAYAtWVA07Tbb1lb1vP+KbWzFMuAQMrKXV +I0sL/gjcwaj18n8vb0NdVU2n4qoW44gBi8ocgbuBntt63J4GHpaIn/I4OHBiwu/2 +IwUrfePEVCI2pAm/sBfw2XiofAclxBhhJniiRoMYPKCOdnPRP1nUWOdotzPJFPJe +1Q== +-----END CERTIFICATE----- + diff --git a/docker/build-test/cacerts/NISTRoot02.crt b/docker/build-test/cacerts/NISTRoot02.crt new file mode 100644 index 00000000..9aeb53c3 --- /dev/null +++ b/docker/build-test/cacerts/NISTRoot02.crt @@ -0,0 +1,29 @@ +-----BEGIN CERTIFICATE----- +MIIFBTCCAu2gAwIBAgIQdRxyg4+47KRFWKY+545EJjANBgkqhkiG9w0BAQsFADAV +MRMwEQYDVQQDEwpOSVNUUm9vdDAyMB4XDTE4MDgwMTE4MTgzOVoXDTM4MDgwMTE4 +Mjc1M1owFTETMBEGA1UEAxMKTklTVFJvb3QwMjCCAiIwDQYJKoZIhvcNAQEBBQAD +ggIPADCCAgoCggIBAJxQaJgDFbHCPwx8YOrjfthNQP7TOra9C4SkeURpetVq3fk1 +AqGgcqYzN3SRxtx9xJUweFBayO83jyBx5d+LLqX9LctaIrS4gU3uLGqDEQJisMST ++r6/mF51H5xF9AaiH8ca6ZopjigYdcv0ivMiUh8UWDvZF8SnPq4BaId4D3UwfVhV +p8Nh9osU04BXGSOIaN5dL4CdNiOleC7IqAl4wXekOMkNfIErp2QeLnq/g1xIFmCv +Dz+4umnPIVAYvuIKa39irNLi7j9XqUpnNcfBAvaypOe9e31RqWEYbHKhYXtFMJ6v +Ui/d+pPPJ0HfoMu2toCZHgMCxzaFnGh0reMkcCrPpH2EQIQzbJaV4QVRFvAfNIF5 +cwvb6mRJ9pqjlIVAoT+//YUy1IsG+4n0TZAEJa9G61G3bGr7Chh+uWYGfmpevY8I +GUTNmhYc5pGma6TFR3Hqil9PwAnPcXYQDnjhwVOGRrC/Ze9LymT7tUIEX0JKmZ0J +ds50u8T0joWwacwK1RYdj0YC4PLeLFB2obqcfust4KCN/Hw7/pvwN3sFhbC1dn2G +YIjqiDaenI7Gsb2t5Q8AOQbMSCJu0RYI9XN8Uzm+v0zseLF4V0+43PSTxDnlBzms +cpjRsMRk563nVnL4oHa+LhJnB/YTBqE86bzieTiIL7SqGW1hH+RJWn55pFtnAgMB +AAGjUTBPMAsGA1UdDwQEAwIBhjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBQl +EQPjYg4e56GOSdev1HJtWx0z+TAQBgkrBgEEAYI3FQEEAwIBADANBgkqhkiG9w0B +AQsFAAOCAgEAHscEbpIIPKe+avqPPxUJxRnnlV9CoBZSN4IJcA3Iox3f7zJdeLra +hMJq8vJBLK0barh9ofLbviX1tBzAqDFd6RnMaMWTfv2BgjtoZNqfFqRp632ErDTI +ONyHbGOnuWGXatwRNXUIhhx2UGeAy38xrIU8Z0ssTCsRY374WSFYaR5Ww7hfunyi +eBmofMY+j6flNxEqckV3BeIarJxWmpEaAihczZxJsnZXW+D0B7h4EKZ/DakOl2QA +59aE740ToPAl+pAF4OhT53xPlju+tqkaLnVJg/kI7Qrc0S2mHGrXnDl1FUya8VFS +Vm8bf3nd483e3nWnSVU+vItlRIrtoHnLQ7xzMkurUNo2pROR+JgsL5WL0+NDGFjv +Ixf9ReYGN9ujrHtojZiFaDLMPUftV6EVk2qc2d8BMEAnVzy8WJk6iqiWsmYaE2uq +wdQHiP8kwQhXXRbqhfFZWwSisga4TIZu65rR88ah08DOGaTLfqKUnb9WD4dzTDFH +XBl6ryuOeJGBoeJVbjy5938ZKHSS/nP3H/zYwve7xBw8CkmKAA1ECLJ47iWFmlyr +mQkr8lkaupRMxgV8LUml35hI4lT2SbvAbdsuP/RvuvrK+mHS2UEDjG/qz4aTuXrm +uMnUuya/1QGPhFD1oztxrhem2ob2jfkRfWT6wbv8UK7Mniw2zBfISXY= +-----END CERTIFICATE----- diff --git a/docker/build-test/cacerts/README.md b/docker/build-test/cacerts/README.md new file mode 100644 index 00000000..b650e217 --- /dev/null +++ b/docker/build-test/cacerts/README.md @@ -0,0 +1,13 @@ +This directory contains non-standard CA certificates needed to build the docker +images. + +Failures building the Docker containers defined in ../ due to SSL certificate +verification errors may be a consequence of your local network's firewall. In +particular, the firewall may be substituting external site certificates with +its own signed by a non-standard CA certficate (chain). If so, you can place +the necessary certificates into this directory; they will be passed into the +containers, allowing them to safely connect to those external sites. + +Be sure the certificates are in PEM format and include a .crt file extension. + +Do not remove this README file; doing so may cause a Docker build faiure. \ No newline at end of file diff --git a/docker/cacerts/README.md b/docker/cacerts/README.md new file mode 100644 index 00000000..b650e217 --- /dev/null +++ b/docker/cacerts/README.md @@ -0,0 +1,13 @@ +This directory contains non-standard CA certificates needed to build the docker +images. + +Failures building the Docker containers defined in ../ due to SSL certificate +verification errors may be a consequence of your local network's firewall. In +particular, the firewall may be substituting external site certificates with +its own signed by a non-standard CA certficate (chain). If so, you can place +the necessary certificates into this directory; they will be passed into the +containers, allowing them to safely connect to those external sites. + +Be sure the certificates are in PEM format and include a .crt file extension. + +Do not remove this README file; doing so may cause a Docker build faiure. \ No newline at end of file From 9a4f79bde47dfa374119f74eba25723ab0e8bf5b Mon Sep 17 00:00:00 2001 From: elmiomar Date: Wed, 31 Jan 2024 11:41:25 -0500 Subject: [PATCH 07/20] create CacheExpiryCheck for automated removal of expired cached data --- .../distrib/cachemgr/CacheExpiryCheck.java | 62 +++++++++++ .../cachemgr/CacheExpiryCheckTest.java | 102 ++++++++++++++++++ 2 files changed, 164 insertions(+) create mode 100644 src/main/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheck.java create mode 100644 src/test/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheckTest.java diff --git a/src/main/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheck.java b/src/main/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheck.java new file mode 100644 index 00000000..64a1bd56 --- /dev/null +++ b/src/main/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheck.java @@ -0,0 +1,62 @@ +package gov.nist.oar.distrib.cachemgr; + +import gov.nist.oar.distrib.StorageVolumeException; + +/** + * Implements a cache object check to identify and remove objects that have been in the cache + * longer than a specified duration, specifically two weeks. This check helps in + * managing cache integrity by ensuring that stale or outdated data are removed + * from the cache. + */ +public class CacheExpiryCheck implements CacheObjectCheck { + + private static final long TWO_WEEKS_MILLIS = 14 * 24 * 60 * 60 * 1000; // 14 days in milliseconds + private StorageInventoryDB inventoryDB; + + public CacheExpiryCheck(StorageInventoryDB inventoryDB) { + this.inventoryDB = inventoryDB; + } + + /** + * Checks whether a cache object has expired based on its last modified time and removes it if expired. + * An object is considered expired if it has been in the cache for more than two weeks. + * + * @param co The CacheObject to be checked for expiry. + * @throws IntegrityException if the cache object's last modified time is unknown. + * @throws StorageVolumeException if there is an issue removing the expired object from the cache volume. + */ + @Override + public void check(CacheObject co) throws IntegrityException, StorageVolumeException { + long currentTime = System.currentTimeMillis(); + long objectLastModifiedTime = co.getLastModified(); + + // Throw an exception if the last modified time is unknown + if (objectLastModifiedTime == -1) { + throw new IntegrityException("Last modified time of cache object is unknown: " + co.name); + } + + // If the cache object is expired, remove it from the cache + if ((currentTime - objectLastModifiedTime) > TWO_WEEKS_MILLIS) { + removeExpiredObject(co); + } + } + + /** + * Removes an expired object from the cache. + * + * @param co The expired CacheObject to be removed. + * @throws StorageVolumeException if there is an issue removing the object from the cache volume. + */ + protected void removeExpiredObject(CacheObject co) throws StorageVolumeException { + CacheVolume volume = co.volume; + if (volume != null && volume.remove(co.name)) { + try { + inventoryDB.removeObject(co.volname, co.name); + } catch (InventoryException e) { + throw new StorageVolumeException("Failed to remove object from inventory database: " + co.name, e); + } + } else { + throw new StorageVolumeException("Failed to remove expired object from cache volume: " + co.name); + } + } +} diff --git a/src/test/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheckTest.java b/src/test/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheckTest.java new file mode 100644 index 00000000..dad89ab7 --- /dev/null +++ b/src/test/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheckTest.java @@ -0,0 +1,102 @@ +package gov.nist.oar.distrib.cachemgr; + +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +import static org.mockito.Mockito.anyString; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class CacheExpiryCheckTest { + + @Mock + private StorageInventoryDB mockInventoryDB; + @Mock + private CacheVolume mockVolume; + @Mock + private CacheObject cacheObject; + private CacheExpiryCheck expiryCheck; + + @Before + public void setUp() { + MockitoAnnotations.initMocks(this); + expiryCheck = new CacheExpiryCheck(mockInventoryDB); + } + + /** + * Test to verify that {@link CacheExpiryCheck} correctly identifies and processes an expired cache object. + * An object is considered expired if its last modified time is more than two weeks ago. + * This test checks that the expired object is appropriately removed from both the cache volume and the inventory + * database. + * + * @throws Exception to handle any exceptions thrown during the test execution + */ + @Test + public void testExpiredObject() throws Exception { + // Setup mock + cacheObject.name = "testObject"; + cacheObject.volname = "testVolume"; + cacheObject.volume = mockVolume; + long lastModified = System.currentTimeMillis() - (15 * 24 * 60 * 60 * 1000); // 15 days in milliseconds + when(cacheObject.getLastModified()).thenReturn(lastModified); + + + when(mockVolume.remove("testObject")).thenReturn(true); + + // Perform the check + expiryCheck.check(cacheObject); + + // Verify the interactions + verify(mockVolume).remove("testObject"); + verify(mockInventoryDB).removeObject(anyString(), anyString()); + } + + + /** + * Test to ensure that {@link CacheExpiryCheck} does not flag a cache object as expired if it has been + * modified within the last two weeks. This test checks that no removal action is taken for a non-expired object. + * + * @throws Exception to handle any exceptions thrown during the test execution + */ + @Test + public void testNonExpiredObject() throws Exception { + // Setup mock + cacheObject.name = "nonExpiredObject"; + cacheObject.volname = "testVolume"; + cacheObject.volume = mockVolume; + + long lastModified = System.currentTimeMillis() - (5 * 24 * 60 * 60 * 1000); // 5 days in milliseconds + when(cacheObject.getLastModified()).thenReturn(lastModified); + + // Perform the check + expiryCheck.check(cacheObject); + + // Verify that the remove method was not called as the object is not expired + verify(mockVolume, never()).remove("nonExpiredObject"); + verify(mockInventoryDB, never()).removeObject("testVolume", "nonExpiredObject"); + } + + /** + * Test to verify that {@link CacheExpiryCheck} throws an {@link IntegrityException} when the last modified time + * of a cache object is unknown (indicated by a value of -1). This situation should be flagged as an error + * as the expiry status of the object cannot be determined. + * + * @throws Exception to handle any exceptions thrown during the test execution + */ + @Test(expected = IntegrityException.class) + public void testUnknownLastModifiedTime() throws Exception { + // Setup mock + cacheObject.name = "unknownLastModifiedObject"; + cacheObject.volname = "testVolume"; + cacheObject.volume = mockVolume; + long lastModified = -1; // Unknown last modified time + when(cacheObject.getLastModified()).thenReturn(lastModified); + + // Perform the check, expecting an IntegrityException + expiryCheck.check(cacheObject); + } + +} From ae225b20224a161ca78e29f89f74c516af7dd8e6 Mon Sep 17 00:00:00 2001 From: elmiomar Date: Wed, 31 Jan 2024 17:04:24 -0500 Subject: [PATCH 08/20] add the CacheExpiryCheck to the list of checks --- .../gov/nist/oar/distrib/web/NISTCacheManagerConfig.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/gov/nist/oar/distrib/web/NISTCacheManagerConfig.java b/src/main/java/gov/nist/oar/distrib/web/NISTCacheManagerConfig.java index cfb74fab..da4d674d 100644 --- a/src/main/java/gov/nist/oar/distrib/web/NISTCacheManagerConfig.java +++ b/src/main/java/gov/nist/oar/distrib/web/NISTCacheManagerConfig.java @@ -12,9 +12,11 @@ package gov.nist.oar.distrib.web; import gov.nist.oar.distrib.cachemgr.BasicCache; +import gov.nist.oar.distrib.cachemgr.CacheExpiryCheck; import gov.nist.oar.distrib.cachemgr.ConfigurableCache; import gov.nist.oar.distrib.cachemgr.CacheManagementException; import gov.nist.oar.distrib.cachemgr.CacheVolume; +import gov.nist.oar.distrib.cachemgr.StorageInventoryDB; import gov.nist.oar.distrib.cachemgr.storage.AWSS3CacheVolume; import gov.nist.oar.distrib.cachemgr.storage.FilesystemCacheVolume; import gov.nist.oar.distrib.cachemgr.VolumeStatus; @@ -485,6 +487,9 @@ public PDRCacheManager createCacheManager(BasicCache cache, PDRDatasetRestorer r List checks = new ArrayList(); checks.add(new ChecksumCheck(false, true)); + // Get the StorageInventoryDB from the cache and add the CacheExpiryCheck to the list of checks + StorageInventoryDB inventoryDB = cache.getInventoryDB(); + checks.add(new CacheExpiryCheck(inventoryDB)); PDRCacheManager out = new PDRCacheManager(cache, rstr, checks, getCheckDutyCycle()*1000, getCheckGracePeriod()*1000, -1, rootdir, logger); if (getMonitorAutoStart()) { From c73ce93baf8a4f852139e682204514787cab8b81 Mon Sep 17 00:00:00 2001 From: elmiomar Date: Tue, 20 Feb 2024 08:43:03 -0500 Subject: [PATCH 09/20] Implement dynamic cache expiration based on expiresIn metadata --- .../distrib/cachemgr/CacheExpiryCheck.java | 58 ++++++++++--------- .../cachemgr/CacheExpiryCheckTest.java | 49 ++++++++++------ 2 files changed, 62 insertions(+), 45 deletions(-) diff --git a/src/main/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheck.java b/src/main/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheck.java index 64a1bd56..5a82d2bd 100644 --- a/src/main/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheck.java +++ b/src/main/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheck.java @@ -2,6 +2,8 @@ import gov.nist.oar.distrib.StorageVolumeException; +import java.time.Instant; + /** * Implements a cache object check to identify and remove objects that have been in the cache * longer than a specified duration, specifically two weeks. This check helps in @@ -10,7 +12,7 @@ */ public class CacheExpiryCheck implements CacheObjectCheck { - private static final long TWO_WEEKS_MILLIS = 14 * 24 * 60 * 60 * 1000; // 14 days in milliseconds + // private static final long TWO_WEEKS_MILLIS = 14 * 24 * 60 * 60 * 1000; // 14 days in milliseconds private StorageInventoryDB inventoryDB; public CacheExpiryCheck(StorageInventoryDB inventoryDB) { @@ -18,45 +20,45 @@ public CacheExpiryCheck(StorageInventoryDB inventoryDB) { } /** - * Checks whether a cache object has expired based on its last modified time and removes it if expired. - * An object is considered expired if it has been in the cache for more than two weeks. + * Checks if a cache object is expired and removes it from the cache if it is. + * The method uses the {@code expiresIn} metadata field to determine the expiration status. + * The expiration time is calculated based on the {@code LastModified} time plus the {@code expiresIn} duration. + * If the current time is past the calculated expiry time, the object is removed from the inventory database. * - * @param co The CacheObject to be checked for expiry. - * @throws IntegrityException if the cache object's last modified time is unknown. - * @throws StorageVolumeException if there is an issue removing the expired object from the cache volume. + * @param co The cache object to check for expiration. + * @throws IntegrityException If the object is found to be corrupted during the check. + * @throws StorageVolumeException If there's an error accessing the storage volume during the check. + * @throws CacheManagementException If there's an error managing the cache, including removing the expired object. */ @Override - public void check(CacheObject co) throws IntegrityException, StorageVolumeException { - long currentTime = System.currentTimeMillis(); - long objectLastModifiedTime = co.getLastModified(); + public void check(CacheObject co) throws IntegrityException, StorageVolumeException, CacheManagementException { + if (co == null || inventoryDB == null) { + throw new IllegalArgumentException("CacheObject or StorageInventoryDB is null"); + } - // Throw an exception if the last modified time is unknown - if (objectLastModifiedTime == -1) { - throw new IntegrityException("Last modified time of cache object is unknown: " + co.name); + if (!co.hasMetadatum("expiresIn")) { + throw new IntegrityException("CacheObject missing 'expiresIn' metadata"); } - // If the cache object is expired, remove it from the cache - if ((currentTime - objectLastModifiedTime) > TWO_WEEKS_MILLIS) { - removeExpiredObject(co); + long expiresInDuration = co.getMetadatumLong("expiresIn", -1L); + if (expiresInDuration == -1L) { + throw new IntegrityException("Invalid 'expiresIn' metadata value"); } - } - /** - * Removes an expired object from the cache. - * - * @param co The expired CacheObject to be removed. - * @throws StorageVolumeException if there is an issue removing the object from the cache volume. - */ - protected void removeExpiredObject(CacheObject co) throws StorageVolumeException { - CacheVolume volume = co.volume; - if (volume != null && volume.remove(co.name)) { + long lastModified = co.getLastModified(); + if (lastModified == -1L) { + throw new IntegrityException("CacheObject 'lastModified' time not available"); + } + + long expiryTime = lastModified + expiresInDuration; + long currentTime = Instant.now().toEpochMilli(); + + if (expiryTime < currentTime) { try { inventoryDB.removeObject(co.volname, co.name); } catch (InventoryException e) { - throw new StorageVolumeException("Failed to remove object from inventory database: " + co.name, e); + throw new CacheManagementException("Error removing expired object from inventory database: " + co.name, e); } - } else { - throw new StorageVolumeException("Failed to remove expired object from cache volume: " + co.name); } } } diff --git a/src/test/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheckTest.java b/src/test/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheckTest.java index dad89ab7..0498092d 100644 --- a/src/test/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheckTest.java +++ b/src/test/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheckTest.java @@ -28,9 +28,9 @@ public void setUp() { /** * Test to verify that {@link CacheExpiryCheck} correctly identifies and processes an expired cache object. - * An object is considered expired if its last modified time is more than two weeks ago. - * This test checks that the expired object is appropriately removed from both the cache volume and the inventory - * database. + * An object is considered expired based on the `expiresIn` metadata, which defines the duration after which + * an object should be considered expired from the time of its last modification. This test ensures that an object + * past its expiration is appropriately removed from the inventory database. * * @throws Exception to handle any exceptions thrown during the test execution */ @@ -39,25 +39,23 @@ public void testExpiredObject() throws Exception { // Setup mock cacheObject.name = "testObject"; cacheObject.volname = "testVolume"; - cacheObject.volume = mockVolume; - long lastModified = System.currentTimeMillis() - (15 * 24 * 60 * 60 * 1000); // 15 days in milliseconds + when(cacheObject.hasMetadatum("expiresIn")).thenReturn(true); + when(cacheObject.getMetadatumLong("expiresIn", -1L)).thenReturn(14 * 24 * 60 * 60 * 1000L); // 14 days in milliseconds + long lastModified = System.currentTimeMillis() - (15 * 24 * 60 * 60 * 1000L); // 15 days ago when(cacheObject.getLastModified()).thenReturn(lastModified); - - when(mockVolume.remove("testObject")).thenReturn(true); - // Perform the check expiryCheck.check(cacheObject); // Verify the interactions - verify(mockVolume).remove("testObject"); - verify(mockInventoryDB).removeObject(anyString(), anyString()); + verify(mockInventoryDB).removeObject(cacheObject.volname, cacheObject.name); } /** - * Test to ensure that {@link CacheExpiryCheck} does not flag a cache object as expired if it has been - * modified within the last two weeks. This test checks that no removal action is taken for a non-expired object. + * Test to ensure that {@link CacheExpiryCheck} does not flag a cache object as expired if the current time has not + * exceeded its `expiresIn` duration since its last modification. This test verifies that no removal action is taken + * for such non-expired objects. * * @throws Exception to handle any exceptions thrown during the test execution */ @@ -66,17 +64,16 @@ public void testNonExpiredObject() throws Exception { // Setup mock cacheObject.name = "nonExpiredObject"; cacheObject.volname = "testVolume"; - cacheObject.volume = mockVolume; - - long lastModified = System.currentTimeMillis() - (5 * 24 * 60 * 60 * 1000); // 5 days in milliseconds + when(cacheObject.hasMetadatum("expiresIn")).thenReturn(true); + when(cacheObject.getMetadatumLong("expiresIn", -1L)).thenReturn(14 * 24 * 60 * 60 * 1000L); // 14 days in milliseconds + long lastModified = System.currentTimeMillis() - (7 * 24 * 60 * 60 * 1000L); // 7 days ago, within expiry period when(cacheObject.getLastModified()).thenReturn(lastModified); // Perform the check expiryCheck.check(cacheObject); // Verify that the remove method was not called as the object is not expired - verify(mockVolume, never()).remove("nonExpiredObject"); - verify(mockInventoryDB, never()).removeObject("testVolume", "nonExpiredObject"); + verify(mockInventoryDB, never()).removeObject(cacheObject.volname, cacheObject.name); } /** @@ -99,4 +96,22 @@ public void testUnknownLastModifiedTime() throws Exception { expiryCheck.check(cacheObject); } + /** + * Test to verify that {@link CacheExpiryCheck} throws an {@link IntegrityException} when a cache object lacks the + * `expiresIn` metadata. This scenario indicates that the object's expiration cannot be determined, necessitating + * error handling. + * + * @throws Exception to handle any exceptions thrown during the test execution + */ + @Test(expected = IntegrityException.class) + public void testObjectWithoutExpiresInMetadata() throws Exception { + // Setup mock to simulate an object without expiresIn metadata + cacheObject.name = "objectWithoutExpiresIn"; + cacheObject.volname = "testVolume"; + when(cacheObject.hasMetadatum("expiresIn")).thenReturn(false); + + // Attempt to check the object, expecting an IntegrityException + expiryCheck.check(cacheObject); + } + } From 53a82a0b6230ae288cb7cee3d2ac7e99edca2dfa Mon Sep 17 00:00:00 2001 From: elmiomar Date: Tue, 20 Feb 2024 08:50:25 -0500 Subject: [PATCH 10/20] Reorder checks - checksum check last --- .../java/gov/nist/oar/distrib/web/NISTCacheManagerConfig.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/gov/nist/oar/distrib/web/NISTCacheManagerConfig.java b/src/main/java/gov/nist/oar/distrib/web/NISTCacheManagerConfig.java index da4d674d..4684f5ae 100644 --- a/src/main/java/gov/nist/oar/distrib/web/NISTCacheManagerConfig.java +++ b/src/main/java/gov/nist/oar/distrib/web/NISTCacheManagerConfig.java @@ -486,10 +486,11 @@ public PDRCacheManager createCacheManager(BasicCache cache, PDRDatasetRestorer r throw new ConfigurationException(rootdir+": Not an existing directory"); List checks = new ArrayList(); - checks.add(new ChecksumCheck(false, true)); // Get the StorageInventoryDB from the cache and add the CacheExpiryCheck to the list of checks StorageInventoryDB inventoryDB = cache.getInventoryDB(); checks.add(new CacheExpiryCheck(inventoryDB)); + checks.add(new ChecksumCheck(false, true)); + PDRCacheManager out = new PDRCacheManager(cache, rstr, checks, getCheckDutyCycle()*1000, getCheckGracePeriod()*1000, -1, rootdir, logger); if (getMonitorAutoStart()) { From f4a279b570eec33b02750ddd11faf4fa82817ab9 Mon Sep 17 00:00:00 2001 From: elmiomar Date: Tue, 20 Feb 2024 11:17:32 -0500 Subject: [PATCH 11/20] update removal logic --- .../distrib/cachemgr/CacheExpiryCheck.java | 57 ++++++++---- .../cachemgr/CacheExpiryCheckTest.java | 88 +++++++++++++------ 2 files changed, 99 insertions(+), 46 deletions(-) diff --git a/src/main/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheck.java b/src/main/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheck.java index 5a82d2bd..c5ef1845 100644 --- a/src/main/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheck.java +++ b/src/main/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheck.java @@ -36,29 +36,48 @@ public void check(CacheObject co) throws IntegrityException, StorageVolumeExcept throw new IllegalArgumentException("CacheObject or StorageInventoryDB is null"); } - if (!co.hasMetadatum("expiresIn")) { - throw new IntegrityException("CacheObject missing 'expiresIn' metadata"); - } - - long expiresInDuration = co.getMetadatumLong("expiresIn", -1L); - if (expiresInDuration == -1L) { - throw new IntegrityException("Invalid 'expiresIn' metadata value"); - } + if (co.hasMetadatum("expiresIn")) { + long expiresInDuration = co.getMetadatumLong("expiresIn", -1L); + if (expiresInDuration == -1L) { + throw new IntegrityException("Invalid 'expiresIn' metadata value"); + } - long lastModified = co.getLastModified(); - if (lastModified == -1L) { - throw new IntegrityException("CacheObject 'lastModified' time not available"); - } + long lastModified = co.getLastModified(); + if (lastModified == -1L) { + throw new IntegrityException("CacheObject 'lastModified' time not available"); + } - long expiryTime = lastModified + expiresInDuration; - long currentTime = Instant.now().toEpochMilli(); + long expiryTime = lastModified + expiresInDuration; + long currentTime = Instant.now().toEpochMilli(); - if (expiryTime < currentTime) { - try { - inventoryDB.removeObject(co.volname, co.name); - } catch (InventoryException e) { - throw new CacheManagementException("Error removing expired object from inventory database: " + co.name, e); + // Check if the object is expired + if (expiryTime < currentTime) { + try { + boolean removed = removeObject(co); + if (!removed) { + throw new CacheManagementException("Failed to remove expired object: " + co.name); + } + } catch (InventoryException e) { + throw new CacheManagementException("Error removing expired object from inventory database: " + co.name, e); + } } } } + + /** + * Attempts to remove a cache object from both its physical volume and the inventory database. + * Synchronization ensures thread-safe removal operations. + * + * @param co The cache object to be removed. + * @return true if the object was successfully removed from its volume, false otherwise. + * @throws StorageVolumeException if an error occurs accessing the storage volume. + * @throws InventoryException if an error occurs updating the inventory database. + */ + protected boolean removeObject(CacheObject co) throws StorageVolumeException, InventoryException { + synchronized (inventoryDB) { + boolean out = co.volume.remove(co.name); + inventoryDB.removeObject(co.volname, co.name); + return out; + } + } } diff --git a/src/test/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheckTest.java b/src/test/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheckTest.java index 0498092d..6796b053 100644 --- a/src/test/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheckTest.java +++ b/src/test/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheckTest.java @@ -5,6 +5,8 @@ import org.mockito.Mock; import org.mockito.MockitoAnnotations; +import java.time.Instant; + import static org.mockito.Mockito.anyString; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; @@ -35,19 +37,17 @@ public void setUp() { * @throws Exception to handle any exceptions thrown during the test execution */ @Test - public void testExpiredObject() throws Exception { - // Setup mock - cacheObject.name = "testObject"; - cacheObject.volname = "testVolume"; + public void testExpiredObjectRemoval() throws Exception { + // Setup an expired cache object + cacheObject.volume = mockVolume; when(cacheObject.hasMetadatum("expiresIn")).thenReturn(true); - when(cacheObject.getMetadatumLong("expiresIn", -1L)).thenReturn(14 * 24 * 60 * 60 * 1000L); // 14 days in milliseconds - long lastModified = System.currentTimeMillis() - (15 * 24 * 60 * 60 * 1000L); // 15 days ago - when(cacheObject.getLastModified()).thenReturn(lastModified); + when(cacheObject.getMetadatumLong("expiresIn", -1L)).thenReturn(1000L); // Expires in 1 second + when(cacheObject.getLastModified()).thenReturn(Instant.now().minusSeconds(10).toEpochMilli()); + when(cacheObject.volume.remove(cacheObject.name)).thenReturn(true); - // Perform the check expiryCheck.check(cacheObject); - // Verify the interactions + // Verify removeObject effect verify(mockInventoryDB).removeObject(cacheObject.volname, cacheObject.name); } @@ -77,41 +77,75 @@ public void testNonExpiredObject() throws Exception { } /** - * Test to verify that {@link CacheExpiryCheck} throws an {@link IntegrityException} when the last modified time - * of a cache object is unknown (indicated by a value of -1). This situation should be flagged as an error - * as the expiry status of the object cannot be determined. + * Tests that no action is taken and no exception is thrown for a cache object without the {@code expiresIn} metadata. + * This verifies that the absence of {@code expiresIn} metadata does not trigger any removal process or result in an error. * * @throws Exception to handle any exceptions thrown during the test execution */ - @Test(expected = IntegrityException.class) - public void testUnknownLastModifiedTime() throws Exception { - // Setup mock - cacheObject.name = "unknownLastModifiedObject"; + @Test + public void testObjectWithoutExpiresInMetadata_NoActionTaken() throws Exception { + cacheObject.name = "objectWithoutExpiresIn"; cacheObject.volname = "testVolume"; - cacheObject.volume = mockVolume; - long lastModified = -1; // Unknown last modified time - when(cacheObject.getLastModified()).thenReturn(lastModified); + when(cacheObject.hasMetadatum("expiresIn")).thenReturn(false); - // Perform the check, expecting an IntegrityException expiryCheck.check(cacheObject); + + verify(mockInventoryDB, never()).removeObject(anyString(), anyString()); } /** - * Test to verify that {@link CacheExpiryCheck} throws an {@link IntegrityException} when a cache object lacks the - * `expiresIn` metadata. This scenario indicates that the object's expiration cannot be determined, necessitating - * error handling. + * Test to ensure that a cache object with an expiration date in the future is not removed from the cache. + * This test verifies the {@code check} method's correct behavior in handling non-expired objects based + * on the {@code expiresIn} metadata. + * + * @throws Exception to handle any exceptions thrown during the test execution. + */ + @Test + public void testNonExpiredObject_NoRemoval() throws Exception { + // Setup a non-expired cache object + when(cacheObject.hasMetadatum("expiresIn")).thenReturn(true); + when(cacheObject.getMetadatumLong("expiresIn", -1L)).thenReturn(System.currentTimeMillis() + 10000L); // Expires in the future + when(cacheObject.getLastModified()).thenReturn(System.currentTimeMillis()); + + expiryCheck.check(cacheObject); + + // Verify no removal happens + verify(mockInventoryDB, never()).removeObject(anyString(), anyString()); + } + + + /** + * Tests that an {@link IntegrityException} is thrown when a cache object has the {@code expiresIn} metadata + * but lacks a valid {@code lastModified} time. * * @throws Exception to handle any exceptions thrown during the test execution */ @Test(expected = IntegrityException.class) - public void testObjectWithoutExpiresInMetadata() throws Exception { - // Setup mock to simulate an object without expiresIn metadata - cacheObject.name = "objectWithoutExpiresIn"; + public void testObjectWithExpiresInButNoLastModified_ThrowsException() throws Exception { + cacheObject.name = "objectWithNoLastModified"; cacheObject.volname = "testVolume"; + when(cacheObject.hasMetadatum("expiresIn")).thenReturn(true); + when(cacheObject.getMetadatumLong("expiresIn", -1L)).thenReturn(1000L); // Expires in 1 second + when(cacheObject.getLastModified()).thenReturn(-1L); // Last modified not available + + expiryCheck.check(cacheObject); + } + + /** + * Test to verify that no action is taken for a cache object missing the {@code expiresIn} metadata. + * This test ensures that the absence of {@code expiresIn} metadata does not trigger any removal or error. + * + * @throws Exception to handle any exceptions thrown during the test execution. + */ + @Test + public void testObjectWithoutExpiresIn_NoAction() throws Exception { + // Setup an object without expiresIn metadata when(cacheObject.hasMetadatum("expiresIn")).thenReturn(false); - // Attempt to check the object, expecting an IntegrityException expiryCheck.check(cacheObject); + + // Verify no action is taken + verify(mockInventoryDB, never()).removeObject(anyString(), anyString()); } } From a71758c00b3d5613d77ae72c75788abfae55f911 Mon Sep 17 00:00:00 2001 From: elmiomar Date: Wed, 21 Feb 2024 12:10:01 -0500 Subject: [PATCH 12/20] create rpa bagstore --- .../distrib/web/NISTDistribServiceConfig.java | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/main/java/gov/nist/oar/distrib/web/NISTDistribServiceConfig.java b/src/main/java/gov/nist/oar/distrib/web/NISTDistribServiceConfig.java index 2ae27f21..e7c3d261 100644 --- a/src/main/java/gov/nist/oar/distrib/web/NISTDistribServiceConfig.java +++ b/src/main/java/gov/nist/oar/distrib/web/NISTDistribServiceConfig.java @@ -179,6 +179,12 @@ public class NISTDistribServiceConfig { @Value("${distrib.packaging.allowedRedirects:1}") int allowedRedirects; + + @Value("${distrib.rpa.bagstore-location}") + private String rpaBagstore; + + @Value("${distrib.rpa.bagstore-mode}") + private String rpaMode; @Autowired AmazonS3 s3client; // set via getter below @Autowired BagStorage lts; // set via getter below @@ -205,6 +211,25 @@ else if (mode.equals("local")) } } + /** + * The storage service to use to access the bags, considering the mode and location. + */ + @Bean + public BagStorage getBagStorageService() throws ConfigurationException { + try { + if ("aws".equals(rpaMode) || "remote".equals(rpaMode)) { + return new AWSS3LongTermStorage(rpaBagstore, s3client); + } else if ("local".equals(rpaMode)) { + return new FilesystemLongTermStorage(rpaBagstore); + } else { + throw new ConfigurationException("distrib.rpa.bagstore-mode", "Unsupported storage mode: " + rpaMode); + } + } catch (FileNotFoundException ex) { + throw new ConfigurationException("distrib.rpa.bagstore-location", + "Storage Location not found: " + ex.getMessage(), ex); + } + } + /** * the client for access S3 storage */ From a0a77f1b6585b1547d3e55a3e9d0826cdbabf6ac Mon Sep 17 00:00:00 2001 From: elmiomar Date: Fri, 23 Feb 2024 06:27:49 -0500 Subject: [PATCH 13/20] add expiresIn metadatum while caching rpa data --- .../cachemgr/pdr/PDRDatasetRestorer.java | 9 ++++ .../pdr/RestrictedDatasetRestorerTest.java | 53 +++++++++++++++++++ 2 files changed, 62 insertions(+) diff --git a/src/main/java/gov/nist/oar/distrib/cachemgr/pdr/PDRDatasetRestorer.java b/src/main/java/gov/nist/oar/distrib/cachemgr/pdr/PDRDatasetRestorer.java index 7aebf76d..fc17faa3 100644 --- a/src/main/java/gov/nist/oar/distrib/cachemgr/pdr/PDRDatasetRestorer.java +++ b/src/main/java/gov/nist/oar/distrib/cachemgr/pdr/PDRDatasetRestorer.java @@ -82,6 +82,8 @@ public class PDRDatasetRestorer implements Restorer, PDRConstants, PDRCacheRoles HeadBagCacheManager hbcm = null; long smszlim = 100000000L; // 100 MB Logger log = null; + private static final long TWO_WEEKS_MILLIS = 14L * 24 * 60 * 60 * 1000; // 2 weeks in milliseconds + /** * create the restorer @@ -650,6 +652,13 @@ protected void cacheFromBagUsingStore(String bagfile, Collection need, C md.put("ediid", resmd.get("ediid")); md.put("cachePrefs", prefs); + // Add expiresIn metadata for files marked as ROLE_RESTRICTED_DATA + if ((prefs & ROLE_RESTRICTED_DATA) != 0) { + // Calculate the expiration time as current time + 2 weeks + long expiresIn = System.currentTimeMillis() + TWO_WEEKS_MILLIS; + md.put("expiresIn", expiresIn); + } + // find space in the cache, and copy the data file into it try { resv = into.reserveSpace(ze.getSize(), prefs); diff --git a/src/test/java/gov/nist/oar/distrib/cachemgr/pdr/RestrictedDatasetRestorerTest.java b/src/test/java/gov/nist/oar/distrib/cachemgr/pdr/RestrictedDatasetRestorerTest.java index dd11fadf..eff08566 100644 --- a/src/test/java/gov/nist/oar/distrib/cachemgr/pdr/RestrictedDatasetRestorerTest.java +++ b/src/test/java/gov/nist/oar/distrib/cachemgr/pdr/RestrictedDatasetRestorerTest.java @@ -406,5 +406,58 @@ public void testCacheFromBag() } + @Test + public void testExpiresIn() throws StorageVolumeException, ResourceNotFoundException, CacheManagementException { + + assertTrue(! cache.isCached("mds1491/trial1.json")); + assertTrue(! cache.isCached("mds1491/trial2.json")); + assertTrue(! cache.isCached("mds1491/trial3/trial3a.json")); + assertTrue(! cache.isCached("mds1491/README.txt")); + + Set cached = rstr.cacheDataset("mds1491", null, cache, true, + PDRCacheRoles.ROLE_RESTRICTED_DATA, null); + assertTrue(cached.contains("trial1.json")); + assertTrue(cached.contains("trial2.json")); + assertTrue(cached.contains("README.txt")); + assertTrue(cached.contains("trial3/trial3a.json")); + assertEquals(4, cached.size()); + + assertTrue(cache.isCached("mds1491/trial1.json")); + assertTrue(cache.isCached("mds1491/trial2.json")); + assertTrue(cache.isCached("mds1491/README.txt")); + assertTrue(cache.isCached("mds1491/trial3/trial3a.json")); + + List found = cache.getInventoryDB().findObject("mds1491/trial1.json", VolumeStatus.VOL_FOR_INFO); + assertEquals(1, found.size()); + assertTrue("CacheObject should contain expiresIn metadata", found.get(0).hasMetadatum("expiresIn")); + + // Verify the "expiresIn" value is approximately 2 weeks from the current time + long TWO_WEEKS_MILLIS = 14L * 24 * 60 * 60 * 1000; + long expectedExpiresIn = System.currentTimeMillis() + TWO_WEEKS_MILLIS; + long actualExpiresIn = found.get(0).getMetadatumLong("expiresIn", 0); + // Check that the absolute difference between the expected and actual expiresIn values is less than 1000ms + assertTrue("expiresIn should be set to 2 weeks from the current time", + Math.abs(expectedExpiresIn - actualExpiresIn) < 1000); + + found = cache.getInventoryDB().findObject("mds1491/trial2.json", VolumeStatus.VOL_FOR_INFO); + assertEquals(1, found.size()); + assertTrue("CacheObject should contain expiresIn metadata", found.get(0).hasMetadatum("expiresIn")); + + expectedExpiresIn = System.currentTimeMillis() + TWO_WEEKS_MILLIS; + actualExpiresIn = found.get(0).getMetadatumLong("expiresIn", 0); + assertTrue("expiresIn should be set to 2 weeks from the current time", + Math.abs(expectedExpiresIn - actualExpiresIn) < 1000); + + found = cache.getInventoryDB().findObject("mds1491/README.txt", VolumeStatus.VOL_FOR_INFO); + assertEquals(1, found.size()); + assertTrue("CacheObject should contain expiresIn metadata", found.get(0).hasMetadatum("expiresIn")); + + expectedExpiresIn = System.currentTimeMillis() + TWO_WEEKS_MILLIS; + actualExpiresIn = found.get(0).getMetadatumLong("expiresIn", 0); + assertTrue("expiresIn should be set to 2 weeks from the current time", + Math.abs(expectedExpiresIn - actualExpiresIn) < 1000); + + } + } From 411bd6b12a9cea80a1fe57b15c34378bddb30214 Mon Sep 17 00:00:00 2001 From: elmiomar Date: Fri, 23 Feb 2024 06:31:51 -0500 Subject: [PATCH 14/20] cleanup: remove the rpa storage bean and unused imports --- .../distrib/web/NISTDistribServiceConfig.java | 32 ------------------- 1 file changed, 32 deletions(-) diff --git a/src/main/java/gov/nist/oar/distrib/web/NISTDistribServiceConfig.java b/src/main/java/gov/nist/oar/distrib/web/NISTDistribServiceConfig.java index e7c3d261..bdca6877 100644 --- a/src/main/java/gov/nist/oar/distrib/web/NISTDistribServiceConfig.java +++ b/src/main/java/gov/nist/oar/distrib/web/NISTDistribServiceConfig.java @@ -17,14 +17,8 @@ import gov.nist.oar.distrib.service.DefaultPreservationBagService; import gov.nist.oar.distrib.service.FileDownloadService; import gov.nist.oar.distrib.service.PreservationBagService; -import gov.nist.oar.distrib.service.RPACachingService; -import gov.nist.oar.distrib.service.rpa.DefaultRPARequestHandlerService; -import gov.nist.oar.distrib.service.rpa.JKSKeyRetriever; -import gov.nist.oar.distrib.service.rpa.KeyRetriever; -import gov.nist.oar.distrib.service.rpa.RPARequestHandlerService; import gov.nist.oar.distrib.storage.AWSS3LongTermStorage; import gov.nist.oar.distrib.storage.FilesystemLongTermStorage; -import gov.nist.oar.distrib.cachemgr.CacheManagementException; import io.swagger.v3.oas.models.Components; import io.swagger.v3.oas.models.OpenAPI; import io.swagger.v3.oas.models.info.Info; @@ -37,7 +31,6 @@ import java.util.List; import java.io.BufferedReader; import java.io.FileNotFoundException; -import java.io.IOException; import javax.activation.MimetypesFileTypeMap; import org.springframework.boot.SpringApplication; @@ -46,7 +39,6 @@ import org.springframework.context.annotation.Bean; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; -import org.springframework.web.client.RestTemplate; import org.springframework.web.servlet.config.annotation.CorsRegistry; import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; import org.springframework.web.servlet.config.annotation.WebMvcConfigurerAdapter; @@ -179,12 +171,6 @@ public class NISTDistribServiceConfig { @Value("${distrib.packaging.allowedRedirects:1}") int allowedRedirects; - - @Value("${distrib.rpa.bagstore-location}") - private String rpaBagstore; - - @Value("${distrib.rpa.bagstore-mode}") - private String rpaMode; @Autowired AmazonS3 s3client; // set via getter below @Autowired BagStorage lts; // set via getter below @@ -211,24 +197,6 @@ else if (mode.equals("local")) } } - /** - * The storage service to use to access the bags, considering the mode and location. - */ - @Bean - public BagStorage getBagStorageService() throws ConfigurationException { - try { - if ("aws".equals(rpaMode) || "remote".equals(rpaMode)) { - return new AWSS3LongTermStorage(rpaBagstore, s3client); - } else if ("local".equals(rpaMode)) { - return new FilesystemLongTermStorage(rpaBagstore); - } else { - throw new ConfigurationException("distrib.rpa.bagstore-mode", "Unsupported storage mode: " + rpaMode); - } - } catch (FileNotFoundException ex) { - throw new ConfigurationException("distrib.rpa.bagstore-location", - "Storage Location not found: " + ex.getMessage(), ex); - } - } /** * the client for access S3 storage From 39fc9a349357ae8edde116b6ec1a02729130ce61 Mon Sep 17 00:00:00 2001 From: elmiomar Date: Tue, 27 Feb 2024 11:01:40 -0500 Subject: [PATCH 15/20] make the new metadata field 'expiresIn' configurable --- .../cachemgr/pdr/PDRDatasetRestorer.java | 22 +++++--- .../pdr/RestrictedDatasetRestorer.java | 56 ++++++++++++------- .../web/RPACachingServiceProvider.java | 4 +- .../oar/distrib/web/RPAConfiguration.java | 11 +++- .../web/RPACachingServiceProviderTest.java | 4 +- .../oar/distrib/web/RPAConfigurationTest.java | 1 + src/test/resources/rpaconfig.json | 3 +- 7 files changed, 68 insertions(+), 33 deletions(-) diff --git a/src/main/java/gov/nist/oar/distrib/cachemgr/pdr/PDRDatasetRestorer.java b/src/main/java/gov/nist/oar/distrib/cachemgr/pdr/PDRDatasetRestorer.java index fc17faa3..7dbfc390 100644 --- a/src/main/java/gov/nist/oar/distrib/cachemgr/pdr/PDRDatasetRestorer.java +++ b/src/main/java/gov/nist/oar/distrib/cachemgr/pdr/PDRDatasetRestorer.java @@ -82,8 +82,6 @@ public class PDRDatasetRestorer implements Restorer, PDRConstants, PDRCacheRoles HeadBagCacheManager hbcm = null; long smszlim = 100000000L; // 100 MB Logger log = null; - private static final long TWO_WEEKS_MILLIS = 14L * 24 * 60 * 60 * 1000; // 2 weeks in milliseconds - /** * create the restorer @@ -652,12 +650,8 @@ protected void cacheFromBagUsingStore(String bagfile, Collection need, C md.put("ediid", resmd.get("ediid")); md.put("cachePrefs", prefs); - // Add expiresIn metadata for files marked as ROLE_RESTRICTED_DATA - if ((prefs & ROLE_RESTRICTED_DATA) != 0) { - // Calculate the expiration time as current time + 2 weeks - long expiresIn = System.currentTimeMillis() + TWO_WEEKS_MILLIS; - md.put("expiresIn", expiresIn); - } + // a hook for handling the expiration logic + updateMetadata(md, prefs); // find space in the cache, and copy the data file into it try { @@ -696,6 +690,18 @@ protected void cacheFromBagUsingStore(String bagfile, Collection need, C fixMissingChecksums(into, fix, manifest); } + /** + * Method intended for customization of metadata before caching. This method can be overridden + * by subclasses to implement specific metadata customization logic as needed. + * + * @param md The metadata JSONObject to be customized. + * @param prefs flags for data roles + */ + protected void updateMetadata(JSONObject md, int prefs) { + // Default implementation does nothing. + // Subclasses can override this to implement specific logic. + } + /** * helper method to generate an ID for the object to be cached */ diff --git a/src/main/java/gov/nist/oar/distrib/cachemgr/pdr/RestrictedDatasetRestorer.java b/src/main/java/gov/nist/oar/distrib/cachemgr/pdr/RestrictedDatasetRestorer.java index da1255c0..d06dff33 100644 --- a/src/main/java/gov/nist/oar/distrib/cachemgr/pdr/RestrictedDatasetRestorer.java +++ b/src/main/java/gov/nist/oar/distrib/cachemgr/pdr/RestrictedDatasetRestorer.java @@ -20,44 +20,23 @@ import gov.nist.oar.distrib.ObjectNotFoundException; import gov.nist.oar.distrib.ResourceNotFoundException; import gov.nist.oar.distrib.BagStorage; -import gov.nist.oar.distrib.Checksum; -import gov.nist.oar.distrib.cachemgr.Restorer; import gov.nist.oar.distrib.cachemgr.Reservation; -import gov.nist.oar.distrib.cachemgr.IntegrityMonitor; -import gov.nist.oar.distrib.cachemgr.BasicCache; import gov.nist.oar.distrib.cachemgr.Cache; import gov.nist.oar.distrib.cachemgr.CacheObject; -import gov.nist.oar.distrib.cachemgr.CacheObjectCheck; -import gov.nist.oar.distrib.cachemgr.StorageInventoryDB; import gov.nist.oar.distrib.cachemgr.CacheManagementException; import gov.nist.oar.distrib.cachemgr.RestorationException; -import gov.nist.oar.distrib.cachemgr.InventoryException; import java.util.Collection; -import java.util.List; -import java.util.ArrayList; import java.util.Set; -import java.util.Map; -import java.util.HashSet; -import java.util.HashMap; -import java.util.zip.ZipInputStream; -import java.util.zip.ZipEntry; -import java.nio.file.Path; -import java.nio.file.Paths; import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.BufferedReader; import java.io.IOException; import java.io.FileNotFoundException; -import java.text.ParseException; import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.json.JSONObject; import org.json.JSONException; -import org.apache.commons.io.FilenameUtils; /** * A {@link gov.nist.oar.distrib.cachemgr.Restorer} for restoring "restricted public" datasets from the @@ -82,6 +61,7 @@ */ public class RestrictedDatasetRestorer extends PDRDatasetRestorer { BagStorage restrictedLtstore = null; + long expiryTime = 1209600000L; // 2 weeks in milliseconds /** * create the restorer @@ -125,6 +105,25 @@ public RestrictedDatasetRestorer(BagStorage publicLtstore, BagStorage restricted this.restrictedLtstore = restrictedLtstore; } + /** + * Retrieves the expiry time for data. + *

+ * This value represents the duration in milliseconds after which the data is considered expired. + * + * @return the expiry time in milliseconds. + */ + public long getExpiryTime() { + return expiryTime; + } + + /** + * Sets the expiry time for restricted/public access content. + * + * @param expiryTime the expiry time in milliseconds to set. + */ + public void setExpiryTime(long expiryTime) { + this.expiryTime = expiryTime; + } /** * return true if an object does not exist in the long term storage system. Returning @@ -285,4 +284,19 @@ protected void cacheFromBag(String bagfile, Collection need, Collection< target, ltstore); } } + + /** + * Updates the metadata for files marked as restricted, by adding an expiration time. + * + * @param md The metadata JSONObject to be customized. + * @param prefs flags for data roles + */ + @Override + protected void updateMetadata(JSONObject md, int prefs) { + if ((prefs & ROLE_RESTRICTED_DATA) != 0) { + // Calculate the expiration time as current time + expiryTime + long expiresIn = System.currentTimeMillis() + expiryTime; + md.put("expiresIn", expiresIn); + } + } } diff --git a/src/main/java/gov/nist/oar/distrib/web/RPACachingServiceProvider.java b/src/main/java/gov/nist/oar/distrib/web/RPACachingServiceProvider.java index 82f31b89..a59e81e3 100644 --- a/src/main/java/gov/nist/oar/distrib/web/RPACachingServiceProvider.java +++ b/src/main/java/gov/nist/oar/distrib/web/RPACachingServiceProvider.java @@ -177,7 +177,9 @@ else if (mode.equals("local")) public RestrictedDatasetRestorer createRPDatasetRestorer() throws ConfigurationException, IOException, CacheManagementException { - return new RestrictedDatasetRestorer(pubstore, getRPBagStorage(), getHeadBagCacheManager()); + RestrictedDatasetRestorer rdr = new RestrictedDatasetRestorer(pubstore, getRPBagStorage(), getHeadBagCacheManager()); + rdr.setExpiryTime(rpacfg.getExpiresAfterMillis()); + return rdr; // return new PDRDatasetRestorer(getRPBagStorage(), getHeadBagCacheManager()); } diff --git a/src/main/java/gov/nist/oar/distrib/web/RPAConfiguration.java b/src/main/java/gov/nist/oar/distrib/web/RPAConfiguration.java index de4e226e..791fdbfc 100644 --- a/src/main/java/gov/nist/oar/distrib/web/RPAConfiguration.java +++ b/src/main/java/gov/nist/oar/distrib/web/RPAConfiguration.java @@ -53,7 +53,8 @@ public class RPAConfiguration { String bagStore = null; @JsonProperty("bagstore-mode") String mode = null; - + @JsonProperty("expiresAfterMillis") + long expiresAfterMillis = 0L; public long getHeadbagCacheSize() { return hbCacheSize; @@ -76,6 +77,14 @@ public void setBagstoreMode(String mode) { this.mode = mode; } + public long getExpiresAfterMillis() { + return expiresAfterMillis; + } + + public void setExpiresAfterMillis(long expiresAfterMillis) { + this.expiresAfterMillis = expiresAfterMillis; + } + public SalesforceJwt getSalesforceJwt() { return salesforceJwt; } diff --git a/src/test/java/gov/nist/oar/distrib/web/RPACachingServiceProviderTest.java b/src/test/java/gov/nist/oar/distrib/web/RPACachingServiceProviderTest.java index 90060b54..1c5d8ac7 100644 --- a/src/test/java/gov/nist/oar/distrib/web/RPACachingServiceProviderTest.java +++ b/src/test/java/gov/nist/oar/distrib/web/RPACachingServiceProviderTest.java @@ -1,5 +1,6 @@ package gov.nist.oar.distrib.web; +import gov.nist.oar.distrib.cachemgr.pdr.RestrictedDatasetRestorer; import gov.nist.oar.distrib.storage.FilesystemLongTermStorage; import gov.nist.oar.distrib.BagStorage; import gov.nist.oar.distrib.cachemgr.CacheManagementException; @@ -81,6 +82,7 @@ public void testGetHeadBagManager() public void testCreateRPDatasetRestorer() throws ConfigurationException, IOException, CacheManagementException { - PDRDatasetRestorer rest = prov.createRPDatasetRestorer(); + RestrictedDatasetRestorer rest = prov.createRPDatasetRestorer(); + assertEquals(rest.getExpiryTime(), 1209600000L); } } diff --git a/src/test/java/gov/nist/oar/distrib/web/RPAConfigurationTest.java b/src/test/java/gov/nist/oar/distrib/web/RPAConfigurationTest.java index f600fb62..0a5ae317 100644 --- a/src/test/java/gov/nist/oar/distrib/web/RPAConfigurationTest.java +++ b/src/test/java/gov/nist/oar/distrib/web/RPAConfigurationTest.java @@ -22,5 +22,6 @@ public void testLoadConfig() throws IOException { assertEquals("local", config.getBagstoreMode()); assertNull(config.getBagstoreLocation()); assertEquals("1234567890 pdr.rpa.2023 1234567890", config.getJwtSecretKey()); + assertEquals(1209600000L, config.getExpiresAfterMillis()); } } diff --git a/src/test/resources/rpaconfig.json b/src/test/resources/rpaconfig.json index 89441de8..19f809b3 100644 --- a/src/test/resources/rpaconfig.json +++ b/src/test/resources/rpaconfig.json @@ -3,5 +3,6 @@ "datacartUrl": "https://localhost/datacart/rpa", "headbagCacheSize": 50000000, "bagstoreMode": "local", - "jwtSecretKey": "1234567890 pdr.rpa.2023 1234567890" + "jwtSecretKey": "1234567890 pdr.rpa.2023 1234567890", + "expiresAfterMillis": 1209600000 } From 364cb8877ef47a4d1e44a8b1090b1b73e6668abb Mon Sep 17 00:00:00 2001 From: elmiomar Date: Wed, 6 Mar 2024 10:57:22 -0500 Subject: [PATCH 16/20] update unit tests --- ...URLConnectionRPARequestHandlerService.java | 4 + ...onnectionRPARequestHandlerServiceTest.java | 149 +++++++++++++++++- 2 files changed, 148 insertions(+), 5 deletions(-) diff --git a/src/main/java/gov/nist/oar/distrib/service/rpa/HttpURLConnectionRPARequestHandlerService.java b/src/main/java/gov/nist/oar/distrib/service/rpa/HttpURLConnectionRPARequestHandlerService.java index 6a391b9e..c75628d9 100644 --- a/src/main/java/gov/nist/oar/distrib/service/rpa/HttpURLConnectionRPARequestHandlerService.java +++ b/src/main/java/gov/nist/oar/distrib/service/rpa/HttpURLConnectionRPARequestHandlerService.java @@ -140,6 +140,10 @@ public void setRecordResponseHandler(RecordResponseHandler recordResponseHandler this.recordResponseHandler = recordResponseHandler; } + public void seRPADatasetCacher(RPADatasetCacher rpaDatasetCacher) { + this.rpaDatasetCacher = rpaDatasetCacher; + } + /** * Constructs a new instance of the service using the given RPA configuration. * diff --git a/src/test/java/gov/nist/oar/distrib/service/rpa/HttpURLConnectionRPARequestHandlerServiceTest.java b/src/test/java/gov/nist/oar/distrib/service/rpa/HttpURLConnectionRPARequestHandlerServiceTest.java index 3d15e46e..369a84b3 100644 --- a/src/test/java/gov/nist/oar/distrib/service/rpa/HttpURLConnectionRPARequestHandlerServiceTest.java +++ b/src/test/java/gov/nist/oar/distrib/service/rpa/HttpURLConnectionRPARequestHandlerServiceTest.java @@ -45,9 +45,13 @@ import java.util.Map; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; @@ -116,6 +120,9 @@ public class HttpURLConnectionRPARequestHandlerServiceTest { @Mock RPACachingService rpaCachingService; + @Mock + RPADatasetCacher rpaDatasetCacher; + private HttpURLConnectionRPARequestHandlerService service; JWTToken testToken = null; Map map = new HashMap() {{ @@ -132,6 +139,7 @@ public void setUp() { service.setRecaptchaHelper(recaptchaHelper); service.setHttpURLConnectionFactory(url -> mockConnection); service.setRecordResponseHandler(recordResponseHandler); + service.seRPADatasetCacher(rpaDatasetCacher); service.setHttpClient(mockHttpClient); // Set up mock behavior for mockJwtHelper @@ -491,21 +499,45 @@ private String getUpdateUrl(String recordId) { return url; } + /** + * Tests successful record update operation when approving a record. + * + *

+ * This test simulates the scenario where a record is approved, ensuring that the + * caching process is invoked, a PATCH request is made to update the record status in + * Salesforce with a newly generated random ID, and the approval status is updated correctly. + * The test verifies that the final approval status matches the expected format. + * 

+     * Status_YYYY-MM-DDTHH:MM:SS.SSSZ_email_randomID
+     *
+ * Where: + *
    + *
  • Status - Indicates the status of the record (e.g., "Approved" or "Declined").
    • + *
  • YYYY-MM-DDTHH:MM:SS.SSSZ - The timestamp of the approval in ISO 8601 format. 'T' separates the date and time, and 'Z' denotes UTC time zone.
    • + *
  • email - The email address associated with the user who approved the record.
    • + *
  • randomID - A unique identifier generated for the approval process or the record itself.
    • + *
+ *

+ * + * @throws Exception if any error occurs during the test execution. + */ @Test public void testUpdateRecord_success() throws Exception { String recordId = "record12345"; String email = "test@example.com"; - String expectedApprovalStatus = "Approved_2023-05-09T15:59:03.872Z_" + email; + String mockRandomId = "mockRandomId123"; + String expectedApprovalStatus = "Approved_2023-05-09T15:59:03.872Z_" + email + "_" + mockRandomId; // Mock behavior of getRecord method doReturn(getTestRecordWrapper(expectedApprovalStatus)).when(service).getRecord("record12345"); + when(rpaDatasetCacher.cache(anyString())).thenReturn(mockRandomId); // Mock HttpResponse CloseableHttpResponse httpResponse = mock(CloseableHttpResponse.class); when(httpResponse.getStatusLine()).thenReturn(mock(StatusLine.class)); when(httpResponse.getStatusLine().getStatusCode()).thenReturn(200); when(httpResponse.getEntity()).thenReturn( - new StringEntity("{\"approvalStatus\":\"Approved_2023-05-09T15:59:03.872Z_test@example.com\"," + + new StringEntity("{\"approvalStatus\":\"Approved_2023-05-09T15:59:03.872Z_test@example.com_mockRandomId123\"," + "\"recordId\":\"record12345\"}", ContentType.APPLICATION_JSON) ); @@ -531,13 +563,24 @@ public void testUpdateRecord_success() throws Exception { // We can't test the exact time as it changes when we run the test, but we can verify the format String patchPayload = EntityUtils.toString(patchRequest.getEntity(), StandardCharsets.UTF_8); JSONObject payloadObject = new JSONObject(patchPayload); - // Pattern to match ISO 8601 format - // This pattern matches a string in the format "Approved_YYYY-MM-DDTHH:MM:SS.SSSZ" - String expectedFormat = "Approved_\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z_[\\w.-]+@[\\w.-]+\\.\\w+"; + // The following regex pattern expects: + // - The "Approved" status followed by a date-time in ISO 8601 format. + // - An email address. + // - A random ID (composed of word characters including underscore, alphanumeric, and possibly -) at the end. + String expectedFormat = "Approved_\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z_[\\w.-]+@[\\w.-]+\\.\\w+_\\w+"; assertTrue(payloadObject.get("Approval_Status__c").toString().matches(expectedFormat)); } + /** + * Tests the updateRecord method's behavior when an unknown status is provided. + *

+ * This test ensures that the method throws an InvalidRequestException when attempting to + * update a record with an unrecognized status. The expected behavior is to validate the + * status input and throw an exception with a specific error message if the status does not + * match expected values (e.g., "Approved" or "Declined"). + *

+ */ @Test public void testUpdateRecord_withUnknownStatus() { String recordId = "record12345"; @@ -545,6 +588,9 @@ public void testUpdateRecord_withUnknownStatus() { String email = "test@example.com"; String expectedErrorMessage = "Invalid approval status: HelloWorld"; + // Mock behavior of getRecord method + doReturn(getTestRecordWrapper("Pending")).when(service).getRecord("record12345"); + // Call the method and catch the exception try { // Act @@ -556,5 +602,98 @@ public void testUpdateRecord_withUnknownStatus() { } } + /** + * Tests the record decline operation for a record that has not been previously approved. + * + *

+ * This test case ensures that when a record is declined without prior approval, the record's + * status is updated accordingly without triggering caching or uncaching operations. It verifies + * the successful update of the record's approval status to "Declined" and confirms that no + * caching or uncaching methods are called, as expected for records not previously approved. + *

+ * + * @throws Exception if any error occurs during the test execution. + */ + @Test + public void testDeclineRecordWithoutPriorApproval_success() throws Exception { + String recordId = "record12345"; + String email = "sme@test.com"; + String status = "Declined"; + + // Mock behavior of getRecord method to simulate a record that has not been approved before + doReturn(getTestRecordWrapper("Pending")).when(service).getRecord(recordId); + + // Mock HttpResponse for the decline operation + CloseableHttpResponse httpResponse = mock(CloseableHttpResponse.class); + when(httpResponse.getStatusLine()).thenReturn(mock(StatusLine.class)); + when(httpResponse.getStatusLine().getStatusCode()).thenReturn(200); + when(httpResponse.getEntity()).thenReturn( + new StringEntity("{\"approvalStatus\":\"Declined\",\"recordId\":\"" + recordId + "\"}", + ContentType.APPLICATION_JSON)); + + // Mock the HttpPatch execution + doReturn(httpResponse).when(mockHttpClient).execute(any(HttpPatch.class)); + + // Act + RecordStatus result = service.updateRecord(recordId, status, email); + + // Assert + assertEquals("Declined", result.getApprovalStatus()); + assertEquals(recordId, result.getRecordId()); + + // Verify that caching and uncaching were not invoked + verify(rpaDatasetCacher, never()).cache(anyString()); + verify(rpaDatasetCacher, never()).uncache(anyString()); + } + + /** + * Tests the decline operation for a record that was previously approved. + * + *

+ * This test checks the behavior of the updateRecord method when declining a record that + * has a prior approval status, including a random ID. It simulates retrieving a previously + * approved record, uncaching the dataset associated with the random ID, and updating the + * record's approval status to "Declined". The test verifies that the uncaching operation + * is executed with the correct random ID and that the record's status is correctly updated. + *

+ * + * @throws Exception if any error occurs during the test execution. + */ + @Test + public void testDeclinePreviouslyApprovedRecord_success() throws Exception { + String recordId = "record12345"; + String email = "sme@test.com"; + String status = "Declined"; + String mockRandomId = "mockRandomId123"; + String initialApprovalStatus = "Approved_2023-05-09T15:59:03.872Z_" + email + "_" + mockRandomId; + + // Mock behavior of getRecord method to simulate retrieving a previously approved record + doReturn(getTestRecordWrapper(initialApprovalStatus)).when(service).getRecord(recordId); + + // Simulate `uncache` returning true + when(rpaDatasetCacher.uncache(anyString())).thenReturn(true); + + // Mock HttpResponse for updating the record to "Declined" + CloseableHttpResponse httpResponse = mock(CloseableHttpResponse.class); + when(httpResponse.getStatusLine()).thenReturn(mock(StatusLine.class)); + when(httpResponse.getStatusLine().getStatusCode()).thenReturn(200); + when(httpResponse.getEntity()).thenReturn( + new StringEntity("{\"approvalStatus\":\"Declined\",\"recordId\":\"" + recordId + "\"}", + ContentType.APPLICATION_JSON)); + + // Mock the HttpPatch execution + doReturn(httpResponse).when(mockHttpClient).execute(any(HttpPatch.class)); + + // Act + RecordStatus result = service.updateRecord(recordId, status, email); + + // Assert + assertEquals("Declined", result.getApprovalStatus()); + assertEquals(recordId, result.getRecordId()); + + // Verify uncaching was invoked with the correct random ID + verify(rpaDatasetCacher).uncache(mockRandomId); + } + } From 03cdf70a0d56bc4d10f93ca08150576936c412a7 Mon Sep 17 00:00:00 2001 From: Omar Ilias EL MIMOUNI Date: Wed, 6 Mar 2024 11:06:41 -0500 Subject: [PATCH 17/20] Update Dockerfile - resolve merge conflicts --- docker/build-test/Dockerfile | 3 --- 1 file changed, 3 deletions(-) diff --git a/docker/build-test/Dockerfile b/docker/build-test/Dockerfile index c6ac3cc1..fba91c27 100644 --- a/docker/build-test/Dockerfile +++ b/docker/build-test/Dockerfile @@ -1,4 +1,3 @@ -<<<<<<< HEAD FROM eclipse-temurin:8-jdk-focal RUN mkdir -p /usr/share/man/man1 @@ -16,14 +15,12 @@ RUN java_certs=$JAVA_HOME/jre/lib/security/cacerts; \ keytool -import -keystore $java_certs -trustcacerts -file $crt \ -storepass changeit -alias $name -noprompt; \ done; -======= FROM eclipse-temurin:8 RUN mkdir -p /usr/share/man/man1 RUN apt-get update && apt-get install -y netcat-openbsd zip git less \ python2 curl maven RUN cd /usr/bin && ln -s python2 python ->>>>>>> origin/integration # Create the user that build/test operations should run as. Normally, # this is set to match identity information of the host user that is From de80e31caf478446a8978851e1406ad838b9608d Mon Sep 17 00:00:00 2001 From: elmiomar Date: Wed, 6 Mar 2024 11:15:12 -0500 Subject: [PATCH 18/20] change 'expiresIn' metadata field name to 'expires' --- .../distrib/cachemgr/CacheExpiryCheck.java | 15 +++--- .../pdr/RestrictedDatasetRestorer.java | 4 +- .../cachemgr/CacheExpiryCheckTest.java | 46 +++++++++---------- .../pdr/RestrictedDatasetRestorerTest.java | 36 +++++++-------- 4 files changed, 50 insertions(+), 51 deletions(-) diff --git a/src/main/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheck.java b/src/main/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheck.java index c5ef1845..2a8f718a 100644 --- a/src/main/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheck.java +++ b/src/main/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheck.java @@ -12,7 +12,6 @@ */ public class CacheExpiryCheck implements CacheObjectCheck { - // private static final long TWO_WEEKS_MILLIS = 14 * 24 * 60 * 60 * 1000; // 14 days in milliseconds private StorageInventoryDB inventoryDB; public CacheExpiryCheck(StorageInventoryDB inventoryDB) { @@ -21,8 +20,8 @@ public CacheExpiryCheck(StorageInventoryDB inventoryDB) { /** * Checks if a cache object is expired and removes it from the cache if it is. - * The method uses the {@code expiresIn} metadata field to determine the expiration status. - * The expiration time is calculated based on the {@code LastModified} time plus the {@code expiresIn} duration. + * The method uses the {@code expires} metadata field to determine the expiration status. + * The expiration time is calculated based on the {@code LastModified} time plus the {@code expires} duration. * If the current time is past the calculated expiry time, the object is removed from the inventory database. * * @param co The cache object to check for expiration. @@ -36,10 +35,10 @@ public void check(CacheObject co) throws IntegrityException, StorageVolumeExcept throw new IllegalArgumentException("CacheObject or StorageInventoryDB is null"); } - if (co.hasMetadatum("expiresIn")) { - long expiresInDuration = co.getMetadatumLong("expiresIn", -1L); - if (expiresInDuration == -1L) { - throw new IntegrityException("Invalid 'expiresIn' metadata value"); + if (co.hasMetadatum("expires")) { + long expiresDuration = co.getMetadatumLong("expires", -1L); + if (expiresDuration == -1L) { + throw new IntegrityException("Invalid 'expires' metadata value"); } long lastModified = co.getLastModified(); @@ -47,7 +46,7 @@ public void check(CacheObject co) throws IntegrityException, StorageVolumeExcept throw new IntegrityException("CacheObject 'lastModified' time not available"); } - long expiryTime = lastModified + expiresInDuration; + long expiryTime = lastModified + expiresDuration; long currentTime = Instant.now().toEpochMilli(); // Check if the object is expired diff --git a/src/main/java/gov/nist/oar/distrib/cachemgr/pdr/RestrictedDatasetRestorer.java b/src/main/java/gov/nist/oar/distrib/cachemgr/pdr/RestrictedDatasetRestorer.java index d06dff33..54bec7ab 100644 --- a/src/main/java/gov/nist/oar/distrib/cachemgr/pdr/RestrictedDatasetRestorer.java +++ b/src/main/java/gov/nist/oar/distrib/cachemgr/pdr/RestrictedDatasetRestorer.java @@ -295,8 +295,8 @@ protected void cacheFromBag(String bagfile, Collection need, Collection< protected void updateMetadata(JSONObject md, int prefs) { if ((prefs & ROLE_RESTRICTED_DATA) != 0) { // Calculate the expiration time as current time + expiryTime - long expiresIn = System.currentTimeMillis() + expiryTime; - md.put("expiresIn", expiresIn); + long expires = System.currentTimeMillis() + expiryTime; + md.put("expires", expires); } } } diff --git a/src/test/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheckTest.java b/src/test/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheckTest.java index 6796b053..0f0c61da 100644 --- a/src/test/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheckTest.java +++ b/src/test/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheckTest.java @@ -30,7 +30,7 @@ public void setUp() { /** * Test to verify that {@link CacheExpiryCheck} correctly identifies and processes an expired cache object. - * An object is considered expired based on the `expiresIn` metadata, which defines the duration after which + * An object is considered expired based on the `expires` metadata, which defines the duration after which * an object should be considered expired from the time of its last modification. This test ensures that an object * past its expiration is appropriately removed from the inventory database. * @@ -40,8 +40,8 @@ public void setUp() { public void testExpiredObjectRemoval() throws Exception { // Setup an expired cache object cacheObject.volume = mockVolume; - when(cacheObject.hasMetadatum("expiresIn")).thenReturn(true); - when(cacheObject.getMetadatumLong("expiresIn", -1L)).thenReturn(1000L); // Expires in 1 second + when(cacheObject.hasMetadatum("expires")).thenReturn(true); + when(cacheObject.getMetadatumLong("expires", -1L)).thenReturn(1000L); // Expires in 1 second when(cacheObject.getLastModified()).thenReturn(Instant.now().minusSeconds(10).toEpochMilli()); when(cacheObject.volume.remove(cacheObject.name)).thenReturn(true); @@ -54,7 +54,7 @@ public void testExpiredObjectRemoval() throws Exception { /** * Test to ensure that {@link CacheExpiryCheck} does not flag a cache object as expired if the current time has not - * exceeded its `expiresIn` duration since its last modification. This test verifies that no removal action is taken + * exceeded its `expires` duration since its last modification. This test verifies that no removal action is taken * for such non-expired objects. * * @throws Exception to handle any exceptions thrown during the test execution @@ -64,8 +64,8 @@ public void testNonExpiredObject() throws Exception { // Setup mock cacheObject.name = "nonExpiredObject"; cacheObject.volname = "testVolume"; - when(cacheObject.hasMetadatum("expiresIn")).thenReturn(true); - when(cacheObject.getMetadatumLong("expiresIn", -1L)).thenReturn(14 * 24 * 60 * 60 * 1000L); // 14 days in milliseconds + when(cacheObject.hasMetadatum("expires")).thenReturn(true); + when(cacheObject.getMetadatumLong("expires", -1L)).thenReturn(14 * 24 * 60 * 60 * 1000L); // 14 days in milliseconds long lastModified = System.currentTimeMillis() - (7 * 24 * 60 * 60 * 1000L); // 7 days ago, within expiry period when(cacheObject.getLastModified()).thenReturn(lastModified); @@ -77,16 +77,16 @@ public void testNonExpiredObject() throws Exception { } /** - * Tests that no action is taken and no exception is thrown for a cache object without the {@code expiresIn} metadata. - * This verifies that the absence of {@code expiresIn} metadata does not trigger any removal process or result in an error. + * Tests that no action is taken and no exception is thrown for a cache object without the {@code expires} metadata. + * This verifies that the absence of {@code expires} metadata does not trigger any removal process or result in an error. * * @throws Exception to handle any exceptions thrown during the test execution */ @Test - public void testObjectWithoutExpiresInMetadata_NoActionTaken() throws Exception { - cacheObject.name = "objectWithoutExpiresIn"; + public void testObjectWithoutExpiresMetadata_NoActionTaken() throws Exception { + cacheObject.name = "objectWithoutExpires"; cacheObject.volname = "testVolume"; - when(cacheObject.hasMetadatum("expiresIn")).thenReturn(false); + when(cacheObject.hasMetadatum("expires")).thenReturn(false); expiryCheck.check(cacheObject); @@ -96,15 +96,15 @@ public void testObjectWithoutExpiresInMetadata_NoActionTaken() throws Exception /** * Test to ensure that a cache object with an expiration date in the future is not removed from the cache. * This test verifies the {@code check} method's correct behavior in handling non-expired objects based - * on the {@code expiresIn} metadata. + * on the {@code expires} metadata. * * @throws Exception to handle any exceptions thrown during the test execution. */ @Test public void testNonExpiredObject_NoRemoval() throws Exception { // Setup a non-expired cache object - when(cacheObject.hasMetadatum("expiresIn")).thenReturn(true); - when(cacheObject.getMetadatumLong("expiresIn", -1L)).thenReturn(System.currentTimeMillis() + 10000L); // Expires in the future + when(cacheObject.hasMetadatum("expires")).thenReturn(true); + when(cacheObject.getMetadatumLong("expires", -1L)).thenReturn(System.currentTimeMillis() + 10000L); // Expires in the future when(cacheObject.getLastModified()).thenReturn(System.currentTimeMillis()); expiryCheck.check(cacheObject); @@ -115,32 +115,32 @@ public void testNonExpiredObject_NoRemoval() throws Exception { /** - * Tests that an {@link IntegrityException} is thrown when a cache object has the {@code expiresIn} metadata + * Tests that an {@link IntegrityException} is thrown when a cache object has the {@code expires} metadata * but lacks a valid {@code lastModified} time. * * @throws Exception to handle any exceptions thrown during the test execution */ @Test(expected = IntegrityException.class) - public void testObjectWithExpiresInButNoLastModified_ThrowsException() throws Exception { + public void testObjectWithExpiresButNoLastModified_ThrowsException() throws Exception { cacheObject.name = "objectWithNoLastModified"; cacheObject.volname = "testVolume"; - when(cacheObject.hasMetadatum("expiresIn")).thenReturn(true); - when(cacheObject.getMetadatumLong("expiresIn", -1L)).thenReturn(1000L); // Expires in 1 second + when(cacheObject.hasMetadatum("expires")).thenReturn(true); + when(cacheObject.getMetadatumLong("expires", -1L)).thenReturn(1000L); // Expires in 1 second when(cacheObject.getLastModified()).thenReturn(-1L); // Last modified not available expiryCheck.check(cacheObject); } /** - * Test to verify that no action is taken for a cache object missing the {@code expiresIn} metadata. - * This test ensures that the absence of {@code expiresIn} metadata does not trigger any removal or error. + * Test to verify that no action is taken for a cache object missing the {@code expires} metadata. + * This test ensures that the absence of {@code expires} metadata does not trigger any removal or error. * * @throws Exception to handle any exceptions thrown during the test execution. */ @Test - public void testObjectWithoutExpiresIn_NoAction() throws Exception { - // Setup an object without expiresIn metadata - when(cacheObject.hasMetadatum("expiresIn")).thenReturn(false); + public void testObjectWithoutExpires_NoAction() throws Exception { + // Setup an object without expires metadata + when(cacheObject.hasMetadatum("expires")).thenReturn(false); expiryCheck.check(cacheObject); diff --git a/src/test/java/gov/nist/oar/distrib/cachemgr/pdr/RestrictedDatasetRestorerTest.java b/src/test/java/gov/nist/oar/distrib/cachemgr/pdr/RestrictedDatasetRestorerTest.java index eff08566..2de7d871 100644 --- a/src/test/java/gov/nist/oar/distrib/cachemgr/pdr/RestrictedDatasetRestorerTest.java +++ b/src/test/java/gov/nist/oar/distrib/cachemgr/pdr/RestrictedDatasetRestorerTest.java @@ -407,7 +407,7 @@ public void testCacheFromBag() } @Test - public void testExpiresIn() throws StorageVolumeException, ResourceNotFoundException, CacheManagementException { + public void testExpires() throws StorageVolumeException, ResourceNotFoundException, CacheManagementException { assertTrue(! cache.isCached("mds1491/trial1.json")); assertTrue(! cache.isCached("mds1491/trial2.json")); @@ -429,33 +429,33 @@ public void testExpiresIn() throws StorageVolumeException, ResourceNotFoundExcep List found = cache.getInventoryDB().findObject("mds1491/trial1.json", VolumeStatus.VOL_FOR_INFO); assertEquals(1, found.size()); - assertTrue("CacheObject should contain expiresIn metadata", found.get(0).hasMetadatum("expiresIn")); + assertTrue("CacheObject should contain expires metadata", found.get(0).hasMetadatum("expires")); - // Verify the "expiresIn" value is approximately 2 weeks from the current time + // Verify the "expires" value is approximately 2 weeks from the current time long TWO_WEEKS_MILLIS = 14L * 24 * 60 * 60 * 1000; - long expectedExpiresIn = System.currentTimeMillis() + TWO_WEEKS_MILLIS; - long actualExpiresIn = found.get(0).getMetadatumLong("expiresIn", 0); - // Check that the absolute difference between the expected and actual expiresIn values is less than 1000ms - assertTrue("expiresIn should be set to 2 weeks from the current time", - Math.abs(expectedExpiresIn - actualExpiresIn) < 1000); + long expectedExpires = System.currentTimeMillis() + TWO_WEEKS_MILLIS; + long actualExpires = found.get(0).getMetadatumLong("expires", 0); + // Check that the absolute difference between the expected and actual expires values is less than 1000ms + assertTrue("expires field should be set to 2 weeks from the current time", + Math.abs(expectedExpires - actualExpires) < 1000); found = cache.getInventoryDB().findObject("mds1491/trial2.json", VolumeStatus.VOL_FOR_INFO); assertEquals(1, found.size()); - assertTrue("CacheObject should contain expiresIn metadata", found.get(0).hasMetadatum("expiresIn")); + assertTrue("CacheObject should contain expires metadata", found.get(0).hasMetadatum("expires")); - expectedExpiresIn = System.currentTimeMillis() + TWO_WEEKS_MILLIS; - actualExpiresIn = found.get(0).getMetadatumLong("expiresIn", 0); - assertTrue("expiresIn should be set to 2 weeks from the current time", - Math.abs(expectedExpiresIn - actualExpiresIn) < 1000); + expectedExpires= System.currentTimeMillis() + TWO_WEEKS_MILLIS; + actualExpires = found.get(0).getMetadatumLong("expires", 0); + assertTrue("expires field should be set to 2 weeks from the current time", + Math.abs(expectedExpires - actualExpires) < 1000); found = cache.getInventoryDB().findObject("mds1491/README.txt", VolumeStatus.VOL_FOR_INFO); assertEquals(1, found.size()); - assertTrue("CacheObject should contain expiresIn metadata", found.get(0).hasMetadatum("expiresIn")); + assertTrue("CacheObject should contain expires metadata", found.get(0).hasMetadatum("expires")); - expectedExpiresIn = System.currentTimeMillis() + TWO_WEEKS_MILLIS; - actualExpiresIn = found.get(0).getMetadatumLong("expiresIn", 0); - assertTrue("expiresIn should be set to 2 weeks from the current time", - Math.abs(expectedExpiresIn - actualExpiresIn) < 1000); + expectedExpires = System.currentTimeMillis() + TWO_WEEKS_MILLIS; + actualExpires = found.get(0).getMetadatumLong("expires", 0); + assertTrue("expires field should be set to 2 weeks from the current time", + Math.abs(expectedExpires - actualExpires) < 1000); } From 980b5fdbe56a48f2f677a3387ea8a802ec97ff78 Mon Sep 17 00:00:00 2001 From: elmiomar Date: Wed, 6 Mar 2024 11:25:55 -0500 Subject: [PATCH 19/20] fix unit tests --- .../service/RPACachingServiceTest.java | 29 +++++++++++-------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/src/test/java/gov/nist/oar/distrib/service/RPACachingServiceTest.java b/src/test/java/gov/nist/oar/distrib/service/RPACachingServiceTest.java index c3d0f108..52829497 100644 --- a/src/test/java/gov/nist/oar/distrib/service/RPACachingServiceTest.java +++ b/src/test/java/gov/nist/oar/distrib/service/RPACachingServiceTest.java @@ -52,8 +52,8 @@ public void testCacheAndGenerateRandomId_validDatasetID() throws Exception { String result = rpaCachingService.cacheAndGenerateRandomId(datasetID, version); assertNotNull(result); - assertEquals(RPACachingService.RANDOM_ID_LENGTH, result.length()); - assertTrue(result.matches("^[a-zA-Z0-9]*$")); // check that the ID is alphanumeric + assertEquals(RPACachingService.RANDOM_ID_LENGTH + 4, result.length()); // 4 for the 'rpa-' prefix + assertTrue(result.matches("^rpa-[a-zA-Z0-9]+$")); // Check that the ID starts with 'rpa-' followed by alphanumeric chars verify(pdrCacheManager).cacheDataset(eq("mds2-2909"), eq(version), eq(true), eq(RPACachingService.ROLE_RESTRICTED_DATA), eq(result)); } @@ -69,8 +69,8 @@ public void testCacheAndGenerateRandomId_validDatasetArkID() throws Exception { String result = rpaCachingService.cacheAndGenerateRandomId(datasetID, version); assertNotNull(result); - assertEquals(RPACachingService.RANDOM_ID_LENGTH, result.length()); - assertTrue(result.matches("^[a-zA-Z0-9]*$")); // check that the ID is alphanumeric + assertEquals(RPACachingService.RANDOM_ID_LENGTH + 4, result.length()); // 4 for the 'rpa-' prefix + assertTrue(result.matches("^rpa-[a-zA-Z0-9]+$")); // Check that the ID starts with 'rpa-' followed by alphanumeric chars verify(pdrCacheManager).cacheDataset(eq("mds2-2909"), eq(version), eq(true), eq(RPACachingService.ROLE_RESTRICTED_DATA), eq(result)); } @@ -85,6 +85,7 @@ public void testCacheAndGenerateRandomId_invalidDatasetArkID() throws Exception @Test public void testRetrieveMetadata_success() throws Exception { String randomID = "randomId123"; + String aipid = "456"; CacheObject cacheObject1 = new CacheObject("object1", new JSONObject() .put("filepath", "path/to/file1.txt") .put("contentType", "text/plain") @@ -95,7 +96,7 @@ public void testRetrieveMetadata_success() throws Exception { .put("checksum", "abc123") .put("version", "v1") .put("ediid", "123") - .put("aipid", "456") + .put("aipid", aipid) .put("sinceDate", "08-05-2023"), "Volume1"); @@ -109,7 +110,7 @@ public void testRetrieveMetadata_success() throws Exception { .put("checksum", "def456") .put("version", "v2") .put("ediid", "123") - .put("aipid", "456") + .put("aipid", aipid) .put("sinceDate", "08-05-2023"), "Volume1"); @@ -124,7 +125,8 @@ public void testRetrieveMetadata_success() throws Exception { Map expected = new HashMap<>(); expected.put("randomId", randomID); expected.put("metadata", new JSONArray() - .put(new JSONObject().put("downloadURL", testBaseDownloadUrl + "/" + randomID + "/path/to/file1.txt") + .put(new JSONObject().put("downloadURL", testBaseDownloadUrl + "/" + randomID + + "/" + aipid + "/path/to/file1.txt") .put("filePath", "path/to/file1.txt") .put("mediaType", "text/plain") .put("size", 100L) @@ -136,7 +138,8 @@ public void testRetrieveMetadata_success() throws Exception { .put("ediid", "123") .put("aipid", "456") .put("sinceDate", "08-05-2023")) - .put(new JSONObject().put("downloadURL", testBaseDownloadUrl + "/" + randomID + "/path/to/file2.txt") + .put(new JSONObject().put("downloadURL", testBaseDownloadUrl + "/" + randomID + + "/" + aipid + "/path/to/file2.txt") .put("filePath", "path/to/file2.txt") .put("mediaType", "text/plain") .put("size", 100L) @@ -159,6 +162,7 @@ public void testRetrieveMetadata_success() throws Exception { @Test public void testRetrieveMetadata_withMissingFilepath() throws Exception { String randomID = "randomId123"; + String aipid = "456"; CacheObject cacheObject1 = new CacheObject("object1", new JSONObject() .put("contentType", "text/plain") .put("size", 100L) @@ -168,7 +172,7 @@ public void testRetrieveMetadata_withMissingFilepath() throws Exception { .put("checksum", "abc123") .put("version", "v1") .put("ediid", "123") - .put("aipid", "456") + .put("aipid", aipid) .put("sinceDate", "08-05-2023"), "Volume1"); @@ -182,7 +186,7 @@ public void testRetrieveMetadata_withMissingFilepath() throws Exception { .put("checksum", "def456") .put("version", "v2") .put("ediid", "123") - .put("aipid", "456") + .put("aipid", aipid) .put("sinceDate", "08-05-2023"), "Volume1"); @@ -197,7 +201,8 @@ public void testRetrieveMetadata_withMissingFilepath() throws Exception { Map expected = new HashMap<>(); expected.put("randomId", randomID); expected.put("metadata", new JSONArray() - .put(new JSONObject().put("downloadURL", testBaseDownloadUrl + "/" + randomID + "/path/to/file2.txt") + .put(new JSONObject().put("downloadURL", testBaseDownloadUrl + "/" + randomID + + "/" + aipid +"/path/to/file2.txt") .put("filePath", "path/to/file2.txt") .put("mediaType", "text/plain") .put("size", 100L) @@ -207,7 +212,7 @@ public void testRetrieveMetadata_withMissingFilepath() throws Exception { .put("checksum", "def456") .put("version", "v2") .put("ediid", "123") - .put("aipid", "456") + .put("aipid", aipid) .put("sinceDate", "08-05-2023")) .toList()); From 8cc659f97642ff792d8b5835a80f40d5d205b65e Mon Sep 17 00:00:00 2001 From: elmiomar Date: Fri, 8 Mar 2024 15:44:34 -0500 Subject: [PATCH 20/20] remove redundant uncaching endpoint --- .../distrib/web/RPADataCachingController.java | 28 ------------------- 1 file changed, 28 deletions(-) diff --git a/src/main/java/gov/nist/oar/distrib/web/RPADataCachingController.java b/src/main/java/gov/nist/oar/distrib/web/RPADataCachingController.java index 4d2520be..e9418f2b 100644 --- a/src/main/java/gov/nist/oar/distrib/web/RPADataCachingController.java +++ b/src/main/java/gov/nist/oar/distrib/web/RPADataCachingController.java @@ -149,34 +149,6 @@ public Map retrieveMetadata(@PathVariable("cacheid") String cach return metadata; } - /** - * This endpoint handles the removal of a dataset from the cache based on its randomly generated ID. - * - * @param randomId The randomly generated ID that was assigned when the dataset was cached. - * - * @return A ResponseEntity containing a status message and HTTP status code. - * - * @throws CacheManagementException If there is an issue with removing the dataset from the cache. - * @throws ResourceNotFoundException If the dataset associated with the randomId is not found in the cache. - * @throws StorageVolumeException If there is an issue with the storage volume. - */ - @PutMapping(value = "/uncache/{randomId}") - public ResponseEntity uncacheDataset( - @PathVariable("randomId") String randomId) - throws CacheManagementException { - - logger.debug("randomId to uncache=" + randomId); - - // Perform uncache operation via the service - boolean success = restrictedSrvc.uncacheById(randomId); - - if (success) { - return new ResponseEntity<>("Dataset was successfully removed from cache.", HttpStatus.OK); - } else { - return new ResponseEntity<>("Failed to uncache the dataset.", HttpStatus.INTERNAL_SERVER_ERROR); - } - } - @ExceptionHandler(MetadataNotFoundException.class) @ResponseStatus(HttpStatus.NOT_FOUND)