From 1a0fcae324feaa4c8b5c97af6353c636aedaa47d Mon Sep 17 00:00:00 2001 From: elmiomar Date: Wed, 11 Oct 2023 12:44:21 -0400 Subject: [PATCH 01/10] add support for uncaching --- .../distrib/service/RPACachingService.java | 7 +++++ .../distrib/web/RPADataCachingController.java | 30 +++++++++++++++++++ .../web/RPARequestHandlerController.java | 4 +-- 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/src/main/java/gov/nist/oar/distrib/service/RPACachingService.java b/src/main/java/gov/nist/oar/distrib/service/RPACachingService.java index ec937291..5e51793a 100644 --- a/src/main/java/gov/nist/oar/distrib/service/RPACachingService.java +++ b/src/main/java/gov/nist/oar/distrib/service/RPACachingService.java @@ -62,6 +62,7 @@ public String cacheAndGenerateRandomId(String datasetID, String version) logger.debug("Request to cache dataset with ID=" + datasetID); + // this is to handle ark IDs String dsid = datasetID; if (datasetID.startsWith("ark:/")) { // Split the dataset ID into components @@ -246,4 +247,10 @@ private String getDownloadUrl(String baseDownloadUrl, String randomId, String pa private String generateRandomID(int length, boolean useLetters, boolean useNumbers) { return RandomStringUtils.random(length, useLetters, useNumbers); } + + public boolean uncacheById(String randomId) throws CacheManagementException { + logger.debug("Request to cache dataset with ID=" + randomId); + this.pdrCacheManager.uncache(randomId); + return false; + } } \ No newline at end of file diff --git a/src/main/java/gov/nist/oar/distrib/web/RPADataCachingController.java b/src/main/java/gov/nist/oar/distrib/web/RPADataCachingController.java index f585194c..53e84289 100644 --- a/src/main/java/gov/nist/oar/distrib/web/RPADataCachingController.java +++ b/src/main/java/gov/nist/oar/distrib/web/RPADataCachingController.java @@ -12,6 +12,7 @@ import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.ExceptionHandler; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; @@ -112,6 +113,35 @@ public Map retrieveMetadata(@PathVariable("cacheid") String cach return metadata; } + /** + * This endpoint handles the removal of a dataset from the cache based on its randomly generated ID. + * + * @param randomId The randomly generated ID that was assigned when the dataset was cached. + * + * @return A ResponseEntity containing a status message and HTTP status code. + * + * @throws CacheManagementException If there is an issue with removing the dataset from the cache. + * @throws ResourceNotFoundException If the dataset associated with the randomId is not found in the cache. + * @throws StorageVolumeException If there is an issue with the storage volume. + */ + @PutMapping(value = "/uncache/{randomId}") + public ResponseEntity uncacheDataset( + @PathVariable("randomId") String randomId) + throws CacheManagementException { + + logger.debug("randomId to uncache=" + randomId); + + // Perform uncache operation via the service + boolean success = restrictedSrvc.uncacheById(randomId); + + if (success) { + return new ResponseEntity<>("Dataset was successfully removed from cache.", HttpStatus.OK); + } else { + return new ResponseEntity<>("Failed to uncache the dataset.", HttpStatus.INTERNAL_SERVER_ERROR); + } + } + + @ExceptionHandler(MetadataNotFoundException.class) @ResponseStatus(HttpStatus.NOT_FOUND) public ErrorInfo handleMetadataNotFoundException(MetadataNotFoundException ex) { diff --git a/src/main/java/gov/nist/oar/distrib/web/RPARequestHandlerController.java b/src/main/java/gov/nist/oar/distrib/web/RPARequestHandlerController.java index 0ddee587..183b3f3a 100644 --- a/src/main/java/gov/nist/oar/distrib/web/RPARequestHandlerController.java +++ b/src/main/java/gov/nist/oar/distrib/web/RPARequestHandlerController.java @@ -299,8 +299,8 @@ public ResponseEntity updateRecord(@PathVariable String id, @RequestBody RecordP LOGGER.debug("Missing required claim detected: " + missingClaimName); throw new InvalidRequestException("JWT token invalid"); } catch (JwtException e) { - LOGGER.debug("Token validation failed due to JwtException: " + e.getMessage()); - throw new RequestProcessingException("JWT token validation failed"); + LOGGER.debug("Token validation failed due to a JwtException: " + e.getMessage()); + throw new UnauthorizedException("JWT token validation failed"); } if (tokenDetails != null) { From 668350e8c7f41b436e2cd932e2f7426039fc809c Mon Sep 17 00:00:00 2001 From: elmiomar Date: Wed, 1 Nov 2023 11:07:07 -0400 Subject: [PATCH 02/10] add rpa- keyword to random id --- .../distrib/service/RPACachingService.java | 45 +++++++++++++++++-- 1 file changed, 41 insertions(+), 4 deletions(-) diff --git a/src/main/java/gov/nist/oar/distrib/service/RPACachingService.java b/src/main/java/gov/nist/oar/distrib/service/RPACachingService.java index 5e51793a..51e04da8 100644 --- a/src/main/java/gov/nist/oar/distrib/service/RPACachingService.java +++ b/src/main/java/gov/nist/oar/distrib/service/RPACachingService.java @@ -74,7 +74,9 @@ public String cacheAndGenerateRandomId(String datasetID, String version) } logger.debug("Caching dataset with dsid=" + dsid); - String randomID = generateRandomID(RANDOM_ID_LENGTH, true, true); + // append "rpa-" with the generated random ID + String randomID = "rpa-" + generateRandomID(RANDOM_ID_LENGTH, true, true); + int prefs = ROLE_RESTRICTED_DATA; if (!version.isEmpty()) @@ -248,9 +250,44 @@ private String generateRandomID(int length, boolean useLetters, boolean useNumbe return RandomStringUtils.random(length, useLetters, useNumbers); } + /** + * Uncache dataset objects using a specified random ID. + * + * @param randomId - The random ID used to fetch and uncache dataset objects. + * @return boolean - True if at least one dataset object was uncached successfully; otherwise, false. + * @throws CacheManagementException if an error occurs during the uncaching process. + */ public boolean uncacheById(String randomId) throws CacheManagementException { - logger.debug("Request to cache dataset with ID=" + randomId); - this.pdrCacheManager.uncache(randomId); - return false; + // Validate input + if (randomId == null || randomId.isEmpty()) { + throw new IllegalArgumentException("Random ID cannot be null or empty."); + } + + logger.debug("Request to uncache dataset with ID=" + randomId); + + // Retrieve dataset objects using the randomId + List objects = this.pdrCacheManager.selectDatasetObjects(randomId, this.pdrCacheManager.VOL_FOR_INFO); + + if (objects.isEmpty()) { + logger.debug("No objects found for ID=" + randomId); + return false; + } + + boolean isUncached = false; + + // Iterate through the retrieved objects and attempt to uncache them + for (CacheObject obj : objects) { + try { + logger.debug("Deleting file with ID=" + obj.id); + this.pdrCacheManager.uncache(obj.id); + isUncached = true; + } catch (CacheManagementException e) { + // Log the exception without throwing it to continue attempting to uncache remaining objects + logger.error("Failed to uncache object with ID=" + obj.id, e); + } + } + + return isUncached; } + } \ No newline at end of file From 878f44fd2be17c0db77a6e6c36d9a0b54e8d7cad Mon Sep 17 00:00:00 2001 From: elmiomar Date: Wed, 1 Nov 2023 11:10:36 -0400 Subject: [PATCH 03/10] update caching logic: cache first then append randomId to status --- ...URLConnectionRPARequestHandlerService.java | 48 ++++++++++++------- .../service/rpa/RecordResponseHandler.java | 2 +- .../rpa/RecordResponseHandlerImpl.java | 18 ++++--- 3 files changed, 39 insertions(+), 29 deletions(-) diff --git a/src/main/java/gov/nist/oar/distrib/service/rpa/HttpURLConnectionRPARequestHandlerService.java b/src/main/java/gov/nist/oar/distrib/service/rpa/HttpURLConnectionRPARequestHandlerService.java index 1ba31865..0de195e1 100644 --- a/src/main/java/gov/nist/oar/distrib/service/rpa/HttpURLConnectionRPARequestHandlerService.java +++ b/src/main/java/gov/nist/oar/distrib/service/rpa/HttpURLConnectionRPARequestHandlerService.java @@ -98,6 +98,7 @@ public class HttpURLConnectionRPARequestHandlerService implements IRPARequestHan */ private RecordResponseHandler recordResponseHandler; + private RPADatasetCacher rpaDatasetCacher; /** * Sets the HTTP URL connection factory. * @@ -164,6 +165,9 @@ public HttpURLConnectionRPARequestHandlerService(RPAConfiguration rpaConfigurati this.recordResponseHandler = new RecordResponseHandlerImpl(this.rpaConfiguration, this.connectionFactory, rpaCachingService); + // Set RPADatasetCacher + this.rpaDatasetCacher = new DefaultRPADatasetCacher(rpaCachingService); + // Set HttpClient this.httpClient = HttpClients.createDefault(); @@ -388,13 +392,16 @@ public RecordStatus updateRecord(String recordId, String status, String smeId) t // Initialize return object RecordStatus recordStatus; - // Get endpoint - String updateRecordUri = getConfig().getSalesforceEndpoints().get(UPDATE_RECORD_ENDPOINT_KEY); + // Cache here before updating the status in SF + LOGGER.info("Starting caching..."); + Record record = this.getRecord(recordId).getRecord(); + String datasetId = record.getUserInfo().getSubject(); + String randomId = this.rpaDatasetCacher.cache(datasetId); + if (randomId == null) + throw new RequestProcessingException("Caching process returned a null randomId"); // Create a valid approval status based on input - String approvalStatus = generateApprovalStatus(status, smeId); - - // TODO: try caching here before updating the status in SF + String approvalStatus = generateApprovalStatus(status, smeId, randomId); // PATCH request payload // Approval_Status__c is how SF service expect the key @@ -404,6 +411,9 @@ public RecordStatus updateRecord(String recordId, String status, String smeId) t // Get token JWTToken token = jwtHelper.getToken(); + // Get endpoint + String updateRecordUri = getConfig().getSalesforceEndpoints().get(UPDATE_RECORD_ENDPOINT_KEY); + // Build request URL String url; try { @@ -448,12 +458,9 @@ public RecordStatus updateRecord(String recordId, String status, String smeId) t throw new RequestProcessingException("I/O error: " + e.getMessage()); } - // Retrieve updated record from SF service - Record record = this.getRecord(recordId).getRecord(); - // Check if status is approved if (recordStatus.getApprovalStatus().toLowerCase().contains("approved")) { - this.recordResponseHandler.onRecordUpdateApproved(record); + this.recordResponseHandler.onRecordUpdateApproved(record, randomId); } else { this.recordResponseHandler.onRecordUpdateDeclined(record); } @@ -462,24 +469,29 @@ public RecordStatus updateRecord(String recordId, String status, String smeId) t } /** - * Generates an approval status string based on the given status and current date/time. - * The date is in ISO 8601 format. + * Generates an approval status string based on the given status, current date/time, and random ID. + * The date is in ISO 8601 format. If the status is "Declined", the randomId will not be appended. * - * @param status the approval status to use, either "Approved" or "Declined" - * @param email the email to append to the status - * @return the generated approval status string, in the format "[status]_[yyyy-MM-dd'T'HH:mm:ss.SSSZ]_[email]" + * @param status the approval status to use, either "Approved" or "Declined" + * @param smeId the SME ID to append to the status + * @param randomId the generated random ID to append (only if status is "Approved") + * @return the generated approval status string. + * If status is "Approved", the format is: + * "[status]_[yyyy-MM-dd'T'HH:mm:ss.SSSZ]_[smeId]_[randomId]". + * If status is "Declined", the format is: + * "[status]_[yyyy-MM-dd'T'HH:mm:ss.SSSZ]_[smeId]" * @throws InvalidRequestException if the provided status is not "Approved" or "Declined" */ - private String generateApprovalStatus(String status, String smeId) throws InvalidRequestException { + private String generateApprovalStatus(String status, String smeId, String randomId) throws InvalidRequestException { String formattedDate = Instant.now().toString(); // ISO 8601 format: 2023-05-09T15:59:03.872Z String approvalStatus; if (status != null) { switch (status.toLowerCase()) { case RECORD_APPROVED_STATUS: - approvalStatus = "Approved_"; + approvalStatus = "Approved_" + formattedDate + "_" + smeId + "_" + randomId; break; case RECORD_DECLINED_STATUS: - approvalStatus = "Declined_"; + approvalStatus = "Declined_" + formattedDate + "_" + smeId; break; default: throw new InvalidRequestException("Invalid approval status: " + status); @@ -487,7 +499,7 @@ private String generateApprovalStatus(String status, String smeId) throws Invali } else { throw new InvalidRequestException("Invalid approval status: status is null"); } - return approvalStatus + formattedDate + "_" + smeId; + return approvalStatus; } } diff --git a/src/main/java/gov/nist/oar/distrib/service/rpa/RecordResponseHandler.java b/src/main/java/gov/nist/oar/distrib/service/rpa/RecordResponseHandler.java index a1941a2e..7e29f588 100644 --- a/src/main/java/gov/nist/oar/distrib/service/rpa/RecordResponseHandler.java +++ b/src/main/java/gov/nist/oar/distrib/service/rpa/RecordResponseHandler.java @@ -23,7 +23,7 @@ public interface RecordResponseHandler { * This method is called when a record update operation is successful and user was approved. * @param record The record that was the user approved for. */ - void onRecordUpdateApproved(Record record); + void onRecordUpdateApproved(Record record, String randomId); /** * This method is called when a record update operation is successful but user was declined. diff --git a/src/main/java/gov/nist/oar/distrib/service/rpa/RecordResponseHandlerImpl.java b/src/main/java/gov/nist/oar/distrib/service/rpa/RecordResponseHandlerImpl.java index 5a456954..7a5315a0 100644 --- a/src/main/java/gov/nist/oar/distrib/service/rpa/RecordResponseHandlerImpl.java +++ b/src/main/java/gov/nist/oar/distrib/service/rpa/RecordResponseHandlerImpl.java @@ -92,20 +92,18 @@ public void onRecordCreationFailure(int statusCode) throws RequestProcessingExce } /** - * Called when a record status was updated to "Approved". - * This uses {@link RPADatasetCacher} to cache the dataset. + * Called when a record update operation has been approved. * - * @param record the record that was updated + * @param record the record that was updated and approved + * @param randomId the ID generated after caching the dataset related to the record + * @throws InvalidRequestException if there is an error in the request + * @throws RequestProcessingException if there is an error while processing the request */ @Override - public void onRecordUpdateApproved(Record record) throws InvalidRequestException, RequestProcessingException { - LOGGER.info("User was approved by SME. Starting caching..."); - String datasetId = record.getUserInfo().getSubject(); - // NEW: case dataset using the RPADatasetCacher - String randomId = this.rpaDatasetCacher.cache(datasetId); - if (randomId == null) - throw new RequestProcessingException("Caching process return a null randomId"); + public void onRecordUpdateApproved(Record record, String randomId) throws InvalidRequestException, RequestProcessingException { + LOGGER.info("Dataset was cached successfully. Sending email to user..."); + // Build Download URL String downloadUrl; try { From 02dbd568bece3f7ac42d2493f5803cd26c856db1 Mon Sep 17 00:00:00 2001 From: elmiomar Date: Wed, 29 Nov 2023 11:56:48 -0500 Subject: [PATCH 04/10] update caching logic --- .../service/rpa/DefaultRPADatasetCacher.java | 13 ++++ .../service/rpa/HttpRPADatasetCacher.java | 5 ++ ...URLConnectionRPARequestHandlerService.java | 69 +++++++++++++++---- .../distrib/service/rpa/RPADatasetCacher.java | 2 + .../rpa/RecordResponseHandlerImpl.java | 2 +- 5 files changed, 77 insertions(+), 14 deletions(-) diff --git a/src/main/java/gov/nist/oar/distrib/service/rpa/DefaultRPADatasetCacher.java b/src/main/java/gov/nist/oar/distrib/service/rpa/DefaultRPADatasetCacher.java index 3ac34daf..0e915b71 100644 --- a/src/main/java/gov/nist/oar/distrib/service/rpa/DefaultRPADatasetCacher.java +++ b/src/main/java/gov/nist/oar/distrib/service/rpa/DefaultRPADatasetCacher.java @@ -37,6 +37,19 @@ public String cache(String datasetId) throws RequestProcessingException { return randomId; } + @Override + public boolean uncache(String randomId) { + boolean uncached = false; + try { + uncached = rpaCachingService.uncacheById(randomId); + } catch (Exception e) { + this.logCachingException(e); + throw new RequestProcessingException(e.getMessage()); + } + + return uncached; + } + /** * Logs the specified exception to the debug log, along with its stack trace. * diff --git a/src/main/java/gov/nist/oar/distrib/service/rpa/HttpRPADatasetCacher.java b/src/main/java/gov/nist/oar/distrib/service/rpa/HttpRPADatasetCacher.java index 9bf5f76f..19204c88 100644 --- a/src/main/java/gov/nist/oar/distrib/service/rpa/HttpRPADatasetCacher.java +++ b/src/main/java/gov/nist/oar/distrib/service/rpa/HttpRPADatasetCacher.java @@ -50,6 +50,11 @@ public String cache(String datasetId) { return sendHttpRequest(datasetId, url); } + @Override + public boolean uncache(String randomId) { + return false; + } + /** * Builds the URL for the given dataset ID and using the given {@link RPAConfiguration} object. * diff --git a/src/main/java/gov/nist/oar/distrib/service/rpa/HttpURLConnectionRPARequestHandlerService.java b/src/main/java/gov/nist/oar/distrib/service/rpa/HttpURLConnectionRPARequestHandlerService.java index 0de195e1..6a391b9e 100644 --- a/src/main/java/gov/nist/oar/distrib/service/rpa/HttpURLConnectionRPARequestHandlerService.java +++ b/src/main/java/gov/nist/oar/distrib/service/rpa/HttpURLConnectionRPARequestHandlerService.java @@ -377,28 +377,52 @@ private String prepareRequestPayload(UserInfoWrapper userInfoWrapper) throws Jso /** - * Updates the status of a record with a given ID. + * Updates the status of a record in the database. + *

+ * This method handles the approval or decline of a record. When a record is approved, it caches the dataset, + * generates a random ID, and appends this ID to the status before updating the record in the database. + * When a record is declined, the dataset is not cached, a null random ID is used, and the record status is + * updated in the database without appending the random ID. + *

+ *

+ * In cases where a record was initially approved (and thus cached with a random ID) but is later declined, + * this method retrieves the random ID from the status, uncaches the dataset using this ID, and updates + * the record status without the random ID. + *

* * @param recordId The ID of the record to update. - * @param status The status to update the record with. - * @return The {@link RecordStatus} object representing the updated record status. + * @param status The new status to be set for the record. Can be 'Approved' or 'Declined'. + * @param smeId The SME ID associated with the record update. + * @return A {@link RecordStatus} object representing the updated record status. * @throws RecordNotFoundException If the record with the given ID is not found. - * @throws InvalidRequestException If the request is invalid. - * @throws RequestProcessingException If there is an error processing the request. + * @throws InvalidRequestException If the provided status is invalid or the request is otherwise invalid. + * @throws RequestProcessingException If there is an error in processing the update request, such as issues + * with caching or communication errors with the database. */ @Override public RecordStatus updateRecord(String recordId, String status, String smeId) throws RecordNotFoundException, InvalidRequestException, RequestProcessingException { // Initialize return object RecordStatus recordStatus; - - // Cache here before updating the status in SF - LOGGER.info("Starting caching..."); Record record = this.getRecord(recordId).getRecord(); String datasetId = record.getUserInfo().getSubject(); - String randomId = this.rpaDatasetCacher.cache(datasetId); - if (randomId == null) - throw new RequestProcessingException("Caching process returned a null randomId"); + String randomId = null; + + // If the record is being approved + if (RECORD_APPROVED_STATUS.equalsIgnoreCase(status)) { + LOGGER.info("Starting caching..."); + randomId = this.rpaDatasetCacher.cache(datasetId); + if (randomId == null) { + throw new RequestProcessingException("Caching process returned a null randomId"); + } + } + // If the record is being declined, check if it needs uncaching + else if (RECORD_DECLINED_STATUS.equalsIgnoreCase(status)) { + randomId = extractRandomIdFromCurrentStatus(record.getUserInfo().getApprovalStatus()); + if (randomId != null) { + this.rpaDatasetCacher.uncache(randomId); + } + } // Create a valid approval status based on input String approvalStatus = generateApprovalStatus(status, smeId, randomId); @@ -468,6 +492,20 @@ public RecordStatus updateRecord(String recordId, String status, String smeId) t return recordStatus; } + private String extractRandomIdFromCurrentStatus(String currentStatus) { + if (currentStatus != null && currentStatus.startsWith("Approved_")) { + String[] parts = currentStatus.split("_"); + + // Since the expected format is "[status]_[yyyy-MM-dd'T'HH:mm:ss.SSSZ]_[smeId]_[randomId]", + // the randomId should be the 4th part, if all parts are present + if (parts.length == 4) { + return parts[3]; + } + } + return null; + } + + /** * Generates an approval status string based on the given status, current date/time, and random ID. * The date is in ISO 8601 format. If the status is "Declined", the randomId will not be appended. @@ -483,12 +521,16 @@ public RecordStatus updateRecord(String recordId, String status, String smeId) t * @throws InvalidRequestException if the provided status is not "Approved" or "Declined" */ private String generateApprovalStatus(String status, String smeId, String randomId) throws InvalidRequestException { - String formattedDate = Instant.now().toString(); // ISO 8601 format: 2023-05-09T15:59:03.872Z + String formattedDate = Instant.now().toString(); String approvalStatus; + if (status != null) { switch (status.toLowerCase()) { case RECORD_APPROVED_STATUS: - approvalStatus = "Approved_" + formattedDate + "_" + smeId + "_" + randomId; + approvalStatus = "Approved_" + formattedDate + "_" + smeId; + if (randomId != null) { + approvalStatus += "_" + randomId; + } break; case RECORD_DECLINED_STATUS: approvalStatus = "Declined_" + formattedDate + "_" + smeId; @@ -502,4 +544,5 @@ private String generateApprovalStatus(String status, String smeId, String random return approvalStatus; } + } diff --git a/src/main/java/gov/nist/oar/distrib/service/rpa/RPADatasetCacher.java b/src/main/java/gov/nist/oar/distrib/service/rpa/RPADatasetCacher.java index 2a2150c3..93395cc8 100644 --- a/src/main/java/gov/nist/oar/distrib/service/rpa/RPADatasetCacher.java +++ b/src/main/java/gov/nist/oar/distrib/service/rpa/RPADatasetCacher.java @@ -14,4 +14,6 @@ public interface RPADatasetCacher { * @throws RequestProcessingException */ String cache(String datasetId) throws RequestProcessingException; + + boolean uncache(String randomId); } diff --git a/src/main/java/gov/nist/oar/distrib/service/rpa/RecordResponseHandlerImpl.java b/src/main/java/gov/nist/oar/distrib/service/rpa/RecordResponseHandlerImpl.java index 7a5315a0..ca7f4150 100644 --- a/src/main/java/gov/nist/oar/distrib/service/rpa/RecordResponseHandlerImpl.java +++ b/src/main/java/gov/nist/oar/distrib/service/rpa/RecordResponseHandlerImpl.java @@ -125,7 +125,7 @@ public void onRecordUpdateApproved(Record record, String randomId) throws Invali * @param record the record that was updated */ @Override - public void onRecordUpdateDeclined(Record record) { + public void onRecordUpdateDeclined(Record record) throws InvalidRequestException, RequestProcessingException { LOGGER.debug("User was declined by SME"); } From 830675c36539eedb75eb5bf44c65ece0673c1150 Mon Sep 17 00:00:00 2001 From: elmiomar Date: Tue, 5 Dec 2023 06:57:16 -0500 Subject: [PATCH 05/10] include aipid in Download URLs --- .../cachemgr/pdr/PDRDatasetRestorer.java | 2 +- .../distrib/service/RPACachingService.java | 43 ++++++++++++++----- 2 files changed, 33 insertions(+), 12 deletions(-) diff --git a/src/main/java/gov/nist/oar/distrib/cachemgr/pdr/PDRDatasetRestorer.java b/src/main/java/gov/nist/oar/distrib/cachemgr/pdr/PDRDatasetRestorer.java index 12b77b2d..f4da33b1 100644 --- a/src/main/java/gov/nist/oar/distrib/cachemgr/pdr/PDRDatasetRestorer.java +++ b/src/main/java/gov/nist/oar/distrib/cachemgr/pdr/PDRDatasetRestorer.java @@ -645,7 +645,7 @@ public String idForObject(String aipid, String filepath, String forVersion, Stri String id; id = aipid + "/" + filepath; if (target != null && !target.isEmpty()) - id = target + "/" + filepath; + id = target + "/" + aipid + "/" + filepath; if (forVersion != null && forVersion.length() > 0) id += "#" + forVersion; return id; diff --git a/src/main/java/gov/nist/oar/distrib/service/RPACachingService.java b/src/main/java/gov/nist/oar/distrib/service/RPACachingService.java index 51e04da8..b3abf742 100644 --- a/src/main/java/gov/nist/oar/distrib/service/RPACachingService.java +++ b/src/main/java/gov/nist/oar/distrib/service/RPACachingService.java @@ -129,6 +129,7 @@ public Map retrieveMetadata(String randomID) throws CacheManagem /** * Formats the metadata from a cache object to a JSON object with an additional field for the download URL. + * The download URL includes the random temporary ID, aipid, and the file path from the metadata. * * @param inMd the metadata from the cache object * @param randomID the random temporary ID associated with the cache object @@ -139,17 +140,27 @@ private JSONObject formatMetadata(JSONObject inMd, String randomID) throws Reque JSONObject outMd = new JSONObject(); List missingFields = new ArrayList<>(); + String aipid = ""; + if (inMd.has("aipid")) { + aipid = inMd.getString("aipid"); + outMd.put("aipid", aipid); + } else { + missingFields.add("aipid"); + } + if (inMd.has("filepath")) { String downloadURL = getDownloadUrl( rpaConfiguration.getBaseDownloadUrl(), randomID, - inMd.get("filepath").toString()); + aipid, + inMd.getString("filepath")); outMd.put("downloadURL", downloadURL); - outMd.put("filePath", inMd.get("filepath")); + outMd.put("filePath", inMd.getString("filepath")); } else { missingFields.add("filepath"); } + if (inMd.has("contentType")) { outMd.put("mediaType", inMd.get("contentType")); } else { @@ -199,12 +210,6 @@ private JSONObject formatMetadata(JSONObject inMd, String randomID) throws Reque missingFields.add("ediid"); } - if (inMd.has("aipid")) { - outMd.put("aipid", inMd.get("aipid")); - } else { - missingFields.add("aipid"); - } - if (inMd.has("sinceDate")) { outMd.put("sinceDate", inMd.get("sinceDate")); } else { @@ -220,28 +225,44 @@ private JSONObject formatMetadata(JSONObject inMd, String randomID) throws Reque /** - * Constructs a download URL using the given base download URL, random ID, and file path from the metadata. + * Constructs a download URL using the given base download URL, random ID, aipid, and file path from the metadata. * * @param baseDownloadUrl the base download URL * @param randomId the random temporary ID + * @param aipid the aipid from the metadata * @param path the file path from the metadata * @return the download URL as a string * @throws RequestProcessingException if there was an error building the download URL */ - private String getDownloadUrl(String baseDownloadUrl, String randomId, String path) throws RequestProcessingException { + + private String getDownloadUrl(String baseDownloadUrl, String randomId, String aipid, String path) throws RequestProcessingException { URL downloadUrl; try { URL url = new URL(baseDownloadUrl); + StringBuilder pathBuilder = new StringBuilder(); + + // append the randomId to the path + pathBuilder.append(randomId); + + // append the aipid if it's not empty + if (!aipid.isEmpty()) { + pathBuilder.append("/").append(aipid); + } + + // append the file path, ensuring it doesn't start with a "/" if (path.startsWith("/")) { path = path.substring(1); } - downloadUrl = new URL(url, randomId + "/" + path); + pathBuilder.append("/").append(path); + + downloadUrl = new URL(url, pathBuilder.toString()); } catch (MalformedURLException e) { throw new RequestProcessingException("Failed to build downloadUrl: " + e.getMessage()); } return downloadUrl.toString(); } + /** * Generate a random alphanumeric string for the dataset to store * This function uses the {@link RandomStringUtils} from Apache Commons. From 9bbd5847f8ca5298b3a299f2e30c0f16302202c4 Mon Sep 17 00:00:00 2001 From: elmiomar Date: Tue, 5 Dec 2023 07:41:36 -0500 Subject: [PATCH 06/10] add certificates --- docker/build-test/Dockerfile | 24 +++++++---- .../cacerts/Forward_Proxy_NIST_CA.crt | 41 +++++++++++++++++++ docker/build-test/cacerts/NISTRoot02.crt | 29 +++++++++++++ docker/build-test/cacerts/README.md | 13 ++++++ docker/cacerts/README.md | 13 ++++++ 5 files changed, 111 insertions(+), 9 deletions(-) create mode 100644 docker/build-test/cacerts/Forward_Proxy_NIST_CA.crt create mode 100644 docker/build-test/cacerts/NISTRoot02.crt create mode 100644 docker/build-test/cacerts/README.md create mode 100644 docker/cacerts/README.md diff --git a/docker/build-test/Dockerfile b/docker/build-test/Dockerfile index 431e9d1c..92068bad 100644 --- a/docker/build-test/Dockerfile +++ b/docker/build-test/Dockerfile @@ -1,14 +1,20 @@ -FROM ibmjava:8-sdk - -# a hack that gets around an installation problem with update-alternatives, openjdk-8-jdk-headless +FROM eclipse-temurin:8-jdk-focal RUN mkdir -p /usr/share/man/man1 - -RUN echo "deb http://archive.debian.org/debian stretch main contrib non-free" > /etc/apt/sources.list - -RUN apt-get update && apt-get upgrade -y && apt-get install -y netcat-openbsd zip git less \ - python2 curl maven -RUN cd /usr/bin && ln -s python2 python +RUN apt-get update && apt-get install -y netcat-openbsd zip git less \ + ca-certificates python3 curl maven gnupg +RUN cd /usr/bin && ln -s python3 python + +COPY cacerts/README.md cacerts/*.crt /usr/local/share/ca-certificates/ +RUN update-ca-certificates +RUN java_certs=$JAVA_HOME/jre/lib/security/cacerts; \ + add_certs=`ls /usr/local/share/ca-certificates/*.crt` && \ + for crt in $add_certs; do \ + name=`basename -s .crt $crt`; \ + echo -n ${name}: " "; \ + keytool -import -keystore $java_certs -trustcacerts -file $crt \ + -storepass changeit -alias $name -noprompt; \ + done; # Create the user that build/test operations should run as. Normally, # this is set to match identity information of the host user that is diff --git a/docker/build-test/cacerts/Forward_Proxy_NIST_CA.crt b/docker/build-test/cacerts/Forward_Proxy_NIST_CA.crt new file mode 100644 index 00000000..6c7a12cb --- /dev/null +++ b/docker/build-test/cacerts/Forward_Proxy_NIST_CA.crt @@ -0,0 +1,41 @@ +-----BEGIN CERTIFICATE----- +MIIG7TCCBNWgAwIBAgITGAAAAAecWWKCXTfeJAAAAAAABzANBgkqhkiG9w0BAQsF +ADAVMRMwEQYDVQQDEwpOSVNUUm9vdDAyMB4XDTIxMTIwMTE4MDU0NloXDTI2MTIw +MTE4MTU0NlowgcIxCzAJBgNVBAYTAlVTMREwDwYDVQQIEwhNYXJ5bGFuZDEVMBMG +A1UEBxMMR2FpdGhlcnNidXJnMTcwNQYDVQQKEy5OYXRpb25hbCBJbnN0aXR1dGUg +b2YgU3RhbmRhcmRzIGFuZCBUZWNobm9sb2d5MQ0wCwYDVQQLEwRPSVNNMR4wHAYD +VQQDDBVGb3J3YXJkX1Byb3h5X05JU1RfQ0ExITAfBgkqhkiG9w0BCQEWEm5ldHNl +Y3VyZUBuaXN0LmdvdjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAL/8 +PgucU6LfbThmVCiQU5zH7HRdJ0QeM8xa9Hy3BnBdD4/CxQklo7dz+AXquaOfI5Br +H8SYZCySWTveFeJW+XvhjmEVpobz8GGrEgdR5nAKg3ZJHvAMPKgGMSnXja227TVj +qqCZX9cIWQifqcM1iWTkS4BW2oZazwXYCqs5dfwy92ey5f/7AYC4dFeL//QtqQs/ +EUFApYabhKLcDLleDh4hwlhbTO9Zjt/eRujB/5f183RVb+igoy/xVZ8S82cNpxHS +2DdO58GZzvAgYMYuXXJkdINkag/fpCXEy9bGaDfydHLpTWviiGz3HfXh/Chb66BG +ZoZJmJrovVO9rSMyptMCAwEAAaOCAoYwggKCMB0GA1UdDgQWBBQquDqJ3U24XoOQ +/6y8kFgbAp9fPDAfBgNVHSMEGDAWgBQlEQPjYg4e56GOSdev1HJtWx0z+TCB/QYD +VR0fBIH1MIHyMIHvoIHsoIHphjFodHRwOi8vbmlzdHBraS5uaXN0Lmdvdi9DZXJ0 +RW5yb2xsL05JU1RSb290MDIuY3JshoGzbGRhcDovLy9DTj1OSVNUUm9vdDAyLENO +PU5JU1Ryb290Q0EwMixDTj1DRFAsQ049UHVibGljJTIwS2V5JTIwU2VydmljZXMs +Q049U2VydmljZXMsQ049Q29uZmlndXJhdGlvbixEQz1OSVNULERDPUdPVj9jZXJ0 +aWZpY2F0ZVJldm9jYXRpb25MaXN0P2Jhc2U/b2JqZWN0Q2xhc3M9Y1JMRGlzdHJp +YnV0aW9uUG9pbnQwggEFBggrBgEFBQcBAQSB+DCB9TBKBggrBgEFBQcwAoY+aHR0 +cDovL25pc3Rwa2kubmlzdC5nb3YvQ2VydEVucm9sbC9OSVNUcm9vdENBMDJfTklT +VFJvb3QwMi5jcnQwgaYGCCsGAQUFBzAChoGZbGRhcDovLy9DTj1OSVNUUm9vdDAy +LENOPUFJQSxDTj1QdWJsaWMlMjBLZXklMjBTZXJ2aWNlcyxDTj1TZXJ2aWNlcyxD +Tj1Db25maWd1cmF0aW9uLERDPU5JU1QsREM9R09WP2NBQ2VydGlmaWNhdGU/YmFz +ZT9vYmplY3RDbGFzcz1jZXJ0aWZpY2F0aW9uQXV0aG9yaXR5MBkGCSsGAQQBgjcU +AgQMHgoAUwB1AGIAQwBBMA8GA1UdEwEB/wQFMAMBAf8wCwYDVR0PBAQDAgGGMA0G +CSqGSIb3DQEBCwUAA4ICAQB3OCkcbjepVN7tbK3PlLzG5HkRBG1QSmFsRnQdUTov +/rWhdLDpHGKO4k/W2zTxNNxPW8ooD1PCy+cIlBLGcq8YcyhvWk0V2Gx1P+/f4+eq +eH4hcUQO/7INohcnh4QXiSVMa7jNaLC+/usqWbsmTvVDbl2aYbQtwizXnUW1qNhz +Bt76OoM7C95rNktNiaJ1VmFmd+Z3rRhzAZiC9XFIwIN1F+um7IG43nsoM4hnCByc +/SBb3LC8R+7vNUYedkrfNPq8SGCHuPuK8H0gJX+8/8hmaaNPtZoe0VZkTdNXitnY +HNof6w5mDoPu9lgLmNO0c36dNrmhHlPAu71EkL3afBhrdgb4Gel0WlENaur2MWf+ +yg6IQz7+aCTu2bMIkW3gm942tp7IrkXMGshUsJjLHFVrpIVkP+70QnO0wGzzQWlI +gt+/gKvj951KGagVzsFyiQtFL9uFYMiS0awLVkSLYtBzdykm8mpG1n6EO5DlEYWe +MOhVSeki05s0+6zUWU6TIhVDgCeUJYvAYAtWVA07Tbb1lb1vP+KbWzFMuAQMrKXV +I0sL/gjcwaj18n8vb0NdVU2n4qoW44gBi8ocgbuBntt63J4GHpaIn/I4OHBiwu/2 +IwUrfePEVCI2pAm/sBfw2XiofAclxBhhJniiRoMYPKCOdnPRP1nUWOdotzPJFPJe +1Q== +-----END CERTIFICATE----- + diff --git a/docker/build-test/cacerts/NISTRoot02.crt b/docker/build-test/cacerts/NISTRoot02.crt new file mode 100644 index 00000000..9aeb53c3 --- /dev/null +++ b/docker/build-test/cacerts/NISTRoot02.crt @@ -0,0 +1,29 @@ +-----BEGIN CERTIFICATE----- +MIIFBTCCAu2gAwIBAgIQdRxyg4+47KRFWKY+545EJjANBgkqhkiG9w0BAQsFADAV +MRMwEQYDVQQDEwpOSVNUUm9vdDAyMB4XDTE4MDgwMTE4MTgzOVoXDTM4MDgwMTE4 +Mjc1M1owFTETMBEGA1UEAxMKTklTVFJvb3QwMjCCAiIwDQYJKoZIhvcNAQEBBQAD +ggIPADCCAgoCggIBAJxQaJgDFbHCPwx8YOrjfthNQP7TOra9C4SkeURpetVq3fk1 +AqGgcqYzN3SRxtx9xJUweFBayO83jyBx5d+LLqX9LctaIrS4gU3uLGqDEQJisMST ++r6/mF51H5xF9AaiH8ca6ZopjigYdcv0ivMiUh8UWDvZF8SnPq4BaId4D3UwfVhV +p8Nh9osU04BXGSOIaN5dL4CdNiOleC7IqAl4wXekOMkNfIErp2QeLnq/g1xIFmCv +Dz+4umnPIVAYvuIKa39irNLi7j9XqUpnNcfBAvaypOe9e31RqWEYbHKhYXtFMJ6v +Ui/d+pPPJ0HfoMu2toCZHgMCxzaFnGh0reMkcCrPpH2EQIQzbJaV4QVRFvAfNIF5 +cwvb6mRJ9pqjlIVAoT+//YUy1IsG+4n0TZAEJa9G61G3bGr7Chh+uWYGfmpevY8I +GUTNmhYc5pGma6TFR3Hqil9PwAnPcXYQDnjhwVOGRrC/Ze9LymT7tUIEX0JKmZ0J +ds50u8T0joWwacwK1RYdj0YC4PLeLFB2obqcfust4KCN/Hw7/pvwN3sFhbC1dn2G +YIjqiDaenI7Gsb2t5Q8AOQbMSCJu0RYI9XN8Uzm+v0zseLF4V0+43PSTxDnlBzms +cpjRsMRk563nVnL4oHa+LhJnB/YTBqE86bzieTiIL7SqGW1hH+RJWn55pFtnAgMB +AAGjUTBPMAsGA1UdDwQEAwIBhjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBQl +EQPjYg4e56GOSdev1HJtWx0z+TAQBgkrBgEEAYI3FQEEAwIBADANBgkqhkiG9w0B +AQsFAAOCAgEAHscEbpIIPKe+avqPPxUJxRnnlV9CoBZSN4IJcA3Iox3f7zJdeLra +hMJq8vJBLK0barh9ofLbviX1tBzAqDFd6RnMaMWTfv2BgjtoZNqfFqRp632ErDTI +ONyHbGOnuWGXatwRNXUIhhx2UGeAy38xrIU8Z0ssTCsRY374WSFYaR5Ww7hfunyi +eBmofMY+j6flNxEqckV3BeIarJxWmpEaAihczZxJsnZXW+D0B7h4EKZ/DakOl2QA +59aE740ToPAl+pAF4OhT53xPlju+tqkaLnVJg/kI7Qrc0S2mHGrXnDl1FUya8VFS +Vm8bf3nd483e3nWnSVU+vItlRIrtoHnLQ7xzMkurUNo2pROR+JgsL5WL0+NDGFjv +Ixf9ReYGN9ujrHtojZiFaDLMPUftV6EVk2qc2d8BMEAnVzy8WJk6iqiWsmYaE2uq +wdQHiP8kwQhXXRbqhfFZWwSisga4TIZu65rR88ah08DOGaTLfqKUnb9WD4dzTDFH +XBl6ryuOeJGBoeJVbjy5938ZKHSS/nP3H/zYwve7xBw8CkmKAA1ECLJ47iWFmlyr +mQkr8lkaupRMxgV8LUml35hI4lT2SbvAbdsuP/RvuvrK+mHS2UEDjG/qz4aTuXrm +uMnUuya/1QGPhFD1oztxrhem2ob2jfkRfWT6wbv8UK7Mniw2zBfISXY= +-----END CERTIFICATE----- diff --git a/docker/build-test/cacerts/README.md b/docker/build-test/cacerts/README.md new file mode 100644 index 00000000..b650e217 --- /dev/null +++ b/docker/build-test/cacerts/README.md @@ -0,0 +1,13 @@ +This directory contains non-standard CA certificates needed to build the docker +images. + +Failures building the Docker containers defined in ../ due to SSL certificate +verification errors may be a consequence of your local network's firewall. In +particular, the firewall may be substituting external site certificates with +its own signed by a non-standard CA certficate (chain). If so, you can place +the necessary certificates into this directory; they will be passed into the +containers, allowing them to safely connect to those external sites. + +Be sure the certificates are in PEM format and include a .crt file extension. + +Do not remove this README file; doing so may cause a Docker build faiure. \ No newline at end of file diff --git a/docker/cacerts/README.md b/docker/cacerts/README.md new file mode 100644 index 00000000..b650e217 --- /dev/null +++ b/docker/cacerts/README.md @@ -0,0 +1,13 @@ +This directory contains non-standard CA certificates needed to build the docker +images. + +Failures building the Docker containers defined in ../ due to SSL certificate +verification errors may be a consequence of your local network's firewall. In +particular, the firewall may be substituting external site certificates with +its own signed by a non-standard CA certficate (chain). If so, you can place +the necessary certificates into this directory; they will be passed into the +containers, allowing them to safely connect to those external sites. + +Be sure the certificates are in PEM format and include a .crt file extension. + +Do not remove this README file; doing so may cause a Docker build faiure. \ No newline at end of file From 364cb8877ef47a4d1e44a8b1090b1b73e6668abb Mon Sep 17 00:00:00 2001 From: elmiomar Date: Wed, 6 Mar 2024 10:57:22 -0500 Subject: [PATCH 07/10] update unit tests --- ...URLConnectionRPARequestHandlerService.java | 4 + ...onnectionRPARequestHandlerServiceTest.java | 149 +++++++++++++++++- 2 files changed, 148 insertions(+), 5 deletions(-) diff --git a/src/main/java/gov/nist/oar/distrib/service/rpa/HttpURLConnectionRPARequestHandlerService.java b/src/main/java/gov/nist/oar/distrib/service/rpa/HttpURLConnectionRPARequestHandlerService.java index 6a391b9e..c75628d9 100644 --- a/src/main/java/gov/nist/oar/distrib/service/rpa/HttpURLConnectionRPARequestHandlerService.java +++ b/src/main/java/gov/nist/oar/distrib/service/rpa/HttpURLConnectionRPARequestHandlerService.java @@ -140,6 +140,10 @@ public void setRecordResponseHandler(RecordResponseHandler recordResponseHandler this.recordResponseHandler = recordResponseHandler; } + public void seRPADatasetCacher(RPADatasetCacher rpaDatasetCacher) { + this.rpaDatasetCacher = rpaDatasetCacher; + } + /** * Constructs a new instance of the service using the given RPA configuration. * diff --git a/src/test/java/gov/nist/oar/distrib/service/rpa/HttpURLConnectionRPARequestHandlerServiceTest.java b/src/test/java/gov/nist/oar/distrib/service/rpa/HttpURLConnectionRPARequestHandlerServiceTest.java index 3d15e46e..369a84b3 100644 --- a/src/test/java/gov/nist/oar/distrib/service/rpa/HttpURLConnectionRPARequestHandlerServiceTest.java +++ b/src/test/java/gov/nist/oar/distrib/service/rpa/HttpURLConnectionRPARequestHandlerServiceTest.java @@ -45,9 +45,13 @@ import java.util.Map; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; @@ -116,6 +120,9 @@ public class HttpURLConnectionRPARequestHandlerServiceTest { @Mock RPACachingService rpaCachingService; + @Mock + RPADatasetCacher rpaDatasetCacher; + private HttpURLConnectionRPARequestHandlerService service; JWTToken testToken = null; Map map = new HashMap() {{ @@ -132,6 +139,7 @@ public void setUp() { service.setRecaptchaHelper(recaptchaHelper); service.setHttpURLConnectionFactory(url -> mockConnection); service.setRecordResponseHandler(recordResponseHandler); + service.seRPADatasetCacher(rpaDatasetCacher); service.setHttpClient(mockHttpClient); // Set up mock behavior for mockJwtHelper @@ -491,21 +499,45 @@ private String getUpdateUrl(String recordId) { return url; } + /** + * Tests successful record update operation when approving a record. + * + *

+ * This test simulates the scenario where a record is approved, ensuring that the + * caching process is invoked, a PATCH request is made to update the record status in + * Salesforce with a newly generated random ID, and the approval status is updated correctly. + * The test verifies that the final approval status matches the expected format. + * 

+     * Status_YYYY-MM-DDTHH:MM:SS.SSSZ_email_randomID
+     *
+ * Where: + *
    + *
  • Status - Indicates the status of the record (e.g., "Approved" or "Declined").
    • + *
  • YYYY-MM-DDTHH:MM:SS.SSSZ - The timestamp of the approval in ISO 8601 format. 'T' separates the date and time, and 'Z' denotes UTC time zone.
    • + *
  • email - The email address associated with the user who approved the record.
    • + *
  • randomID - A unique identifier generated for the approval process or the record itself.
    • + *
+ *

+ * + * @throws Exception if any error occurs during the test execution. + */ @Test public void testUpdateRecord_success() throws Exception { String recordId = "record12345"; String email = "test@example.com"; - String expectedApprovalStatus = "Approved_2023-05-09T15:59:03.872Z_" + email; + String mockRandomId = "mockRandomId123"; + String expectedApprovalStatus = "Approved_2023-05-09T15:59:03.872Z_" + email + "_" + mockRandomId; // Mock behavior of getRecord method doReturn(getTestRecordWrapper(expectedApprovalStatus)).when(service).getRecord("record12345"); + when(rpaDatasetCacher.cache(anyString())).thenReturn(mockRandomId); // Mock HttpResponse CloseableHttpResponse httpResponse = mock(CloseableHttpResponse.class); when(httpResponse.getStatusLine()).thenReturn(mock(StatusLine.class)); when(httpResponse.getStatusLine().getStatusCode()).thenReturn(200); when(httpResponse.getEntity()).thenReturn( - new StringEntity("{\"approvalStatus\":\"Approved_2023-05-09T15:59:03.872Z_test@example.com\"," + + new StringEntity("{\"approvalStatus\":\"Approved_2023-05-09T15:59:03.872Z_test@example.com_mockRandomId123\"," + "\"recordId\":\"record12345\"}", ContentType.APPLICATION_JSON) ); @@ -531,13 +563,24 @@ public void testUpdateRecord_success() throws Exception { // We can't test the exact time as it changes when we run the test, but we can verify the format String patchPayload = EntityUtils.toString(patchRequest.getEntity(), StandardCharsets.UTF_8); JSONObject payloadObject = new JSONObject(patchPayload); - // Pattern to match ISO 8601 format - // This pattern matches a string in the format "Approved_YYYY-MM-DDTHH:MM:SS.SSSZ" - String expectedFormat = "Approved_\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z_[\\w.-]+@[\\w.-]+\\.\\w+"; + // The following regex pattern expects: + // - The "Approved" status followed by a date-time in ISO 8601 format. + // - An email address. + // - A random ID (composed of word characters including underscore, alphanumeric, and possibly -) at the end. + String expectedFormat = "Approved_\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z_[\\w.-]+@[\\w.-]+\\.\\w+_\\w+"; assertTrue(payloadObject.get("Approval_Status__c").toString().matches(expectedFormat)); } + /** + * Tests the updateRecord method's behavior when an unknown status is provided. + *

+ * This test ensures that the method throws an InvalidRequestException when attempting to + * update a record with an unrecognized status. The expected behavior is to validate the + * status input and throw an exception with a specific error message if the status does not + * match expected values (e.g., "Approved" or "Declined"). + *

+ */ @Test public void testUpdateRecord_withUnknownStatus() { String recordId = "record12345"; @@ -545,6 +588,9 @@ public void testUpdateRecord_withUnknownStatus() { String email = "test@example.com"; String expectedErrorMessage = "Invalid approval status: HelloWorld"; + // Mock behavior of getRecord method + doReturn(getTestRecordWrapper("Pending")).when(service).getRecord("record12345"); + // Call the method and catch the exception try { // Act @@ -556,5 +602,98 @@ public void testUpdateRecord_withUnknownStatus() { } } + /** + * Tests the record decline operation for a record that has not been previously approved. + * + *

+ * This test case ensures that when a record is declined without prior approval, the record's + * status is updated accordingly without triggering caching or uncaching operations. It verifies + * the successful update of the record's approval status to "Declined" and confirms that no + * caching or uncaching methods are called, as expected for records not previously approved. + *

+ * + * @throws Exception if any error occurs during the test execution. + */ + @Test + public void testDeclineRecordWithoutPriorApproval_success() throws Exception { + String recordId = "record12345"; + String email = "sme@test.com"; + String status = "Declined"; + + // Mock behavior of getRecord method to simulate a record that has not been approved before + doReturn(getTestRecordWrapper("Pending")).when(service).getRecord(recordId); + + // Mock HttpResponse for the decline operation + CloseableHttpResponse httpResponse = mock(CloseableHttpResponse.class); + when(httpResponse.getStatusLine()).thenReturn(mock(StatusLine.class)); + when(httpResponse.getStatusLine().getStatusCode()).thenReturn(200); + when(httpResponse.getEntity()).thenReturn( + new StringEntity("{\"approvalStatus\":\"Declined\",\"recordId\":\"" + recordId + "\"}", + ContentType.APPLICATION_JSON)); + + // Mock the HttpPatch execution + doReturn(httpResponse).when(mockHttpClient).execute(any(HttpPatch.class)); + + // Act + RecordStatus result = service.updateRecord(recordId, status, email); + + // Assert + assertEquals("Declined", result.getApprovalStatus()); + assertEquals(recordId, result.getRecordId()); + + // Verify that caching and uncaching were not invoked + verify(rpaDatasetCacher, never()).cache(anyString()); + verify(rpaDatasetCacher, never()).uncache(anyString()); + } + + /** + * Tests the decline operation for a record that was previously approved. + * + *

+ * This test checks the behavior of the updateRecord method when declining a record that + * has a prior approval status, including a random ID. It simulates retrieving a previously + * approved record, uncaching the dataset associated with the random ID, and updating the + * record's approval status to "Declined". The test verifies that the uncaching operation + * is executed with the correct random ID and that the record's status is correctly updated. + *

+ * + * @throws Exception if any error occurs during the test execution. + */ + @Test + public void testDeclinePreviouslyApprovedRecord_success() throws Exception { + String recordId = "record12345"; + String email = "sme@test.com"; + String status = "Declined"; + String mockRandomId = "mockRandomId123"; + String initialApprovalStatus = "Approved_2023-05-09T15:59:03.872Z_" + email + "_" + mockRandomId; + + // Mock behavior of getRecord method to simulate retrieving a previously approved record + doReturn(getTestRecordWrapper(initialApprovalStatus)).when(service).getRecord(recordId); + + // Simulate `uncache` returning true + when(rpaDatasetCacher.uncache(anyString())).thenReturn(true); + + // Mock HttpResponse for updating the record to "Declined" + CloseableHttpResponse httpResponse = mock(CloseableHttpResponse.class); + when(httpResponse.getStatusLine()).thenReturn(mock(StatusLine.class)); + when(httpResponse.getStatusLine().getStatusCode()).thenReturn(200); + when(httpResponse.getEntity()).thenReturn( + new StringEntity("{\"approvalStatus\":\"Declined\",\"recordId\":\"" + recordId + "\"}", + ContentType.APPLICATION_JSON)); + + // Mock the HttpPatch execution + doReturn(httpResponse).when(mockHttpClient).execute(any(HttpPatch.class)); + + // Act + RecordStatus result = service.updateRecord(recordId, status, email); + + // Assert + assertEquals("Declined", result.getApprovalStatus()); + assertEquals(recordId, result.getRecordId()); + + // Verify uncaching was invoked with the correct random ID + verify(rpaDatasetCacher).uncache(mockRandomId); + } + } From 03cdf70a0d56bc4d10f93ca08150576936c412a7 Mon Sep 17 00:00:00 2001 From: Omar Ilias EL MIMOUNI Date: Wed, 6 Mar 2024 11:06:41 -0500 Subject: [PATCH 08/10] Update Dockerfile - resolve merge conflicts --- docker/build-test/Dockerfile | 3 --- 1 file changed, 3 deletions(-) diff --git a/docker/build-test/Dockerfile b/docker/build-test/Dockerfile index c6ac3cc1..fba91c27 100644 --- a/docker/build-test/Dockerfile +++ b/docker/build-test/Dockerfile @@ -1,4 +1,3 @@ -<<<<<<< HEAD FROM eclipse-temurin:8-jdk-focal RUN mkdir -p /usr/share/man/man1 @@ -16,14 +15,12 @@ RUN java_certs=$JAVA_HOME/jre/lib/security/cacerts; \ keytool -import -keystore $java_certs -trustcacerts -file $crt \ -storepass changeit -alias $name -noprompt; \ done; -======= FROM eclipse-temurin:8 RUN mkdir -p /usr/share/man/man1 RUN apt-get update && apt-get install -y netcat-openbsd zip git less \ python2 curl maven RUN cd /usr/bin && ln -s python2 python ->>>>>>> origin/integration # Create the user that build/test operations should run as. Normally, # this is set to match identity information of the host user that is From 980b5fdbe56a48f2f677a3387ea8a802ec97ff78 Mon Sep 17 00:00:00 2001 From: elmiomar Date: Wed, 6 Mar 2024 11:25:55 -0500 Subject: [PATCH 09/10] fix unit tests --- .../service/RPACachingServiceTest.java | 29 +++++++++++-------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/src/test/java/gov/nist/oar/distrib/service/RPACachingServiceTest.java b/src/test/java/gov/nist/oar/distrib/service/RPACachingServiceTest.java index c3d0f108..52829497 100644 --- a/src/test/java/gov/nist/oar/distrib/service/RPACachingServiceTest.java +++ b/src/test/java/gov/nist/oar/distrib/service/RPACachingServiceTest.java @@ -52,8 +52,8 @@ public void testCacheAndGenerateRandomId_validDatasetID() throws Exception { String result = rpaCachingService.cacheAndGenerateRandomId(datasetID, version); assertNotNull(result); - assertEquals(RPACachingService.RANDOM_ID_LENGTH, result.length()); - assertTrue(result.matches("^[a-zA-Z0-9]*$")); // check that the ID is alphanumeric + assertEquals(RPACachingService.RANDOM_ID_LENGTH + 4, result.length()); // 4 for the 'rpa-' prefix + assertTrue(result.matches("^rpa-[a-zA-Z0-9]+$")); // Check that the ID starts with 'rpa-' followed by alphanumeric chars verify(pdrCacheManager).cacheDataset(eq("mds2-2909"), eq(version), eq(true), eq(RPACachingService.ROLE_RESTRICTED_DATA), eq(result)); } @@ -69,8 +69,8 @@ public void testCacheAndGenerateRandomId_validDatasetArkID() throws Exception { String result = rpaCachingService.cacheAndGenerateRandomId(datasetID, version); assertNotNull(result); - assertEquals(RPACachingService.RANDOM_ID_LENGTH, result.length()); - assertTrue(result.matches("^[a-zA-Z0-9]*$")); // check that the ID is alphanumeric + assertEquals(RPACachingService.RANDOM_ID_LENGTH + 4, result.length()); // 4 for the 'rpa-' prefix + assertTrue(result.matches("^rpa-[a-zA-Z0-9]+$")); // Check that the ID starts with 'rpa-' followed by alphanumeric chars verify(pdrCacheManager).cacheDataset(eq("mds2-2909"), eq(version), eq(true), eq(RPACachingService.ROLE_RESTRICTED_DATA), eq(result)); } @@ -85,6 +85,7 @@ public void testCacheAndGenerateRandomId_invalidDatasetArkID() throws Exception @Test public void testRetrieveMetadata_success() throws Exception { String randomID = "randomId123"; + String aipid = "456"; CacheObject cacheObject1 = new CacheObject("object1", new JSONObject() .put("filepath", "path/to/file1.txt") .put("contentType", "text/plain") @@ -95,7 +96,7 @@ public void testRetrieveMetadata_success() throws Exception { .put("checksum", "abc123") .put("version", "v1") .put("ediid", "123") - .put("aipid", "456") + .put("aipid", aipid) .put("sinceDate", "08-05-2023"), "Volume1"); @@ -109,7 +110,7 @@ public void testRetrieveMetadata_success() throws Exception { .put("checksum", "def456") .put("version", "v2") .put("ediid", "123") - .put("aipid", "456") + .put("aipid", aipid) .put("sinceDate", "08-05-2023"), "Volume1"); @@ -124,7 +125,8 @@ public void testRetrieveMetadata_success() throws Exception { Map expected = new HashMap<>(); expected.put("randomId", randomID); expected.put("metadata", new JSONArray() - .put(new JSONObject().put("downloadURL", testBaseDownloadUrl + "/" + randomID + "/path/to/file1.txt") + .put(new JSONObject().put("downloadURL", testBaseDownloadUrl + "/" + randomID + + "/" + aipid + "/path/to/file1.txt") .put("filePath", "path/to/file1.txt") .put("mediaType", "text/plain") .put("size", 100L) @@ -136,7 +138,8 @@ public void testRetrieveMetadata_success() throws Exception { .put("ediid", "123") .put("aipid", "456") .put("sinceDate", "08-05-2023")) - .put(new JSONObject().put("downloadURL", testBaseDownloadUrl + "/" + randomID + "/path/to/file2.txt") + .put(new JSONObject().put("downloadURL", testBaseDownloadUrl + "/" + randomID + + "/" + aipid + "/path/to/file2.txt") .put("filePath", "path/to/file2.txt") .put("mediaType", "text/plain") .put("size", 100L) @@ -159,6 +162,7 @@ public void testRetrieveMetadata_success() throws Exception { @Test public void testRetrieveMetadata_withMissingFilepath() throws Exception { String randomID = "randomId123"; + String aipid = "456"; CacheObject cacheObject1 = new CacheObject("object1", new JSONObject() .put("contentType", "text/plain") .put("size", 100L) @@ -168,7 +172,7 @@ public void testRetrieveMetadata_withMissingFilepath() throws Exception { .put("checksum", "abc123") .put("version", "v1") .put("ediid", "123") - .put("aipid", "456") + .put("aipid", aipid) .put("sinceDate", "08-05-2023"), "Volume1"); @@ -182,7 +186,7 @@ public void testRetrieveMetadata_withMissingFilepath() throws Exception { .put("checksum", "def456") .put("version", "v2") .put("ediid", "123") - .put("aipid", "456") + .put("aipid", aipid) .put("sinceDate", "08-05-2023"), "Volume1"); @@ -197,7 +201,8 @@ public void testRetrieveMetadata_withMissingFilepath() throws Exception { Map expected = new HashMap<>(); expected.put("randomId", randomID); expected.put("metadata", new JSONArray() - .put(new JSONObject().put("downloadURL", testBaseDownloadUrl + "/" + randomID + "/path/to/file2.txt") + .put(new JSONObject().put("downloadURL", testBaseDownloadUrl + "/" + randomID + + "/" + aipid +"/path/to/file2.txt") .put("filePath", "path/to/file2.txt") .put("mediaType", "text/plain") .put("size", 100L) @@ -207,7 +212,7 @@ public void testRetrieveMetadata_withMissingFilepath() throws Exception { .put("checksum", "def456") .put("version", "v2") .put("ediid", "123") - .put("aipid", "456") + .put("aipid", aipid) .put("sinceDate", "08-05-2023")) .toList()); From 8cc659f97642ff792d8b5835a80f40d5d205b65e Mon Sep 17 00:00:00 2001 From: elmiomar Date: Fri, 8 Mar 2024 15:44:34 -0500 Subject: [PATCH 10/10] remove redundant uncaching endpoint --- .../distrib/web/RPADataCachingController.java | 28 ------------------- 1 file changed, 28 deletions(-) diff --git a/src/main/java/gov/nist/oar/distrib/web/RPADataCachingController.java b/src/main/java/gov/nist/oar/distrib/web/RPADataCachingController.java index 4d2520be..e9418f2b 100644 --- a/src/main/java/gov/nist/oar/distrib/web/RPADataCachingController.java +++ b/src/main/java/gov/nist/oar/distrib/web/RPADataCachingController.java @@ -149,34 +149,6 @@ public Map retrieveMetadata(@PathVariable("cacheid") String cach return metadata; } - /** - * This endpoint handles the removal of a dataset from the cache based on its randomly generated ID. - * - * @param randomId The randomly generated ID that was assigned when the dataset was cached. - * - * @return A ResponseEntity containing a status message and HTTP status code. - * - * @throws CacheManagementException If there is an issue with removing the dataset from the cache. - * @throws ResourceNotFoundException If the dataset associated with the randomId is not found in the cache. - * @throws StorageVolumeException If there is an issue with the storage volume. - */ - @PutMapping(value = "/uncache/{randomId}") - public ResponseEntity uncacheDataset( - @PathVariable("randomId") String randomId) - throws CacheManagementException { - - logger.debug("randomId to uncache=" + randomId); - - // Perform uncache operation via the service - boolean success = restrictedSrvc.uncacheById(randomId); - - if (success) { - return new ResponseEntity<>("Dataset was successfully removed from cache.", HttpStatus.OK); - } else { - return new ResponseEntity<>("Failed to uncache the dataset.", HttpStatus.INTERNAL_SERVER_ERROR); - } - } - @ExceptionHandler(MetadataNotFoundException.class) @ResponseStatus(HttpStatus.NOT_FOUND)