diff --git a/docker/build-test/Dockerfile b/docker/build-test/Dockerfile index 9f86a5d5..fba91c27 100644 --- a/docker/build-test/Dockerfile +++ b/docker/build-test/Dockerfile @@ -1,3 +1,20 @@ +FROM eclipse-temurin:8-jdk-focal + +RUN mkdir -p /usr/share/man/man1 +RUN apt-get update && apt-get install -y netcat-openbsd zip git less \ + ca-certificates python3 curl maven gnupg +RUN cd /usr/bin && ln -s python3 python + +COPY cacerts/README.md cacerts/*.crt /usr/local/share/ca-certificates/ +RUN update-ca-certificates +RUN java_certs=$JAVA_HOME/jre/lib/security/cacerts; \ + add_certs=`ls /usr/local/share/ca-certificates/*.crt` && \ + for crt in $add_certs; do \ + name=`basename -s .crt $crt`; \ + echo -n ${name}: " "; \ + keytool -import -keystore $java_certs -trustcacerts -file $crt \ + -storepass changeit -alias $name -noprompt; \ + done; FROM eclipse-temurin:8 RUN mkdir -p /usr/share/man/man1 diff --git a/docker/build-test/cacerts/Forward_Proxy_NIST_CA.crt b/docker/build-test/cacerts/Forward_Proxy_NIST_CA.crt new file mode 100644 index 00000000..6c7a12cb --- /dev/null +++ b/docker/build-test/cacerts/Forward_Proxy_NIST_CA.crt @@ -0,0 +1,41 @@ +-----BEGIN CERTIFICATE----- +MIIG7TCCBNWgAwIBAgITGAAAAAecWWKCXTfeJAAAAAAABzANBgkqhkiG9w0BAQsF +ADAVMRMwEQYDVQQDEwpOSVNUUm9vdDAyMB4XDTIxMTIwMTE4MDU0NloXDTI2MTIw +MTE4MTU0NlowgcIxCzAJBgNVBAYTAlVTMREwDwYDVQQIEwhNYXJ5bGFuZDEVMBMG +A1UEBxMMR2FpdGhlcnNidXJnMTcwNQYDVQQKEy5OYXRpb25hbCBJbnN0aXR1dGUg +b2YgU3RhbmRhcmRzIGFuZCBUZWNobm9sb2d5MQ0wCwYDVQQLEwRPSVNNMR4wHAYD +VQQDDBVGb3J3YXJkX1Byb3h5X05JU1RfQ0ExITAfBgkqhkiG9w0BCQEWEm5ldHNl +Y3VyZUBuaXN0LmdvdjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAL/8 +PgucU6LfbThmVCiQU5zH7HRdJ0QeM8xa9Hy3BnBdD4/CxQklo7dz+AXquaOfI5Br +H8SYZCySWTveFeJW+XvhjmEVpobz8GGrEgdR5nAKg3ZJHvAMPKgGMSnXja227TVj +qqCZX9cIWQifqcM1iWTkS4BW2oZazwXYCqs5dfwy92ey5f/7AYC4dFeL//QtqQs/ +EUFApYabhKLcDLleDh4hwlhbTO9Zjt/eRujB/5f183RVb+igoy/xVZ8S82cNpxHS +2DdO58GZzvAgYMYuXXJkdINkag/fpCXEy9bGaDfydHLpTWviiGz3HfXh/Chb66BG +ZoZJmJrovVO9rSMyptMCAwEAAaOCAoYwggKCMB0GA1UdDgQWBBQquDqJ3U24XoOQ +/6y8kFgbAp9fPDAfBgNVHSMEGDAWgBQlEQPjYg4e56GOSdev1HJtWx0z+TCB/QYD +VR0fBIH1MIHyMIHvoIHsoIHphjFodHRwOi8vbmlzdHBraS5uaXN0Lmdvdi9DZXJ0 +RW5yb2xsL05JU1RSb290MDIuY3JshoGzbGRhcDovLy9DTj1OSVNUUm9vdDAyLENO +PU5JU1Ryb290Q0EwMixDTj1DRFAsQ049UHVibGljJTIwS2V5JTIwU2VydmljZXMs +Q049U2VydmljZXMsQ049Q29uZmlndXJhdGlvbixEQz1OSVNULERDPUdPVj9jZXJ0 +aWZpY2F0ZVJldm9jYXRpb25MaXN0P2Jhc2U/b2JqZWN0Q2xhc3M9Y1JMRGlzdHJp +YnV0aW9uUG9pbnQwggEFBggrBgEFBQcBAQSB+DCB9TBKBggrBgEFBQcwAoY+aHR0 +cDovL25pc3Rwa2kubmlzdC5nb3YvQ2VydEVucm9sbC9OSVNUcm9vdENBMDJfTklT +VFJvb3QwMi5jcnQwgaYGCCsGAQUFBzAChoGZbGRhcDovLy9DTj1OSVNUUm9vdDAy +LENOPUFJQSxDTj1QdWJsaWMlMjBLZXklMjBTZXJ2aWNlcyxDTj1TZXJ2aWNlcyxD +Tj1Db25maWd1cmF0aW9uLERDPU5JU1QsREM9R09WP2NBQ2VydGlmaWNhdGU/YmFz +ZT9vYmplY3RDbGFzcz1jZXJ0aWZpY2F0aW9uQXV0aG9yaXR5MBkGCSsGAQQBgjcU +AgQMHgoAUwB1AGIAQwBBMA8GA1UdEwEB/wQFMAMBAf8wCwYDVR0PBAQDAgGGMA0G +CSqGSIb3DQEBCwUAA4ICAQB3OCkcbjepVN7tbK3PlLzG5HkRBG1QSmFsRnQdUTov +/rWhdLDpHGKO4k/W2zTxNNxPW8ooD1PCy+cIlBLGcq8YcyhvWk0V2Gx1P+/f4+eq +eH4hcUQO/7INohcnh4QXiSVMa7jNaLC+/usqWbsmTvVDbl2aYbQtwizXnUW1qNhz +Bt76OoM7C95rNktNiaJ1VmFmd+Z3rRhzAZiC9XFIwIN1F+um7IG43nsoM4hnCByc +/SBb3LC8R+7vNUYedkrfNPq8SGCHuPuK8H0gJX+8/8hmaaNPtZoe0VZkTdNXitnY +HNof6w5mDoPu9lgLmNO0c36dNrmhHlPAu71EkL3afBhrdgb4Gel0WlENaur2MWf+ +yg6IQz7+aCTu2bMIkW3gm942tp7IrkXMGshUsJjLHFVrpIVkP+70QnO0wGzzQWlI +gt+/gKvj951KGagVzsFyiQtFL9uFYMiS0awLVkSLYtBzdykm8mpG1n6EO5DlEYWe +MOhVSeki05s0+6zUWU6TIhVDgCeUJYvAYAtWVA07Tbb1lb1vP+KbWzFMuAQMrKXV +I0sL/gjcwaj18n8vb0NdVU2n4qoW44gBi8ocgbuBntt63J4GHpaIn/I4OHBiwu/2 +IwUrfePEVCI2pAm/sBfw2XiofAclxBhhJniiRoMYPKCOdnPRP1nUWOdotzPJFPJe +1Q== +-----END CERTIFICATE----- + diff --git a/docker/build-test/cacerts/NISTRoot02.crt b/docker/build-test/cacerts/NISTRoot02.crt new file mode 100644 index 00000000..9aeb53c3 --- /dev/null +++ b/docker/build-test/cacerts/NISTRoot02.crt @@ -0,0 +1,29 @@ +-----BEGIN CERTIFICATE----- +MIIFBTCCAu2gAwIBAgIQdRxyg4+47KRFWKY+545EJjANBgkqhkiG9w0BAQsFADAV +MRMwEQYDVQQDEwpOSVNUUm9vdDAyMB4XDTE4MDgwMTE4MTgzOVoXDTM4MDgwMTE4 +Mjc1M1owFTETMBEGA1UEAxMKTklTVFJvb3QwMjCCAiIwDQYJKoZIhvcNAQEBBQAD +ggIPADCCAgoCggIBAJxQaJgDFbHCPwx8YOrjfthNQP7TOra9C4SkeURpetVq3fk1 +AqGgcqYzN3SRxtx9xJUweFBayO83jyBx5d+LLqX9LctaIrS4gU3uLGqDEQJisMST ++r6/mF51H5xF9AaiH8ca6ZopjigYdcv0ivMiUh8UWDvZF8SnPq4BaId4D3UwfVhV +p8Nh9osU04BXGSOIaN5dL4CdNiOleC7IqAl4wXekOMkNfIErp2QeLnq/g1xIFmCv +Dz+4umnPIVAYvuIKa39irNLi7j9XqUpnNcfBAvaypOe9e31RqWEYbHKhYXtFMJ6v +Ui/d+pPPJ0HfoMu2toCZHgMCxzaFnGh0reMkcCrPpH2EQIQzbJaV4QVRFvAfNIF5 +cwvb6mRJ9pqjlIVAoT+//YUy1IsG+4n0TZAEJa9G61G3bGr7Chh+uWYGfmpevY8I +GUTNmhYc5pGma6TFR3Hqil9PwAnPcXYQDnjhwVOGRrC/Ze9LymT7tUIEX0JKmZ0J +ds50u8T0joWwacwK1RYdj0YC4PLeLFB2obqcfust4KCN/Hw7/pvwN3sFhbC1dn2G +YIjqiDaenI7Gsb2t5Q8AOQbMSCJu0RYI9XN8Uzm+v0zseLF4V0+43PSTxDnlBzms +cpjRsMRk563nVnL4oHa+LhJnB/YTBqE86bzieTiIL7SqGW1hH+RJWn55pFtnAgMB +AAGjUTBPMAsGA1UdDwQEAwIBhjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBQl +EQPjYg4e56GOSdev1HJtWx0z+TAQBgkrBgEEAYI3FQEEAwIBADANBgkqhkiG9w0B +AQsFAAOCAgEAHscEbpIIPKe+avqPPxUJxRnnlV9CoBZSN4IJcA3Iox3f7zJdeLra +hMJq8vJBLK0barh9ofLbviX1tBzAqDFd6RnMaMWTfv2BgjtoZNqfFqRp632ErDTI +ONyHbGOnuWGXatwRNXUIhhx2UGeAy38xrIU8Z0ssTCsRY374WSFYaR5Ww7hfunyi +eBmofMY+j6flNxEqckV3BeIarJxWmpEaAihczZxJsnZXW+D0B7h4EKZ/DakOl2QA +59aE740ToPAl+pAF4OhT53xPlju+tqkaLnVJg/kI7Qrc0S2mHGrXnDl1FUya8VFS +Vm8bf3nd483e3nWnSVU+vItlRIrtoHnLQ7xzMkurUNo2pROR+JgsL5WL0+NDGFjv +Ixf9ReYGN9ujrHtojZiFaDLMPUftV6EVk2qc2d8BMEAnVzy8WJk6iqiWsmYaE2uq +wdQHiP8kwQhXXRbqhfFZWwSisga4TIZu65rR88ah08DOGaTLfqKUnb9WD4dzTDFH +XBl6ryuOeJGBoeJVbjy5938ZKHSS/nP3H/zYwve7xBw8CkmKAA1ECLJ47iWFmlyr +mQkr8lkaupRMxgV8LUml35hI4lT2SbvAbdsuP/RvuvrK+mHS2UEDjG/qz4aTuXrm +uMnUuya/1QGPhFD1oztxrhem2ob2jfkRfWT6wbv8UK7Mniw2zBfISXY= +-----END CERTIFICATE----- diff --git a/docker/build-test/cacerts/README.md b/docker/build-test/cacerts/README.md new file mode 100644 index 00000000..b650e217 --- /dev/null +++ b/docker/build-test/cacerts/README.md @@ -0,0 +1,13 @@ +This directory contains non-standard CA certificates needed to build the docker +images. + +Failures building the Docker containers defined in ../ due to SSL certificate +verification errors may be a consequence of your local network's firewall. In +particular, the firewall may be substituting external site certificates with +its own signed by a non-standard CA certficate (chain). If so, you can place +the necessary certificates into this directory; they will be passed into the +containers, allowing them to safely connect to those external sites. + +Be sure the certificates are in PEM format and include a .crt file extension. + +Do not remove this README file; doing so may cause a Docker build faiure. \ No newline at end of file diff --git a/docker/cacerts/README.md b/docker/cacerts/README.md new file mode 100644 index 00000000..b650e217 --- /dev/null +++ b/docker/cacerts/README.md @@ -0,0 +1,13 @@ +This directory contains non-standard CA certificates needed to build the docker +images. + +Failures building the Docker containers defined in ../ due to SSL certificate +verification errors may be a consequence of your local network's firewall. In +particular, the firewall may be substituting external site certificates with +its own signed by a non-standard CA certficate (chain). If so, you can place +the necessary certificates into this directory; they will be passed into the +containers, allowing them to safely connect to those external sites. + +Be sure the certificates are in PEM format and include a .crt file extension. + +Do not remove this README file; doing so may cause a Docker build faiure. \ No newline at end of file diff --git a/src/main/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheck.java b/src/main/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheck.java new file mode 100644 index 00000000..2a8f718a --- /dev/null +++ b/src/main/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheck.java @@ -0,0 +1,82 @@ +package gov.nist.oar.distrib.cachemgr; + +import gov.nist.oar.distrib.StorageVolumeException; + +import java.time.Instant; + +/** + * Implements a cache object check to identify and remove objects that have been in the cache + * longer than a specified duration, specifically two weeks. This check helps in + * managing cache integrity by ensuring that stale or outdated data are removed + * from the cache. + */ +public class CacheExpiryCheck implements CacheObjectCheck { + + private StorageInventoryDB inventoryDB; + + public CacheExpiryCheck(StorageInventoryDB inventoryDB) { + this.inventoryDB = inventoryDB; + } + + /** + * Checks if a cache object is expired and removes it from the cache if it is. + * The method uses the {@code expires} metadata field to determine the expiration status. + * The expiration time is calculated based on the {@code LastModified} time plus the {@code expires} duration. + * If the current time is past the calculated expiry time, the object is removed from the inventory database. + * + * @param co The cache object to check for expiration. + * @throws IntegrityException If the object is found to be corrupted during the check. + * @throws StorageVolumeException If there's an error accessing the storage volume during the check. + * @throws CacheManagementException If there's an error managing the cache, including removing the expired object. + */ + @Override + public void check(CacheObject co) throws IntegrityException, StorageVolumeException, CacheManagementException { + if (co == null || inventoryDB == null) { + throw new IllegalArgumentException("CacheObject or StorageInventoryDB is null"); + } + + if (co.hasMetadatum("expires")) { + long expiresDuration = co.getMetadatumLong("expires", -1L); + if (expiresDuration == -1L) { + throw new IntegrityException("Invalid 'expires' metadata value"); + } + + long lastModified = co.getLastModified(); + if (lastModified == -1L) { + throw new IntegrityException("CacheObject 'lastModified' time not available"); + } + + long expiryTime = lastModified + expiresDuration; + long currentTime = Instant.now().toEpochMilli(); + + // Check if the object is expired + if (expiryTime < currentTime) { + try { + boolean removed = removeObject(co); + if (!removed) { + throw new CacheManagementException("Failed to remove expired object: " + co.name); + } + } catch (InventoryException e) { + throw new CacheManagementException("Error removing expired object from inventory database: " + co.name, e); + } + } + } + } + + /** + * Attempts to remove a cache object from both its physical volume and the inventory database. + * Synchronization ensures thread-safe removal operations. + * + * @param co The cache object to be removed. + * @return true if the object was successfully removed from its volume, false otherwise. + * @throws StorageVolumeException if an error occurs accessing the storage volume. + * @throws InventoryException if an error occurs updating the inventory database. + */ + protected boolean removeObject(CacheObject co) throws StorageVolumeException, InventoryException { + synchronized (inventoryDB) { + boolean out = co.volume.remove(co.name); + inventoryDB.removeObject(co.volname, co.name); + return out; + } + } +} diff --git a/src/main/java/gov/nist/oar/distrib/cachemgr/pdr/PDRDatasetRestorer.java b/src/main/java/gov/nist/oar/distrib/cachemgr/pdr/PDRDatasetRestorer.java index 7aebf76d..aa7ba637 100644 --- a/src/main/java/gov/nist/oar/distrib/cachemgr/pdr/PDRDatasetRestorer.java +++ b/src/main/java/gov/nist/oar/distrib/cachemgr/pdr/PDRDatasetRestorer.java @@ -650,6 +650,9 @@ protected void cacheFromBagUsingStore(String bagfile, Collection need, C md.put("ediid", resmd.get("ediid")); md.put("cachePrefs", prefs); + // a hook for handling the expiration logic + updateMetadata(md, prefs); + // find space in the cache, and copy the data file into it try { resv = into.reserveSpace(ze.getSize(), prefs); @@ -687,6 +690,18 @@ protected void cacheFromBagUsingStore(String bagfile, Collection need, C fixMissingChecksums(into, fix, manifest); } + /** + * Method intended for customization of metadata before caching. This method can be overridden + * by subclasses to implement specific metadata customization logic as needed. + * + * @param md The metadata JSONObject to be customized. + * @param prefs flags for data roles + */ + protected void updateMetadata(JSONObject md, int prefs) { + // Default implementation does nothing. + // Subclasses can override this to implement specific logic. + } + /** * helper method to generate an ID for the object to be cached */ @@ -694,7 +709,7 @@ public String idForObject(String aipid, String filepath, String forVersion, Stri String id; id = aipid + "/" + filepath; if (target != null && !target.isEmpty()) - id = target + "/" + filepath; + id = target + "/" + aipid + "/" + filepath; if (forVersion != null && forVersion.length() > 0) id += "#" + forVersion; return id; diff --git a/src/main/java/gov/nist/oar/distrib/cachemgr/pdr/RestrictedDatasetRestorer.java b/src/main/java/gov/nist/oar/distrib/cachemgr/pdr/RestrictedDatasetRestorer.java index da1255c0..54bec7ab 100644 --- a/src/main/java/gov/nist/oar/distrib/cachemgr/pdr/RestrictedDatasetRestorer.java +++ b/src/main/java/gov/nist/oar/distrib/cachemgr/pdr/RestrictedDatasetRestorer.java @@ -20,44 +20,23 @@ import gov.nist.oar.distrib.ObjectNotFoundException; import gov.nist.oar.distrib.ResourceNotFoundException; import gov.nist.oar.distrib.BagStorage; -import gov.nist.oar.distrib.Checksum; -import gov.nist.oar.distrib.cachemgr.Restorer; import gov.nist.oar.distrib.cachemgr.Reservation; -import gov.nist.oar.distrib.cachemgr.IntegrityMonitor; -import gov.nist.oar.distrib.cachemgr.BasicCache; import gov.nist.oar.distrib.cachemgr.Cache; import gov.nist.oar.distrib.cachemgr.CacheObject; -import gov.nist.oar.distrib.cachemgr.CacheObjectCheck; -import gov.nist.oar.distrib.cachemgr.StorageInventoryDB; import gov.nist.oar.distrib.cachemgr.CacheManagementException; import gov.nist.oar.distrib.cachemgr.RestorationException; -import gov.nist.oar.distrib.cachemgr.InventoryException; import java.util.Collection; -import java.util.List; -import java.util.ArrayList; import java.util.Set; -import java.util.Map; -import java.util.HashSet; -import java.util.HashMap; -import java.util.zip.ZipInputStream; -import java.util.zip.ZipEntry; -import java.nio.file.Path; -import java.nio.file.Paths; import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.BufferedReader; import java.io.IOException; import java.io.FileNotFoundException; -import java.text.ParseException; import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.json.JSONObject; import org.json.JSONException; -import org.apache.commons.io.FilenameUtils; /** * A {@link gov.nist.oar.distrib.cachemgr.Restorer} for restoring "restricted public" datasets from the @@ -82,6 +61,7 @@ */ public class RestrictedDatasetRestorer extends PDRDatasetRestorer { BagStorage restrictedLtstore = null; + long expiryTime = 1209600000L; // 2 weeks in milliseconds /** * create the restorer @@ -125,6 +105,25 @@ public RestrictedDatasetRestorer(BagStorage publicLtstore, BagStorage restricted this.restrictedLtstore = restrictedLtstore; } + /** + * Retrieves the expiry time for data. + *

+ * This value represents the duration in milliseconds after which the data is considered expired. + * + * @return the expiry time in milliseconds. + */ + public long getExpiryTime() { + return expiryTime; + } + + /** + * Sets the expiry time for restricted/public access content. + * + * @param expiryTime the expiry time in milliseconds to set. + */ + public void setExpiryTime(long expiryTime) { + this.expiryTime = expiryTime; + } /** * return true if an object does not exist in the long term storage system. Returning @@ -285,4 +284,19 @@ protected void cacheFromBag(String bagfile, Collection need, Collection< target, ltstore); } } + + /** + * Updates the metadata for files marked as restricted, by adding an expiration time. + * + * @param md The metadata JSONObject to be customized. + * @param prefs flags for data roles + */ + @Override + protected void updateMetadata(JSONObject md, int prefs) { + if ((prefs & ROLE_RESTRICTED_DATA) != 0) { + // Calculate the expiration time as current time + expiryTime + long expires = System.currentTimeMillis() + expiryTime; + md.put("expires", expires); + } + } } diff --git a/src/main/java/gov/nist/oar/distrib/service/RPACachingService.java b/src/main/java/gov/nist/oar/distrib/service/RPACachingService.java index 18017a4f..8f87dc83 100644 --- a/src/main/java/gov/nist/oar/distrib/service/RPACachingService.java +++ b/src/main/java/gov/nist/oar/distrib/service/RPACachingService.java @@ -62,6 +62,7 @@ public String cacheAndGenerateRandomId(String datasetID, String version) logger.debug("Request to cache dataset with ID=" + datasetID); + // this is to handle ark IDs String dsid = datasetID; if (datasetID.startsWith("ark:/")) { // Split the dataset ID into components @@ -73,7 +74,9 @@ public String cacheAndGenerateRandomId(String datasetID, String version) } logger.debug("Caching dataset with dsid=" + dsid); - String randomID = generateRandomID(RANDOM_ID_LENGTH, true, true); + // append "rpa-" with the generated random ID + String randomID = "rpa-" + generateRandomID(RANDOM_ID_LENGTH, true, true); + int prefs = ROLE_RESTRICTED_DATA; if (!version.isEmpty()) @@ -126,6 +129,7 @@ public Map retrieveMetadata(String randomID) throws CacheManagem /** * Formats the metadata from a cache object to a JSON object with an additional field for the download URL. + * The download URL includes the random temporary ID, aipid, and the file path from the metadata. * * @param inMd the metadata from the cache object * @param randomID the random temporary ID associated with the cache object @@ -136,17 +140,27 @@ private JSONObject formatMetadata(JSONObject inMd, String randomID) throws Reque JSONObject outMd = new JSONObject(); List missingFields = new ArrayList<>(); + String aipid = ""; + if (inMd.has("aipid")) { + aipid = inMd.getString("aipid"); + outMd.put("aipid", aipid); + } else { + missingFields.add("aipid"); + } + if (inMd.has("filepath")) { String downloadURL = getDownloadUrl( rpaConfiguration.getBaseDownloadUrl(), randomID, - inMd.get("filepath").toString()); + aipid, + inMd.getString("filepath")); outMd.put("downloadURL", downloadURL); - outMd.put("filePath", inMd.get("filepath")); + outMd.put("filePath", inMd.getString("filepath")); } else { missingFields.add("filepath"); } + if (inMd.has("contentType")) { outMd.put("mediaType", inMd.get("contentType")); } else { @@ -196,12 +210,6 @@ private JSONObject formatMetadata(JSONObject inMd, String randomID) throws Reque missingFields.add("ediid"); } - if (inMd.has("aipid")) { - outMd.put("aipid", inMd.get("aipid")); - } else { - missingFields.add("aipid"); - } - if (inMd.has("sinceDate")) { outMd.put("sinceDate", inMd.get("sinceDate")); } else { @@ -217,28 +225,44 @@ private JSONObject formatMetadata(JSONObject inMd, String randomID) throws Reque /** - * Constructs a download URL using the given base download URL, random ID, and file path from the metadata. + * Constructs a download URL using the given base download URL, random ID, aipid, and file path from the metadata. * * @param baseDownloadUrl the base download URL * @param randomId the random temporary ID + * @param aipid the aipid from the metadata * @param path the file path from the metadata * @return the download URL as a string * @throws RequestProcessingException if there was an error building the download URL */ - private String getDownloadUrl(String baseDownloadUrl, String randomId, String path) throws RequestProcessingException { + + private String getDownloadUrl(String baseDownloadUrl, String randomId, String aipid, String path) throws RequestProcessingException { URL downloadUrl; try { URL url = new URL(baseDownloadUrl); + StringBuilder pathBuilder = new StringBuilder(); + + // append the randomId to the path + pathBuilder.append(randomId); + + // append the aipid if it's not empty + if (!aipid.isEmpty()) { + pathBuilder.append("/").append(aipid); + } + + // append the file path, ensuring it doesn't start with a "/" if (path.startsWith("/")) { path = path.substring(1); } - downloadUrl = new URL(url, randomId + "/" + path); + pathBuilder.append("/").append(path); + + downloadUrl = new URL(url, pathBuilder.toString()); } catch (MalformedURLException e) { throw new RequestProcessingException("Failed to build downloadUrl: " + e.getMessage()); } return downloadUrl.toString(); } + /** * Generate a random alphanumeric string for the dataset to store * This function uses the {@link RandomStringUtils} from Apache Commons. @@ -246,4 +270,45 @@ private String getDownloadUrl(String baseDownloadUrl, String randomId, String pa private String generateRandomID(int length, boolean useLetters, boolean useNumbers) { return RandomStringUtils.random(length, useLetters, useNumbers); } + + /** + * Uncache dataset objects using a specified random ID. + * + * @param randomId - The random ID used to fetch and uncache dataset objects. + * @return boolean - True if at least one dataset object was uncached successfully; otherwise, false. + * @throws CacheManagementException if an error occurs during the uncaching process. + */ + public boolean uncacheById(String randomId) throws CacheManagementException { + // Validate input + if (randomId == null || randomId.isEmpty()) { + throw new IllegalArgumentException("Random ID cannot be null or empty."); + } + + logger.debug("Request to uncache dataset with ID=" + randomId); + + // Retrieve dataset objects using the randomId + List objects = this.pdrCacheManager.selectDatasetObjects(randomId, this.pdrCacheManager.VOL_FOR_INFO); + + if (objects.isEmpty()) { + logger.debug("No objects found for ID=" + randomId); + return false; + } + + boolean isUncached = false; + + // Iterate through the retrieved objects and attempt to uncache them + for (CacheObject obj : objects) { + try { + logger.debug("Deleting file with ID=" + obj.id); + this.pdrCacheManager.uncache(obj.id); + isUncached = true; + } catch (CacheManagementException e) { + // Log the exception without throwing it to continue attempting to uncache remaining objects + logger.error("Failed to uncache object with ID=" + obj.id, e); + } + } + + return isUncached; + } + } diff --git a/src/main/java/gov/nist/oar/distrib/service/rpa/DefaultRPADatasetCacher.java b/src/main/java/gov/nist/oar/distrib/service/rpa/DefaultRPADatasetCacher.java index 3ac34daf..0e915b71 100644 --- a/src/main/java/gov/nist/oar/distrib/service/rpa/DefaultRPADatasetCacher.java +++ b/src/main/java/gov/nist/oar/distrib/service/rpa/DefaultRPADatasetCacher.java @@ -37,6 +37,19 @@ public String cache(String datasetId) throws RequestProcessingException { return randomId; } + @Override + public boolean uncache(String randomId) { + boolean uncached = false; + try { + uncached = rpaCachingService.uncacheById(randomId); + } catch (Exception e) { + this.logCachingException(e); + throw new RequestProcessingException(e.getMessage()); + } + + return uncached; + } + /** * Logs the specified exception to the debug log, along with its stack trace. * diff --git a/src/main/java/gov/nist/oar/distrib/service/rpa/DefaultRPARequestHandlerService.java b/src/main/java/gov/nist/oar/distrib/service/rpa/DefaultRPARequestHandlerService.java deleted file mode 100644 index 2acf61f9..00000000 --- a/src/main/java/gov/nist/oar/distrib/service/rpa/DefaultRPARequestHandlerService.java +++ /dev/null @@ -1,397 +0,0 @@ -package gov.nist.oar.distrib.service.rpa; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import gov.nist.oar.distrib.service.rpa.exceptions.FailedRecordUpdateException; -import gov.nist.oar.distrib.service.rpa.exceptions.InvalidRecaptchaException; -import gov.nist.oar.distrib.service.rpa.exceptions.InvalidRequestException; -import gov.nist.oar.distrib.service.rpa.exceptions.RecordNotFoundException; -import gov.nist.oar.distrib.service.rpa.exceptions.UnauthorizedException; -import gov.nist.oar.distrib.service.rpa.model.EmailInfo; -import gov.nist.oar.distrib.service.rpa.model.EmailInfoWrapper; -import gov.nist.oar.distrib.service.rpa.model.JWTToken; -import gov.nist.oar.distrib.service.rpa.model.RecaptchaResponse; -import gov.nist.oar.distrib.service.rpa.model.Record; -import gov.nist.oar.distrib.service.rpa.model.RecordStatus; -import gov.nist.oar.distrib.service.rpa.model.RecordWrapper; -import gov.nist.oar.distrib.service.rpa.model.UserInfoWrapper; -import gov.nist.oar.distrib.web.RPAConfiguration; -import io.jsonwebtoken.Jwts; -import io.jsonwebtoken.SignatureAlgorithm; -import lombok.RequiredArgsConstructor; -import org.apache.http.client.HttpClient; -import org.apache.http.impl.client.HttpClientBuilder; -import org.json.JSONObject; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.http.HttpEntity; -import org.springframework.http.HttpHeaders; -import org.springframework.http.HttpMethod; -import org.springframework.http.HttpStatus; -import org.springframework.http.MediaType; -import org.springframework.http.ResponseEntity; -import org.springframework.http.client.HttpComponentsClientHttpRequestFactory; -import org.springframework.web.client.HttpStatusCodeException; -import org.springframework.web.client.RestTemplate; -import org.springframework.web.util.UriComponentsBuilder; - -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.util.Date; - -/** - * Default implementation of the RPARequestHandlerService. - */ -@RequiredArgsConstructor -public class DefaultRPARequestHandlerService implements RPARequestHandlerService { - - private final static Logger LOGGER = LoggerFactory.getLogger(DefaultRPARequestHandlerService.class); - private final static String API_TEST_ENDPOINT_KEY = "api-test-endpoint"; - private final static String GET_RECORD_ENDPOINT_KEY = "get-record-endpoint"; - private final static String CREATE_RECORD_ENDPOINT_KEY = "create-record-endpoint"; - private final static String UPDATE_RECORD_ENDPOINT_KEY = "update-record-endpoint"; - private final static String SEND_EMAIL_ENDPOINT_KEY = "send-email-endpoint"; - - private RPAConfiguration rpaConfiguration = null; - private KeyRetriever keyRetriever = null; - private RestTemplate restTemplate; - - /** - * Constructs a new DefaultRPARequestHandlerService object with the given rpaConfiguration and keyRetriever. - * @param rpaConfiguration The Restricted Public Access (RPA) configuration object - * @param keyRetriever The private key retriever object - */ - public DefaultRPARequestHandlerService(RPAConfiguration rpaConfiguration, RestTemplate restTemplate) { - this.rpaConfiguration = rpaConfiguration; - this.keyRetriever = new JKSKeyRetriever(); - this.restTemplate = new RestTemplate(); - // We need to include HttpComponentsClientHttpRequestFactory because The standard JDK HTTP library - // does not support HTTP PATCH. We need to use the Apache HttpComponents or OkHttp request factory. - HttpClient httpClient = HttpClientBuilder.create().build(); - this.restTemplate.setRequestFactory(new HttpComponentsClientHttpRequestFactory(httpClient)); - - LOGGER.debug("RPA_CONFIGURATION=" + this.rpaConfiguration.toString()); - } - - /** - * Returns the RPA configuration. This is a bean that is injected into the service. - * The RPA configuration is loaded from the config server. - * - * @return The RPA configuration - */ - public RPAConfiguration getConfig() { return this.rpaConfiguration; } - - - /** - * Get information about a specific record. - * - * This asks for an access token, then uses it to add the bearer auth http header. - * Constructs the URL, and sends a GET request to fetch information about the record. - * - * @param recordId the identifier for the record. - * - * @exception UnauthorizedException if there is an issue with the access token. - * @exception RecordNotFoundException if record is not found. - * - * @return RecordWrapper -- the requested record wrapped within a "record" envelope. - */ - @Override - public RecordWrapper getRecord(String recordId) throws RecordNotFoundException, UnauthorizedException { - String getRecordUri = getConfig().getSalesforceEndpoints().get(GET_RECORD_ENDPOINT_KEY); - JWTToken token = getToken(); - HttpHeaders headers = getHttpHeaders(token, null); - String url = UriComponentsBuilder.fromUriString(token.getInstanceUrl()) - .path(getRecordUri + "/" + recordId) - .toUriString(); - ResponseEntity response = restTemplate.exchange(url, HttpMethod.GET, - new HttpEntity<>(headers), RecordWrapper.class); - if (response.getStatusCode() == HttpStatus.NOT_FOUND) { - throw RecordNotFoundException.fromRecordId(recordId); - } - return response.getBody(); - } - - /** - * Create a new record. - * - * This asks for an access token, then uses it to add the bearer auth http header. - * Constructs the URL, and sends a POST request to create a new record. - * - * @param userInfoWrapper the information provided by the user. - * - * @exception UnauthorizedException if there is an issue with the access token. - * @exception InvalidRecaptchaException if there is an issue processing the recaptcha. - * @exception InvalidRequestException if there is an issue with the request fields. - * - * @return RecordWrapper -- the newly created record wrapped within a "record" envelope. - */ - @Override - public RecordWrapper createRecord(UserInfoWrapper userInfoWrapper) - throws InvalidRecaptchaException, InvalidRequestException, UnauthorizedException { - - String createRecordUri = getConfig().getSalesforceEndpoints().get(CREATE_RECORD_ENDPOINT_KEY); - JWTToken token = getToken(); - HttpHeaders headers = getHttpHeaders(token, MediaType.APPLICATION_JSON); - HttpEntity request; - ObjectMapper mapper = new ObjectMapper(); - - // first check if recaptcha is valid - RecaptchaResponse recaptchaResponse = verifyRecaptcha(userInfoWrapper.getRecaptcha()); - if (!recaptchaResponse.isSuccess()) { - throw new InvalidRecaptchaException("failed to verify the reCaptcha: " + recaptchaResponse.getErrorCodes()); - } - try { - // cleaning form input from any HTML - UserInfoWrapper cleanUserInfoWrapper = HTMLCleaner.clean(userInfoWrapper); - String payload = mapper.writeValueAsString(cleanUserInfoWrapper); - LOGGER.debug("PAYLOAD=" + payload); - request = new HttpEntity<>(payload, headers); - } catch (JsonProcessingException e) { - throw new InvalidRequestException("could not clean form fields: " + e.getMessage()); - } - String url = token.getInstanceUrl() + createRecordUri; - ResponseEntity responseEntity = restTemplate.exchange(url, HttpMethod.POST, request, RecordWrapper.class); - if (responseEntity.getStatusCode().equals(HttpStatus.OK)) { - onSuccessfulRecordCreation(responseEntity.getBody().getRecord()); - } else { - onFailedRecordCreation(responseEntity.getStatusCode()); - } - return responseEntity.getBody(); - } - - /** - * Function to verify the recaptcha token. - */ - private RecaptchaResponse verifyRecaptcha(String token) { - String url = UriComponentsBuilder.fromUriString("https://www.google.com") - .path("/recaptcha/api/siteverify") - .queryParam("secret", getConfig().getRecaptchaSecret()) - .queryParam("response", token) - .toUriString(); - - LOGGER.debug("RECAPTCHA_URL=" + url); - HttpHeaders headers = new HttpHeaders(); - return restTemplate.postForObject(url, new HttpEntity<>(null, headers), RecaptchaResponse.class); - } - - - /** - * On successful creation of a record, send a confirmation email to the user, and another email to the approver. - * Check if sending of emails was successful. - */ - private void onSuccessfulRecordCreation(Record record) throws UnauthorizedException { - LOGGER.debug("Record created successfully! Now sending emails..."); - if (sendConfirmationEmailToEndUser(record).equals(HttpStatus.OK)) { - LOGGER.debug("Confirmation email sent to end user successfully!"); - } - if (sendApprovalEmailToSME(record).equals(HttpStatus.OK)) { - LOGGER.debug("Request approval email sent to subject matter expert successfully!"); - } - } - - /** - * On failed creation of a record - * TODO: add logic when creation fails - */ - private void onFailedRecordCreation(HttpStatus statusCode) { - // handle failed record creation - } - - - /** - * Update the status of a specific record. - * - * This asks for an access token, then uses it to add the bearer auth http header. - * Constructs the URL, and sends a PATCH request to update a record. - * - * @param recordId the identifier for the record. - * @param status the new status. - * - * @exception UnauthorizedException if there is an issue with the access token. - * @exception RecordNotFoundException if record is not found. - * @exception FailedRecordUpdateException if the record update failed. - * - * @return RecordStatus -- the updated status of the record. - */ - @Override - public RecordStatus updateRecord(String recordId, String status) - throws RecordNotFoundException, UnauthorizedException, FailedRecordUpdateException { - - String updateRecordUri = getConfig().getSalesforceEndpoints().get(UPDATE_RECORD_ENDPOINT_KEY); - JSONObject updateBody = new JSONObject(); - updateBody.put("Approval_Status__c", status); - JWTToken token = getToken(); - HttpHeaders headers = getHttpHeaders(token, MediaType.APPLICATION_JSON); - String patch = updateBody.toString(); - LOGGER.debug("PATCH_DATA=" + patch); - HttpEntity request = new HttpEntity<>(patch, headers); - String url = UriComponentsBuilder.fromUriString(token.getInstanceUrl()) - .path(updateRecordUri + "/" + recordId) - .toUriString(); - LOGGER.debug("UPDATE_URL=" + url); - ResponseEntity responseEntity = null; - try { - responseEntity= restTemplate.exchange( - url, HttpMethod.PATCH, request, RecordStatus.class - ); - } catch (HttpStatusCodeException e) { - LOGGER.debug("failed to update record: " + e.getResponseBodyAsString()); - throw FailedRecordUpdateException.forID(recordId); - } - - // check if status is approved and trigger the caching process - LOGGER.debug("APPROVAL_STATUS=" + responseEntity.getBody().getApprovalStatus()); - if (responseEntity.getBody().getApprovalStatus().toLowerCase().contains("approved")) { - onEndUserApproved(recordId); - } - // check if status is declined and send email notification to user - if (responseEntity.getBody().getApprovalStatus().toLowerCase().contains("declined")) { - // Don't do anything if User is declined - // onEndUserDeclined(recordId); - } - return responseEntity.getBody(); - } - - private HttpStatus onEndUserApproved(String recordId) throws RecordNotFoundException, UnauthorizedException { - LOGGER.info("User was approved by SME. Starting caching..."); - Record record = getRecord(recordId).getRecord(); - String datasetId = record.getUserInfo().getSubject(); - String randomId = startCaching(datasetId); - String downloadUrl = UriComponentsBuilder.fromUriString(getConfig().getDatacartUrl()) - .queryParam("id", randomId) - .toUriString(); - LOGGER.info("Dataset was cached successfully. Sending email to user..."); - return sendDownloadEmailToEndUser(record, downloadUrl); - } - - private HttpStatus onEndUserDeclined(String recordId) throws RecordNotFoundException, UnauthorizedException { - LOGGER.info("User was declined by SME. Sending declined email to user..."); - Record record = getRecord(recordId).getRecord(); - return sendDeclinedEmailToEndUser(record); - } - - // Function to call the caching the service to start the caching process - // This function returns a temporary URL to the datacart that contains the cached dataset - private String startCaching(String datasetId) { - String url = getConfig().getPdrCachingUrl() + "/cache/" + datasetId; - HttpEntity request = new HttpEntity<>(null, new HttpHeaders()); - ResponseEntity responseEntity = restTemplate.exchange(url, HttpMethod.PUT, request, String.class); - return responseEntity.getBody(); - } - - private HttpStatus sendApprovalEmailToSME(Record record) throws UnauthorizedException { - EmailInfo emailInfo = EmailHelper.getSMEApprovalEmailInfo(record, getConfig()); - LOGGER.debug("EMAIL_INFO=" + emailInfo); - ResponseEntity responseEntity = this.sendEmail(emailInfo); - return responseEntity.getStatusCode(); - } - - private HttpStatus sendConfirmationEmailToEndUser(Record record) throws UnauthorizedException { - EmailInfo emailInfo = EmailHelper.getEndUserConfirmationEmailInfo(record, getConfig()); - LOGGER.debug("EMAIL_INFO=" + emailInfo); - ResponseEntity responseEntity = this.sendEmail(emailInfo); - return responseEntity.getStatusCode(); - } - - private HttpStatus sendDownloadEmailToEndUser(Record record, String downloadUrl) throws UnauthorizedException { - EmailInfo emailInfo = EmailHelper.getEndUserDownloadEmailInfo(record, getConfig(), downloadUrl); - LOGGER.info("EMAIL_INFO=" + emailInfo); - ResponseEntity responseEntity = this.sendEmail(emailInfo); - return responseEntity.getStatusCode(); - } - - private HttpStatus sendDeclinedEmailToEndUser(Record record) throws UnauthorizedException { - EmailInfo emailInfo = EmailHelper.getEndUserDeclinedEmailInfo(record, getConfig()); - LOGGER.debug("EMAIL_INFO=" + emailInfo); - ResponseEntity responseEntity = this.sendEmail(emailInfo); - return responseEntity.getStatusCode(); - } - - - private ResponseEntity sendEmail(EmailInfo emailInfo) throws UnauthorizedException { - String sendEmailUri = getConfig().getSalesforceEndpoints().get(SEND_EMAIL_ENDPOINT_KEY); - JWTToken token = getToken(); - HttpHeaders headers = getHttpHeaders(token, MediaType.APPLICATION_JSON); - ObjectMapper mapper = new ObjectMapper(); - HttpEntity request; - try { - request = new HttpEntity<>(mapper.writeValueAsString(emailInfo), headers); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - String url = token.getInstanceUrl() + sendEmailUri; - return restTemplate.exchange(url, HttpMethod.POST, request, EmailInfoWrapper.class); - } - - - /** - * Test connection to Salesforce service to make sure the Connected App is working. - */ - public String testSalesforceAPIConnection() throws UnauthorizedException { - String testUri = getConfig().getSalesforceEndpoints().get(API_TEST_ENDPOINT_KEY); - JWTToken token = getToken(); - HttpHeaders headers = getHttpHeaders(token, null); - ResponseEntity response = restTemplate.exchange(token.getInstanceUrl() + testUri, - HttpMethod.GET, new HttpEntity<>(headers), String.class); - return response.getBody(); - } - - private HttpHeaders getHttpHeaders(JWTToken token, MediaType mediaType) { - HttpHeaders headers = new HttpHeaders(); - headers.setBearerAuth(token.getAccessToken()); - if (mediaType != null) - headers.setContentType(mediaType); - return headers; - } - - // JWT methods - - // Retrieve the JWT Access Token - private JWTToken getToken() throws UnauthorizedException { - return sendTokenRequest(createAssertion()); - } - - // Create the jwt assertion. - private String createAssertion() { - LocalDateTime localDateTime = LocalDateTime.now().plusMinutes(getConfig().getSalesforceJwt().getExpirationInMinutes()); - return Jwts.builder() - .setIssuer(getConfig().getSalesforceJwt().getClientId()) - .setAudience(getConfig().getSalesforceJwt().getAudience()) - .setSubject(getConfig().getSalesforceJwt().getSubject()) - .setExpiration(Date.from(localDateTime.atZone(ZoneId.systemDefault()).toInstant())) - .signWith(SignatureAlgorithm.RS256, keyRetriever.getKey(getConfig())) - .compact(); - } - - /** - * Send token request as per rfc7523. - * - * @param assertion - the assertion token containing a JWT token - * - * @return JWTToken - the access token - */ - private JWTToken sendTokenRequest(String assertion) throws UnauthorizedException { - String url = UriComponentsBuilder.fromUriString(getConfig().getSalesforceInstanceUrl()) - .path("/services/oauth2/token") - .queryParam("grant_type", getConfig().getSalesforceJwt().getGrantType()) - .queryParam("assertion", assertion) - .toUriString(); - HttpHeaders headers = new HttpHeaders(); - headers.setContentType(MediaType.APPLICATION_FORM_URLENCODED); - ResponseEntity response; - try { - response = restTemplate.exchange( - url, - HttpMethod.POST, - new HttpEntity<>(null, headers), - JWTToken.class - ); - } catch (HttpStatusCodeException e) { - LOGGER.debug("access token request is invalid: " + e.getResponseBodyAsString()); - throw new UnauthorizedException("access token request is invalid: " + e.getResponseBodyAsString()); - } - return response.getBody(); - } - - -} diff --git a/src/main/java/gov/nist/oar/distrib/service/rpa/HttpRPADatasetCacher.java b/src/main/java/gov/nist/oar/distrib/service/rpa/HttpRPADatasetCacher.java index 9bf5f76f..19204c88 100644 --- a/src/main/java/gov/nist/oar/distrib/service/rpa/HttpRPADatasetCacher.java +++ b/src/main/java/gov/nist/oar/distrib/service/rpa/HttpRPADatasetCacher.java @@ -50,6 +50,11 @@ public String cache(String datasetId) { return sendHttpRequest(datasetId, url); } + @Override + public boolean uncache(String randomId) { + return false; + } + /** * Builds the URL for the given dataset ID and using the given {@link RPAConfiguration} object. * diff --git a/src/main/java/gov/nist/oar/distrib/service/rpa/HttpURLConnectionRPARequestHandlerService.java b/src/main/java/gov/nist/oar/distrib/service/rpa/HttpURLConnectionRPARequestHandlerService.java index ed69fe06..7b49b3c8 100644 --- a/src/main/java/gov/nist/oar/distrib/service/rpa/HttpURLConnectionRPARequestHandlerService.java +++ b/src/main/java/gov/nist/oar/distrib/service/rpa/HttpURLConnectionRPARequestHandlerService.java @@ -3,15 +3,10 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import gov.nist.oar.distrib.service.RPACachingService; -import gov.nist.oar.distrib.service.rpa.exceptions.InvalidRecaptchaException; import gov.nist.oar.distrib.service.rpa.exceptions.InvalidRequestException; -import gov.nist.oar.distrib.service.rpa.exceptions.RecaptchaClientException; -import gov.nist.oar.distrib.service.rpa.exceptions.RecaptchaServerException; -import gov.nist.oar.distrib.service.rpa.exceptions.RecaptchaVerificationFailedException; import gov.nist.oar.distrib.service.rpa.exceptions.RecordNotFoundException; import gov.nist.oar.distrib.service.rpa.exceptions.RequestProcessingException; import gov.nist.oar.distrib.service.rpa.model.JWTToken; -import gov.nist.oar.distrib.service.rpa.model.RecaptchaResponse; import gov.nist.oar.distrib.service.rpa.model.Record; import gov.nist.oar.distrib.service.rpa.model.RecordStatus; import gov.nist.oar.distrib.service.rpa.model.RecordWrapper; @@ -48,10 +43,14 @@ /** - * An implementation of the RPARequestHandlerService that uses HttpURLConnection to send HTTP requests and - * receives responses from the Salesforce service. + * An implementation of the {@link RPARequestHandler} interface that uses HttpURLConnection + * to send HTTP requests and receive responses from the Salesforce service. This class serves + * as a bridge between the application and Salesforce, acting as the data source for managing + * records. Through this service, records can be created, retrieved, and updated directly in + * Salesforce, using the platform's API for record management. + * */ -public class HttpURLConnectionRPARequestHandlerService implements IRPARequestHandler { +public class HttpURLConnectionRPARequestHandlerService implements RPARequestHandler { private final static Logger LOGGER = LoggerFactory.getLogger(HttpURLConnectionRPARequestHandlerService.class); @@ -102,6 +101,7 @@ public class HttpURLConnectionRPARequestHandlerService implements IRPARequestHan */ private RecordResponseHandler recordResponseHandler; + private RPADatasetCacher rpaDatasetCacher; /** * Sets the HTTP URL connection factory. * @@ -143,6 +143,10 @@ public void setRecordResponseHandler(RecordResponseHandler recordResponseHandler this.recordResponseHandler = recordResponseHandler; } + public void seRPADatasetCacher(RPADatasetCacher rpaDatasetCacher) { + this.rpaDatasetCacher = rpaDatasetCacher; + } + /** * Constructs a new instance of the service using the given RPA configuration. * @@ -168,6 +172,9 @@ public HttpURLConnectionRPARequestHandlerService(RPAConfiguration rpaConfigurati this.recordResponseHandler = new RecordResponseHandlerImpl(this.rpaConfiguration, this.connectionFactory, rpaCachingService); + // Set RPADatasetCacher + this.rpaDatasetCacher = new DefaultRPADatasetCacher(rpaCachingService); + // Set HttpClient this.httpClient = HttpClients.createDefault(); @@ -431,28 +438,55 @@ private String prepareRequestPayload(UserInfoWrapper userInfoWrapper) throws Jso /** - * Updates the status of a record with a given ID. + * Updates the status of a record in the database. + *

+ * This method handles the approval or decline of a record. When a record is approved, it caches the dataset, + * generates a random ID, and appends this ID to the status before updating the record in the database. + * When a record is declined, the dataset is not cached, a null random ID is used, and the record status is + * updated in the database without appending the random ID. + *

+ *

+ * In cases where a record was initially approved (and thus cached with a random ID) but is later declined, + * this method retrieves the random ID from the status, uncaches the dataset using this ID, and updates + * the record status without the random ID. + *

* * @param recordId The ID of the record to update. - * @param status The status to update the record with. - * @return The {@link RecordStatus} object representing the updated record status. + * @param status The new status to be set for the record. Can be 'Approved' or 'Declined'. + * @param smeId The SME ID associated with the record update. + * @return A {@link RecordStatus} object representing the updated record status. * @throws RecordNotFoundException If the record with the given ID is not found. - * @throws InvalidRequestException If the request is invalid. - * @throws RequestProcessingException If there is an error processing the request. + * @throws InvalidRequestException If the provided status is invalid or the request is otherwise invalid. + * @throws RequestProcessingException If there is an error in processing the update request, such as issues + * with caching or communication errors with the database. */ @Override public RecordStatus updateRecord(String recordId, String status, String smeId) throws RecordNotFoundException, InvalidRequestException, RequestProcessingException { // Initialize return object RecordStatus recordStatus; - - // Get endpoint - String updateRecordUri = getConfig().getSalesforceEndpoints().get(UPDATE_RECORD_ENDPOINT_KEY); + Record record = this.getRecord(recordId).getRecord(); + String datasetId = record.getUserInfo().getSubject(); + String randomId = null; + + // If the record is being approved + if (RECORD_APPROVED_STATUS.equalsIgnoreCase(status)) { + LOGGER.info("Starting caching..."); + randomId = this.rpaDatasetCacher.cache(datasetId); + if (randomId == null) { + throw new RequestProcessingException("Caching process returned a null randomId"); + } + } + // If the record is being declined, check if it needs uncaching + else if (RECORD_DECLINED_STATUS.equalsIgnoreCase(status)) { + randomId = extractRandomIdFromCurrentStatus(record.getUserInfo().getApprovalStatus()); + if (randomId != null) { + this.rpaDatasetCacher.uncache(randomId); + } + } // Create a valid approval status based on input - String approvalStatus = generateApprovalStatus(status, smeId); - - // TODO: try caching here before updating the status in SF + String approvalStatus = generateApprovalStatus(status, smeId, randomId); // PATCH request payload // Approval_Status__c is how SF service expect the key @@ -462,6 +496,9 @@ public RecordStatus updateRecord(String recordId, String status, String smeId) t // Get token JWTToken token = jwtHelper.getToken(); + // Get endpoint + String updateRecordUri = getConfig().getSalesforceEndpoints().get(UPDATE_RECORD_ENDPOINT_KEY); + // Build request URL String url; try { @@ -506,12 +543,9 @@ public RecordStatus updateRecord(String recordId, String status, String smeId) t throw new RequestProcessingException("I/O error: " + e.getMessage()); } - // Retrieve updated record from SF service - Record record = this.getRecord(recordId).getRecord(); - // Check if status is approved if (recordStatus.getApprovalStatus().toLowerCase().contains("approved")) { - this.recordResponseHandler.onRecordUpdateApproved(record); + this.recordResponseHandler.onRecordUpdateApproved(record, randomId); } else { this.recordResponseHandler.onRecordUpdateDeclined(record); } @@ -519,25 +553,48 @@ public RecordStatus updateRecord(String recordId, String status, String smeId) t return recordStatus; } + private String extractRandomIdFromCurrentStatus(String currentStatus) { + if (currentStatus != null && currentStatus.startsWith("Approved_")) { + String[] parts = currentStatus.split("_"); + + // Since the expected format is "[status]_[yyyy-MM-dd'T'HH:mm:ss.SSSZ]_[smeId]_[randomId]", + // the randomId should be the 4th part, if all parts are present + if (parts.length == 4) { + return parts[3]; + } + } + return null; + } + + /** - * Generates an approval status string based on the given status and current date/time. - * The date is in ISO 8601 format. + * Generates an approval status string based on the given status, current date/time, and random ID. + * The date is in ISO 8601 format. If the status is "Declined", the randomId will not be appended. * - * @param status the approval status to use, either "Approved" or "Declined" - * @param email the email to append to the status - * @return the generated approval status string, in the format "[status]_[yyyy-MM-dd'T'HH:mm:ss.SSSZ]_[email]" + * @param status the approval status to use, either "Approved" or "Declined" + * @param smeId the SME ID to append to the status + * @param randomId the generated random ID to append (only if status is "Approved") + * @return the generated approval status string. + * If status is "Approved", the format is: + * "[status]_[yyyy-MM-dd'T'HH:mm:ss.SSSZ]_[smeId]_[randomId]". + * If status is "Declined", the format is: + * "[status]_[yyyy-MM-dd'T'HH:mm:ss.SSSZ]_[smeId]" * @throws InvalidRequestException if the provided status is not "Approved" or "Declined" */ - private String generateApprovalStatus(String status, String smeId) throws InvalidRequestException { - String formattedDate = Instant.now().toString(); // ISO 8601 format: 2023-05-09T15:59:03.872Z + private String generateApprovalStatus(String status, String smeId, String randomId) throws InvalidRequestException { + String formattedDate = Instant.now().toString(); String approvalStatus; + if (status != null) { switch (status.toLowerCase()) { case RECORD_APPROVED_STATUS: - approvalStatus = "Approved_"; + approvalStatus = "Approved_" + formattedDate + "_" + smeId; + if (randomId != null) { + approvalStatus += "_" + randomId; + } break; case RECORD_DECLINED_STATUS: - approvalStatus = "Declined_"; + approvalStatus = "Declined_" + formattedDate + "_" + smeId; break; default: throw new InvalidRequestException("Invalid approval status: " + status); @@ -545,7 +602,8 @@ private String generateApprovalStatus(String status, String smeId) throws Invali } else { throw new InvalidRequestException("Invalid approval status: status is null"); } - return approvalStatus + formattedDate + "_" + smeId; + return approvalStatus; } + } diff --git a/src/main/java/gov/nist/oar/distrib/service/rpa/RPADatasetCacher.java b/src/main/java/gov/nist/oar/distrib/service/rpa/RPADatasetCacher.java index 2a2150c3..93395cc8 100644 --- a/src/main/java/gov/nist/oar/distrib/service/rpa/RPADatasetCacher.java +++ b/src/main/java/gov/nist/oar/distrib/service/rpa/RPADatasetCacher.java @@ -14,4 +14,6 @@ public interface RPADatasetCacher { * @throws RequestProcessingException */ String cache(String datasetId) throws RequestProcessingException; + + boolean uncache(String randomId); } diff --git a/src/main/java/gov/nist/oar/distrib/service/rpa/IRPARequestHandler.java b/src/main/java/gov/nist/oar/distrib/service/rpa/RPARequestHandler.java similarity index 87% rename from src/main/java/gov/nist/oar/distrib/service/rpa/IRPARequestHandler.java rename to src/main/java/gov/nist/oar/distrib/service/rpa/RPARequestHandler.java index 7029b560..ac647073 100644 --- a/src/main/java/gov/nist/oar/distrib/service/rpa/IRPARequestHandler.java +++ b/src/main/java/gov/nist/oar/distrib/service/rpa/RPARequestHandler.java @@ -9,9 +9,11 @@ import gov.nist.oar.distrib.service.rpa.model.UserInfoWrapper; /** - * An interface for handling requests to manage records. + * An interface for handling requests to manage records. This includes operations + * such as creating, retrieving, and updating records. Implementations of this interface + * are responsible for the interaction with a data source to perform these operations. */ -public interface IRPARequestHandler { +public interface RPARequestHandler { /** * Updates the status of a record by ID. diff --git a/src/main/java/gov/nist/oar/distrib/service/rpa/RPARequestHandlerService.java b/src/main/java/gov/nist/oar/distrib/service/rpa/RPARequestHandlerService.java deleted file mode 100644 index 0bc5660c..00000000 --- a/src/main/java/gov/nist/oar/distrib/service/rpa/RPARequestHandlerService.java +++ /dev/null @@ -1,49 +0,0 @@ -package gov.nist.oar.distrib.service.rpa; - -import gov.nist.oar.distrib.service.rpa.exceptions.FailedRecordUpdateException; -import gov.nist.oar.distrib.service.rpa.exceptions.InvalidRecaptchaException; -import gov.nist.oar.distrib.service.rpa.exceptions.InvalidRequestException; -import gov.nist.oar.distrib.service.rpa.exceptions.RecaptchaClientException; -import gov.nist.oar.distrib.service.rpa.exceptions.RecaptchaVerificationFailedException; -import gov.nist.oar.distrib.service.rpa.exceptions.RecordNotFoundException; -import gov.nist.oar.distrib.service.rpa.exceptions.UnauthorizedException; -import gov.nist.oar.distrib.service.rpa.model.RecordStatus; -import gov.nist.oar.distrib.service.rpa.model.RecordWrapper; -import gov.nist.oar.distrib.service.rpa.model.UserInfoWrapper; - - -/** - * Service interface for handling RPA request submissions by end users. - * When an end user submits a new request, a new record will be created. - */ -public interface RPARequestHandlerService { - - /** - * Get information about a specific record. - * - * @param recordId the identifier for the record. - * @return RecordWrapper -- the requested record wrapped within a "record" envelope. - */ - RecordWrapper getRecord(String recordId) throws RecordNotFoundException, UnauthorizedException; - - /** - * Create a new record. - * - * @param userInfoWrapper the information provided by the user. - * @return RecordWrapper -- the newly created record wrapped within a "record" envelope. - */ - RecordWrapper createRecord(UserInfoWrapper userInfoWrapper) throws InvalidRecaptchaException, - InvalidRequestException, UnauthorizedException, RecaptchaVerificationFailedException, - RecaptchaClientException; - - /** - * Update the status of a specific record. - * - * @param recordId the identifier for the record. - * @param status the new status. - * @return RecordStatus -- the updated status of the record. - */ - RecordStatus updateRecord(String recordId, String status) throws RecordNotFoundException, UnauthorizedException, - FailedRecordUpdateException; - -} diff --git a/src/main/java/gov/nist/oar/distrib/service/rpa/RecordResponseHandler.java b/src/main/java/gov/nist/oar/distrib/service/rpa/RecordResponseHandler.java index a1941a2e..7e29f588 100644 --- a/src/main/java/gov/nist/oar/distrib/service/rpa/RecordResponseHandler.java +++ b/src/main/java/gov/nist/oar/distrib/service/rpa/RecordResponseHandler.java @@ -23,7 +23,7 @@ public interface RecordResponseHandler { * This method is called when a record update operation is successful and user was approved. * @param record The record that was the user approved for. */ - void onRecordUpdateApproved(Record record); + void onRecordUpdateApproved(Record record, String randomId); /** * This method is called when a record update operation is successful but user was declined. diff --git a/src/main/java/gov/nist/oar/distrib/service/rpa/RecordResponseHandlerImpl.java b/src/main/java/gov/nist/oar/distrib/service/rpa/RecordResponseHandlerImpl.java index 5a456954..ca7f4150 100644 --- a/src/main/java/gov/nist/oar/distrib/service/rpa/RecordResponseHandlerImpl.java +++ b/src/main/java/gov/nist/oar/distrib/service/rpa/RecordResponseHandlerImpl.java @@ -92,20 +92,18 @@ public void onRecordCreationFailure(int statusCode) throws RequestProcessingExce } /** - * Called when a record status was updated to "Approved". - * This uses {@link RPADatasetCacher} to cache the dataset. + * Called when a record update operation has been approved. * - * @param record the record that was updated + * @param record the record that was updated and approved + * @param randomId the ID generated after caching the dataset related to the record + * @throws InvalidRequestException if there is an error in the request + * @throws RequestProcessingException if there is an error while processing the request */ @Override - public void onRecordUpdateApproved(Record record) throws InvalidRequestException, RequestProcessingException { - LOGGER.info("User was approved by SME. Starting caching..."); - String datasetId = record.getUserInfo().getSubject(); - // NEW: case dataset using the RPADatasetCacher - String randomId = this.rpaDatasetCacher.cache(datasetId); - if (randomId == null) - throw new RequestProcessingException("Caching process return a null randomId"); + public void onRecordUpdateApproved(Record record, String randomId) throws InvalidRequestException, RequestProcessingException { + LOGGER.info("Dataset was cached successfully. Sending email to user..."); + // Build Download URL String downloadUrl; try { @@ -127,7 +125,7 @@ public void onRecordUpdateApproved(Record record) throws InvalidRequestException * @param record the record that was updated */ @Override - public void onRecordUpdateDeclined(Record record) { + public void onRecordUpdateDeclined(Record record) throws InvalidRequestException, RequestProcessingException { LOGGER.debug("User was declined by SME"); } diff --git a/src/main/java/gov/nist/oar/distrib/web/CacheManagementController.java b/src/main/java/gov/nist/oar/distrib/web/CacheManagementController.java index d17ef602..0d21e3d1 100644 --- a/src/main/java/gov/nist/oar/distrib/web/CacheManagementController.java +++ b/src/main/java/gov/nist/oar/distrib/web/CacheManagementController.java @@ -11,6 +11,7 @@ */ package gov.nist.oar.distrib.web; +import gov.nist.oar.distrib.cachemgr.VolumeStatus; import gov.nist.oar.distrib.cachemgr.pdr.PDRCacheManager; import gov.nist.oar.distrib.cachemgr.CacheManagementException; import gov.nist.oar.distrib.cachemgr.CacheObject; @@ -230,14 +231,17 @@ else if (":cached".equals(selector)) } List files = mgr.selectDatasetObjects(dsid, mgr.VOL_FOR_INFO); - if (files.size() == 0) - throw new ResourceNotFoundException(dsid); if (filepath.length() > 0) files = mgr.selectFileObjects(dsid, filepath, purpose); else if (purpose != mgr.VOL_FOR_INFO) files = mgr.selectDatasetObjects(dsid, purpose); + // Ensure that a ResourceNotFoundException is thrown if the files list is empty + // after all the selection logic has been applied + if (files.size() == 0) + throw new ResourceNotFoundException(dsid); + if (selector == null && filepath.length() > 0) { // return a single JSON object; get the one that's cached List use = files.stream().filter(c -> c.cached).collect(Collectors.toList()); @@ -331,6 +335,72 @@ else if (":checked".equals(selector)) return new ResponseEntity("Method not allowed on URL", HttpStatus.METHOD_NOT_ALLOWED); } + /** + * Removes a dataset or specific files within a dataset from the cache based on the provided dataset identifier (dsid). + * This endpoint supports selective removal using the ":cached" selector in the URL path, allowing for more granular + * control over cache management. + * + * If the ":cached" selector is present, and no specific file path is provided, the entire dataset identified by the dsid + * is removed from the cache. If a specific file path is provided, only the specified file within the dataset will be removed. + * + * @param dsid the dataset identifier + * @param request used to extract the optional file path from the URL + * @return ResponseEntity with the result of the operation + */ + @DeleteMapping(value="/objects/{dsid}/**") + public ResponseEntity removeFromCache(@PathVariable("dsid") String dsid, HttpServletRequest request) { + try { + + _checkForManager(); + log.debug("Attempting to remove files from cache for Dataset ID: {}", dsid); + + // Extract the optional file path from the request URL + String path = (String) request.getAttribute(HandlerMapping.PATH_WITHIN_HANDLER_MAPPING_ATTRIBUTE); + String prefix = "/cache/objects/" + dsid; + String filepath = path.startsWith(prefix) ? path.substring(prefix.length()) : ""; + + String selector = null; + Matcher selmatch = SEL_PATH_FIELD.matcher(filepath); + if (selmatch.find()) { + selector = selmatch.group(1); + filepath = filepath.substring(0, selmatch.start()); + } + if (filepath.startsWith("/")) { + filepath = filepath.substring(1); // Remove leading slash + } + + if (":cached".equals(selector)) { + if (filepath.isEmpty() || filepath.equals("/")) { + log.debug("Removing entire dataset from cache for ID: {}", dsid); + List files = mgr.selectDatasetObjects(dsid, VolumeStatus.VOL_FOR_UPDATE); + for (CacheObject file : files) { + log.debug("Uncaching file: {}", file.id); + mgr.uncache(file.id); + } + return ResponseEntity.ok("Dataset " + dsid + " removed from cache"); + } else { + log.debug("Removing file(s) from cache for dataset ID: {} and path: {}", dsid, filepath); + List files = mgr.selectFileObjects(dsid, filepath, VolumeStatus.VOL_FOR_UPDATE); + for (CacheObject file : files) { + log.debug("Uncaching file: {}", file.id); + mgr.uncache(file.id); + } + return ResponseEntity.ok("File(s) " + filepath + " in dataset " + dsid + " removed from cache"); + } + } else { + log.warn("Operation not allowed: URL does not contain ':cached' selector"); + return new ResponseEntity("Operation not allowed on URL without :cached selector", HttpStatus.METHOD_NOT_ALLOWED); + } + + } catch (NotOperatingException e) { + log.error("Cache manager is not operational", e); + return ResponseEntity.status(HttpStatus.SERVICE_UNAVAILABLE).body("Cache manager is not operational"); + } catch (CacheManagementException e) { + log.error("Error processing cache removal request", e); + return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body("Error processing cache removal request: " + e.getMessage()); + } + } + /** * return status information about the caching queue. The caching queue is a queue of data items * waiting to be cached diff --git a/src/main/java/gov/nist/oar/distrib/web/NISTCacheManagerConfig.java b/src/main/java/gov/nist/oar/distrib/web/NISTCacheManagerConfig.java index cfb74fab..4684f5ae 100644 --- a/src/main/java/gov/nist/oar/distrib/web/NISTCacheManagerConfig.java +++ b/src/main/java/gov/nist/oar/distrib/web/NISTCacheManagerConfig.java @@ -12,9 +12,11 @@ package gov.nist.oar.distrib.web; import gov.nist.oar.distrib.cachemgr.BasicCache; +import gov.nist.oar.distrib.cachemgr.CacheExpiryCheck; import gov.nist.oar.distrib.cachemgr.ConfigurableCache; import gov.nist.oar.distrib.cachemgr.CacheManagementException; import gov.nist.oar.distrib.cachemgr.CacheVolume; +import gov.nist.oar.distrib.cachemgr.StorageInventoryDB; import gov.nist.oar.distrib.cachemgr.storage.AWSS3CacheVolume; import gov.nist.oar.distrib.cachemgr.storage.FilesystemCacheVolume; import gov.nist.oar.distrib.cachemgr.VolumeStatus; @@ -484,7 +486,11 @@ public PDRCacheManager createCacheManager(BasicCache cache, PDRDatasetRestorer r throw new ConfigurationException(rootdir+": Not an existing directory"); List checks = new ArrayList(); + // Get the StorageInventoryDB from the cache and add the CacheExpiryCheck to the list of checks + StorageInventoryDB inventoryDB = cache.getInventoryDB(); + checks.add(new CacheExpiryCheck(inventoryDB)); checks.add(new ChecksumCheck(false, true)); + PDRCacheManager out = new PDRCacheManager(cache, rstr, checks, getCheckDutyCycle()*1000, getCheckGracePeriod()*1000, -1, rootdir, logger); if (getMonitorAutoStart()) { diff --git a/src/main/java/gov/nist/oar/distrib/web/NISTDistribServiceConfig.java b/src/main/java/gov/nist/oar/distrib/web/NISTDistribServiceConfig.java index 05586a2c..bdca6877 100644 --- a/src/main/java/gov/nist/oar/distrib/web/NISTDistribServiceConfig.java +++ b/src/main/java/gov/nist/oar/distrib/web/NISTDistribServiceConfig.java @@ -197,6 +197,7 @@ else if (mode.equals("local")) } } + /** * the client for access S3 storage */ diff --git a/src/main/java/gov/nist/oar/distrib/web/RPACachingServiceProvider.java b/src/main/java/gov/nist/oar/distrib/web/RPACachingServiceProvider.java index 82f31b89..a59e81e3 100644 --- a/src/main/java/gov/nist/oar/distrib/web/RPACachingServiceProvider.java +++ b/src/main/java/gov/nist/oar/distrib/web/RPACachingServiceProvider.java @@ -177,7 +177,9 @@ else if (mode.equals("local")) public RestrictedDatasetRestorer createRPDatasetRestorer() throws ConfigurationException, IOException, CacheManagementException { - return new RestrictedDatasetRestorer(pubstore, getRPBagStorage(), getHeadBagCacheManager()); + RestrictedDatasetRestorer rdr = new RestrictedDatasetRestorer(pubstore, getRPBagStorage(), getHeadBagCacheManager()); + rdr.setExpiryTime(rpacfg.getExpiresAfterMillis()); + return rdr; // return new PDRDatasetRestorer(getRPBagStorage(), getHeadBagCacheManager()); } diff --git a/src/main/java/gov/nist/oar/distrib/web/RPAConfiguration.java b/src/main/java/gov/nist/oar/distrib/web/RPAConfiguration.java index 1bc75b27..1b4c55a6 100644 --- a/src/main/java/gov/nist/oar/distrib/web/RPAConfiguration.java +++ b/src/main/java/gov/nist/oar/distrib/web/RPAConfiguration.java @@ -56,10 +56,10 @@ public class RPAConfiguration { String mode = null; @JsonProperty("disallowedEmails") private List disallowedEmails = new ArrayList<>(); - @JsonProperty("disallowedCountries") private List disallowedCountries = new ArrayList<>(); - + @JsonProperty("expiresAfterMillis") + long expiresAfterMillis = 0L; public long getHeadbagCacheSize() { return hbCacheSize; @@ -82,6 +82,14 @@ public void setBagstoreMode(String mode) { this.mode = mode; } + public long getExpiresAfterMillis() { + return expiresAfterMillis; + } + + public void setExpiresAfterMillis(long expiresAfterMillis) { + this.expiresAfterMillis = expiresAfterMillis; + } + public SalesforceJwt getSalesforceJwt() { return salesforceJwt; } diff --git a/src/main/java/gov/nist/oar/distrib/web/RPADataCachingController.java b/src/main/java/gov/nist/oar/distrib/web/RPADataCachingController.java index 4c48ecfe..e9418f2b 100644 --- a/src/main/java/gov/nist/oar/distrib/web/RPADataCachingController.java +++ b/src/main/java/gov/nist/oar/distrib/web/RPADataCachingController.java @@ -12,6 +12,7 @@ import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.ExceptionHandler; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; @@ -148,6 +149,7 @@ public Map retrieveMetadata(@PathVariable("cacheid") String cach return metadata; } + @ExceptionHandler(MetadataNotFoundException.class) @ResponseStatus(HttpStatus.NOT_FOUND) public ErrorInfo handleMetadataNotFoundException(MetadataNotFoundException ex) { diff --git a/src/main/java/gov/nist/oar/distrib/web/RPARequestHandlerController.java b/src/main/java/gov/nist/oar/distrib/web/RPARequestHandlerController.java index 44bd6022..d61175c0 100644 --- a/src/main/java/gov/nist/oar/distrib/web/RPARequestHandlerController.java +++ b/src/main/java/gov/nist/oar/distrib/web/RPARequestHandlerController.java @@ -1,7 +1,7 @@ package gov.nist.oar.distrib.web; import gov.nist.oar.distrib.service.RPACachingService; -import gov.nist.oar.distrib.service.rpa.IRPARequestHandler; +import gov.nist.oar.distrib.service.rpa.RPARequestHandler; import gov.nist.oar.distrib.service.rpa.exceptions.InvalidRequestException; import gov.nist.oar.distrib.service.rpa.exceptions.RecaptchaClientException; import gov.nist.oar.distrib.service.rpa.exceptions.RecaptchaServerException; @@ -62,7 +62,7 @@ public class RPARequestHandlerController { /** * The primary service for handling RPA requests. */ - IRPARequestHandler service = null; + RPARequestHandler service = null; /** * The sanitizer for incoming requests. @@ -114,7 +114,7 @@ public RPARequestHandlerController(RPAServiceProvider rpaServiceProvider, RPACachingService cachingService) { if (cachingService != null && rpaServiceProvider != null) - this.service = rpaServiceProvider.getIRPARequestHandler(cachingService); + this.service = rpaServiceProvider.getRPARequestHandler(cachingService); this.requestSanitizer = new RequestSanitizer(); this.configuration = rpaServiceProvider.getRpaConfiguration(); this.jwtTokenValidator = new JwtTokenValidator(this.configuration); @@ -346,8 +346,8 @@ public ResponseEntity updateRecord(@PathVariable String id, @RequestBody RecordP LOGGER.debug("Missing required claim detected: " + missingClaimName); throw new InvalidRequestException("JWT token invalid"); } catch (JwtException e) { - LOGGER.debug("Token validation failed due to JwtException: " + e.getMessage()); - throw new RequestProcessingException("JWT token validation failed"); + LOGGER.debug("Token validation failed due to a JwtException: " + e.getMessage()); + throw new UnauthorizedException("JWT token validation failed"); } if (tokenDetails != null) { diff --git a/src/main/java/gov/nist/oar/distrib/web/RPAServiceProvider.java b/src/main/java/gov/nist/oar/distrib/web/RPAServiceProvider.java index 6466351e..fb3300b4 100644 --- a/src/main/java/gov/nist/oar/distrib/web/RPAServiceProvider.java +++ b/src/main/java/gov/nist/oar/distrib/web/RPAServiceProvider.java @@ -1,12 +1,8 @@ package gov.nist.oar.distrib.web; import gov.nist.oar.distrib.service.RPACachingService; -import gov.nist.oar.distrib.service.rpa.DefaultRPARequestHandlerService; import gov.nist.oar.distrib.service.rpa.HttpURLConnectionRPARequestHandlerService; -import gov.nist.oar.distrib.service.rpa.IRPARequestHandler; -import gov.nist.oar.distrib.service.rpa.RPARequestHandlerService; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.web.client.RestTemplate; +import gov.nist.oar.distrib.service.rpa.RPARequestHandler; public class RPAServiceProvider { RPAConfiguration rpaConfiguration; @@ -15,19 +11,11 @@ public RPAServiceProvider(RPAConfiguration rpaConfiguration) { this.rpaConfiguration = rpaConfiguration; } - public RPARequestHandlerService getRPARequestHandlerService(RestTemplate restTemplate) { - return this.getDefaultRPARequestHandlerService(restTemplate); - } - - private DefaultRPARequestHandlerService getDefaultRPARequestHandlerService(RestTemplate restTemplate) { - return new DefaultRPARequestHandlerService(this.rpaConfiguration, restTemplate); - } - - public IRPARequestHandler getIRPARequestHandler(RPACachingService rpaCachingService) { + public RPARequestHandler getRPARequestHandler(RPACachingService rpaCachingService) { return this.getHttpURLConnectionRPARequestHandler(rpaCachingService); } - private IRPARequestHandler getHttpURLConnectionRPARequestHandler(RPACachingService rpaCachingService) { + private RPARequestHandler getHttpURLConnectionRPARequestHandler(RPACachingService rpaCachingService) { return new HttpURLConnectionRPARequestHandlerService(this.rpaConfiguration, rpaCachingService); } diff --git a/src/test/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheckTest.java b/src/test/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheckTest.java new file mode 100644 index 00000000..0f0c61da --- /dev/null +++ b/src/test/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheckTest.java @@ -0,0 +1,151 @@ +package gov.nist.oar.distrib.cachemgr; + +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +import java.time.Instant; + +import static org.mockito.Mockito.anyString; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class CacheExpiryCheckTest { + + @Mock + private StorageInventoryDB mockInventoryDB; + @Mock + private CacheVolume mockVolume; + @Mock + private CacheObject cacheObject; + private CacheExpiryCheck expiryCheck; + + @Before + public void setUp() { + MockitoAnnotations.initMocks(this); + expiryCheck = new CacheExpiryCheck(mockInventoryDB); + } + + /** + * Test to verify that {@link CacheExpiryCheck} correctly identifies and processes an expired cache object. + * An object is considered expired based on the `expires` metadata, which defines the duration after which + * an object should be considered expired from the time of its last modification. This test ensures that an object + * past its expiration is appropriately removed from the inventory database. + * + * @throws Exception to handle any exceptions thrown during the test execution + */ + @Test + public void testExpiredObjectRemoval() throws Exception { + // Setup an expired cache object + cacheObject.volume = mockVolume; + when(cacheObject.hasMetadatum("expires")).thenReturn(true); + when(cacheObject.getMetadatumLong("expires", -1L)).thenReturn(1000L); // Expires in 1 second + when(cacheObject.getLastModified()).thenReturn(Instant.now().minusSeconds(10).toEpochMilli()); + when(cacheObject.volume.remove(cacheObject.name)).thenReturn(true); + + expiryCheck.check(cacheObject); + + // Verify removeObject effect + verify(mockInventoryDB).removeObject(cacheObject.volname, cacheObject.name); + } + + + /** + * Test to ensure that {@link CacheExpiryCheck} does not flag a cache object as expired if the current time has not + * exceeded its `expires` duration since its last modification. This test verifies that no removal action is taken + * for such non-expired objects. + * + * @throws Exception to handle any exceptions thrown during the test execution + */ + @Test + public void testNonExpiredObject() throws Exception { + // Setup mock + cacheObject.name = "nonExpiredObject"; + cacheObject.volname = "testVolume"; + when(cacheObject.hasMetadatum("expires")).thenReturn(true); + when(cacheObject.getMetadatumLong("expires", -1L)).thenReturn(14 * 24 * 60 * 60 * 1000L); // 14 days in milliseconds + long lastModified = System.currentTimeMillis() - (7 * 24 * 60 * 60 * 1000L); // 7 days ago, within expiry period + when(cacheObject.getLastModified()).thenReturn(lastModified); + + // Perform the check + expiryCheck.check(cacheObject); + + // Verify that the remove method was not called as the object is not expired + verify(mockInventoryDB, never()).removeObject(cacheObject.volname, cacheObject.name); + } + + /** + * Tests that no action is taken and no exception is thrown for a cache object without the {@code expires} metadata. + * This verifies that the absence of {@code expires} metadata does not trigger any removal process or result in an error. + * + * @throws Exception to handle any exceptions thrown during the test execution + */ + @Test + public void testObjectWithoutExpiresMetadata_NoActionTaken() throws Exception { + cacheObject.name = "objectWithoutExpires"; + cacheObject.volname = "testVolume"; + when(cacheObject.hasMetadatum("expires")).thenReturn(false); + + expiryCheck.check(cacheObject); + + verify(mockInventoryDB, never()).removeObject(anyString(), anyString()); + } + + /** + * Test to ensure that a cache object with an expiration date in the future is not removed from the cache. + * This test verifies the {@code check} method's correct behavior in handling non-expired objects based + * on the {@code expires} metadata. + * + * @throws Exception to handle any exceptions thrown during the test execution. + */ + @Test + public void testNonExpiredObject_NoRemoval() throws Exception { + // Setup a non-expired cache object + when(cacheObject.hasMetadatum("expires")).thenReturn(true); + when(cacheObject.getMetadatumLong("expires", -1L)).thenReturn(System.currentTimeMillis() + 10000L); // Expires in the future + when(cacheObject.getLastModified()).thenReturn(System.currentTimeMillis()); + + expiryCheck.check(cacheObject); + + // Verify no removal happens + verify(mockInventoryDB, never()).removeObject(anyString(), anyString()); + } + + + /** + * Tests that an {@link IntegrityException} is thrown when a cache object has the {@code expires} metadata + * but lacks a valid {@code lastModified} time. + * + * @throws Exception to handle any exceptions thrown during the test execution + */ + @Test(expected = IntegrityException.class) + public void testObjectWithExpiresButNoLastModified_ThrowsException() throws Exception { + cacheObject.name = "objectWithNoLastModified"; + cacheObject.volname = "testVolume"; + when(cacheObject.hasMetadatum("expires")).thenReturn(true); + when(cacheObject.getMetadatumLong("expires", -1L)).thenReturn(1000L); // Expires in 1 second + when(cacheObject.getLastModified()).thenReturn(-1L); // Last modified not available + + expiryCheck.check(cacheObject); + } + + /** + * Test to verify that no action is taken for a cache object missing the {@code expires} metadata. + * This test ensures that the absence of {@code expires} metadata does not trigger any removal or error. + * + * @throws Exception to handle any exceptions thrown during the test execution. + */ + @Test + public void testObjectWithoutExpires_NoAction() throws Exception { + // Setup an object without expires metadata + when(cacheObject.hasMetadatum("expires")).thenReturn(false); + + expiryCheck.check(cacheObject); + + // Verify no action is taken + verify(mockInventoryDB, never()).removeObject(anyString(), anyString()); + } + +} diff --git a/src/test/java/gov/nist/oar/distrib/cachemgr/pdr/RestrictedDatasetRestorerTest.java b/src/test/java/gov/nist/oar/distrib/cachemgr/pdr/RestrictedDatasetRestorerTest.java index dd11fadf..2de7d871 100644 --- a/src/test/java/gov/nist/oar/distrib/cachemgr/pdr/RestrictedDatasetRestorerTest.java +++ b/src/test/java/gov/nist/oar/distrib/cachemgr/pdr/RestrictedDatasetRestorerTest.java @@ -406,5 +406,58 @@ public void testCacheFromBag() } + @Test + public void testExpires() throws StorageVolumeException, ResourceNotFoundException, CacheManagementException { + + assertTrue(! cache.isCached("mds1491/trial1.json")); + assertTrue(! cache.isCached("mds1491/trial2.json")); + assertTrue(! cache.isCached("mds1491/trial3/trial3a.json")); + assertTrue(! cache.isCached("mds1491/README.txt")); + + Set cached = rstr.cacheDataset("mds1491", null, cache, true, + PDRCacheRoles.ROLE_RESTRICTED_DATA, null); + assertTrue(cached.contains("trial1.json")); + assertTrue(cached.contains("trial2.json")); + assertTrue(cached.contains("README.txt")); + assertTrue(cached.contains("trial3/trial3a.json")); + assertEquals(4, cached.size()); + + assertTrue(cache.isCached("mds1491/trial1.json")); + assertTrue(cache.isCached("mds1491/trial2.json")); + assertTrue(cache.isCached("mds1491/README.txt")); + assertTrue(cache.isCached("mds1491/trial3/trial3a.json")); + + List found = cache.getInventoryDB().findObject("mds1491/trial1.json", VolumeStatus.VOL_FOR_INFO); + assertEquals(1, found.size()); + assertTrue("CacheObject should contain expires metadata", found.get(0).hasMetadatum("expires")); + + // Verify the "expires" value is approximately 2 weeks from the current time + long TWO_WEEKS_MILLIS = 14L * 24 * 60 * 60 * 1000; + long expectedExpires = System.currentTimeMillis() + TWO_WEEKS_MILLIS; + long actualExpires = found.get(0).getMetadatumLong("expires", 0); + // Check that the absolute difference between the expected and actual expires values is less than 1000ms + assertTrue("expires field should be set to 2 weeks from the current time", + Math.abs(expectedExpires - actualExpires) < 1000); + + found = cache.getInventoryDB().findObject("mds1491/trial2.json", VolumeStatus.VOL_FOR_INFO); + assertEquals(1, found.size()); + assertTrue("CacheObject should contain expires metadata", found.get(0).hasMetadatum("expires")); + + expectedExpires= System.currentTimeMillis() + TWO_WEEKS_MILLIS; + actualExpires = found.get(0).getMetadatumLong("expires", 0); + assertTrue("expires field should be set to 2 weeks from the current time", + Math.abs(expectedExpires - actualExpires) < 1000); + + found = cache.getInventoryDB().findObject("mds1491/README.txt", VolumeStatus.VOL_FOR_INFO); + assertEquals(1, found.size()); + assertTrue("CacheObject should contain expires metadata", found.get(0).hasMetadatum("expires")); + + expectedExpires = System.currentTimeMillis() + TWO_WEEKS_MILLIS; + actualExpires = found.get(0).getMetadatumLong("expires", 0); + assertTrue("expires field should be set to 2 weeks from the current time", + Math.abs(expectedExpires - actualExpires) < 1000); + + } + } diff --git a/src/test/java/gov/nist/oar/distrib/service/RPACachingServiceTest.java b/src/test/java/gov/nist/oar/distrib/service/RPACachingServiceTest.java index c3d0f108..52829497 100644 --- a/src/test/java/gov/nist/oar/distrib/service/RPACachingServiceTest.java +++ b/src/test/java/gov/nist/oar/distrib/service/RPACachingServiceTest.java @@ -52,8 +52,8 @@ public void testCacheAndGenerateRandomId_validDatasetID() throws Exception { String result = rpaCachingService.cacheAndGenerateRandomId(datasetID, version); assertNotNull(result); - assertEquals(RPACachingService.RANDOM_ID_LENGTH, result.length()); - assertTrue(result.matches("^[a-zA-Z0-9]*$")); // check that the ID is alphanumeric + assertEquals(RPACachingService.RANDOM_ID_LENGTH + 4, result.length()); // 4 for the 'rpa-' prefix + assertTrue(result.matches("^rpa-[a-zA-Z0-9]+$")); // Check that the ID starts with 'rpa-' followed by alphanumeric chars verify(pdrCacheManager).cacheDataset(eq("mds2-2909"), eq(version), eq(true), eq(RPACachingService.ROLE_RESTRICTED_DATA), eq(result)); } @@ -69,8 +69,8 @@ public void testCacheAndGenerateRandomId_validDatasetArkID() throws Exception { String result = rpaCachingService.cacheAndGenerateRandomId(datasetID, version); assertNotNull(result); - assertEquals(RPACachingService.RANDOM_ID_LENGTH, result.length()); - assertTrue(result.matches("^[a-zA-Z0-9]*$")); // check that the ID is alphanumeric + assertEquals(RPACachingService.RANDOM_ID_LENGTH + 4, result.length()); // 4 for the 'rpa-' prefix + assertTrue(result.matches("^rpa-[a-zA-Z0-9]+$")); // Check that the ID starts with 'rpa-' followed by alphanumeric chars verify(pdrCacheManager).cacheDataset(eq("mds2-2909"), eq(version), eq(true), eq(RPACachingService.ROLE_RESTRICTED_DATA), eq(result)); } @@ -85,6 +85,7 @@ public void testCacheAndGenerateRandomId_invalidDatasetArkID() throws Exception @Test public void testRetrieveMetadata_success() throws Exception { String randomID = "randomId123"; + String aipid = "456"; CacheObject cacheObject1 = new CacheObject("object1", new JSONObject() .put("filepath", "path/to/file1.txt") .put("contentType", "text/plain") @@ -95,7 +96,7 @@ public void testRetrieveMetadata_success() throws Exception { .put("checksum", "abc123") .put("version", "v1") .put("ediid", "123") - .put("aipid", "456") + .put("aipid", aipid) .put("sinceDate", "08-05-2023"), "Volume1"); @@ -109,7 +110,7 @@ public void testRetrieveMetadata_success() throws Exception { .put("checksum", "def456") .put("version", "v2") .put("ediid", "123") - .put("aipid", "456") + .put("aipid", aipid) .put("sinceDate", "08-05-2023"), "Volume1"); @@ -124,7 +125,8 @@ public void testRetrieveMetadata_success() throws Exception { Map expected = new HashMap<>(); expected.put("randomId", randomID); expected.put("metadata", new JSONArray() - .put(new JSONObject().put("downloadURL", testBaseDownloadUrl + "/" + randomID + "/path/to/file1.txt") + .put(new JSONObject().put("downloadURL", testBaseDownloadUrl + "/" + randomID + + "/" + aipid + "/path/to/file1.txt") .put("filePath", "path/to/file1.txt") .put("mediaType", "text/plain") .put("size", 100L) @@ -136,7 +138,8 @@ public void testRetrieveMetadata_success() throws Exception { .put("ediid", "123") .put("aipid", "456") .put("sinceDate", "08-05-2023")) - .put(new JSONObject().put("downloadURL", testBaseDownloadUrl + "/" + randomID + "/path/to/file2.txt") + .put(new JSONObject().put("downloadURL", testBaseDownloadUrl + "/" + randomID + + "/" + aipid + "/path/to/file2.txt") .put("filePath", "path/to/file2.txt") .put("mediaType", "text/plain") .put("size", 100L) @@ -159,6 +162,7 @@ public void testRetrieveMetadata_success() throws Exception { @Test public void testRetrieveMetadata_withMissingFilepath() throws Exception { String randomID = "randomId123"; + String aipid = "456"; CacheObject cacheObject1 = new CacheObject("object1", new JSONObject() .put("contentType", "text/plain") .put("size", 100L) @@ -168,7 +172,7 @@ public void testRetrieveMetadata_withMissingFilepath() throws Exception { .put("checksum", "abc123") .put("version", "v1") .put("ediid", "123") - .put("aipid", "456") + .put("aipid", aipid) .put("sinceDate", "08-05-2023"), "Volume1"); @@ -182,7 +186,7 @@ public void testRetrieveMetadata_withMissingFilepath() throws Exception { .put("checksum", "def456") .put("version", "v2") .put("ediid", "123") - .put("aipid", "456") + .put("aipid", aipid) .put("sinceDate", "08-05-2023"), "Volume1"); @@ -197,7 +201,8 @@ public void testRetrieveMetadata_withMissingFilepath() throws Exception { Map expected = new HashMap<>(); expected.put("randomId", randomID); expected.put("metadata", new JSONArray() - .put(new JSONObject().put("downloadURL", testBaseDownloadUrl + "/" + randomID + "/path/to/file2.txt") + .put(new JSONObject().put("downloadURL", testBaseDownloadUrl + "/" + randomID + + "/" + aipid +"/path/to/file2.txt") .put("filePath", "path/to/file2.txt") .put("mediaType", "text/plain") .put("size", 100L) @@ -207,7 +212,7 @@ public void testRetrieveMetadata_withMissingFilepath() throws Exception { .put("checksum", "def456") .put("version", "v2") .put("ediid", "123") - .put("aipid", "456") + .put("aipid", aipid) .put("sinceDate", "08-05-2023")) .toList()); diff --git a/src/test/java/gov/nist/oar/distrib/service/rpa/HttpURLConnectionRPARequestHandlerServiceTest.java b/src/test/java/gov/nist/oar/distrib/service/rpa/HttpURLConnectionRPARequestHandlerServiceTest.java index 94ae38e7..10d6a214 100644 --- a/src/test/java/gov/nist/oar/distrib/service/rpa/HttpURLConnectionRPARequestHandlerServiceTest.java +++ b/src/test/java/gov/nist/oar/distrib/service/rpa/HttpURLConnectionRPARequestHandlerServiceTest.java @@ -45,6 +45,7 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; @@ -114,6 +115,9 @@ public class HttpURLConnectionRPARequestHandlerServiceTest { @Mock RPACachingService rpaCachingService; + @Mock + RPADatasetCacher rpaDatasetCacher; + private HttpURLConnectionRPARequestHandlerService service; JWTToken testToken = null; Map map = new HashMap() {{ @@ -130,6 +134,7 @@ public void setUp() { service.setRecaptchaHelper(recaptchaHelper); service.setHttpURLConnectionFactory(url -> mockConnection); service.setRecordResponseHandler(recordResponseHandler); + service.seRPADatasetCacher(rpaDatasetCacher); service.setHttpClient(mockHttpClient); // Set up mock behavior for mockJwtHelper @@ -615,22 +620,46 @@ private String getUpdateUrl(String recordId) { return url; } + /** + * Tests successful record update operation when approving a record. + * + *

+ * This test simulates the scenario where a record is approved, ensuring that the + * caching process is invoked, a PATCH request is made to update the record status in + * Salesforce with a newly generated random ID, and the approval status is updated correctly. + * The test verifies that the final approval status matches the expected format. + * 

+     * Status_YYYY-MM-DDTHH:MM:SS.SSSZ_email_randomID
+     *
+ * Where: + *
    + *
  • Status - Indicates the status of the record (e.g., "Approved" or "Declined").
    • + *
  • YYYY-MM-DDTHH:MM:SS.SSSZ - The timestamp of the approval in ISO 8601 format. 'T' separates the date and time, and 'Z' denotes UTC time zone.
    • + *
  • email - The email address associated with the user who approved the record.
    • + *
  • randomID - A unique identifier generated for the approval process or the record itself.
    • + *
+ *

+ * + * @throws Exception if any error occurs during the test execution. + */ @Test public void testUpdateRecord_success() throws Exception { String recordId = "record12345"; String email = "test@example.com"; String country = "United States"; - String expectedApprovalStatus = "Approved_2023-05-09T15:59:03.872Z_" + email; - + String mockRandomId = "mockRandomId123"; + String expectedApprovalStatus = "Approved_2023-05-09T15:59:03.872Z_" + email + "_" + mockRandomId; // Mock behavior of getRecord method doReturn(getTestRecordWrapper(expectedApprovalStatus, email, country)).when(service).getRecord("record12345"); + when(rpaDatasetCacher.cache(anyString())).thenReturn(mockRandomId); + // Mock HttpResponse CloseableHttpResponse httpResponse = mock(CloseableHttpResponse.class); when(httpResponse.getStatusLine()).thenReturn(mock(StatusLine.class)); when(httpResponse.getStatusLine().getStatusCode()).thenReturn(200); when(httpResponse.getEntity()).thenReturn( - new StringEntity("{\"approvalStatus\":\"Approved_2023-05-09T15:59:03.872Z_test@example.com\"," + + new StringEntity("{\"approvalStatus\":\"Approved_2023-05-09T15:59:03.872Z_test@example.com_mockRandomId123\"," + "\"recordId\":\"record12345\"}", ContentType.APPLICATION_JSON) ); @@ -656,20 +685,35 @@ public void testUpdateRecord_success() throws Exception { // We can't test the exact time as it changes when we run the test, but we can verify the format String patchPayload = EntityUtils.toString(patchRequest.getEntity(), StandardCharsets.UTF_8); JSONObject payloadObject = new JSONObject(patchPayload); - // Pattern to match ISO 8601 format - // This pattern matches a string in the format "Approved_YYYY-MM-DDTHH:MM:SS.SSSZ" - String expectedFormat = "Approved_\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z_[\\w.-]+@[\\w.-]+\\.\\w+"; + // The following regex pattern expects: + // - The "Approved" status followed by a date-time in ISO 8601 format. + // - An email address. + // - A random ID (composed of word characters including underscore, alphanumeric, and possibly -) at the end. + String expectedFormat = "Approved_\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z_[\\w.-]+@[\\w.-]+\\.\\w+_\\w+"; assertTrue(payloadObject.get("Approval_Status__c").toString().matches(expectedFormat)); } + /** + * Tests the updateRecord method's behavior when an unknown status is provided. + *

+ * This test ensures that the method throws an InvalidRequestException when attempting to + * update a record with an unrecognized status. The expected behavior is to validate the + * status input and throw an exception with a specific error message if the status does not + * match expected values (e.g., "Approved" or "Declined"). + *

+ */ @Test public void testUpdateRecord_withUnknownStatus() { String recordId = "record12345"; String status = "HelloWorld"; String email = "test@example.com"; + String country = "United States"; String expectedErrorMessage = "Invalid approval status: HelloWorld"; + // Mock behavior of getRecord method + doReturn(getTestRecordWrapper("Pending", email, country)).when(service).getRecord("record12345"); + // Call the method and catch the exception try { // Act @@ -681,5 +725,99 @@ public void testUpdateRecord_withUnknownStatus() { } } + /** + * Tests the record decline operation for a record that has not been previously approved. + * + *

+ * This test case ensures that when a record is declined without prior approval, the record's + * status is updated accordingly without triggering caching or uncaching operations. It verifies + * the successful update of the record's approval status to "Declined" and confirms that no + * caching or uncaching methods are called, as expected for records not previously approved. + *

+ * + * @throws Exception if any error occurs during the test execution. + */ + @Test + public void testDeclineRecordWithoutPriorApproval_success() throws Exception { + String recordId = "record12345"; + String email = "sme@test.com"; + String status = "Declined"; + String country = "United States"; + // Mock behavior of getRecord method to simulate a record that has not been approved before + doReturn(getTestRecordWrapper("Pending", email, country)).when(service).getRecord(recordId); + + // Mock HttpResponse for the decline operation + CloseableHttpResponse httpResponse = mock(CloseableHttpResponse.class); + when(httpResponse.getStatusLine()).thenReturn(mock(StatusLine.class)); + when(httpResponse.getStatusLine().getStatusCode()).thenReturn(200); + when(httpResponse.getEntity()).thenReturn( + new StringEntity("{\"approvalStatus\":\"Declined\",\"recordId\":\"" + recordId + "\"}", + ContentType.APPLICATION_JSON)); + + // Mock the HttpPatch execution + doReturn(httpResponse).when(mockHttpClient).execute(any(HttpPatch.class)); + + // Act + RecordStatus result = service.updateRecord(recordId, status, email); + + // Assert + assertEquals("Declined", result.getApprovalStatus()); + assertEquals(recordId, result.getRecordId()); + + // Verify that caching and uncaching were not invoked + verify(rpaDatasetCacher, never()).cache(anyString()); + verify(rpaDatasetCacher, never()).uncache(anyString()); + } + + /** + * Tests the decline operation for a record that was previously approved. + * + *

+ * This test checks the behavior of the updateRecord method when declining a record that + * has a prior approval status, including a random ID. It simulates retrieving a previously + * approved record, uncaching the dataset associated with the random ID, and updating the + * record's approval status to "Declined". The test verifies that the uncaching operation + * is executed with the correct random ID and that the record's status is correctly updated. + *

+ * + * @throws Exception if any error occurs during the test execution. + */ + @Test + public void testDeclinePreviouslyApprovedRecord_success() throws Exception { + String recordId = "record12345"; + String email = "sme@test.com"; + String status = "Declined"; + String mockRandomId = "mockRandomId123"; + String country = "United States"; + String initialApprovalStatus = "Approved_2023-05-09T15:59:03.872Z_" + email + "_" + mockRandomId; + + // Mock behavior of getRecord method to simulate retrieving a previously approved record + doReturn(getTestRecordWrapper(initialApprovalStatus, email, country)).when(service).getRecord(recordId); + + // Simulate `uncache` returning true + when(rpaDatasetCacher.uncache(anyString())).thenReturn(true); + + // Mock HttpResponse for updating the record to "Declined" + CloseableHttpResponse httpResponse = mock(CloseableHttpResponse.class); + when(httpResponse.getStatusLine()).thenReturn(mock(StatusLine.class)); + when(httpResponse.getStatusLine().getStatusCode()).thenReturn(200); + when(httpResponse.getEntity()).thenReturn( + new StringEntity("{\"approvalStatus\":\"Declined\",\"recordId\":\"" + recordId + "\"}", + ContentType.APPLICATION_JSON)); + + // Mock the HttpPatch execution + doReturn(httpResponse).when(mockHttpClient).execute(any(HttpPatch.class)); + + // Act + RecordStatus result = service.updateRecord(recordId, status, email); + + // Assert + assertEquals("Declined", result.getApprovalStatus()); + assertEquals(recordId, result.getRecordId()); + + // Verify uncaching was invoked with the correct random ID + verify(rpaDatasetCacher).uncache(mockRandomId); + } + } diff --git a/src/test/java/gov/nist/oar/distrib/web/CacheManagementControllerTest.java b/src/test/java/gov/nist/oar/distrib/web/CacheManagementControllerTest.java index 9aac4d53..02e05598 100644 --- a/src/test/java/gov/nist/oar/distrib/web/CacheManagementControllerTest.java +++ b/src/test/java/gov/nist/oar/distrib/web/CacheManagementControllerTest.java @@ -404,6 +404,44 @@ public void testStartMonitor() throws ConfigurationException { assertFalse("Monitor failed to finish?", status.getBoolean("running")); } + @Test + public void testRemoveFromCache() { + HttpEntity req = new HttpEntity<>(null, headers); + + // First, ensure the dataset or file initially exists in the cache by sending a GET request + // This is a check to confirm presence before deletion + ResponseEntity resp = websvc.exchange(getBaseURL() + "/cache/objects/mds1491/:cached", + HttpMethod.GET, req, String.class); + assertEquals(HttpStatus.OK, resp.getStatusCode()); + + // Attempt to remove a specific file (trial2.json) from the cache using the DELETE method + // The file path and the ':cached' selector are included in the URL + ResponseEntity removeResp = websvc.exchange(getBaseURL() + + "/cache/objects/mds1491/trial2.json/:cached", + HttpMethod.DELETE, req, String.class); + assertEquals(HttpStatus.OK, removeResp.getStatusCode()); + + // After deletion, verify that the specific file (trial2.json) is no longer accessible + // Expect a NOT_FOUND status code to confirm successful deletion + resp = websvc.exchange(getBaseURL() + "/cache/objects/mds1491/trial2.json/:cached", + HttpMethod.GET, req, String.class); + assertEquals(HttpStatus.NOT_FOUND, resp.getStatusCode()); + + // Now, attempt to remove the entire dataset ('mds1491') from the cache + // This tests the capability to delete all files under a dataset identifier + removeResp = websvc.exchange(getBaseURL() + + "/cache/objects/mds1491/:cached", + HttpMethod.DELETE, req, String.class); + assertEquals(HttpStatus.OK, removeResp.getStatusCode()); + + // Confirm that the entire dataset ('mds1491') is no longer available in the cache + // A GET request should return a NOT_FOUND status, meaning the dataset has been successfully removed from cache + resp = websvc.exchange(getBaseURL() + "/cache/objects/mds1491/:cached", + HttpMethod.GET, req, String.class); + assertEquals(HttpStatus.NOT_FOUND, resp.getStatusCode()); + } + + private String getBaseURL() { return "http://localhost:" + port + "/od"; } diff --git a/src/test/java/gov/nist/oar/distrib/web/RPACachingServiceProviderTest.java b/src/test/java/gov/nist/oar/distrib/web/RPACachingServiceProviderTest.java index 90060b54..1c5d8ac7 100644 --- a/src/test/java/gov/nist/oar/distrib/web/RPACachingServiceProviderTest.java +++ b/src/test/java/gov/nist/oar/distrib/web/RPACachingServiceProviderTest.java @@ -1,5 +1,6 @@ package gov.nist.oar.distrib.web; +import gov.nist.oar.distrib.cachemgr.pdr.RestrictedDatasetRestorer; import gov.nist.oar.distrib.storage.FilesystemLongTermStorage; import gov.nist.oar.distrib.BagStorage; import gov.nist.oar.distrib.cachemgr.CacheManagementException; @@ -81,6 +82,7 @@ public void testGetHeadBagManager() public void testCreateRPDatasetRestorer() throws ConfigurationException, IOException, CacheManagementException { - PDRDatasetRestorer rest = prov.createRPDatasetRestorer(); + RestrictedDatasetRestorer rest = prov.createRPDatasetRestorer(); + assertEquals(rest.getExpiryTime(), 1209600000L); } } diff --git a/src/test/java/gov/nist/oar/distrib/web/RPAConfigurationTest.java b/src/test/java/gov/nist/oar/distrib/web/RPAConfigurationTest.java index b23e57be..55754187 100644 --- a/src/test/java/gov/nist/oar/distrib/web/RPAConfigurationTest.java +++ b/src/test/java/gov/nist/oar/distrib/web/RPAConfigurationTest.java @@ -33,5 +33,6 @@ public void testLoadConfig() throws IOException { assertEquals(2, config.getDisallowedCountries().size()); assertTrue(config.getDisallowedCountries().contains("Cuba")); assertTrue(config.getDisallowedCountries().contains("North Korea")); + assertEquals(1209600000L, config.getExpiresAfterMillis()); } } diff --git a/src/test/java/gov/nist/oar/distrib/web/RPARequestHandlerControllerTest.java b/src/test/java/gov/nist/oar/distrib/web/RPARequestHandlerControllerTest.java index 5b5a14e5..5e8e1f6f 100644 --- a/src/test/java/gov/nist/oar/distrib/web/RPARequestHandlerControllerTest.java +++ b/src/test/java/gov/nist/oar/distrib/web/RPARequestHandlerControllerTest.java @@ -2,7 +2,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import gov.nist.oar.distrib.service.RPACachingService; -import gov.nist.oar.distrib.service.rpa.IRPARequestHandler; +import gov.nist.oar.distrib.service.rpa.RPARequestHandler; import gov.nist.oar.distrib.service.rpa.RecaptchaHelper; import gov.nist.oar.distrib.service.rpa.exceptions.InvalidRequestException; import gov.nist.oar.distrib.service.rpa.exceptions.RecaptchaVerificationFailedException; @@ -51,7 +51,7 @@ @RunWith(MockitoJUnitRunner.class) public class RPARequestHandlerControllerTest { @Mock - private IRPARequestHandler service; + private RPARequestHandler service; @Mock RPAServiceProvider mockRPAServiceProvider; @@ -79,7 +79,7 @@ public class RPARequestHandlerControllerTest { @Before public void setup() { MockitoAnnotations.initMocks(this); - when(mockRPAServiceProvider.getIRPARequestHandler(mockRPACachingService)).thenReturn(service); + when(mockRPAServiceProvider.getRPARequestHandler(mockRPACachingService)).thenReturn(service); // create a test instance of the RPARequestHandlerController class controller = new RPARequestHandlerController(mockRPAServiceProvider, mockRPACachingService); controller.setRequestSanitizer(mockRequestSanitizer); diff --git a/src/test/resources/rpaconfig.json b/src/test/resources/rpaconfig.json index f7815b0c..3630f198 100644 --- a/src/test/resources/rpaconfig.json +++ b/src/test/resources/rpaconfig.json @@ -11,5 +11,6 @@ "disallowedCountries": [ "Cuba", "North Korea" - ] + ], + "expiresAfterMillis": 1209600000 }