Merge pull request #102 from usnistgov/feature/rpa_uncaching

Enhancing RPACachingService with uncaching capability

docker/build-test/Dockerfile

@@ -1,3 +1,20 @@
+FROM eclipse-temurin:8-jdk-focal 
+
+RUN mkdir -p /usr/share/man/man1
+RUN apt-get update && apt-get install -y netcat-openbsd zip git less \
+                                            ca-certificates python3 curl maven gnupg
+RUN cd /usr/bin && ln -s python3 python
+
+COPY cacerts/README.md cacerts/*.crt /usr/local/share/ca-certificates/
+RUN update-ca-certificates
+RUN java_certs=$JAVA_HOME/jre/lib/security/cacerts; \
+    add_certs=`ls /usr/local/share/ca-certificates/*.crt` && \
+    for crt in $add_certs; do \
+        name=`basename -s .crt $crt`; \
+        echo -n ${name}: " "; \
+        keytool -import -keystore $java_certs -trustcacerts -file $crt \
+                -storepass changeit -alias $name -noprompt; \
+    done;
 FROM eclipse-temurin:8
 
 RUN mkdir -p /usr/share/man/man1

docker/build-test/cacerts/Forward_Proxy_NIST_CA.crt

@@ -0,0 +1,41 @@
+-----BEGIN CERTIFICATE-----
+MIIG7TCCBNWgAwIBAgITGAAAAAecWWKCXTfeJAAAAAAABzANBgkqhkiG9w0BAQsF
+ADAVMRMwEQYDVQQDEwpOSVNUUm9vdDAyMB4XDTIxMTIwMTE4MDU0NloXDTI2MTIw
+MTE4MTU0NlowgcIxCzAJBgNVBAYTAlVTMREwDwYDVQQIEwhNYXJ5bGFuZDEVMBMG
+A1UEBxMMR2FpdGhlcnNidXJnMTcwNQYDVQQKEy5OYXRpb25hbCBJbnN0aXR1dGUg
+b2YgU3RhbmRhcmRzIGFuZCBUZWNobm9sb2d5MQ0wCwYDVQQLEwRPSVNNMR4wHAYD
+VQQDDBVGb3J3YXJkX1Byb3h5X05JU1RfQ0ExITAfBgkqhkiG9w0BCQEWEm5ldHNl
+Y3VyZUBuaXN0LmdvdjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAL/8
+PgucU6LfbThmVCiQU5zH7HRdJ0QeM8xa9Hy3BnBdD4/CxQklo7dz+AXquaOfI5Br
+H8SYZCySWTveFeJW+XvhjmEVpobz8GGrEgdR5nAKg3ZJHvAMPKgGMSnXja227TVj
+qqCZX9cIWQifqcM1iWTkS4BW2oZazwXYCqs5dfwy92ey5f/7AYC4dFeL//QtqQs/
+EUFApYabhKLcDLleDh4hwlhbTO9Zjt/eRujB/5f183RVb+igoy/xVZ8S82cNpxHS
+2DdO58GZzvAgYMYuXXJkdINkag/fpCXEy9bGaDfydHLpTWviiGz3HfXh/Chb66BG
+ZoZJmJrovVO9rSMyptMCAwEAAaOCAoYwggKCMB0GA1UdDgQWBBQquDqJ3U24XoOQ
+/6y8kFgbAp9fPDAfBgNVHSMEGDAWgBQlEQPjYg4e56GOSdev1HJtWx0z+TCB/QYD
+VR0fBIH1MIHyMIHvoIHsoIHphjFodHRwOi8vbmlzdHBraS5uaXN0Lmdvdi9DZXJ0
+RW5yb2xsL05JU1RSb290MDIuY3JshoGzbGRhcDovLy9DTj1OSVNUUm9vdDAyLENO
+PU5JU1Ryb290Q0EwMixDTj1DRFAsQ049UHVibGljJTIwS2V5JTIwU2VydmljZXMs
+Q049U2VydmljZXMsQ049Q29uZmlndXJhdGlvbixEQz1OSVNULERDPUdPVj9jZXJ0
+aWZpY2F0ZVJldm9jYXRpb25MaXN0P2Jhc2U/b2JqZWN0Q2xhc3M9Y1JMRGlzdHJp
+YnV0aW9uUG9pbnQwggEFBggrBgEFBQcBAQSB+DCB9TBKBggrBgEFBQcwAoY+aHR0
+cDovL25pc3Rwa2kubmlzdC5nb3YvQ2VydEVucm9sbC9OSVNUcm9vdENBMDJfTklT
+VFJvb3QwMi5jcnQwgaYGCCsGAQUFBzAChoGZbGRhcDovLy9DTj1OSVNUUm9vdDAy
+LENOPUFJQSxDTj1QdWJsaWMlMjBLZXklMjBTZXJ2aWNlcyxDTj1TZXJ2aWNlcyxD
+Tj1Db25maWd1cmF0aW9uLERDPU5JU1QsREM9R09WP2NBQ2VydGlmaWNhdGU/YmFz
+ZT9vYmplY3RDbGFzcz1jZXJ0aWZpY2F0aW9uQXV0aG9yaXR5MBkGCSsGAQQBgjcU
+AgQMHgoAUwB1AGIAQwBBMA8GA1UdEwEB/wQFMAMBAf8wCwYDVR0PBAQDAgGGMA0G
+CSqGSIb3DQEBCwUAA4ICAQB3OCkcbjepVN7tbK3PlLzG5HkRBG1QSmFsRnQdUTov
+/rWhdLDpHGKO4k/W2zTxNNxPW8ooD1PCy+cIlBLGcq8YcyhvWk0V2Gx1P+/f4+eq
+eH4hcUQO/7INohcnh4QXiSVMa7jNaLC+/usqWbsmTvVDbl2aYbQtwizXnUW1qNhz
+Bt76OoM7C95rNktNiaJ1VmFmd+Z3rRhzAZiC9XFIwIN1F+um7IG43nsoM4hnCByc
+/SBb3LC8R+7vNUYedkrfNPq8SGCHuPuK8H0gJX+8/8hmaaNPtZoe0VZkTdNXitnY
+HNof6w5mDoPu9lgLmNO0c36dNrmhHlPAu71EkL3afBhrdgb4Gel0WlENaur2MWf+
+yg6IQz7+aCTu2bMIkW3gm942tp7IrkXMGshUsJjLHFVrpIVkP+70QnO0wGzzQWlI
+gt+/gKvj951KGagVzsFyiQtFL9uFYMiS0awLVkSLYtBzdykm8mpG1n6EO5DlEYWe
+MOhVSeki05s0+6zUWU6TIhVDgCeUJYvAYAtWVA07Tbb1lb1vP+KbWzFMuAQMrKXV
+I0sL/gjcwaj18n8vb0NdVU2n4qoW44gBi8ocgbuBntt63J4GHpaIn/I4OHBiwu/2
+IwUrfePEVCI2pAm/sBfw2XiofAclxBhhJniiRoMYPKCOdnPRP1nUWOdotzPJFPJe
+1Q==
+-----END CERTIFICATE-----
+

docker/build-test/cacerts/NISTRoot02.crt

@@ -0,0 +1,29 @@
+-----BEGIN CERTIFICATE-----
+MIIFBTCCAu2gAwIBAgIQdRxyg4+47KRFWKY+545EJjANBgkqhkiG9w0BAQsFADAV
+MRMwEQYDVQQDEwpOSVNUUm9vdDAyMB4XDTE4MDgwMTE4MTgzOVoXDTM4MDgwMTE4
+Mjc1M1owFTETMBEGA1UEAxMKTklTVFJvb3QwMjCCAiIwDQYJKoZIhvcNAQEBBQAD
+ggIPADCCAgoCggIBAJxQaJgDFbHCPwx8YOrjfthNQP7TOra9C4SkeURpetVq3fk1
+AqGgcqYzN3SRxtx9xJUweFBayO83jyBx5d+LLqX9LctaIrS4gU3uLGqDEQJisMST
++r6/mF51H5xF9AaiH8ca6ZopjigYdcv0ivMiUh8UWDvZF8SnPq4BaId4D3UwfVhV
+p8Nh9osU04BXGSOIaN5dL4CdNiOleC7IqAl4wXekOMkNfIErp2QeLnq/g1xIFmCv
+Dz+4umnPIVAYvuIKa39irNLi7j9XqUpnNcfBAvaypOe9e31RqWEYbHKhYXtFMJ6v
+Ui/d+pPPJ0HfoMu2toCZHgMCxzaFnGh0reMkcCrPpH2EQIQzbJaV4QVRFvAfNIF5
+cwvb6mRJ9pqjlIVAoT+//YUy1IsG+4n0TZAEJa9G61G3bGr7Chh+uWYGfmpevY8I
+GUTNmhYc5pGma6TFR3Hqil9PwAnPcXYQDnjhwVOGRrC/Ze9LymT7tUIEX0JKmZ0J
+ds50u8T0joWwacwK1RYdj0YC4PLeLFB2obqcfust4KCN/Hw7/pvwN3sFhbC1dn2G
+YIjqiDaenI7Gsb2t5Q8AOQbMSCJu0RYI9XN8Uzm+v0zseLF4V0+43PSTxDnlBzms
+cpjRsMRk563nVnL4oHa+LhJnB/YTBqE86bzieTiIL7SqGW1hH+RJWn55pFtnAgMB
+AAGjUTBPMAsGA1UdDwQEAwIBhjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBQl
+EQPjYg4e56GOSdev1HJtWx0z+TAQBgkrBgEEAYI3FQEEAwIBADANBgkqhkiG9w0B
+AQsFAAOCAgEAHscEbpIIPKe+avqPPxUJxRnnlV9CoBZSN4IJcA3Iox3f7zJdeLra
+hMJq8vJBLK0barh9ofLbviX1tBzAqDFd6RnMaMWTfv2BgjtoZNqfFqRp632ErDTI
+ONyHbGOnuWGXatwRNXUIhhx2UGeAy38xrIU8Z0ssTCsRY374WSFYaR5Ww7hfunyi
+eBmofMY+j6flNxEqckV3BeIarJxWmpEaAihczZxJsnZXW+D0B7h4EKZ/DakOl2QA
+59aE740ToPAl+pAF4OhT53xPlju+tqkaLnVJg/kI7Qrc0S2mHGrXnDl1FUya8VFS
+Vm8bf3nd483e3nWnSVU+vItlRIrtoHnLQ7xzMkurUNo2pROR+JgsL5WL0+NDGFjv
+Ixf9ReYGN9ujrHtojZiFaDLMPUftV6EVk2qc2d8BMEAnVzy8WJk6iqiWsmYaE2uq
+wdQHiP8kwQhXXRbqhfFZWwSisga4TIZu65rR88ah08DOGaTLfqKUnb9WD4dzTDFH
+XBl6ryuOeJGBoeJVbjy5938ZKHSS/nP3H/zYwve7xBw8CkmKAA1ECLJ47iWFmlyr
+mQkr8lkaupRMxgV8LUml35hI4lT2SbvAbdsuP/RvuvrK+mHS2UEDjG/qz4aTuXrm
+uMnUuya/1QGPhFD1oztxrhem2ob2jfkRfWT6wbv8UK7Mniw2zBfISXY=
+-----END CERTIFICATE-----

docker/build-test/cacerts/README.md

@@ -0,0 +1,13 @@
+This directory contains non-standard CA certificates needed to build the docker
+images. 
+
+Failures building the Docker containers defined in ../ due to SSL certificate
+verification errors may be a consequence of your local network's firewall.  In
+particular, the firewall may be substituting external site certificates with
+its own signed by a non-standard CA certficate (chain).  If so, you can place 
+the necessary certificates into this directory; they will be passed into the
+containers, allowing them to safely connect to those external sites.
+
+Be sure the certificates are in PEM format and include a .crt file extension.
+
+Do not remove this README file; doing so may cause a Docker build faiure.

docker/cacerts/README.md

@@ -0,0 +1,13 @@
+This directory contains non-standard CA certificates needed to build the docker
+images. 
+
+Failures building the Docker containers defined in ../ due to SSL certificate
+verification errors may be a consequence of your local network's firewall.  In
+particular, the firewall may be substituting external site certificates with
+its own signed by a non-standard CA certficate (chain).  If so, you can place 
+the necessary certificates into this directory; they will be passed into the
+containers, allowing them to safely connect to those external sites.
+
+Be sure the certificates are in PEM format and include a .crt file extension.
+
+Do not remove this README file; doing so may cause a Docker build faiure.

src/main/java/gov/nist/oar/distrib/cachemgr/pdr/PDRDatasetRestorer.java

@@ -694,7 +694,7 @@ public String idForObject(String aipid, String filepath, String forVersion, Stri
         String id;
         id = aipid + "/" + filepath;
         if (target != null && !target.isEmpty())
-            id = target + "/" + filepath;
+            id = target + "/" + aipid + "/" + filepath;
         if (forVersion != null && forVersion.length() > 0)
             id += "#" + forVersion;
         return id;

src/main/java/gov/nist/oar/distrib/service/RPACachingService.java

@@ -62,6 +62,7 @@ public String cacheAndGenerateRandomId(String datasetID, String version)
 
         logger.debug("Request to cache dataset with ID=" + datasetID);
 
+        // this is to handle ark IDs
         String dsid = datasetID;
         if (datasetID.startsWith("ark:/")) {
             // Split the dataset ID into components
@@ -73,7 +74,9 @@ public String cacheAndGenerateRandomId(String datasetID, String version)
         }
 
         logger.debug("Caching dataset with dsid=" + dsid);
-        String randomID = generateRandomID(RANDOM_ID_LENGTH, true, true);
+        // append "rpa-" with the generated random ID
+        String randomID = "rpa-" + generateRandomID(RANDOM_ID_LENGTH, true, true);
+
 
         int prefs = ROLE_RESTRICTED_DATA;
         if (!version.isEmpty())
@@ -126,6 +129,7 @@ public Map<String, Object> retrieveMetadata(String randomID) throws CacheManagem
 
     /**
      * Formats the metadata from a cache object to a JSON object with an additional field for the download URL.
+     * The download URL includes the random temporary ID, aipid, and the file path from the metadata.
      *
      * @param inMd     the metadata from the cache object
      * @param randomID the random temporary ID associated with the cache object
@@ -136,17 +140,27 @@ private JSONObject formatMetadata(JSONObject inMd, String randomID) throws Reque
         JSONObject outMd = new JSONObject();
         List<String> missingFields = new ArrayList<>();
 
+        String aipid = "";
+        if (inMd.has("aipid")) {
+            aipid = inMd.getString("aipid");
+            outMd.put("aipid", aipid);
+        } else {
+            missingFields.add("aipid");
+        }
+
         if (inMd.has("filepath")) {
             String downloadURL = getDownloadUrl(
                     rpaConfiguration.getBaseDownloadUrl(),
                     randomID,
-                    inMd.get("filepath").toString());
+                    aipid,
+                    inMd.getString("filepath"));
             outMd.put("downloadURL", downloadURL);
-            outMd.put("filePath", inMd.get("filepath"));
+            outMd.put("filePath", inMd.getString("filepath"));
         } else {
             missingFields.add("filepath");
         }
 
+
         if (inMd.has("contentType")) {
             outMd.put("mediaType", inMd.get("contentType"));
         } else {
@@ -196,12 +210,6 @@ private JSONObject formatMetadata(JSONObject inMd, String randomID) throws Reque
             missingFields.add("ediid");
         }
 
-        if (inMd.has("aipid")) {
-            outMd.put("aipid", inMd.get("aipid"));
-        } else {
-            missingFields.add("aipid");
-        }
-
         if (inMd.has("sinceDate")) {
             outMd.put("sinceDate", inMd.get("sinceDate"));
         } else {
@@ -217,33 +225,90 @@ private JSONObject formatMetadata(JSONObject inMd, String randomID) throws Reque
 
 
     /**
-     * Constructs a download URL using the given base download URL, random ID, and file path from the metadata.
+     * Constructs a download URL using the given base download URL, random ID, aipid, and file path from the metadata.
      *
      * @param baseDownloadUrl the base download URL
      * @param randomId        the random temporary ID
+     * @param aipid           the aipid from the metadata
      * @param path            the file path from the metadata
      * @return the download URL as a string
      * @throws RequestProcessingException if there was an error building the download URL
      */
-    private String getDownloadUrl(String baseDownloadUrl, String randomId, String path) throws RequestProcessingException {
+
+    private String getDownloadUrl(String baseDownloadUrl, String randomId, String aipid, String path) throws RequestProcessingException {
         URL downloadUrl;
         try {
             URL url = new URL(baseDownloadUrl);
+            StringBuilder pathBuilder = new StringBuilder();
+
+            // append the randomId to the path
+            pathBuilder.append(randomId);
+
+            // append the aipid if it's not empty
+            if (!aipid.isEmpty()) {
+                pathBuilder.append("/").append(aipid);
+            }
+
+            // append the file path, ensuring it doesn't start with a "/"
             if (path.startsWith("/")) {
                 path = path.substring(1);
             }
-            downloadUrl = new URL(url, randomId + "/" + path);
+            pathBuilder.append("/").append(path);
+
+            downloadUrl = new URL(url, pathBuilder.toString());
         } catch (MalformedURLException e) {
             throw new RequestProcessingException("Failed to build downloadUrl: " + e.getMessage());
         }
         return downloadUrl.toString();
     }
 
+
     /**
      * Generate a random alphanumeric string for the dataset to store
      * This function uses the {@link RandomStringUtils} from Apache Commons.
      **/
     private String generateRandomID(int length, boolean useLetters, boolean useNumbers) {
         return RandomStringUtils.random(length, useLetters, useNumbers);
     }
+
+    /**
+     * Uncache dataset objects using a specified random ID.
+     *
+     * @param randomId - The random ID used to fetch and uncache dataset objects.
+     * @return boolean - True if at least one dataset object was uncached successfully; otherwise, false.
+     * @throws CacheManagementException if an error occurs during the uncaching process.
+     */
+    public boolean uncacheById(String randomId) throws CacheManagementException {
+        // Validate input
+        if (randomId == null || randomId.isEmpty()) {
+            throw new IllegalArgumentException("Random ID cannot be null or empty.");
+        }
+
+        logger.debug("Request to uncache dataset with ID=" + randomId);
+
+        // Retrieve dataset objects using the randomId
+        List<CacheObject> objects = this.pdrCacheManager.selectDatasetObjects(randomId, this.pdrCacheManager.VOL_FOR_INFO);
+
+        if (objects.isEmpty()) {
+            logger.debug("No objects found for ID=" + randomId);
+            return false;
+        }
+
+        boolean isUncached = false;
+
+        // Iterate through the retrieved objects and attempt to uncache them
+        for (CacheObject obj : objects) {
+            try {
+                logger.debug("Deleting file with ID=" + obj.id);
+                this.pdrCacheManager.uncache(obj.id);
+                isUncached = true;
+            } catch (CacheManagementException e) {
+                // Log the exception without throwing it to continue attempting to uncache remaining objects
+                logger.error("Failed to uncache object with ID=" + obj.id, e);
+            }
+        }
+
+        return isUncached;
+    }
+
 }

src/main/java/gov/nist/oar/distrib/service/rpa/DefaultRPADatasetCacher.java

@@ -37,6 +37,19 @@ public String cache(String datasetId) throws RequestProcessingException {
         return randomId;
     }
 
+    @Override
+    public boolean uncache(String randomId) {
+        boolean uncached = false;
+        try {
+            uncached = rpaCachingService.uncacheById(randomId);
+        } catch (Exception e) {
+            this.logCachingException(e);
+            throw new RequestProcessingException(e.getMessage());
+        }
+
+        return uncached;
+    }
+
     /**
      * Logs the specified exception to the debug log, along with its stack trace.
      *

src/main/java/gov/nist/oar/distrib/service/rpa/HttpRPADatasetCacher.java

@@ -50,6 +50,11 @@ public String cache(String datasetId) {
         return sendHttpRequest(datasetId, url);
     }
 
+    @Override
+    public boolean uncache(String randomId) {
+        return false;
+    }
+
     /**
      * Builds the URL for the given dataset ID and using the given {@link RPAConfiguration} object.
      *