merge integration into feature/email-country-blacklists-v2

docker/build-test/Dockerfile

@@ -1,3 +1,20 @@
+FROM eclipse-temurin:8-jdk-focal 
+
+RUN mkdir -p /usr/share/man/man1
+RUN apt-get update && apt-get install -y netcat-openbsd zip git less \
+                                            ca-certificates python3 curl maven gnupg
+RUN cd /usr/bin && ln -s python3 python
+
+COPY cacerts/README.md cacerts/*.crt /usr/local/share/ca-certificates/
+RUN update-ca-certificates
+RUN java_certs=$JAVA_HOME/jre/lib/security/cacerts; \
+    add_certs=`ls /usr/local/share/ca-certificates/*.crt` && \
+    for crt in $add_certs; do \
+        name=`basename -s .crt $crt`; \
+        echo -n ${name}: " "; \
+        keytool -import -keystore $java_certs -trustcacerts -file $crt \
+                -storepass changeit -alias $name -noprompt; \
+    done;
 FROM eclipse-temurin:8
 
 RUN mkdir -p /usr/share/man/man1

docker/build-test/cacerts/Forward_Proxy_NIST_CA.crt

@@ -0,0 +1,41 @@
+-----BEGIN CERTIFICATE-----
+MIIG7TCCBNWgAwIBAgITGAAAAAecWWKCXTfeJAAAAAAABzANBgkqhkiG9w0BAQsF
+ADAVMRMwEQYDVQQDEwpOSVNUUm9vdDAyMB4XDTIxMTIwMTE4MDU0NloXDTI2MTIw
+MTE4MTU0NlowgcIxCzAJBgNVBAYTAlVTMREwDwYDVQQIEwhNYXJ5bGFuZDEVMBMG
+A1UEBxMMR2FpdGhlcnNidXJnMTcwNQYDVQQKEy5OYXRpb25hbCBJbnN0aXR1dGUg
+b2YgU3RhbmRhcmRzIGFuZCBUZWNobm9sb2d5MQ0wCwYDVQQLEwRPSVNNMR4wHAYD
+VQQDDBVGb3J3YXJkX1Byb3h5X05JU1RfQ0ExITAfBgkqhkiG9w0BCQEWEm5ldHNl
+Y3VyZUBuaXN0LmdvdjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAL/8
+PgucU6LfbThmVCiQU5zH7HRdJ0QeM8xa9Hy3BnBdD4/CxQklo7dz+AXquaOfI5Br
+H8SYZCySWTveFeJW+XvhjmEVpobz8GGrEgdR5nAKg3ZJHvAMPKgGMSnXja227TVj
+qqCZX9cIWQifqcM1iWTkS4BW2oZazwXYCqs5dfwy92ey5f/7AYC4dFeL//QtqQs/
+EUFApYabhKLcDLleDh4hwlhbTO9Zjt/eRujB/5f183RVb+igoy/xVZ8S82cNpxHS
+2DdO58GZzvAgYMYuXXJkdINkag/fpCXEy9bGaDfydHLpTWviiGz3HfXh/Chb66BG
+ZoZJmJrovVO9rSMyptMCAwEAAaOCAoYwggKCMB0GA1UdDgQWBBQquDqJ3U24XoOQ
+/6y8kFgbAp9fPDAfBgNVHSMEGDAWgBQlEQPjYg4e56GOSdev1HJtWx0z+TCB/QYD
+VR0fBIH1MIHyMIHvoIHsoIHphjFodHRwOi8vbmlzdHBraS5uaXN0Lmdvdi9DZXJ0
+RW5yb2xsL05JU1RSb290MDIuY3JshoGzbGRhcDovLy9DTj1OSVNUUm9vdDAyLENO
+PU5JU1Ryb290Q0EwMixDTj1DRFAsQ049UHVibGljJTIwS2V5JTIwU2VydmljZXMs
+Q049U2VydmljZXMsQ049Q29uZmlndXJhdGlvbixEQz1OSVNULERDPUdPVj9jZXJ0
+aWZpY2F0ZVJldm9jYXRpb25MaXN0P2Jhc2U/b2JqZWN0Q2xhc3M9Y1JMRGlzdHJp
+YnV0aW9uUG9pbnQwggEFBggrBgEFBQcBAQSB+DCB9TBKBggrBgEFBQcwAoY+aHR0
+cDovL25pc3Rwa2kubmlzdC5nb3YvQ2VydEVucm9sbC9OSVNUcm9vdENBMDJfTklT
+VFJvb3QwMi5jcnQwgaYGCCsGAQUFBzAChoGZbGRhcDovLy9DTj1OSVNUUm9vdDAy
+LENOPUFJQSxDTj1QdWJsaWMlMjBLZXklMjBTZXJ2aWNlcyxDTj1TZXJ2aWNlcyxD
+Tj1Db25maWd1cmF0aW9uLERDPU5JU1QsREM9R09WP2NBQ2VydGlmaWNhdGU/YmFz
+ZT9vYmplY3RDbGFzcz1jZXJ0aWZpY2F0aW9uQXV0aG9yaXR5MBkGCSsGAQQBgjcU
+AgQMHgoAUwB1AGIAQwBBMA8GA1UdEwEB/wQFMAMBAf8wCwYDVR0PBAQDAgGGMA0G
+CSqGSIb3DQEBCwUAA4ICAQB3OCkcbjepVN7tbK3PlLzG5HkRBG1QSmFsRnQdUTov
+/rWhdLDpHGKO4k/W2zTxNNxPW8ooD1PCy+cIlBLGcq8YcyhvWk0V2Gx1P+/f4+eq
+eH4hcUQO/7INohcnh4QXiSVMa7jNaLC+/usqWbsmTvVDbl2aYbQtwizXnUW1qNhz
+Bt76OoM7C95rNktNiaJ1VmFmd+Z3rRhzAZiC9XFIwIN1F+um7IG43nsoM4hnCByc
+/SBb3LC8R+7vNUYedkrfNPq8SGCHuPuK8H0gJX+8/8hmaaNPtZoe0VZkTdNXitnY
+HNof6w5mDoPu9lgLmNO0c36dNrmhHlPAu71EkL3afBhrdgb4Gel0WlENaur2MWf+
+yg6IQz7+aCTu2bMIkW3gm942tp7IrkXMGshUsJjLHFVrpIVkP+70QnO0wGzzQWlI
+gt+/gKvj951KGagVzsFyiQtFL9uFYMiS0awLVkSLYtBzdykm8mpG1n6EO5DlEYWe
+MOhVSeki05s0+6zUWU6TIhVDgCeUJYvAYAtWVA07Tbb1lb1vP+KbWzFMuAQMrKXV
+I0sL/gjcwaj18n8vb0NdVU2n4qoW44gBi8ocgbuBntt63J4GHpaIn/I4OHBiwu/2
+IwUrfePEVCI2pAm/sBfw2XiofAclxBhhJniiRoMYPKCOdnPRP1nUWOdotzPJFPJe
+1Q==
+-----END CERTIFICATE-----
+

docker/build-test/cacerts/NISTRoot02.crt

@@ -0,0 +1,29 @@
+-----BEGIN CERTIFICATE-----
+MIIFBTCCAu2gAwIBAgIQdRxyg4+47KRFWKY+545EJjANBgkqhkiG9w0BAQsFADAV
+MRMwEQYDVQQDEwpOSVNUUm9vdDAyMB4XDTE4MDgwMTE4MTgzOVoXDTM4MDgwMTE4
+Mjc1M1owFTETMBEGA1UEAxMKTklTVFJvb3QwMjCCAiIwDQYJKoZIhvcNAQEBBQAD
+ggIPADCCAgoCggIBAJxQaJgDFbHCPwx8YOrjfthNQP7TOra9C4SkeURpetVq3fk1
+AqGgcqYzN3SRxtx9xJUweFBayO83jyBx5d+LLqX9LctaIrS4gU3uLGqDEQJisMST
++r6/mF51H5xF9AaiH8ca6ZopjigYdcv0ivMiUh8UWDvZF8SnPq4BaId4D3UwfVhV
+p8Nh9osU04BXGSOIaN5dL4CdNiOleC7IqAl4wXekOMkNfIErp2QeLnq/g1xIFmCv
+Dz+4umnPIVAYvuIKa39irNLi7j9XqUpnNcfBAvaypOe9e31RqWEYbHKhYXtFMJ6v
+Ui/d+pPPJ0HfoMu2toCZHgMCxzaFnGh0reMkcCrPpH2EQIQzbJaV4QVRFvAfNIF5
+cwvb6mRJ9pqjlIVAoT+//YUy1IsG+4n0TZAEJa9G61G3bGr7Chh+uWYGfmpevY8I
+GUTNmhYc5pGma6TFR3Hqil9PwAnPcXYQDnjhwVOGRrC/Ze9LymT7tUIEX0JKmZ0J
+ds50u8T0joWwacwK1RYdj0YC4PLeLFB2obqcfust4KCN/Hw7/pvwN3sFhbC1dn2G
+YIjqiDaenI7Gsb2t5Q8AOQbMSCJu0RYI9XN8Uzm+v0zseLF4V0+43PSTxDnlBzms
+cpjRsMRk563nVnL4oHa+LhJnB/YTBqE86bzieTiIL7SqGW1hH+RJWn55pFtnAgMB
+AAGjUTBPMAsGA1UdDwQEAwIBhjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBQl
+EQPjYg4e56GOSdev1HJtWx0z+TAQBgkrBgEEAYI3FQEEAwIBADANBgkqhkiG9w0B
+AQsFAAOCAgEAHscEbpIIPKe+avqPPxUJxRnnlV9CoBZSN4IJcA3Iox3f7zJdeLra
+hMJq8vJBLK0barh9ofLbviX1tBzAqDFd6RnMaMWTfv2BgjtoZNqfFqRp632ErDTI
+ONyHbGOnuWGXatwRNXUIhhx2UGeAy38xrIU8Z0ssTCsRY374WSFYaR5Ww7hfunyi
+eBmofMY+j6flNxEqckV3BeIarJxWmpEaAihczZxJsnZXW+D0B7h4EKZ/DakOl2QA
+59aE740ToPAl+pAF4OhT53xPlju+tqkaLnVJg/kI7Qrc0S2mHGrXnDl1FUya8VFS
+Vm8bf3nd483e3nWnSVU+vItlRIrtoHnLQ7xzMkurUNo2pROR+JgsL5WL0+NDGFjv
+Ixf9ReYGN9ujrHtojZiFaDLMPUftV6EVk2qc2d8BMEAnVzy8WJk6iqiWsmYaE2uq
+wdQHiP8kwQhXXRbqhfFZWwSisga4TIZu65rR88ah08DOGaTLfqKUnb9WD4dzTDFH
+XBl6ryuOeJGBoeJVbjy5938ZKHSS/nP3H/zYwve7xBw8CkmKAA1ECLJ47iWFmlyr
+mQkr8lkaupRMxgV8LUml35hI4lT2SbvAbdsuP/RvuvrK+mHS2UEDjG/qz4aTuXrm
+uMnUuya/1QGPhFD1oztxrhem2ob2jfkRfWT6wbv8UK7Mniw2zBfISXY=
+-----END CERTIFICATE-----

docker/build-test/cacerts/README.md

@@ -0,0 +1,13 @@
+This directory contains non-standard CA certificates needed to build the docker
+images. 
+
+Failures building the Docker containers defined in ../ due to SSL certificate
+verification errors may be a consequence of your local network's firewall.  In
+particular, the firewall may be substituting external site certificates with
+its own signed by a non-standard CA certficate (chain).  If so, you can place 
+the necessary certificates into this directory; they will be passed into the
+containers, allowing them to safely connect to those external sites.
+
+Be sure the certificates are in PEM format and include a .crt file extension.
+
+Do not remove this README file; doing so may cause a Docker build faiure.

docker/cacerts/README.md

@@ -0,0 +1,13 @@
+This directory contains non-standard CA certificates needed to build the docker
+images. 
+
+Failures building the Docker containers defined in ../ due to SSL certificate
+verification errors may be a consequence of your local network's firewall.  In
+particular, the firewall may be substituting external site certificates with
+its own signed by a non-standard CA certficate (chain).  If so, you can place 
+the necessary certificates into this directory; they will be passed into the
+containers, allowing them to safely connect to those external sites.
+
+Be sure the certificates are in PEM format and include a .crt file extension.
+
+Do not remove this README file; doing so may cause a Docker build faiure.

src/main/java/gov/nist/oar/distrib/cachemgr/CacheExpiryCheck.java

@@ -0,0 +1,82 @@
+package gov.nist.oar.distrib.cachemgr;
+
+import gov.nist.oar.distrib.StorageVolumeException;
+
+import java.time.Instant;
+
+/**
+ * Implements a cache object check to identify and remove objects that have been in the cache
+ * longer than a specified duration, specifically two weeks. This check helps in
+ * managing cache integrity by ensuring that stale or outdated data are removed
+ * from the cache.
+ */
+public class CacheExpiryCheck implements CacheObjectCheck {
+
+    private StorageInventoryDB inventoryDB;
+
+    public CacheExpiryCheck(StorageInventoryDB inventoryDB) {
+        this.inventoryDB = inventoryDB;
+    }
+
+    /**
+     * Checks if a cache object is expired and removes it from the cache if it is.
+     * The method uses the {@code expires} metadata field to determine the expiration status.
+     * The expiration time is calculated based on the {@code LastModified} time plus the {@code expires} duration.
+     * If the current time is past the calculated expiry time, the object is removed from the inventory database.
+     *
+     * @param co The cache object to check for expiration.
+     * @throws IntegrityException If the object is found to be corrupted during the check.
+     * @throws StorageVolumeException If there's an error accessing the storage volume during the check.
+     * @throws CacheManagementException If there's an error managing the cache, including removing the expired object.
+     */
+    @Override
+    public void check(CacheObject co) throws IntegrityException, StorageVolumeException, CacheManagementException {
+        if (co == null || inventoryDB == null) {
+            throw new IllegalArgumentException("CacheObject or StorageInventoryDB is null");
+        }
+
+        if (co.hasMetadatum("expires")) {
+            long expiresDuration = co.getMetadatumLong("expires", -1L);
+            if (expiresDuration == -1L) {
+                throw new IntegrityException("Invalid 'expires' metadata value");
+            }
+
+            long lastModified = co.getLastModified();
+            if (lastModified == -1L) {
+                throw new IntegrityException("CacheObject 'lastModified' time not available");
+            }
+
+            long expiryTime = lastModified + expiresDuration;
+            long currentTime = Instant.now().toEpochMilli();
+
+            // Check if the object is expired
+            if (expiryTime < currentTime) {
+                try {
+                    boolean removed = removeObject(co);
+                    if (!removed) {
+                        throw new CacheManagementException("Failed to remove expired object: " + co.name);
+                    }
+                } catch (InventoryException e) {
+                    throw new CacheManagementException("Error removing expired object from inventory database: " + co.name, e);
+                }
+            }
+        }
+    }
+
+    /**
+     * Attempts to remove a cache object from both its physical volume and the inventory database.
+     * Synchronization ensures thread-safe removal operations.
+     *
+     * @param co The cache object to be removed.
+     * @return true if the object was successfully removed from its volume, false otherwise.
+     * @throws StorageVolumeException if an error occurs accessing the storage volume.
+     * @throws InventoryException if an error occurs updating the inventory database.
+     */
+    protected boolean removeObject(CacheObject co) throws StorageVolumeException, InventoryException {
+        synchronized (inventoryDB) {
+            boolean out = co.volume.remove(co.name);
+            inventoryDB.removeObject(co.volname, co.name);
+            return out;
+        }
+    }
+}

src/main/java/gov/nist/oar/distrib/cachemgr/pdr/PDRDatasetRestorer.java

@@ -650,6 +650,9 @@ protected void cacheFromBagUsingStore(String bagfile, Collection<String> need, C
                     md.put("ediid", resmd.get("ediid"));
                 md.put("cachePrefs", prefs);
 
+                // a hook for handling the expiration logic
+                updateMetadata(md, prefs);
+
                 // find space in the cache, and copy the data file into it
                 try {
                     resv = into.reserveSpace(ze.getSize(), prefs);
@@ -687,14 +690,26 @@ protected void cacheFromBagUsingStore(String bagfile, Collection<String> need, C
             fixMissingChecksums(into, fix, manifest);
     }
 
+    /**
+     * Method intended for customization of metadata before caching. This method can be overridden
+     * by subclasses to implement specific metadata customization logic as needed.
+     *
+     * @param md The metadata JSONObject to be customized.
+     * @param prefs flags for data roles
+     */
+    protected void updateMetadata(JSONObject md, int prefs) {
+        // Default implementation does nothing.
+        // Subclasses can override this to implement specific logic.
+    }
+
     /**
      * helper method to generate an ID for the object to be cached
      */
     public String idForObject(String aipid, String filepath, String forVersion, String target) {
         String id;
         id = aipid + "/" + filepath;
         if (target != null && !target.isEmpty())
-            id = target + "/" + filepath;
+            id = target + "/" + aipid + "/" + filepath;
         if (forVersion != null && forVersion.length() > 0)
             id += "#" + forVersion;
         return id;

src/main/java/gov/nist/oar/distrib/cachemgr/pdr/RestrictedDatasetRestorer.java

@@ -20,44 +20,23 @@
 import gov.nist.oar.distrib.ObjectNotFoundException;
 import gov.nist.oar.distrib.ResourceNotFoundException;
 import gov.nist.oar.distrib.BagStorage;
-import gov.nist.oar.distrib.Checksum;
-import gov.nist.oar.distrib.cachemgr.Restorer;
 import gov.nist.oar.distrib.cachemgr.Reservation;
-import gov.nist.oar.distrib.cachemgr.IntegrityMonitor;
-import gov.nist.oar.distrib.cachemgr.BasicCache;
 import gov.nist.oar.distrib.cachemgr.Cache;
 import gov.nist.oar.distrib.cachemgr.CacheObject;
-import gov.nist.oar.distrib.cachemgr.CacheObjectCheck;
-import gov.nist.oar.distrib.cachemgr.StorageInventoryDB;
 import gov.nist.oar.distrib.cachemgr.CacheManagementException;
 import gov.nist.oar.distrib.cachemgr.RestorationException;
-import gov.nist.oar.distrib.cachemgr.InventoryException;
 
 import java.util.Collection;
-import java.util.List;
-import java.util.ArrayList;
 import java.util.Set;
-import java.util.Map;
-import java.util.HashSet;
-import java.util.HashMap;
-import java.util.zip.ZipInputStream;
-import java.util.zip.ZipEntry;
-import java.nio.file.Path;
-import java.nio.file.Paths;
 import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.FileNotFoundException;
-import java.text.ParseException;
 
 import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 import org.json.JSONObject;
 import org.json.JSONException;
 
-import org.apache.commons.io.FilenameUtils;
 
 /**
  * A {@link gov.nist.oar.distrib.cachemgr.Restorer} for restoring "restricted public" datasets from the 
@@ -82,6 +61,7 @@
  */
 public class RestrictedDatasetRestorer extends PDRDatasetRestorer {
     BagStorage restrictedLtstore = null;
+    long expiryTime = 1209600000L; // 2 weeks in milliseconds
 
     /**
      * create the restorer
@@ -125,6 +105,25 @@ public RestrictedDatasetRestorer(BagStorage publicLtstore, BagStorage restricted
         this.restrictedLtstore = restrictedLtstore;
     }
 
+    /**
+     * Retrieves the expiry time for data.
+     * <p>
+     * This value represents the duration in milliseconds after which the data is considered expired.
+     *
+     * @return the expiry time in milliseconds.
+     */
+    public long getExpiryTime() {
+        return expiryTime;
+    }
+
+    /**
+     * Sets the expiry time for restricted/public access content.
+     *
+     * @param expiryTime the expiry time in milliseconds to set.
+     */
+    public void setExpiryTime(long expiryTime) {
+        this.expiryTime = expiryTime;
+    }
 
     /**
      * return true if an object does <i>not</i> exist in the long term storage system.  Returning
@@ -285,4 +284,19 @@ protected void cacheFromBag(String bagfile, Collection<String> need, Collection<
                                    target, ltstore);
         }
     }
+
+    /**
+     * Updates the metadata for files marked as restricted, by adding an expiration time.
+     *
+     * @param md The metadata JSONObject to be customized.
+     * @param prefs flags for data roles
+     */
+    @Override
+    protected void updateMetadata(JSONObject md, int prefs) {
+        if ((prefs & ROLE_RESTRICTED_DATA) != 0) {
+            // Calculate the expiration time as current time + expiryTime
+            long expires = System.currentTimeMillis() + expiryTime;
+            md.put("expires", expires);
+        }
+    }
 }