Merge pull request #104 from usnistgov/feature/uncaching-endpoint

Add a new endpoint for removing datasets or files from cache

src/main/java/gov/nist/oar/distrib/service/rpa/HttpURLConnectionRPARequestHandlerService.java

@@ -3,15 +3,10 @@
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import gov.nist.oar.distrib.service.RPACachingService;
-import gov.nist.oar.distrib.service.rpa.exceptions.InvalidRecaptchaException;
 import gov.nist.oar.distrib.service.rpa.exceptions.InvalidRequestException;
-import gov.nist.oar.distrib.service.rpa.exceptions.RecaptchaClientException;
-import gov.nist.oar.distrib.service.rpa.exceptions.RecaptchaServerException;
-import gov.nist.oar.distrib.service.rpa.exceptions.RecaptchaVerificationFailedException;
 import gov.nist.oar.distrib.service.rpa.exceptions.RecordNotFoundException;
 import gov.nist.oar.distrib.service.rpa.exceptions.RequestProcessingException;
 import gov.nist.oar.distrib.service.rpa.model.JWTToken;
-import gov.nist.oar.distrib.service.rpa.model.RecaptchaResponse;
 import gov.nist.oar.distrib.service.rpa.model.Record;
 import gov.nist.oar.distrib.service.rpa.model.RecordStatus;
 import gov.nist.oar.distrib.service.rpa.model.RecordWrapper;
@@ -45,10 +40,14 @@
 
 
 /**
- * An implementation of the RPARequestHandlerService that uses HttpURLConnection to send HTTP requests and
- * receives responses from the Salesforce service.
+ * An implementation of the {@link RPARequestHandler} interface that uses HttpURLConnection
+ * to send HTTP requests and receive responses from the Salesforce service. This class serves
+ * as a bridge between the application and Salesforce, acting as the data source for managing
+ * records. Through this service, records can be created, retrieved, and updated directly in
+ * Salesforce, using the platform's API for record management.
+ *
  */
-public class HttpURLConnectionRPARequestHandlerService implements IRPARequestHandler {
+public class HttpURLConnectionRPARequestHandlerService implements RPARequestHandler {
 
     private final static Logger LOGGER = LoggerFactory.getLogger(HttpURLConnectionRPARequestHandlerService.class);
 

src/main/java/gov/nist/oar/distrib/service/rpa/IRPARequestHandler.java → src/main/java/gov/nist/oar/distrib/service/rpa/RPARequestHandler.java

@@ -9,9 +9,11 @@
 import gov.nist.oar.distrib.service.rpa.model.UserInfoWrapper;
 
 /**
- * An interface for handling requests to manage records.
+ * An interface for handling requests to manage records. This includes operations
+ * such as creating, retrieving, and updating records. Implementations of this interface
+ * are responsible for the interaction with a data source to perform these operations.
  */
-public interface IRPARequestHandler {
+public interface RPARequestHandler {
 
     /**
      * Updates the status of a record by ID.

src/main/java/gov/nist/oar/distrib/web/CacheManagementController.java

@@ -11,6 +11,7 @@
  */
 package gov.nist.oar.distrib.web;
 
+import gov.nist.oar.distrib.cachemgr.VolumeStatus;
 import gov.nist.oar.distrib.cachemgr.pdr.PDRCacheManager;
 import gov.nist.oar.distrib.cachemgr.CacheManagementException;
 import gov.nist.oar.distrib.cachemgr.CacheObject;
@@ -230,14 +231,17 @@ else if (":cached".equals(selector))
         }
 
         List<CacheObject> files = mgr.selectDatasetObjects(dsid, mgr.VOL_FOR_INFO);
-        if (files.size() == 0)
-            throw new ResourceNotFoundException(dsid);
 
         if (filepath.length() > 0)
             files = mgr.selectFileObjects(dsid, filepath, purpose);
         else if (purpose != mgr.VOL_FOR_INFO)
             files = mgr.selectDatasetObjects(dsid, purpose);
 
+        // Ensure that a ResourceNotFoundException is thrown if the files list is empty
+        // after all the selection logic has been applied
+        if (files.size() == 0)
+            throw new ResourceNotFoundException(dsid);
+
         if (selector == null && filepath.length() > 0) {
             // return a single JSON object; get the one that's cached
             List<CacheObject> use = files.stream().filter(c -> c.cached).collect(Collectors.toList());
@@ -331,6 +335,72 @@ else if (":checked".equals(selector))
             return new ResponseEntity<String>("Method not allowed on URL", HttpStatus.METHOD_NOT_ALLOWED);
     }
 
+    /**
+     * Removes a dataset or specific files within a dataset from the cache based on the provided dataset identifier (dsid).
+     * This endpoint supports selective removal using the ":cached" selector in the URL path, allowing for more granular
+     * control over cache management.
+     *
+     * If the ":cached" selector is present, and no specific file path is provided, the entire dataset identified by the dsid
+     * is removed from the cache. If a specific file path is provided, only the specified file within the dataset will be removed.
+     *
+     * @param dsid the dataset identifier
+     * @param request used to extract the optional file path from the URL
+     * @return ResponseEntity with the result of the operation
+     */
+    @DeleteMapping(value="/objects/{dsid}/**")
+    public ResponseEntity<String> removeFromCache(@PathVariable("dsid") String dsid, HttpServletRequest request) {
+        try {
+
+            _checkForManager();
+            log.debug("Attempting to remove files from cache for Dataset ID: {}", dsid);
+
+            // Extract the optional file path from the request URL
+            String path = (String) request.getAttribute(HandlerMapping.PATH_WITHIN_HANDLER_MAPPING_ATTRIBUTE);
+            String prefix = "/cache/objects/" + dsid;
+            String filepath = path.startsWith(prefix) ? path.substring(prefix.length()) : "";
+
+            String selector = null;
+            Matcher selmatch = SEL_PATH_FIELD.matcher(filepath);
+            if (selmatch.find()) {
+                selector = selmatch.group(1);
+                filepath = filepath.substring(0, selmatch.start());
+            }
+            if (filepath.startsWith("/")) {
+                filepath = filepath.substring(1); // Remove leading slash
+            }
+
+            if (":cached".equals(selector)) {
+                if (filepath.isEmpty() || filepath.equals("/")) {
+                    log.debug("Removing entire dataset from cache for ID: {}", dsid);
+                    List<CacheObject> files = mgr.selectDatasetObjects(dsid, VolumeStatus.VOL_FOR_UPDATE);
+                    for (CacheObject file : files) {
+                        log.debug("Uncaching file: {}", file.id);
+                        mgr.uncache(file.id);
+                    }
+                    return ResponseEntity.ok("Dataset " + dsid + " removed from cache");
+                } else {
+                    log.debug("Removing file(s) from cache for dataset ID: {} and path: {}", dsid, filepath);
+                    List<CacheObject> files = mgr.selectFileObjects(dsid, filepath, VolumeStatus.VOL_FOR_UPDATE);
+                    for (CacheObject file : files) {
+                        log.debug("Uncaching file: {}", file.id);
+                        mgr.uncache(file.id);
+                    }
+                    return ResponseEntity.ok("File(s) " + filepath + " in dataset " + dsid + " removed from cache");
+                }
+            } else {
+                log.warn("Operation not allowed: URL does not contain ':cached' selector");
+                return new ResponseEntity<String>("Operation not allowed on URL without :cached selector", HttpStatus.METHOD_NOT_ALLOWED);
+            }
+
+        } catch (NotOperatingException e) {
+            log.error("Cache manager is not operational", e);
+            return ResponseEntity.status(HttpStatus.SERVICE_UNAVAILABLE).body("Cache manager is not operational");
+        } catch (CacheManagementException e) {
+            log.error("Error processing cache removal request", e);
+            return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body("Error processing cache removal request: " + e.getMessage());
+        }
+    }
+
     /**
      * return status information about the caching queue.  The caching queue is a queue of data items 
      * waiting to be cached

src/main/java/gov/nist/oar/distrib/web/NISTDistribServiceConfig.java

@@ -17,6 +17,9 @@
 import gov.nist.oar.distrib.service.DefaultPreservationBagService;
 import gov.nist.oar.distrib.service.FileDownloadService;
 import gov.nist.oar.distrib.service.PreservationBagService;
+import gov.nist.oar.distrib.service.RPACachingService;
+import gov.nist.oar.distrib.service.rpa.JKSKeyRetriever;
+import gov.nist.oar.distrib.service.rpa.KeyRetriever;
 import gov.nist.oar.distrib.storage.AWSS3LongTermStorage;
 import gov.nist.oar.distrib.storage.FilesystemLongTermStorage;
 import io.swagger.v3.oas.models.Components;

src/main/java/gov/nist/oar/distrib/web/RPARequestHandlerController.java

@@ -1,7 +1,7 @@
 package gov.nist.oar.distrib.web;
 
 import gov.nist.oar.distrib.service.RPACachingService;
-import gov.nist.oar.distrib.service.rpa.IRPARequestHandler;
+import gov.nist.oar.distrib.service.rpa.RPARequestHandler;
 import gov.nist.oar.distrib.service.rpa.exceptions.InvalidRequestException;
 import gov.nist.oar.distrib.service.rpa.exceptions.RecaptchaClientException;
 import gov.nist.oar.distrib.service.rpa.exceptions.RecaptchaServerException;
@@ -62,7 +62,7 @@ public class RPARequestHandlerController {
     /**
      * The primary service for handling RPA requests.
      */
-    IRPARequestHandler service = null;
+    RPARequestHandler service = null;
 
     /**
      * The sanitizer for incoming requests.
@@ -114,7 +114,7 @@ public RPARequestHandlerController(RPAServiceProvider rpaServiceProvider,
                                        RPACachingService cachingService)
     {
         if (cachingService != null && rpaServiceProvider != null)
-            this.service = rpaServiceProvider.getIRPARequestHandler(cachingService);
+            this.service = rpaServiceProvider.getRPARequestHandler(cachingService);
         this.requestSanitizer = new RequestSanitizer();
         this.configuration = rpaServiceProvider.getRpaConfiguration();
         this.jwtTokenValidator = new JwtTokenValidator(this.configuration);

src/main/java/gov/nist/oar/distrib/web/RPAServiceProvider.java

@@ -1,12 +1,8 @@
 package gov.nist.oar.distrib.web;
 
 import gov.nist.oar.distrib.service.RPACachingService;
-import gov.nist.oar.distrib.service.rpa.DefaultRPARequestHandlerService;
 import gov.nist.oar.distrib.service.rpa.HttpURLConnectionRPARequestHandlerService;
-import gov.nist.oar.distrib.service.rpa.IRPARequestHandler;
-import gov.nist.oar.distrib.service.rpa.RPARequestHandlerService;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.web.client.RestTemplate;
+import gov.nist.oar.distrib.service.rpa.RPARequestHandler;
 
 public class RPAServiceProvider {
     RPAConfiguration rpaConfiguration;
@@ -15,19 +11,11 @@ public RPAServiceProvider(RPAConfiguration rpaConfiguration) {
         this.rpaConfiguration = rpaConfiguration;
     }
 
-    public RPARequestHandlerService getRPARequestHandlerService(RestTemplate restTemplate) {
-        return this.getDefaultRPARequestHandlerService(restTemplate);
-    }
-
-    private DefaultRPARequestHandlerService getDefaultRPARequestHandlerService(RestTemplate restTemplate) {
-        return new DefaultRPARequestHandlerService(this.rpaConfiguration, restTemplate);
-    }
-
-    public IRPARequestHandler getIRPARequestHandler(RPACachingService rpaCachingService) {
+    public RPARequestHandler getRPARequestHandler(RPACachingService rpaCachingService) {
         return this.getHttpURLConnectionRPARequestHandler(rpaCachingService);
     }
 
-    private IRPARequestHandler getHttpURLConnectionRPARequestHandler(RPACachingService rpaCachingService) {
+    private RPARequestHandler getHttpURLConnectionRPARequestHandler(RPACachingService rpaCachingService) {
         return new HttpURLConnectionRPARequestHandlerService(this.rpaConfiguration, rpaCachingService);
     }
 

src/test/java/gov/nist/oar/distrib/web/CacheManagementControllerTest.java

@@ -404,6 +404,44 @@ public void testStartMonitor() throws ConfigurationException {
         assertFalse("Monitor failed to finish?", status.getBoolean("running"));
     }
 
+    @Test
+    public void testRemoveFromCache() {
+        HttpEntity<String> req = new HttpEntity<>(null, headers);
+
+        // First, ensure the dataset or file initially exists in the cache by sending a GET request
+        // This is a check to confirm presence before deletion
+        ResponseEntity<String> resp = websvc.exchange(getBaseURL() + "/cache/objects/mds1491/:cached",
+                HttpMethod.GET, req, String.class);
+        assertEquals(HttpStatus.OK, resp.getStatusCode());
+
+        // Attempt to remove a specific file (trial2.json) from the cache using the DELETE method
+        // The file path and the ':cached' selector are included in the URL
+        ResponseEntity<String> removeResp = websvc.exchange(getBaseURL() +
+                        "/cache/objects/mds1491/trial2.json/:cached",
+                HttpMethod.DELETE, req, String.class);
+        assertEquals(HttpStatus.OK, removeResp.getStatusCode());
+
+        // After deletion, verify that the specific file (trial2.json) is no longer accessible
+        // Expect a NOT_FOUND status code to confirm successful deletion
+        resp = websvc.exchange(getBaseURL() + "/cache/objects/mds1491/trial2.json/:cached",
+                HttpMethod.GET, req, String.class);
+        assertEquals(HttpStatus.NOT_FOUND, resp.getStatusCode());
+
+        // Now, attempt to remove the entire dataset ('mds1491') from the cache
+        // This tests the capability to delete all files under a dataset identifier
+        removeResp = websvc.exchange(getBaseURL() +
+                        "/cache/objects/mds1491/:cached",
+                HttpMethod.DELETE, req, String.class);
+        assertEquals(HttpStatus.OK, removeResp.getStatusCode());
+
+        // Confirm that the entire dataset ('mds1491') is no longer available in the cache
+        // A GET request should return a NOT_FOUND status, meaning the dataset has been successfully removed from cache
+        resp = websvc.exchange(getBaseURL() + "/cache/objects/mds1491/:cached",
+                HttpMethod.GET, req, String.class);
+        assertEquals(HttpStatus.NOT_FOUND, resp.getStatusCode());
+    }
+
+
     private String getBaseURL() {
         return "http://localhost:" + port + "/od";
     }

src/test/java/gov/nist/oar/distrib/web/RPARequestHandlerControllerTest.java

@@ -2,7 +2,7 @@
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 import gov.nist.oar.distrib.service.RPACachingService;
-import gov.nist.oar.distrib.service.rpa.IRPARequestHandler;
+import gov.nist.oar.distrib.service.rpa.RPARequestHandler;
 import gov.nist.oar.distrib.service.rpa.RecaptchaHelper;
 import gov.nist.oar.distrib.service.rpa.exceptions.InvalidRequestException;
 import gov.nist.oar.distrib.service.rpa.exceptions.RecaptchaVerificationFailedException;
@@ -51,7 +51,7 @@
 @RunWith(MockitoJUnitRunner.class)
 public class RPARequestHandlerControllerTest {
     @Mock
-    private IRPARequestHandler service;
+    private RPARequestHandler service;
 
     @Mock
     RPAServiceProvider mockRPAServiceProvider;
@@ -79,7 +79,7 @@ public class RPARequestHandlerControllerTest {
     @Before
     public void setup() {
         MockitoAnnotations.initMocks(this);
-        when(mockRPAServiceProvider.getIRPARequestHandler(mockRPACachingService)).thenReturn(service);
+        when(mockRPAServiceProvider.getRPARequestHandler(mockRPACachingService)).thenReturn(service);
         // create a test instance of the RPARequestHandlerController class
         controller = new RPARequestHandlerController(mockRPAServiceProvider, mockRPACachingService);
         controller.setRequestSanitizer(mockRequestSanitizer);