enable cavern to handle links to external files

cadc-test-vos/build.gradle

@@ -16,7 +16,7 @@ sourceCompatibility = 11
 
 group = 'org.opencadc'
 
-version = '2.1.10'
+version = '2.1.11'
 
 description = 'OpenCADC VOSpace test library'
 def git_url = 'https://github.com/opencadc/vos'

cadc-test-vos/src/main/java/org/opencadc/conformance/vos/FilesTest.java

@@ -96,9 +96,9 @@
 import org.junit.Assume;
 import org.junit.Test;
 import org.opencadc.vospace.DataNode;
+import org.opencadc.vospace.LinkNode;
 import org.opencadc.vospace.VOS;
 import org.opencadc.vospace.VOSURI;
-import org.opencadc.vospace.io.NodeReader;
 import org.opencadc.vospace.transfer.Direction;
 import org.opencadc.vospace.transfer.Protocol;
 import org.opencadc.vospace.transfer.Transfer;
@@ -111,6 +111,7 @@ public class FilesTest extends VOSTest {
     protected final URL filesServiceURL;
     protected Subject altSubject;
     protected boolean enablePassthroughParamTest = false;
+    protected boolean linkExternalFile = false;
 
     protected FilesTest(URI resourceID, File testCert) {
         super(resourceID, testCert);
@@ -364,6 +365,56 @@ public void testDataNodePermission() {
         }
     }
 
+    @Test
+    public void testLinkNodeExternalFile() {
+        if (!linkExternalFile) {
+            log.info("linkExternalFile not enabled");
+            return;
+        }
+
+        String[] targets = new String[] {
+            "file:///path/to/external/data",
+            "file:///../../something-above"
+        };
+
+        try {
+            for (String t : targets) {
+                // create a simple link node
+                String name = "testLinkNodeExternalFile";
+                URL nodeURL = getNodeURL(nodesServiceURL, name);
+                VOSURI nodeURI = getVOSURI(name);
+                URI targetURI = URI.create(t);
+                log.info("link node: " + nodeURI + " -> " + targetURI);
+
+                // cleanup
+                delete(nodeURL, false);
+
+                // not found
+                get(nodeURL, 404, TEXT_CONTENT_TYPE);
+
+                // PUT the node
+                log.info("put: " + nodeURI + " -> " + nodeURL);
+                LinkNode testNode = new LinkNode(name, targetURI);
+                put(nodeURL, nodeURI, testNode);
+
+                URL fileURL = getNodeURL(filesServiceURL, name);
+                log.info("files: " + fileURL);
+
+                // GET should fail
+                HttpGet get = new HttpGet(fileURL, true);
+                Subject.doAs(authSubject, new RunnableAction(get));
+                log.info("get: " + get.getResponseCode() + " " + get.getThrowable());
+                Assert.assertEquals(400, get.getResponseCode());
+
+                delete(nodeURL);
+            }
+
+        } catch (Exception e) {
+            log.error("Unexpected error", e);
+            Assert.fail("Unexpected error: " + e);
+        }
+    }
+
     protected static URI computeChecksumURI(byte[] input) throws NoSuchAlgorithmException, IOException {
         MessageDigest md = MessageDigest.getInstance("MD5");
         md.update(input);

cadc-test-vos/src/main/java/org/opencadc/conformance/vos/NodesTest.java

@@ -109,6 +109,7 @@ public class NodesTest extends VOSTest {
     private GroupURI group1;
     private GroupURI group2;
 
+    protected boolean linkExternalFile = false; // try to make link -> file:///path/to/something
     protected boolean linkNodeProps = true;
     protected boolean paginationSupported = true;
     protected boolean paginationLimitIgnored = false;
@@ -388,6 +389,69 @@ public void testLinkNode() {
         }
     }
 
+    @Test
+    public void testLinkNodeExternalFile() {
+        if (!linkExternalFile) {
+            log.info("linkExternalFile not enabled");
+            return;
+        }
+
+        try {
+            // create a simple link node
+            String name = "testLinkNodeExternalFile";
+            URL nodeURL = getNodeURL(nodesServiceURL, name);
+            VOSURI nodeURI = getVOSURI(name);
+            URI targetURI = URI.create("file:///path/to/external/data");
+            log.info("link node: " + nodeURI + " -> " + targetURI);
+
+            // cleanup
+            delete(nodeURL, false);
+
+            // not found
+            get(nodeURL, 404, TEXT_CONTENT_TYPE);
+
+            // PUT the node
+            log.info("put: " + nodeURI + " -> " + nodeURL);
+            LinkNode testNode = new LinkNode(name, targetURI);
+            put(nodeURL, nodeURI, testNode);
+
+            // GET the new node
+            NodeReader.NodeReaderResult result = get(nodeURL, 200, XML_CONTENT_TYPE);
+            log.info("found: " + result.vosURI + " owner: " + result.node.ownerDisplay);
+            Assert.assertTrue(result.node instanceof LinkNode);
+            LinkNode persistedNode = (LinkNode) result.node;
+            Assert.assertEquals(testNode, persistedNode);
+            Assert.assertEquals(nodeURI, result.vosURI);
+            Assert.assertEquals(testNode.getTarget(), persistedNode.getTarget());
+            Assert.assertEquals(testNode.getName(), persistedNode.getName());
+
+            if (linkNodeProps) {
+                // POST an update to the node
+                NodeProperty nodeProperty = new NodeProperty(VOS.PROPERTY_URI_LANGUAGE, "English");
+                testNode.getProperties().add(nodeProperty);
+                post(nodeURL, nodeURI, testNode);
+
+                // GET the updated node
+                result = get(nodeURL, 200, XML_CONTENT_TYPE);
+                Assert.assertTrue(result.node instanceof LinkNode);
+                LinkNode updatedNode = (LinkNode) result.node;
+                Assert.assertEquals(testNode, updatedNode);
+                Assert.assertEquals(nodeURI, result.vosURI);
+                Assert.assertEquals(testNode.getTarget(), updatedNode.getTarget());
+                Assert.assertEquals(testNode.getName(), updatedNode.getName());
+                Assert.assertTrue(updatedNode.getProperties().contains(nodeProperty));
+            }
+
+            if (cleanupOnSuccess) {
+                delete(nodeURL);
+            }
+
+        } catch (Exception e) {
+            log.error("Unexpected error", e);
+            Assert.fail("Unexpected error: " + e);
+        }
+    }
+
     @Test
     public void testNodePropertyUpdates() {
         try {

cadc-vos-server/build.gradle

@@ -16,7 +16,7 @@ sourceCompatibility = 11
 
 group = 'org.opencadc'
 
-version = '2.0.16'
+version = '2.0.17'
 
 description = 'OpenCADC VOSpace server'
 def git_url = 'https://github.com/opencadc/vos'

cadc-vos-server/src/main/java/org/opencadc/vospace/server/PathResolver.java

@@ -71,6 +71,7 @@
 
 import ca.nrc.cadc.auth.AuthenticationUtil;
 import ca.nrc.cadc.util.StringUtil;
+import java.net.URI;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Iterator;
@@ -244,11 +245,15 @@ private ResolvedNode resolveNode(String nodePath, boolean resolveLeafLink, List<
      */
     public VOSURI validateTargetURI(LinkNode linkNode) throws Exception {
         LocalServiceURI localServiceURI = new LocalServiceURI(nodePersistence.getResourceID());
-        VOSURI targetURI = new VOSURI(linkNode.getTarget());
+        URI turi = linkNode.getTarget();
+        if (!"vos".equals(turi.getScheme())) {
+            throw NodeFault.InvalidArgument.getStatus("cannot navigate external link " + turi);
+        }
+        VOSURI targetURI = new VOSURI(turi);
 
         log.debug("Validating target: " + targetURI.getServiceURI() + " vs " + localServiceURI.getVOSBase().getServiceURI());
         if (!localServiceURI.getVOSBase().getServiceURI().equals(targetURI.getServiceURI())) {
-            throw NodeFault.InvalidArgument.getStatus("External link " + targetURI.getServiceURI().toASCIIString());
+            throw NodeFault.InvalidArgument.getStatus("cannot navigate external vospace link " + turi);
         }
         return targetURI;
     }

cavern/VERSION

@@ -1,6 +1,6 @@
 ## deployable containers have a semantic and build tag
 # semantic version tag: major.minor
 # build version tag: timestamp
-VER=0.7.10
+VER=0.7.11
 TAGS="${VER} ${VER}-$(date -u +"%Y%m%dT%H%M%S")"
 unset VER

cavern/build.gradle

@@ -43,7 +43,7 @@ dependencies {
     implementation 'org.opencadc:cadc-dali:[1.0,)'
     implementation 'org.opencadc:cadc-pkg-server:[1.2.3,)'
     implementation 'org.opencadc:cadc-vos:[2.0.7,)'
-    implementation 'org.opencadc:cadc-vos-server:[2.0.15,)'
+    implementation 'org.opencadc:cadc-vos-server:[2.0.17,)'
 
     runtimeOnly 'org.opencadc:cadc-access-control-identity:[1.2.0,)'
 

cavern/src/intTest/java/org/opencadc/cavern/FilesTest.java

@@ -89,6 +89,8 @@ public FilesTest() {
 
         // enables testDataNodePermission
         super.enableTestDataNodePermission(Constants.AUTH_TEST_CERT);
+
+        super.linkExternalFile = true;
     }
 
 }

cavern/src/intTest/java/org/opencadc/cavern/NodesTest.java

@@ -95,6 +95,7 @@ public class NodesTest extends org.opencadc.conformance.vos.NodesTest {
     public NodesTest() {
         super(Constants.RESOURCE_ID, Constants.TEST_CERT);
         super.linkNodeProps = false;       // xattrs on links not supported in posix filesystem
+        super.linkExternalFile = true;     // allow links to external filesystem using file:///path/target
         super.paginationSupported = false; // not implemented because it is not scalable
         super.nodelockSupported = false;   // not implemented, maybe never
 

cavern/src/main/java/org/opencadc/cavern/nodes/FileSystemNodePersistence.java

@@ -441,14 +441,16 @@ public Node put(Node node) throws NodeNotSupportedException, TransientException
 
         if (node instanceof LinkNode) {
             LinkNode ln = (LinkNode) node;
-            try {
-                PathResolver ps = new PathResolver(this, new VOSpaceAuthorizer(this));
-                ps.validateTargetURI(ln);
-            } catch (Exception ex) {
-                throw new UnsupportedOperationException("link to external resource", ex);
+            if (!"file".equals(ln.getTarget().getScheme())) {
+                try {
+                    PathResolver ps = new PathResolver(this, new VOSpaceAuthorizer(this));
+                    ps.validateTargetURI(ln);
+                } catch (Exception ex) {
+                    throw new UnsupportedOperationException("link to external resource", ex);
+                }
             }
         }
-
+        
         // this is a complicated way to get the Path
         LocalServiceURI loc = new LocalServiceURI(getResourceID());
         VOSURI vu = loc.getURI(node);

cavern/src/main/java/org/opencadc/cavern/nodes/NodeUtil.java

@@ -112,6 +112,8 @@
 import org.opencadc.vospace.NodeProperty;
 import org.opencadc.vospace.VOS;
 import org.opencadc.vospace.VOSURI;
+import org.opencadc.vospace.server.PathResolver;
+import org.opencadc.vospace.server.auth.VOSpaceAuthorizer;
 
 /**
  * Utility methods for interacting with nodes. This is now like a DAO class
@@ -230,12 +232,20 @@ public void put(Node node, VOSURI uri)
             } else if (node instanceof LinkNode) {
                 log.debug("[create] link: " + np);
                 LinkNode ln = (LinkNode) node;
-                String targPath = ln.getTarget().getPath().substring(1);
-                Path absPath = root.resolve(targPath);
-                Path rel = np.getParent().relativize(absPath);
-                log.debug("[create] link: " + np + "\ntarget: " + targPath
-                        + "\nabs: " + absPath + "\nrel: " + rel);
-                ret = Files.createSymbolicLink(np, rel);
+                URI target = ln.getTarget();
+                if ("file".equals(target.getScheme())) {
+                    // link to external filesystem object
+                    String path = target.getPath();
+                    log.debug("[create] external link: " + np + "\ntarget: " + target + "\nabs: " + path);
+                    ret = Files.createSymbolicLink(np, root.getFileSystem().getPath(path));
+                } else {
+                    String targPath = ln.getTarget().getPath().substring(1);
+                    Path absPath = root.resolve(targPath);
+                    Path rel = np.getParent().relativize(absPath);
+                    log.debug("[create] link: " + np + "\ntarget: " + targPath
+                            + "\nabs: " + absPath + "\nrel: " + rel);
+                    ret = Files.createSymbolicLink(np, rel);
+                }
             } else {
                 throw new UnsupportedOperationException(
                         "unexpected node type: " + node.getClass().getName());
@@ -433,7 +443,7 @@ public void move(VOSURI source, VOSURI destDir, PosixPrincipal owner, String des
         }
         VOSURI destWithName = new VOSURI(URI.create(destDir.toString() + "/" + destName));
         Path destPath = nodeToPath(root, destWithName);
-        log.warn("atomic move: " + sourcePath + " -> " + destPath);
+        log.debug("atomic move: " + sourcePath + " -> " + destPath);
         try {
             Files.move(sourcePath, destPath, StandardCopyOption.ATOMIC_MOVE);
         } catch (AtomicMoveNotSupportedException atomicMoveNotSupportedException) {
@@ -498,11 +508,26 @@ Node pathToNode(Path p, boolean getAttrs)
             Path tp = Files.readSymbolicLink(p);
             Path abs = p.getParent().resolve(tp);
             Path rel = root.relativize(abs);
-            URI turi = URI.create(rootURI.getScheme() + "://"
+            log.debug("[pathToNode] link: " +  p + "\ntarget: " + tp
+                + "\nabs: " + abs
+                + "\nrel: " + rel);
+            if (!abs.startsWith(root)) {
+                // absolute link to an additional mounted filesystem (double slash: omit host from URI)
+                URI turi = URI.create("file://" + abs.toString());
+                log.debug("[pathToNode] link: " + abs + " -> " + turi);
+                ret = new LinkNode(p.getFileName().toString(), turi);
+            } else if (!rel.startsWith("..")) {
+                // link inside vos filesystem
+                URI turi = URI.create(rootURI.getScheme() + "://"
                     + rootURI.getAuthority() + "/" + rel.toString());
-            log.debug("[pathToNode] link: " + p + "\ntarget: " + tp + "\nabs: "
-                    + abs + "\nrel: " + rel + "\nuri: " + turi);
-            ret = new LinkNode(p.getFileName().toString(), turi);
+                log.debug("[pathToNode] link: " + abs + " -> " + turi);
+                ret = new LinkNode(p.getFileName().toString(), turi);
+            } else {
+                // relative link to target outside the vos filesystem
+                URI turi = URI.create("file://" + tp.toString());
+                log.debug("[pathToNode] external relative link: " + abs + " -> " + turi);
+                ret = new LinkNode(p.getFileName().toString(), turi);
+            }
         } else {
             throw new IllegalStateException(
                     "found unexpected file system object: " + p);