Accept links to external targets (CADC-13802) (#229)

* Support for external links

---------

Co-authored-by: Adrian Damian <[email protected]>

.github/workflows/cibuild.yml

@@ -13,10 +13,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - name: Set up Python 3.10
+      - name: Set up Python 3.11
         uses: actions/setup-python@v5
         with:
-          python-version: "3.10"
+          python-version: "3.11"
       - name: Install tox
         run: python -m pip install --upgrade tox
       - name: egg-info vos
@@ -28,7 +28,7 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        python-version: ["3.7","3.8","3.9","3.10","3.11","3.12"]
+        python-version: ["3.8","3.9","3.10","3.11","3.12", "3.13"]
         package: [vos]
     steps:
     - name: Checkout code
@@ -54,10 +54,10 @@ jobs:
     needs: tests
     steps:
       - uses: actions/checkout@v4
-      - name: Set up Python 3.10
+      - name: Set up Python 3.11
         uses: actions/setup-python@v5
         with:
-          python-version: "3.10"
+          python-version: "3.11"
       - name: Setup Graphviz
         uses: ts-graphviz/setup-graphviz@v2
       - name: Install tox

vos/setup.cfg

@@ -52,11 +52,11 @@ edit_on_github = False
 github_project = opencadc/vostools
 install_requires =
     html2text>=2016.5.29
-    cadcutils>=1.5.3
+    cadcutils>=1.5.4
     aenum
 
 # version should be PEP440 compatible (http://www.python.org/dev/peps/pep-0440)
-version = 3.6.1.1
+version = 3.6.2
 
 [options.extras_require]
 test =

vos/test/scripts_local/vospace-link-atest.tcsh

@@ -136,9 +136,8 @@ foreach resource ($resources)
   echo " [OK]"
 
   echo -n "create link to unknown scheme in URI"
-  #TODO not sure why this is not working anymore
-  #$LNCMD $CERT unknown://cadc.nrc.ca~vault/CADCAuthtest1 $CONTAINER/e2link >& /dev/null && echo " [FAIL]" && exit -1
-  echo " [SKIPPED - TODO]"
+  $LNCMD $CERT unknown://cadc.nrc.ca~vault/CADCAuthtest1 $CONTAINER/e2link >& /dev/null && echo " [FAIL]" && exit -1
+  echo " [OK]"
 
   echo -n "follow the invalid link and fail"
   $CPCMD $CERT $CONTAINER/e2link/somefile /tmp  >& /dev/null && echo " [FAIL]" && exit -1

vos/test/scripts_prod/vospace-link-atest.tcsh

@@ -130,9 +130,8 @@ foreach resource ($resources)
   echo " [OK]"
 
   echo -n "create link to unknown scheme in URI"
-  #TODO not sure why this is not working anymore
-  #$LNCMD $CERT unknown://cadc.nrc.ca~vault/CADCRegtest1 $CONTAINER/e2link >& /dev/null && echo " [FAIL]" && exit -1
-  echo " [SKIPPED - TODO]"
+  $LNCMD $CERT file:///cadc.nrc.ca~vault/CADCRegtest1 $CONTAINER/e2link >& /dev/null || echo " [FAIL]" && exit -1
+  echo " [OK]"
 
   echo -n "follow the invalid link and fail"
   $CPCMD $CERT $CONTAINER/e2link/somefile /tmp  >& /dev/null && echo " [FAIL]" && exit -1

vos/tox.ini

@@ -5,7 +5,7 @@ name = vos
 
 [tox]
 envlist =
-    py{37,38,39,310,311,312}
+    py{38,39,310,311,312,313}
 requires =
     pip >= 19.3.1
 

vos/vos/commands/vln.py

@@ -2,7 +2,7 @@
 # ******************  CANADIAN ASTRONOMY DATA CENTRE  *******************
 # *************  CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES  **************
 #
-#  (c) 2022.                            (c) 2022.
+#  (c) 2025.                            (c) 2025.
 #  Government of Canada                 Gouvernement du Canada
 #  National Research Council            Conseil national de recherches
 #  Ottawa, Canada, K1A 0R6              Ottawa, Canada, K1A 0R6
@@ -89,7 +89,8 @@
 
     vln vos:vospace/junk.txt vos:vospace/linkToJunk.txt
     vln vos:vospace/directory vos:vospace/linkToDirectory
-    vln http://external.data.source vos:vospace/linkToExternalDataSource
+    vln https://external.data.source vos:vospace/linkToExternalDataSource
+    vln file:///data/localfile vos:vospace/linkToLocalFile
 
 """.format(URI_DESCRIPTION)
 
@@ -106,11 +107,10 @@ def vln():
             vospace_certfile=opt.certfile,
             vospace_token=opt.token,
             insecure=opt.insecure)
-        if not client.is_remote_file(opt.source) or \
-                not client.is_remote_file(opt.target):
+        if not client.is_remote_file(opt.target):
             raise ArgumentError(
                 None,
-                "source must be vos node or http url, target must be vos node")
+                "target must be vos node")
 
         client.link(opt.source, opt.target)
     except ArgumentError as ex:

vos/vos/tests/test_vos.py

@@ -330,51 +330,36 @@ def test_glob(self):
         # /anode/abc /anode/def - > anode/a* should return
         # /anode/adc
 
-        mock_node = MagicMock(type='vos:ContainerNode')
-        mock_node.configure_mock(name='anode')
-        mock_child_node1 = Mock(type='vos:DataNode')
-        mock_child_node1.name = 'abc'
-        mock_child_node2 = Mock(type='vos:DataNode')
-        mock_child_node2.name = 'def'
-        # because we use wild characters in the root node,
-        # we need to create a corresponding node for the base node
-        mock_base_node = Mock(type='vos:ContainerNode')
-        mock_base_node.name = 'vos:'
-        mock_base_node.node_list = [mock_node]
-        mock_node.node_list = [mock_base_node, mock_child_node1,
-                               mock_child_node2]
         client = Client()
-        client.get_node = Mock(
-            side_effect=[mock_node, mock_base_node, mock_node])
+        client.listdir = Mock(
+            return_value=['abc', 'def'])  # list parent dir and anode dir
         self.assertEqual(['vos:/anode/abc'], client.glob('vos:/anode/a*'))
+        client.listdir = Mock(
+            return_value=['abc', 'def'])  # list parent dir and anode dir
+        self.assertEqual([], client.glob('vos:/anode/m*'))
+        client.access = Mock()
+        client.listdir = Mock(
+            side_effect=[['anode'], ['abc', 'def']])
         self.assertEqual(['vos:/anode/abc'], client.glob('vos:/*node/abc'))
+        client.listdir = Mock(
+            side_effect=[['anode'], ['abc', 'def']])
+        self.assertEqual([], client.glob('vos:/*foo/abc'))
 
         # test nodes:
         # /anode/.test1 /bnode/sometests /bnode/blah
-        # /[a,c]node/*test* should return /bnode/somtests (.test1 is filtered
+        # /[a,c]node/*test* should return /bnode/sometests (.test1 is filtered
         # out as a special file)
 
-        mock_node1 = MagicMock(type='vos:ContainerNode')
-        mock_node1.configure_mock(name='anode')
-        mock_node1.node_list = [mock_child_node1]
-
-        mock_child_node2 = Mock(type='vos:DataNode')
-        mock_child_node2.name = 'sometests'
-        mock_child_node3 = Mock(type='vos:DataNode')
-        mock_child_node3.name = 'blah'
-        mock_node2 = MagicMock(type='vos:ContainerNode')
-        mock_node2.configure_mock(name='bnode')
-        mock_node2.node_list = [mock_child_node2, mock_child_node3]
-
         # because we use wild characters in the root node,
         # we need to create a corresponding node for the base node
-        mock_base_node = Mock(type='vos:DataNode')
-        mock_base_node.name = 'vos:'
-        mock_base_node.node_list = [mock_node1, mock_node2]
         client = Client()
         client.is_remote_file = Mock()
-        client.get_node = Mock(
-            side_effect=[mock_base_node, mock_node1, mock_node2])
+        dirs = ['anode', 'bnode']
+        anode = ['.test']
+        bnode = ['sometests', 'blah']
+        client.listdir = Mock(
+            side_effect=[dirs, anode, bnode])
+        client.access = Mock()
         self.assertEqual(['vos:/bnode/sometests'],
                          client.glob('vos:/[a,b]node/*test*'))
 

vos/vos/vos.py

@@ -2,7 +2,7 @@
 # ******************  CANADIAN ASTRONOMY DATA CENTRE  *******************
 # *************  CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES  **************
 #
-#  (c) 2024.                            (c) 2024.
+#  (c) 2025.                            (c) 2025.
 #  Government of Canada                 Gouvernement du Canada
 #  National Research Council            Conseil national de recherches
 #  Ottawa, Canada, K1A 0R6              Ottawa, Canada, K1A 0R6
@@ -811,7 +811,7 @@ def get_children(self, client, sort, order, limit=None):
         """ Gets an iterator over the nodes held to by a ContainerNode"""
         # IF THE CALLER KNOWS THEY DON'T NEED THE CHILDREN THEY
         # CAN SET LIMIT=0 IN THE CALL Also, if the number of nodes
-        # on the firt call was less than 500, we likely got them
+        # on the first call was less than 500, we likely got them
         # all during the init
         if not self.isdir():
             return
@@ -1890,7 +1890,7 @@ def copy(self, source, destination, send_md5=False, disposition=False,
                     # purposes:
                     #   1. Check if source is identical to destination and
                     #   avoid sending the bytes again.
-                    #   2. send info to the service so it can recover in case
+                    #   2. send info to the service so that it can recover in case
                     #   the bytes got corrupted on the way
                     src_md5 = md5_cache.MD5Cache.compute_md5(source)
                     if src_md5 == dest_node_md5:
@@ -2186,8 +2186,7 @@ def get_node_url(self, uri, method='GET', view=None, limit=None,
     def link(self, src_uri, link_uri):
         """Make link_uri point to src_uri.
 
-        :param src_uri: the existing resource, either a vospace uri or a http
-        url
+        :param src_uri: the existing resource to link to
         :type src_uri: unicode
         :param link_uri: the vospace node to create that will be a link to
         src_uri
@@ -2197,7 +2196,8 @@ def link(self, src_uri, link_uri):
         HttpException exceptions declared in the cadcutils.exceptions module
         """
         link_uri = self.fix_uri(link_uri)
-        src_uri = self.fix_uri(src_uri)
+        if "://" not in src_uri:
+            src_uri = self.fix_uri(src_uri)
 
         # if the link_uri points at an existing directory then we try and
         # make a link into that directory
@@ -2723,24 +2723,23 @@ def get_info_list(self, uri):
 
     def listdir(self, uri, force=False):
         """
-        Walk through the directory structure a la os.walk.
-        Setting force=True will make sure no cached results are used.
+        Return a list with the content of the directory
         Follows LinksNodes to their destination location.
+        Note: this method returns a list of children names. For larger
+        directories, use get_children_info() to iterate through it and
+        avoid loading the entire content into memory.
 
         :param force: don't use cached values, retrieve from service.
         :param uri: The ContainerNode to get a listing of.
         :rtype [unicode]
         """
-        names = []
         logger.debug(str(uri))
-        node = self.get_node(uri, limit=None, force=force)
+        node = self.get_node(uri, limit=0, force=force)
         while node.type == "vos:LinkNode":
             uri = node.target
             # logger.debug(uri)
-            node = self.get_node(uri, limit=None, force=force)
-        for thisNode in node.node_list:
-            names.append(thisNode.name)
-        return names
+            node = self.get_node(uri, limit=0, force=force)
+        return [i.name for i in self.get_children_info(node.uri, force=force)]
 
     def _node_type(self, uri):
         """