diff --git a/.github/workflows/cibuild.yml b/.github/workflows/cibuild.yml index 28540c00..bf58a226 100644 --- a/.github/workflows/cibuild.yml +++ b/.github/workflows/cibuild.yml @@ -13,10 +13,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Set up Python 3.10 + - name: Set up Python 3.11 uses: actions/setup-python@v5 with: - python-version: "3.10" + python-version: "3.11" - name: Install tox run: python -m pip install --upgrade tox - name: egg-info vos @@ -28,7 +28,7 @@ jobs: strategy: fail-fast: true matrix: - python-version: ["3.7","3.8","3.9","3.10","3.11","3.12"] + python-version: ["3.8","3.9","3.10","3.11","3.12", "3.13"] package: [vos] steps: - name: Checkout code @@ -54,10 +54,10 @@ jobs: needs: tests steps: - uses: actions/checkout@v4 - - name: Set up Python 3.10 + - name: Set up Python 3.11 uses: actions/setup-python@v5 with: - python-version: "3.10" + python-version: "3.11" - name: Setup Graphviz uses: ts-graphviz/setup-graphviz@v2 - name: Install tox diff --git a/vos/setup.cfg b/vos/setup.cfg index 873e0d07..66ff6010 100644 --- a/vos/setup.cfg +++ b/vos/setup.cfg @@ -52,11 +52,11 @@ edit_on_github = False github_project = opencadc/vostools install_requires = html2text>=2016.5.29 - cadcutils>=1.5.3 + cadcutils>=1.5.4 aenum # version should be PEP440 compatible (http://www.python.org/dev/peps/pep-0440) -version = 3.6.1.1 +version = 3.6.2 [options.extras_require] test = diff --git a/vos/test/scripts_local/vospace-link-atest.tcsh b/vos/test/scripts_local/vospace-link-atest.tcsh index a6639422..ea2ce3d6 100755 --- a/vos/test/scripts_local/vospace-link-atest.tcsh +++ b/vos/test/scripts_local/vospace-link-atest.tcsh @@ -136,9 +136,8 @@ foreach resource ($resources) echo " [OK]" echo -n "create link to unknown scheme in URI" - #TODO not sure why this is not working anymore - #$LNCMD $CERT unknown://cadc.nrc.ca~vault/CADCAuthtest1 $CONTAINER/e2link >& /dev/null && echo " [FAIL]" && exit -1 - echo " [SKIPPED - TODO]" + $LNCMD $CERT unknown://cadc.nrc.ca~vault/CADCAuthtest1 $CONTAINER/e2link >& /dev/null && echo " [FAIL]" && exit -1 + echo " [OK]" echo -n "follow the invalid link and fail" $CPCMD $CERT $CONTAINER/e2link/somefile /tmp >& /dev/null && echo " [FAIL]" && exit -1 diff --git a/vos/test/scripts_prod/vospace-link-atest.tcsh b/vos/test/scripts_prod/vospace-link-atest.tcsh index 61bdd3ed..c7507a40 100755 --- a/vos/test/scripts_prod/vospace-link-atest.tcsh +++ b/vos/test/scripts_prod/vospace-link-atest.tcsh @@ -130,9 +130,8 @@ foreach resource ($resources) echo " [OK]" echo -n "create link to unknown scheme in URI" - #TODO not sure why this is not working anymore - #$LNCMD $CERT unknown://cadc.nrc.ca~vault/CADCRegtest1 $CONTAINER/e2link >& /dev/null && echo " [FAIL]" && exit -1 - echo " [SKIPPED - TODO]" + $LNCMD $CERT file:///cadc.nrc.ca~vault/CADCRegtest1 $CONTAINER/e2link >& /dev/null || echo " [FAIL]" && exit -1 + echo " [OK]" echo -n "follow the invalid link and fail" $CPCMD $CERT $CONTAINER/e2link/somefile /tmp >& /dev/null && echo " [FAIL]" && exit -1 diff --git a/vos/tox.ini b/vos/tox.ini index 892d3425..80a01ed6 100644 --- a/vos/tox.ini +++ b/vos/tox.ini @@ -5,7 +5,7 @@ name = vos [tox] envlist = - py{37,38,39,310,311,312} + py{38,39,310,311,312,313} requires = pip >= 19.3.1 diff --git a/vos/vos/commands/vln.py b/vos/vos/commands/vln.py index 0d405d46..c38dcabf 100755 --- a/vos/vos/commands/vln.py +++ b/vos/vos/commands/vln.py @@ -2,7 +2,7 @@ # ****************** CANADIAN ASTRONOMY DATA CENTRE ******************* # ************* CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** # -# (c) 2022. (c) 2022. +# (c) 2025. (c) 2025. # Government of Canada Gouvernement du Canada # National Research Council Conseil national de recherches # Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 @@ -89,7 +89,8 @@ vln vos:vospace/junk.txt vos:vospace/linkToJunk.txt vln vos:vospace/directory vos:vospace/linkToDirectory - vln http://external.data.source vos:vospace/linkToExternalDataSource + vln https://external.data.source vos:vospace/linkToExternalDataSource + vln file:///data/localfile vos:vospace/linkToLocalFile """.format(URI_DESCRIPTION) @@ -106,11 +107,10 @@ def vln(): vospace_certfile=opt.certfile, vospace_token=opt.token, insecure=opt.insecure) - if not client.is_remote_file(opt.source) or \ - not client.is_remote_file(opt.target): + if not client.is_remote_file(opt.target): raise ArgumentError( None, - "source must be vos node or http url, target must be vos node") + "target must be vos node") client.link(opt.source, opt.target) except ArgumentError as ex: diff --git a/vos/vos/tests/test_vos.py b/vos/vos/tests/test_vos.py index c13ee090..29743971 100644 --- a/vos/vos/tests/test_vos.py +++ b/vos/vos/tests/test_vos.py @@ -330,51 +330,36 @@ def test_glob(self): # /anode/abc /anode/def - > anode/a* should return # /anode/adc - mock_node = MagicMock(type='vos:ContainerNode') - mock_node.configure_mock(name='anode') - mock_child_node1 = Mock(type='vos:DataNode') - mock_child_node1.name = 'abc' - mock_child_node2 = Mock(type='vos:DataNode') - mock_child_node2.name = 'def' - # because we use wild characters in the root node, - # we need to create a corresponding node for the base node - mock_base_node = Mock(type='vos:ContainerNode') - mock_base_node.name = 'vos:' - mock_base_node.node_list = [mock_node] - mock_node.node_list = [mock_base_node, mock_child_node1, - mock_child_node2] client = Client() - client.get_node = Mock( - side_effect=[mock_node, mock_base_node, mock_node]) + client.listdir = Mock( + return_value=['abc', 'def']) # list parent dir and anode dir self.assertEqual(['vos:/anode/abc'], client.glob('vos:/anode/a*')) + client.listdir = Mock( + return_value=['abc', 'def']) # list parent dir and anode dir + self.assertEqual([], client.glob('vos:/anode/m*')) + client.access = Mock() + client.listdir = Mock( + side_effect=[['anode'], ['abc', 'def']]) self.assertEqual(['vos:/anode/abc'], client.glob('vos:/*node/abc')) + client.listdir = Mock( + side_effect=[['anode'], ['abc', 'def']]) + self.assertEqual([], client.glob('vos:/*foo/abc')) # test nodes: # /anode/.test1 /bnode/sometests /bnode/blah - # /[a,c]node/*test* should return /bnode/somtests (.test1 is filtered + # /[a,c]node/*test* should return /bnode/sometests (.test1 is filtered # out as a special file) - mock_node1 = MagicMock(type='vos:ContainerNode') - mock_node1.configure_mock(name='anode') - mock_node1.node_list = [mock_child_node1] - - mock_child_node2 = Mock(type='vos:DataNode') - mock_child_node2.name = 'sometests' - mock_child_node3 = Mock(type='vos:DataNode') - mock_child_node3.name = 'blah' - mock_node2 = MagicMock(type='vos:ContainerNode') - mock_node2.configure_mock(name='bnode') - mock_node2.node_list = [mock_child_node2, mock_child_node3] - # because we use wild characters in the root node, # we need to create a corresponding node for the base node - mock_base_node = Mock(type='vos:DataNode') - mock_base_node.name = 'vos:' - mock_base_node.node_list = [mock_node1, mock_node2] client = Client() client.is_remote_file = Mock() - client.get_node = Mock( - side_effect=[mock_base_node, mock_node1, mock_node2]) + dirs = ['anode', 'bnode'] + anode = ['.test'] + bnode = ['sometests', 'blah'] + client.listdir = Mock( + side_effect=[dirs, anode, bnode]) + client.access = Mock() self.assertEqual(['vos:/bnode/sometests'], client.glob('vos:/[a,b]node/*test*')) diff --git a/vos/vos/vos.py b/vos/vos/vos.py index 95a77bb1..08b11825 100644 --- a/vos/vos/vos.py +++ b/vos/vos/vos.py @@ -2,7 +2,7 @@ # ****************** CANADIAN ASTRONOMY DATA CENTRE ******************* # ************* CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** # -# (c) 2024. (c) 2024. +# (c) 2025. (c) 2025. # Government of Canada Gouvernement du Canada # National Research Council Conseil national de recherches # Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 @@ -811,7 +811,7 @@ def get_children(self, client, sort, order, limit=None): """ Gets an iterator over the nodes held to by a ContainerNode""" # IF THE CALLER KNOWS THEY DON'T NEED THE CHILDREN THEY # CAN SET LIMIT=0 IN THE CALL Also, if the number of nodes - # on the firt call was less than 500, we likely got them + # on the first call was less than 500, we likely got them # all during the init if not self.isdir(): return @@ -1890,7 +1890,7 @@ def copy(self, source, destination, send_md5=False, disposition=False, # purposes: # 1. Check if source is identical to destination and # avoid sending the bytes again. - # 2. send info to the service so it can recover in case + # 2. send info to the service so that it can recover in case # the bytes got corrupted on the way src_md5 = md5_cache.MD5Cache.compute_md5(source) if src_md5 == dest_node_md5: @@ -2186,8 +2186,7 @@ def get_node_url(self, uri, method='GET', view=None, limit=None, def link(self, src_uri, link_uri): """Make link_uri point to src_uri. - :param src_uri: the existing resource, either a vospace uri or a http - url + :param src_uri: the existing resource to link to :type src_uri: unicode :param link_uri: the vospace node to create that will be a link to src_uri @@ -2197,7 +2196,8 @@ def link(self, src_uri, link_uri): HttpException exceptions declared in the cadcutils.exceptions module """ link_uri = self.fix_uri(link_uri) - src_uri = self.fix_uri(src_uri) + if "://" not in src_uri: + src_uri = self.fix_uri(src_uri) # if the link_uri points at an existing directory then we try and # make a link into that directory @@ -2723,24 +2723,23 @@ def get_info_list(self, uri): def listdir(self, uri, force=False): """ - Walk through the directory structure a la os.walk. - Setting force=True will make sure no cached results are used. + Return a list with the content of the directory Follows LinksNodes to their destination location. + Note: this method returns a list of children names. For larger + directories, use get_children_info() to iterate through it and + avoid loading the entire content into memory. :param force: don't use cached values, retrieve from service. :param uri: The ContainerNode to get a listing of. :rtype [unicode] """ - names = [] logger.debug(str(uri)) - node = self.get_node(uri, limit=None, force=force) + node = self.get_node(uri, limit=0, force=force) while node.type == "vos:LinkNode": uri = node.target # logger.debug(uri) - node = self.get_node(uri, limit=None, force=force) - for thisNode in node.node_list: - names.append(thisNode.name) - return names + node = self.get_node(uri, limit=0, force=force) + return [i.name for i in self.get_children_info(node.uri, force=force)] def _node_type(self, uri): """