From be22d50b1b10be76b8edb394a35ad0b09b234020 Mon Sep 17 00:00:00 2001 From: Ausbeth Date: Wed, 23 Oct 2024 12:12:40 +0100 Subject: [PATCH] Cleaning up (removing previous code on upgrade) --- shpc/client/__init__.py | 34 +--- shpc/client/help.py | 33 ---- shpc/client/upgrade.py | 190 ---------------------- shpc/main/client.py | 10 -- shpc/tests/test_client.py | 325 -------------------------------------- shpc/tests/test_client.sh | 11 -- 6 files changed, 1 insertion(+), 602 deletions(-) delete mode 100644 shpc/client/upgrade.py diff --git a/shpc/client/__init__.py b/shpc/client/__init__.py index 890f98c02..c7186862d 100644 --- a/shpc/client/__init__.py +++ b/shpc/client/__init__.py @@ -332,35 +332,6 @@ def get_parser(): action="store_true", ) - # Upgrade a software to its latest version - upgrade = subparsers.add_parser( - "upgrade", - description=help.upgrade_description, - formatter_class=argparse.RawTextHelpFormatter, - ) - upgrade.add_argument( - "upgrade_recipe", - help="software to upgrade", - nargs="?", - ) - - upgrade.add_argument( - "--all", - "-a", - help="upgrade all installed software.", - dest="upgrade_all", - action="store_true", - ) - - upgrade.add_argument( - "--force", - "-f", - dest="force", - help="force upgrade without prompting for confirmation to uninstall current version(s) or install latest version to view(s).", - default=False, - action="store_true", - ) - # Update gets latest tags from OCI registries update = subparsers.add_parser( "update", @@ -414,7 +385,7 @@ def get_parser(): action="store_true", ) - for command in update, upgrade, install, sync: + for command in update, install, sync: command.add_argument( "--dry-run", "-d", @@ -437,7 +408,6 @@ def get_parser(): shell, test, uninstall, - upgrade, view, ]: command.add_argument( @@ -585,8 +555,6 @@ def help(return_code=0): from .uninstall import main elif args.command == "update": from .update import main - elif args.command == "upgrade": - from .upgrade import main elif args.command == "sync-registry": from .sync import sync_registry as main diff --git a/shpc/client/help.py b/shpc/client/help.py index ceb451a5c..34405d2b3 100644 --- a/shpc/client/help.py +++ b/shpc/client/help.py @@ -253,36 +253,3 @@ # Filter all modules to those with "python" $ shpc show --filter python """ - -upgrade_description = """Upgrade software to the latest version. - - # Upgrade a software to its latest version and give option to uninstall older versions or not. - # Do not include the version in the command - $ shpc upgrade quay.io/biocontainers/samtools - - # Upgrade all software - $ shpc upgrade --all - OR - $ shpc upgrade -a - - # Valid arguement combinations: - # Perform dry-run on a software to check if the latest is installed or not without upgrading it. - $ shpc upgrade quay.io/biocontainers/samtools --dry-run - OR - $ shpc upgrade quay.io/biocontainers/samtools -d - - # Perform dry-run to show version details of all installed software, to check if the latest version is installed or not without upgrading them. - $ shpc upgrade --all --dry-run - OR - $ shpc upgrade -a -d - - # Invalid arguement combinations: - $ shpc upgrade quay.io/biocontainers/samtools --all - OR - $ shpc upgrade quay.io/biocontainers/samtools -a - - $ shpc upgrade quay.io/biocontainers/samtools --all --dry-run - OR - $ shpc upgrade quay.io/biocontainers/samtools -a -d - -""" diff --git a/shpc/client/upgrade.py b/shpc/client/upgrade.py deleted file mode 100644 index 78df6b3f1..000000000 --- a/shpc/client/upgrade.py +++ /dev/null @@ -1,190 +0,0 @@ -__author__ = "Vanessa Sochat" -__copyright__ = "Copyright 2021-2024, Vanessa Sochat" -__license__ = "MPL 2.0" - -import shpc.utils as utils -from shpc.logger import logger - - -def main(args, parser, extra, subparser): - from shpc.main import get_client - - utils.ensure_no_extra(extra) - - cli = get_client(quiet=args.quiet, settings_file=args.settings_file) - - # Update config settings on the fly - cli.settings.update_params(args.config_params) - - # Check if user entered an incomplete command - if not args.upgrade_recipe and not args.upgrade_all: - subparser.error( - "Incomplete command. The following arguments are required: upgrade_recipe, --all, or -h for more details" - ) - - # Get the list of installed software - installed_software = cli.list(return_modules=True) - - # Ensure the user has software installed before carrying out upgrade - if not installed_software: - logger.exit( - "Cannot perform shpc upgrade because you currently do not have any software installed.", - 0, - ) - - # Upgrade a specific installed software - if args.upgrade_recipe: - # Check if the provided recipe is known in any registry - try: - cli._load_container(args.upgrade_recipe) - except SystemExit: - # Give additional messages relating to shpc upgrade, to the original exit message in _load_container function - logger.exit( - "This means it cannot be upgraded because it is not installed, and cannot be installed because it is not known in any registry.\nPlease check the name or try a different recipe." - ) - # Check if the user typed an invalid argument combination - if args.upgrade_all: - logger.exit( - "Cannot use '--all' with a specific recipe. Please choose one option." - ) - # Check if the user specified a version - if ":" in args.upgrade_recipe: - logger.exit("Please use 'shpc upgrade recipe' without including a version.") - # Check if the specific software is installed - if args.upgrade_recipe not in installed_software: - logger.exit( - f"You currently do not have {args.upgrade_recipe} installed.\nYou can install it with this command: shpc install {args.upgrade_recipe}", - 0, - ) - - # Does the user just want a dry-run of the specific software? - if args.dryrun: - version_info = upgrade( - args.upgrade_recipe, cli, args, dryrun=True - ) # This returns the latest version if its available, else returns None - if version_info: - logger.info( - f"You do not have the latest version installed.\nLatest version avaiable is {version_info}" - ) - else: - logger.info( - f"You have the latest version of {args.upgrade_recipe} installed." - ) - - # Upgade the software - else: - upgrade(args.upgrade_recipe, cli, args, dryrun=False, force=args.force) - - # Upgrade all installed software - elif args.upgrade_all: - # Store the number of all outdated software - num_outdated = 0 - - # Does the user just want a dry-run of all software? - if args.dryrun: - print("Performing a dry-run on all your software...") - for software in installed_software.keys(): - version_info = upgrade(software, cli, args, dryrun=True) - if version_info: - logger.info( - f"{software} is outdated. Latest version available is {version_info}" - ) - num_outdated += 1 - else: - logger.info(f"{software} is up to date.") - # Provide a report on the dry-run - if num_outdated == 0: - logger.info("All your software are currently up to date.") - else: - logger.info(f"You have a total of {num_outdated} outdated software.") - - # Upgrade all software - else: - print("Checking your list to upgrade outdated software...") - for software in installed_software.keys(): - # Attempt upgrade on each software - upgrade_info = upgrade( - software, cli, args, dryrun=False, force=args.force - ) - # Count actual upgrades - if upgrade_info: - num_outdated += 1 - if num_outdated == 0: - logger.info("No upgrade needed. All your software are up to date.") - else: - logger.info( - f"Updated {num_outdated} outdated software from your list\nAll your software are now up to date." - ) - - -def upgrade(name, cli, args, dryrun=False, force=False): - """ - Upgrade a software to its latest version. Or preview available upgrades from the user's software list - """ - # Add namespace - name = cli.add_namespace(name) - - # Load the container configuration for the specified recipe - config = cli._load_container(name) - - # Store the installed versions and the latest version tag - installed_versions = cli.list(pattern=name, return_modules=True) - latest_version_tag = get_latest_version(name, config) - - # Compare the latest version with the user's installed version - if any(latest_version_tag in versions for versions in installed_versions.values()): - if not dryrun: - logger.info("You have the latest version of " + name + " installed already") - return None # No upgrade necessary - - else: - if dryrun: - return ( - latest_version_tag # Return the latest version for upgrade information - ) - print( - "Upgrading " - + name - + " to its latest version. Version " - + latest_version_tag - ) - - # Get the list of views the software was in - views_with_module = set() - view_dir = cli.new_module(name).module_dir - for view_name, entry in cli.views.items(): - if entry.exists(view_dir): - views_with_module.add(view_name) - - # Ask if the user wants to unintall old versions - if not cli.uninstall(name, force=force): - logger.info("Old versions of " + name + " were preserved") - - # Install the latest version - cli.install(name) - - # Install the latest version to views where the outdated version was found - if views_with_module: - msg = f"Do you also want to install the latest version of {name} to the view(s) of the previous version(s)?" - if utils.confirm_action(msg, force=force): - for view_name in views_with_module: - cli.view_install(view_name, name) - logger.info( - f"Installed the latest version of {name} to view: {view_name}" - ) - - return latest_version_tag # Upgrade occured - - -def get_latest_version(name, config): - """ - Given an added namespace of a recipe and a loaded container configuration of that namespace, - Retrieve the latest version tag. - """ - latest_version_info = config.get("latest") - if not latest_version_info: - logger.exit(f"No latest version found for {name}") - - # Extract the latest version tag - latest_version_tag = list(latest_version_info.keys())[0] - return latest_version_tag diff --git a/shpc/main/client.py b/shpc/main/client.py index a221d93f8..7ef602b92 100644 --- a/shpc/main/client.py +++ b/shpc/main/client.py @@ -62,16 +62,6 @@ def install(self, name, tag=None, **kwargs): """ raise NotImplementedError - def upgrade(self, name, dryrun=False, force=False): - """ - Upgrade an outdated software - """ - from shpc.client.upgrade import upgrade - - cli = self - args = {} - upgrade(name, cli, args, dryrun=dryrun, force=force) - def uninstall(self, name, tag=None): """ Uninstall must also implemented by the subclass (e.g., lmod) diff --git a/shpc/tests/test_client.py b/shpc/tests/test_client.py index c2315862c..6cdfc85cf 100644 --- a/shpc/tests/test_client.py +++ b/shpc/tests/test_client.py @@ -9,14 +9,11 @@ import io import os import shutil -from unittest import mock import pytest -import shpc.main.modules.views as views import shpc.main.registry as registry import shpc.utils -from shpc.client.upgrade import get_latest_version as glv from .helpers import here, init_client @@ -413,325 +410,3 @@ def test_remove(tmp_path): # Remove the module (with force) client.remove(module, force=True) assert client.registry.exists(module) is None - - -@pytest.mark.parametrize( - "module_sys, module_file, container_tech, remote, dryrun", - [ - ("lmod", "module.lua", "singularity", False, False), - ("lmod", "module.lua", "podman", False, False), - ("tcl", "module.tcl", "singularity", False, False), - ("tcl", "module.tcl", "podman", False, False), - ("lmod", "module.lua", "singularity", True, False), - ("lmod", "module.lua", "podman", True, False), - ("tcl", "module.tcl", "singularity", True, False), - ("tcl", "module.tcl", "podman", True, False), - ("lmod", "module.lua", "singularity", False, True), - ("lmod", "module.lua", "podman", False, True), - ("tcl", "module.tcl", "singularity", False, True), - ("tcl", "module.tcl", "podman", False, True), - ("lmod", "module.lua", "singularity", True, True), - ("lmod", "module.lua", "podman", True, True), - ("tcl", "module.tcl", "singularity", True, True), - ("tcl", "module.tcl", "podman", True, True), - ], -) -def test_upgrade_software_with_force( - tmp_path, module_sys, module_file, container_tech, remote, dryrun -): - """ - Test upgrading a software where uninstalling older versions and installing latest version to view is also done. - """ - client = init_client(str(tmp_path), module_sys, container_tech, remote=remote) - - # Install an outdated version of a software - client.install("quay.io/biocontainers/samtools:1.18--h50ea8bc_1") - - # Create the default view if it doesn't exist - view_handler = views.ViewsHandler( - settings_file=client.settings.settings_file, module_sys=module_sys - ) - assert "mpi" not in client.views - view_handler.create("mpi") - client.detect_views() - assert "mpi" in client.views - view = client.views["mpi"] - assert view.path == os.path.join(tmp_path, "views", "mpi") and os.path.exists( - view.path - ) - assert os.path.exists(view.config_path) - assert view._config["view"]["name"] == "mpi" - assert not view._config["view"]["modules"] - assert not view._config["view"]["system_modules"] - - # Install the software to the view - client.view_install("mpi", "quay.io/biocontainers/samtools:1.18--h50ea8bc_1") - - # Upgrade the software to its latest version - client.upgrade("quay.io/biocontainers/samtools", dryrun=dryrun, force=True) - - # Load the container configuration for the software and get its latest version tag - name = client.add_namespace("quay.io/biocontainers/samtools") - config = client._load_container(name) - latest_version = glv(name, config) - - # Verify if the latest version of the software was installed - if not dryrun: - # Verify the module's directory exists and module files were installed - module_dir = os.path.join( - client.settings.module_base, - "quay.io/biocontainers/samtools", - latest_version, - ) - assert os.path.exists(module_dir), "Latest version directiory should exist." - module_file_path = os.path.join(module_dir, module_file) - assert os.path.exists( - module_file_path - ), "Latest version's module files should be installed." - - # Check if the older version's module directory was removed and if its module files were uninstalled - module_dir_old = os.path.join( - client.settings.module_base, - "quay.io/biocontainers/samtools", - "1.18--h50ea8bc_1", - ) - assert not os.path.exists(module_dir_old), "Older version should be uninstalled" - module_file_path = os.path.join(module_dir_old, module_file) - assert not os.path.exists( - module_file_path - ), "Older version's module files should be uninstalled." - - # Ensure the latest version was added to the view - assert client.views["mpi"].exists( - module_dir - ), "Upgraded software should be added to the view 'mpi'" - - # Verify that the latest version of the software was not installed if dry-run is TRUE - else: - module_dir = os.path.join( - client.settings.module_base, - "quay.io/biocontainers/samtools", - latest_version, - ) - assert not os.path.exists(module_dir), "Latest version should not be installed." - module_file_path = os.path.join(module_dir, module_file) - assert not os.path.exists( - module_file_path - ), "Latest version's module files should not be installed." - - -@pytest.mark.parametrize( - "module_sys,module_file,container_tech,remote", - [ - ("lmod", "module.lua", "singularity", False), - ("lmod", "module.lua", "podman", False), - ("tcl", "module.tcl", "singularity", False), - ("tcl", "module.tcl", "podman", False), - ("lmod", "module.lua", "singularity", True), - ("lmod", "module.lua", "podman", True), - ("tcl", "module.tcl", "singularity", True), - ("tcl", "module.tcl", "podman", True), - ], -) -@mock.patch("shpc.utils.confirm_action") -def test_upgrade_software_without_force( - mock_confirm_action, tmp_path, module_sys, module_file, container_tech, remote -): - """ - Test upgrading a software where uninstalling older versions and installing latest version to view is not done. - """ - client = init_client(str(tmp_path), module_sys, container_tech, remote=remote) - - # Install an outdated version of a software - client.install("quay.io/biocontainers/samtools:1.18--h50ea8bc_1") - - # Create the default view if it doesn't exist - view_handler = views.ViewsHandler( - settings_file=client.settings.settings_file, module_sys=module_sys - ) - assert "mpi" not in client.views - view_handler.create("mpi") - client.detect_views() - assert "mpi" in client.views - view = client.views["mpi"] - assert view.path == os.path.join(tmp_path, "views", "mpi") and os.path.exists( - view.path - ) - assert os.path.exists(view.config_path) - assert view._config["view"]["name"] == "mpi" - assert not view._config["view"]["modules"] - assert not view._config["view"]["system_modules"] - - # Install the software to the view - client.view_install("mpi", "quay.io/biocontainers/samtools:1.18--h50ea8bc_1") - - # Simulate user's choice for uninstalling older versions and installing latest version to the views of the older versions - mock_confirm_action.return_value = False - - # Upgrade the software to its latest version - client.upgrade("quay.io/biocontainers/samtools", dryrun=False, force=False) - - # Load the container configuration for the software and get its latest version tag - name = client.add_namespace("quay.io/biocontainers/samtools") - config = client._load_container(name) - latest_version = glv(name, config) - - # Verify the module's directory exists and module files were installed - module_dir = os.path.join( - client.settings.module_base, "quay.io/biocontainers/samtools", latest_version - ) - assert os.path.exists(module_dir), "Latest version directiory should exist." - module_file_path = os.path.join(module_dir, module_file) - assert os.path.exists( - module_file_path - ), "Latest version's module files should be installed." - - # Ensure the older version's module directory still exists and its module files were not uninstalled - module_dir_old = os.path.join( - client.settings.module_base, - "quay.io/biocontainers/samtools", - "1.18--h50ea8bc_1", - ) - assert os.path.exists(module_dir_old), "Old version should not be uninstalled" - module_file_path = os.path.join(module_dir_old, module_file) - assert os.path.exists( - module_file_path - ), "Older version's module files should not be uninstalled." - - # Ensure the latest version was not added to the view - assert not client.views["mpi"].exists( - module_dir - ), "Upgraded software should not added to the view 'mpi'" - - -@pytest.mark.parametrize( - "module_sys,module_file,container_tech,remote", - [ - ("lmod", "module.lua", "singularity", False), - ("lmod", "module.lua", "podman", False), - ("tcl", "module.tcl", "singularity", False), - ("tcl", "module.tcl", "podman", False), - ("lmod", "module.lua", "singularity", True), - ("lmod", "module.lua", "podman", True), - ("tcl", "module.tcl", "singularity", True), - ("tcl", "module.tcl", "podman", True), - ], -) -def test_upgrade_with_latest_already_installed( - tmp_path, module_sys, module_file, container_tech, remote -): - client = init_client(str(tmp_path), module_sys, container_tech, remote=remote) - - # Load the container configuration for the software and get is latest version - name = client.add_namespace("quay.io/biocontainers/samtools") - config = client._load_container(name) - latest_version = glv(name, config) - - # Install an outdated and the latest version of a software - client.install(f"quay.io/biocontainers/samtools:{latest_version}") - client.install("quay.io/biocontainers/samtools:1.18--h50ea8bc_1") - - # Verify the latest version's module directory exists and module files were installed - module_dir = os.path.join( - client.settings.module_base, "quay.io/biocontainers/samtools", latest_version - ) - assert os.path.exists(module_dir), "Latest version directiory should exist." - module_file_path = os.path.join(module_dir, module_file) - assert os.path.exists(module_file_path), "Latest version module files should exist." - - # Capture the time the directory was created - module_dir_mtime_before = os.path.getmtime(module_dir) - - # Perform upgrade - client.upgrade("quay.io/biocontainers/samtools", dryrun=False, force=True) - - # Capture the time of the directory after upgrade was done - module_dir_mtime_after = os.path.getmtime(module_dir) - - # Ensure the directory did not change, to signify upgrade was not performed when the latest was already installed - assert ( - module_dir_mtime_after == module_dir_mtime_before - ), "Upgrade should not occur if latest is installed already." - - -@pytest.mark.parametrize( - "module_sys,module_file,container_tech,remote", - [ - ("lmod", "module.lua", "singularity", False), - ("lmod", "module.lua", "podman", False), - ("tcl", "module.tcl", "singularity", False), - ("tcl", "module.tcl", "podman", False), - ("lmod", "module.lua", "singularity", True), - ("lmod", "module.lua", "podman", True), - ("tcl", "module.tcl", "singularity", True), - ("tcl", "module.tcl", "podman", True), - ], -) -def test_upgrade_all_software( - tmp_path, module_sys, module_file, container_tech, remote -): - """ - Test upgrading a software where uninstalling older versions and installing latest version to view is also done. - """ - client = init_client(str(tmp_path), module_sys, container_tech, remote=remote) - - # Install two different outdated software - client.install("quay.io/biocontainers/samtools:1.18--h50ea8bc_1") - client.install("quay.io/biocontainers/bwa:0.7.18--he4a0461_0") - - # Create the default view if it doesn't exist - view_handler = views.ViewsHandler( - settings_file=client.settings.settings_file, module_sys=module_sys - ) - assert "mpi" not in client.views - view_handler.create("mpi") - client.detect_views() - assert "mpi" in client.views - view = client.views["mpi"] - assert view.path == os.path.join(tmp_path, "views", "mpi") and os.path.exists( - view.path - ) - assert os.path.exists(view.config_path) - assert view._config["view"]["name"] == "mpi" - assert not view._config["view"]["modules"] - assert not view._config["view"]["system_modules"] - - # Install the software to the view - client.view_install("mpi", "quay.io/biocontainers/samtools:1.18--h50ea8bc_1") - client.view_install("mpi", "quay.io/biocontainers/bwa:0.7.18--he4a0461_0") - - # Upgrade all software to thei latest version - installed_software = client.list(return_modules=True) - for software, versions in installed_software.items(): - client.upgrade(software, dryrun=False, force=True) - - # Load the container configuration for the software and get their latest version tag - name = client.add_namespace(software) - config = client._load_container(name) - latest_version = glv(name, config) - - # Verify the module's directory exists and module files were installed - module_dir = os.path.join(client.settings.module_base, software, latest_version) - assert os.path.exists(module_dir), "Latest version directiory should exist." - module_file_path = os.path.join(module_dir, module_file) - assert os.path.exists( - module_file_path - ), "Latest version's module files should be installed." - - for older_version in versions: - # Check if the older version's module directory was removed and if its module files were uninstalled - module_dir_old = os.path.join( - client.settings.module_base, software, older_version - ) - assert not os.path.exists( - module_dir_old - ), "Older version should be uninstalled" - module_file_path = os.path.join(module_dir_old, module_file) - assert not os.path.exists( - module_file_path - ), "Older version's module files should be uninstalled." - - # Ensure the latest versions were added to the view - assert client.views["mpi"].exists( - module_dir - ), "Upgraded software should be added to the view 'mpi'" diff --git a/shpc/tests/test_client.sh b/shpc/tests/test_client.sh index 761745c73..c7db9ece2 100755 --- a/shpc/tests/test_client.sh +++ b/shpc/tests/test_client.sh @@ -67,17 +67,6 @@ runTest 0 $output shpc --settings-file $settings install python:3.9.2-slim runTest 0 $output shpc --settings-file $settings install vanessa/salad:latest runTest 0 $output shpc --settings-file $settings install --container_tech podman python -echo -echo "#### Testing upgrade " -runTest 0 $output shpc --settings-file $settings upgrade --help -runTest 0 $output shpc --settings-file $settings install quay.io/biocontainers/samtools:1.20--h50ea8bc_0 -runTest 0 $output shpc --settings-file $settings install quay.io/biocontainers/bioconductor-bags:2.40.0--r43ha9d7317_0 -runTest 0 $output shpc --settings-file $settings install quay.io/biocontainers/bwa:0.7.18--he4a0461_1 -runTest 0 $output shpc --settings-file $settings upgrade quay.io/biocontainers/samtools --dry-run -runTest 0 $output shpc --settings-file $settings upgrade quay.io/biocontainers/samtools --force -runTest 0 $output shpc --settings-file $settings upgrade --all --dry-run -runTest 0 $output shpc --settings-file $settings upgrade --all --force - echo echo "#### Testing get " runTest 0 $output shpc --settings-file $settings get --help