From 74c4a55e697e373193a8c67e6b596e1100496e69 Mon Sep 17 00:00:00 2001 From: Marek Hrvol Date: Thu, 6 Feb 2025 13:40:14 +0100 Subject: [PATCH 1/5] Changed default ubuntu version of Nextflow applets --- CHANGELOG.md | 8 +++ src/python/dxpy/nextflow/nextflow_builder.py | 57 ++++++++++--------- src/python/dxpy/nextflow/nextflow_utils.py | 29 ++++++++++ .../templating/templates/nextflow/dxapp.json | 2 +- src/python/test/test_nextflow.py | 3 +- 5 files changed, 69 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b01a833935..ce8a4eed93 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,14 @@ Categories for each release: Added, Changed, Deprecated, Removed, Fixed, Securit ## Unreleased +### Added + +* Ubuntu version can be overridden for Nextflow pipelines during build time with `dx build --nextflow --extra-args={"runSpec":{"release":"..."}}` + +### Changed + +* Default Ubuntu version of Nextflow pipelines is now 24.04 +* ## [389.0] - beta ### Fixed diff --git a/src/python/dxpy/nextflow/nextflow_builder.py b/src/python/dxpy/nextflow/nextflow_builder.py index 15a06f17bf..ab66dd3caa 100644 --- a/src/python/dxpy/nextflow/nextflow_builder.py +++ b/src/python/dxpy/nextflow/nextflow_builder.py @@ -6,9 +6,11 @@ from glob import glob import shutil import tempfile +from functools import partial from dxpy.nextflow.nextflow_templates import (get_nextflow_dxapp, get_nextflow_src) -from dxpy.nextflow.nextflow_utils import (get_template_dir, write_exec, write_dxapp, get_importer_name, create_readme) +from dxpy.nextflow.nextflow_utils import (get_template_dir, write_exec, write_dxapp, get_importer_name, + create_readme, get_nested, get_allowed_extra_fields_mapping) from dxpy.cli.exec_io import parse_obj from dxpy.cli import try_call from dxpy.utils.resolver import resolve_existing_path @@ -49,45 +51,44 @@ def build_pipeline_with_npi( Runs the Nextflow Pipeline Importer app, which creates a Nextflow applet from a given Git repository. """ - def parse_extra_args(extra_args): - dx_input = {} - if extra_args.get("name") is not None: - dx_input["name"] = extra_args.get("name") - if extra_args.get("title") is not None: - dx_input["title"] = extra_args.get("title") - if extra_args.get("summary") is not None: - dx_input["summary"] = extra_args.get("summary") - if extra_args.get("runSpec", {}).get("timeoutPolicy") is not None: - dx_input["timeout_policy"] = extra_args.get("runSpec", {}).get("timeoutPolicy") - if extra_args.get("details", {}).get("whatsNew") is not None: - dx_input["whats_new"] = extra_args.get("details", {}).get("whatsNew") - return dx_input + def parse_extra_args(args): + """ + Returns overridable fields from extra_args + :param args: extra args from command input + :return: + """ + return { + target_key: val + for arg_path, target_key in get_allowed_extra_fields_mapping() + if (val := get_nested(args, arg_path)) is not None + } extra_args = extra_args or {} build_project_id = dxpy.WORKSPACE_ID build_folder = None input_hash = parse_extra_args(extra_args) input_hash["repository_url"] = repository - if tag: - input_hash["repository_tag"] = tag - if profile: - input_hash["config_profile"] = profile - if git_creds: - input_hash["github_credentials"] = parse_obj(git_creds, "file") + + # { NPI_input_name: (raw value, transformation function),...} + input_updates = { + "repository_tag": (tag, None), + "config_profile": (profile, None), + "cache_docker": (cache_docker, None), + "nextflow_pipeline_params": (nextflow_pipeline_params, None), + "docker_secrets": (docker_secrets, partial(parse_obj, klass="file")), + "github_credentials": (git_creds, partial(parse_obj, klass="file")), + } + for key, (raw_value, transform) in input_updates.items(): + if raw_value: + input_hash[key] = transform(raw_value) if transform else raw_value + if destination: build_project_id, build_folder, _ = try_call(resolve_existing_path, destination, expected='folder') - if docker_secrets: - input_hash["docker_secrets"] = parse_obj(docker_secrets, "file") - if cache_docker: - input_hash["cache_docker"] = cache_docker - if nextflow_pipeline_params: - input_hash["nextflow_pipeline_params"] = nextflow_pipeline_params - if build_project_id is None: parser.error( "Can't create an applet without specifying a destination project; please use the -d/--destination flag to explicitly specify a project") - nf_builder_job = dxpy.DXApp(name=get_importer_name()).run(app_input=input_hash, project=build_project_id, + nf_builder_job = dxpy.DXApplet("applet-GyXBgzj01xy03BB454gXJxP8").run(applet_input=input_hash, project=build_project_id, folder=build_folder, name="Nextflow build of %s" % (repository), detach=True) diff --git a/src/python/dxpy/nextflow/nextflow_utils.py b/src/python/dxpy/nextflow/nextflow_utils.py index a9a577b764..513caa4f67 100644 --- a/src/python/dxpy/nextflow/nextflow_utils.py +++ b/src/python/dxpy/nextflow/nextflow_utils.py @@ -155,3 +155,32 @@ def get_nextflow_assets(region): with open(nextaur_assets, 'r') as nextaur_f, open(nextflow_assets, 'r') as nextflow_f, open(awscli_assets, 'r') as awscli_f: return json.load(nextaur_f)[region], json.load(nextflow_f)[region], json.load(awscli_f)[region] + +def get_nested(d, path): + """ + Given a dict 'd' and a tuple or list of keys in 'path', + returns the nested value if it exists, otherwise None. + """ + for key in path: + if not isinstance(d, dict): + return None + d = d.get(key) + if d is None: + return None + return d + + +def get_allowed_extra_fields_mapping(): + """ + :return: tuple (arg_path, target_key) + arg_path is a list of a dxapp.json location of an allowed extra_arg, target_key is name of an argument for a remote build + """ + return [ + (["name"], "name"), + (["title"], "title"), + (["summary"], "summary"), + (["runSpec", "timeoutPolicy"], "timeout_policy"), + (["runSpec", "release"], "release"), + (["details", "whatsNew"], "whats_new"), + ] + diff --git a/src/python/dxpy/templating/templates/nextflow/dxapp.json b/src/python/dxpy/templating/templates/nextflow/dxapp.json index fb67420031..1bdd122378 100644 --- a/src/python/dxpy/templating/templates/nextflow/dxapp.json +++ b/src/python/dxpy/templating/templates/nextflow/dxapp.json @@ -94,7 +94,7 @@ "interpreter": "bash", "execDepends": [], "distribution": "Ubuntu", - "release": "20.04", + "release": "24.04", "version": "0" }, "details": { diff --git a/src/python/test/test_nextflow.py b/src/python/test/test_nextflow.py index 11e29baf39..21633836d3 100755 --- a/src/python/test/test_nextflow.py +++ b/src/python/test/test_nextflow.py @@ -241,7 +241,7 @@ def test_dx_build_nextflow_with_extra_args(self): pipeline_name, existing_nf_file_path=self.base_nextflow_nf) # Override metadata values - extra_args = '{"name": "name-9Oxvx2tCZe", "title": "Title VsnhPeFBqt", "summary": "Summary 3E7fFfEXdB"}' + extra_args = '{"name": "name-9Oxvx2tCZe", "title": "Title VsnhPeFBqt", "summary": "Summary 3E7fFfEXdB", "runSpec":{"release":"20.04"}}' applet_id = json.loads(run( "dx build --nextflow '{}' --json --extra-args '{}'".format(applet_dir, extra_args)))["id"] @@ -250,6 +250,7 @@ def test_dx_build_nextflow_with_extra_args(self): self.assertEqual(desc["name"], json.loads(extra_args)["name"]) self.assertEqual(desc["title"], json.loads(extra_args)["title"]) self.assertEqual(desc["summary"], json.loads(extra_args)["summary"]) + self.assertEqual(desc["runSpec"]["release"], json.loads(extra_args)["runSpec"]["release"]) details = applet.get_details() self.assertEqual(details["repository"], "local") From d088784381ac8ad43735952bbf06df8109630f6a Mon Sep 17 00:00:00 2001 From: Marek Hrvol <52779097+mhrvol@users.noreply.github.com> Date: Mon, 10 Feb 2025 12:33:12 +0100 Subject: [PATCH 2/5] Update src/python/dxpy/nextflow/nextflow_utils.py Co-authored-by: Jan Dvorsky --- src/python/dxpy/nextflow/nextflow_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python/dxpy/nextflow/nextflow_utils.py b/src/python/dxpy/nextflow/nextflow_utils.py index 513caa4f67..beccbba6b9 100644 --- a/src/python/dxpy/nextflow/nextflow_utils.py +++ b/src/python/dxpy/nextflow/nextflow_utils.py @@ -172,7 +172,7 @@ def get_nested(d, path): def get_allowed_extra_fields_mapping(): """ - :return: tuple (arg_path, target_key) + :returns: tuple (arg_path, target_key) arg_path is a list of a dxapp.json location of an allowed extra_arg, target_key is name of an argument for a remote build """ return [ From 610897ec3419677732acdbf1504d04bdc90da29a Mon Sep 17 00:00:00 2001 From: Marek Hrvol <52779097+mhrvol@users.noreply.github.com> Date: Mon, 10 Feb 2025 12:33:21 +0100 Subject: [PATCH 3/5] Update src/python/test/test_nextflow.py Co-authored-by: Jan Dvorsky --- src/python/test/test_nextflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python/test/test_nextflow.py b/src/python/test/test_nextflow.py index 21633836d3..bac1bbbfb4 100755 --- a/src/python/test/test_nextflow.py +++ b/src/python/test/test_nextflow.py @@ -241,7 +241,7 @@ def test_dx_build_nextflow_with_extra_args(self): pipeline_name, existing_nf_file_path=self.base_nextflow_nf) # Override metadata values - extra_args = '{"name": "name-9Oxvx2tCZe", "title": "Title VsnhPeFBqt", "summary": "Summary 3E7fFfEXdB", "runSpec":{"release":"20.04"}}' + extra_args = '{"name": "name-9Oxvx2tCZe", "title": "Title VsnhPeFBqt", "summary": "Summary 3E7fFfEXdB", "runSpec": {"release": "20.04"}}' applet_id = json.loads(run( "dx build --nextflow '{}' --json --extra-args '{}'".format(applet_dir, extra_args)))["id"] From 7c44b2b73cc44af2590bfebbf7b7cdadccbc69a8 Mon Sep 17 00:00:00 2001 From: Marek Hrvol <52779097+mhrvol@users.noreply.github.com> Date: Mon, 10 Feb 2025 14:35:59 +0100 Subject: [PATCH 4/5] Update src/python/dxpy/nextflow/nextflow_builder.py Co-authored-by: jsitarova-dnanexus <100697844+jsitarova-dnanexus@users.noreply.github.com> --- src/python/dxpy/nextflow/nextflow_builder.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/python/dxpy/nextflow/nextflow_builder.py b/src/python/dxpy/nextflow/nextflow_builder.py index ab66dd3caa..e64406277e 100644 --- a/src/python/dxpy/nextflow/nextflow_builder.py +++ b/src/python/dxpy/nextflow/nextflow_builder.py @@ -53,9 +53,8 @@ def build_pipeline_with_npi( def parse_extra_args(args): """ - Returns overridable fields from extra_args :param args: extra args from command input - :return: + :returns: overridable fields from extra_args """ return { target_key: val From 944bd2851a1a3cef7ab46fc41ebfa61d31def722 Mon Sep 17 00:00:00 2001 From: Marek Hrvol Date: Mon, 10 Feb 2025 14:50:14 +0100 Subject: [PATCH 5/5] updated var names + comment --- src/python/dxpy/nextflow/nextflow_utils.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/python/dxpy/nextflow/nextflow_utils.py b/src/python/dxpy/nextflow/nextflow_utils.py index beccbba6b9..c50c6d256f 100644 --- a/src/python/dxpy/nextflow/nextflow_utils.py +++ b/src/python/dxpy/nextflow/nextflow_utils.py @@ -156,18 +156,21 @@ def get_nextflow_assets(region): with open(nextaur_assets, 'r') as nextaur_f, open(nextflow_assets, 'r') as nextflow_f, open(awscli_assets, 'r') as awscli_f: return json.load(nextaur_f)[region], json.load(nextflow_f)[region], json.load(awscli_f)[region] -def get_nested(d, path): +def get_nested(args, arg_path): """ - Given a dict 'd' and a tuple or list of keys in 'path', - returns the nested value if it exists, otherwise None. + :param args: extra args from command input + :type args: dict + :param arg_path: list of a dxapp.json location of an allowed extra_arg (eg. ["runSpec", "timeoutPolicy"]) + :type arg_path: tuple/list + :returns: nested arg value if it exists in args, otherwise None """ - for key in path: - if not isinstance(d, dict): + for key in arg_path: + if not isinstance(args, dict): return None - d = d.get(key) - if d is None: + args = args.get(key) + if args is None: return None - return d + return args def get_allowed_extra_fields_mapping():