DEVEX-2461 Noble in NF pipelines by default

CHANGELOG.md

@@ -6,6 +6,14 @@ Categories for each release: Added, Changed, Deprecated, Removed, Fixed, Securit
 
 ## Unreleased
 
+### Added
+
+* Ubuntu version can be overridden for Nextflow pipelines during build time with `dx build --nextflow --extra-args={"runSpec":{"release":"..."}}`
+
+### Changed
+
+* Default Ubuntu version of Nextflow pipelines is now 24.04
+* 
 ## [389.0] - beta
 
 ### Fixed

src/python/dxpy/nextflow/nextflow_builder.py

@@ -6,9 +6,11 @@
 from glob import glob
 import shutil
 import tempfile
+from functools import partial
 
 from dxpy.nextflow.nextflow_templates import (get_nextflow_dxapp, get_nextflow_src)
-from dxpy.nextflow.nextflow_utils import (get_template_dir, write_exec, write_dxapp, get_importer_name, create_readme)
+from dxpy.nextflow.nextflow_utils import (get_template_dir, write_exec, write_dxapp, get_importer_name,
+                                          create_readme, get_nested, get_allowed_extra_fields_mapping)
 from dxpy.cli.exec_io import parse_obj
 from dxpy.cli import try_call
 from dxpy.utils.resolver import resolve_existing_path
@@ -49,45 +51,43 @@ def build_pipeline_with_npi(
     Runs the Nextflow Pipeline Importer app, which creates a Nextflow applet from a given Git repository.
     """
 
-    def parse_extra_args(extra_args):
-        dx_input = {}
-        if extra_args.get("name") is not None:
-            dx_input["name"] = extra_args.get("name")
-        if extra_args.get("title") is not None:
-            dx_input["title"] = extra_args.get("title")
-        if extra_args.get("summary") is not None:
-            dx_input["summary"] = extra_args.get("summary")
-        if extra_args.get("runSpec", {}).get("timeoutPolicy") is not None:
-            dx_input["timeout_policy"] = extra_args.get("runSpec", {}).get("timeoutPolicy")
-        if extra_args.get("details", {}).get("whatsNew") is not None:
-            dx_input["whats_new"] = extra_args.get("details", {}).get("whatsNew")
-        return dx_input
+    def parse_extra_args(args):
+        """
+        :param args: extra args from command input
+        :returns: overridable fields from extra_args
+        """
+        return {
+            target_key: val
+            for arg_path, target_key in get_allowed_extra_fields_mapping()
+            if (val := get_nested(args, arg_path)) is not None
+        }
 
     extra_args = extra_args or {}
     build_project_id = dxpy.WORKSPACE_ID
     build_folder = None
     input_hash = parse_extra_args(extra_args)
     input_hash["repository_url"] = repository
-    if tag:
-        input_hash["repository_tag"] = tag
-    if profile:
-        input_hash["config_profile"] = profile
-    if git_creds:
-        input_hash["github_credentials"] = parse_obj(git_creds, "file")
+
+    # { NPI_input_name: (raw value, transformation function),...}
+    input_updates = {
+        "repository_tag": (tag, None),
+        "config_profile": (profile, None),
+        "cache_docker": (cache_docker, None),
+        "nextflow_pipeline_params": (nextflow_pipeline_params, None),
+        "docker_secrets": (docker_secrets, partial(parse_obj, klass="file")),
+        "github_credentials": (git_creds, partial(parse_obj, klass="file")),
+    }
+    for key, (raw_value, transform) in input_updates.items():
+        if raw_value:
+            input_hash[key] = transform(raw_value) if transform else raw_value
+
     if destination:
         build_project_id, build_folder, _ = try_call(resolve_existing_path, destination, expected='folder')
-    if docker_secrets:
-        input_hash["docker_secrets"] = parse_obj(docker_secrets, "file")
-    if cache_docker:
-        input_hash["cache_docker"] = cache_docker
-    if nextflow_pipeline_params:
-        input_hash["nextflow_pipeline_params"] = nextflow_pipeline_params
-
     if build_project_id is None:
         parser.error(
             "Can't create an applet without specifying a destination project; please use the -d/--destination flag to explicitly specify a project")
 
-    nf_builder_job = dxpy.DXApp(name=get_importer_name()).run(app_input=input_hash, project=build_project_id,
+    nf_builder_job = dxpy.DXApplet("applet-GyXBgzj01xy03BB454gXJxP8").run(applet_input=input_hash, project=build_project_id,
                                                               folder=build_folder,
                                                               name="Nextflow build of %s" % (repository), detach=True)
 

src/python/dxpy/nextflow/nextflow_utils.py

@@ -155,3 +155,35 @@ def get_nextflow_assets(region):
 
         with open(nextaur_assets, 'r') as nextaur_f, open(nextflow_assets, 'r') as nextflow_f, open(awscli_assets, 'r') as awscli_f:
             return json.load(nextaur_f)[region], json.load(nextflow_f)[region], json.load(awscli_f)[region]
+
+def get_nested(args, arg_path):
+    """
+    :param args: extra args from command input
+    :type args: dict
+    :param arg_path: list of a dxapp.json location of an allowed extra_arg (eg. ["runSpec", "timeoutPolicy"])
+    :type arg_path: tuple/list
+    :returns: nested arg value if it exists in args, otherwise None
+    """
+    for key in arg_path:
+        if not isinstance(args, dict):
+            return None
+        args = args.get(key)
+        if args is None:
+            return None
+    return args
+
+
+def get_allowed_extra_fields_mapping():
+    """
+    :returns: tuple (arg_path, target_key)
+        arg_path is a list of a dxapp.json location of an allowed extra_arg, target_key is name of an argument for a remote build
+    """
+    return [
+        (["name"], "name"),
+        (["title"], "title"),
+        (["summary"], "summary"),
+        (["runSpec", "timeoutPolicy"], "timeout_policy"),
+        (["runSpec", "release"], "release"),
+        (["details", "whatsNew"], "whats_new"),
+    ]
+

src/python/dxpy/templating/templates/nextflow/dxapp.json

@@ -94,7 +94,7 @@
     "interpreter": "bash",
     "execDepends": [],
     "distribution": "Ubuntu",
-    "release": "20.04",
+    "release": "24.04",
     "version": "0"
   },
   "details": {

src/python/test/test_nextflow.py

@@ -241,7 +241,7 @@ def test_dx_build_nextflow_with_extra_args(self):
             pipeline_name, existing_nf_file_path=self.base_nextflow_nf)
 
         # Override metadata values
-        extra_args = '{"name": "name-9Oxvx2tCZe", "title": "Title VsnhPeFBqt", "summary": "Summary 3E7fFfEXdB"}'
+        extra_args = '{"name": "name-9Oxvx2tCZe", "title": "Title VsnhPeFBqt", "summary": "Summary 3E7fFfEXdB", "runSpec": {"release": "20.04"}}'
         applet_id = json.loads(run(
             "dx build --nextflow '{}' --json --extra-args '{}'".format(applet_dir, extra_args)))["id"]
 
@@ -250,6 +250,7 @@ def test_dx_build_nextflow_with_extra_args(self):
         self.assertEqual(desc["name"], json.loads(extra_args)["name"])
         self.assertEqual(desc["title"], json.loads(extra_args)["title"])
         self.assertEqual(desc["summary"], json.loads(extra_args)["summary"])
+        self.assertEqual(desc["runSpec"]["release"], json.loads(extra_args)["runSpec"]["release"])
 
         details = applet.get_details()
         self.assertEqual(details["repository"], "local")