Merge pull request cylc#5564 from cylc/8.1.x-deconflict

🤖 Merge 8.1.x-deconflict into master

CHANGES.md

@@ -32,14 +32,19 @@ short option for `--no-detach` to `-N`; add `-r` as a short option for
 [#5525](https://github.com/cylc/cylc-flow/pull/5525) - Jobs can use scripts
 in `share/bin` and Python modules in `share/lib/python`.
 
-### Fixes
-
 [#5328](https://github.com/cylc/cylc-flow/pull/5328) -
 Efficiency improvements to reduce task management overheads on the Scheduler.
 
 -------------------------------------------------------------------------------
 ## __cylc-8.1.5 (<span actions:bind='release-date'>Upcoming</span>)__
 
+### Breaking Changes
+
+[#5600](https://github.com/cylc/cylc-flow/pull/5600) -
+The `CYLC_TASK_DEPENDENCIES` environment variable will no longer be exported
+in job environments if there are more than 50 dependencies. This avoids an
+issue which could cause jobs to fail if this variable became too long.
+
 ### Enhancements
 
 [#5546](https://github.com/cylc/cylc-flow/pull/5546) -
@@ -48,13 +53,13 @@ This can be overridden using the new `--exit-zero` flag.
 
 ### Fixes
 
+[#5524](https://github.com/cylc/cylc-flow/pull/5524) - Logging includes timestamps
+for `cylc play` when called by `cylc vip` or `cylc vr`.
+
 [#5228](https://github.com/cylc/cylc-flow/pull/5228) -
 Enabled the "stop", "poll", "kill" and "message" commands to be issued from
 the UI whilst the workflow is in the process of shutting down.
 
-[#5524](https://github.com/cylc/cylc-flow/pull/5524) - Logging includes timestamps
-for `cylc play` when called by `cylc vip` or `cylc vr`.
-
 -------------------------------------------------------------------------------
 ## __cylc-8.1.4 (<span actions:bind='release-date'>Released 2023-05-04</span>)__
 

cylc/flow/job_file.py

@@ -28,6 +28,13 @@
 from cylc.flow.option_parsers import verbosity_to_env
 from cylc.flow.config import interpolate_template, ParamExpandError
 
+# the maximum number of task dependencies which Cylc will list before
+# omitting the CYLC_TASK_DEPENDENCIES environment variable
+# see: https://github.com/cylc/cylc-flow/issues/5551
+# NOTE: please update `src/reference/job-script-vars/var-list.txt`
+#       in cylc-doc if changing this value
+MAX_CYLC_TASK_DEPENDENCIES_LEN = 50
+
 
 class JobFileWriter:
 
@@ -220,9 +227,18 @@ def _write_task_environment(self, handle, job_conf):
         handle.write(
             '\n    export CYLC_TASK_NAMESPACE_HIERARCHY="%s"' %
             ' '.join(job_conf['namespace_hierarchy']))
-        handle.write(
-            '\n    export CYLC_TASK_DEPENDENCIES="%s"' %
-            ' '.join(job_conf['dependencies']))
+        if len(job_conf['dependencies']) <= MAX_CYLC_TASK_DEPENDENCIES_LEN:
+            handle.write(
+                '\n    export CYLC_TASK_DEPENDENCIES="%s"' %
+                ' '.join(job_conf['dependencies']))
+        else:
+            # redact the CYLC_TASK_DEPENDENCIES variable but leave a note
+            # explaining why
+            # see: https://github.com/cylc/cylc-flow/issues/5551
+            handle.write(
+                '\n    # CYLC_TASK_DEPENDENCIES=disabled'
+                f' (more than {MAX_CYLC_TASK_DEPENDENCIES_LEN} dependencies)'
+            )
         handle.write(
             '\n    export CYLC_TASK_TRY_NUMBER=%s' % job_conf['try_num'])
         handle.write(

tests/unit/test_job_file.py

@@ -29,7 +29,10 @@
     __version__,
     __file__ as cylc_flow_file,
 )
-from cylc.flow.job_file import JobFileWriter
+from cylc.flow.job_file import (
+    JobFileWriter,
+    MAX_CYLC_TASK_DEPENDENCIES_LEN,
+)
 from cylc.flow.platforms import platform_from_name
 
 
@@ -533,3 +536,47 @@ def test_homeless_platform(fixture_get_platform):
         job_sh_txt = job_sh.read()
         if 'HOME' in job_sh_txt:
             raise Exception('$HOME found in job.sh\n{job_sh_txt}')
+
+
+def test_cylc_task_dependencies_length():
+    f"""Test CYLC_TASK_DEPENDENCIES variable toggle.
+
+    The CYLC_TASK_DEPENDENCIES variriable should only be exported if there are
+    { MAX_CYLC_TASK_DEPENDENCIES_LEN } or fewer dependencies.
+
+    See: https://github.com/cylc/cylc-flow/issues/5551
+
+    """
+    job_conf = {
+        'platform': {'communication method': 'zmq'},
+        'job_d': 'a/b/c',
+        'namespace_hierarchy': ['a', 'b'],
+        # the maximum permitted number of dependencies before the environment
+        # variable is omitted
+        'dependencies': ['a'] * (MAX_CYLC_TASK_DEPENDENCIES_LEN),
+        'try_num': 1,
+        'flow_nums': {1},
+        'param_var': {},
+        'work_d': 'b/c/d',
+    }
+
+    # write the job environment
+    with io.StringIO() as fake_file:
+        JobFileWriter()._write_task_environment(fake_file, job_conf)
+        output = fake_file.getvalue()
+
+    # assert the env var is exported
+    lines = [line.strip().split('=')[0] for line in output.splitlines()]
+    assert 'export CYLC_TASK_DEPENDENCIES' in lines
+
+    # add an extra dependency to push it over the limit
+    job_conf['dependencies'] += ['b']
+
+    # write the job environment
+    with io.StringIO() as fake_file:
+        JobFileWriter()._write_task_environment(fake_file, job_conf)
+        output = fake_file.getvalue()
+
+    # assert the env var is redacted
+    lines = [line.strip().split('=')[0] for line in output.splitlines()]
+    assert '# CYLC_TASK_DEPENDENCIES' in lines  # var should be commented out