Change RMUTL to use ZOAU shell commands for compatibility

galaxy.yml

@@ -8,7 +8,7 @@ namespace: ibm
 name: ibm_zos_cics
 
 # The collection version
-version: 2.1.0
+version: 2.2.0
 
 # Collection README file
 readme: README.md

plugins/module_utils/_data_set_utils.py

@@ -13,7 +13,8 @@
 from ansible_collections.ibm.ibm_zos_core.plugins.module_utils.ansible_module import AnsibleModuleHelper
 from ansible_collections.ibm.ibm_zos_cics.plugins.module_utils._response import _execution, MVSExecutionException
 from ansible_collections.ibm.ibm_zos_core.plugins.module_utils.zos_mvs_raw import MVSCmd
-from ansible_collections.ibm.ibm_zos_core.plugins.module_utils.dd_statement import DDStatement, StdoutDefinition, DatasetDefinition, StdinDefinition
+from ansible_collections.ibm.ibm_zos_core.plugins.module_utils.dd_statement import DDStatement, StdoutDefinition, StdinDefinition
+from ansible_collections.ibm.ibm_zos_core.plugins.module_utils.job import job_output
 
 MVS_CMD_RETRY_ATTEMPTS = 10
 
@@ -273,6 +274,85 @@ def _execute_command(command):
     module = AnsibleModuleHelper(argument_spec={})
     return module.run_command(command)
 
+def _submit_jcl(jcl_uss_path, job_name):
+    executions =[]
+    command = "jsub -f '{0}'".format(jcl_uss_path)
+
+    rc, stdout,stderr = _execute_command(command)
+    executions.append(
+    _execution(
+        name="Submit jcl job {0} for {1}".format(jcl_uss_path, job_name),
+        rc=rc,
+        stdout=stdout,
+        stderr=stderr))
+    if rc != 0:
+        raise MVSExecutionException(
+            "RC {0} when submitting jcl from {1}".format(
+                rc, jcl_uss_path), executions)
+    return executions
+
+
+def _get_job_output(job_id, job_name):
+    executions = []
+
+    try:
+        jobs = job_output(job_id=job_id, job_name=job_name)
+
+        #There should only be one job found for the JCL submitted
+        if (len(jobs) != 1):
+            raise MVSExecutionException(
+                "Query for job status for {0} with job id {1} returned more than one result. Jobs returned: {2}".format(
+                    job_name,
+                    job_id, 
+                    jobs), executions)
+
+        executions.append(
+            _execution(
+                name="Get job output for {0}".format(job_id),
+                rc=jobs[0].get("ret_code").get("code"),
+                stdout=jobs[0].get("ret_code").get("msg", ""),
+                stderr=jobs[0].get("ret_code").get("msg_txt", "")))
+    except Exception as e:
+        raise MVSExecutionException(
+            "Query for {0} job submitted under Job ID {1} failed. An exception occured: {2}".format(
+            job_name, 
+            job_id,
+            e), executions)
+
+    try:
+        # job output fails to get the ddname content in its response. Call direct into zoau to do it
+        for i in range(len(jobs[0].get("ddnames"))):
+            dd_executions, job_stdout = _get_job_dd(job_id, jobs[0]["ddnames"][i]["ddname"])
+
+            #Put the content back in the job response object
+            jobs[0]["ddnames"][i]["content"] = job_stdout
+            executions.append(dd_executions)
+
+        return jobs[0], executions
+    except Exception as e:
+        raise MVSExecutionException(
+            "Could not get all job DDs for {0}. An exception occured: {1}".format(
+            job_name, 
+            e), executions)
+
+
+def _get_job_dd(job_id, dd_name):
+    executions = []
+    command = "pjdd {0} {1}".format(job_id, dd_name)
+
+    rc, stdout, stderr = _execute_command(command)
+    executions.append(
+        _execution(
+            name="Get job dd {0} output for {1}".format(dd_name, job_id),
+            rc=rc,
+            stdout=stdout,
+            stderr=stderr))
+    if rc != 0:
+        raise MVSExecutionException(
+            "RC {0} when getting job output for {1} from {2}".format(
+                rc, dd_name, job_id), executions)
+    return executions, stdout
+
 
 def _read_data_set_content(data_set_name):
     executions = []

plugins/module_utils/_global_catalog.py

@@ -9,11 +9,10 @@
 __metaclass__ = type
 
 
-from ansible_collections.ibm.ibm_zos_core.plugins.module_utils.zos_mvs_raw import MVSCmd, MVSCmdResponse
-from ansible_collections.ibm.ibm_zos_core.plugins.module_utils.dd_statement import StdoutDefinition, DatasetDefinition, DDStatement, InputDefinition
 from ansible_collections.ibm.ibm_zos_cics.plugins.module_utils._response import MVSExecutionException, _execution
-from ansible_collections.ibm.ibm_zos_cics.plugins.module_utils._data_set_utils import MVS_CMD_RETRY_ATTEMPTS
+from ansible_collections.ibm.ibm_zos_cics.plugins.module_utils._data_set_utils import MVS_CMD_RETRY_ATTEMPTS, _submit_jcl, _get_job_output
 import tempfile
+from time import sleep
 
 def _get_value_from_line(line):  # type: (list[str]) -> str | None
     val = None
@@ -26,18 +25,6 @@ def _get_filtered_list(elements, target):  # type: (list[str],str) -> list[str]
     return list(filter(lambda x: target in x, elements))
 
 
-def _get_rmutl_dds(
-        location,
-        sdfhload,
-        cmd):  # type: (str, str, str) -> list[DDStatement]
-    return [
-        DDStatement('steplib', DatasetDefinition(sdfhload)),
-        DDStatement('dfhgcd', DatasetDefinition(location)),
-        DDStatement('sysin', InputDefinition(content=cmd)),
-        DDStatement('sysprint', StdoutDefinition()),
-    ]
-
-
 def _get_reason_code(stdout_lines_arr):  # type: (list[str]) -> str | None
     if len(stdout_lines_arr) == 0:
         return None
@@ -74,16 +61,27 @@ def _create_dfhrmutl_jcl(location, sdfhload, cmd=""):
     _validate_line_length(steplib_line, sdfhload)
     _validate_line_length(dfhgcd_line, location)
 
-    jcl = f'''
-//RMUTL1 JOB
+    jcl = ""
+    if (cmd ==""):
+        jcl = f'''
+//DFHRMUTL JOB
+//RMUTL    EXEC PGM=DFHRMUTL,REGION=1M
+{steplib_line}
+//SYSPRINT DD SYSOUT=*
+{dfhgcd_line}
+//SYSIN    DD *
+/*
+'''
+    else:
+        jcl = f'''
+//DFHRMUTL JOB
 //RMUTL    EXEC PGM=DFHRMUTL,REGION=1M
 {steplib_line}
-//SYSPRINT DD SYSOUT=A
+//SYSPRINT DD SYSOUT=*
 {dfhgcd_line}
 //SYSIN    DD *
     {cmd}
 /*
-//
 '''
 
     # Create a temporary file
@@ -127,22 +125,35 @@ def _run_dfhrmutl(
     executions = []
 
     for x in range(MVS_CMD_RETRY_ATTEMPTS):
-        dfhrmutl_response = _execute_dfhrmutl(location, sdfhload, cmd)
+        dfhrmutl_response, jcl_executions = _execute_dfhrmutl(qualified_file_path)
+        dfhrmutl_rc = dfhrmutl_response.get("ret_code").get("code")
+
+        allContent = []
+        for ddname in dfhrmutl_response.get("ddnames"):
+                allContent += ddname.get("content")
+        stdout_raw = "".join(allContent)
+
+        executions.append(jcl_executions)
         executions.append(
             _execution(
                 name="DFHRMUTL - {0} - Run {1}".format(
                     "Get current catalog" if cmd == "" else "Updating autostart override",
                     x + 1),
-                rc=dfhrmutl_response.rc,
-                stdout=dfhrmutl_response.stdout,
-                stderr=dfhrmutl_response.stderr))
+                rc=dfhrmutl_rc,
+                stdout=stdout_raw,
+                stderr=dfhrmutl_response.get("ret_code").get("msg_txt", "")))
+
+        if dfhrmutl_rc not in (0, 16):
+            raise MVSExecutionException(
+                "DFHRMUTL failed with RC {0}".format(
+                    dfhrmutl_rc), executions)
 
-        if dfhrmutl_response.rc == 0:
+        if dfhrmutl_rc == 0:
             break
-        if dfhrmutl_response.rc == 16:
+        if dfhrmutl_rc == 16:
             formatted_stdout_lines = [
                 "{0}".format(element.replace(" ", "").upper())
-                for element in dfhrmutl_response.stdout.split("\n")
+                for element in stdout_raw.split("\n")
             ]
             stdout_with_rc = list(filter(lambda x: "REASON:X" in x, formatted_stdout_lines))
 
@@ -157,23 +168,23 @@ def _run_dfhrmutl(
                     executions,
                 )
 
-        else:
-            raise MVSExecutionException(
-                "DFHRMUTL failed with RC {0}".format(
-                    dfhrmutl_response.rc), executions)
-
     if cmd != "":
         return executions
 
-    return executions, _get_catalog_records(dfhrmutl_response.stdout)
+    return executions, _get_catalog_records(stdout_raw)
+
+
+def _execute_dfhrmutl(rmutl_jcl_path, job_name="DFHRMUTL"):
+    executions = _submit_jcl(rmutl_jcl_path, job_name)
+    job_id = executions[0].get("stdout").strip()
+
+    #Give RMUTL a second to run
+    sleep(1)
 
+    job, job_executions = _get_job_output(job_id, job_name)
+    executions.append(job_executions)
 
-def _execute_dfhrmutl(location, sdfhload, cmd=""):   # type: (str, str, str) -> MVSCmdResponse
-    return MVSCmd.execute(
-        pgm="DFHRMUTL",
-        dds=_get_rmutl_dds(location=location, sdfhload=sdfhload, cmd=cmd),
-        verbose=True,
-        debug=False)
+    return job, executions
 
 
 def _get_idcams_cmd_gcd(dataset):   # type: (dict) -> dict

tests/integration/targets/cics_cmci/playbooks/cmci_bas_install.yml

@@ -496,7 +496,7 @@
           - result.cpsm_response == 'OK'
           - result.record_count == 1
           - result.records[0].group == target_scope_special_char
-          - result.records[0].changeusrid == cmci_user
+          - result.records[0].changeusrid == cmci_user | upper
 
     - name: Delete System group with special chars
       delegate_to: localhost

tests/unit/helpers/data_set_helper.py

@@ -418,6 +418,71 @@ def CSDUP_add_group_stdout(data_set_name):
         DFH5109 I END OF DFHCSDUP UTILITY JOB. HIGHEST RETURN CODE WAS: 0
     """.format(data_set_name)
 
+def get_sample_job_output(content="", rc=0, err="CC"):
+    return {
+        "class": "",
+        "content_type": "",
+        "ddnames": [
+            {
+                "byte_count": 0,
+                "content": [
+                    "{0}".format(content)
+                ],
+                "ddname": "JESMSGLG",
+                "id": "?",
+                "proctep": "",
+                "record_count": "",
+                "stepname": "JES2"
+            },
+            {
+                "byte_count": 0,
+                "content": [
+                    ""
+                ],
+                "ddname": "JESJCL",
+                "id": "?",
+                "proctep": "",
+                "record_count": "",
+                "stepname": "JES2"
+            },
+            {
+                "byte_count": 0,
+                "content": [
+                    ""
+                ],
+                "ddname": "JESYSMSG",
+                "id": "?",
+                "proctep": "",
+                "record_count": "",
+                "stepname": "JES2"
+            },
+            {
+                "byte_count": 0,
+                "content": [
+                    ""
+                ],
+                "ddname": "SYSPRINT",
+                "id": "?",
+                "proctep": "",
+                "record_count": "",
+                "stepname": "RMUTL"
+            }
+        ],
+        "duration": 0,
+        "job_id": "JOB12345",
+        "job_name": "DFHRMUTL",
+        "owner": "IBMUSER",
+        "ret_code": {
+            "code": rc,
+            "msg": "CC",
+            "msg_code": "0000",
+            "msg_txt": err,
+            "steps": []
+        },
+        "subsystem": "",
+        "system": ""
+    }
+
 
 def read_data_set_content_run_name(data_set_name):
     return "Read data set {0}".format(data_set_name)