From 6f20db284513d342ec905273d1af342a6ee21d61 Mon Sep 17 00:00:00 2001 From: ddimatos Date: Thu, 23 Mar 2023 13:22:13 -0700 Subject: [PATCH 1/9] Update zos_job_submit to handle some edge cases and return the job log nearly always Signed-off-by: ddimatos --- plugins/action/zos_job_submit.py | 16 +++++ plugins/module_utils/job.py | 11 ++-- plugins/modules/zos_job_submit.py | 106 ++++++++++++++++++++---------- 3 files changed, 94 insertions(+), 39 deletions(-) diff --git a/plugins/action/zos_job_submit.py b/plugins/action/zos_job_submit.py index dcda43125..d7573137e 100644 --- a/plugins/action/zos_job_submit.py +++ b/plugins/action/zos_job_submit.py @@ -135,4 +135,20 @@ def run(self, tmp=None, task_vars=None): ) ) + def delete_dict_entries(entries, dictionary): + """ Deletes entries from a dictionary when provided key and dictionary. + + Arguments: + entries (tuple) - entries to delete from dictionary + dictionary (dic) - dictionary to remove entries + """ + for key in entries: + if key in dictionary: + del dictionary[key] + + # Currently the direction is undecided if we the action plugin should + # continue to use the community modules or move to FTP so this code + # can remain should we want to clean up unrelated response values. + # entries = ('checksum', 'dest', 'gid', 'group', 'md5sum', 'mode', 'owner', 'size', 'src', 'state', 'uid') + # delete_dict_entries(entries, result) return result diff --git a/plugins/module_utils/job.py b/plugins/module_utils/job.py index d7c156673..a466c7aca 100644 --- a/plugins/module_utils/job.py +++ b/plugins/module_utils/job.py @@ -35,10 +35,13 @@ def job_output(job_id=None, owner=None, job_name=None, dd_name=None, duration=0, """Get the output from a z/OS job based on various search criteria. Keyword Arguments: - job_id {str} -- The job ID to search for (default: {None}) - owner {str} -- The owner of the job (default: {None}) - job_name {str} -- The job name search for (default: {None}) - dd_name {str} -- The data definition to retrieve (default: {None}) + job_id (str) -- The job ID to search for (default: {None}) + owner (str) -- The owner of the job (default: {None}) + job_name (str) -- The job name search for (default: {None}) + dd_name (str) -- The data definition to retrieve (default: {None}) + duration (int) -- The time the submitted job ran for + timeout (int) - how long to wait in seconds for a job to complete + start_time (int) - time the JCL started its submission Returns: list[dict] -- The output information for a list of jobs matching specified criteria. diff --git a/plugins/modules/zos_job_submit.py b/plugins/modules/zos_job_submit.py index 6b3df1506..6d4d18a63 100644 --- a/plugins/modules/zos_job_submit.py +++ b/plugins/modules/zos_job_submit.py @@ -59,17 +59,23 @@ default: false type: bool description: + - Setting this option will yield no change, it is deprecated. There is no + no need to set I(wait); setting I(wait_times_s) is the correct way to + configure the amount of tme to wait for a job to execute. - Configuring wait used by the L(zos_job_submit,./zos_job_submit.html) module has been deprecated and will be removed in ibm.ibm_zos_core collection. - - Setting this option will yield no change, it is deprecated. - - See option ``wait_time_s``. + - See option I(wait_time_s). wait_time_s: required: false default: 10 type: int description: - - When I(wait) is true, the module will wait for the number of seconds for Job completion. - - User can set the wait time manually with this option. + - Option I(wait_time_s) is the total time that module + L(zos_job_submit,./zos_job_submit.html) will wait for a submitted job + to complete. The time begins when the module is executed on the managed + node. + - I(wait_time_s) is measured in seconds and must be a value greater than 0 + and less than 86400. max_rc: required: false type: int @@ -584,25 +590,27 @@ MAX_WAIT_TIME_S = 86400 -def submit_src_jcl(module, src, timeout=0, hfs=True, volume=None, start_time=timer()): +def submit_src_jcl(module, src, src_name=None, timeout=0, hfs=True, volume=None, start_time=timer()): """ Submit src JCL whether JCL is local (Ansible Controller), USS or in a data set. Arguments: module - module instnace to access the module api - src (str) - JCL, can be relative or absolute paths either on controller or USS - - Data set, can be PS, PDS, PDSE Member - timeout (int) - how long to wait in seconds for a job to complete - hfs (boolean) - True if JCL is a file in USS, otherwise False; Note that all - JCL local to a controller is transfered to USS thus would be - True - volume (str) - volume the data set JCL is located on that will be cataloged before - being submitted - start_time - time the JCL started its submission + src (str) - JCL, can be relative or absolute paths either on controller or USS + - Data set, can be PS, PDS, PDSE Member + src_name (str) - the src name that was provided in the module because through + the runtime src could be replace with a temporary file name + timeout (int) - how long to wait in seconds for a job to complete + hfs (boolean) - True if JCL is a file in USS, otherwise False; Note that all + JCL local to a controller is transfered to USS thus would be + True + volume (str) - volume the data set JCL is located on that will be cataloged before + being submitted + start_time - time the JCL started its submission Returns: - job_submitted_id - the JCL job ID returned from submitting a job, else if no - job submits, None will be returned - duration - how long the job ran for in this method + job_submitted_id - the JCL job ID returned from submitting a job, else if no + job submits, None will be returned + duration - how long the job ran for in this method """ kwargs = { @@ -660,16 +668,20 @@ def submit_src_jcl(module, src, timeout=0, hfs=True, volume=None, start_time=tim job_listing_rc = jobs.listing(job_submitted.id)[0].rc job_listing_status = jobs.listing(job_submitted.id)[0].status - # ZOAU throws a ZOAUException when the job sumbission fails, not when the - # JCL is non-zero, for non-zero JCL RCs that is caught in the job_output - # processing + # ZOAU throws a ZOAUException when the job sumbission fails thus there is no + # JCL RC to share with the user, if there is a RC, that will be processed + # in the job_output parser. except ZOAUException as err: result["changed"] = False result["failed"] = True result["stderr"] = str(err) - result["msg"] = ("Unable to submit job {0}, a job sumission has returned " - "a non-zero return code, please review the standard error " - "and contact a system administrator.".format(src)) + result["duration"] = duration + result["job_id"] = job_submitted.id if job_submitted else None + result["msg"] = ("Unable to submit job {0}, the job submission has failed. " + "Without the job id, the error can not be determined, " + "consider using module `zos_job_query` to poll for the " + "job by name or review the system log for purged jobs " + "resulting from an abend.".format(src_name)) module.fail_json(**result) # ZOAU throws a JobSubmitException when timeout has execeeded in that no job_id @@ -684,7 +696,29 @@ def submit_src_jcl(module, src, timeout=0, hfs=True, volume=None, start_time=tim "within the allocated time of {1} seconds. Consider using " " module zos_job_query to poll for a long running " "jobs or increasing the value for " - "'wait_times_s`.".format(src, str(timeout))) + "`wait_times_s`.".format(src_name, str(timeout))) + module.fail_json(**result) + + # Between getting a job_submitted and the jobs.listing(job_submitted.id)[0].rc + # is enough time for the system to purge an invalid job, so catch it and let + # it fall through to the catchall. + except IndexError: + job_submitted = None + + # There appears to be a small fraction of time when ZOAU has a handle on the + # job and and suddenly its purged, this check is to ensure the job is there + # long after the purge else we throw an error here if its been purged. + if job_submitted is None: + result["changed"] = False + result["failed"] = True + result["duration"] = duration + result["job_id"] = job_submitted.id if job_submitted else None + result["msg"] = ("The job {0} has been submitted and no job id was returned " + "within the allocated time of {1} seconds. Without the " + "job id, the error can not be determined, consider using " + "module `zos_job_query` to poll for the job by name or " + "review the system log for purged jobs resulting from an " + "abend.".format(src_name, str(timeout))) module.fail_json(**result) return job_submitted.id if job_submitted else None, duration @@ -786,27 +820,29 @@ def run_module(): # temporary file names for copied files when user sets location to LOCAL temp_file = parsed_args.get("temp_file") temp_file_encoded = None - if temp_file: - temp_file_encoded = NamedTemporaryFile(delete=True) # Default 'changed' is False in case the module is not able to execute result = dict(changed=False) if wait_time_s <= 0 or wait_time_s > MAX_WAIT_TIME_S: result["failed"] = True - result["msg"] = ("The value for option wait_time_s is not valid, it must " - "be greater than 0 and less than " + MAX_WAIT_TIME_S) + result["msg"] = ("The value for option `wait_time_s` is not valid, it must " + "be greater than 0 and less than {0}.".format(str(MAX_WAIT_TIME_S))) module.fail_json(**result) + if temp_file: + temp_file_encoded = NamedTemporaryFile(delete=True) + job_submitted_id = None duration = 0 start_time = timer() + if location == "DATA_SET": job_submitted_id, duration = submit_src_jcl( - module, src, wait_time_s, False, volume, start_time=start_time) + module, src, src_name=src, timeout=wait_time_s, hfs=False, volume=volume, start_time=start_time) elif location == "USS": - job_submitted_id, duration = submit_src_jcl(module, src, wait_time_s, True) + job_submitted_id, duration = submit_src_jcl(module, src, src_name=src, timeout=wait_time_s, hfs=True) else: # added -c to iconv to prevent '\r' from erroring as invalid chars to EBCDIC conv_str = "iconv -c -f {0} -t {1} {2} > {3}".format( @@ -823,7 +859,7 @@ def run_module(): if conv_rc == 0: job_submitted_id, duration = submit_src_jcl( - module, temp_file_encoded.name, wait_time_s, True) + module, temp_file_encoded.name, src_name=src, timeout=wait_time_s, hfs=True) else: result["failed"] = True result["stdout"] = stdout @@ -847,6 +883,8 @@ def run_module(): if duration >= wait_time_s: result["failed"] = True result["changed"] = False + if job_output_txt is not None: + result["jobs"] = job_output_txt result["msg"] = ( "The JCL submitted with job id {0} but appears to be a long " "running job that exceeded its maximum wait time of {1} " @@ -860,6 +898,7 @@ def run_module(): is_changed = True if job_output_txt: + result["jobs"] = job_output_txt job_ret_code = job_output_txt[0].get("ret_code") if job_ret_code: @@ -893,8 +932,6 @@ def run_module(): raise Exception("The job return code {0} was non-zero in the " "job output, this job has failed.".format(str(job_code))) - result["jobs"] = job_output_txt - if not return_output: for job in result.get("jobs", []): job["ddnames"] = [] @@ -914,7 +951,7 @@ def run_module(): result["changed"] = False result["msg"] = ("The JCL submitted with job id {0} but " "there was an error, please review " - "the error for further details: {1}.".format + "the error for further details: {1}".format (str(job_submitted_id), str(err))) module.exit_json(**result) @@ -959,7 +996,6 @@ def assert_valid_return_code(max_rc, job_rc, ret_code): return True - def main(): run_module() From 97031659362c2afd4bc8cfcf3140bf0814717c61 Mon Sep 17 00:00:00 2001 From: ddimatos Date: Thu, 23 Mar 2023 13:23:09 -0700 Subject: [PATCH 2/9] Linting correction Signed-off-by: ddimatos --- plugins/modules/zos_job_submit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/modules/zos_job_submit.py b/plugins/modules/zos_job_submit.py index 6d4d18a63..510882c57 100644 --- a/plugins/modules/zos_job_submit.py +++ b/plugins/modules/zos_job_submit.py @@ -837,7 +837,6 @@ def run_module(): duration = 0 start_time = timer() - if location == "DATA_SET": job_submitted_id, duration = submit_src_jcl( module, src, src_name=src, timeout=wait_time_s, hfs=False, volume=volume, start_time=start_time) @@ -996,6 +995,7 @@ def assert_valid_return_code(max_rc, job_rc, ret_code): return True + def main(): run_module() From 63411305100393d3a6398df37a821d7f04d0cb74 Mon Sep 17 00:00:00 2001 From: ddimatos Date: Thu, 23 Mar 2023 16:03:22 -0700 Subject: [PATCH 3/9] Add changelog fragement for pr 683 Signed-off-by: ddimatos --- .../fragments/683-zos_job_submit-bugs.yml | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 changelogs/fragments/683-zos_job_submit-bugs.yml diff --git a/changelogs/fragments/683-zos_job_submit-bugs.yml b/changelogs/fragments/683-zos_job_submit-bugs.yml new file mode 100644 index 000000000..8502b5f3e --- /dev/null +++ b/changelogs/fragments/683-zos_job_submit-bugs.yml @@ -0,0 +1,20 @@ +bugfixes: +- zos_job_submit - Fixes the issue when invalid JCL syntax is submitted that a + stack trace would result in the response, issue 623. + (https://github.com/ansible-collections/ibm_zos_core/pull/683) +- zos_job_submit - Fixes the issue when a job is purged by the system that a + stack trace would result in the response, issue 681. + (https://github.com/ansible-collections/ibm_zos_core/pull/683) +- zos_job_submit - Fixes the issue where the response did not include the + job log when a non-zero return code would occur, issue 655. + (https://github.com/ansible-collections/ibm_zos_core/pull/683) +- zos_job_submit - Fixes the issue when resources (data sets) identified in JCL + did not exist such that a stack trace would result in the response, issue 624. + (https://github.com/ansible-collections/ibm_zos_core/pull/683) +- zos_job_submit - Fixes the issue when `wait_time_s` was set to 0 that would + result in a `type` error that a stack trace would result in the response, + issue 670. (https://github.com/ansible-collections/ibm_zos_core/pull/683) +trivial: +- zos_job_submit - Update documentation to for deprecated `wait` option and + expand on the `wait_time_s` description, issue 670. + (https://github.com/ansible-collections/ibm_zos_core/pull/683) From 595ed808ee2dafc04ae0b3fc022fe585cce444df Mon Sep 17 00:00:00 2001 From: ddimatos Date: Thu, 23 Mar 2023 16:06:28 -0700 Subject: [PATCH 4/9] Clean up comments in code Signed-off-by: ddimatos --- plugins/action/zos_job_submit.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/action/zos_job_submit.py b/plugins/action/zos_job_submit.py index d7573137e..8541c74a7 100644 --- a/plugins/action/zos_job_submit.py +++ b/plugins/action/zos_job_submit.py @@ -146,8 +146,8 @@ def delete_dict_entries(entries, dictionary): if key in dictionary: del dictionary[key] - # Currently the direction is undecided if we the action plugin should - # continue to use the community modules or move to FTP so this code + # Currently the direction is undecided if we should continue to use the + # community action plugins or transition to SFTP, so this code # can remain should we want to clean up unrelated response values. # entries = ('checksum', 'dest', 'gid', 'group', 'md5sum', 'mode', 'owner', 'size', 'src', 'state', 'uid') # delete_dict_entries(entries, result) From e96b1a28e4f8e1ec2f550ae692e13fdf586e9c64 Mon Sep 17 00:00:00 2001 From: ddimatos Date: Thu, 23 Mar 2023 16:21:38 -0700 Subject: [PATCH 5/9] Update restructured text for modules and changelog fragment Signed-off-by: ddimatos --- changelogs/fragments/683-zos_job_submit-bugs.yml | 2 ++ docs/source/modules/zos_job_submit.rst | 10 +++++----- docs/source/modules/zos_operator.rst | 16 +++++++++------- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/changelogs/fragments/683-zos_job_submit-bugs.yml b/changelogs/fragments/683-zos_job_submit-bugs.yml index 8502b5f3e..679e1245c 100644 --- a/changelogs/fragments/683-zos_job_submit-bugs.yml +++ b/changelogs/fragments/683-zos_job_submit-bugs.yml @@ -18,3 +18,5 @@ trivial: - zos_job_submit - Update documentation to for deprecated `wait` option and expand on the `wait_time_s` description, issue 670. (https://github.com/ansible-collections/ibm_zos_core/pull/683) +- zos_operator - Update restructured text to include the updated examples. + (https://github.com/ansible-collections/ibm_zos_core/pull/683) \ No newline at end of file diff --git a/docs/source/modules/zos_job_submit.rst b/docs/source/modules/zos_job_submit.rst index bcf0c5383..f51a7a37e 100644 --- a/docs/source/modules/zos_job_submit.rst +++ b/docs/source/modules/zos_job_submit.rst @@ -58,20 +58,20 @@ location wait - Configuring wait used by the `zos_job_submit <./zos_job_submit.html>`_ module has been deprecated and will be removed in ibm.ibm_zos_core collection. + Setting this option will yield no change, it is deprecated. There is no no need to set *wait*; setting *wait_times_s* is the correct way to configure the amount of tme to wait for a job to execute. - Setting this option will yield no change, it is deprecated. + Configuring wait used by the `zos_job_submit <./zos_job_submit.html>`_ module has been deprecated and will be removed in ibm.ibm_zos_core collection. - See option ``wait_time_s``. + See option *wait_time_s*. | **required**: False | **type**: bool wait_time_s - When *wait* is true, the module will wait for the number of seconds for Job completion. + Option *wait_time_s* is the total time that module `zos_job_submit <./zos_job_submit.html>`_ will wait for a submitted job to complete. The time begins when the module is executed on the managed node. - User can set the wait time manually with this option. + *wait_time_s* is measured in seconds and must be a value greater than 0 and less than 86400. | **required**: False | **type**: int diff --git a/docs/source/modules/zos_operator.rst b/docs/source/modules/zos_operator.rst index 7742e60cd..868c78a10 100644 --- a/docs/source/modules/zos_operator.rst +++ b/docs/source/modules/zos_operator.rst @@ -59,10 +59,12 @@ wait_time_s wait - Configuring wait used by the `zos_operator <./zos_operator.html>`_ module has been deprecated and will be removed in ibm.ibm_zos_core collection. + Configuring wait used by the `zos_operator <./zos_operator.html>`_ module has been deprecated and will be removed in a future ibm.ibm_zos_core collection. Setting this option will yield no change, it is deprecated. + Review option *wait_time_s* to instruct operator commands to wait. + | **required**: False | **type**: bool | **default**: True @@ -76,13 +78,13 @@ Examples .. code-block:: yaml+jinja - - name: Execute an operator command to show active jobs + - name: Execute an operator command to show device status and allocation zos_operator: - cmd: 'd u,all' + cmd: 'd u' - - name: Execute an operator command to show active jobs with verbose information + - name: Execute an operator command to show device status and allocation with verbose information zos_operator: - cmd: 'd u,all' + cmd: 'd u' verbose: true - name: Execute an operator command to purge all job logs (requires escaping) @@ -91,12 +93,12 @@ Examples - name: Execute operator command to show jobs, waiting up to 5 seconds for response zos_operator: - cmd: 'd u,all' + cmd: 'd a,all' wait_time_s: 5 - name: Execute operator command to show jobs, always waiting 7 seconds for response zos_operator: - cmd: 'd u,all' + cmd: 'd a,all' wait_time_s: 7 From 453ec15f7aaf89fed08aadcaadaadde9b9ffdd52 Mon Sep 17 00:00:00 2001 From: ddimatos Date: Fri, 24 Mar 2023 10:48:06 -0700 Subject: [PATCH 6/9] Add support to check for security exception to job submit module Signed-off-by: ddimatos --- plugins/module_utils/job.py | 5 +++-- plugins/modules/zos_job_submit.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/plugins/module_utils/job.py b/plugins/module_utils/job.py index a466c7aca..478a605e5 100644 --- a/plugins/module_utils/job.py +++ b/plugins/module_utils/job.py @@ -223,7 +223,8 @@ def _get_job_status(job_id="*", owner="*", job_name="*", dd_name=None, duration= job["ret_code"] = {} job["ret_code"]["msg"] = entry.status + " " + entry.rc job["ret_code"]["msg_code"] = entry.rc - job["ret_code"]["code"] = "" + # Why was this set to an empty string? + job["ret_code"]["code"] = None if len(entry.rc) > 0: if entry.rc.isdigit(): job["ret_code"]["code"] = int(entry.rc) @@ -315,7 +316,7 @@ def _get_job_status(job_id="*", owner="*", job_name="*", dd_name=None, duration= job["ret_code"]["msg"] = tmptext.strip() job["ret_code"]["msg_code"] = None job["ret_code"]["code"] = None - if len(list_of_dds) > 1: + if len(list_of_dds) > 0: # The duration should really only be returned for job submit but the code # is used job_output as well, for now we can ignore this point unless # we want to offer a wait_time_s for job output which might be reasonable. diff --git a/plugins/modules/zos_job_submit.py b/plugins/modules/zos_job_submit.py index 510882c57..6045908c5 100644 --- a/plugins/modules/zos_job_submit.py +++ b/plugins/modules/zos_job_submit.py @@ -586,7 +586,7 @@ JOB_COMPLETION_MESSAGES = frozenset(["CC", "ABEND", "SEC ERROR", "JCL ERROR", "JCLERR"]) -JOB_ERROR_MESSAGES = frozenset(["ABEND", "SEC ERROR", "JCL ERROR", "JCLERR"]) +JOB_ERROR_MESSAGES = frozenset(["ABEND", "SEC ERROR", "SEC", "JCL ERROR", "JCLERR"]) MAX_WAIT_TIME_S = 86400 From f0b9d29e4252f3c635148290ee17351718b5e117 Mon Sep 17 00:00:00 2001 From: ddimatos Date: Fri, 24 Mar 2023 17:44:50 -0700 Subject: [PATCH 7/9] Add additonal logic for jobs that use typerun=scan Signed-off-by: ddimatos --- docs/source/modules/zos_job_submit.rst | 16 +- plugins/modules/zos_job_submit.py | 20 +- .../modules/test_zos_job_submit_func.py | 192 +++++++++++++++++- 3 files changed, 204 insertions(+), 24 deletions(-) diff --git a/docs/source/modules/zos_job_submit.rst b/docs/source/modules/zos_job_submit.rst index f51a7a37e..bb438f8a5 100644 --- a/docs/source/modules/zos_job_submit.rst +++ b/docs/source/modules/zos_job_submit.rst @@ -16,9 +16,9 @@ zos_job_submit -- Submit JCL Synopsis -------- -- Submit JCL from DATA_SET , USS, or LOCAL location. -- Submit a job and optionally monitor for its execution. -- Optionally wait a designated time until the job finishes. +- Submit JCL from a data set, USS, or from the controller. +- Submit a job and optionally monitor for completion. +- Optionally, wait a designated time until the job finishes. - For an uncataloged dataset, specify the volume serial number. @@ -32,7 +32,7 @@ Parameters src The source file or data set containing the JCL to submit. - It could be physical sequential data set or a partitioned data set qualified by a member or a path. (e.g "USER.TEST","USER.JCL(TEST)") + It could be a physical sequential data set, a partitioned data set qualified by a member or a path. (e.g "USER.TEST","USER.JCL(TEST)") Or a USS file. (e.g "/u/tester/demo/sample.jcl") @@ -100,7 +100,7 @@ volume When configured, the `zos_job_submit <./zos_job_submit.html>`_ will try to catalog the data set for the volume serial. If it is not able to, the module will fail. - Ignored for USS and LOCAL. + Ignored for *location=USS* and *location=LOCAL*. | **required**: False | **type**: str @@ -548,18 +548,18 @@ jobs } msg - Return code resulting from the job submission. + Return code resulting from the job submission. Jobs that take longer to assign a value can have a value of '?'. | **type**: str | **sample**: CC 0000 msg_code - Return code extracted from the `msg` so that it can be evaluated as a string. + Return code extracted from the `msg` so that it can be evaluated as a string. Jobs that take longer to assign a value can have a value of '?'. | **type**: str msg_txt - Returns additional information related to the job. + Returns additional information related to the job. Jobs that take longer to assign a value can have a value of '?'. | **type**: str | **sample**: The job completion code (CC) was not available in the job output, please review the job log." diff --git a/plugins/modules/zos_job_submit.py b/plugins/modules/zos_job_submit.py index 6045908c5..924a204c7 100644 --- a/plugins/modules/zos_job_submit.py +++ b/plugins/modules/zos_job_submit.py @@ -25,9 +25,9 @@ - "Demetrios Dimatos (@ddimatos)" short_description: Submit JCL description: - - Submit JCL from DATA_SET , USS, or LOCAL location. - - Submit a job and optionally monitor for its execution. - - Optionally wait a designated time until the job finishes. + - Submit JCL from a data set, USS, or from the controller. + - Submit a job and optionally monitor for completion. + - Optionally, wait a designated time until the job finishes. - For an uncataloged dataset, specify the volume serial number. version_added: "1.0.0" options: @@ -36,7 +36,7 @@ type: str description: - The source file or data set containing the JCL to submit. - - It could be physical sequential data set or a partitioned data set + - It could be a physical sequential data set, a partitioned data set qualified by a member or a path. (e.g "USER.TEST","USER.JCL(TEST)") - Or a USS file. (e.g "/u/tester/demo/sample.jcl") - Or a LOCAL file in ansible control node. @@ -97,7 +97,7 @@ - When configured, the L(zos_job_submit,./zos_job_submit.html) will try to catalog the data set for the volume serial. If it is not able to, the module will fail. - - Ignored for USS and LOCAL. + - Ignored for I(location=USS) and I(location=LOCAL). encoding: description: - Specifies which encoding the local JCL file should be converted from @@ -224,18 +224,21 @@ contains: msg: description: - Return code resulting from the job submission. + Return code resulting from the job submission. Jobs that take + longer to assign a value can have a value of '?'. type: str sample: CC 0000 msg_code: description: Return code extracted from the `msg` so that it can be evaluated - as a string. + as a string. Jobs that take longer to assign a value can have a + value of '?'. type: str sample: 0000 msg_txt: description: - Returns additional information related to the job. + Returns additional information related to the job. Jobs that take + longer to assign a value can have a value of '?'. type: str sample: The job completion code (CC) was not available in the job output, please review the job log." @@ -660,6 +663,7 @@ def submit_src_jcl(module, src, src_name=None, timeout=0, hfs=True, volume=None, # drop through and get analyzed in the main as it will scan the job ouput # Any match to JOB_ERROR_MESSAGES ends our processing and wait times while (job_listing_status not in JOB_ERROR_MESSAGES and + job_listing_status == 'AC' and ((job_listing_rc is None or len(job_listing_rc) == 0 or job_listing_rc == '?') and duration < timeout)): current_time = timer() diff --git a/tests/functional/modules/test_zos_job_submit_func.py b/tests/functional/modules/test_zos_job_submit_func.py index 3106aa292..888281712 100644 --- a/tests/functional/modules/test_zos_job_submit_func.py +++ b/tests/functional/modules/test_zos_job_submit_func.py @@ -19,8 +19,24 @@ import tempfile import pytest import re +from pprint import pprint -JCL_FILE_CONTENTS = """//HELLO JOB (T043JM,JM00,1,0,0,0),'HELLO WORLD - JRM',CLASS=R, + + +# ############################################################################## +# Configure the job card as needed, most common keyword parameters: +# CLASS: Used to achieve a balance between different types of jobs and avoid +# contention between jobs that use the same resources. +# MSGLEVEL: controls hpw the allocation messages and termination messages are +# printed in the job's output listing (SYSOUT). +# MSGCLASS: assign an output class for your output listing (SYSOUT) +# ############################################################################## + +JCL_FILE_CONTENTS = """//* +//****************************************************************************** +//* Happy path job that prints hello world, returns RC 0 as is. +//****************************************************************************** +//HELLO JOB (T043JM,JM00,1,0,0,0),'HELLO WORLD - JRM',CLASS=R, // MSGCLASS=X,MSGLEVEL=1,NOTIFY=S0JM //STEP0001 EXEC PGM=IEBGENER //SYSIN DD DUMMY @@ -31,7 +47,13 @@ //SYSUT2 DD SYSOUT=* // """ -JCL_FILE_CONTENTS_R = """//HELLO JOB (T043JM,JM00,1,0,0,0),'HELLO WORLD - JRM',CLASS=R, + +JCL_FILE_CONTENTS_BACKSLASH_R = """//* +//****************************************************************************** +//* Happy path job containing backslash r's, returns RC 0 after +//* zos_job_sbumit strips backslash r's, prints Hello world. +//****************************************************************************** +//HELLOR JOB (T043JM,JM00,1,0,0,0),'HELLO WORLD - JRM',CLASS=R, // MSGCLASS=X,MSGLEVEL=1,NOTIFY=S0JM //STEP0001 EXEC PGM=IEBGENER //SYSIN DD DUMMY @@ -42,7 +64,18 @@ //SYSUT2 DD SYSOUT=* // """ -JCL_FILE_CONTENTS_BAD = """//HELLO JOB (T043JM,JM00,1,0,0,0),'HELLO WORLD - JRM',CLASS=R, + +JCL_FILE_CONTENTS_BAD = """//* +//****************************************************************************** +//* Negative path job containing !!'s. +//* Returns: +//* ret_code->(code=null, msg=JCL ERROR , msg_text=JCLERR) +//* msg --> The JCL submitted with job id JOB00604 but there was an error, +//* please review the error for further details: The job completion +//* code (CC) was not in the job log. Please review the error +//* JCL ERROR 555 and the job log.", +//****************************************************************************** +//HELLO JOB (T043JM,JM00,1,0,0,0),'HELLO WORLD - JRM',CLASS=R, // MSGCLASS=X,MSGLEVEL=1,NOTIFY=S0JM //STEP0001 EXEC PGM=IEBGENER //SYSIN DD DUMMY @@ -54,7 +87,7 @@ // """ -JCL_FILE_CONTENTS_30_SEC = """//BPXSLEEP JOB MSGCLASS=A,MSGLEVEL=(1,1),NOTIFY=&SYSUID,REGION=0M +JCL_FILE_CONTENTS_30_SEC = """//SLEEP30 JOB MSGCLASS=A,MSGLEVEL=(1,1),NOTIFY=&SYSUID,REGION=0M //USSCMD EXEC PGM=BPXBATCH //STDERR DD SYSOUT=* //STDOUT DD SYSOUT=* @@ -65,7 +98,7 @@ // """ -JCL_FILE_CONTENTS_05_SEC = """//BPXSLEEP JOB MSGCLASS=A,MSGLEVEL=(1,1),NOTIFY=&SYSUID,REGION=0M +JCL_FILE_CONTENTS_05_SEC = """//SLEEP05 JOB MSGCLASS=A,MSGLEVEL=(1,1),NOTIFY=&SYSUID,REGION=0M //USSCMD EXEC PGM=BPXBATCH //STDERR DD SYSOUT=* //STDOUT DD SYSOUT=* @@ -75,6 +108,7 @@ /* // """ + # Should return a max RC of 8 JCL_FILE_CONTENTS_RC_8 = """//RCBADJCL JOB MSGCLASS=A,MSGLEVEL=(1,1),NOTIFY=&SYSUID,REGION=0M //S1 EXEC PGM=IDCAMS @@ -84,6 +118,105 @@ /* """ +JCL_FILE_CONTENTS_NO_DSN = """//* +//****************************************************************************** +//* Job containing a non existent DSN that will force an error. +//* Returns: +//* ret_code->(code=null, msg=JCLERR ?, msg_text=JCLERR, msg_code=?) +//* msg --> The JCL submitted with job id JOB00532 but there was an error, +//* please review the error for further details: The job completion +//* code (CC) was not in the job log. Please review the error +//* JCLERR ? and the job log.", +//****************************************************************************** +//JOBLIBPM JOB MSGCLASS=A,MSGLEVEL=(1,1),NOTIFY=&SYSUID,REGION=0M +//JOBLIB DD DSN=DATASET.NOT.EXIST,DISP=SHR +//STEP1 EXEC PGM=HELLOPGM +//SYSPRINT DD SYSOUT=* +//SYSOUT DD SYSOUT=* +// +""" + +# Do not use this test case, although its fine, the problem is it does not trigger +# the correct behavior because ZOAU has a bug such that it will return the last +# job when it can not find what we requested, so this causes the wrong job +# go be found and analyzed. See JCL_FILE_CONTENTS_JCL_ERROR_INT that does actually +# force the code properly find the correct job. +# Fix coming in zoau 1.2.3 +# JCL_FILE_CONTENTS_NO_JOB_CARD = """//STEP0001 EXEC PGM=IEBGENER +# //SYSIN DD DUMMY +# //SYSPRINT DD SYSOUT=* +# //SYSUT1 DD * +# HELLO, WORLD +# /* +# //SYSUT2 DD SYSOUT=* +# // +# """ + + +JCL_FILE_CONTENTS_JCL_ERROR_INT = """//* +//****************************************************************************** +//* Another job containing no job card resulting in a JCLERROR with an value. It +//* won't always be 952, it will increment. +//* Returns: +//* ret_code->(code=null, msg=JCL ERROR 952, msg_text=JCLERR, msg_code=null) +//* msg --> The JCL submitted with job id JOB00728 but there was an error, +//* please review the error for further details: The job completion +//* code (CC) was not in the job log. Please review the error +//* JCL ERROR 952 and the job log. +//****************************************************************************** +//CLGP JOB +//CLG EXEC IGYWCLG +//COBOL.SYSIN DD DSN=IBMUSER.ANSIBLE.COBOL(HELLO),DISP=SHR +""" + +JCL_FILE_CONTENTS_INVALID_USER = """//* +//****************************************************************************** +//* Job containing a USER=FOOBAR that will cause JES to return a SEC ERROR which +//* is a security error. +//* Returns: +//* ret_code->(code=null, msg=SEC ?, msg_text=SEC, msg_code=?) +//* msg --> The JCL submitted with job id JOB00464 but there was an error, +//* please review the error for further details: The job return code +//* was not available in the job log, please review the job log +//* and error SEC ?.", +//****************************************************************************** +//INVUSER JOB (T043JM,JM00,1,0,0,0),'HELLO WORLD - JRM',CLASS=R, +// MSGCLASS=X,MSGLEVEL=1,NOTIFY=S0JM,USER=FOOBAR +//STEP0001 EXEC PGM=IEBGENER +//SYSIN DD DUMMY +//SYSPRINT DD SYSOUT=* +//SYSUT1 DD * +HELLO, WORLD +/* +//SYSUT2 DD SYSOUT=* +// +""" + + +JCL_FILE_CONTENTS_TYPRUN_SCAN = """//* +//****************************************************************************** +//* Job containing a TYPRUN=SCAN that will cause JES to run a syntax check and +//* not actually run the JCL. +//* Returns: +//* ret_code->(code=null, msg=? ?, msg_text=?, msg_code=?) +//* msg --> The JCL submitted with job id JOB00620 but there was an error, +//* please review the error for further details: The job return code +//* was not available in the job log, please review the job log +//* and error ? ?.", +//****************************************************************************** +//TYPESCAN JOB (T043JM,JM00,1,0,0,0),'HELLO WORLD - JRM',CLASS=R, +// MSGCLASS=X,MSGLEVEL=1,NOTIFY=S0JM,TYPRUN=SCAN +//STEP0001 EXEC PGM=IEBGENER +//SYSIN DD DUMMY +//SYSPRINT DD SYSOUT=* +//SYSUT1 DD * +HELLO, WORLD +/* +//SYSUT2 DD SYSOUT=* +// +""" + + TEMP_PATH = "/tmp/jcl" DATA_SET_NAME = "imstestl.ims1.test05" DATA_SET_NAME_SPECIAL_CHARS = "imstestl.im@1.xxx05" @@ -177,7 +310,7 @@ def test_job_submit_LOCAL(ansible_zos_module): def test_job_submit_LOCAL_extraR(ansible_zos_module): tmp_file = tempfile.NamedTemporaryFile(delete=True) with open(tmp_file.name, "w") as f: - f.write(JCL_FILE_CONTENTS_R) + f.write(JCL_FILE_CONTENTS_BACKSLASH_R) hosts = ansible_zos_module results = hosts.all.zos_job_submit(src=tmp_file.name, location="LOCAL", wait=True) @@ -262,6 +395,7 @@ def test_job_submit_PDS_5_SEC_JOB_WAIT_15(ansible_zos_module): hosts.all.file(path=TEMP_PATH, state="absent") hosts.all.zos_data_set(name=DATA_SET_NAME, state="absent") + def test_job_submit_PDS_30_SEC_JOB_WAIT_60(ansible_zos_module): try: hosts = ansible_zos_module @@ -375,6 +509,48 @@ def test_job_submit_max_rc(ansible_zos_module, args): assert result.get("msg") is None assert result.get('changed') is False assert result.get("jobs")[0].get("ret_code").get("code") < 12 - finally: - hosts.all.file(path=tmp_file.name, state="absent") \ No newline at end of file + hosts.all.file(path=tmp_file.name, state="absent") + + +def test_negative_job_submit_local_jcl_no_dsn(ansible_zos_module): + tmp_file = tempfile.NamedTemporaryFile(delete=True) + with open(tmp_file.name, "w") as f: + f.write(JCL_FILE_CONTENTS_NO_DSN) + hosts = ansible_zos_module + results = hosts.all.zos_job_submit(src=tmp_file.name, location="LOCAL") + for result in results.contacted.values(): + # Expecting: The job completion code (CC) was not in the job log....." + assert result.get("changed") is False + assert re.search(r'completion code', repr(result.get("msg"))) + assert result.get("jobs")[0].get("job_id") is not None + + +# Should have a JCL ERROR +def test_negative_job_submit_local_jcl_invalid_user(ansible_zos_module): + tmp_file = tempfile.NamedTemporaryFile(delete=True) + with open(tmp_file.name, "w") as f: + f.write(JCL_FILE_CONTENTS_INVALID_USER) + hosts = ansible_zos_module + results = hosts.all.zos_job_submit(src=tmp_file.name, location="LOCAL") + for result in results.contacted.values(): + # Expecting: The job completion code (CC) was not in the job log....." + assert result.get("changed") is False + assert re.search(r'return code was not available', repr(result.get("msg"))) + assert re.search(r'error SEC', repr(result.get("msg"))) + assert result.get("jobs")[0].get("job_id") is not None + assert re.search(r'SEC', repr(result.get("jobs")[0].get("ret_code").get("msg_text"))) + +def test_negative_job_submit_local_jcl_typrun_scan(ansible_zos_module): + tmp_file = tempfile.NamedTemporaryFile(delete=True) + with open(tmp_file.name, "w") as f: + f.write(JCL_FILE_CONTENTS_TYPRUN_SCAN) + hosts = ansible_zos_module + results = hosts.all.zos_job_submit(src=tmp_file.name, location="LOCAL") + for result in results.contacted.values(): + # Expecting: The job completion code (CC) was not in the job log....." + assert result.get("changed") is False + assert re.search(r'return code was not available', repr(result.get("msg"))) + assert re.search(r'error ? ?', repr(result.get("msg"))) + assert result.get("jobs")[0].get("job_id") is not None + assert result.get("jobs")[0].get("ret_code").get("msg_text") == "?" From b3c57b0012a96ebd75320fe934fda428034c09c4 Mon Sep 17 00:00:00 2001 From: ddimatos Date: Fri, 24 Mar 2023 19:54:47 -0700 Subject: [PATCH 8/9] Update chnagelog fragment Signed-off-by: ddimatos --- changelogs/fragments/683-zos_job_submit-bugs.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/changelogs/fragments/683-zos_job_submit-bugs.yml b/changelogs/fragments/683-zos_job_submit-bugs.yml index 679e1245c..b77fbdbc9 100644 --- a/changelogs/fragments/683-zos_job_submit-bugs.yml +++ b/changelogs/fragments/683-zos_job_submit-bugs.yml @@ -14,9 +14,22 @@ bugfixes: - zos_job_submit - Fixes the issue when `wait_time_s` was set to 0 that would result in a `type` error that a stack trace would result in the response, issue 670. (https://github.com/ansible-collections/ibm_zos_core/pull/683) +- zos_job_submit - Fixes the issue when a job encounters a security exception no + job log would would result in the response, issue 684. + (https://github.com/ansible-collections/ibm_zos_core/pull/683) +- zos_job_submit - Fixes the issue when a job is configured for a syntax check + using TYPRUN=SCAN that it would wait the full duration set by `wait_time_s` + to return a response, issue 685. + (https://github.com/ansible-collections/ibm_zos_core/pull/683) +- zos_job_submit - Fixes the issue when a job is configured for a syntax check + using TYPRUN=SCAN that no job log would result in the response, issue 685. + (https://github.com/ansible-collections/ibm_zos_core/pull/683) trivial: - zos_job_submit - Update documentation to for deprecated `wait` option and expand on the `wait_time_s` description, issue 670. (https://github.com/ansible-collections/ibm_zos_core/pull/683) +- zos_job_submit - Update documentation to describing the significance of '?' + for the 'ret_code' properties 'msg_text', 'msg_code' and 'msg', issue 685. + (https://github.com/ansible-collections/ibm_zos_core/pull/683) - zos_operator - Update restructured text to include the updated examples. (https://github.com/ansible-collections/ibm_zos_core/pull/683) \ No newline at end of file From 7078d67b8c873e085cf3d74842de3905d6d2301a Mon Sep 17 00:00:00 2001 From: ddimatos Date: Mon, 27 Mar 2023 15:02:35 -0700 Subject: [PATCH 9/9] Correct run on sentence Signed-off-by: ddimatos --- plugins/modules/zos_job_submit.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/modules/zos_job_submit.py b/plugins/modules/zos_job_submit.py index 924a204c7..a58e138a1 100644 --- a/plugins/modules/zos_job_submit.py +++ b/plugins/modules/zos_job_submit.py @@ -682,8 +682,8 @@ def submit_src_jcl(module, src, src_name=None, timeout=0, hfs=True, volume=None, result["duration"] = duration result["job_id"] = job_submitted.id if job_submitted else None result["msg"] = ("Unable to submit job {0}, the job submission has failed. " - "Without the job id, the error can not be determined, " - "consider using module `zos_job_query` to poll for the " + "Without the job id, the error can not be determined. " + "Consider using module `zos_job_query` to poll for the " "job by name or review the system log for purged jobs " "resulting from an abend.".format(src_name)) module.fail_json(**result)