From dcdc616eac9aa756311ce29db7b88be012b0c23b Mon Sep 17 00:00:00 2001 From: tmcguinness Date: Tue, 23 Apr 2024 15:24:01 +0000 Subject: [PATCH 1/2] Hot fix to disable STALLED as an error (only reports as status at par with RUNNING) and added message to user on Case fail when there is also not an error log present (bug had no message out in that case) --- ci/Jenkinsfile | 4 +++- ci/scripts/utils/rocotostat.py | 6 +++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile index b1334e5344..eba25ca99a 100644 --- a/ci/Jenkinsfile +++ b/ci/Jenkinsfile @@ -19,6 +19,7 @@ pipeline { stages { // This initial stage is used to get the Machine name from the GitHub labels on the PR // which is used to designate the Nodes in the Jenkins Controler by the agent label // Each Jenknis Node is connected to said machine via an JAVA agent via an ssh tunnel + // no op 2 stage('Get Machine') { agent { label 'built-in' } @@ -111,7 +112,7 @@ pipeline { try { sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_BUILD_${env.CHANGE_ID}") gist_url=sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_BUILD_${env.CHANGE_ID}", returnStdout: true).trim() - sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Build **FAILED** on **${Machine}** with error logs:\n\\`\\`\\`${error_logs_message}\n\\`\\`\\`\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})" """) + sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Build **FAILED** on **${Machine}** with error logs:\n\\`\\`\\`\n${error_logs_message}\\`\\`\\`\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})" """) } catch (Exception error_comment) { echo "Failed to comment on PR: ${error_comment.getMessage()}" } @@ -215,6 +216,7 @@ pipeline { STATUS = 'Failed' try { sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Running" --add-label "CI-${Machine}-${STATUS}" """, returnStatus: true) + sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Experiment ${Case} **FAILED** on ${Machine}\nin\\`${HOME}/RUNTESTS/${pslot}\\`" """) } catch (Exception e) { echo "Failed to update label from Running to ${STATUS}: ${e.getMessage()}" } diff --git a/ci/scripts/utils/rocotostat.py b/ci/scripts/utils/rocotostat.py index 730def1d27..5443bd6a99 100755 --- a/ci/scripts/utils/rocotostat.py +++ b/ci/scripts/utils/rocotostat.py @@ -95,7 +95,11 @@ def rocoto_statcount(): error_return = rocoto_status['UNKNOWN'] rocoto_state = 'UNKNOWN' elif rocoto_status['RUNNING'] + rocoto_status['SUBMITTING'] + rocoto_status['QUEUED'] == 0: - error_return = -3 + # + # TODO for now a STALLED state will be just a warning as it can + # produce a false negative if there is a timestamp on a file dependency. + # + #error_return = -3 rocoto_state = 'STALLED' else: rocoto_state = 'RUNNING' From 5b4155f42c633996facb6894986f18d832cca4ea Mon Sep 17 00:00:00 2001 From: tmcguinness Date: Tue, 23 Apr 2024 15:36:58 +0000 Subject: [PATCH 2/2] comment indentation fix for pynorms --- ci/scripts/utils/rocotostat.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ci/scripts/utils/rocotostat.py b/ci/scripts/utils/rocotostat.py index 5443bd6a99..9d7a7ee6b9 100755 --- a/ci/scripts/utils/rocotostat.py +++ b/ci/scripts/utils/rocotostat.py @@ -95,11 +95,11 @@ def rocoto_statcount(): error_return = rocoto_status['UNKNOWN'] rocoto_state = 'UNKNOWN' elif rocoto_status['RUNNING'] + rocoto_status['SUBMITTING'] + rocoto_status['QUEUED'] == 0: - # - # TODO for now a STALLED state will be just a warning as it can - # produce a false negative if there is a timestamp on a file dependency. - # - #error_return = -3 + # + # TODO for now a STALLED state will be just a warning as it can + # produce a false negative if there is a timestamp on a file dependency. + # + # error_return = -3 rocoto_state = 'STALLED' else: rocoto_state = 'RUNNING'