From e5d857b060361cea5e5692ae6bb33e4299f0a08e Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Fri, 3 Jan 2025 18:44:43 -0500 Subject: [PATCH] Set runtime shell requirements within Jenkins Pipeline (#3171) This PR updates the Jenkins Pipeline script to source `gw_setup.sh` from within the PR itself for when ever a shell command is issued that needs it. Currently the runtime bash shell environment requirements for running the **global-workflow** (set by sourcing `$HOMEgfs/workflow/gw_setup.sh`) is established by the user's shell that makes the connection from the Java remote agent to the Jenkins Controller. This update simply sources `gw_setup.sh` directly during run time when ever needed instead. **NOTE:** This does not apply for **GitHub CLI** and **pyGitHub** since they are not currently supported by **Spack Stack** and therefore not incorporated in to `gw_setup.sh`. These remaining dependances are the responsibility of the _role account_ holders that manage the Jenkins Controller to Node Agent connections. --------- Co-authored-by: Terry McGuinness Co-authored-by: Rahul Mahajan --- ci/Jenkinsfile | 45 +++++++++++++++++++++------ ci/scripts/utils/launch_java_agent.sh | 15 ++++----- 2 files changed, 44 insertions(+), 16 deletions(-) diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile index b7a29e15b0..67ac0fdb17 100644 --- a/ci/Jenkinsfile +++ b/ci/Jenkinsfile @@ -138,8 +138,14 @@ pipeline { } } try { - sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_BUILD_${env.CHANGE_ID}") - gist_url=sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_BUILD_${env.CHANGE_ID}", returnStdout: true).trim() + sh(script: """ + source ${HOMEgfs}/workflow/gw_setup.sh + ${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_BUILD_${env.CHANGE_ID} + """) + gist_url=sh(script: """ + source ${HOMEgfs}/workflow/gw_setup.sh + ${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_BUILD_${env.CHANGE_ID} + """, returnStdout: true).trim() sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Build **FAILED** on **${Machine}** in Build# ${env.BUILD_NUMBER} with error logs:\n\\`\\`\\`\n${error_logs_message}\\`\\`\\`\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})" """) } catch (Exception error_comment) { echo "Failed to comment on PR: ${error_comment.getMessage()}" @@ -158,7 +164,10 @@ pipeline { } } // Get a list of CI cases to run - CI_CASES = sh(script: "${HOMEgfs}/ci/scripts/utils/get_host_case_list.py ${machine}", returnStdout: true).trim().split() + CI_CASES = sh(script: """ + source ${HOMEgfs}/workflow/gw_setup.sh + ${HOMEgfs}/ci/scripts/utils/get_host_case_list.py ${machine} + """, returnStdout: true).trim().split() echo "Cases to run: ${CI_CASES}" } } @@ -179,7 +188,10 @@ pipeline { script { env.RUNTESTS = "${CUSTOM_WORKSPACE}/RUNTESTS" try { - error_output = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${caseName}.yaml", returnStdout: true).trim() + error_output = sh(script: """ + source ${HOMEgfs}/workflow/gw_setup.sh + ${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${caseName}.yaml + """, returnStdout: true).trim() } catch (Exception error_create) { sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "${Case} **FAILED** to create experiment on ${Machine} in BUILD# ${env.BUILD_NUMBER}\n with the error:\n\\`\\`\\`\n${error_output}\\`\\`\\`" """) error("Case ${caseName} failed to create experiment directory") @@ -194,10 +206,19 @@ pipeline { def error_file = "${CUSTOM_WORKSPACE}/RUNTESTS/${pslot}_error.logs" sh(script: " rm -f ${error_file}") try { - sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} 'global-workflow'") - sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cleanup_experiment ${CUSTOM_WORKSPACE}/RUNTESTS/EXPDIR/${pslot}") + sh(script: """ + source ${HOMEgfs}/workflow/gw_setup.sh + ${HOMEgfs}/ci/scripts/run-check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} 'global-workflow' + """) + sh(script: """ + source ${HOMEgfs}/workflow/gw_setup.sh + ${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cleanup_experiment ${CUSTOM_WORKSPACE}/RUNTESTS/EXPDIR/${pslot} + """) } catch (Exception error_experment) { - sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cancel_batch_jobs ${pslot}") + sh(script: """ + source ${HOMEgfs}/workflow/gw_setup.sh + ${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cancel_batch_jobs ${pslot} + """) ws(CUSTOM_WORKSPACE) { def error_logs = "" def error_logs_message = "" @@ -217,9 +238,15 @@ pipeline { } } try { - gist_url = sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_${env.CHANGE_ID}", returnStdout: true).trim() + gist_url = sh(script: """ + source ${HOMEgfs}/workflow/gw_setup.sh + ${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_${env.CHANGE_ID} + """, returnStdout: true).trim() sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Experiment ${caseName} **FAILED** on ${Machine} in Build# ${env.BUILD_NUMBER} with error logs:\n\\`\\`\\`\n${error_logs_message}\\`\\`\\`\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})" """) - sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_${env.CHANGE_ID}") + sh(script: """ + source ${HOMEgfs}/workflow/gw_setup.sh + ${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_${env.CHANGE_ID} + """) } catch (Exception error_comment) { echo "Failed to comment on PR: ${error_comment.getMessage()}" } diff --git a/ci/scripts/utils/launch_java_agent.sh b/ci/scripts/utils/launch_java_agent.sh index eb78d3b1ef..ad79a75cbd 100755 --- a/ci/scripts/utils/launch_java_agent.sh +++ b/ci/scripts/utils/launch_java_agent.sh @@ -65,14 +65,14 @@ controller_url="https://jenkins.epic.oarcloud.noaa.gov" controller_user=${controller_user:-"terry.mcguinness"} controller_user_auth_token="jenkins_token" -HOMEgfs="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." >/dev/null 2>&1 && pwd )" +HOMEGFS_="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." >/dev/null 2>&1 && pwd )" host=$(hostname) ######################################################################### # Set up runtime environment varibles for accounts on supproted machines ######################################################################### -source "${HOMEgfs}/ush/detect_machine.sh" +source "${HOMEGFS_}/ush/detect_machine.sh" case ${MACHINE_ID} in hera | orion | hercules | wcoss2 | gaea) echo "Launch Jenkins Java Controler on ${MACHINE_ID}";; @@ -84,10 +84,10 @@ esac LOG=lanuched_agent-$(date +%Y%m%d%M).log rm -f "${LOG}" -source "${HOMEgfs}/ush/module-setup.sh" -module use "${HOMEgfs}/modulefiles" +source "${HOMEGFS_}/ush/module-setup.sh" +module use "${HOMEGFS_}/modulefiles" module load "module_gwsetup.${MACHINE_ID}" -source "${HOMEgfs}/ci/platforms/config.${MACHINE_ID}" +source "${HOMEGFS_}/ci/platforms/config.${MACHINE_ID}" JAVA_HOME="${JENKINS_AGENT_LANUCH_DIR}/JAVA/jdk-17.0.10" if [[ ! -d "${JAVA_HOME}" ]]; then @@ -102,9 +102,10 @@ JAVA="${JAVA_HOME}/bin/java" echo "JAVA VERSION: " ${JAVA} -version -export GH="${HOME}/bin/gh" -[[ -f "${GH}" ]] || echo "gh is not installed in ${HOME}/bin" +GH=$(command -v gh || echo "${HOME}/bin/gh") +[[ -f "${GH}" ]] || ( echo "ERROR: GitHub CLI (gh) not found. (exiting with error)"; exit 1 ) ${GH} --version +export GH check_mark=$("${GH}" auth status -t 2>&1 | grep "Token:" | awk '{print $1}') || true if [[ "${check_mark}" != "✓" ]]; then