forked from NOAA-EMC/global-workflow
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge remote-tracking branch 'upstream/develop' into feature/use_jcb_atm
* upstream/develop: Add CCPP suite and FASTER option to UFS build (NOAA-EMC#2521) New "atmanlfv3inc" Rocoto job (NOAA-EMC#2420) Hotfix to disable STALLED in CI as an error (NOAA-EMC#2523) Add restart on failure capability for the forecast executable (NOAA-EMC#2510) Update parm/transfer list files to match vetted GFSv16 set (NOAA-EMC#2517) Update gdas_gsibec_ver to 20240416 (NOAA-EMC#2497) Adding more cycles to gempak script gfs_meta_sa2.sh (NOAA-EMC#2518) Update gsi_enkf.sh hash to 457510c (NOAA-EMC#2514) Enable using the FV3_global_nest_v1 CCPP suite (NOAA-EMC#2512) CI Refactoring and STALLED case detection (NOAA-EMC#2488) Add C768 and C1152 S2SW test cases (NOAA-EMC#2509) Fix paths for refactored prepocnobs task (NOAA-EMC#2504)
- Loading branch information
Showing
121 changed files
with
2,033 additions
and
1,321 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -4,18 +4,22 @@ def HOME = 'none' | |
| def caseList = '' | ||
| // Location of the custom workspaces for each machine in the CI system. They are persitent for each iteration of the PR. | ||
| def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/stmp/CI/HERCULES'] | ||
| def repo_url = '[email protected]:NOAA-EMC/global-workflow.git' | ||
| def STATUS = 'Passed' | ||
|
|
||
| pipeline { | ||
|
|
||
| agent { label 'built-in' } | ||
|
|
||
| options { | ||
| skipDefaultCheckout() | ||
| parallelsAlwaysFailFast() | ||
| //parallelsAlwaysFailFast() | ||
| } | ||
|
|
||
| stages { // This initial stage is used to get the Machine name from the GitHub labels on the PR | ||
| // which is used to designate the Nodes in the Jenkins Controler by the agent label | ||
| // Each Jenknis Node is connected to said machine via an JAVA agent via an ssh tunnel | ||
| // no op 2 | ||
|
|
||
| stage('Get Machine') { | ||
| agent { label 'built-in' } | ||
|
|
@@ -46,10 +50,7 @@ pipeline { | |
| properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) | ||
| HOME = "${WORKSPACE}" | ||
| sh(script: "mkdir -p ${HOME}/RUNTESTS;rm -Rf ${HOME}/RUNTESTS/error.logs") | ||
| pullRequest.addLabel("CI-${Machine}-Building") | ||
| if (pullRequest.labels.any { value -> value.matches("CI-${Machine}-Ready") }) { | ||
| pullRequest.removeLabel("CI-${Machine}-Ready") | ||
| } | ||
| sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Building" --remove-label "CI-${Machine}-Ready" """) | ||
| } | ||
| echo "Building and running on ${Machine} in directory ${HOME}" | ||
| } | ||
|
|
@@ -108,26 +109,23 @@ pipeline { | |
| catch (Exception error_arch) { echo "Failed to archive error log ${line}: ${error_arch.getMessage()}" } | ||
| } | ||
| } | ||
| repo_url=sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_BUILD_${env.CHANGE_ID}", returnStdout: true).trim() | ||
| gist_url=sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_BUILD_${env.CHANGE_ID}", returnStdout: true).trim() | ||
| try { | ||
| pullRequest.comment("Build failed on **${Machine}** with error logs:${error_logs_message}\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})") | ||
| sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_BUILD_${env.CHANGE_ID}") | ||
| gist_url=sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_BUILD_${env.CHANGE_ID}", returnStdout: true).trim() | ||
| sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Build **FAILED** on **${Machine}** with error logs:\n\\`\\`\\`\n${error_logs_message}\\`\\`\\`\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})" """) | ||
| } catch (Exception error_comment) { | ||
| echo "Failed to comment on PR: ${error_comment.getMessage()}" | ||
| } | ||
| error("Failed to build system on ${Machine}") | ||
| } | ||
| } | ||
| sh(script: './link_workflow.sh') | ||
| // sh(script: "echo ${HOMEgfs} > BUILT_semaphor") | ||
| sh(script: "echo ${HOMEgfs} > BUILT_semaphor") | ||
| } | ||
| } | ||
| if (env.CHANGE_ID && system == 'gfs') { | ||
| try { | ||
| if (pullRequest.labels.any { value -> value.matches("CI-${Machine}-Building") }) { | ||
| pullRequest.removeLabel("CI-${Machine}-Building") | ||
| } | ||
| pullRequest.addLabel("CI-${Machine}-Running") | ||
| sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Running" --remove-label "CI-${Machine}-Building" """) | ||
| } catch (Exception e) { | ||
| echo "Failed to update label from Buildng to Running: ${e.getMessage()}" | ||
| } | ||
|
|
@@ -144,6 +142,7 @@ pipeline { | |
| } | ||
|
|
||
| stage('Run Tests') { | ||
| failFast false | ||
| matrix { | ||
| agent { label "${machine}-emc" } | ||
| axes { | ||
|
|
@@ -175,19 +174,22 @@ pipeline { | |
| when { | ||
| expression { return caseList.contains(Case) } | ||
| } | ||
| failFast false | ||
| steps { | ||
| script { | ||
| HOMEgfs = "${HOME}/gfs" // common HOMEgfs is used to launch the scripts that run the experiments | ||
| pslot = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true).trim() | ||
| def pslot = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true).trim() | ||
| def error_file = "${HOME}/RUNTESTS/${pslot}_error.logs" | ||
| sh(script: " rm -f ${error_file}") | ||
| try { | ||
| sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}") | ||
| sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot} ${system}") | ||
| } catch (Exception error_experment) { | ||
| sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cancel_all_batch_jobs ${HOME}/RUNTESTS") | ||
| sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cancel_batch_jobs ${pslot}") | ||
| ws(HOME) { | ||
| def error_logs = "" | ||
| def error_logs_message = "" | ||
| if (fileExists("RUNTESTS/error.logs")) { | ||
| def fileContent = readFile 'RUNTESTS/error.logs' | ||
| if (fileExists(error_file)) { | ||
| def fileContent = readFile error_file | ||
| def lines = fileContent.readLines() | ||
| for (line in lines) { | ||
| echo "archiving: ${line}" | ||
|
|
@@ -201,15 +203,22 @@ pipeline { | |
| } | ||
| } | ||
| } | ||
| repo_url = sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_${env.CHANGE_ID}", returnStdout: true).trim() | ||
| gist_url = sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_${env.CHANGE_ID}", returnStdout: true).trim() | ||
| try { | ||
| pullRequest.comment("Experiment ${Case} failed on ${Machine} with error logs: ${error_logs_message}\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})") | ||
| gist_url = sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_${env.CHANGE_ID}", returnStdout: true).trim() | ||
| sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Experiment ${Case} **FAILED** on ${Machine} with error logs:\n\\`\\`\\`\n${error_logs_message}\\`\\`\\`\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})" """) | ||
| sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_${env.CHANGE_ID}") | ||
| } catch (Exception error_comment) { | ||
| echo "Failed to comment on PR: ${error_comment.getMessage()}" | ||
| } | ||
| } else { | ||
| echo "No error logs found for failed cases in $HOME/RUNTESTS/error.logs" | ||
| echo "No error logs found for failed cases in $HOME/RUNTESTS/${pslot}_error.logs" | ||
| } | ||
| STATUS = 'Failed' | ||
| try { | ||
| sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Running" --add-label "CI-${Machine}-${STATUS}" """, returnStatus: true) | ||
| sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Experiment ${Case} **FAILED** on ${Machine}\nin\\`${HOME}/RUNTESTS/${pslot}\\`" """) | ||
| } catch (Exception e) { | ||
| echo "Failed to update label from Running to ${STATUS}: ${e.getMessage()}" | ||
| } | ||
| error("Failed to run experiments ${Case} on ${Machine}") | ||
| } | ||
|
|
@@ -220,49 +229,23 @@ pipeline { | |
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| post { | ||
| always { | ||
| script { | ||
| if(env.CHANGE_ID) { | ||
| try { | ||
| for (label in pullRequest.labels) { | ||
| if (label.contains("${Machine}")) { | ||
| pullRequest.removeLabel(label) | ||
| } | ||
| } | ||
| } catch (Exception e) { | ||
| echo "Failed to remove labels: ${e.getMessage()}" | ||
| } | ||
| } | ||
| } | ||
| } | ||
| success { | ||
| script { | ||
| if(env.CHANGE_ID) { | ||
| try { | ||
| pullRequest.addLabel("CI-${Machine}-Passed") | ||
| def timestamp = new Date().format('MM dd HH:mm:ss', TimeZone.getTimeZone('America/New_York')) | ||
| pullRequest.comment("**CI SUCCESS** ${Machine} at ${timestamp}\n\nBuilt and ran in directory `${HOME}`") | ||
| } catch (Exception e) { | ||
| echo "Failed to add success label or comment: ${e.getMessage()}" | ||
| } | ||
| stage( 'FINALIZE' ) { | ||
| when { | ||
| expression { | ||
| STATUS == 'Passed' | ||
| } | ||
| } | ||
| } | ||
| failure { | ||
| script { | ||
| if(env.CHANGE_ID) { | ||
| agent { label "${machine}-emc" } | ||
| steps { | ||
| script { | ||
| try { | ||
| pullRequest.addLabel("CI-${Machine}-Failed") | ||
| def timestamp = new Date().format('MM dd HH:mm:ss', TimeZone.getTimeZone('America/New_York')) | ||
| pullRequest.comment("**CI FAILED** ${Machine} at ${timestamp}<br>Built and ran in directory `${HOME}`") | ||
| sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Running" --remove-label "CI-${Machine}-Building" --add-label "CI-${Machine}-${STATUS}" """, returnStatus: true) | ||
| sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "**CI ${STATUS}** ${Machine} at <br>Built and ran in directory \\`${HOME}\\`" """, returnStatus: true) | ||
| } catch (Exception e) { | ||
| echo "Failed to add failure label or comment: ${e.getMessage()}" | ||
| echo "Failed to update label from Running to ${STATUS}: ${e.getMessage()}" | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,14 @@ | ||
| experiment: | ||
| system: gfs | ||
| mode: forecast-only | ||
|
|
||
| arguments: | ||
| pslot: {{ 'pslot' | getenv }} | ||
| app: S2SW | ||
| resdetatmos: 1152 | ||
| resdetocean: 0.25 | ||
| comroot: {{ 'RUNTESTS' | getenv }}/COMROOT | ||
| expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR | ||
| idate: 2019120300 | ||
| edate: 2019120300 | ||
| yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_defaults_ci.yaml |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,14 @@ | ||
| experiment: | ||
| system: gfs | ||
| mode: forecast-only | ||
|
|
||
| arguments: | ||
| pslot: {{ 'pslot' | getenv }} | ||
| app: S2SW | ||
| resdetatmos: 768 | ||
| resdetocean: 0.25 | ||
| comroot: {{ 'RUNTESTS' | getenv }}/COMROOT | ||
| expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR | ||
| idate: 2019120300 | ||
| edate: 2019120300 | ||
| yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_defaults_ci.yaml |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.