From 9f725c1107f96aca6ee02021aa07090f07f1f65f Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 21 May 2020 17:02:37 +0100 Subject: [PATCH 01/25] ci: fix warnings with wildcards --- .ci/scripts/pre-archive-test.sh | 15 +++++++++++++++ Jenkinsfile | 7 +++++-- 2 files changed, 20 insertions(+), 2 deletions(-) create mode 100755 .ci/scripts/pre-archive-test.sh diff --git a/.ci/scripts/pre-archive-test.sh b/.ci/scripts/pre-archive-test.sh new file mode 100755 index 000000000000..a45031390b0d --- /dev/null +++ b/.ci/scripts/pre-archive-test.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -exuo pipefail + +FOLDER=${1:-.build} + +if [ -d "${FOLDER}" ] ; then + rm -rf "${FOLDER}" +fi +mkdir -p "${FOLDER}" +find . -name "build" -type d -print0 | while read -r -d $'\0' build +do + base=$(basename "$(dirname "${build}")") + mkdir -p "${FOLDER}/${base}" + cp -rf "${build}" "${FOLDER}/${base}" +done \ No newline at end of file diff --git a/Jenkinsfile b/Jenkinsfile index 068b1ba4cc96..996ca60dd2e6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -848,8 +848,11 @@ def withBeatsEnv(boolean archive, Closure body) { } finally { if (archive) { catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') { - junitAndStore(allowEmptyResults: true, keepLongStdio: true, testResults: "**/build/TEST*.xml") - archiveArtifacts(allowEmptyArchive: true, artifacts: '**/build/TEST*.out') + sh(label: 'Prepare test output', script: '.ci/scripts/pre-archive-test.sh .build') + dir('.build') { + junitAndStore(allowEmptyResults: true, keepLongStdio: true, testResults: '**/build/TEST*.xml') + archiveArtifacts(allowEmptyArchive: true, artifacts: '**/build/TEST*.out') + } } } reportCoverage() From ecceee2988aa5d12f8374fbbc7554f314ff03daf Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 27 May 2020 14:54:27 +0100 Subject: [PATCH 02/25] ci: use make as a wrapper and python as the implementation --- .ci/scripts/pre-archive-test.sh | 15 --------------- .ci/scripts/pre_archive_test.py | 18 ++++++++++++++++++ Jenkinsfile | 11 +++++++---- Makefile | 8 ++++++++ 4 files changed, 33 insertions(+), 19 deletions(-) delete mode 100755 .ci/scripts/pre-archive-test.sh create mode 100755 .ci/scripts/pre_archive_test.py diff --git a/.ci/scripts/pre-archive-test.sh b/.ci/scripts/pre-archive-test.sh deleted file mode 100755 index a45031390b0d..000000000000 --- a/.ci/scripts/pre-archive-test.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash -set -exuo pipefail - -FOLDER=${1:-.build} - -if [ -d "${FOLDER}" ] ; then - rm -rf "${FOLDER}" -fi -mkdir -p "${FOLDER}" -find . -name "build" -type d -print0 | while read -r -d $'\0' build -do - base=$(basename "$(dirname "${build}")") - mkdir -p "${FOLDER}/${base}" - cp -rf "${build}" "${FOLDER}/${base}" -done \ No newline at end of file diff --git a/.ci/scripts/pre_archive_test.py b/.ci/scripts/pre_archive_test.py new file mode 100755 index 000000000000..e37042feda2c --- /dev/null +++ b/.ci/scripts/pre_archive_test.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 + +import os +import distutils +from distutils import dir_util + +if __name__ == "__main__": + + if not os.path.exists('build'): + os.makedirs('build') + + ## Top level folders to be excluded + EXCLUDE = set(['.ci', '.git', '.github', 'vendor', 'dev-tools']) + for root, dirs, files in os.walk('.'): + dirs[:] = [d for d in dirs if d not in EXCLUDE] + if root.endswith(('build')) and not root.startswith(('./build')): + dest = './build/' + root.replace('./', '') + distutils.dir_util.copy_tree(root, dest) diff --git a/Jenkinsfile b/Jenkinsfile index 996ca60dd2e6..062e82a1dc07 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -848,8 +848,8 @@ def withBeatsEnv(boolean archive, Closure body) { } finally { if (archive) { catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') { - sh(label: 'Prepare test output', script: '.ci/scripts/pre-archive-test.sh .build') - dir('.build') { + sh(label: 'Prepare test output', script: 'make prepare-archive-test') + dir('build') { junitAndStore(allowEmptyResults: true, keepLongStdio: true, testResults: '**/build/TEST*.xml') archiveArtifacts(allowEmptyArchive: true, artifacts: '**/build/TEST*.out') } @@ -885,8 +885,11 @@ def withBeatsEnvWin(Closure body) { } } finally { catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') { - junitAndStore(allowEmptyResults: true, keepLongStdio: true, testResults: "**\\build\\TEST*.xml") - archiveArtifacts(allowEmptyArchive: true, artifacts: '**\\build\\TEST*.out') + bat(label: 'Prepare test output', script: 'make prepare-archive-test') + dir('build') { + junitAndStore(allowEmptyResults: true, keepLongStdio: true, testResults: "**\\build\\TEST*.xml") + archiveArtifacts(allowEmptyArchive: true, artifacts: '**\\build\\TEST*.out') + } } } } diff --git a/Makefile b/Makefile index b03fd92ba726..def1a2f38bc0 100644 --- a/Makefile +++ b/Makefile @@ -165,6 +165,14 @@ python-env: test-apm: sh ./script/test_apm.sh +### CI specific #### + +## prepare-archive-test : Move the test output to the top level .build folder +.PHONY: prepare-archive-test +prepare-archive-test: python-env + @${PYTHON_ENV_EXE} .ci/scripts/pre_archive_test.py . + + ### Packaging targets #### ## snapshot : Builds a snapshot release. From a9c95d936ca9d7baff2e0edb1a67bc5f9f9ae3cf Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 27 May 2020 15:05:10 +0100 Subject: [PATCH 03/25] ci: support windows --- .ci/scripts/pre_archive_test.py | 2 +- Jenkinsfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/scripts/pre_archive_test.py b/.ci/scripts/pre_archive_test.py index e37042feda2c..33fbc8406896 100755 --- a/.ci/scripts/pre_archive_test.py +++ b/.ci/scripts/pre_archive_test.py @@ -14,5 +14,5 @@ for root, dirs, files in os.walk('.'): dirs[:] = [d for d in dirs if d not in EXCLUDE] if root.endswith(('build')) and not root.startswith(('./build')): - dest = './build/' + root.replace('./', '') + dest = os.path.join('build', root.replace('./', '')) distutils.dir_util.copy_tree(root, dest) diff --git a/Jenkinsfile b/Jenkinsfile index 062e82a1dc07..1a38b5b98e0a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -885,7 +885,7 @@ def withBeatsEnvWin(Closure body) { } } finally { catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') { - bat(label: 'Prepare test output', script: 'make prepare-archive-test') + bat(label: 'Prepare test output', script: 'python .ci/scripts/pre_archive_test.py') dir('build') { junitAndStore(allowEmptyResults: true, keepLongStdio: true, testResults: "**\\build\\TEST*.xml") archiveArtifacts(allowEmptyArchive: true, artifacts: '**\\build\\TEST*.out') From 3f2c1bc2a33900e881b29eda860d804fb799502c Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 27 May 2020 15:30:34 +0100 Subject: [PATCH 04/25] fix: autopep8 lint --- .ci/scripts/pre_archive_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/scripts/pre_archive_test.py b/.ci/scripts/pre_archive_test.py index 33fbc8406896..33707f732178 100755 --- a/.ci/scripts/pre_archive_test.py +++ b/.ci/scripts/pre_archive_test.py @@ -9,7 +9,7 @@ if not os.path.exists('build'): os.makedirs('build') - ## Top level folders to be excluded + # Top level folders to be excluded EXCLUDE = set(['.ci', '.git', '.github', 'vendor', 'dev-tools']) for root, dirs, files in os.walk('.'): dirs[:] = [d for d in dirs if d not in EXCLUDE] From 809b9069f28b04ea21cad3571ae05f86f4aa8d81 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 28 May 2020 16:04:03 +0100 Subject: [PATCH 05/25] fix symlinks, permissions and use python instead make --- .ci/scripts/pre_archive_test.py | 2 +- Jenkinsfile | 3 ++- Makefile | 8 -------- 3 files changed, 3 insertions(+), 10 deletions(-) diff --git a/.ci/scripts/pre_archive_test.py b/.ci/scripts/pre_archive_test.py index 33707f732178..a6a5ae68a429 100755 --- a/.ci/scripts/pre_archive_test.py +++ b/.ci/scripts/pre_archive_test.py @@ -15,4 +15,4 @@ dirs[:] = [d for d in dirs if d not in EXCLUDE] if root.endswith(('build')) and not root.startswith(('./build')): dest = os.path.join('build', root.replace('./', '')) - distutils.dir_util.copy_tree(root, dest) + distutils.dir_util.copy_tree(root, dest, preserve_symlinks=1) diff --git a/Jenkinsfile b/Jenkinsfile index 1a38b5b98e0a..9b755ead18dc 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -848,7 +848,8 @@ def withBeatsEnv(boolean archive, Closure body) { } finally { if (archive) { catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') { - sh(label: 'Prepare test output', script: 'make prepare-archive-test') + fixPermissions("${WORKSPACE}") + sh(label: 'Prepare test output', script: 'python .ci/scripts/pre_archive_test.py') dir('build') { junitAndStore(allowEmptyResults: true, keepLongStdio: true, testResults: '**/build/TEST*.xml') archiveArtifacts(allowEmptyArchive: true, artifacts: '**/build/TEST*.out') diff --git a/Makefile b/Makefile index def1a2f38bc0..b03fd92ba726 100644 --- a/Makefile +++ b/Makefile @@ -165,14 +165,6 @@ python-env: test-apm: sh ./script/test_apm.sh -### CI specific #### - -## prepare-archive-test : Move the test output to the top level .build folder -.PHONY: prepare-archive-test -prepare-archive-test: python-env - @${PYTHON_ENV_EXE} .ci/scripts/pre_archive_test.py . - - ### Packaging targets #### ## snapshot : Builds a snapshot release. From b66aa230a5a91b49266f26038fbc1f207ee01310 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 28 May 2020 17:31:58 +0100 Subject: [PATCH 06/25] more verbose, skip some folders that causes issues in windows --- .ci/scripts/pre_archive_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.ci/scripts/pre_archive_test.py b/.ci/scripts/pre_archive_test.py index a6a5ae68a429..e5273b410c44 100755 --- a/.ci/scripts/pre_archive_test.py +++ b/.ci/scripts/pre_archive_test.py @@ -10,9 +10,10 @@ os.makedirs('build') # Top level folders to be excluded - EXCLUDE = set(['.ci', '.git', '.github', 'vendor', 'dev-tools']) + EXCLUDE = set(['.ci', '.git', '.github', 'vendor', 'dev-tools', 'system-tests']) for root, dirs, files in os.walk('.'): dirs[:] = [d for d in dirs if d not in EXCLUDE] if root.endswith(('build')) and not root.startswith(('./build')): dest = os.path.join('build', root.replace('./', '')) + print("Copy {} into {}".format(root, dest)) distutils.dir_util.copy_tree(root, dest, preserve_symlinks=1) From 660ddd11b83d41eccfe03776c17b729335ab5a90 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 29 May 2020 08:46:46 +0100 Subject: [PATCH 07/25] Support multiplatform and overriide copy_tree to filter problematic folders --- .ci/scripts/pre_archive_test.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/.ci/scripts/pre_archive_test.py b/.ci/scripts/pre_archive_test.py index e5273b410c44..38ef8b57aa59 100755 --- a/.ci/scripts/pre_archive_test.py +++ b/.ci/scripts/pre_archive_test.py @@ -4,16 +4,31 @@ import distutils from distutils import dir_util +# Support filtering in the distutils.dir_util.copy_tree +ORIG_COPY_TREE = distutils.dir_util.copy_tree + + +def my_copy_tree(src, *args, **kwargs): + '''function my_copy_tree + Override distutils.dir_util.copy_tree to filter the system-tests + ''' + if src.endswith('system-tests'): + return [] + return ORIG_COPY_TREE(src, *args, **kwargs) + + +distutils.dir_util.copy_tree = my_copy_tree + if __name__ == "__main__": if not os.path.exists('build'): os.makedirs('build') # Top level folders to be excluded - EXCLUDE = set(['.ci', '.git', '.github', 'vendor', 'dev-tools', 'system-tests']) + EXCLUDE = set(['.ci', '.git', '.github', 'vendor', 'dev-tools']) for root, dirs, files in os.walk('.'): dirs[:] = [d for d in dirs if d not in EXCLUDE] if root.endswith(('build')) and not root.startswith(('./build')): - dest = os.path.join('build', root.replace('./', '')) + dest = os.path.join('build', root.replace(".{}".format(os.sep), '')) print("Copy {} into {}".format(root, dest)) distutils.dir_util.copy_tree(root, dest, preserve_symlinks=1) From 594eb004d0c85d0388f73cf0e747acb56ffdd2e4 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 29 May 2020 16:22:04 +0200 Subject: [PATCH 08/25] platform agnostic --- .ci/scripts/pre_archive_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/scripts/pre_archive_test.py b/.ci/scripts/pre_archive_test.py index 38ef8b57aa59..61c730a993e4 100755 --- a/.ci/scripts/pre_archive_test.py +++ b/.ci/scripts/pre_archive_test.py @@ -28,7 +28,7 @@ def my_copy_tree(src, *args, **kwargs): EXCLUDE = set(['.ci', '.git', '.github', 'vendor', 'dev-tools']) for root, dirs, files in os.walk('.'): dirs[:] = [d for d in dirs if d not in EXCLUDE] - if root.endswith(('build')) and not root.startswith(('./build')): + if root.endswith(('build')) and not root.startswith((".{}build".format(os.sep))): dest = os.path.join('build', root.replace(".{}".format(os.sep), '')) print("Copy {} into {}".format(root, dest)) distutils.dir_util.copy_tree(root, dest, preserve_symlinks=1) From f5c4ad8ed1e1b882094ec0a86c2e3a98820a80fd Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 29 May 2020 15:25:28 +0100 Subject: [PATCH 09/25] comment to support system-tests --- .ci/scripts/pre_archive_test.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.ci/scripts/pre_archive_test.py b/.ci/scripts/pre_archive_test.py index 61c730a993e4..0975f44e4cd2 100755 --- a/.ci/scripts/pre_archive_test.py +++ b/.ci/scripts/pre_archive_test.py @@ -5,19 +5,19 @@ from distutils import dir_util # Support filtering in the distutils.dir_util.copy_tree -ORIG_COPY_TREE = distutils.dir_util.copy_tree - - -def my_copy_tree(src, *args, **kwargs): - '''function my_copy_tree - Override distutils.dir_util.copy_tree to filter the system-tests - ''' - if src.endswith('system-tests'): - return [] - return ORIG_COPY_TREE(src, *args, **kwargs) - - -distutils.dir_util.copy_tree = my_copy_tree +#ORIG_COPY_TREE = distutils.dir_util.copy_tree +# +# +# def my_copy_tree(src, *args, **kwargs): +# '''function my_copy_tree +# Override distutils.dir_util.copy_tree to filter the system-tests +# ''' +# if src.endswith('system-tests'): +# return [] +# return ORIG_COPY_TREE(src, *args, **kwargs) +# +# +#distutils.dir_util.copy_tree = my_copy_tree if __name__ == "__main__": From 57541c36cf8594fb0f985c31f979a582708c4b0d Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 29 May 2020 15:31:18 +0100 Subject: [PATCH 10/25] prepare context to filter what to archive --- Jenkinsfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index 9b755ead18dc..947193935e2b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -853,6 +853,11 @@ def withBeatsEnv(boolean archive, Closure body) { dir('build') { junitAndStore(allowEmptyResults: true, keepLongStdio: true, testResults: '**/build/TEST*.xml') archiveArtifacts(allowEmptyArchive: true, artifacts: '**/build/TEST*.out') + catchError(buildResult: 'SUCCESS', message: 'Failed to archive the build test results', stageResult: 'SUCCESS') { + sh(label: 'Find', script: ''' + pwd + find . -path ./python-env -prune -o -type f''') + } } } } From dc124f590ad010c8b99222978e918671483afcd8 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 29 May 2020 16:48:40 +0100 Subject: [PATCH 11/25] refactor and support system-tests search for Linux --- Jenkinsfile | 58 +++++++++++++++++++++++++++++++++++------------------ 1 file changed, 38 insertions(+), 20 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 947193935e2b..03da6195d230 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -847,19 +847,7 @@ def withBeatsEnv(boolean archive, Closure body) { } } finally { if (archive) { - catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') { - fixPermissions("${WORKSPACE}") - sh(label: 'Prepare test output', script: 'python .ci/scripts/pre_archive_test.py') - dir('build') { - junitAndStore(allowEmptyResults: true, keepLongStdio: true, testResults: '**/build/TEST*.xml') - archiveArtifacts(allowEmptyArchive: true, artifacts: '**/build/TEST*.out') - catchError(buildResult: 'SUCCESS', message: 'Failed to archive the build test results', stageResult: 'SUCCESS') { - sh(label: 'Find', script: ''' - pwd - find . -path ./python-env -prune -o -type f''') - } - } - } + archiveTestOutput(testResults: '**/build/TEST*.xml', artifacts: '**/build/TEST*.out') } reportCoverage() } @@ -867,6 +855,42 @@ def withBeatsEnv(boolean archive, Closure body) { } } +/** + This method archive and report the tests output. + It searches for certain folders to bypass some issues when working with big repositories. In addition, + + OUTPUT: + - JUnit + - Artifacts in Jenkins + - Artifacts in GCP +*/ +def archiveTestOutput(Map args = [:]) { + catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') { + def labelName = 'Prepare test output' + def command = 'python .ci/scripts/pre_archive_test.py' + if (isUnix()) { + fixPermissions("${WORKSPACE}") + sh(label: labelName, script: command) + } else { + bat(label: labelName, script: command) + } + dir('build') { + junitAndStore(allowEmptyResults: true, keepLongStdio: true, testResults: args.testResults) + archiveArtifacts(allowEmptyArchive: true, artifacts: args.artifacts) + catchError(buildResult: 'SUCCESS', message: 'Failed to archive the build test results', stageResult: 'SUCCESS') { + // TODO: support windows. (required to support the windows platform in the tar step) + if (isUnix()) { + def folder = sh(label: 'Find system-tests', returnStdout: true, script: 'find . -name system-tests -type d') + if (folder.trim()) { + def name = folder.replaceAll('./', '').replaceAll('/', '-').replaceAll('build/', '') + tar(file: "${name}.tgz", archive: true, dir: folder) + } + } + } + } + } +} + def withBeatsEnvWin(Closure body) { final String chocoPath = 'C:\\ProgramData\\chocolatey\\bin' final String chocoPython3Path = 'C:\\Python38;C:\\Python38\\Scripts' @@ -890,13 +914,7 @@ def withBeatsEnvWin(Closure body) { body() } } finally { - catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') { - bat(label: 'Prepare test output', script: 'python .ci/scripts/pre_archive_test.py') - dir('build') { - junitAndStore(allowEmptyResults: true, keepLongStdio: true, testResults: "**\\build\\TEST*.xml") - archiveArtifacts(allowEmptyArchive: true, artifacts: '**\\build\\TEST*.out') - } - } + archiveTestOutput(testResults: "**\\build\\TEST*.xml", artifacts: "**\\build\\TEST*.out") } } } From 714c8edc61269085bcf843f7bf04864a6985bffc Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 29 May 2020 17:02:41 +0100 Subject: [PATCH 12/25] Support system-tests archiving in windows --- .ci/scripts/search_system_tests.py | 10 ++++++++ Jenkinsfile | 37 +++++++++++++++++++----------- 2 files changed, 34 insertions(+), 13 deletions(-) create mode 100755 .ci/scripts/search_system_tests.py diff --git a/.ci/scripts/search_system_tests.py b/.ci/scripts/search_system_tests.py new file mode 100755 index 000000000000..06a3e698f5a7 --- /dev/null +++ b/.ci/scripts/search_system_tests.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python3 + +import os + + +if __name__ == "__main__": + + for root, dirs, files in os.walk('.'): + if root.endswith(('system-tests')): + print("{}".format(root.replace(".{}".format(os.sep), ''))) diff --git a/Jenkinsfile b/Jenkinsfile index 03da6195d230..c9583e7b03b9 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -856,7 +856,7 @@ def withBeatsEnv(boolean archive, Closure body) { } /** - This method archive and report the tests output. + This method archives and report the tests output. It searches for certain folders to bypass some issues when working with big repositories. In addition, OUTPUT: @@ -866,31 +866,42 @@ def withBeatsEnv(boolean archive, Closure body) { */ def archiveTestOutput(Map args = [:]) { catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') { - def labelName = 'Prepare test output' - def command = 'python .ci/scripts/pre_archive_test.py' if (isUnix()) { fixPermissions("${WORKSPACE}") - sh(label: labelName, script: command) - } else { - bat(label: labelName, script: command) } + shOrBat(label: 'Prepare test output', script: 'python .ci/scripts/pre_archive_test.py') dir('build') { junitAndStore(allowEmptyResults: true, keepLongStdio: true, testResults: args.testResults) archiveArtifacts(allowEmptyArchive: true, artifacts: args.artifacts) catchError(buildResult: 'SUCCESS', message: 'Failed to archive the build test results', stageResult: 'SUCCESS') { - // TODO: support windows. (required to support the windows platform in the tar step) - if (isUnix()) { - def folder = sh(label: 'Find system-tests', returnStdout: true, script: 'find . -name system-tests -type d') - if (folder.trim()) { - def name = folder.replaceAll('./', '').replaceAll('/', '-').replaceAll('build/', '') - tar(file: "${name}.tgz", archive: true, dir: folder) - } + def folder = shOrBat(label: 'Find system-tests', returnStdout: true, script: 'python .ci/scripts/search_system_tests.py') + if (folder.trim()) { + def name = folder.replaceAll('/', '-').replaceAll('\\', '-').replaceAll('build', '') + tar(file: "${name}.tgz", archive: true, dir: folder) } } } } } +/** + This method encapsulates the OS shell. +*/ +def shOrBat(Map args = [:]) { + if (isUnix()) { + return sh(args) + } else { + def command = args.script + if (args.containsKey('returnStdout') && args.returnStdout) { + command = """@echo off + ${args.script} + """ + args.script = command + } + return bat(args) + } +} + def withBeatsEnvWin(Closure body) { final String chocoPath = 'C:\\ProgramData\\chocolatey\\bin' final String chocoPython3Path = 'C:\\Python38;C:\\Python38\\Scripts' From 324870ced06c37d24742023202f8c1ddbfbb418e Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 29 May 2020 17:33:42 +0100 Subject: [PATCH 13/25] fix: script path --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index c9583e7b03b9..ac12c6843af5 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -874,7 +874,7 @@ def archiveTestOutput(Map args = [:]) { junitAndStore(allowEmptyResults: true, keepLongStdio: true, testResults: args.testResults) archiveArtifacts(allowEmptyArchive: true, artifacts: args.artifacts) catchError(buildResult: 'SUCCESS', message: 'Failed to archive the build test results', stageResult: 'SUCCESS') { - def folder = shOrBat(label: 'Find system-tests', returnStdout: true, script: 'python .ci/scripts/search_system_tests.py') + def folder = shOrBat(label: 'Find system-tests', returnStdout: true, script: 'python ../.ci/scripts/search_system_tests.py') if (folder.trim()) { def name = folder.replaceAll('/', '-').replaceAll('\\', '-').replaceAll('build', '') tar(file: "${name}.tgz", archive: true, dir: folder) From 8605d364c29a355bb5de483d84e202f4f1ad3173 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 29 May 2020 18:15:13 +0100 Subject: [PATCH 14/25] Force where to look for --- .ci/scripts/search_system_tests.py | 4 ++-- Jenkinsfile | 13 +++++++------ 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.ci/scripts/search_system_tests.py b/.ci/scripts/search_system_tests.py index 06a3e698f5a7..0e3896d9ff4c 100755 --- a/.ci/scripts/search_system_tests.py +++ b/.ci/scripts/search_system_tests.py @@ -5,6 +5,6 @@ if __name__ == "__main__": - for root, dirs, files in os.walk('.'): + for root, dirs, files in os.walk('build'): if root.endswith(('system-tests')): - print("{}".format(root.replace(".{}".format(os.sep), ''))) + print(root.replace(".{}".format(os.sep), '')) diff --git a/Jenkinsfile b/Jenkinsfile index ac12c6843af5..676c07d22347 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -873,12 +873,13 @@ def archiveTestOutput(Map args = [:]) { dir('build') { junitAndStore(allowEmptyResults: true, keepLongStdio: true, testResults: args.testResults) archiveArtifacts(allowEmptyArchive: true, artifacts: args.artifacts) - catchError(buildResult: 'SUCCESS', message: 'Failed to archive the build test results', stageResult: 'SUCCESS') { - def folder = shOrBat(label: 'Find system-tests', returnStdout: true, script: 'python ../.ci/scripts/search_system_tests.py') - if (folder.trim()) { - def name = folder.replaceAll('/', '-').replaceAll('\\', '-').replaceAll('build', '') - tar(file: "${name}.tgz", archive: true, dir: folder) - } + } + catchError(buildResult: 'SUCCESS', message: 'Failed to archive the build test results', stageResult: 'SUCCESS') { + def folder = shOrBat(label: 'Find system-tests', returnStdout: true, script: 'python .ci/scripts/search_system_tests.py') + log(level: 'INFO', text: "system-tests has been found in ${folder}") + if (folder.trim()) { + def name = folder.replaceAll('/', '-').replaceAll('\\', '-').replaceAll('build', '') + tar(file: "${name}.tgz", archive: true, dir: folder) } } } From 7de9a9cd41a7da34930353587883e5a1bd7c424c Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Tue, 2 Jun 2020 08:20:38 +0100 Subject: [PATCH 15/25] Update .ci/scripts/pre_archive_test.py --- .ci/scripts/pre_archive_test.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/.ci/scripts/pre_archive_test.py b/.ci/scripts/pre_archive_test.py index 0975f44e4cd2..8fd8cb75ea1e 100755 --- a/.ci/scripts/pre_archive_test.py +++ b/.ci/scripts/pre_archive_test.py @@ -4,20 +4,6 @@ import distutils from distutils import dir_util -# Support filtering in the distutils.dir_util.copy_tree -#ORIG_COPY_TREE = distutils.dir_util.copy_tree -# -# -# def my_copy_tree(src, *args, **kwargs): -# '''function my_copy_tree -# Override distutils.dir_util.copy_tree to filter the system-tests -# ''' -# if src.endswith('system-tests'): -# return [] -# return ORIG_COPY_TREE(src, *args, **kwargs) -# -# -#distutils.dir_util.copy_tree = my_copy_tree if __name__ == "__main__": From 8b737dadb26edfdf4e3baa6c0e11860dbdc01d6e Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Tue, 2 Jun 2020 17:39:26 +0100 Subject: [PATCH 16/25] ci: use cmd step --- Jenkinsfile | 22 ++-------------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 676c07d22347..2639fc16c6aa 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -869,13 +869,13 @@ def archiveTestOutput(Map args = [:]) { if (isUnix()) { fixPermissions("${WORKSPACE}") } - shOrBat(label: 'Prepare test output', script: 'python .ci/scripts/pre_archive_test.py') + cmd(label: 'Prepare test output', script: 'python .ci/scripts/pre_archive_test.py') dir('build') { junitAndStore(allowEmptyResults: true, keepLongStdio: true, testResults: args.testResults) archiveArtifacts(allowEmptyArchive: true, artifacts: args.artifacts) } catchError(buildResult: 'SUCCESS', message: 'Failed to archive the build test results', stageResult: 'SUCCESS') { - def folder = shOrBat(label: 'Find system-tests', returnStdout: true, script: 'python .ci/scripts/search_system_tests.py') + def folder = cmd(label: 'Find system-tests', returnStdout: true, script: 'python .ci/scripts/search_system_tests.py') log(level: 'INFO', text: "system-tests has been found in ${folder}") if (folder.trim()) { def name = folder.replaceAll('/', '-').replaceAll('\\', '-').replaceAll('build', '') @@ -885,24 +885,6 @@ def archiveTestOutput(Map args = [:]) { } } -/** - This method encapsulates the OS shell. -*/ -def shOrBat(Map args = [:]) { - if (isUnix()) { - return sh(args) - } else { - def command = args.script - if (args.containsKey('returnStdout') && args.returnStdout) { - command = """@echo off - ${args.script} - """ - args.script = command - } - return bat(args) - } -} - def withBeatsEnvWin(Closure body) { final String chocoPath = 'C:\\ProgramData\\chocolatey\\bin' final String chocoPython3Path = 'C:\\Python38;C:\\Python38\\Scripts' From 5150beb7cf95ba2deb20aa3ffec7bdf0f21160f8 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Tue, 2 Jun 2020 17:40:33 +0100 Subject: [PATCH 17/25] ci: rephrase a bit the docs --- Jenkinsfile | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 2639fc16c6aa..48f4ae2f1f12 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -856,13 +856,8 @@ def withBeatsEnv(boolean archive, Closure body) { } /** - This method archives and report the tests output. - It searches for certain folders to bypass some issues when working with big repositories. In addition, - - OUTPUT: - - JUnit - - Artifacts in Jenkins - - Artifacts in GCP + This method archives and report the tests output, for such, it searches in certain folders + to bypass some issues when working with big repositories. */ def archiveTestOutput(Map args = [:]) { catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') { From b6606c44d9e063bd1b4f301ea02ef595603f04fb Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 3 Jun 2020 07:18:35 +0100 Subject: [PATCH 18/25] Update Jenkinsfile --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 48f4ae2f1f12..f0c6dd2c8e6b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -870,7 +870,7 @@ def archiveTestOutput(Map args = [:]) { archiveArtifacts(allowEmptyArchive: true, artifacts: args.artifacts) } catchError(buildResult: 'SUCCESS', message: 'Failed to archive the build test results', stageResult: 'SUCCESS') { - def folder = cmd(label: 'Find system-tests', returnStdout: true, script: 'python .ci/scripts/search_system_tests.py') + def folder = cmd(label: 'Find system-tests', returnStdout: true, script: 'python .ci/scripts/search_system_tests.py').trim() log(level: 'INFO', text: "system-tests has been found in ${folder}") if (folder.trim()) { def name = folder.replaceAll('/', '-').replaceAll('\\', '-').replaceAll('build', '') From ee7f5706e3688c524a939d3b791e44d73bca609f Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 3 Jun 2020 08:47:56 +0100 Subject: [PATCH 19/25] fix: use hardcode paths --- .ci/scripts/search_system_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/scripts/search_system_tests.py b/.ci/scripts/search_system_tests.py index 0e3896d9ff4c..c2d802a4fb19 100755 --- a/.ci/scripts/search_system_tests.py +++ b/.ci/scripts/search_system_tests.py @@ -7,4 +7,4 @@ for root, dirs, files in os.walk('build'): if root.endswith(('system-tests')): - print(root.replace(".{}".format(os.sep), '')) + print(root.replace("./", '').replace(".\\", '')) From 6557388c90cd8bc07d621d8a708cd75ff7ecdc4a Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 12 Jun 2020 15:30:17 +0100 Subject: [PATCH 20/25] ci: cosmetic change in the log --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index d013d5167959..a22bbfdf627a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -934,7 +934,7 @@ def archiveTestOutput(Map args = [:]) { } catchError(buildResult: 'SUCCESS', message: 'Failed to archive the build test results', stageResult: 'SUCCESS') { def folder = cmd(label: 'Find system-tests', returnStdout: true, script: 'python .ci/scripts/search_system_tests.py').trim() - log(level: 'INFO', text: "system-tests has been found in ${folder}") + log(level: 'INFO', text: "system-tests='${folder}'. If no empty then let's create a tarbal") if (folder.trim()) { def name = folder.replaceAll('/', '-').replaceAll('\\', '-').replaceAll('build', '') tar(file: "${name}.tgz", archive: true, dir: folder) From bd8d0d546a5fd156d70f410b26b87a7bd60584ae Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 12 Jun 2020 17:02:33 +0100 Subject: [PATCH 21/25] fix: the evil regex in groovy with backslashes --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index a22bbfdf627a..3b7a57f2fe6a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -934,9 +934,9 @@ def archiveTestOutput(Map args = [:]) { } catchError(buildResult: 'SUCCESS', message: 'Failed to archive the build test results', stageResult: 'SUCCESS') { def folder = cmd(label: 'Find system-tests', returnStdout: true, script: 'python .ci/scripts/search_system_tests.py').trim() - log(level: 'INFO', text: "system-tests='${folder}'. If no empty then let's create a tarbal") + log(level: 'INFO', text: "system-tests='${folder}'. If no empty then let's create a tarball") if (folder.trim()) { - def name = folder.replaceAll('/', '-').replaceAll('\\', '-').replaceAll('build', '') + def name = folder.replaceAll('/', '-').replaceAll('\\\\', '-').replaceAll('build', '') tar(file: "${name}.tgz", archive: true, dir: folder) } } From 626840de94d41dd4a6a7d1a1d6b63d83b079420f Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 12 Jun 2020 17:03:36 +0100 Subject: [PATCH 22/25] Revert "fix: use hardcode paths" This reverts commit ee7f5706e3688c524a939d3b791e44d73bca609f. --- .ci/scripts/search_system_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/scripts/search_system_tests.py b/.ci/scripts/search_system_tests.py index c2d802a4fb19..0e3896d9ff4c 100755 --- a/.ci/scripts/search_system_tests.py +++ b/.ci/scripts/search_system_tests.py @@ -7,4 +7,4 @@ for root, dirs, files in os.walk('build'): if root.endswith(('system-tests')): - print(root.replace("./", '').replace(".\\", '')) + print(root.replace(".{}".format(os.sep), '')) From 6af8dfb0e921454b9125f50b1773dc6839542bbd Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 12 Jun 2020 17:50:26 +0100 Subject: [PATCH 23/25] Support OS in the name of the archived files --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 3b7a57f2fe6a..aafa025b9f60 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -936,7 +936,7 @@ def archiveTestOutput(Map args = [:]) { def folder = cmd(label: 'Find system-tests', returnStdout: true, script: 'python .ci/scripts/search_system_tests.py').trim() log(level: 'INFO', text: "system-tests='${folder}'. If no empty then let's create a tarball") if (folder.trim()) { - def name = folder.replaceAll('/', '-').replaceAll('\\\\', '-').replaceAll('build', '') + def name = folder.replaceAll('/', '-').replaceAll('\\\\', '-').replaceAll('build', '') + goos() tar(file: "${name}.tgz", archive: true, dir: folder) } } From f7a03a6b4b285fc382b72258a05b32a0fec97295 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Mon, 15 Jun 2020 13:31:43 +0100 Subject: [PATCH 24/25] ci: platform agnostic --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 69ceb5cb3874..fa8c8cbddbb4 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -936,7 +936,7 @@ def archiveTestOutput(Map args = [:]) { def folder = cmd(label: 'Find system-tests', returnStdout: true, script: 'python .ci/scripts/search_system_tests.py').trim() log(level: 'INFO', text: "system-tests='${folder}'. If no empty then let's create a tarball") if (folder.trim()) { - def name = folder.replaceAll('/', '-').replaceAll('\\\\', '-').replaceAll('build', '') + goos() + def name = folder.replaceAll(File.separator, '-').replaceAll('build', '').replaceAll('^-', '') + '-' + goos() tar(file: "${name}.tgz", archive: true, dir: folder) } } From 1838113e3bd5cbf54a905b8509364222ffdd66dc Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Mon, 15 Jun 2020 14:19:28 +0100 Subject: [PATCH 25/25] ci: fix the script approval for File.Separator --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index fa8c8cbddbb4..852bdef64988 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -936,7 +936,7 @@ def archiveTestOutput(Map args = [:]) { def folder = cmd(label: 'Find system-tests', returnStdout: true, script: 'python .ci/scripts/search_system_tests.py').trim() log(level: 'INFO', text: "system-tests='${folder}'. If no empty then let's create a tarball") if (folder.trim()) { - def name = folder.replaceAll(File.separator, '-').replaceAll('build', '').replaceAll('^-', '') + '-' + goos() + def name = folder.replaceAll('/', '-').replaceAll('\\\\', '-').replaceAll('build', '').replaceAll('^-', '') + '-' + goos() tar(file: "${name}.tgz", archive: true, dir: folder) } }