From 9c1284e81d7a1ea68aff65389365b9882ee04de6 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 28 Jul 2021 14:11:23 -0700 Subject: [PATCH 01/66] Add superpmi-replay pipeline file --- eng/pipelines/coreclr/superpmi-replay.yml | 30 +++++++++++++++++++ .../coreclr/templates/build-jit-job.yml | 10 ++++--- 2 files changed, 36 insertions(+), 4 deletions(-) create mode 100644 eng/pipelines/coreclr/superpmi-replay.yml diff --git a/eng/pipelines/coreclr/superpmi-replay.yml b/eng/pipelines/coreclr/superpmi-replay.yml new file mode 100644 index 00000000000000..f0786192ba946d --- /dev/null +++ b/eng/pipelines/coreclr/superpmi-replay.yml @@ -0,0 +1,30 @@ +trigger: + batch: false + branches: + include: + - main + paths: + include: + - src/coreclr/jit/* + - src/coreclr/inc/jiteeversionguid.h + +# This pipeline is supposed to be run only on merged changes +# and should not be triggerable from a PR. +pr: none + +jobs: + +- template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/build-jit-job.yml + buildConfig: checked + skipBinariesUpload: true + platforms: + - OSX_arm64 + - OSX_x64 + - Linux_arm + - Linux_arm64 + - Linux_x64 + - windows_x64 + - windows_x86 + - windows_arm64 diff --git a/eng/pipelines/coreclr/templates/build-jit-job.yml b/eng/pipelines/coreclr/templates/build-jit-job.yml index 74df781d1ed188..6ae36ea31512ae 100644 --- a/eng/pipelines/coreclr/templates/build-jit-job.yml +++ b/eng/pipelines/coreclr/templates/build-jit-job.yml @@ -11,6 +11,7 @@ parameters: timeoutInMinutes: '' variables: {} dependOnEvaluatePaths: false + skipBinariesUpload: false ### Product build jobs: @@ -110,10 +111,11 @@ jobs: - script: $(PipScript) install --user --upgrade pip && $(PipScript) install --user azure.storage.blob==12.5.0 --force-reinstall displayName: Upgrade Pip to latest and install azure-storage-blob Python package - - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/jitrollingbuild.py upload -build_type $(buildConfig) -arch $(archType) -host_os $(osGroup) -git_hash $(Build.SourceVersion) - displayName: Upload JIT to Azure Storage - env: - CLRJIT_AZ_KEY: $(clrjit_key1) # secret key stored as variable in pipeline + - ${{ if eq(parameters.skipBinariesUpload, 'false') }}: + - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/jitrollingbuild.py upload -build_type $(buildConfig) -arch $(archType) -host_os $(osGroup) -git_hash $(Build.SourceVersion) + displayName: Upload JIT to Azure Storage + env: + CLRJIT_AZ_KEY: $(clrjit_key1) # secret key stored as variable in pipeline # Publish Logs - task: PublishPipelineArtifact@1 From 60b44b40385f8fd4d72ab5565a2c16db77da1c3a Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 28 Jul 2021 16:07:41 -0700 Subject: [PATCH 02/66] Add superpmi-replay-job.yml that downloads collection --- eng/pipelines/coreclr/superpmi-replay-job.yml | 34 +++++++++++++++++++ eng/pipelines/coreclr/superpmi-replay.yml | 17 ++++++---- .../coreclr/templates/build-jit-job.yml | 8 ++--- 3 files changed, 49 insertions(+), 10 deletions(-) create mode 100644 eng/pipelines/coreclr/superpmi-replay-job.yml diff --git a/eng/pipelines/coreclr/superpmi-replay-job.yml b/eng/pipelines/coreclr/superpmi-replay-job.yml new file mode 100644 index 00000000000000..4c116b6d5c8ae5 --- /dev/null +++ b/eng/pipelines/coreclr/superpmi-replay-job.yml @@ -0,0 +1,34 @@ +parameters: + buildConfig: '' # required -- build configuration + archType: '' # required -- targeting CPU architecture + osGroup: '' # required -- operating system for the job + osSubgroup: '' # optional -- operating system subgroup + dependsOn: '' # optional -- dependencies of the job + +jobs: +- template: xplat-pipeline-job.yml + parameters: + dependsOn: ${{ parameters.dependsOn }} + buildConfig: ${{ parameters.buildConfig }} + archType: ${{ parameters.archType }} + osGroup: ${{ parameters.osGroup }} + osSubgroup: ${{ parameters.osSubgroup }} + + variables: + - name: PythonScript + value: 'py -3' + - name: PipScript + value: 'py -3 -m pip' + - name: Core_Root_Dir + value: '$(Build.SourcesDirectory)\artifacts\tests\coreclr\${{ parameters.osGroup }}.${{ parameters.archType }}.${{ parameters.buildConfig }}\Tests\Core_Root' + - name: SpmiCollectionLocation + value: '$(Build.SourcesDirectory)\artifacts\spmi\' + + steps: + - script: | + mkdir -p $(SpmiLogsLocation) + displayName: Create directory for SPMI collection + + - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi.py download -spmi_location $(SpmiCollectionLocation) + displayName: Download superpmi collection + \ No newline at end of file diff --git a/eng/pipelines/coreclr/superpmi-replay.yml b/eng/pipelines/coreclr/superpmi-replay.yml index f0786192ba946d..355cb49c3bc6ce 100644 --- a/eng/pipelines/coreclr/superpmi-replay.yml +++ b/eng/pipelines/coreclr/superpmi-replay.yml @@ -20,11 +20,16 @@ jobs: buildConfig: checked skipBinariesUpload: true platforms: - - OSX_arm64 - - OSX_x64 - - Linux_arm - - Linux_arm64 - - Linux_x64 - windows_x64 - windows_x86 - - windows_arm64 + +- template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/superpmi-replay-job.yml + buildConfig: checked + platforms: + - windows_x64 + helixQueueGroup: ci + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + jobParameters: + testGroup: outerloop \ No newline at end of file diff --git a/eng/pipelines/coreclr/templates/build-jit-job.yml b/eng/pipelines/coreclr/templates/build-jit-job.yml index 6ae36ea31512ae..302fb8d02a5804 100644 --- a/eng/pipelines/coreclr/templates/build-jit-job.yml +++ b/eng/pipelines/coreclr/templates/build-jit-job.yml @@ -107,11 +107,11 @@ jobs: - script: set __TestIntermediateDir=int&&$(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) -ci -nopgooptimize -skiprestoreoptdata -component alljits displayName: Build CoreCLR JIT - # Ensure the Python azure-storage-blob package is installed before doing the upload. - - script: $(PipScript) install --user --upgrade pip && $(PipScript) install --user azure.storage.blob==12.5.0 --force-reinstall - displayName: Upgrade Pip to latest and install azure-storage-blob Python package - - ${{ if eq(parameters.skipBinariesUpload, 'false') }}: + # Ensure the Python azure-storage-blob package is installed before doing the upload. + - script: $(PipScript) install --user --upgrade pip && $(PipScript) install --user azure.storage.blob==12.5.0 --force-reinstall + displayName: Upgrade Pip to latest and install azure-storage-blob Python package + - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/jitrollingbuild.py upload -build_type $(buildConfig) -arch $(archType) -host_os $(osGroup) -git_hash $(Build.SourceVersion) displayName: Upload JIT to Azure Storage env: From 1919da8f804e614367933e5192bc95924de3a66d Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 28 Jul 2021 16:09:28 -0700 Subject: [PATCH 03/66] Moved superpmi-replay-job to right location --- eng/pipelines/coreclr/{ => templates}/superpmi-replay-job.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename eng/pipelines/coreclr/{ => templates}/superpmi-replay-job.yml (100%) diff --git a/eng/pipelines/coreclr/superpmi-replay-job.yml b/eng/pipelines/coreclr/templates/superpmi-replay-job.yml similarity index 100% rename from eng/pipelines/coreclr/superpmi-replay-job.yml rename to eng/pipelines/coreclr/templates/superpmi-replay-job.yml From f85e0729d9f9e6ac88dcc4736a319e1b478358d5 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 28 Jul 2021 16:16:40 -0700 Subject: [PATCH 04/66] make sure to have spmi collection after jit build --- eng/pipelines/coreclr/superpmi-replay.yml | 10 +--------- .../coreclr/templates/superpmi-replay-job.yml | 5 ++++- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/eng/pipelines/coreclr/superpmi-replay.yml b/eng/pipelines/coreclr/superpmi-replay.yml index 355cb49c3bc6ce..20fc5f9ed2f2b6 100644 --- a/eng/pipelines/coreclr/superpmi-replay.yml +++ b/eng/pipelines/coreclr/superpmi-replay.yml @@ -14,21 +14,13 @@ pr: none jobs: -- template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/build-jit-job.yml - buildConfig: checked - skipBinariesUpload: true - platforms: - - windows_x64 - - windows_x86 - - template: /eng/pipelines/common/platform-matrix.yml parameters: jobTemplate: /eng/pipelines/coreclr/templates/superpmi-replay-job.yml buildConfig: checked platforms: - windows_x64 + - windows_x86 helixQueueGroup: ci helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml jobParameters: diff --git a/eng/pipelines/coreclr/templates/superpmi-replay-job.yml b/eng/pipelines/coreclr/templates/superpmi-replay-job.yml index 4c116b6d5c8ae5..2813fe1555f393 100644 --- a/eng/pipelines/coreclr/templates/superpmi-replay-job.yml +++ b/eng/pipelines/coreclr/templates/superpmi-replay-job.yml @@ -4,9 +4,10 @@ parameters: osGroup: '' # required -- operating system for the job osSubgroup: '' # optional -- operating system subgroup dependsOn: '' # optional -- dependencies of the job + runJobTemplate: '/eng/pipelines/coreclr/templates/build-jit-job.yml' jobs: -- template: xplat-pipeline-job.yml +- template: ${{ parameters.runJobTemplate }} parameters: dependsOn: ${{ parameters.dependsOn }} buildConfig: ${{ parameters.buildConfig }} @@ -25,6 +26,8 @@ jobs: value: '$(Build.SourcesDirectory)\artifacts\spmi\' steps: + # Extra steps that will be passed to the superpmi template and run before sending the job to helix (all of which is done in the template) + - script: | mkdir -p $(SpmiLogsLocation) displayName: Create directory for SPMI collection From 224adcddb8b7ea04f4ae901db13baa4491453ad5 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 28 Jul 2021 16:30:05 -0700 Subject: [PATCH 05/66] annotate displayName with windows/OSX --- eng/pipelines/coreclr/templates/build-jit-job.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/coreclr/templates/build-jit-job.yml b/eng/pipelines/coreclr/templates/build-jit-job.yml index 302fb8d02a5804..a1b51e9ba8a3c2 100644 --- a/eng/pipelines/coreclr/templates/build-jit-job.yml +++ b/eng/pipelines/coreclr/templates/build-jit-job.yml @@ -86,11 +86,11 @@ jobs: # preinstalled, so we only need this step for OSX and Windows. - ${{ if eq(parameters.osGroup, 'OSX') }}: - script: $(Build.SourcesDirectory)/eng/install-native-dependencies.sh $(osGroup) ${{ parameters.archType }} azDO - displayName: Install native dependencies + displayName: Install native dependencies (OSX) - ${{ if eq(parameters.osGroup, 'windows') }}: # Necessary to install python - script: $(Build.SourcesDirectory)\eng\common\init-tools-native.cmd -InstallDirectory $(Build.SourcesDirectory)\native-tools -Force - displayName: Install native dependencies + displayName: Install native dependencies (windows) # Install internal tools on official builds # Since our internal tools are behind an authenticated feed, From 33072e3d6bfa09bd04d6fc414863f447ab1aab2c Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 28 Jul 2021 18:08:36 -0700 Subject: [PATCH 06/66] Make osGroup=windows --- eng/pipelines/coreclr/templates/superpmi-replay-job.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/coreclr/templates/superpmi-replay-job.yml b/eng/pipelines/coreclr/templates/superpmi-replay-job.yml index 2813fe1555f393..5cfdc9d9f557b1 100644 --- a/eng/pipelines/coreclr/templates/superpmi-replay-job.yml +++ b/eng/pipelines/coreclr/templates/superpmi-replay-job.yml @@ -1,7 +1,7 @@ parameters: buildConfig: '' # required -- build configuration archType: '' # required -- targeting CPU architecture - osGroup: '' # required -- operating system for the job + osGroup: 'windows' # required -- operating system for the job osSubgroup: '' # optional -- operating system subgroup dependsOn: '' # optional -- dependencies of the job runJobTemplate: '/eng/pipelines/coreclr/templates/build-jit-job.yml' From 207ce4156b9402754ab0156e7ace0d575089dc44 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 28 Jul 2021 18:12:54 -0700 Subject: [PATCH 07/66] Add helixType (if that matters) --- eng/pipelines/coreclr/templates/superpmi-replay-job.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/eng/pipelines/coreclr/templates/superpmi-replay-job.yml b/eng/pipelines/coreclr/templates/superpmi-replay-job.yml index 5cfdc9d9f557b1..41eda15fcc84a9 100644 --- a/eng/pipelines/coreclr/templates/superpmi-replay-job.yml +++ b/eng/pipelines/coreclr/templates/superpmi-replay-job.yml @@ -1,7 +1,7 @@ parameters: buildConfig: '' # required -- build configuration archType: '' # required -- targeting CPU architecture - osGroup: 'windows' # required -- operating system for the job + osGroup: '' # required -- operating system for the job osSubgroup: '' # optional -- operating system subgroup dependsOn: '' # optional -- dependencies of the job runJobTemplate: '/eng/pipelines/coreclr/templates/build-jit-job.yml' @@ -14,6 +14,11 @@ jobs: archType: ${{ parameters.archType }} osGroup: ${{ parameters.osGroup }} osSubgroup: ${{ parameters.osSubgroup }} + helixType: 'build/product/' + enableMicrobuild: true + stagedBuild: ${{ parameters.stagedBuild }} + pool: ${{ parameters.pool }} + dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths }} variables: - name: PythonScript From 84e3dfec260423cbe6893d8330bfc2f39d75a3f7 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Thu, 29 Jul 2021 07:26:42 -0700 Subject: [PATCH 08/66] Add some more variables --- eng/pipelines/coreclr/templates/superpmi-replay-job.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/eng/pipelines/coreclr/templates/superpmi-replay-job.yml b/eng/pipelines/coreclr/templates/superpmi-replay-job.yml index 41eda15fcc84a9..c8ae53aebc461e 100644 --- a/eng/pipelines/coreclr/templates/superpmi-replay-job.yml +++ b/eng/pipelines/coreclr/templates/superpmi-replay-job.yml @@ -4,6 +4,12 @@ parameters: osGroup: '' # required -- operating system for the job osSubgroup: '' # optional -- operating system subgroup dependsOn: '' # optional -- dependencies of the job + pool: '' + stagedBuild: false + timeoutInMinutes: '' + variables: {} + dependOnEvaluatePaths: false + skipBinariesUpload: false runJobTemplate: '/eng/pipelines/coreclr/templates/build-jit-job.yml' jobs: From 2b232d09d16aaf5b304199682e5370cecf6acc69 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Thu, 29 Jul 2021 07:46:17 -0700 Subject: [PATCH 09/66] skip restore internal tools --- eng/pipelines/coreclr/templates/superpmi-replay-job.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eng/pipelines/coreclr/templates/superpmi-replay-job.yml b/eng/pipelines/coreclr/templates/superpmi-replay-job.yml index c8ae53aebc461e..eb5f275731767d 100644 --- a/eng/pipelines/coreclr/templates/superpmi-replay-job.yml +++ b/eng/pipelines/coreclr/templates/superpmi-replay-job.yml @@ -25,6 +25,7 @@ jobs: stagedBuild: ${{ parameters.stagedBuild }} pool: ${{ parameters.pool }} dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths }} + _skipRestoreInternalTools: true variables: - name: PythonScript From 971d198e8e9f4f9eb9fe0c648ce642ecdf510b3a Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Thu, 29 Jul 2021 07:54:02 -0700 Subject: [PATCH 10/66] Just use build-jit-job directly --- eng/pipelines/coreclr/superpmi-replay.yml | 26 ++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/eng/pipelines/coreclr/superpmi-replay.yml b/eng/pipelines/coreclr/superpmi-replay.yml index 20fc5f9ed2f2b6..7734a5bb46532a 100644 --- a/eng/pipelines/coreclr/superpmi-replay.yml +++ b/eng/pipelines/coreclr/superpmi-replay.yml @@ -16,12 +16,28 @@ jobs: - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/coreclr/templates/superpmi-replay-job.yml + jobTemplate: /eng/pipelines/coreclr/templates/build-jit-job.yml buildConfig: checked platforms: - windows_x64 - windows_x86 - helixQueueGroup: ci - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testGroup: outerloop \ No newline at end of file + + variables: + - name: PythonScript + value: 'py -3' + - name: PipScript + value: 'py -3 -m pip' + - name: Core_Root_Dir + value: '$(Build.SourcesDirectory)\artifacts\tests\coreclr\${{ parameters.osGroup }}.${{ parameters.archType }}.${{ parameters.buildConfig }}\Tests\Core_Root' + - name: SpmiCollectionLocation + value: '$(Build.SourcesDirectory)\artifacts\spmi\' + + steps: + # Extra steps that will be passed to the superpmi template and run before sending the job to helix (all of which is done in the template) + + - script: | + mkdir -p $(SpmiLogsLocation) + displayName: Create directory for SPMI collection + + - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi.py download -spmi_location $(SpmiCollectionLocation) + displayName: Download superpmi collection \ No newline at end of file From bc571f9fb5504cab6c0f27fabe00145aee406f5b Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Thu, 29 Jul 2021 07:56:39 -0700 Subject: [PATCH 11/66] Hardcode windows.x64.checked --- eng/pipelines/coreclr/superpmi-replay.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/coreclr/superpmi-replay.yml b/eng/pipelines/coreclr/superpmi-replay.yml index 7734a5bb46532a..d24d20930677c3 100644 --- a/eng/pipelines/coreclr/superpmi-replay.yml +++ b/eng/pipelines/coreclr/superpmi-replay.yml @@ -20,7 +20,7 @@ jobs: buildConfig: checked platforms: - windows_x64 - - windows_x86 + # - windows_x86 variables: - name: PythonScript @@ -28,7 +28,8 @@ jobs: - name: PipScript value: 'py -3 -m pip' - name: Core_Root_Dir - value: '$(Build.SourcesDirectory)\artifacts\tests\coreclr\${{ parameters.osGroup }}.${{ parameters.archType }}.${{ parameters.buildConfig }}\Tests\Core_Root' + value: '$(Build.SourcesDirectory)\artifacts\tests\coreclr\windows.x64.checked\Tests\Core_Root' + # value: '$(Build.SourcesDirectory)\artifacts\tests\coreclr\${{ parameters.osGroup }}.${{ parameters.archType }}.${{ parameters.buildConfig }}\Tests\Core_Root' - name: SpmiCollectionLocation value: '$(Build.SourcesDirectory)\artifacts\spmi\' From f7f77ae75a8327a97c180b8b4ef3f940d68b55e9 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Thu, 29 Jul 2021 08:11:11 -0700 Subject: [PATCH 12/66] Add -download_all -download_raw options to superpmi.py --- src/coreclr/scripts/superpmi.py | 43 ++++++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 9 deletions(-) diff --git a/src/coreclr/scripts/superpmi.py b/src/coreclr/scripts/superpmi.py index eb1defc6ba775f..cc6437a963b5df 100755 --- a/src/coreclr/scripts/superpmi.py +++ b/src/coreclr/scripts/superpmi.py @@ -194,6 +194,14 @@ target directory already exists. """ +download_raw_help = """\ +If specified 'raw', then mch files will not be decompressed. +""" + +download_all_help = """\ +If specified, it will override the target_arch/target_os and download mch files for all arch/OS. +""" + merge_mch_pattern_help = """\ A pattern to describing files to merge, passed through directly to `mcs -merge`. Acceptable patterns include `*.mch`, `file*.mch`, and `c:\\my\\directory\\*.mch`. @@ -337,6 +345,8 @@ download_parser.add_argument("-jit_ee_version", help=jit_ee_version_help) download_parser.add_argument("--skip_cleanup", action="store_true", help=skip_cleanup_help) download_parser.add_argument("--force_download", action="store_true", help=force_download_help) +download_parser.add_argument("--download_raw", action="store_true", help=download_raw_help) +download_parser.add_argument("--download_all", action="store_true", help=download_all_help) download_parser.add_argument("-mch_files", metavar="MCH_FILE", nargs='+', help=replay_mch_files_help) download_parser.add_argument("-private_store", action="append", help=private_store_help) @@ -2645,7 +2655,7 @@ def download_mch_from_azure(coreclr_args, target_dir): list containing the local path of files downloaded """ - blob_filter_string = "{}/{}/{}/".format(coreclr_args.jit_ee_version, coreclr_args.target_os, coreclr_args.mch_arch).lower() + blob_filter_string = "{}/".format(coreclr_args.jit_ee_version).lower() if coreclr_args.download_all else "{}/{}/{}/".format(coreclr_args.jit_ee_version, coreclr_args.target_os, coreclr_args.mch_arch).lower() # Determine if a URL in Azure Storage should be allowed. The path looks like: # jit-ee-guid/Linux/x64/Linux.x64.Checked.frameworks.mch.zip @@ -2664,13 +2674,14 @@ def filter_superpmi_collections(path): blob_url_prefix = "{}/{}/".format(az_blob_storage_superpmi_container_uri, az_collections_root_folder) urls = [blob_url_prefix + path for path in paths] - return download_files(urls, target_dir) + return download_files(urls, target_dir, uncompress=not coreclr_args.download_raw) -def download_files(paths, target_dir, verbose=True, fail_if_not_found=True): +def download_files(paths, target_dir, uncompress=True, verbose=True, fail_if_not_found=True): """ Download a set of files, specified as URLs or paths (such as Windows UNC paths), to a target directory. If a file is a .ZIP file, then uncompress the file and - copy all its contents to the target directory. + copy all its contents to the target directory. If skipUncompress is specified, then + simply copy the .ZIP file to the target directory without uncompressing. Args: paths (list): the URLs and paths to download @@ -2725,13 +2736,17 @@ def download_files(paths, target_dir, verbose=True, fail_if_not_found=True): logging.info("Download: %s -> %s", item_path, download_path) shutil.copy2(item_path, download_path) - if verbose: - logging.info("Uncompress %s", download_path) - with zipfile.ZipFile(download_path, "r") as file_handle: - file_handle.extractall(temp_location) + if uncompress: + if verbose: + logging.info("Uncompress %s", download_path) + with zipfile.ZipFile(download_path, "r") as file_handle: + file_handle.extractall(temp_location) # Copy everything that was extracted to the target directory. - items = [ os.path.join(temp_location, item) for item in os.listdir(temp_location) if not item.endswith(".zip") ] + if uncompress: + items = [ os.path.join(temp_location, item) for item in os.listdir(temp_location) if not item.endswith(".zip") ] + else: + items = [ os.path.join(temp_location, item) for item in os.listdir(temp_location) if item.endswith(".zip") ] for item in items: target_path = os.path.join(target_dir, os.path.basename(item)) if verbose: @@ -3774,6 +3789,16 @@ def verify_replay_common_args(): lambda unused: True, "Unable to set force_download") + coreclr_args.verify(args, + "download_all", + lambda unused: True, + "Unable to set download_all") + + coreclr_args.verify(args, + "download_raw", + lambda unused: True, + "Unable to set download_raw") + coreclr_args.verify(args, "filter", lambda unused: True, From a7bdc5db3de8ec43416b8cfc255e14b971b110f4 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Thu, 29 Jul 2021 12:50:23 -0700 Subject: [PATCH 13/66] Seperate 2 jobs --- eng/pipelines/coreclr/superpmi-replay.yml | 31 +++----- .../coreclr/templates/build-jit-job.yml | 14 +++- .../templates/run-superpmi-replay-job.yml | 76 +++++++++++++++++++ .../coreclr/templates/superpmi-replay-job.yml | 41 ++++------ 4 files changed, 113 insertions(+), 49 deletions(-) create mode 100644 eng/pipelines/coreclr/templates/run-superpmi-replay-job.yml diff --git a/eng/pipelines/coreclr/superpmi-replay.yml b/eng/pipelines/coreclr/superpmi-replay.yml index d24d20930677c3..73202c97df43cb 100644 --- a/eng/pipelines/coreclr/superpmi-replay.yml +++ b/eng/pipelines/coreclr/superpmi-replay.yml @@ -20,25 +20,14 @@ jobs: buildConfig: checked platforms: - windows_x64 - # - windows_x86 + - windows_x86 - variables: - - name: PythonScript - value: 'py -3' - - name: PipScript - value: 'py -3 -m pip' - - name: Core_Root_Dir - value: '$(Build.SourcesDirectory)\artifacts\tests\coreclr\windows.x64.checked\Tests\Core_Root' - # value: '$(Build.SourcesDirectory)\artifacts\tests\coreclr\${{ parameters.osGroup }}.${{ parameters.archType }}.${{ parameters.buildConfig }}\Tests\Core_Root' - - name: SpmiCollectionLocation - value: '$(Build.SourcesDirectory)\artifacts\spmi\' - - steps: - # Extra steps that will be passed to the superpmi template and run before sending the job to helix (all of which is done in the template) - - - script: | - mkdir -p $(SpmiLogsLocation) - displayName: Create directory for SPMI collection - - - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi.py download -spmi_location $(SpmiCollectionLocation) - displayName: Download superpmi collection \ No newline at end of file +- template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/superpmi-replay-job.yml + buildConfig: checked + platforms: + - windows_x64 + - windows_x86 + helixQueueGroup: ci + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml \ No newline at end of file diff --git a/eng/pipelines/coreclr/templates/build-jit-job.yml b/eng/pipelines/coreclr/templates/build-jit-job.yml index a1b51e9ba8a3c2..f1bad18d34a50a 100644 --- a/eng/pipelines/coreclr/templates/build-jit-job.yml +++ b/eng/pipelines/coreclr/templates/build-jit-job.yml @@ -107,7 +107,7 @@ jobs: - script: set __TestIntermediateDir=int&&$(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) -ci -nopgooptimize -skiprestoreoptdata -component alljits displayName: Build CoreCLR JIT - - ${{ if eq(parameters.skipBinariesUpload, 'false') }}: + - ${{ if eq(parameters.uploadAsArtifacts, 'false') }}: # Ensure the Python azure-storage-blob package is installed before doing the upload. - script: $(PipScript) install --user --upgrade pip && $(PipScript) install --user azure.storage.blob==12.5.0 --force-reinstall displayName: Upgrade Pip to latest and install azure-storage-blob Python package @@ -117,6 +117,18 @@ jobs: env: CLRJIT_AZ_KEY: $(clrjit_key1) # secret key stored as variable in pipeline + - ${{ if eq(parameters.uploadAsArtifacts, 'true') }}: + # Publish product output directory for consumption by tests. + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(buildProductRootFolderPath) + includeRootFolder: false + archiveType: $(archiveType) + tarCompression: $(tarCompression) + archiveExtension: $(archiveExtension) + artifactName: $(buildProductArtifactName) + displayName: 'product build' + # Publish Logs - task: PublishPipelineArtifact@1 displayName: Publish Logs diff --git a/eng/pipelines/coreclr/templates/run-superpmi-replay-job.yml b/eng/pipelines/coreclr/templates/run-superpmi-replay-job.yml new file mode 100644 index 00000000000000..376f9d8e698d2e --- /dev/null +++ b/eng/pipelines/coreclr/templates/run-superpmi-replay-job.yml @@ -0,0 +1,76 @@ +parameters: + steps: [] # optional -- any additional steps that need to happen before pulling down the jitutils repo and sending the jitutils to helix (ie building your repo) + variables: [] # optional -- list of additional variables to send to the template + jobName: '' # required -- job name + displayName: '' # optional -- display name for the job. Will use jobName if not passed + pool: '' # required -- name of the Build pool + container: '' # required -- name of the container + buildConfig: '' # required -- build configuration + archType: '' # required -- targeting CPU architecture + osGroup: '' # required -- operating system for the job + osSubgroup: '' # optional -- operating system subgroup + extraSetupParameters: '' # optional -- extra arguments to pass to the setup script + frameworks: ['netcoreapp3.0'] # optional -- list of frameworks to run against + continueOnError: 'false' # optional -- determines whether to continue the build if the step errors + dependsOn: '' # optional -- dependencies of the job + timeoutInMinutes: 320 # optional -- timeout for the job + enableTelemetry: false # optional -- enable for telemetry + liveLibrariesBuildConfig: '' # optional -- live-live libraries configuration to use for the run + runtimeType: 'coreclr' # optional -- Sets the runtime as coreclr or mono + codeGenType: 'JIT' # optional -- Decides on the codegen technology if running on mono + projectFile: 'superpmi.proj' # optional -- project file to build helix workitems + runKind: '' # required -- test category + collectionType: '' + collectionName: '' + dependOnEvaluatePaths: false + +jobs: +- template: xplat-pipeline-job.yml + parameters: + dependsOn: ${{ parameters.dependsOn }} + buildConfig: ${{ parameters.buildConfig }} + archType: ${{ parameters.archType }} + osGroup: ${{ parameters.osGroup }} + osSubgroup: ${{ parameters.osSubgroup }} + liveLibrariesBuildConfig: ${{ parameters.liveLibrariesBuildConfig }} + enableTelemetry: ${{ parameters.enableTelemetry }} + enablePublishBuildArtifacts: true + continueOnError: ${{ parameters.continueOnError }} + collectionType: $ {{ parameters.collectionType }} + collectionName: ${{ parameters.collectionName }} + dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths }} + + ${{ if ne(parameters.displayName, '') }}: + displayName: '${{ parameters.displayName }}' + ${{ if eq(parameters.displayName, '') }}: + displayName: '${{ parameters.jobName }}' + + variables: + - name: PythonScript + value: 'py -3' + - name: PipScript + value: 'py -3 -m pip' + - name: Core_Root_Dir + value: '$(Build.SourcesDirectory)\artifacts\tests\coreclr\${{ parameters.osGroup }}.${{ parameters.archType }}.${{ parameters.buildConfig }}\Tests\Core_Root' + - name: SpmiCollectionLocation + value: '$(Build.SourcesDirectory)\artifacts\spmi\' + + workspace: + clean: all + pool: + ${{ parameters.pool }} + container: ${{ parameters.container }} + strategy: + matrix: + ${{ each framework in parameters.frameworks }}: + ${{ framework }}: + _Framework: ${{ framework }} + steps: + - ${{ parameters.steps }} + + - script: | + mkdir -p $(SpmiLogsLocation) + displayName: Create directory for SPMI collection + + - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi.py download --download_all --download_raw -spmi_location $(SpmiCollectionLocation) + displayName: Download superpmi collection \ No newline at end of file diff --git a/eng/pipelines/coreclr/templates/superpmi-replay-job.yml b/eng/pipelines/coreclr/templates/superpmi-replay-job.yml index eb5f275731767d..3dadc947fbb8c7 100644 --- a/eng/pipelines/coreclr/templates/superpmi-replay-job.yml +++ b/eng/pipelines/coreclr/templates/superpmi-replay-job.yml @@ -3,47 +3,34 @@ parameters: archType: '' # required -- targeting CPU architecture osGroup: '' # required -- operating system for the job osSubgroup: '' # optional -- operating system subgroup - dependsOn: '' # optional -- dependencies of the job pool: '' stagedBuild: false timeoutInMinutes: '' variables: {} dependOnEvaluatePaths: false - skipBinariesUpload: false - runJobTemplate: '/eng/pipelines/coreclr/templates/build-jit-job.yml' + runJobTemplate: '/eng/pipelines/coreclr/templates/run-superpmi-replay-job.yml' jobs: - template: ${{ parameters.runJobTemplate }} parameters: - dependsOn: ${{ parameters.dependsOn }} + jobName: ${{ format('superpmibuild_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + displayName: ${{ format('SuperPMI replay {0}{1} {2} {3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + pool: ${{ parameters.pool }} buildConfig: ${{ parameters.buildConfig }} archType: ${{ parameters.archType }} osGroup: ${{ parameters.osGroup }} osSubgroup: ${{ parameters.osSubgroup }} - helixType: 'build/product/' - enableMicrobuild: true - stagedBuild: ${{ parameters.stagedBuild }} - pool: ${{ parameters.pool }} dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths }} - _skipRestoreInternalTools: true - - variables: - - name: PythonScript - value: 'py -3' - - name: PipScript - value: 'py -3 -m pip' - - name: Core_Root_Dir - value: '$(Build.SourcesDirectory)\artifacts\tests\coreclr\${{ parameters.osGroup }}.${{ parameters.archType }}.${{ parameters.buildConfig }}\Tests\Core_Root' - - name: SpmiCollectionLocation - value: '$(Build.SourcesDirectory)\artifacts\spmi\' + uploadAsArtifacts: true + dependsOn: + - ${{ format('coreclr_jit_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} steps: - # Extra steps that will be passed to the superpmi template and run before sending the job to helix (all of which is done in the template) - - - script: | - mkdir -p $(SpmiLogsLocation) - displayName: Create directory for SPMI collection - - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi.py download -spmi_location $(SpmiCollectionLocation) - displayName: Download superpmi collection - \ No newline at end of file + # Download jit builds + - template: /eng/pipelines/common/download-artifact-step.yml + parameters: + unpackFolder: $(buildProductRootFolderPath) + artifactFileName: '$(buildProductArtifactName)$(archiveExtension)' + artifactName: '$(buildProductArtifactName)' + displayName: 'JIT product build' \ No newline at end of file From fabab8a5a7663d42fa3bd148c9312e60d8dbbf3a Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Thu, 29 Jul 2021 13:05:26 -0700 Subject: [PATCH 14/66] Fix Upload artifacts --- eng/pipelines/coreclr/templates/build-jit-job.yml | 1 + eng/pipelines/coreclr/templates/superpmi-replay-job.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/eng/pipelines/coreclr/templates/build-jit-job.yml b/eng/pipelines/coreclr/templates/build-jit-job.yml index f1bad18d34a50a..7f7114bb754c0c 100644 --- a/eng/pipelines/coreclr/templates/build-jit-job.yml +++ b/eng/pipelines/coreclr/templates/build-jit-job.yml @@ -12,6 +12,7 @@ parameters: variables: {} dependOnEvaluatePaths: false skipBinariesUpload: false + uploadAsArtifacts: false ### Product build jobs: diff --git a/eng/pipelines/coreclr/templates/superpmi-replay-job.yml b/eng/pipelines/coreclr/templates/superpmi-replay-job.yml index 3dadc947fbb8c7..f830eab582e521 100644 --- a/eng/pipelines/coreclr/templates/superpmi-replay-job.yml +++ b/eng/pipelines/coreclr/templates/superpmi-replay-job.yml @@ -14,7 +14,7 @@ jobs: - template: ${{ parameters.runJobTemplate }} parameters: jobName: ${{ format('superpmibuild_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - displayName: ${{ format('SuperPMI replay {0}{1} {2} {3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + displayName: ${{ format('SuperPMI replay {0}', parameters.archType) }} pool: ${{ parameters.pool }} buildConfig: ${{ parameters.buildConfig }} archType: ${{ parameters.archType }} From 9f8a0baa0522b6580636512891fdd27c92e57a18 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Thu, 29 Jul 2021 13:09:40 -0700 Subject: [PATCH 15/66] Fix uploadAsArtifacts --- eng/pipelines/coreclr/superpmi-replay.yml | 1 + eng/pipelines/coreclr/templates/superpmi-replay-job.yml | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/coreclr/superpmi-replay.yml b/eng/pipelines/coreclr/superpmi-replay.yml index 73202c97df43cb..79c225648ce55b 100644 --- a/eng/pipelines/coreclr/superpmi-replay.yml +++ b/eng/pipelines/coreclr/superpmi-replay.yml @@ -18,6 +18,7 @@ jobs: parameters: jobTemplate: /eng/pipelines/coreclr/templates/build-jit-job.yml buildConfig: checked + uploadAsArtifacts: true platforms: - windows_x64 - windows_x86 diff --git a/eng/pipelines/coreclr/templates/superpmi-replay-job.yml b/eng/pipelines/coreclr/templates/superpmi-replay-job.yml index f830eab582e521..1c457ac47dc3e1 100644 --- a/eng/pipelines/coreclr/templates/superpmi-replay-job.yml +++ b/eng/pipelines/coreclr/templates/superpmi-replay-job.yml @@ -21,7 +21,6 @@ jobs: osGroup: ${{ parameters.osGroup }} osSubgroup: ${{ parameters.osSubgroup }} dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths }} - uploadAsArtifacts: true dependsOn: - ${{ format('coreclr_jit_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} From 08650f610445a9b8e306ce0a9a34acd79aa23a32 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Thu, 29 Jul 2021 13:20:28 -0700 Subject: [PATCH 16/66] Fix to uploadAsArtifacts --- eng/pipelines/coreclr/templates/build-jit-job.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eng/pipelines/coreclr/templates/build-jit-job.yml b/eng/pipelines/coreclr/templates/build-jit-job.yml index 7f7114bb754c0c..08a26619fdb664 100644 --- a/eng/pipelines/coreclr/templates/build-jit-job.yml +++ b/eng/pipelines/coreclr/templates/build-jit-job.yml @@ -27,6 +27,7 @@ jobs: stagedBuild: ${{ parameters.stagedBuild }} pool: ${{ parameters.pool }} dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths }} + uploadAsArtifacts: ${{ parameters.uploadAsArtifacts }} # Compute job name from template parameters name: ${{ format('coreclr_jit_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} From d14111fe8c07b80fea78811dd67cfde231b39c1b Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Thu, 29 Jul 2021 13:31:25 -0700 Subject: [PATCH 17/66] comment upload to azure storage temporarily --- .../coreclr/templates/build-jit-job.yml | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/eng/pipelines/coreclr/templates/build-jit-job.yml b/eng/pipelines/coreclr/templates/build-jit-job.yml index 08a26619fdb664..53455b2d9206a9 100644 --- a/eng/pipelines/coreclr/templates/build-jit-job.yml +++ b/eng/pipelines/coreclr/templates/build-jit-job.yml @@ -11,7 +11,6 @@ parameters: timeoutInMinutes: '' variables: {} dependOnEvaluatePaths: false - skipBinariesUpload: false uploadAsArtifacts: false ### Product build @@ -109,17 +108,17 @@ jobs: - script: set __TestIntermediateDir=int&&$(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) -ci -nopgooptimize -skiprestoreoptdata -component alljits displayName: Build CoreCLR JIT - - ${{ if eq(parameters.uploadAsArtifacts, 'false') }}: - # Ensure the Python azure-storage-blob package is installed before doing the upload. - - script: $(PipScript) install --user --upgrade pip && $(PipScript) install --user azure.storage.blob==12.5.0 --force-reinstall - displayName: Upgrade Pip to latest and install azure-storage-blob Python package + # - ${{ if eq(parameters.uploadAsArtifacts, 'false') }}: + # # Ensure the Python azure-storage-blob package is installed before doing the upload. + # - script: $(PipScript) install --user --upgrade pip && $(PipScript) install --user azure.storage.blob==12.5.0 --force-reinstall + # displayName: Upgrade Pip to latest and install azure-storage-blob Python package - - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/jitrollingbuild.py upload -build_type $(buildConfig) -arch $(archType) -host_os $(osGroup) -git_hash $(Build.SourceVersion) - displayName: Upload JIT to Azure Storage - env: - CLRJIT_AZ_KEY: $(clrjit_key1) # secret key stored as variable in pipeline + # - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/jitrollingbuild.py upload -build_type $(buildConfig) -arch $(archType) -host_os $(osGroup) -git_hash $(Build.SourceVersion) + # displayName: Upload JIT to Azure Storage + # env: + # CLRJIT_AZ_KEY: $(clrjit_key1) # secret key stored as variable in pipeline - - ${{ if eq(parameters.uploadAsArtifacts, 'true') }}: + # - ${{ if eq(parameters.uploadAsArtifacts, 'true') }}: # Publish product output directory for consumption by tests. - template: /eng/pipelines/common/upload-artifact-step.yml parameters: From a3916fe1a8a628c6fa932f2f68368e9e50d40752 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Thu, 29 Jul 2021 16:41:56 -0700 Subject: [PATCH 18/66] Set variables inside superpmi-replay-job --- eng/pipelines/coreclr/templates/superpmi-replay-job.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eng/pipelines/coreclr/templates/superpmi-replay-job.yml b/eng/pipelines/coreclr/templates/superpmi-replay-job.yml index 1c457ac47dc3e1..25fc4dc841d986 100644 --- a/eng/pipelines/coreclr/templates/superpmi-replay-job.yml +++ b/eng/pipelines/coreclr/templates/superpmi-replay-job.yml @@ -24,6 +24,8 @@ jobs: dependsOn: - ${{ format('coreclr_jit_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + variables: ${{ parameters.variables }} + steps: # Download jit builds From fcd98555a976590bbb6c94e44d8608bf59971530 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 30 Jul 2021 00:53:46 -0700 Subject: [PATCH 19/66] Entire pipeline --- .../coreclr/templates/run-superpmi-job.yml | 2 +- .../templates/run-superpmi-replay-job.yml | 52 +++++- .../templates/superpmi-send-to-helix.yml | 3 +- src/coreclr/scripts/superpmi-replay.proj | 45 ++++++ src/coreclr/scripts/superpmi-replay.py | 135 ++++++++++++++++ src/coreclr/scripts/superpmi.py | 8 +- src/coreclr/scripts/superpmi_replay_setup.py | 148 ++++++++++++++++++ 7 files changed, 386 insertions(+), 7 deletions(-) create mode 100644 src/coreclr/scripts/superpmi-replay.proj create mode 100644 src/coreclr/scripts/superpmi-replay.py create mode 100644 src/coreclr/scripts/superpmi_replay_setup.py diff --git a/eng/pipelines/coreclr/templates/run-superpmi-job.yml b/eng/pipelines/coreclr/templates/run-superpmi-job.yml index 2f22a42f98e3dd..649646bd928d1f 100644 --- a/eng/pipelines/coreclr/templates/run-superpmi-job.yml +++ b/eng/pipelines/coreclr/templates/run-superpmi-job.yml @@ -144,7 +144,7 @@ jobs: WorkItemTimeout: 4:00 # 4 hours WorkItemDirectory: '$(WorkItemDirectory)' CorrelationPayloadDirectory: '$(CorrelationPayloadDirectory)' - ProjectFile: ${{ parameters.projectFile }} + ProjectFile: 'superpmi.proj' BuildConfig: ${{ parameters.buildConfig }} osGroup: ${{ parameters.osGroup }} InputArtifacts: '$(InputArtifacts)' diff --git a/eng/pipelines/coreclr/templates/run-superpmi-replay-job.yml b/eng/pipelines/coreclr/templates/run-superpmi-replay-job.yml index 376f9d8e698d2e..cfe2007edc543f 100644 --- a/eng/pipelines/coreclr/templates/run-superpmi-replay-job.yml +++ b/eng/pipelines/coreclr/templates/run-superpmi-replay-job.yml @@ -39,6 +39,7 @@ jobs: collectionType: $ {{ parameters.collectionType }} collectionName: ${{ parameters.collectionName }} dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths }} + helixType: 'build/product/' ${{ if ne(parameters.displayName, '') }}: displayName: '${{ parameters.displayName }}' @@ -54,6 +55,10 @@ jobs: value: '$(Build.SourcesDirectory)\artifacts\tests\coreclr\${{ parameters.osGroup }}.${{ parameters.archType }}.${{ parameters.buildConfig }}\Tests\Core_Root' - name: SpmiCollectionLocation value: '$(Build.SourcesDirectory)\artifacts\spmi\' + - name: SpmiLogsLocation + value: '$(Build.SourcesDirectory)\artifacts\spmi_logs\' + - name: HelixResultLocation + value: '$(Build.SourcesDirectory)\artifacts\helixresults\' workspace: clean: all @@ -69,8 +74,51 @@ jobs: - ${{ parameters.steps }} - script: | - mkdir -p $(SpmiLogsLocation) + mkdir -p $(SpmiCollectionLocation) displayName: Create directory for SPMI collection - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi.py download --download_all --download_raw -spmi_location $(SpmiCollectionLocation) - displayName: Download superpmi collection \ No newline at end of file + displayName: Download superpmi collection + + - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi_replay_setup.py -source_directory $(Build.SourcesDirectory) -product_directory $(Core_Root_Dir) -arch $(archType) -mch_directory $(SpmiCollectionLocation) + displayName: ${{ format('SuperPMI replay setup ({0})', parameters.osGroup) }} + + # Run superpmi replay in helix + - template: /eng/pipelines/coreclr/templates/superpmi-send-to-helix.yml + parameters: + HelixSource: '$(HelixSourcePrefix)/$(Build.Repository.Name)/$(Build.SourceBranch)' # sources must start with pr/, official/, prodcon/, or agent/ + HelixAccessToken: $(HelixApiAccessToken) + HelixTargetQueues: $(Queue) + HelixPreCommands: $(HelixPreCommand) + Creator: $(Creator) + WorkItemTimeout: 4:00 # 4 hours + WorkItemDirectory: '$(WorkItemDirectory)' + CorrelationPayloadDirectory: '$(CorrelationPayloadDirectory)' + ProjectFile: 'superpmi-replay.proj' + BuildConfig: ${{ parameters.buildConfig }} + osGroup: ${{ parameters.osGroup }} + archType: ${{ parameters.archType }} + continueOnError: true # Run the future step i.e. upload superpmi logs + + # Always upload the available logs for diagnostics + - task: CopyFiles@2 + displayName: Copying superpmi.log of all partitions + inputs: + sourceFolder: '$(HelixResultLocation)' + contents: '**/superpmi_*.log' + targetFolder: '$(SpmiLogsLocation)' + condition: always() + + - task: PublishPipelineArtifact@1 + displayName: Publish Superpmi logs + inputs: + targetPath: $(SpmiLogsLocation) + artifactName: 'SuperPMI_Logs_$(archType)_$(buildConfig)' + condition: always() + + - task: PublishPipelineArtifact@1 + displayName: Publish SuperPMI build logs + inputs: + targetPath: $(Build.SourcesDirectory)/artifacts/log + artifactName: 'SuperPMI_BuildLogs__$(archType)_$(buildConfig)' + condition: always() \ No newline at end of file diff --git a/eng/pipelines/coreclr/templates/superpmi-send-to-helix.yml b/eng/pipelines/coreclr/templates/superpmi-send-to-helix.yml index fe8eb69b3544ce..ed321d0244ec36 100644 --- a/eng/pipelines/coreclr/templates/superpmi-send-to-helix.yml +++ b/eng/pipelines/coreclr/templates/superpmi-send-to-helix.yml @@ -22,12 +22,13 @@ parameters: InputArtifacts: '' CollectionType: '' CollectionName: '' + ProjectFile: '' steps: - template: /eng/pipelines/common/templates/runtimes/send-to-helix-inner-step.yml parameters: osGroup: ${{ parameters.osGroup }} - sendParams: $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi.proj /restore /t:Test /bl:$(Build.SourcesDirectory)/artifacts/log/$(BuildConfig)/SendToHelix.binlog + sendParams: $(Build.SourcesDirectory)/src/coreclr/scripts/${{ parameters.ProjectFile }} /restore /t:Test /bl:$(Build.SourcesDirectory)/artifacts/log/$(BuildConfig)/SendToHelix.binlog displayName: ${{ parameters.DisplayNamePrefix }} condition: ${{ parameters.condition }} shouldContinueOnError: ${{ parameters.continueOnError }} diff --git a/src/coreclr/scripts/superpmi-replay.proj b/src/coreclr/scripts/superpmi-replay.proj new file mode 100644 index 00000000000000..be3261482c6ab4 --- /dev/null +++ b/src/coreclr/scripts/superpmi-replay.proj @@ -0,0 +1,45 @@ + + + + + + + %HELIX_PYTHONPATH% + $HELIX_CORRELATION_PAYLOAD + $HELIX_WORKITEM_UPLOAD_ROOT + $(Python) $(ProductDirectory)\superpmi-replay.py -arch $(Architecture) -jit_directory $(ProductDirectory) + + + + + + + + + + $(WorkItemCommand) -mch_directory $(WorkItemDirectory)\%(Identity) -partition_index %(Identity) -log_directory $(SuperpmiLogsLocation) + $(WorkItemTimeout) + superpmi_%(Identity).log + + + diff --git a/src/coreclr/scripts/superpmi-replay.py b/src/coreclr/scripts/superpmi-replay.py new file mode 100644 index 00000000000000..a4813cbb00f7ab --- /dev/null +++ b/src/coreclr/scripts/superpmi-replay.py @@ -0,0 +1,135 @@ + +jitstressregs_values = ['1', '2', '3', '4', '8', '0x10', '0x80', '0x1000'] + +import argparse +from os import path, system +import os +import shutil +import stat +import subprocess +import tempfile +import re + +from os.path import isfile, join +from os import listdir +from coreclr_arguments import * +from superpmi_setup import copy_directory, copy_files, set_pipeline_variable +from superpmi_setup import run_command + +parser = argparse.ArgumentParser(description="description") + +parser.add_argument("-arch", help="Architecture") +parser.add_argument("-partition_index", help="Partition index for logs") +parser.add_argument("-jit_directory", help="path to the directory containing clrjit binaries") +parser.add_argument("-mch_directory", help="path to the directory containing mch files") +parser.add_argument("-log_directory", help="path to the directory containing superpmi log files") + + +def setup_args(args): + """ Setup the args for SuperPMI to use. + + Args: + args (ArgParse): args parsed by arg parser + + Returns: + args (CoreclrArguments) + + """ + coreclr_args = CoreclrArguments(args, require_built_core_root=False, require_built_product_dir=False, + require_built_test_dir=False, default_build_type="Checked") + + coreclr_args.verify(args, + "arch", + lambda unused: True, + "Unable to set arch") + + coreclr_args.verify(args, + "partition_index", + lambda partition_index: partition_index.isnumeric(), + "Unable to set partition_index") + + coreclr_args.verify(args, + "jit_directory", + lambda jit_directory: os.path.isdir(jit_directory), + "jit_directory doesn't exist") + + coreclr_args.verify(args, + "mch_directory", + lambda mch_directory: os.path.isdir(mch_directory), + "mch_directory doesn't exist") + + coreclr_args.verify(args, + "log_directory", + lambda log_directory: os.path.isdir(log_directory), + "log_directory doesn't exist") + + return coreclr_args + +def main(main_args): + """Main entrypoint + + Args: + main_args ([type]): Arguments to the script + """ + + python_path = sys.executable + cwd = os.path.dirname(os.path.realpath(__file__)) + coreclr_args = setup_args(main_args) + mch_directory = coreclr_args.mch_directory + log_directory = coreclr_args.log_directory + mch_filename = '' + + for f in listdir(mch_directory): + if f.endswith(".mch.zip"): + mch_filename = f + + match_mch_elems = re.search('(.*)\.(windows|Linux)\.(x64|x86|arm|arm64)\.', mch_filename) + collection_name = os_name = match_mch_elems.group(1).replace('.','_') + os_name = "win" if match_mch_elems.group(2) == "windows" else "unix" + target_arch_name = match_mch_elems.group(3) + run_name = "{}_{}_{}".format(os_name, target_arch_name, coreclr_args.arch) + + # populate based on zip file name and jit_directory + jit_path = path.join(coreclr_args.jit_directory, 'clrjit_{}.dll'.format(run_name)) + run_id = 0 + for jitstressregs in jitstressregs_values: + # TODO: This should be DownloadFilesFromResults + log_file = path.join(log_directory, 'superpmi_{}_{}_flag_{}.log'.format(collection_name, run_name, jitstressregs)) + + # print(' '.join([ + # python_path, path.join(cwd, "superpmi.py"), "replay", "-core_root", cwd, + # "-jitoption", "JitStressRegs=" + jitstressregs, "-jitoption", "TieredCompilation=0", + # "-jit_path", jit_path, "-mch_files", mch_directory, "-spmi_location", mch_directory, + # "-log_level", "debug", "-log_file", log_path])) + + # In first iteration, mch files are unzipped and placed in "mch" folder. Going forward, use the + # uncompressed mch files + if run_id == 1: + mch_directory = path.join(mch_directory, "mch") + + run_command([ + python_path, path.join(cwd, "superpmi.py"), "replay", "-core_root", cwd, + "-jitoption", "JitStressRegs=" + jitstressregs, "-jitoption", "TieredCompilation=0", + "-jit_path", jit_path, "-mch_files", mch_directory, "-spmi_location", mch_directory, + "-log_level", "debug", "-log_file", log_file], + _exit_on_fail=True) + + run_id += 1 + + # Consolidate all superpmi.logs in superpmi_partition_index.log + final_log_name = path.join(log_directory, "superpmi_{}.log".format(coreclr_args.partition_index)) + with open(final_log_name, "a") as final_superpmi_log: + for superpmi_log in listdir(log_directory): + if not f.startswith("superpmi_") and not f.endswith(".log"): + continue + + final_superpmi_log.write("======================================================={}".format(os.linesep)) + final_superpmi_log.write("Contents from {}{}".format(superpmi_log, os.linesep)) + final_superpmi_log.write("======================================================={}".format(os.linesep)) + with open(path.join(log_directory, superpmi_log), "r") as current_superpmi_log: + contents = current_superpmi_log.read() + final_superpmi_log.write(contents) + +if __name__ == "__main__": + args = parser.parse_args() + sys.exit(main(args)) diff --git a/src/coreclr/scripts/superpmi.py b/src/coreclr/scripts/superpmi.py index cc6437a963b5df..5afc583d120be7 100755 --- a/src/coreclr/scripts/superpmi.py +++ b/src/coreclr/scripts/superpmi.py @@ -2542,7 +2542,7 @@ def process_local_mch_files(coreclr_args, mch_files, mch_cache_dir): for item in mch_files: # On Windows only, see if any of the mch_files are UNC paths (i.e., "\\server\share\..."). # If so, download and cache all the files found there to our usual local cache location, to avoid future network access. - if coreclr_args.host_os == "windows" and item.startswith("\\\\"): + if coreclr_args.host_os == "windows":# and item.startswith("\\\\"): # Special case: if the user specifies a .mch file, we'll also look for and cache a .mch.mct file next to it, if one exists. # This happens naturally if a directory is passed and we search for all .mch and .mct files in that directory. mch_file = os.path.abspath(item) @@ -2655,7 +2655,9 @@ def download_mch_from_azure(coreclr_args, target_dir): list containing the local path of files downloaded """ - blob_filter_string = "{}/".format(coreclr_args.jit_ee_version).lower() if coreclr_args.download_all else "{}/{}/{}/".format(coreclr_args.jit_ee_version, coreclr_args.target_os, coreclr_args.mch_arch).lower() + download_all = hasattr(coreclr_args, "download_all") and coreclr_args.download_all + download_raw = hasattr(coreclr_args, "download_raw") and coreclr_args.download_raw + blob_filter_string = "{}/".format(coreclr_args.jit_ee_version).lower() if download_all else "{}/{}/{}/".format(coreclr_args.jit_ee_version, coreclr_args.target_os, coreclr_args.mch_arch).lower() # Determine if a URL in Azure Storage should be allowed. The path looks like: # jit-ee-guid/Linux/x64/Linux.x64.Checked.frameworks.mch.zip @@ -2674,7 +2676,7 @@ def filter_superpmi_collections(path): blob_url_prefix = "{}/{}/".format(az_blob_storage_superpmi_container_uri, az_collections_root_folder) urls = [blob_url_prefix + path for path in paths] - return download_files(urls, target_dir, uncompress=not coreclr_args.download_raw) + return download_files(urls, target_dir, uncompress=not download_raw) def download_files(paths, target_dir, uncompress=True, verbose=True, fail_if_not_found=True): diff --git a/src/coreclr/scripts/superpmi_replay_setup.py b/src/coreclr/scripts/superpmi_replay_setup.py new file mode 100644 index 00000000000000..a1052a8928abd3 --- /dev/null +++ b/src/coreclr/scripts/superpmi_replay_setup.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 +# +# Licensed to the .NET Foundation under one or more agreements. +# The .NET Foundation licenses this file to you under the MIT license. +# +## +# Title : superpmi_replay_setup.py +# +# Notes: +# +# Script to setup directory structure required to perform SuperPMI replay in CI. +# It does the following steps: +# 1. It creates `correlation_payload_directory` that contains clrjit*_x64.dll and clrjit*_x86.dll +# 2. The script takes `input_artifacts` parameter which contains *.mch.zip and *.mct.zip. It will then +# partition it by moving each pair of *.mch.zip/*.mct.zip into its own folder under 'payload' +# directory. +################################################################################ +################################################################################ + +import argparse +from os import path +import shutil +import stat +import subprocess +import tempfile + +from os.path import isfile, join +from coreclr_arguments import * +from superpmi_setup import copy_directory, copy_files, set_pipeline_variable + +parser = argparse.ArgumentParser(description="description") + +parser.add_argument("-arch", help="Architecture") +parser.add_argument("-source_directory", help="path to the directory containing binaries") +parser.add_argument("-product_directory", help="path to the directory containing binaries") +parser.add_argument("-mch_directory", help="path to directory containing compressed mch files") + + +def setup_args(args): + """ Setup the args for SuperPMI to use. + + Args: + args (ArgParse): args parsed by arg parser + + Returns: + args (CoreclrArguments) + + """ + coreclr_args = CoreclrArguments(args, require_built_core_root=False, require_built_product_dir=False, + require_built_test_dir=False, default_build_type="Checked") + + coreclr_args.verify(args, + "arch", + lambda unused: True, + "Unable to set arch") + + coreclr_args.verify(args, + "source_directory", + lambda source_directory: os.path.isdir(source_directory), + "source_directory doesn't exist") + + coreclr_args.verify(args, + "product_directory", + lambda product_directory: os.path.isdir(product_directory), + "product_directory doesn't exist") + + coreclr_args.verify(args, + "mch_directory", + lambda mch_directory: os.path.isdir(mch_directory), + "mch_directory doesn't exist") + return coreclr_args + + +def partition_mch(mch_directory, dst_directory): + from os import listdir + + all_zip_files = [f for f in listdir(mch_directory) if isfile(join(mch_directory, f))] + mch_zip_files = filter(lambda zip_file: zip_file.endswith(".mch.zip"), all_zip_files) + index = 1 + for mch_file in mch_zip_files: + file_names = [] + file_names += [path.join(mch_directory, mch_file)] + file_names += [path.join(mch_directory, mch_file.replace(".mch.zip", ".mch.mct.zip"))] + curr_dst_path = path.join(dst_directory, "partitions", str(index)) + copy_files(mch_directory, curr_dst_path, file_names) + index += 1 + + +def match_correlation_files(full_path): + file_name = os.path.basename(full_path) + + if file_name.startswith("clrjit_") and file_name.endswith(".dll") and file_name.find( + "osx") == -1 and file_name.find("armel") == -1: + return True + + if file_name == "superpmi.exe" or file_name == "mcs.exe": + return True + + return False + +def main(main_args): + """Main entrypoint + + Args: + main_args ([type]): Arguments to the script + """ + coreclr_args = setup_args(main_args) + + arch = coreclr_args.arch + source_directory = coreclr_args.source_directory + product_directory = coreclr_args.product_directory + mch_directory = coreclr_args.mch_directory + + # CorrelationPayload directories + correlation_payload_directory = path.join(coreclr_args.source_directory, "payload") + superpmi_src_directory = path.join(source_directory, 'src', 'coreclr', 'scripts') + + # Workitem directories + workitem_directory = path.join(source_directory, "workitem") + + helix_source_prefix = "official" + creator = "" + ci = True + helix_queue = "Windows.10.Amd64.X86" + + # Copy *.py to CorrelationPayload + print('Copying {} -> {}'.format(superpmi_src_directory, correlation_payload_directory)) + copy_directory(superpmi_src_directory, correlation_payload_directory, match_func=lambda path: any(path.endswith(extension) for extension in [".py"])) + + # Copy clrjit*_arch.dll binaries to CorrelationPayload + print('Copying clrjit*_{}.dll {} -> {}'.format(arch, product_directory, correlation_payload_directory)) + copy_directory(product_directory, correlation_payload_directory, match_func=match_correlation_files) + + # Partition mch/mct zip files + partition_mch(mch_directory, workitem_directory) + + # Set variables + print('Setting pipeline variables:') + set_pipeline_variable("CorrelationPayloadDirectory", correlation_payload_directory) + set_pipeline_variable("WorkItemDirectory", workitem_directory) + set_pipeline_variable("Architecture", arch) + set_pipeline_variable("Creator", creator) + set_pipeline_variable("Queue", helix_queue) + set_pipeline_variable("HelixSourcePrefix", helix_source_prefix) + +if __name__ == "__main__": + args = parser.parse_args() + sys.exit(main(args)) From dd120708a2d91376b6ec7690fc45f79c4c9aa0a4 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 30 Jul 2021 01:05:14 -0700 Subject: [PATCH 20/66] minor fix --- eng/pipelines/coreclr/templates/run-superpmi-replay-job.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/eng/pipelines/coreclr/templates/run-superpmi-replay-job.yml b/eng/pipelines/coreclr/templates/run-superpmi-replay-job.yml index cfe2007edc543f..fb6302835232bb 100644 --- a/eng/pipelines/coreclr/templates/run-superpmi-replay-job.yml +++ b/eng/pipelines/coreclr/templates/run-superpmi-replay-job.yml @@ -51,8 +51,6 @@ jobs: value: 'py -3' - name: PipScript value: 'py -3 -m pip' - - name: Core_Root_Dir - value: '$(Build.SourcesDirectory)\artifacts\tests\coreclr\${{ parameters.osGroup }}.${{ parameters.archType }}.${{ parameters.buildConfig }}\Tests\Core_Root' - name: SpmiCollectionLocation value: '$(Build.SourcesDirectory)\artifacts\spmi\' - name: SpmiLogsLocation @@ -80,8 +78,8 @@ jobs: - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi.py download --download_all --download_raw -spmi_location $(SpmiCollectionLocation) displayName: Download superpmi collection - - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi_replay_setup.py -source_directory $(Build.SourcesDirectory) -product_directory $(Core_Root_Dir) -arch $(archType) -mch_directory $(SpmiCollectionLocation) - displayName: ${{ format('SuperPMI replay setup ({0})', parameters.osGroup) }} + - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi_replay_setup.py -source_directory $(Build.SourcesDirectory) -product_directory $(buildProductRootFolderPath) -arch $(archType) -mch_directory $(SpmiCollectionLocation) + displayName: ${{ format('SuperPMI replay setup ({0})', parameters.archType) }} # Run superpmi replay in helix - template: /eng/pipelines/coreclr/templates/superpmi-send-to-helix.yml From b136c9d62cdee64b9dda21be35d2bcf108dfca41 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 30 Jul 2021 01:25:58 -0700 Subject: [PATCH 21/66] Add some logging --- .../coreclr/templates/run-superpmi-replay-job.yml | 8 +++++++- eng/pipelines/coreclr/templates/superpmi-replay-job.yml | 5 +++++ src/coreclr/scripts/superpmi_replay_setup.py | 2 ++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/eng/pipelines/coreclr/templates/run-superpmi-replay-job.yml b/eng/pipelines/coreclr/templates/run-superpmi-replay-job.yml index fb6302835232bb..5ba64cf65f760c 100644 --- a/eng/pipelines/coreclr/templates/run-superpmi-replay-job.yml +++ b/eng/pipelines/coreclr/templates/run-superpmi-replay-job.yml @@ -39,7 +39,6 @@ jobs: collectionType: $ {{ parameters.collectionType }} collectionName: ${{ parameters.collectionName }} dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths }} - helixType: 'build/product/' ${{ if ne(parameters.displayName, '') }}: displayName: '${{ parameters.displayName }}' @@ -47,6 +46,13 @@ jobs: displayName: '${{ parameters.jobName }}' variables: + - ${{ each variable in parameters.variables }}: + - ${{ if ne(variable.name, '') }}: + - name: ${{ variable.name }} + value: ${{ variable.value }} + - ${{ if ne(variable.group, '') }}: + - group: ${{ variable.group }} + - name: PythonScript value: 'py -3' - name: PipScript diff --git a/eng/pipelines/coreclr/templates/superpmi-replay-job.yml b/eng/pipelines/coreclr/templates/superpmi-replay-job.yml index 25fc4dc841d986..3bca67c6004d68 100644 --- a/eng/pipelines/coreclr/templates/superpmi-replay-job.yml +++ b/eng/pipelines/coreclr/templates/superpmi-replay-job.yml @@ -6,6 +6,7 @@ parameters: pool: '' stagedBuild: false timeoutInMinutes: '' + framework: net5.0 # Specify the appropriate framework when running release branches (ie netcoreapp3.0 for release/3.0) variables: {} dependOnEvaluatePaths: false runJobTemplate: '/eng/pipelines/coreclr/templates/run-superpmi-replay-job.yml' @@ -21,11 +22,15 @@ jobs: osGroup: ${{ parameters.osGroup }} osSubgroup: ${{ parameters.osSubgroup }} dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths }} + additionalSetupParameters: ${{ parameters.additionalSetupParameters }} dependsOn: - ${{ format('coreclr_jit_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} variables: ${{ parameters.variables }} + frameworks: + - ${{ parameters.framework }} + steps: # Download jit builds diff --git a/src/coreclr/scripts/superpmi_replay_setup.py b/src/coreclr/scripts/superpmi_replay_setup.py index a1052a8928abd3..ec2847678dd7e4 100644 --- a/src/coreclr/scripts/superpmi_replay_setup.py +++ b/src/coreclr/scripts/superpmi_replay_setup.py @@ -74,10 +74,12 @@ def setup_args(args): def partition_mch(mch_directory, dst_directory): from os import listdir + print("Inside partition_mch") all_zip_files = [f for f in listdir(mch_directory) if isfile(join(mch_directory, f))] mch_zip_files = filter(lambda zip_file: zip_file.endswith(".mch.zip"), all_zip_files) index = 1 for mch_file in mch_zip_files: + print("Processing {}".format(mch_file)) file_names = [] file_names += [path.join(mch_directory, mch_file)] file_names += [path.join(mch_directory, mch_file.replace(".mch.zip", ".mch.mct.zip"))] From 333927ef9132db7d2930ac5000cb3f0fc41af29a Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 30 Jul 2021 08:42:05 -0700 Subject: [PATCH 22/66] Fix the path --- src/coreclr/scripts/superpmi-replay.proj | 5 +++-- src/coreclr/scripts/superpmi_replay_setup.py | 16 +++++++++++++--- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/src/coreclr/scripts/superpmi-replay.proj b/src/coreclr/scripts/superpmi-replay.proj index be3261482c6ab4..4d19b0481bed71 100644 --- a/src/coreclr/scripts/superpmi-replay.proj +++ b/src/coreclr/scripts/superpmi-replay.proj @@ -26,18 +26,19 @@ %HELIX_PYTHONPATH% $HELIX_CORRELATION_PAYLOAD + $(WorkItemDirectory)\partitions $HELIX_WORKITEM_UPLOAD_ROOT $(Python) $(ProductDirectory)\superpmi-replay.py -arch $(Architecture) -jit_directory $(ProductDirectory) - + - $(WorkItemCommand) -mch_directory $(WorkItemDirectory)\%(Identity) -partition_index %(Identity) -log_directory $(SuperpmiLogsLocation) + $(WorkItemCommand) -mch_directory $(SpmiCollectionFolder)\%(Identity)\mch -partition_index %(Identity) -log_directory $(SuperpmiLogsLocation) $(WorkItemTimeout) superpmi_%(Identity).log diff --git a/src/coreclr/scripts/superpmi_replay_setup.py b/src/coreclr/scripts/superpmi_replay_setup.py index ec2847678dd7e4..caaeda03cd88ec 100644 --- a/src/coreclr/scripts/superpmi_replay_setup.py +++ b/src/coreclr/scripts/superpmi_replay_setup.py @@ -18,7 +18,7 @@ ################################################################################ import argparse -from os import path +from os import path, walk import shutil import stat import subprocess @@ -75,8 +75,18 @@ def partition_mch(mch_directory, dst_directory): from os import listdir print("Inside partition_mch") - all_zip_files = [f for f in listdir(mch_directory) if isfile(join(mch_directory, f))] - mch_zip_files = filter(lambda zip_file: zip_file.endswith(".mch.zip"), all_zip_files) + mch_zip_files = [] + for file_path, dirs, files in walk(mch_directory, topdown=True): + for name in files: + curr_file_path = path.join(file_path, name) + + if not isfile(curr_file_path): + continue + if not name.endswith(".mch.zip"): + continue + + mch_zip_files.append(curr_file_path) + index = 1 for mch_file in mch_zip_files: print("Processing {}".format(mch_file)) From c8f75f56cb7529090a16f3a7d5a47b1e8aa95d9b Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Mon, 2 Aug 2021 07:41:52 -0700 Subject: [PATCH 23/66] fix file names and partition ids --- src/coreclr/scripts/superpmi-replay.proj | 4 ++-- src/coreclr/scripts/superpmi_replay_setup.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/coreclr/scripts/superpmi-replay.proj b/src/coreclr/scripts/superpmi-replay.proj index 4d19b0481bed71..2543c79aa5c341 100644 --- a/src/coreclr/scripts/superpmi-replay.proj +++ b/src/coreclr/scripts/superpmi-replay.proj @@ -11,7 +11,7 @@ + + + + - $(WorkItemCommand) -mch_directory $(SpmiCollectionFolder)\%(HelixWorkItem.PartitionId)\mch -partition_index %(HelixWorkItem.PartitionId) -log_directory $(SuperpmiLogsLocation) - $(WorkItemDirectory) + $(WorkItemCommand) -mch_directory foo.txt -partition_index 1 -log_directory $(SuperpmiLogsLocation) + + $(WorkItemTimeout) superpmi_%(Identity).log diff --git a/src/coreclr/scripts/superpmi-replay.py b/src/coreclr/scripts/superpmi-replay.py index de0c826df02b52..cf60460397dbfb 100644 --- a/src/coreclr/scripts/superpmi-replay.py +++ b/src/coreclr/scripts/superpmi-replay.py @@ -55,7 +55,7 @@ def setup_args(args): coreclr_args.verify(args, "mch_directory", - lambda mch_directory: os.path.isdir(mch_directory), + lambda mch_directory: True, #os.path.isdir(mch_directory), "mch_directory doesn't exist") coreclr_args.verify(args, @@ -79,6 +79,9 @@ def main(main_args): log_directory = coreclr_args.log_directory mch_filename = '' + print("=============> Running superpmi.py download") + run_command([python_path, path.join(cwd, "superpmi.py"), "download", "-f", "benchmarks"]) + for f in listdir(mch_directory): if f.endswith(".mch.zip"): mch_filename = f @@ -89,8 +92,6 @@ def main(main_args): target_arch_name = match_mch_elems.group(3) run_name = "{}_{}_{}".format(os_name, target_arch_name, coreclr_args.arch) - print("=============> Running superpmi.py download") - run_command([python_path, path.join(cwd, "superpmi.py"), "download", "-f", "benchmarks"]) # populate based on zip file name and jit_directory jit_path = path.join(coreclr_args.jit_directory, 'clrjit_{}.dll'.format(run_name)) diff --git a/src/coreclr/scripts/superpmi_replay_setup.py b/src/coreclr/scripts/superpmi_replay_setup.py index af000e4fa3a5ab..fce9bb04767966 100644 --- a/src/coreclr/scripts/superpmi_replay_setup.py +++ b/src/coreclr/scripts/superpmi_replay_setup.py @@ -66,7 +66,7 @@ def setup_args(args): coreclr_args.verify(args, "mch_directory", - lambda mch_directory: os.path.isdir(mch_directory), + lambda mch_directory: True, #os.path.isdir(mch_directory), "mch_directory doesn't exist") return coreclr_args @@ -144,7 +144,7 @@ def main(main_args): copy_directory(product_directory, correlation_payload_directory, match_func=match_correlation_files) # Partition mch/mct zip files - partition_mch(mch_directory, workitem_directory) + # partition_mch(mch_directory, workitem_directory) # Print correlation_payload_directory and workitem_directory print("==> correlation_payload_directory:") From 3ec4a84dea2d8daa9589f27bdc83e36511c64bfc Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 3 Aug 2021 10:41:46 -0700 Subject: [PATCH 34/66] fix core_root --- src/coreclr/scripts/superpmi-replay.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/scripts/superpmi-replay.py b/src/coreclr/scripts/superpmi-replay.py index cf60460397dbfb..5e3517eafd1131 100644 --- a/src/coreclr/scripts/superpmi-replay.py +++ b/src/coreclr/scripts/superpmi-replay.py @@ -80,7 +80,7 @@ def main(main_args): mch_filename = '' print("=============> Running superpmi.py download") - run_command([python_path, path.join(cwd, "superpmi.py"), "download", "-f", "benchmarks"]) + run_command([python_path, path.join(cwd, "superpmi.py"), "download", "-f", "benchmarks", "-core_root", cwd]) for f in listdir(mch_directory): if f.endswith(".mch.zip"): From 966a53008fe0f45d80270fa35a78329ef72bee29 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 3 Aug 2021 12:00:50 -0700 Subject: [PATCH 35/66] Make download happen on helix --- .../templates/run-superpmi-replay-job.yml | 4 +- src/coreclr/scripts/superpmi-replay.proj | 8 +-- src/coreclr/scripts/superpmi-replay.py | 58 +++++++++++-------- src/coreclr/scripts/superpmi_replay_setup.py | 49 ++++++++++------ 4 files changed, 71 insertions(+), 48 deletions(-) diff --git a/eng/pipelines/coreclr/templates/run-superpmi-replay-job.yml b/eng/pipelines/coreclr/templates/run-superpmi-replay-job.yml index ed492336fc4d1b..720cc99d19905a 100644 --- a/eng/pipelines/coreclr/templates/run-superpmi-replay-job.yml +++ b/eng/pipelines/coreclr/templates/run-superpmi-replay-job.yml @@ -84,8 +84,8 @@ jobs: # - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi.py download --download_all --download_raw -spmi_location $(SpmiCollectionLocation) # displayName: Download superpmi collection - - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi_replay_setup.py -source_directory $(Build.SourcesDirectory) -product_directory $(buildProductRootFolderPath) -arch $(archType) -mch_directory $(SpmiCollectionLocation)\mch - displayName: ${{ format('SuperPMI replay setup ({0})', parameters.archType) }} + - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi_replay_setup.py -source_directory $(Build.SourcesDirectory) -product_directory $(buildProductRootFolderPath) -platform $(osGroup) -arch $(archType) + displayName: ${{ format('SuperPMI replay setup ({0} {1})', parameters.osGroup, parameters.archType) }} # Run superpmi replay in helix - template: /eng/pipelines/coreclr/templates/superpmi-send-to-helix.yml diff --git a/src/coreclr/scripts/superpmi-replay.proj b/src/coreclr/scripts/superpmi-replay.proj index 09ce82ffe16a37..059fe539e2f186 100644 --- a/src/coreclr/scripts/superpmi-replay.proj +++ b/src/coreclr/scripts/superpmi-replay.proj @@ -43,17 +43,17 @@ --> - - + + - $(WorkItemCommand) -mch_directory foo.txt -partition_index 1 -log_directory $(SuperpmiLogsLocation) + $(WorkItemCommand) -arch %(HelixWorkItem.Architecture) -platform %(HelixWorkItem.Platform) -log_directory $(SuperpmiLogsLocation) $(WorkItemTimeout) - superpmi_%(Identity).log + superpmi_%(HelixWorkItem.Platform)_%(HelixWorkItem.Architecture).log diff --git a/src/coreclr/scripts/superpmi-replay.py b/src/coreclr/scripts/superpmi-replay.py index 5e3517eafd1131..aad0e2d624f4e8 100644 --- a/src/coreclr/scripts/superpmi-replay.py +++ b/src/coreclr/scripts/superpmi-replay.py @@ -19,6 +19,7 @@ parser = argparse.ArgumentParser(description="description") parser.add_argument("-arch", help="Architecture") +parser.add_argument("-platform", help="OS platform") parser.add_argument("-partition_index", help="Partition index for logs") parser.add_argument("-jit_directory", help="path to the directory containing clrjit binaries") parser.add_argument("-mch_directory", help="path to the directory containing mch files") @@ -44,19 +45,24 @@ def setup_args(args): "Unable to set arch") coreclr_args.verify(args, - "partition_index", - lambda partition_index: True, #partition_index.isnumeric(), - "Unable to set partition_index") + "platform", + lambda unused: True, + "Unable to set platform") + + # coreclr_args.verify(args, + # "partition_index", + # lambda partition_index: True, #partition_index.isnumeric(), + # "Unable to set partition_index") coreclr_args.verify(args, "jit_directory", lambda jit_directory: os.path.isdir(jit_directory), "jit_directory doesn't exist") - coreclr_args.verify(args, - "mch_directory", - lambda mch_directory: True, #os.path.isdir(mch_directory), - "mch_directory doesn't exist") + # coreclr_args.verify(args, + # "mch_directory", + # lambda mch_directory: True, #os.path.isdir(mch_directory), + # "mch_directory doesn't exist") coreclr_args.verify(args, "log_directory", @@ -75,30 +81,34 @@ def main(main_args): python_path = sys.executable cwd = os.path.dirname(os.path.realpath(__file__)) coreclr_args = setup_args(main_args) - mch_directory = coreclr_args.mch_directory + # mch_directory = coreclr_args.mch_directory + spmi_location = path.join(cwd, "artifacts", "spmi") log_directory = coreclr_args.log_directory mch_filename = '' print("=============> Running superpmi.py download") - run_command([python_path, path.join(cwd, "superpmi.py"), "download", "-f", "benchmarks", "-core_root", cwd]) + run_command([python_path, path.join(cwd, "superpmi.py"), "download", "-f", "benchmarks", "-core_root", cwd, "-spmi_location", spmi_location]) - for f in listdir(mch_directory): + for f in listdir(spmi_location): if f.endswith(".mch.zip"): mch_filename = f - match_mch_elems = re.search('(.*)\.(windows|Linux)\.(x64|x86|arm|arm64)\.', mch_filename) - collection_name = os_name = match_mch_elems.group(1).replace('.','_') - os_name = "win" if match_mch_elems.group(2) == "windows" else "unix" - target_arch_name = match_mch_elems.group(3) - run_name = "{}_{}_{}".format(os_name, target_arch_name, coreclr_args.arch) + # match_mch_elems = re.search('(.*)\.(windows|Linux)\.(x64|x86|arm|arm64)\.', mch_filename) + # collection_name = os_name = match_mch_elems.group(1).replace('.','_') + # os_name = "win" if match_mch_elems.group(2) == "windows" else "unix" + # target_arch_name = match_mch_elems.group(3) + # run_name = "{}_{}_{}".format(os_name, target_arch_name, coreclr_args.arch) # populate based on zip file name and jit_directory - jit_path = path.join(coreclr_args.jit_directory, 'clrjit_{}.dll'.format(run_name)) - run_id = 0 + os_name = "win" if coreclr_args.platform.lower() == "windows" else "unix" + arch_name = coreclr_args.arch + host_arch_name = "x64" if arch_name.endswith("x64") else "x86" + jit_path = path.join(coreclr_args.jit_directory, 'clrjit_{}_{}_{}.dll'.format(os_name, arch_name, host_arch_name)) + # run_id = 0 for jitstressregs in jitstressregs_values: # TODO: This should be DownloadFilesFromResults - log_file = path.join(log_directory, 'superpmi_{}_{}_flag_{}.log'.format(collection_name, run_name, jitstressregs)) + log_file = path.join(log_directory, 'superpmi_flag_{}.log'.format(jitstressregs)) # print(' '.join([ # python_path, path.join(cwd, "superpmi.py"), "replay", "-core_root", cwd, @@ -108,23 +118,23 @@ def main(main_args): # In first iteration, mch files are unzipped and placed in "mch" folder. Going forward, use the # uncompressed mch files - if run_id == 1: - mch_directory = path.join(mch_directory, "mch") + # if run_id == 1: + # mch_directory = path.join(mch_directory, "mch") run_command([ python_path, path.join(cwd, "superpmi.py"), "replay", "-core_root", cwd, "-jitoption", "JitStressRegs=" + jitstressregs, "-jitoption", "TieredCompilation=0", - "-jit_path", jit_path, "-mch_files", mch_directory, "-spmi_location", mch_directory, + "-jit_path", jit_path, "-spmi_location", spmi_location, "-log_level", "debug", "-log_file", log_file], _exit_on_fail=True) - run_id += 1 + # run_id += 1 # Consolidate all superpmi.logs in superpmi_partition_index.log - final_log_name = path.join(log_directory, "superpmi_{}.log".format(coreclr_args.partition_index)) + final_log_name = path.join(log_directory, "superpmi_{}_{}.log".format(coreclr_args.platform, arch_name)) with open(final_log_name, "a") as final_superpmi_log: for superpmi_log in listdir(log_directory): - if not f.startswith("superpmi_") and not f.endswith(".log"): + if not f.startswith("superpmi_flag") and not f.endswith(".log"): continue final_superpmi_log.write("======================================================={}".format(os.linesep)) diff --git a/src/coreclr/scripts/superpmi_replay_setup.py b/src/coreclr/scripts/superpmi_replay_setup.py index fce9bb04767966..a524ccbba38bd5 100644 --- a/src/coreclr/scripts/superpmi_replay_setup.py +++ b/src/coreclr/scripts/superpmi_replay_setup.py @@ -31,9 +31,10 @@ parser = argparse.ArgumentParser(description="description") parser.add_argument("-arch", help="Architecture") +parser.add_argument("-platform", help="OS platform") parser.add_argument("-source_directory", help="path to the directory containing binaries") parser.add_argument("-product_directory", help="path to the directory containing binaries") -parser.add_argument("-mch_directory", help="path to directory containing compressed mch files") +# parser.add_argument("-mch_directory", help="path to directory containing compressed mch files") def setup_args(args): @@ -54,6 +55,11 @@ def setup_args(args): lambda unused: True, "Unable to set arch") + coreclr_args.verify(args, + "platform", + lambda unused: True, + "Unable to set platform") + coreclr_args.verify(args, "source_directory", lambda source_directory: os.path.isdir(source_directory), @@ -64,10 +70,10 @@ def setup_args(args): lambda product_directory: os.path.isdir(product_directory), "product_directory doesn't exist") - coreclr_args.verify(args, - "mch_directory", - lambda mch_directory: True, #os.path.isdir(mch_directory), - "mch_directory doesn't exist") + # coreclr_args.verify(args, + # "mch_directory", + # lambda mch_directory: True, #os.path.isdir(mch_directory), + # "mch_directory doesn't exist") return coreclr_args @@ -119,9 +125,10 @@ def main(main_args): coreclr_args = setup_args(main_args) arch = coreclr_args.arch + platform = coreclr_args.platform source_directory = coreclr_args.source_directory product_directory = coreclr_args.product_directory - mch_directory = coreclr_args.mch_directory + # mch_directory = coreclr_args.mch_directory # CorrelationPayload directories correlation_payload_directory = path.join(coreclr_args.source_directory, "payload") @@ -140,29 +147,35 @@ def main(main_args): copy_directory(superpmi_src_directory, correlation_payload_directory, match_func=lambda path: any(path.endswith(extension) for extension in [".py"])) # Copy clrjit*_arch.dll binaries to CorrelationPayload - print('Copying clrjit*_{}.dll {} -> {}'.format(arch, product_directory, correlation_payload_directory)) + print('Copying binaries {} -> {}'.format(arch, product_directory, correlation_payload_directory)) copy_directory(product_directory, correlation_payload_directory, match_func=match_correlation_files) + #TODO: Just send appropriate clrjit*.dll files to workitem_directory + # # Copy clrjit*_arch.dll binaries to workitem_directory + # print('Copying clrjit_{}_{}.dll {} -> {}'.format(arch, product_directory, correlation_payload_directory)) + # copy_directory(product_directory, correlation_payload_directory, match_func=match_correlation_files) + # Partition mch/mct zip files # partition_mch(mch_directory, workitem_directory) - # Print correlation_payload_directory and workitem_directory - print("==> correlation_payload_directory:") - for file_path, dirs, files in walk(correlation_payload_directory, topdown=True): - for name in files: - curr_file_path = path.join(file_path, name) - print(curr_file_path) + # # Print correlation_payload_directory and workitem_directory + # print("==> correlation_payload_directory:") + # for file_path, dirs, files in walk(correlation_payload_directory, topdown=True): + # for name in files: + # curr_file_path = path.join(file_path, name) + # print(curr_file_path) - print("==> workitem_directory:") - for file_path, dirs, files in walk(workitem_directory, topdown=True): - for name in files: - curr_file_path = path.join(file_path, name) - print(curr_file_path) + # print("==> workitem_directory:") + # for file_path, dirs, files in walk(workitem_directory, topdown=True): + # for name in files: + # curr_file_path = path.join(file_path, name) + # print(curr_file_path) # Set variables print('Setting pipeline variables:') set_pipeline_variable("CorrelationPayloadDirectory", correlation_payload_directory) set_pipeline_variable("WorkItemDirectory", workitem_directory) + set_pipeline_variable("Platform", platform) set_pipeline_variable("Architecture", arch) set_pipeline_variable("Creator", creator) set_pipeline_variable("Queue", helix_queue) From 494e7abd22a9c3b8ed57f8547699c3df63d93206 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 3 Aug 2021 12:18:30 -0700 Subject: [PATCH 36/66] Add windows x86 --- eng/pipelines/coreclr/superpmi-replay.yml | 1 + src/coreclr/scripts/superpmi-replay.proj | 6 ++--- src/coreclr/scripts/superpmi-replay.py | 30 ++++++++++++++--------- 3 files changed, 23 insertions(+), 14 deletions(-) diff --git a/eng/pipelines/coreclr/superpmi-replay.yml b/eng/pipelines/coreclr/superpmi-replay.yml index fd72c23adc18d1..5028926d930727 100644 --- a/eng/pipelines/coreclr/superpmi-replay.yml +++ b/eng/pipelines/coreclr/superpmi-replay.yml @@ -29,5 +29,6 @@ jobs: buildConfig: checked platforms: - windows_x64 + - windows_x86 helixQueueGroup: ci helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml \ No newline at end of file diff --git a/src/coreclr/scripts/superpmi-replay.proj b/src/coreclr/scripts/superpmi-replay.proj index 059fe539e2f186..fe9adb60779f73 100644 --- a/src/coreclr/scripts/superpmi-replay.proj +++ b/src/coreclr/scripts/superpmi-replay.proj @@ -28,7 +28,7 @@ %HELIX_CORRELATION_PAYLOAD% $(WorkItemDirectory)\partitions %HELIX_WORKITEM_UPLOAD_ROOT% - $(Python) $(ProductDirectory)\superpmi-replay.py -arch $(Architecture) -jit_directory $(ProductDirectory) + $(Python) $(ProductDirectory)\superpmi-replay.py -jit_directory $(ProductDirectory) @@ -42,10 +42,10 @@ --> - + diff --git a/src/coreclr/scripts/superpmi-replay.py b/src/coreclr/scripts/superpmi-replay.py index aad0e2d624f4e8..2780d4ee51bcd6 100644 --- a/src/coreclr/scripts/superpmi-replay.py +++ b/src/coreclr/scripts/superpmi-replay.py @@ -1,6 +1,4 @@ -jitstressregs_values = ['1', '2', '3', '4', '8', '0x10', '0x80', '0x1000'] - import argparse from os import path, system import os @@ -25,6 +23,17 @@ parser.add_argument("-mch_directory", help="path to the directory containing mch files") parser.add_argument("-log_directory", help="path to the directory containing superpmi log files") +jit_flags = [ + "JitStressRegs=0", + "JitStressRegs=1", + "JitStressRegs=2", + "JitStressRegs=3", + "JitStressRegs=4", + "JitStressRegs=8", + "JitStressRegs=0x10", + "JitStressRegs=0x80", + "JitStressRegs=0x1000", + ] def setup_args(args): """ Setup the args for SuperPMI to use. @@ -86,7 +95,7 @@ def main(main_args): log_directory = coreclr_args.log_directory mch_filename = '' - print("=============> Running superpmi.py download") + print("Running superpmi.py download") run_command([python_path, path.join(cwd, "superpmi.py"), "download", "-f", "benchmarks", "-core_root", cwd, "-spmi_location", spmi_location]) for f in listdir(spmi_location): @@ -106,10 +115,10 @@ def main(main_args): host_arch_name = "x64" if arch_name.endswith("x64") else "x86" jit_path = path.join(coreclr_args.jit_directory, 'clrjit_{}_{}_{}.dll'.format(os_name, arch_name, host_arch_name)) # run_id = 0 - for jitstressregs in jitstressregs_values: + for jit_flag in jit_flags: # TODO: This should be DownloadFilesFromResults - log_file = path.join(log_directory, 'superpmi_flag_{}.log'.format(jitstressregs)) - + log_file = path.join(log_directory, 'superpmi_{}.log'.format(jit_flag.replace("=", "_"))) + print("Running superpmi.py replay for {}".format(jit_flag)) # print(' '.join([ # python_path, path.join(cwd, "superpmi.py"), "replay", "-core_root", cwd, # "-jitoption", "JitStressRegs=" + jitstressregs, "-jitoption", "TieredCompilation=0", @@ -123,18 +132,17 @@ def main(main_args): run_command([ python_path, path.join(cwd, "superpmi.py"), "replay", "-core_root", cwd, - "-jitoption", "JitStressRegs=" + jitstressregs, "-jitoption", "TieredCompilation=0", + "-jitoption", jit_flag, "-jitoption", "TieredCompilation=0", "-jit_path", jit_path, "-spmi_location", spmi_location, - "-log_level", "debug", "-log_file", log_file], - _exit_on_fail=True) + "-log_level", "debug", "-log_file", log_file]) # run_id += 1 - # Consolidate all superpmi.logs in superpmi_partition_index.log + # Consolidate all superpmi_*.logs in superpmi_platform_architecture.log final_log_name = path.join(log_directory, "superpmi_{}_{}.log".format(coreclr_args.platform, arch_name)) with open(final_log_name, "a") as final_superpmi_log: for superpmi_log in listdir(log_directory): - if not f.startswith("superpmi_flag") and not f.endswith(".log"): + if not f.startswith("superpmi_Jit") and not f.endswith(".log"): continue final_superpmi_log.write("======================================================={}".format(os.linesep)) From e7ab4994089bf794eb8a8f657331ebba9f08195b Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 3 Aug 2021 12:43:04 -0700 Subject: [PATCH 37/66] Use -target_os and pass 1 item to helix --- src/coreclr/scripts/superpmi-replay.proj | 12 ++++-------- src/coreclr/scripts/superpmi-replay.py | 13 ++++++++----- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/src/coreclr/scripts/superpmi-replay.proj b/src/coreclr/scripts/superpmi-replay.proj index fe9adb60779f73..1adce357c36d83 100644 --- a/src/coreclr/scripts/superpmi-replay.proj +++ b/src/coreclr/scripts/superpmi-replay.proj @@ -28,7 +28,7 @@ %HELIX_CORRELATION_PAYLOAD% $(WorkItemDirectory)\partitions %HELIX_WORKITEM_UPLOAD_ROOT% - $(Python) $(ProductDirectory)\superpmi-replay.py -jit_directory $(ProductDirectory) + $(Python) $(ProductDirectory)\superpmi-replay.py -arch $(Architecture) -platform $(Platform) -log_directory $(SuperpmiLogsLocation) -jit_directory $(ProductDirectory) @@ -42,14 +42,10 @@ --> - - - - $(WorkItemCommand) -arch %(HelixWorkItem.Architecture) -platform %(HelixWorkItem.Platform) -log_directory $(SuperpmiLogsLocation) + + $(WorkItemCommand) + $(WorkItemTimeout) diff --git a/src/coreclr/scripts/superpmi-replay.py b/src/coreclr/scripts/superpmi-replay.py index 2780d4ee51bcd6..7bcb0b282d752b 100644 --- a/src/coreclr/scripts/superpmi-replay.py +++ b/src/coreclr/scripts/superpmi-replay.py @@ -94,9 +94,13 @@ def main(main_args): spmi_location = path.join(cwd, "artifacts", "spmi") log_directory = coreclr_args.log_directory mch_filename = '' + os_name = "win" if coreclr_args.platform.lower() == "windows" else "unix" + arch_name = coreclr_args.arch + host_arch_name = "x64" if arch_name.endswith("x64") else "x86" + jit_path = path.join(coreclr_args.jit_directory, 'clrjit_{}_{}_{}.dll'.format(os_name, arch_name, host_arch_name)) print("Running superpmi.py download") - run_command([python_path, path.join(cwd, "superpmi.py"), "download", "-f", "benchmarks", "-core_root", cwd, "-spmi_location", spmi_location]) + run_command([python_path, path.join(cwd, "superpmi.py"), "download", "-f", "benchmarks", "-target_os", coreclr_args.platform.lower(), "-target_arch", arch_name, "-core_root", cwd, "-spmi_location", spmi_location]) for f in listdir(spmi_location): if f.endswith(".mch.zip"): @@ -110,10 +114,7 @@ def main(main_args): # populate based on zip file name and jit_directory - os_name = "win" if coreclr_args.platform.lower() == "windows" else "unix" - arch_name = coreclr_args.arch - host_arch_name = "x64" if arch_name.endswith("x64") else "x86" - jit_path = path.join(coreclr_args.jit_directory, 'clrjit_{}_{}_{}.dll'.format(os_name, arch_name, host_arch_name)) + # run_id = 0 for jit_flag in jit_flags: # TODO: This should be DownloadFilesFromResults @@ -133,6 +134,8 @@ def main(main_args): run_command([ python_path, path.join(cwd, "superpmi.py"), "replay", "-core_root", cwd, "-jitoption", jit_flag, "-jitoption", "TieredCompilation=0", + "-target_os", coreclr_args.platform.lower(), "-target_arch", arch_name, + "-arch", host_arch_name, "-jit_path", jit_path, "-spmi_location", spmi_location, "-log_level", "debug", "-log_file", log_file]) From eea085fb757675109e3a51bd93c5b984a70d0bb4 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 3 Aug 2021 13:38:22 -0700 Subject: [PATCH 38/66] Add some logging, have different work item ID --- src/coreclr/scripts/superpmi-replay.proj | 6 ++++-- src/coreclr/scripts/superpmi-replay.py | 2 ++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/coreclr/scripts/superpmi-replay.proj b/src/coreclr/scripts/superpmi-replay.proj index 1adce357c36d83..e4cce85c4aa00e 100644 --- a/src/coreclr/scripts/superpmi-replay.proj +++ b/src/coreclr/scripts/superpmi-replay.proj @@ -12,6 +12,8 @@ - + $(WorkItemCommand) $(WorkItemTimeout) - superpmi_%(HelixWorkItem.Platform)_%(HelixWorkItem.Architecture).log + superpmi_$(Platform)_$(Architecture).log diff --git a/src/coreclr/scripts/superpmi-replay.py b/src/coreclr/scripts/superpmi-replay.py index 7bcb0b282d752b..b21fac0c3151af 100644 --- a/src/coreclr/scripts/superpmi-replay.py +++ b/src/coreclr/scripts/superpmi-replay.py @@ -143,11 +143,13 @@ def main(main_args): # Consolidate all superpmi_*.logs in superpmi_platform_architecture.log final_log_name = path.join(log_directory, "superpmi_{}_{}.log".format(coreclr_args.platform, arch_name)) + print("Consolidating final {}".format(final_log_name)) with open(final_log_name, "a") as final_superpmi_log: for superpmi_log in listdir(log_directory): if not f.startswith("superpmi_Jit") and not f.endswith(".log"): continue + print("Appending {}".format(superpmi_log)) final_superpmi_log.write("======================================================={}".format(os.linesep)) final_superpmi_log.write("Contents from {}{}".format(superpmi_log, os.linesep)) final_superpmi_log.write("======================================================={}".format(os.linesep)) From 283f211e898ec92906970e1ec5a401b314b3e025 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 3 Aug 2021 14:00:05 -0700 Subject: [PATCH 39/66] fix the superpmi.log upload --- src/coreclr/scripts/superpmi-replay.proj | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/coreclr/scripts/superpmi-replay.proj b/src/coreclr/scripts/superpmi-replay.proj index e4cce85c4aa00e..e6828610182064 100644 --- a/src/coreclr/scripts/superpmi-replay.proj +++ b/src/coreclr/scripts/superpmi-replay.proj @@ -30,6 +30,8 @@ %HELIX_CORRELATION_PAYLOAD% $(WorkItemDirectory)\partitions %HELIX_WORKITEM_UPLOAD_ROOT% + + $(BUILD_SOURCESDIRECTORY)\artifacts\helixresults $(Python) $(ProductDirectory)\superpmi-replay.py -arch $(Architecture) -platform $(Platform) -log_directory $(SuperpmiLogsLocation) -jit_directory $(ProductDirectory) From f1e914b6e490e4884b42221a4af518ccbf654676 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 3 Aug 2021 14:39:29 -0700 Subject: [PATCH 40/66] Fix the log location - Also include Linux_x64 --- eng/pipelines/coreclr/superpmi-replay.yml | 1 + src/coreclr/scripts/superpmi-replay.proj | 4 ++-- src/coreclr/scripts/superpmi-replay.py | 9 +++++---- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/eng/pipelines/coreclr/superpmi-replay.yml b/eng/pipelines/coreclr/superpmi-replay.yml index 5028926d930727..35d48a956dc326 100644 --- a/eng/pipelines/coreclr/superpmi-replay.yml +++ b/eng/pipelines/coreclr/superpmi-replay.yml @@ -30,5 +30,6 @@ jobs: platforms: - windows_x64 - windows_x86 + - Linux_x64 helixQueueGroup: ci helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml \ No newline at end of file diff --git a/src/coreclr/scripts/superpmi-replay.proj b/src/coreclr/scripts/superpmi-replay.proj index e6828610182064..d605d6c5e57673 100644 --- a/src/coreclr/scripts/superpmi-replay.proj +++ b/src/coreclr/scripts/superpmi-replay.proj @@ -32,7 +32,7 @@ %HELIX_WORKITEM_UPLOAD_ROOT% $(BUILD_SOURCESDIRECTORY)\artifacts\helixresults - $(Python) $(ProductDirectory)\superpmi-replay.py -arch $(Architecture) -platform $(Platform) -log_directory $(SuperpmiLogsLocation) -jit_directory $(ProductDirectory) + $(Python) $(ProductDirectory)\superpmi-replay.py -arch $(Architecture) -platform $(Platform) -jit_directory $(ProductDirectory) @@ -48,7 +48,7 @@ - $(WorkItemCommand) + $(WorkItemCommand) -log_directory $(SuperpmiLogsLocation)\$(Platform)_$(Architecture) diff --git a/src/coreclr/scripts/superpmi-replay.py b/src/coreclr/scripts/superpmi-replay.py index b21fac0c3151af..bbc5bc2b02326c 100644 --- a/src/coreclr/scripts/superpmi-replay.py +++ b/src/coreclr/scripts/superpmi-replay.py @@ -94,13 +94,14 @@ def main(main_args): spmi_location = path.join(cwd, "artifacts", "spmi") log_directory = coreclr_args.log_directory mch_filename = '' - os_name = "win" if coreclr_args.platform.lower() == "windows" else "unix" + platform_name = coreclr_args.platform + os_name = "win" if platform_name.lower() == "windows" else "unix" arch_name = coreclr_args.arch host_arch_name = "x64" if arch_name.endswith("x64") else "x86" jit_path = path.join(coreclr_args.jit_directory, 'clrjit_{}_{}_{}.dll'.format(os_name, arch_name, host_arch_name)) print("Running superpmi.py download") - run_command([python_path, path.join(cwd, "superpmi.py"), "download", "-f", "benchmarks", "-target_os", coreclr_args.platform.lower(), "-target_arch", arch_name, "-core_root", cwd, "-spmi_location", spmi_location]) + run_command([python_path, path.join(cwd, "superpmi.py"), "download", "-f", "benchmarks", "-target_os", platform_name, "-target_arch", arch_name, "-core_root", cwd, "-spmi_location", spmi_location]) for f in listdir(spmi_location): if f.endswith(".mch.zip"): @@ -134,7 +135,7 @@ def main(main_args): run_command([ python_path, path.join(cwd, "superpmi.py"), "replay", "-core_root", cwd, "-jitoption", jit_flag, "-jitoption", "TieredCompilation=0", - "-target_os", coreclr_args.platform.lower(), "-target_arch", arch_name, + "-target_os", platform_name, "-target_arch", arch_name, "-arch", host_arch_name, "-jit_path", jit_path, "-spmi_location", spmi_location, "-log_level", "debug", "-log_file", log_file]) @@ -142,7 +143,7 @@ def main(main_args): # run_id += 1 # Consolidate all superpmi_*.logs in superpmi_platform_architecture.log - final_log_name = path.join(log_directory, "superpmi_{}_{}.log".format(coreclr_args.platform, arch_name)) + final_log_name = path.join(log_directory, "superpmi_{}_{}.log".format(platform_name, arch_name)) print("Consolidating final {}".format(final_log_name)) with open(final_log_name, "a") as final_superpmi_log: for superpmi_log in listdir(log_directory): From 92e3d553703b79e1f7bf31bf01af62e08aace1e0 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 3 Aug 2021 15:14:34 -0700 Subject: [PATCH 41/66] one more fix to log_directory --- .../coreclr/templates/superpmi-replay-job.yml | 17 ++++++++++++----- src/coreclr/scripts/superpmi-replay.py | 2 +- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/eng/pipelines/coreclr/templates/superpmi-replay-job.yml b/eng/pipelines/coreclr/templates/superpmi-replay-job.yml index 05a265b75147d5..61c6c6db75e040 100644 --- a/eng/pipelines/coreclr/templates/superpmi-replay-job.yml +++ b/eng/pipelines/coreclr/templates/superpmi-replay-job.yml @@ -15,7 +15,7 @@ jobs: - template: ${{ parameters.runJobTemplate }} parameters: jobName: ${{ format('superpmibuild_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - displayName: ${{ format('SuperPMI replay {0}', parameters.archType) }} + displayName: ${{ format('SuperPMI replay {0} {1}', parameters.osGroup, parameters.archType) }} pool: ${{ parameters.pool }} buildConfig: ${{ parameters.buildConfig }} archType: ${{ parameters.archType }} @@ -26,7 +26,14 @@ jobs: # dependsOn: # - ${{ format('coreclr_jit_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - variables: ${{ parameters.variables }} + variables: + - ${{ parameters.variables }} + + - name: ClrJitRootFolder + value: '$(Build.SourcesDirectory)/artifacts/bin/coreclr/windows.$(archType).$(buildConfigUpper)' + + - name: ClrJitArtifactName + value: 'CoreCLRProduct_${{ parameters.pgoType }}_${{ parameters.runtimeVariant }}_windows$(osSubgroup)_$(archType)_$(buildConfig)' frameworks: - ${{ parameters.framework }} @@ -36,7 +43,7 @@ jobs: # Download jit builds - template: /eng/pipelines/common/download-specific-artifact-step.yml parameters: - unpackFolder: $(buildProductRootFolderPath) - artifactFileName: '$(buildProductArtifactName)$(archiveExtension)' - artifactName: '$(buildProductArtifactName)' + unpackFolder: '$(ClrJitRootFolder)' + artifactFileName: '$(ClrJitArtifactName)$(archiveExtension)' + artifactName: '$(ClrJitArtifactName)' displayName: 'JIT product build' \ No newline at end of file diff --git a/src/coreclr/scripts/superpmi-replay.py b/src/coreclr/scripts/superpmi-replay.py index bbc5bc2b02326c..85d44aaad09858 100644 --- a/src/coreclr/scripts/superpmi-replay.py +++ b/src/coreclr/scripts/superpmi-replay.py @@ -75,7 +75,7 @@ def setup_args(args): coreclr_args.verify(args, "log_directory", - lambda log_directory: os.path.isdir(log_directory), + lambda log_directory: True, #os.path.isdir(log_directory), "log_directory doesn't exist") return coreclr_args From 93edb832c846b0230f3fd74bde3fb838ea7c3e81 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 3 Aug 2021 15:52:21 -0700 Subject: [PATCH 42/66] try to fix download results --- eng/pipelines/coreclr/templates/superpmi-replay-job.yml | 4 ++-- src/coreclr/scripts/superpmi-replay.proj | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/pipelines/coreclr/templates/superpmi-replay-job.yml b/eng/pipelines/coreclr/templates/superpmi-replay-job.yml index 61c6c6db75e040..2b700e7360d5b3 100644 --- a/eng/pipelines/coreclr/templates/superpmi-replay-job.yml +++ b/eng/pipelines/coreclr/templates/superpmi-replay-job.yml @@ -24,7 +24,7 @@ jobs: dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths }} additionalSetupParameters: ${{ parameters.additionalSetupParameters }} # dependsOn: - # - ${{ format('coreclr_jit_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + # - ${{ format('coreclr_jit_build_windows{0}_{1}_{2}', parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} variables: - ${{ parameters.variables }} @@ -44,6 +44,6 @@ jobs: - template: /eng/pipelines/common/download-specific-artifact-step.yml parameters: unpackFolder: '$(ClrJitRootFolder)' - artifactFileName: '$(ClrJitArtifactName)$(archiveExtension)' + artifactFileName: '$(ClrJitArtifactName).zip' artifactName: '$(ClrJitArtifactName)' displayName: 'JIT product build' \ No newline at end of file diff --git a/src/coreclr/scripts/superpmi-replay.proj b/src/coreclr/scripts/superpmi-replay.proj index d605d6c5e57673..f72065e445b21f 100644 --- a/src/coreclr/scripts/superpmi-replay.proj +++ b/src/coreclr/scripts/superpmi-replay.proj @@ -31,7 +31,7 @@ $(WorkItemDirectory)\partitions %HELIX_WORKITEM_UPLOAD_ROOT% - $(BUILD_SOURCESDIRECTORY)\artifacts\helixresults + $(Python) $(ProductDirectory)\superpmi-replay.py -arch $(Architecture) -platform $(Platform) -jit_directory $(ProductDirectory) From 20db184e21b798c825f69faaa46f52b5f410d40d Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 3 Aug 2021 16:03:54 -0700 Subject: [PATCH 43/66] Move the matrix to proj file --- eng/pipelines/coreclr/superpmi-replay.yml | 1 - .../templates/run-superpmi-replay-job.yml | 2 +- .../coreclr/templates/superpmi-replay-job.yml | 17 +++++----------- src/coreclr/scripts/superpmi-replay.proj | 20 +++++++++++++++---- src/coreclr/scripts/superpmi_replay_setup.py | 12 +++++------ 5 files changed, 27 insertions(+), 25 deletions(-) diff --git a/eng/pipelines/coreclr/superpmi-replay.yml b/eng/pipelines/coreclr/superpmi-replay.yml index 35d48a956dc326..5028926d930727 100644 --- a/eng/pipelines/coreclr/superpmi-replay.yml +++ b/eng/pipelines/coreclr/superpmi-replay.yml @@ -30,6 +30,5 @@ jobs: platforms: - windows_x64 - windows_x86 - - Linux_x64 helixQueueGroup: ci helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml \ No newline at end of file diff --git a/eng/pipelines/coreclr/templates/run-superpmi-replay-job.yml b/eng/pipelines/coreclr/templates/run-superpmi-replay-job.yml index 720cc99d19905a..df6fb45fa39e34 100644 --- a/eng/pipelines/coreclr/templates/run-superpmi-replay-job.yml +++ b/eng/pipelines/coreclr/templates/run-superpmi-replay-job.yml @@ -84,7 +84,7 @@ jobs: # - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi.py download --download_all --download_raw -spmi_location $(SpmiCollectionLocation) # displayName: Download superpmi collection - - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi_replay_setup.py -source_directory $(Build.SourcesDirectory) -product_directory $(buildProductRootFolderPath) -platform $(osGroup) -arch $(archType) + - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi_replay_setup.py -source_directory $(Build.SourcesDirectory) -product_directory $(buildProductRootFolderPath) -arch $(archType) displayName: ${{ format('SuperPMI replay setup ({0} {1})', parameters.osGroup, parameters.archType) }} # Run superpmi replay in helix diff --git a/eng/pipelines/coreclr/templates/superpmi-replay-job.yml b/eng/pipelines/coreclr/templates/superpmi-replay-job.yml index 2b700e7360d5b3..3dda9d5ae44e5c 100644 --- a/eng/pipelines/coreclr/templates/superpmi-replay-job.yml +++ b/eng/pipelines/coreclr/templates/superpmi-replay-job.yml @@ -24,16 +24,9 @@ jobs: dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths }} additionalSetupParameters: ${{ parameters.additionalSetupParameters }} # dependsOn: - # - ${{ format('coreclr_jit_build_windows{0}_{1}_{2}', parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + # - ${{ format('coreclr_jit_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - variables: - - ${{ parameters.variables }} - - - name: ClrJitRootFolder - value: '$(Build.SourcesDirectory)/artifacts/bin/coreclr/windows.$(archType).$(buildConfigUpper)' - - - name: ClrJitArtifactName - value: 'CoreCLRProduct_${{ parameters.pgoType }}_${{ parameters.runtimeVariant }}_windows$(osSubgroup)_$(archType)_$(buildConfig)' + variables: ${{ parameters.variables }} frameworks: - ${{ parameters.framework }} @@ -43,7 +36,7 @@ jobs: # Download jit builds - template: /eng/pipelines/common/download-specific-artifact-step.yml parameters: - unpackFolder: '$(ClrJitRootFolder)' - artifactFileName: '$(ClrJitArtifactName).zip' - artifactName: '$(ClrJitArtifactName)' + unpackFolder: $(buildProductRootFolderPath) + artifactFileName: '$(buildProductArtifactName)$(archiveExtension)' + artifactName: '$(buildProductArtifactName)' displayName: 'JIT product build' \ No newline at end of file diff --git a/src/coreclr/scripts/superpmi-replay.proj b/src/coreclr/scripts/superpmi-replay.proj index f72065e445b21f..c18f27dd1cf6fa 100644 --- a/src/coreclr/scripts/superpmi-replay.proj +++ b/src/coreclr/scripts/superpmi-replay.proj @@ -31,8 +31,8 @@ $(WorkItemDirectory)\partitions %HELIX_WORKITEM_UPLOAD_ROOT% - - $(Python) $(ProductDirectory)\superpmi-replay.py -arch $(Architecture) -platform $(Platform) -jit_directory $(ProductDirectory) + $(BUILD_SOURCESDIRECTORY)\artifacts\helixresults + $(Python) $(ProductDirectory)\superpmi-replay.py -jit_directory $(ProductDirectory) @@ -46,9 +46,21 @@ --> + + + + + + + + + + + + - - $(WorkItemCommand) -log_directory $(SuperpmiLogsLocation)\$(Platform)_$(Architecture) + + $(WorkItemCommand) -arch $(HelixWorkItem.Architecture) -platform $(HelixWorkItem.Platform) -log_directory $(SuperpmiLogsLocation) diff --git a/src/coreclr/scripts/superpmi_replay_setup.py b/src/coreclr/scripts/superpmi_replay_setup.py index a524ccbba38bd5..abc6b957dec8e0 100644 --- a/src/coreclr/scripts/superpmi_replay_setup.py +++ b/src/coreclr/scripts/superpmi_replay_setup.py @@ -31,7 +31,7 @@ parser = argparse.ArgumentParser(description="description") parser.add_argument("-arch", help="Architecture") -parser.add_argument("-platform", help="OS platform") +# parser.add_argument("-platform", help="OS platform") parser.add_argument("-source_directory", help="path to the directory containing binaries") parser.add_argument("-product_directory", help="path to the directory containing binaries") # parser.add_argument("-mch_directory", help="path to directory containing compressed mch files") @@ -55,10 +55,10 @@ def setup_args(args): lambda unused: True, "Unable to set arch") - coreclr_args.verify(args, - "platform", - lambda unused: True, - "Unable to set platform") + # coreclr_args.verify(args, + # "platform", + # lambda unused: True, + # "Unable to set platform") coreclr_args.verify(args, "source_directory", @@ -125,7 +125,6 @@ def main(main_args): coreclr_args = setup_args(main_args) arch = coreclr_args.arch - platform = coreclr_args.platform source_directory = coreclr_args.source_directory product_directory = coreclr_args.product_directory # mch_directory = coreclr_args.mch_directory @@ -175,7 +174,6 @@ def main(main_args): print('Setting pipeline variables:') set_pipeline_variable("CorrelationPayloadDirectory", correlation_payload_directory) set_pipeline_variable("WorkItemDirectory", workitem_directory) - set_pipeline_variable("Platform", platform) set_pipeline_variable("Architecture", arch) set_pipeline_variable("Creator", creator) set_pipeline_variable("Queue", helix_queue) From bcc6c3629075c85335fecf4aae1431a9955ba135 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 3 Aug 2021 16:16:33 -0700 Subject: [PATCH 44/66] fix superpmi-replay.proj --- src/coreclr/scripts/superpmi-replay.proj | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/coreclr/scripts/superpmi-replay.proj b/src/coreclr/scripts/superpmi-replay.proj index c18f27dd1cf6fa..17e4ab46bd7a4c 100644 --- a/src/coreclr/scripts/superpmi-replay.proj +++ b/src/coreclr/scripts/superpmi-replay.proj @@ -47,25 +47,25 @@ --> - - - - + + + + - - + + - $(WorkItemCommand) -arch $(HelixWorkItem.Architecture) -platform $(HelixWorkItem.Platform) -log_directory $(SuperpmiLogsLocation) + $(WorkItemCommand) -arch %(HelixWorkItem.Architecture) -platform %(HelixWorkItem.Platform) -log_directory $(SuperpmiLogsLocation) $(WorkItemTimeout) - superpmi_$(Platform)_$(Architecture).log + superpmi_%(HelixWorkItem.Platform)_%(HelixWorkItem.Architecture).log From 8b50c09162aa0cb8e15ef4d5d68389a1d6f801fb Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 3 Aug 2021 16:49:13 -0700 Subject: [PATCH 45/66] Cleanup --- eng/pipelines/coreclr/superpmi-replay.yml | 16 +++--- .../templates/run-superpmi-replay-job.yml | 3 -- .../coreclr/templates/superpmi-replay-job.yml | 6 +-- src/coreclr/scripts/superpmi-replay.proj | 8 --- src/coreclr/scripts/superpmi-replay.py | 53 ++----------------- 5 files changed, 15 insertions(+), 71 deletions(-) diff --git a/eng/pipelines/coreclr/superpmi-replay.yml b/eng/pipelines/coreclr/superpmi-replay.yml index 5028926d930727..79c225648ce55b 100644 --- a/eng/pipelines/coreclr/superpmi-replay.yml +++ b/eng/pipelines/coreclr/superpmi-replay.yml @@ -14,14 +14,14 @@ pr: none jobs: -# - template: /eng/pipelines/common/platform-matrix.yml -# parameters: -# jobTemplate: /eng/pipelines/coreclr/templates/build-jit-job.yml -# buildConfig: checked -# uploadAsArtifacts: true -# platforms: -# - windows_x64 -# - windows_x86 +- template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/build-jit-job.yml + buildConfig: checked + uploadAsArtifacts: true + platforms: + - windows_x64 + - windows_x86 - template: /eng/pipelines/common/platform-matrix.yml parameters: diff --git a/eng/pipelines/coreclr/templates/run-superpmi-replay-job.yml b/eng/pipelines/coreclr/templates/run-superpmi-replay-job.yml index df6fb45fa39e34..ee244e23856d8a 100644 --- a/eng/pipelines/coreclr/templates/run-superpmi-replay-job.yml +++ b/eng/pipelines/coreclr/templates/run-superpmi-replay-job.yml @@ -81,9 +81,6 @@ jobs: mkdir -p $(SpmiCollectionLocation) displayName: Create directory for SPMI collection - # - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi.py download --download_all --download_raw -spmi_location $(SpmiCollectionLocation) - # displayName: Download superpmi collection - - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi_replay_setup.py -source_directory $(Build.SourcesDirectory) -product_directory $(buildProductRootFolderPath) -arch $(archType) displayName: ${{ format('SuperPMI replay setup ({0} {1})', parameters.osGroup, parameters.archType) }} diff --git a/eng/pipelines/coreclr/templates/superpmi-replay-job.yml b/eng/pipelines/coreclr/templates/superpmi-replay-job.yml index 3dda9d5ae44e5c..90e7f69fa7496c 100644 --- a/eng/pipelines/coreclr/templates/superpmi-replay-job.yml +++ b/eng/pipelines/coreclr/templates/superpmi-replay-job.yml @@ -23,8 +23,8 @@ jobs: osSubgroup: ${{ parameters.osSubgroup }} dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths }} additionalSetupParameters: ${{ parameters.additionalSetupParameters }} - # dependsOn: - # - ${{ format('coreclr_jit_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + dependsOn: + - ${{ format('coreclr_jit_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} variables: ${{ parameters.variables }} @@ -34,7 +34,7 @@ jobs: steps: # Download jit builds - - template: /eng/pipelines/common/download-specific-artifact-step.yml + - template: /eng/pipelines/common/download-artifact-step.yml parameters: unpackFolder: $(buildProductRootFolderPath) artifactFileName: '$(buildProductArtifactName)$(archiveExtension)' diff --git a/src/coreclr/scripts/superpmi-replay.proj b/src/coreclr/scripts/superpmi-replay.proj index 17e4ab46bd7a4c..950dc6a420d808 100644 --- a/src/coreclr/scripts/superpmi-replay.proj +++ b/src/coreclr/scripts/superpmi-replay.proj @@ -41,11 +41,6 @@ - - @@ -61,9 +56,6 @@ $(WorkItemCommand) -arch %(HelixWorkItem.Architecture) -platform %(HelixWorkItem.Platform) -log_directory $(SuperpmiLogsLocation) - - - $(WorkItemTimeout) superpmi_%(HelixWorkItem.Platform)_%(HelixWorkItem.Architecture).log diff --git a/src/coreclr/scripts/superpmi-replay.py b/src/coreclr/scripts/superpmi-replay.py index 85d44aaad09858..fc9a7eb99d98a4 100644 --- a/src/coreclr/scripts/superpmi-replay.py +++ b/src/coreclr/scripts/superpmi-replay.py @@ -1,26 +1,16 @@ import argparse -from os import path, system +from os import path import os -import shutil -import stat -import subprocess -import tempfile -import re - -from os.path import isfile, join from os import listdir from coreclr_arguments import * -from superpmi_setup import copy_directory, copy_files, set_pipeline_variable from superpmi_setup import run_command parser = argparse.ArgumentParser(description="description") parser.add_argument("-arch", help="Architecture") parser.add_argument("-platform", help="OS platform") -parser.add_argument("-partition_index", help="Partition index for logs") parser.add_argument("-jit_directory", help="path to the directory containing clrjit binaries") -parser.add_argument("-mch_directory", help="path to the directory containing mch files") parser.add_argument("-log_directory", help="path to the directory containing superpmi log files") jit_flags = [ @@ -58,24 +48,14 @@ def setup_args(args): lambda unused: True, "Unable to set platform") - # coreclr_args.verify(args, - # "partition_index", - # lambda partition_index: True, #partition_index.isnumeric(), - # "Unable to set partition_index") - coreclr_args.verify(args, "jit_directory", lambda jit_directory: os.path.isdir(jit_directory), "jit_directory doesn't exist") - # coreclr_args.verify(args, - # "mch_directory", - # lambda mch_directory: True, #os.path.isdir(mch_directory), - # "mch_directory doesn't exist") - coreclr_args.verify(args, "log_directory", - lambda log_directory: True, #os.path.isdir(log_directory), + lambda log_directory: True, "log_directory doesn't exist") return coreclr_args @@ -97,40 +77,17 @@ def main(main_args): platform_name = coreclr_args.platform os_name = "win" if platform_name.lower() == "windows" else "unix" arch_name = coreclr_args.arch - host_arch_name = "x64" if arch_name.endswith("x64") else "x86" + host_arch_name = "x64" if arch_name.endswith("64") else "x86" jit_path = path.join(coreclr_args.jit_directory, 'clrjit_{}_{}_{}.dll'.format(os_name, arch_name, host_arch_name)) print("Running superpmi.py download") - run_command([python_path, path.join(cwd, "superpmi.py"), "download", "-f", "benchmarks", "-target_os", platform_name, "-target_arch", arch_name, "-core_root", cwd, "-spmi_location", spmi_location]) - - for f in listdir(spmi_location): - if f.endswith(".mch.zip"): - mch_filename = f - - # match_mch_elems = re.search('(.*)\.(windows|Linux)\.(x64|x86|arm|arm64)\.', mch_filename) - # collection_name = os_name = match_mch_elems.group(1).replace('.','_') - # os_name = "win" if match_mch_elems.group(2) == "windows" else "unix" - # target_arch_name = match_mch_elems.group(3) - # run_name = "{}_{}_{}".format(os_name, target_arch_name, coreclr_args.arch) - - - # populate based on zip file name and jit_directory + run_command([python_path, path.join(cwd, "superpmi.py"), "download", "-target_os", platform_name, "-target_arch", arch_name, "-core_root", cwd, "-spmi_location", spmi_location]) # run_id = 0 for jit_flag in jit_flags: # TODO: This should be DownloadFilesFromResults log_file = path.join(log_directory, 'superpmi_{}.log'.format(jit_flag.replace("=", "_"))) print("Running superpmi.py replay for {}".format(jit_flag)) - # print(' '.join([ - # python_path, path.join(cwd, "superpmi.py"), "replay", "-core_root", cwd, - # "-jitoption", "JitStressRegs=" + jitstressregs, "-jitoption", "TieredCompilation=0", - # "-jit_path", jit_path, "-mch_files", mch_directory, "-spmi_location", mch_directory, - # "-log_level", "debug", "-log_file", log_path])) - - # In first iteration, mch files are unzipped and placed in "mch" folder. Going forward, use the - # uncompressed mch files - # if run_id == 1: - # mch_directory = path.join(mch_directory, "mch") run_command([ python_path, path.join(cwd, "superpmi.py"), "replay", "-core_root", cwd, @@ -140,8 +97,6 @@ def main(main_args): "-jit_path", jit_path, "-spmi_location", spmi_location, "-log_level", "debug", "-log_file", log_file]) - # run_id += 1 - # Consolidate all superpmi_*.logs in superpmi_platform_architecture.log final_log_name = path.join(log_directory, "superpmi_{}_{}.log".format(platform_name, arch_name)) print("Consolidating final {}".format(final_log_name)) From e35562ef8f8519911bd69da1c36031012766605b Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 4 Aug 2021 06:45:41 -0700 Subject: [PATCH 46/66] Go back to download specific --- .../common/download-specific-artifact-step.yml | 2 +- eng/pipelines/coreclr/superpmi-replay.yml | 16 ++++++++-------- .../coreclr/templates/superpmi-replay-job.yml | 6 +++--- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/eng/pipelines/common/download-specific-artifact-step.yml b/eng/pipelines/common/download-specific-artifact-step.yml index 27860e5874925f..ae1fe1e32b4950 100644 --- a/eng/pipelines/common/download-specific-artifact-step.yml +++ b/eng/pipelines/common/download-specific-artifact-step.yml @@ -15,7 +15,7 @@ steps: pipeline: 'superpmi-replay' buildVersionToDownload: specific branchName: 'kpathak/spmi-replay' - buildId: '1271380' + buildId: '1274841' downloadType: single downloadPath: '$(Build.SourcesDirectory)/__download__' artifactName: '${{ parameters.artifactName }}' diff --git a/eng/pipelines/coreclr/superpmi-replay.yml b/eng/pipelines/coreclr/superpmi-replay.yml index 79c225648ce55b..5028926d930727 100644 --- a/eng/pipelines/coreclr/superpmi-replay.yml +++ b/eng/pipelines/coreclr/superpmi-replay.yml @@ -14,14 +14,14 @@ pr: none jobs: -- template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/build-jit-job.yml - buildConfig: checked - uploadAsArtifacts: true - platforms: - - windows_x64 - - windows_x86 +# - template: /eng/pipelines/common/platform-matrix.yml +# parameters: +# jobTemplate: /eng/pipelines/coreclr/templates/build-jit-job.yml +# buildConfig: checked +# uploadAsArtifacts: true +# platforms: +# - windows_x64 +# - windows_x86 - template: /eng/pipelines/common/platform-matrix.yml parameters: diff --git a/eng/pipelines/coreclr/templates/superpmi-replay-job.yml b/eng/pipelines/coreclr/templates/superpmi-replay-job.yml index 90e7f69fa7496c..3dda9d5ae44e5c 100644 --- a/eng/pipelines/coreclr/templates/superpmi-replay-job.yml +++ b/eng/pipelines/coreclr/templates/superpmi-replay-job.yml @@ -23,8 +23,8 @@ jobs: osSubgroup: ${{ parameters.osSubgroup }} dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths }} additionalSetupParameters: ${{ parameters.additionalSetupParameters }} - dependsOn: - - ${{ format('coreclr_jit_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + # dependsOn: + # - ${{ format('coreclr_jit_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} variables: ${{ parameters.variables }} @@ -34,7 +34,7 @@ jobs: steps: # Download jit builds - - template: /eng/pipelines/common/download-artifact-step.yml + - template: /eng/pipelines/common/download-specific-artifact-step.yml parameters: unpackFolder: $(buildProductRootFolderPath) artifactFileName: '$(buildProductArtifactName)$(archiveExtension)' From 7033211735bc3a44bd98b1eb564485d626e13c01 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 4 Aug 2021 07:01:00 -0700 Subject: [PATCH 47/66] WorkTimeout for 5 hours --- src/coreclr/scripts/superpmi-replay.proj | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/coreclr/scripts/superpmi-replay.proj b/src/coreclr/scripts/superpmi-replay.proj index 950dc6a420d808..71c2eb495fbee8 100644 --- a/src/coreclr/scripts/superpmi-replay.proj +++ b/src/coreclr/scripts/superpmi-replay.proj @@ -33,6 +33,9 @@ $(BUILD_SOURCESDIRECTORY)\artifacts\helixresults $(Python) $(ProductDirectory)\superpmi-replay.py -jit_directory $(ProductDirectory) + false + false + 5:00 From 4edf41d3217bff3f7365821c44b92165c0ef0a0a Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 4 Aug 2021 10:51:20 -0700 Subject: [PATCH 48/66] Remove timeoutInMinutes --- eng/pipelines/coreclr/templates/superpmi-replay-job.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/eng/pipelines/coreclr/templates/superpmi-replay-job.yml b/eng/pipelines/coreclr/templates/superpmi-replay-job.yml index 3dda9d5ae44e5c..4f159f67b756eb 100644 --- a/eng/pipelines/coreclr/templates/superpmi-replay-job.yml +++ b/eng/pipelines/coreclr/templates/superpmi-replay-job.yml @@ -5,7 +5,6 @@ parameters: osSubgroup: '' # optional -- operating system subgroup pool: '' stagedBuild: false - timeoutInMinutes: '' framework: net5.0 # Specify the appropriate framework when running release branches (ie netcoreapp3.0 for release/3.0) variables: {} dependOnEvaluatePaths: false From 244693f5b59d6871273db066f55421217b7756a7 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 4 Aug 2021 10:55:49 -0700 Subject: [PATCH 49/66] fix errors related to logging --- src/coreclr/scripts/superpmi-replay.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/scripts/superpmi-replay.py b/src/coreclr/scripts/superpmi-replay.py index fc9a7eb99d98a4..f90a65ed6b7ecf 100644 --- a/src/coreclr/scripts/superpmi-replay.py +++ b/src/coreclr/scripts/superpmi-replay.py @@ -102,7 +102,7 @@ def main(main_args): print("Consolidating final {}".format(final_log_name)) with open(final_log_name, "a") as final_superpmi_log: for superpmi_log in listdir(log_directory): - if not f.startswith("superpmi_Jit") and not f.endswith(".log"): + if not superpmi_log.startswith("superpmi_Jit") and not superpmi_log.endswith(".log"): continue print("Appending {}".format(superpmi_log)) From 9794f4c6bdbef3a7211a594054cdbd8dce589647 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 4 Aug 2021 11:23:19 -0700 Subject: [PATCH 50/66] return failed run result --- .../templates/run-superpmi-replay-job.yml | 1 + .../coreclr/templates/superpmi-replay-job.yml | 2 ++ src/coreclr/scripts/superpmi-replay.py | 18 +++++++++++++++--- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/eng/pipelines/coreclr/templates/run-superpmi-replay-job.yml b/eng/pipelines/coreclr/templates/run-superpmi-replay-job.yml index ee244e23856d8a..e74e3946da3311 100644 --- a/eng/pipelines/coreclr/templates/run-superpmi-replay-job.yml +++ b/eng/pipelines/coreclr/templates/run-superpmi-replay-job.yml @@ -39,6 +39,7 @@ jobs: collectionType: $ {{ parameters.collectionType }} collectionName: ${{ parameters.collectionName }} dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths }} + timeoutInMinutes: ${{ parameters.timeoutInMinutes }} ${{ if ne(parameters.displayName, '') }}: displayName: '${{ parameters.displayName }}' diff --git a/eng/pipelines/coreclr/templates/superpmi-replay-job.yml b/eng/pipelines/coreclr/templates/superpmi-replay-job.yml index 4f159f67b756eb..3f667f1107e7a2 100644 --- a/eng/pipelines/coreclr/templates/superpmi-replay-job.yml +++ b/eng/pipelines/coreclr/templates/superpmi-replay-job.yml @@ -5,6 +5,7 @@ parameters: osSubgroup: '' # optional -- operating system subgroup pool: '' stagedBuild: false + timeoutInMinutes: 320 # build timeout framework: net5.0 # Specify the appropriate framework when running release branches (ie netcoreapp3.0 for release/3.0) variables: {} dependOnEvaluatePaths: false @@ -21,6 +22,7 @@ jobs: osGroup: ${{ parameters.osGroup }} osSubgroup: ${{ parameters.osSubgroup }} dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths }} + timeoutInMinutes: ${{ parameters.timeoutInMinutes }} additionalSetupParameters: ${{ parameters.additionalSetupParameters }} # dependsOn: # - ${{ format('coreclr_jit_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} diff --git a/src/coreclr/scripts/superpmi-replay.py b/src/coreclr/scripts/superpmi-replay.py index f90a65ed6b7ecf..696305472535e9 100644 --- a/src/coreclr/scripts/superpmi-replay.py +++ b/src/coreclr/scripts/superpmi-replay.py @@ -81,15 +81,15 @@ def main(main_args): jit_path = path.join(coreclr_args.jit_directory, 'clrjit_{}_{}_{}.dll'.format(os_name, arch_name, host_arch_name)) print("Running superpmi.py download") - run_command([python_path, path.join(cwd, "superpmi.py"), "download", "-target_os", platform_name, "-target_arch", arch_name, "-core_root", cwd, "-spmi_location", spmi_location]) + run_command([python_path, path.join(cwd, "superpmi.py"), "download", "-target_os", platform_name, "-target_arch", arch_name, "-core_root", cwd, "-spmi_location", spmi_location], _exit_on_fail=True) - # run_id = 0 + failed_runs = [] for jit_flag in jit_flags: # TODO: This should be DownloadFilesFromResults log_file = path.join(log_directory, 'superpmi_{}.log'.format(jit_flag.replace("=", "_"))) print("Running superpmi.py replay for {}".format(jit_flag)) - run_command([ + _, _, return_code = run_command([ python_path, path.join(cwd, "superpmi.py"), "replay", "-core_root", cwd, "-jitoption", jit_flag, "-jitoption", "TieredCompilation=0", "-target_os", platform_name, "-target_arch", arch_name, @@ -97,6 +97,9 @@ def main(main_args): "-jit_path", jit_path, "-spmi_location", spmi_location, "-log_level", "debug", "-log_file", log_file]) + if return_code != 0: + failed_runs.append("Failure in {}".format(log_file)) + # Consolidate all superpmi_*.logs in superpmi_platform_architecture.log final_log_name = path.join(log_directory, "superpmi_{}_{}.log".format(platform_name, arch_name)) print("Consolidating final {}".format(final_log_name)) @@ -113,6 +116,15 @@ def main(main_args): contents = current_superpmi_log.read() final_superpmi_log.write(contents) + if len(failed_runs) > 0: + final_superpmi_log.write(os.linesep) + final_superpmi_log.write(os.linesep) + final_superpmi_log.write("========Failed runs summary========".format(os.linesep)) + final_superpmi_log.write(os.linesep.join(failed_runs)) + + return 0 if len(failed_runs) == 0 else 1 + + if __name__ == "__main__": args = parser.parse_args() sys.exit(main(args)) From dc5ca486576a4a11292dfc5e51d47039b71950c8 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 4 Aug 2021 12:08:23 -0700 Subject: [PATCH 51/66] fix the appending logic --- src/coreclr/scripts/superpmi-replay.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/scripts/superpmi-replay.py b/src/coreclr/scripts/superpmi-replay.py index 696305472535e9..02d32ade6e5449 100644 --- a/src/coreclr/scripts/superpmi-replay.py +++ b/src/coreclr/scripts/superpmi-replay.py @@ -105,7 +105,7 @@ def main(main_args): print("Consolidating final {}".format(final_log_name)) with open(final_log_name, "a") as final_superpmi_log: for superpmi_log in listdir(log_directory): - if not superpmi_log.startswith("superpmi_Jit") and not superpmi_log.endswith(".log"): + if not superpmi_log.startswith("superpmi_Jit") or not superpmi_log.endswith(".log"): continue print("Appending {}".format(superpmi_log)) From b9ba24124ec11ae612079ab7bc1d32b3b1883077 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 4 Aug 2021 14:21:18 -0700 Subject: [PATCH 52/66] Remove download-all and download-raw and instead add -no-progres --- src/coreclr/scripts/superpmi.py | 62 ++++++++++++--------------------- 1 file changed, 23 insertions(+), 39 deletions(-) diff --git a/src/coreclr/scripts/superpmi.py b/src/coreclr/scripts/superpmi.py index 5afc583d120be7..9ebf5c36345303 100755 --- a/src/coreclr/scripts/superpmi.py +++ b/src/coreclr/scripts/superpmi.py @@ -194,12 +194,8 @@ target directory already exists. """ -download_raw_help = """\ -If specified 'raw', then mch files will not be decompressed. -""" - -download_all_help = """\ -If specified, it will override the target_arch/target_os and download mch files for all arch/OS. +download_no_progress_help = """\ +If specified, then download progress will not be shown. """ merge_mch_pattern_help = """\ @@ -345,8 +341,7 @@ download_parser.add_argument("-jit_ee_version", help=jit_ee_version_help) download_parser.add_argument("--skip_cleanup", action="store_true", help=skip_cleanup_help) download_parser.add_argument("--force_download", action="store_true", help=force_download_help) -download_parser.add_argument("--download_raw", action="store_true", help=download_raw_help) -download_parser.add_argument("--download_all", action="store_true", help=download_all_help) +download_parser.add_argument("--no_progress", action="store_true", help=download_no_progress_help) download_parser.add_argument("-mch_files", metavar="MCH_FILE", nargs='+', help=replay_mch_files_help) download_parser.add_argument("-private_store", action="append", help=private_store_help) @@ -434,7 +429,7 @@ def download_progress_hook(count, block_size, total_size): sys.stdout.flush() -def download_with_progress_urlretrieve(uri, target_location, fail_if_not_found=True): +def download_with_progress_urlretrieve(uri, target_location, fail_if_not_found=True, display_progress=True): """ Do an URI download using urllib.request.urlretrieve with a progress hook. Args: @@ -449,7 +444,8 @@ def download_with_progress_urlretrieve(uri, target_location, fail_if_not_found=T ok = True try: - urllib.request.urlretrieve(uri, target_location, reporthook=download_progress_hook) + progress_display_method = download_progress_hook if display_progress else None + urllib.request.urlretrieve(uri, target_location, reporthook=progress_display_method) except urllib.error.HTTPError as httperror: if (httperror == 404) and fail_if_not_found: logging.error("HTTP 404 error") @@ -508,7 +504,7 @@ def download_with_azure(uri, target_location, fail_if_not_found=True): return ok -def download_one_url(uri, target_location, fail_if_not_found=True): +def download_one_url(uri, target_location, fail_if_not_found=True, display_progress=True): """ Do an URI download using urllib.request.urlretrieve or Azure Storage APIs. Args: @@ -522,7 +518,7 @@ def download_one_url(uri, target_location, fail_if_not_found=True): if authenticate_using_azure: return download_with_azure(uri, target_location, fail_if_not_found) else: - return download_with_progress_urlretrieve(uri, target_location, fail_if_not_found) + return download_with_progress_urlretrieve(uri, target_location, fail_if_not_found, display_progress) def is_zero_length_file(fpath): @@ -2568,7 +2564,7 @@ def filter_local_path(path): # Download all the urls at once, and add the local cache filenames to our accumulated list of local file names. if len(urls) != 0: - local_mch_files += download_files(urls, mch_cache_dir) + local_mch_files += download_files(urls, mch_cache_dir, display_progress=not coreclr_args.no_progress) # Special case: walk the URLs list and for every ".mch" or ".mch.zip" file, check to see that either the associated ".mct" file is already # in the list, or add it to a new list to attempt to download (but don't fail the download if it doesn't exist). @@ -2579,7 +2575,7 @@ def filter_local_path(path): if mct_url not in urls: mct_urls.append(mct_url) if len(mct_urls) != 0: - local_mch_files += download_files(mct_urls, mch_cache_dir, fail_if_not_found=False) + local_mch_files += download_files(mct_urls, mch_cache_dir, fail_if_not_found=False, display_progress=not coreclr_args.no_progress) # Even though we might have downloaded MCT files, only return the set of MCH files. local_mch_files = [file for file in local_mch_files if any(file.lower().endswith(extension) for extension in [".mch"])] @@ -2655,9 +2651,7 @@ def download_mch_from_azure(coreclr_args, target_dir): list containing the local path of files downloaded """ - download_all = hasattr(coreclr_args, "download_all") and coreclr_args.download_all - download_raw = hasattr(coreclr_args, "download_raw") and coreclr_args.download_raw - blob_filter_string = "{}/".format(coreclr_args.jit_ee_version).lower() if download_all else "{}/{}/{}/".format(coreclr_args.jit_ee_version, coreclr_args.target_os, coreclr_args.mch_arch).lower() + blob_filter_string = "{}/{}/{}/".format(coreclr_args.jit_ee_version, coreclr_args.target_os, coreclr_args.mch_arch).lower() # Determine if a URL in Azure Storage should be allowed. The path looks like: # jit-ee-guid/Linux/x64/Linux.x64.Checked.frameworks.mch.zip @@ -2676,14 +2670,13 @@ def filter_superpmi_collections(path): blob_url_prefix = "{}/{}/".format(az_blob_storage_superpmi_container_uri, az_collections_root_folder) urls = [blob_url_prefix + path for path in paths] - return download_files(urls, target_dir, uncompress=not download_raw) + return download_files(urls, target_dir, display_progress=not coreclr_args.no_progress) -def download_files(paths, target_dir, uncompress=True, verbose=True, fail_if_not_found=True): +def download_files(paths, target_dir, verbose=True, fail_if_not_found=True, display_progress=True): """ Download a set of files, specified as URLs or paths (such as Windows UNC paths), to a target directory. If a file is a .ZIP file, then uncompress the file and - copy all its contents to the target directory. If skipUncompress is specified, then - simply copy the .ZIP file to the target directory without uncompressing. + copy all its contents to the target directory. Args: paths (list): the URLs and paths to download @@ -2729,7 +2722,7 @@ def download_files(paths, target_dir, uncompress=True, verbose=True, fail_if_not download_path = os.path.join(temp_location, item_name) if is_item_url: - ok = download_one_url(item_path, download_path, fail_if_not_found) + ok = download_one_url(item_path, download_path, fail_if_not_found, display_progress) if not ok: continue else: @@ -2738,17 +2731,13 @@ def download_files(paths, target_dir, uncompress=True, verbose=True, fail_if_not logging.info("Download: %s -> %s", item_path, download_path) shutil.copy2(item_path, download_path) - if uncompress: - if verbose: - logging.info("Uncompress %s", download_path) - with zipfile.ZipFile(download_path, "r") as file_handle: - file_handle.extractall(temp_location) + if verbose: + logging.info("Uncompress %s", download_path) + with zipfile.ZipFile(download_path, "r") as file_handle: + file_handle.extractall(temp_location) # Copy everything that was extracted to the target directory. - if uncompress: - items = [ os.path.join(temp_location, item) for item in os.listdir(temp_location) if not item.endswith(".zip") ] - else: - items = [ os.path.join(temp_location, item) for item in os.listdir(temp_location) if item.endswith(".zip") ] + items = [ os.path.join(temp_location, item) for item in os.listdir(temp_location) if not item.endswith(".zip") ] for item in items: target_path = os.path.join(target_dir, os.path.basename(item)) if verbose: @@ -2759,7 +2748,7 @@ def download_files(paths, target_dir, uncompress=True, verbose=True, fail_if_not # Not a zip file; download directory to target directory download_path = os.path.join(target_dir, item_name) if is_item_url: - ok = download_one_url(item_path, download_path, fail_if_not_found) + ok = download_one_url(item_path, download_path, fail_if_not_found, display_progress) if not ok: continue else: @@ -3792,14 +3781,9 @@ def verify_replay_common_args(): "Unable to set force_download") coreclr_args.verify(args, - "download_all", - lambda unused: True, - "Unable to set download_all") - - coreclr_args.verify(args, - "download_raw", + "no_progress", lambda unused: True, - "Unable to set download_raw") + "Unable to set no_progress") coreclr_args.verify(args, "filter", From 4d97688c2073f57999235066c611102c1523436a Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 4 Aug 2021 14:40:40 -0700 Subject: [PATCH 53/66] do not show download progress, try fixing download file --- src/coreclr/scripts/superpmi-replay.proj | 2 +- src/coreclr/scripts/superpmi-replay.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/coreclr/scripts/superpmi-replay.proj b/src/coreclr/scripts/superpmi-replay.proj index 71c2eb495fbee8..737b236e3eff6c 100644 --- a/src/coreclr/scripts/superpmi-replay.proj +++ b/src/coreclr/scripts/superpmi-replay.proj @@ -60,7 +60,7 @@ $(WorkItemCommand) -arch %(HelixWorkItem.Architecture) -platform %(HelixWorkItem.Platform) -log_directory $(SuperpmiLogsLocation) $(WorkItemTimeout) - superpmi_%(HelixWorkItem.Platform)_%(HelixWorkItem.Architecture).log + 1/superpmi_%(HelixWorkItem.Platform)_%(HelixWorkItem.Architecture).log diff --git a/src/coreclr/scripts/superpmi-replay.py b/src/coreclr/scripts/superpmi-replay.py index 02d32ade6e5449..a2fb9866121100 100644 --- a/src/coreclr/scripts/superpmi-replay.py +++ b/src/coreclr/scripts/superpmi-replay.py @@ -81,7 +81,7 @@ def main(main_args): jit_path = path.join(coreclr_args.jit_directory, 'clrjit_{}_{}_{}.dll'.format(os_name, arch_name, host_arch_name)) print("Running superpmi.py download") - run_command([python_path, path.join(cwd, "superpmi.py"), "download", "-target_os", platform_name, "-target_arch", arch_name, "-core_root", cwd, "-spmi_location", spmi_location], _exit_on_fail=True) + run_command([python_path, path.join(cwd, "superpmi.py"), "download", "--no_progress", "-target_os", platform_name, "-target_arch", arch_name, "-core_root", cwd, "-spmi_location", spmi_location], _exit_on_fail=True) failed_runs = [] for jit_flag in jit_flags: From 53eb39679ed95a5fb41f6ddf11f671821cf02cd4 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 4 Aug 2021 14:41:25 -0700 Subject: [PATCH 54/66] TO REVERT: Just try benchmark on windows x64/x86 --- src/coreclr/scripts/superpmi-replay.proj | 6 +++--- src/coreclr/scripts/superpmi-replay.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/coreclr/scripts/superpmi-replay.proj b/src/coreclr/scripts/superpmi-replay.proj index 737b236e3eff6c..5b6ac38afa1de6 100644 --- a/src/coreclr/scripts/superpmi-replay.proj +++ b/src/coreclr/scripts/superpmi-replay.proj @@ -46,14 +46,14 @@ - + - + diff --git a/src/coreclr/scripts/superpmi-replay.py b/src/coreclr/scripts/superpmi-replay.py index a2fb9866121100..fb35559a8cd6b9 100644 --- a/src/coreclr/scripts/superpmi-replay.py +++ b/src/coreclr/scripts/superpmi-replay.py @@ -81,7 +81,7 @@ def main(main_args): jit_path = path.join(coreclr_args.jit_directory, 'clrjit_{}_{}_{}.dll'.format(os_name, arch_name, host_arch_name)) print("Running superpmi.py download") - run_command([python_path, path.join(cwd, "superpmi.py"), "download", "--no_progress", "-target_os", platform_name, "-target_arch", arch_name, "-core_root", cwd, "-spmi_location", spmi_location], _exit_on_fail=True) + run_command([python_path, path.join(cwd, "superpmi.py"), "download", "--no_progress", "-f", "benchmark", "-target_os", platform_name, "-target_arch", arch_name, "-core_root", cwd, "-spmi_location", spmi_location], _exit_on_fail=True) failed_runs = [] for jit_flag in jit_flags: From 0c12c610f00e67e40079957b62246218d14d007a Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 4 Aug 2021 15:58:36 -0700 Subject: [PATCH 55/66] Use uploadAsArtifacts in build-jit-job --- .../coreclr/templates/build-jit-job.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/eng/pipelines/coreclr/templates/build-jit-job.yml b/eng/pipelines/coreclr/templates/build-jit-job.yml index 53455b2d9206a9..5d6e3e5985ee4c 100644 --- a/eng/pipelines/coreclr/templates/build-jit-job.yml +++ b/eng/pipelines/coreclr/templates/build-jit-job.yml @@ -108,17 +108,17 @@ jobs: - script: set __TestIntermediateDir=int&&$(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) -ci -nopgooptimize -skiprestoreoptdata -component alljits displayName: Build CoreCLR JIT - # - ${{ if eq(parameters.uploadAsArtifacts, 'false') }}: - # # Ensure the Python azure-storage-blob package is installed before doing the upload. - # - script: $(PipScript) install --user --upgrade pip && $(PipScript) install --user azure.storage.blob==12.5.0 --force-reinstall - # displayName: Upgrade Pip to latest and install azure-storage-blob Python package + - ${{ if eq(parameters.uploadAsArtifacts, 'false') }}: + # Ensure the Python azure-storage-blob package is installed before doing the upload. + - script: $(PipScript) install --user --upgrade pip && $(PipScript) install --user azure.storage.blob==12.5.0 --force-reinstall + displayName: Upgrade Pip to latest and install azure-storage-blob Python package - # - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/jitrollingbuild.py upload -build_type $(buildConfig) -arch $(archType) -host_os $(osGroup) -git_hash $(Build.SourceVersion) - # displayName: Upload JIT to Azure Storage - # env: - # CLRJIT_AZ_KEY: $(clrjit_key1) # secret key stored as variable in pipeline + - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/jitrollingbuild.py upload -build_type $(buildConfig) -arch $(archType) -host_os $(osGroup) -git_hash $(Build.SourceVersion) + displayName: Upload JIT to Azure Storage + env: + CLRJIT_AZ_KEY: $(clrjit_key1) # secret key stored as variable in pipeline - # - ${{ if eq(parameters.uploadAsArtifacts, 'true') }}: + - ${{ if eq(parameters.uploadAsArtifacts, 'true') }}: # Publish product output directory for consumption by tests. - template: /eng/pipelines/common/upload-artifact-step.yml parameters: From 4a48c3702be3d03574442267a235aa66d4eaddb5 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 4 Aug 2021 16:03:27 -0700 Subject: [PATCH 56/66] Create dummy.txt in workitem directory --- src/coreclr/scripts/superpmi-replay.proj | 6 +++--- src/coreclr/scripts/superpmi_replay_setup.py | 4 ++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/coreclr/scripts/superpmi-replay.proj b/src/coreclr/scripts/superpmi-replay.proj index 5b6ac38afa1de6..504f796846ef51 100644 --- a/src/coreclr/scripts/superpmi-replay.proj +++ b/src/coreclr/scripts/superpmi-replay.proj @@ -6,7 +6,7 @@ Once you've done that you can run this to see the results: - dotnet msbuild .\superpmi.proj /v:n + dotnet msbuild .\superpmi-replay.proj /v:n --> $(BUILD_SOURCESDIRECTORY)\artifacts\helixresults @@ -59,8 +58,9 @@ $(WorkItemCommand) -arch %(HelixWorkItem.Architecture) -platform %(HelixWorkItem.Platform) -log_directory $(SuperpmiLogsLocation) + $(WorkItemDirectory) $(WorkItemTimeout) - 1/superpmi_%(HelixWorkItem.Platform)_%(HelixWorkItem.Architecture).log + superpmi_%(HelixWorkItem.Platform)_%(HelixWorkItem.Architecture).log diff --git a/src/coreclr/scripts/superpmi_replay_setup.py b/src/coreclr/scripts/superpmi_replay_setup.py index abc6b957dec8e0..f948f65f49a951 100644 --- a/src/coreclr/scripts/superpmi_replay_setup.py +++ b/src/coreclr/scripts/superpmi_replay_setup.py @@ -149,6 +149,10 @@ def main(main_args): print('Copying binaries {} -> {}'.format(arch, product_directory, correlation_payload_directory)) copy_directory(product_directory, correlation_payload_directory, match_func=match_correlation_files) + dummy_workitem_file = path.join(workitem_directory, "dummy.txt") + with open(dummy_workitem_file, "a") as dummy_file: + dummy_file.write("Hello World!") + #TODO: Just send appropriate clrjit*.dll files to workitem_directory # # Copy clrjit*_arch.dll binaries to workitem_directory # print('Copying clrjit_{}_{}.dll {} -> {}'.format(arch, product_directory, correlation_payload_directory)) From 1c1414693e47ef7e38b564c8b807e4ca5c2a5c90 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 4 Aug 2021 17:32:42 -0700 Subject: [PATCH 57/66] create workitem_directory --- src/coreclr/scripts/superpmi_replay_setup.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/coreclr/scripts/superpmi_replay_setup.py b/src/coreclr/scripts/superpmi_replay_setup.py index f948f65f49a951..6c1ad8f7cdfc5d 100644 --- a/src/coreclr/scripts/superpmi_replay_setup.py +++ b/src/coreclr/scripts/superpmi_replay_setup.py @@ -19,6 +19,7 @@ import argparse from os import path, walk +import os import shutil import stat import subprocess @@ -149,6 +150,8 @@ def main(main_args): print('Copying binaries {} -> {}'.format(arch, product_directory, correlation_payload_directory)) copy_directory(product_directory, correlation_payload_directory, match_func=match_correlation_files) + if not os.path.exists(workitem_directory): + os.makedirs(workitem_directory) dummy_workitem_file = path.join(workitem_directory, "dummy.txt") with open(dummy_workitem_file, "a") as dummy_file: dummy_file.write("Hello World!") From 8fbcacefc9abadb91b09a9af5c5471bc9e12a029 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 4 Aug 2021 18:47:07 -0700 Subject: [PATCH 58/66] Build clrjit --- eng/pipelines/coreclr/superpmi-replay.yml | 16 ++++++++-------- .../coreclr/templates/superpmi-replay-job.yml | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/eng/pipelines/coreclr/superpmi-replay.yml b/eng/pipelines/coreclr/superpmi-replay.yml index 5028926d930727..79c225648ce55b 100644 --- a/eng/pipelines/coreclr/superpmi-replay.yml +++ b/eng/pipelines/coreclr/superpmi-replay.yml @@ -14,14 +14,14 @@ pr: none jobs: -# - template: /eng/pipelines/common/platform-matrix.yml -# parameters: -# jobTemplate: /eng/pipelines/coreclr/templates/build-jit-job.yml -# buildConfig: checked -# uploadAsArtifacts: true -# platforms: -# - windows_x64 -# - windows_x86 +- template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/build-jit-job.yml + buildConfig: checked + uploadAsArtifacts: true + platforms: + - windows_x64 + - windows_x86 - template: /eng/pipelines/common/platform-matrix.yml parameters: diff --git a/eng/pipelines/coreclr/templates/superpmi-replay-job.yml b/eng/pipelines/coreclr/templates/superpmi-replay-job.yml index 3f667f1107e7a2..5afcf06bd946e0 100644 --- a/eng/pipelines/coreclr/templates/superpmi-replay-job.yml +++ b/eng/pipelines/coreclr/templates/superpmi-replay-job.yml @@ -35,7 +35,7 @@ jobs: steps: # Download jit builds - - template: /eng/pipelines/common/download-specific-artifact-step.yml + - template: /eng/pipelines/common/download-artifact-step.yml parameters: unpackFolder: $(buildProductRootFolderPath) artifactFileName: '$(buildProductArtifactName)$(archiveExtension)' From 5a3168db736a56a01ce18c796bd448c6b6a04e8a Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 4 Aug 2021 22:50:23 -0700 Subject: [PATCH 59/66] fix build-jit-job --- eng/pipelines/coreclr/templates/build-jit-job.yml | 4 ++-- src/coreclr/scripts/superpmi-replay.proj | 1 - src/coreclr/scripts/superpmi_replay_setup.py | 6 ------ 3 files changed, 2 insertions(+), 9 deletions(-) diff --git a/eng/pipelines/coreclr/templates/build-jit-job.yml b/eng/pipelines/coreclr/templates/build-jit-job.yml index 5d6e3e5985ee4c..805b4c9f2727ce 100644 --- a/eng/pipelines/coreclr/templates/build-jit-job.yml +++ b/eng/pipelines/coreclr/templates/build-jit-job.yml @@ -108,7 +108,7 @@ jobs: - script: set __TestIntermediateDir=int&&$(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) -ci -nopgooptimize -skiprestoreoptdata -component alljits displayName: Build CoreCLR JIT - - ${{ if eq(parameters.uploadAsArtifacts, 'false') }}: + - ${{ if eq(parameters.uploadAsArtifacts, false) }}: # Ensure the Python azure-storage-blob package is installed before doing the upload. - script: $(PipScript) install --user --upgrade pip && $(PipScript) install --user azure.storage.blob==12.5.0 --force-reinstall displayName: Upgrade Pip to latest and install azure-storage-blob Python package @@ -118,7 +118,7 @@ jobs: env: CLRJIT_AZ_KEY: $(clrjit_key1) # secret key stored as variable in pipeline - - ${{ if eq(parameters.uploadAsArtifacts, 'true') }}: + - ${{ if eq(parameters.uploadAsArtifacts, true) }}: # Publish product output directory for consumption by tests. - template: /eng/pipelines/common/upload-artifact-step.yml parameters: diff --git a/src/coreclr/scripts/superpmi-replay.proj b/src/coreclr/scripts/superpmi-replay.proj index 504f796846ef51..7b1d8b74f11473 100644 --- a/src/coreclr/scripts/superpmi-replay.proj +++ b/src/coreclr/scripts/superpmi-replay.proj @@ -58,7 +58,6 @@ $(WorkItemCommand) -arch %(HelixWorkItem.Architecture) -platform %(HelixWorkItem.Platform) -log_directory $(SuperpmiLogsLocation) - $(WorkItemDirectory) $(WorkItemTimeout) superpmi_%(HelixWorkItem.Platform)_%(HelixWorkItem.Architecture).log diff --git a/src/coreclr/scripts/superpmi_replay_setup.py b/src/coreclr/scripts/superpmi_replay_setup.py index 6c1ad8f7cdfc5d..e39a96544ebbc2 100644 --- a/src/coreclr/scripts/superpmi_replay_setup.py +++ b/src/coreclr/scripts/superpmi_replay_setup.py @@ -150,12 +150,6 @@ def main(main_args): print('Copying binaries {} -> {}'.format(arch, product_directory, correlation_payload_directory)) copy_directory(product_directory, correlation_payload_directory, match_func=match_correlation_files) - if not os.path.exists(workitem_directory): - os.makedirs(workitem_directory) - dummy_workitem_file = path.join(workitem_directory, "dummy.txt") - with open(dummy_workitem_file, "a") as dummy_file: - dummy_file.write("Hello World!") - #TODO: Just send appropriate clrjit*.dll files to workitem_directory # # Copy clrjit*_arch.dll binaries to workitem_directory # print('Copying clrjit_{}_{}.dll {} -> {}'.format(arch, product_directory, correlation_payload_directory)) From f796b6a6b27fe1cdf6ea976da4bc8350bfa387be Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 4 Aug 2021 22:51:55 -0700 Subject: [PATCH 60/66] add dependsOn --- eng/pipelines/coreclr/templates/superpmi-replay-job.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/coreclr/templates/superpmi-replay-job.yml b/eng/pipelines/coreclr/templates/superpmi-replay-job.yml index 5afcf06bd946e0..ab481ee6a78f3d 100644 --- a/eng/pipelines/coreclr/templates/superpmi-replay-job.yml +++ b/eng/pipelines/coreclr/templates/superpmi-replay-job.yml @@ -24,8 +24,8 @@ jobs: dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths }} timeoutInMinutes: ${{ parameters.timeoutInMinutes }} additionalSetupParameters: ${{ parameters.additionalSetupParameters }} - # dependsOn: - # - ${{ format('coreclr_jit_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + dependsOn: + - ${{ format('coreclr_jit_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} variables: ${{ parameters.variables }} From deeb6254cb9025eb590003b26645f6fb26c086b4 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 4 Aug 2021 23:06:05 -0700 Subject: [PATCH 61/66] print params --- eng/pipelines/coreclr/templates/build-jit-job.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/eng/pipelines/coreclr/templates/build-jit-job.yml b/eng/pipelines/coreclr/templates/build-jit-job.yml index 805b4c9f2727ce..22aec6312cf752 100644 --- a/eng/pipelines/coreclr/templates/build-jit-job.yml +++ b/eng/pipelines/coreclr/templates/build-jit-job.yml @@ -26,7 +26,6 @@ jobs: stagedBuild: ${{ parameters.stagedBuild }} pool: ${{ parameters.pool }} dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths }} - uploadAsArtifacts: ${{ parameters.uploadAsArtifacts }} # Compute job name from template parameters name: ${{ format('coreclr_jit_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} @@ -51,6 +50,8 @@ jobs: - name: publishLogsArtifactPrefix value: 'BuildLogs_CoreCLR_JIT' + - name: uploadAsArtifacts + value: ${{ parameters.uploadAsArtifacts }} - name: compilerArg value: '' @@ -80,6 +81,9 @@ jobs: - ${{ parameters.variables }} steps: + - script: | + echo $(uploadAsArtifacts) + echo ${{ parameters.uploadAsArtifacts }} # Install native dependencies # Linux builds use docker images with dependencies preinstalled, From faa8294643eb697cb6b1aa6aa15ae3622507d5d2 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 4 Aug 2021 23:06:41 -0700 Subject: [PATCH 62/66] TO REVERT: Just run jitbuild win-x64 --- eng/pipelines/coreclr/superpmi-replay.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/eng/pipelines/coreclr/superpmi-replay.yml b/eng/pipelines/coreclr/superpmi-replay.yml index 79c225648ce55b..217388c1763e5a 100644 --- a/eng/pipelines/coreclr/superpmi-replay.yml +++ b/eng/pipelines/coreclr/superpmi-replay.yml @@ -21,14 +21,14 @@ jobs: uploadAsArtifacts: true platforms: - windows_x64 - - windows_x86 + # - windows_x86 -- template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/superpmi-replay-job.yml - buildConfig: checked - platforms: - - windows_x64 - - windows_x86 - helixQueueGroup: ci - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml \ No newline at end of file +# - template: /eng/pipelines/common/platform-matrix.yml +# parameters: +# jobTemplate: /eng/pipelines/coreclr/templates/superpmi-replay-job.yml +# buildConfig: checked +# platforms: +# - windows_x64 +# - windows_x86 +# helixQueueGroup: ci +# helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml \ No newline at end of file From 7c73aa6f0ecb5392a1c3ce49b76d7c0165623955 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 4 Aug 2021 23:07:20 -0700 Subject: [PATCH 63/66] minor --- eng/pipelines/coreclr/templates/build-jit-job.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/coreclr/templates/build-jit-job.yml b/eng/pipelines/coreclr/templates/build-jit-job.yml index 22aec6312cf752..5bdd49b0451aba 100644 --- a/eng/pipelines/coreclr/templates/build-jit-job.yml +++ b/eng/pipelines/coreclr/templates/build-jit-job.yml @@ -82,8 +82,8 @@ jobs: steps: - script: | - echo $(uploadAsArtifacts) - echo ${{ parameters.uploadAsArtifacts }} + echo $(uploadAsArtifacts) + echo ${{ parameters.uploadAsArtifacts }} # Install native dependencies # Linux builds use docker images with dependencies preinstalled, From b06725faa0ac4d8acb17abef7287f7f56578e9bf Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 4 Aug 2021 23:34:47 -0700 Subject: [PATCH 64/66] move under jobParameters --- eng/pipelines/coreclr/superpmi-replay.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eng/pipelines/coreclr/superpmi-replay.yml b/eng/pipelines/coreclr/superpmi-replay.yml index 217388c1763e5a..22074115066904 100644 --- a/eng/pipelines/coreclr/superpmi-replay.yml +++ b/eng/pipelines/coreclr/superpmi-replay.yml @@ -18,10 +18,11 @@ jobs: parameters: jobTemplate: /eng/pipelines/coreclr/templates/build-jit-job.yml buildConfig: checked - uploadAsArtifacts: true platforms: - windows_x64 # - windows_x86 + jobParameters: + uploadAsArtifacts: true # - template: /eng/pipelines/common/platform-matrix.yml # parameters: From a7297948531d770698018d18cad958a6912e6f53 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 4 Aug 2021 23:50:43 -0700 Subject: [PATCH 65/66] Cleanup --- eng/pipelines/coreclr/superpmi-replay.yml | 20 ++++---- .../coreclr/templates/build-jit-job.yml | 3 -- src/coreclr/scripts/superpmi-replay.proj | 7 ++- src/coreclr/scripts/superpmi-replay.py | 37 ++++++++++---- src/coreclr/scripts/superpmi_replay_setup.py | 50 +++---------------- 5 files changed, 45 insertions(+), 72 deletions(-) diff --git a/eng/pipelines/coreclr/superpmi-replay.yml b/eng/pipelines/coreclr/superpmi-replay.yml index 22074115066904..bdb31881d1aab4 100644 --- a/eng/pipelines/coreclr/superpmi-replay.yml +++ b/eng/pipelines/coreclr/superpmi-replay.yml @@ -20,16 +20,16 @@ jobs: buildConfig: checked platforms: - windows_x64 - # - windows_x86 + - windows_x86 jobParameters: uploadAsArtifacts: true -# - template: /eng/pipelines/common/platform-matrix.yml -# parameters: -# jobTemplate: /eng/pipelines/coreclr/templates/superpmi-replay-job.yml -# buildConfig: checked -# platforms: -# - windows_x64 -# - windows_x86 -# helixQueueGroup: ci -# helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml \ No newline at end of file +- template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/superpmi-replay-job.yml + buildConfig: checked + platforms: + - windows_x64 + - windows_x86 + helixQueueGroup: ci + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml \ No newline at end of file diff --git a/eng/pipelines/coreclr/templates/build-jit-job.yml b/eng/pipelines/coreclr/templates/build-jit-job.yml index 5bdd49b0451aba..8992ee01df2f19 100644 --- a/eng/pipelines/coreclr/templates/build-jit-job.yml +++ b/eng/pipelines/coreclr/templates/build-jit-job.yml @@ -81,9 +81,6 @@ jobs: - ${{ parameters.variables }} steps: - - script: | - echo $(uploadAsArtifacts) - echo ${{ parameters.uploadAsArtifacts }} # Install native dependencies # Linux builds use docker images with dependencies preinstalled, diff --git a/src/coreclr/scripts/superpmi-replay.proj b/src/coreclr/scripts/superpmi-replay.proj index 7b1d8b74f11473..a44bc0889c2558 100644 --- a/src/coreclr/scripts/superpmi-replay.proj +++ b/src/coreclr/scripts/superpmi-replay.proj @@ -11,7 +11,6 @@ + - + diff --git a/src/coreclr/scripts/superpmi-replay.py b/src/coreclr/scripts/superpmi-replay.py index fb35559a8cd6b9..5b10e569de8510 100644 --- a/src/coreclr/scripts/superpmi-replay.py +++ b/src/coreclr/scripts/superpmi-replay.py @@ -1,3 +1,17 @@ +#!/usr/bin/env python3 +# +# Licensed to the .NET Foundation under one or more agreements. +# The .NET Foundation licenses this file to you under the MIT license. +# +## +# Title : superpmi_setup.py +# +# Notes: +# +# Script to run "superpmi replay" for various collections under various COMPlus_JitStressRegs value. +################################################################################ +################################################################################ + import argparse from os import path @@ -23,7 +37,8 @@ "JitStressRegs=0x10", "JitStressRegs=0x80", "JitStressRegs=0x1000", - ] +] + def setup_args(args): """ Setup the args for SuperPMI to use. @@ -60,6 +75,7 @@ def setup_args(args): return coreclr_args + def main(main_args): """Main entrypoint @@ -70,10 +86,8 @@ def main(main_args): python_path = sys.executable cwd = os.path.dirname(os.path.realpath(__file__)) coreclr_args = setup_args(main_args) - # mch_directory = coreclr_args.mch_directory spmi_location = path.join(cwd, "artifacts", "spmi") log_directory = coreclr_args.log_directory - mch_filename = '' platform_name = coreclr_args.platform os_name = "win" if platform_name.lower() == "windows" else "unix" arch_name = coreclr_args.arch @@ -81,21 +95,21 @@ def main(main_args): jit_path = path.join(coreclr_args.jit_directory, 'clrjit_{}_{}_{}.dll'.format(os_name, arch_name, host_arch_name)) print("Running superpmi.py download") - run_command([python_path, path.join(cwd, "superpmi.py"), "download", "--no_progress", "-f", "benchmark", "-target_os", platform_name, "-target_arch", arch_name, "-core_root", cwd, "-spmi_location", spmi_location], _exit_on_fail=True) + run_command([python_path, path.join(cwd, "superpmi.py"), "download", "--no_progress", "-target_os", platform_name, + "-target_arch", arch_name, "-core_root", cwd, "-spmi_location", spmi_location], _exit_on_fail=True) failed_runs = [] for jit_flag in jit_flags: - # TODO: This should be DownloadFilesFromResults log_file = path.join(log_directory, 'superpmi_{}.log'.format(jit_flag.replace("=", "_"))) print("Running superpmi.py replay for {}".format(jit_flag)) _, _, return_code = run_command([ - python_path, path.join(cwd, "superpmi.py"), "replay", "-core_root", cwd, - "-jitoption", jit_flag, "-jitoption", "TieredCompilation=0", - "-target_os", platform_name, "-target_arch", arch_name, - "-arch", host_arch_name, - "-jit_path", jit_path, "-spmi_location", spmi_location, - "-log_level", "debug", "-log_file", log_file]) + python_path, path.join(cwd, "superpmi.py"), "replay", "-core_root", cwd, + "-jitoption", jit_flag, "-jitoption", "TieredCompilation=0", + "-target_os", platform_name, "-target_arch", arch_name, + "-arch", host_arch_name, + "-jit_path", jit_path, "-spmi_location", spmi_location, + "-log_level", "debug", "-log_file", log_file]) if return_code != 0: failed_runs.append("Failure in {}".format(log_file)) @@ -116,6 +130,7 @@ def main(main_args): contents = current_superpmi_log.read() final_superpmi_log.write(contents) + # Log failures summary if len(failed_runs) > 0: final_superpmi_log.write(os.linesep) final_superpmi_log.write(os.linesep) diff --git a/src/coreclr/scripts/superpmi_replay_setup.py b/src/coreclr/scripts/superpmi_replay_setup.py index e39a96544ebbc2..34cc8301fc5fd4 100644 --- a/src/coreclr/scripts/superpmi_replay_setup.py +++ b/src/coreclr/scripts/superpmi_replay_setup.py @@ -9,11 +9,7 @@ # Notes: # # Script to setup directory structure required to perform SuperPMI replay in CI. -# It does the following steps: -# 1. It creates `correlation_payload_directory` that contains clrjit*_x64.dll and clrjit*_x86.dll -# 2. The script takes `input_artifacts` parameter which contains *.mch.zip and *.mct.zip. It will then -# partition it by moving each pair of *.mch.zip/*.mct.zip into its own folder under 'payload' -# directory. +# It creates `correlation_payload_directory` that contains clrjit*_x64.dll and clrjit*_x86.dll ################################################################################ ################################################################################ @@ -32,10 +28,8 @@ parser = argparse.ArgumentParser(description="description") parser.add_argument("-arch", help="Architecture") -# parser.add_argument("-platform", help="OS platform") parser.add_argument("-source_directory", help="path to the directory containing binaries") parser.add_argument("-product_directory", help="path to the directory containing binaries") -# parser.add_argument("-mch_directory", help="path to directory containing compressed mch files") def setup_args(args): @@ -56,11 +50,6 @@ def setup_args(args): lambda unused: True, "Unable to set arch") - # coreclr_args.verify(args, - # "platform", - # lambda unused: True, - # "Unable to set platform") - coreclr_args.verify(args, "source_directory", lambda source_directory: os.path.isdir(source_directory), @@ -71,10 +60,6 @@ def setup_args(args): lambda product_directory: os.path.isdir(product_directory), "product_directory doesn't exist") - # coreclr_args.verify(args, - # "mch_directory", - # lambda mch_directory: True, #os.path.isdir(mch_directory), - # "mch_directory doesn't exist") return coreclr_args @@ -117,6 +102,7 @@ def match_correlation_files(full_path): return False + def main(main_args): """Main entrypoint @@ -128,15 +114,11 @@ def main(main_args): arch = coreclr_args.arch source_directory = coreclr_args.source_directory product_directory = coreclr_args.product_directory - # mch_directory = coreclr_args.mch_directory # CorrelationPayload directories correlation_payload_directory = path.join(coreclr_args.source_directory, "payload") superpmi_src_directory = path.join(source_directory, 'src', 'coreclr', 'scripts') - # Workitem directories - workitem_directory = path.join(source_directory, "workitem") - helix_source_prefix = "official" creator = "" ci = True @@ -144,42 +126,22 @@ def main(main_args): # Copy *.py to CorrelationPayload print('Copying {} -> {}'.format(superpmi_src_directory, correlation_payload_directory)) - copy_directory(superpmi_src_directory, correlation_payload_directory, match_func=lambda path: any(path.endswith(extension) for extension in [".py"])) - + copy_directory(superpmi_src_directory, correlation_payload_directory, + match_func=lambda path: any(path.endswith(extension) for extension in [".py"])) + # Copy clrjit*_arch.dll binaries to CorrelationPayload print('Copying binaries {} -> {}'.format(arch, product_directory, correlation_payload_directory)) copy_directory(product_directory, correlation_payload_directory, match_func=match_correlation_files) - #TODO: Just send appropriate clrjit*.dll files to workitem_directory - # # Copy clrjit*_arch.dll binaries to workitem_directory - # print('Copying clrjit_{}_{}.dll {} -> {}'.format(arch, product_directory, correlation_payload_directory)) - # copy_directory(product_directory, correlation_payload_directory, match_func=match_correlation_files) - - # Partition mch/mct zip files - # partition_mch(mch_directory, workitem_directory) - - # # Print correlation_payload_directory and workitem_directory - # print("==> correlation_payload_directory:") - # for file_path, dirs, files in walk(correlation_payload_directory, topdown=True): - # for name in files: - # curr_file_path = path.join(file_path, name) - # print(curr_file_path) - - # print("==> workitem_directory:") - # for file_path, dirs, files in walk(workitem_directory, topdown=True): - # for name in files: - # curr_file_path = path.join(file_path, name) - # print(curr_file_path) - # Set variables print('Setting pipeline variables:') set_pipeline_variable("CorrelationPayloadDirectory", correlation_payload_directory) - set_pipeline_variable("WorkItemDirectory", workitem_directory) set_pipeline_variable("Architecture", arch) set_pipeline_variable("Creator", creator) set_pipeline_variable("Queue", helix_queue) set_pipeline_variable("HelixSourcePrefix", helix_source_prefix) + if __name__ == "__main__": args = parser.parse_args() sys.exit(main(args)) From c8dc88a27e81ea6f20ede7689c9215f9a66eef57 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 11 Aug 2021 11:26:25 -0700 Subject: [PATCH 66/66] review feedback --- .../common/download-specific-artifact-step.yml | 4 ++-- eng/pipelines/coreclr/superpmi-replay.yml | 2 +- eng/pipelines/coreclr/templates/build-jit-job.yml | 10 +++++----- eng/pipelines/coreclr/templates/run-superpmi-job.yml | 1 - .../coreclr/templates/run-superpmi-replay-job.yml | 1 - eng/pipelines/coreclr/templates/superpmi-job.yml | 1 - 6 files changed, 8 insertions(+), 11 deletions(-) diff --git a/eng/pipelines/common/download-specific-artifact-step.yml b/eng/pipelines/common/download-specific-artifact-step.yml index ae1fe1e32b4950..f3d1bf7d0f52c4 100644 --- a/eng/pipelines/common/download-specific-artifact-step.yml +++ b/eng/pipelines/common/download-specific-artifact-step.yml @@ -11,10 +11,10 @@ steps: displayName: 'Download specific ${{ parameters.displayName }}' inputs: buildType: specific - project: 'internal' + project: 'internal' # 'internal' or 'public' pipeline: 'superpmi-replay' buildVersionToDownload: specific - branchName: 'kpathak/spmi-replay' + branchName: 'your/branch/having/artifacts' buildId: '1274841' downloadType: single downloadPath: '$(Build.SourcesDirectory)/__download__' diff --git a/eng/pipelines/coreclr/superpmi-replay.yml b/eng/pipelines/coreclr/superpmi-replay.yml index bdb31881d1aab4..83b4d2ac6237b9 100644 --- a/eng/pipelines/coreclr/superpmi-replay.yml +++ b/eng/pipelines/coreclr/superpmi-replay.yml @@ -22,7 +22,7 @@ jobs: - windows_x64 - windows_x86 jobParameters: - uploadAsArtifacts: true + uploadAs: 'pipelineArtifacts' - template: /eng/pipelines/common/platform-matrix.yml parameters: diff --git a/eng/pipelines/coreclr/templates/build-jit-job.yml b/eng/pipelines/coreclr/templates/build-jit-job.yml index 8992ee01df2f19..5ac3f54679d862 100644 --- a/eng/pipelines/coreclr/templates/build-jit-job.yml +++ b/eng/pipelines/coreclr/templates/build-jit-job.yml @@ -11,7 +11,7 @@ parameters: timeoutInMinutes: '' variables: {} dependOnEvaluatePaths: false - uploadAsArtifacts: false + uploadAs: 'azureBlob' ### Product build jobs: @@ -50,8 +50,8 @@ jobs: - name: publishLogsArtifactPrefix value: 'BuildLogs_CoreCLR_JIT' - - name: uploadAsArtifacts - value: ${{ parameters.uploadAsArtifacts }} + - name: uploadAs + value: ${{ parameters.uploadAs }} - name: compilerArg value: '' @@ -109,7 +109,7 @@ jobs: - script: set __TestIntermediateDir=int&&$(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) -ci -nopgooptimize -skiprestoreoptdata -component alljits displayName: Build CoreCLR JIT - - ${{ if eq(parameters.uploadAsArtifacts, false) }}: + - ${{ if eq(parameters.uploadAs, 'azureBlob') }}: # Ensure the Python azure-storage-blob package is installed before doing the upload. - script: $(PipScript) install --user --upgrade pip && $(PipScript) install --user azure.storage.blob==12.5.0 --force-reinstall displayName: Upgrade Pip to latest and install azure-storage-blob Python package @@ -119,7 +119,7 @@ jobs: env: CLRJIT_AZ_KEY: $(clrjit_key1) # secret key stored as variable in pipeline - - ${{ if eq(parameters.uploadAsArtifacts, true) }}: + - ${{ if eq(parameters.uploadAs, 'pipelineArtifacts') }}: # Publish product output directory for consumption by tests. - template: /eng/pipelines/common/upload-artifact-step.yml parameters: diff --git a/eng/pipelines/coreclr/templates/run-superpmi-job.yml b/eng/pipelines/coreclr/templates/run-superpmi-job.yml index 649646bd928d1f..78a4e778c733f2 100644 --- a/eng/pipelines/coreclr/templates/run-superpmi-job.yml +++ b/eng/pipelines/coreclr/templates/run-superpmi-job.yml @@ -18,7 +18,6 @@ parameters: liveLibrariesBuildConfig: '' # optional -- live-live libraries configuration to use for the run runtimeType: 'coreclr' # optional -- Sets the runtime as coreclr or mono codeGenType: 'JIT' # optional -- Decides on the codegen technology if running on mono - projectFile: 'superpmi.proj' # optional -- project file to build helix workitems runKind: '' # required -- test category collectionType: '' collectionName: '' diff --git a/eng/pipelines/coreclr/templates/run-superpmi-replay-job.yml b/eng/pipelines/coreclr/templates/run-superpmi-replay-job.yml index e74e3946da3311..f267329a86760d 100644 --- a/eng/pipelines/coreclr/templates/run-superpmi-replay-job.yml +++ b/eng/pipelines/coreclr/templates/run-superpmi-replay-job.yml @@ -18,7 +18,6 @@ parameters: liveLibrariesBuildConfig: '' # optional -- live-live libraries configuration to use for the run runtimeType: 'coreclr' # optional -- Sets the runtime as coreclr or mono codeGenType: 'JIT' # optional -- Decides on the codegen technology if running on mono - projectFile: 'superpmi.proj' # optional -- project file to build helix workitems runKind: '' # required -- test category collectionType: '' collectionName: '' diff --git a/eng/pipelines/coreclr/templates/superpmi-job.yml b/eng/pipelines/coreclr/templates/superpmi-job.yml index 133dabafdfe3c5..c51ca723c9e602 100644 --- a/eng/pipelines/coreclr/templates/superpmi-job.yml +++ b/eng/pipelines/coreclr/templates/superpmi-job.yml @@ -39,7 +39,6 @@ jobs: liveLibrariesBuildConfig: ${{ parameters.liveLibrariesBuildConfig }} runtimeType: ${{ parameters.runtimeType }} codeGenType: ${{ parameters.codeGenType }} - projectFile: ${{ parameters.projectFile }} runKind: ${{ parameters.runKind }} testGroup: ${{ parameters.testGroup }} collectionType: ${{ parameters.collectionType }}