From 38aeb2c1f8dce4825b783a25c56a90349496795b Mon Sep 17 00:00:00 2001 From: "Scott Beddall (from Dev Box)" Date: Fri, 19 Jul 2024 15:20:48 -0700 Subject: [PATCH 001/119] update venv reference methodology --- .../steps/build-package-artifacts.yml | 28 +++---------------- eng/scripts/create-venv.ps1 | 1 + 2 files changed, 5 insertions(+), 24 deletions(-) diff --git a/eng/pipelines/templates/steps/build-package-artifacts.yml b/eng/pipelines/templates/steps/build-package-artifacts.yml index 5b88ab4c2c39..05a21121d000 100644 --- a/eng/pipelines/templates/steps/build-package-artifacts.yml +++ b/eng/pipelines/templates/steps/build-package-artifacts.yml @@ -57,21 +57,12 @@ steps: VirtualEnvironmentName: "venv" Activate: false - - bash: | - source $(VENV_LOCATION)/bin/activate - which python - python -m pip install --force -r eng/ci_tools.txt - python -m pip freeze --all - displayName: 'Prep Environment Linux/Mac' - condition: and(succeeded(), or(eq(variables['Agent.OS'], 'Linux'), eq(variables['Agent.OS'], 'Darwin'))) - - pwsh: | - . $(VENV_LOCATION)/Scripts/Activate.ps1 + $(VENV_ACTIVATION_SCRIPT) which python python -m pip install --force -r eng/ci_tools.txt python -m pip freeze --all - displayName: 'Prep Environment Windows' - condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) + displayName: 'Prep Environment' - template: set-dev-build.yml@self parameters: @@ -96,22 +87,11 @@ steps: displayName: 'Install QEMU Dependencies' condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux')) - - bash: | - source $(VENV_LOCATION)/bin/activate - which python - sdk_build -d "$(Build.ArtifactStagingDirectory)" "$(TargetingString)" --service=${{parameters.ServiceDirectory}} --inactive - displayName: 'Generate Packages Linux/Mac' - condition: and(succeeded(), or(eq(variables['Agent.OS'], 'Linux'), eq(variables['Agent.OS'], 'Darwin'))) - timeoutInMinutes: 80 - env: - CIBW_BUILD_VERBOSITY: 3 - - pwsh: | - . $(VENV_LOCATION)/Scripts/Activate.ps1 + $(VENV_ACTIVATION_SCRIPT) which python sdk_build -d "$(Build.ArtifactStagingDirectory)" "$(TargetingString)" --service=${{parameters.ServiceDirectory}} --inactive - displayName: 'Generate Packages Windows' - condition: and(succeededOrFailed(), eq(variables['Agent.OS'], 'Windows_NT')) + displayName: 'Generate Packages Linux/Mac' timeoutInMinutes: 80 env: CIBW_BUILD_VERBOSITY: 3 diff --git a/eng/scripts/create-venv.ps1 b/eng/scripts/create-venv.ps1 index fa4e9a54357c..919b3ba968fa 100644 --- a/eng/scripts/create-venv.ps1 +++ b/eng/scripts/create-venv.ps1 @@ -27,6 +27,7 @@ if (!(Test-Path $venvPath)) { python -m virtualenv "$venvPath" Write-Host "Virtual environment '$VenvName' created." Write-Host "##vso[task.setvariable variable=$($VenvName)_LOCATION]$venvPath" + Write-Host "##vso[task.setvariable variable=$($VenvName)_ACTIVATION_SCRIPT]if(`$IsWindows){. $venvPath/Scripts/Activate.ps1;}else {. $venvPath/bin/activate.ps1}" } else { Write-Host "Virtual environment '$VenvName' already exists. Skipping creation." From 8a2b75c46c09a880dcf0ce0b538c828b48172ea8 Mon Sep 17 00:00:00 2001 From: "Scott Beddall (from Dev Box)" Date: Fri, 19 Jul 2024 15:22:00 -0700 Subject: [PATCH 002/119] remove platform from build step --- eng/pipelines/templates/steps/build-package-artifacts.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/steps/build-package-artifacts.yml b/eng/pipelines/templates/steps/build-package-artifacts.yml index 05a21121d000..80a0e49f1b11 100644 --- a/eng/pipelines/templates/steps/build-package-artifacts.yml +++ b/eng/pipelines/templates/steps/build-package-artifacts.yml @@ -91,7 +91,7 @@ steps: $(VENV_ACTIVATION_SCRIPT) which python sdk_build -d "$(Build.ArtifactStagingDirectory)" "$(TargetingString)" --service=${{parameters.ServiceDirectory}} --inactive - displayName: 'Generate Packages Linux/Mac' + displayName: 'Generate Packages' timeoutInMinutes: 80 env: CIBW_BUILD_VERBOSITY: 3 From 1c8ddf8f33ff9eb8050140db3f07e68c2c4469ac Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Mon, 22 Jul 2024 13:18:56 -0700 Subject: [PATCH 003/119] a _bunch_ of build changes. this is to get our builds all running the same way --- eng/pipelines/templates/jobs/ci.yml | 111 +++--------------- .../templates/stages/archetype-sdk-client.yml | 4 - .../steps/build-package-artifacts.yml | 12 +- eng/pipelines/templates/steps/build.yml | 36 ++++++ .../steps/resolve-build-platforms.yml | 15 +++ eng/pipelines/templates/steps/use-venv.yml | 5 + sdk/storage/ci.yml | 1 - 7 files changed, 80 insertions(+), 104 deletions(-) create mode 100644 eng/pipelines/templates/steps/build.yml create mode 100644 eng/pipelines/templates/steps/resolve-build-platforms.yml diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index 91f32e7c45f7..388303537e2a 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -61,34 +61,8 @@ parameters: - name: GenerateApiReviewForManualOnly type: boolean default: false - - name: AdvancedBuild - type: boolean - default: false jobs: - - - ${{ if eq(parameters['AdvancedBuild'], false) }}: - - job: 'Build' - timeoutInMinutes: 90 - - pool: - name: $(LINUXPOOL) - image: $(LINUXVMIMAGE) - os: linux - - steps: - - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml - parameters: - BuildTargetingString: ${{ parameters.BuildTargetingString }} - - - template: ../steps/build-package-artifacts.yml - parameters: - ServiceDirectory: ${{ parameters.ServiceDirectory }} - BeforePublishSteps: ${{ parameters.BeforePublishSteps }} - TestPipeline: ${{ parameters.TestPipeline }} - Artifacts: ${{ parameters.Artifacts }} - - - ${{ if eq(parameters['AdvancedBuild'], true) }}: - job: 'Build_Linux' timeoutInMinutes: 90 @@ -98,18 +72,14 @@ jobs: os: linux steps: - - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml - parameters: - BuildTargetingString: ${{ parameters.BuildTargetingString }} - - - template: ../steps/build-package-artifacts.yml + - template: /eng/pipelines/templates/steps/build.yml parameters: ServiceDirectory: ${{ parameters.ServiceDirectory }} + BuildTargetingString: ${{ parameters.BuildTargetingString }} BeforePublishSteps: ${{ parameters.BeforePublishSteps }} TestPipeline: ${{ parameters.TestPipeline }} - Artifacts: ${{ parameters.Artifacts }} ArtifactSuffix: linux - UseVirtualEnvironment: false + Artifacts: ${{ parameters.Artifacts }} - job: 'Build_Windows' timeoutInMinutes: 90 @@ -120,18 +90,14 @@ jobs: os: windows steps: - - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml - parameters: - BuildTargetingString: ${{ parameters.BuildTargetingString }} - - - template: ../steps/build-package-artifacts.yml + - template: /eng/pipelines/templates/steps/build.yml parameters: ServiceDirectory: ${{ parameters.ServiceDirectory }} + BuildTargetingString: ${{ parameters.BuildTargetingString }} BeforePublishSteps: ${{ parameters.BeforePublishSteps }} TestPipeline: ${{ parameters.TestPipeline }} - Artifacts: ${{ parameters.Artifacts }} ArtifactSuffix: windows - UseVirtualEnvironment: false + Artifacts: ${{ parameters.Artifacts }} - job: 'Build_MacOS' timeoutInMinutes: 90 @@ -142,60 +108,21 @@ jobs: os: macOS steps: - - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml - parameters: - BuildTargetingString: ${{ parameters.BuildTargetingString }} - - - template: ../steps/build-package-artifacts.yml + - template: /eng/pipelines/templates/steps/build.yml parameters: ServiceDirectory: ${{ parameters.ServiceDirectory }} + BuildTargetingString: ${{ parameters.BuildTargetingString }} BeforePublishSteps: ${{ parameters.BeforePublishSteps }} TestPipeline: ${{ parameters.TestPipeline }} - Artifacts: ${{ parameters.Artifacts }} ArtifactSuffix: mac - UseVirtualEnvironment: false - - - job: 'CoalesceBuildArtifacts' - displayName: Combine Built Artifacts - dependsOn: - - 'Build_Linux' - - 'Build_Windows' - - 'Build_MacOS' - timeoutInMinutes: 90 - - pool: - name: $(LINUXPOOL) - image: $(LINUXVMIMAGE) - os: linux - - steps: - - task: DownloadPipelineArtifact@2 - inputs: - artifactName: 'packages_windows' - targetPath: $(Build.ArtifactStagingDirectory)/packages - - - task: DownloadPipelineArtifact@2 - inputs: - artifactName: 'packages_mac' - targetPath: $(Build.ArtifactStagingDirectory)/packages - - - task: DownloadPipelineArtifact@2 - inputs: - artifactName: 'packages_linux' - targetPath: $(Build.ArtifactStagingDirectory)/packages - - - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml - parameters: - ArtifactPath: '$(Build.ArtifactStagingDirectory)/packages' - ArtifactName: 'packages' + Artifacts: ${{ parameters.Artifacts }} - job: 'Build_Extended' displayName: Build Extended dependsOn: - - ${{ if eq(parameters['AdvancedBuild'], true) }}: - - 'CoalesceBuildArtifacts' - - ${{ else }}: - - 'Build' + - 'Build_Linux' + - 'Build_Windows' + - 'Build_MacOS' timeoutInMinutes: 90 @@ -272,10 +199,9 @@ jobs: OsVmImage: azsdk-pool-mms-ubuntu-2004-1espt Pool: azsdk-pool-mms-ubuntu-2004-general DependsOn: - - ${{ if eq(parameters['AdvancedBuild'], true) }}: - - 'CoalesceBuildArtifacts' - - ${{ else }}: - - 'Build' + - 'Build_Linux' + - 'Build_Windows' + - 'Build_MacOS' MatrixConfigs: ${{ parameters.MatrixConfigs }} MatrixFilters: ${{ parameters.MatrixFilters }} MatrixReplace: ${{ parameters.MatrixReplace }} @@ -327,10 +253,9 @@ jobs: CloudConfig: Cloud: Public DependsOn: - - ${{ if eq(parameters['AdvancedBuild'], true) }}: - - 'CoalesceBuildArtifacts' - - ${{ else }}: - - 'Build' + - 'Build_Linux' + - 'Build_Windows' + - 'Build_MacOS' AdditionalParameters: BuildTargetingString: ${{ parameters.BuildTargetingString }} ServiceDirectory: ${{ parameters.ServiceDirectory }} diff --git a/eng/pipelines/templates/stages/archetype-sdk-client.yml b/eng/pipelines/templates/stages/archetype-sdk-client.yml index 0e0d435be3c6..3f264af14dd9 100644 --- a/eng/pipelines/templates/stages/archetype-sdk-client.yml +++ b/eng/pipelines/templates/stages/archetype-sdk-client.yml @@ -72,9 +72,6 @@ parameters: - name: GenerateApiReviewForManualOnly type: boolean default: false - - name: AdvancedBuild - type: boolean - default: false - name: oneESTemplateTag type: string default: release @@ -113,7 +110,6 @@ extends: ValidateFormatting: ${{ parameters.ValidateFormatting }} TestProxy: ${{ parameters.TestProxy }} GenerateApiReviewForManualOnly: ${{ parameters.GenerateApiReviewForManualOnly }} - AdvancedBuild: ${{ parameters.AdvancedBuild }} variables: - template: /eng/pipelines/templates/variables/globals.yml diff --git a/eng/pipelines/templates/steps/build-package-artifacts.yml b/eng/pipelines/templates/steps/build-package-artifacts.yml index 80a0e49f1b11..8fb5a908018b 100644 --- a/eng/pipelines/templates/steps/build-package-artifacts.yml +++ b/eng/pipelines/templates/steps/build-package-artifacts.yml @@ -17,25 +17,25 @@ parameters: - name: ArtifactSuffix type: string default: '' - - name: UseVirtualEnvironment - type: boolean - default: true # The variable TargetingString is set by template `eng/pipelines/templates/steps/targeting-string-resolve.yml`. This template is invoked from yml files: # eng/pipelines/templates/jobs/ci.tests.yml # eng/pipelines/templates/jobs/ci.yml # eng/pipelines/templates/jobs/live.test.yml -# Please use `$(TargetingString)` to refer to the python packages glob string. This was previously `${{ parameters.BuildTargetingString }}`. +# Please use `$(TargetingString)` to refer to the python packages glob string. This was previously `${{ parameters.BuildTargetingString }}`. This variable +# will be updated by PR diff detection as well. -# The variable '$(ArtifactName)' is set at runtime by the strategy for the `Build` job. +# The parameter 'ArtifactSuffix' is set at compile time by the strategy for the `Build` job. # Specifically, one of 3 values `linux`, `mac`, or `windows` will always be populated when invoked from `jobs/ci.yml`. # However, please note that this variable will not ALWAYS be set. If using `build-artifacts.yml`, ensure that this variable is set AT LEAST # to "linux", otherwise the primary tasks will not be run as expected. APIStub, Docs, etc are NOT run on machines that aren't linux. + steps: - task: UsePythonVersion@0 displayName: 'Use Python $(PythonVersion)' + condition: and(succeeded(), $()) inputs: versionSpec: $(PythonVersion) @@ -78,7 +78,7 @@ steps: pwsh: true workingDirectory: $(Pipeline.Workspace) displayName: Update package properties with dev version - condition: and(succeeded(),eq(variables['SetDevVersion'],'true')) + condition: and(succeeded(), eq(variables['SetDevVersion'],'true')) - script: | sudo apt-get update diff --git a/eng/pipelines/templates/steps/build.yml b/eng/pipelines/templates/steps/build.yml new file mode 100644 index 000000000000..f43911bf543c --- /dev/null +++ b/eng/pipelines/templates/steps/build.yml @@ -0,0 +1,36 @@ +parameters: + - name: ServiceDirectory + type: string + default: '' + - name: BuildTargetingString + type: string + default: 'azure-*' + - name: BeforePublishSteps + type: object + default: [] + - name: TestPipeline + type: boolean + default: false + - name: ArtifactSuffix + type: string + default: '' + - name: Artifacts + type: object + default: [] + +steps: + - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml + parameters: + BuildTargetingString: ${{ parameters.BuildTargetingString }} + + - template: /eng/pipelines/templates/steps/resolve-build-platforms.yml + parameters: + ServiceDirectory: ${{ parameters.ServiceDirectory }} + + - template: ../steps/build-package-artifacts.yml + parameters: + ServiceDirectory: ${{ parameters.ServiceDirectory }} + BeforePublishSteps: ${{ parameters.BeforePublishSteps }} + TestPipeline: ${{ parameters.TestPipeline }} + Artifacts: ${{ parameters.Artifacts }} + ArtifactSuffix: ${{ parameters.ArtifactSuffix }} diff --git a/eng/pipelines/templates/steps/resolve-build-platforms.yml b/eng/pipelines/templates/steps/resolve-build-platforms.yml new file mode 100644 index 000000000000..569a80cdccdb --- /dev/null +++ b/eng/pipelines/templates/steps/resolve-build-platforms.yml @@ -0,0 +1,15 @@ +parameters: + - name: ServiceDirectory + type: string + default: '' + +steps: + # when we merge pipeline v3, this check will change to examining the targeting string $(TargetingString) + # as the generate-pr-diff call + resolution will be present in targeting-string-resolve.yml. + # until then, we simply check to see if we're targeting storage service directory + - pwsh: | + if ("${{ parameters.ServiceDirectory }}" -eq "storage") { + Write-Host "Targeting storage, enabling extension build." + Write-Host "##vso[task.setvariable variable=ENABLE_EXTENSION_BUILD]true" + } + displayName: Check extension package presence diff --git a/eng/pipelines/templates/steps/use-venv.yml b/eng/pipelines/templates/steps/use-venv.yml index 65d0b0d0b565..0a0defa16b3f 100644 --- a/eng/pipelines/templates/steps/use-venv.yml +++ b/eng/pipelines/templates/steps/use-venv.yml @@ -5,6 +5,9 @@ parameters: - name: Activate type: boolean default: true + - name: Condition + type: boolean + default: succeeded() steps: - pwsh: | @@ -12,6 +15,7 @@ steps: -VenvName "${{ parameters.VirtualEnvironmentName }}" ` -RepoRoot "$(Build.SourcesDirectory)" displayName: Create virtual environment + condition: ${{ parameters.Condition }} - ${{ if eq(parameters.Activate, true) }}: - pwsh: | @@ -19,3 +23,4 @@ steps: -VenvName "${{ parameters.VirtualEnvironmentName }}" ` -RepoRoot "$(Build.SourcesDirectory)" displayName: Use ${{ parameters.VirtualEnvironmentName }} Virtual Environment + condition: ${{ parameters.Condition }} diff --git a/sdk/storage/ci.yml b/sdk/storage/ci.yml index f35fcab9966a..a22b506e4ad1 100644 --- a/sdk/storage/ci.yml +++ b/sdk/storage/ci.yml @@ -36,7 +36,6 @@ extends: Path: eng/pipelines/templates/stages/platform-matrix-excluding-pypy.json Selection: sparse GenerateVMJobs: true - AdvancedBuild: true Artifacts: - name: azure-storage-blob safeName: azurestorageblob From 354213e1c59254dad590713b6d046c9253e17768 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Mon, 22 Jul 2024 13:20:49 -0700 Subject: [PATCH 004/119] ok we aren't honoring the conditionals, but we can test the actual build runs. --- eng/pipelines/templates/jobs/ci.tests.yml | 14 ++++++++++++-- eng/pipelines/templates/jobs/ci.yml | 15 +++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/templates/jobs/ci.tests.yml b/eng/pipelines/templates/jobs/ci.tests.yml index 806d7d0cade7..f2141b4a5a16 100644 --- a/eng/pipelines/templates/jobs/ci.tests.yml +++ b/eng/pipelines/templates/jobs/ci.tests.yml @@ -124,8 +124,18 @@ jobs: BeforeTestSteps: - task: DownloadPipelineArtifact@2 inputs: - artifactName: 'packages' - targetPath: $(Build.ArtifactStagingDirectory) + artifactName: 'packages_windows' + targetPath: $(Build.ArtifactStagingDirectory)/packages + + - task: DownloadPipelineArtifact@2 + inputs: + artifactName: 'packages_mac' + targetPath: $(Build.ArtifactStagingDirectory)/packages + + - task: DownloadPipelineArtifact@2 + inputs: + artifactName: 'packages_linux' + targetPath: $(Build.ArtifactStagingDirectory)/packages - template: ../steps/set-dev-build.yml parameters: diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index 388303537e2a..c0ce8c7c0cf8 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -132,6 +132,21 @@ jobs: os: linux steps: + - task: DownloadPipelineArtifact@2 + inputs: + artifactName: 'packages_windows' + targetPath: $(Build.ArtifactStagingDirectory)/packages + + - task: DownloadPipelineArtifact@2 + inputs: + artifactName: 'packages_mac' + targetPath: $(Build.ArtifactStagingDirectory)/packages + + - task: DownloadPipelineArtifact@2 + inputs: + artifactName: 'packages_linux' + targetPath: $(Build.ArtifactStagingDirectory)/packages + - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml parameters: BuildTargetingString: ${{ parameters.BuildTargetingString }} From d84253d3f35491cf1f204cfbcfc8c15b5ede9625 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Mon, 22 Jul 2024 13:31:16 -0700 Subject: [PATCH 005/119] update indentation --- eng/pipelines/templates/jobs/ci.yml | 90 ++++++++++++++--------------- 1 file changed, 45 insertions(+), 45 deletions(-) diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index c0ce8c7c0cf8..8db954259ad5 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -63,59 +63,59 @@ parameters: default: false jobs: - - job: 'Build_Linux' - timeoutInMinutes: 90 + - job: 'Build_Linux' + timeoutInMinutes: 90 - pool: - name: $(LINUXPOOL) - image: $(LINUXVMIMAGE) - os: linux + pool: + name: $(LINUXPOOL) + image: $(LINUXVMIMAGE) + os: linux - steps: - - template: /eng/pipelines/templates/steps/build.yml - parameters: - ServiceDirectory: ${{ parameters.ServiceDirectory }} - BuildTargetingString: ${{ parameters.BuildTargetingString }} - BeforePublishSteps: ${{ parameters.BeforePublishSteps }} - TestPipeline: ${{ parameters.TestPipeline }} - ArtifactSuffix: linux - Artifacts: ${{ parameters.Artifacts }} + steps: + - template: /eng/pipelines/templates/steps/build.yml + parameters: + ServiceDirectory: ${{ parameters.ServiceDirectory }} + BuildTargetingString: ${{ parameters.BuildTargetingString }} + BeforePublishSteps: ${{ parameters.BeforePublishSteps }} + TestPipeline: ${{ parameters.TestPipeline }} + ArtifactSuffix: linux + Artifacts: ${{ parameters.Artifacts }} - - job: 'Build_Windows' - timeoutInMinutes: 90 + - job: 'Build_Windows' + timeoutInMinutes: 90 - pool: - name: $(WINDOWSPOOL) - image: $(WINDOWSVMIMAGE) - os: windows + pool: + name: $(WINDOWSPOOL) + image: $(WINDOWSVMIMAGE) + os: windows - steps: - - template: /eng/pipelines/templates/steps/build.yml - parameters: - ServiceDirectory: ${{ parameters.ServiceDirectory }} - BuildTargetingString: ${{ parameters.BuildTargetingString }} - BeforePublishSteps: ${{ parameters.BeforePublishSteps }} - TestPipeline: ${{ parameters.TestPipeline }} - ArtifactSuffix: windows - Artifacts: ${{ parameters.Artifacts }} + steps: + - template: /eng/pipelines/templates/steps/build.yml + parameters: + ServiceDirectory: ${{ parameters.ServiceDirectory }} + BuildTargetingString: ${{ parameters.BuildTargetingString }} + BeforePublishSteps: ${{ parameters.BeforePublishSteps }} + TestPipeline: ${{ parameters.TestPipeline }} + ArtifactSuffix: windows + Artifacts: ${{ parameters.Artifacts }} - - job: 'Build_MacOS' - timeoutInMinutes: 90 + - job: 'Build_MacOS' + timeoutInMinutes: 90 - pool: - name: $(MACPOOL) - vmImage: $(MACVMIMAGE) - os: macOS + pool: + name: $(MACPOOL) + vmImage: $(MACVMIMAGE) + os: macOS - steps: - - template: /eng/pipelines/templates/steps/build.yml - parameters: - ServiceDirectory: ${{ parameters.ServiceDirectory }} - BuildTargetingString: ${{ parameters.BuildTargetingString }} - BeforePublishSteps: ${{ parameters.BeforePublishSteps }} - TestPipeline: ${{ parameters.TestPipeline }} - ArtifactSuffix: mac - Artifacts: ${{ parameters.Artifacts }} + steps: + - template: /eng/pipelines/templates/steps/build.yml + parameters: + ServiceDirectory: ${{ parameters.ServiceDirectory }} + BuildTargetingString: ${{ parameters.BuildTargetingString }} + BeforePublishSteps: ${{ parameters.BeforePublishSteps }} + TestPipeline: ${{ parameters.TestPipeline }} + ArtifactSuffix: mac + Artifacts: ${{ parameters.Artifacts }} - job: 'Build_Extended' displayName: Build Extended From 0442b241b84897ab15feedfafc0f1261f62bb744 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Mon, 22 Jul 2024 13:34:30 -0700 Subject: [PATCH 006/119] update condition usage in use-venv --- eng/pipelines/templates/steps/use-venv.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/steps/use-venv.yml b/eng/pipelines/templates/steps/use-venv.yml index 0a0defa16b3f..a25459c7a156 100644 --- a/eng/pipelines/templates/steps/use-venv.yml +++ b/eng/pipelines/templates/steps/use-venv.yml @@ -6,7 +6,7 @@ parameters: type: boolean default: true - name: Condition - type: boolean + type: string default: succeeded() steps: From 2cf398e43e701ad7b5641ddd2be67e9774309e98 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Mon, 22 Jul 2024 13:38:59 -0700 Subject: [PATCH 007/119] remove wonky condition --- eng/pipelines/templates/steps/build-package-artifacts.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/eng/pipelines/templates/steps/build-package-artifacts.yml b/eng/pipelines/templates/steps/build-package-artifacts.yml index 8fb5a908018b..e875d064af81 100644 --- a/eng/pipelines/templates/steps/build-package-artifacts.yml +++ b/eng/pipelines/templates/steps/build-package-artifacts.yml @@ -35,7 +35,6 @@ parameters: steps: - task: UsePythonVersion@0 displayName: 'Use Python $(PythonVersion)' - condition: and(succeeded(), $()) inputs: versionSpec: $(PythonVersion) From c0ffa9c1dd0dc4d25608a3e1994cdfcfa3d826f8 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Mon, 22 Jul 2024 14:19:36 -0700 Subject: [PATCH 008/119] adjust packages usage in build-extended-artifacts --- eng/pipelines/templates/steps/build-extended-artifacts.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/eng/pipelines/templates/steps/build-extended-artifacts.yml b/eng/pipelines/templates/steps/build-extended-artifacts.yml index 7e68a218e1a2..5cfdb56e2a48 100644 --- a/eng/pipelines/templates/steps/build-extended-artifacts.yml +++ b/eng/pipelines/templates/steps/build-extended-artifacts.yml @@ -24,11 +24,6 @@ parameters: # Please use `$(TargetingString)` to refer to the python packages glob string. This was previously `${{ parameters.BuildTargetingString }}`. steps: - - task: DownloadPipelineArtifact@2 - inputs: - artifactName: 'packages' - targetPath: $(Build.ArtifactStagingDirectory) - - task: UsePythonVersion@0 displayName: 'Use Python 3.11' inputs: From 1a7dcc12fcdab127e70306cca90f4f1d8001b69e Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Mon, 22 Jul 2024 17:39:21 -0700 Subject: [PATCH 009/119] why is this thing having such a hard time processing getting corehttp package properties? --- eng/pipelines/templates/steps/build-extended-artifacts.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/eng/pipelines/templates/steps/build-extended-artifacts.yml b/eng/pipelines/templates/steps/build-extended-artifacts.yml index 5cfdb56e2a48..db3d2634e12c 100644 --- a/eng/pipelines/templates/steps/build-extended-artifacts.yml +++ b/eng/pipelines/templates/steps/build-extended-artifacts.yml @@ -57,6 +57,9 @@ steps: - template: auth-dev-feed.yml parameters: DevFeedName: ${{ parameters.DevFeedName }} + - pwsh: | + Get-ChildItem -R "$(Build.ArtifactStagingDirectory)" | % { Write-Host $_.FullName } + displayName: display visible files - task: PythonScript@0 displayName: 'Generate Docs' From 425b744003e0458b2bc61c4fbcdaa2609553ed17 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Mon, 22 Jul 2024 18:23:56 -0700 Subject: [PATCH 010/119] waiting for confirmation, but pretty certain this will work --- eng/pipelines/templates/jobs/ci.tests.yml | 15 +-------------- eng/pipelines/templates/jobs/ci.yml | 15 +-------------- .../steps/download-package-artifacts.yml | 15 +++++++++++++++ 3 files changed, 17 insertions(+), 28 deletions(-) create mode 100644 eng/pipelines/templates/steps/download-package-artifacts.yml diff --git a/eng/pipelines/templates/jobs/ci.tests.yml b/eng/pipelines/templates/jobs/ci.tests.yml index f2141b4a5a16..7219acb71eb3 100644 --- a/eng/pipelines/templates/jobs/ci.tests.yml +++ b/eng/pipelines/templates/jobs/ci.tests.yml @@ -122,20 +122,7 @@ jobs: InjectedPackages: $(InjectedPackages) TestProxy: ${{ parameters.TestProxy }} BeforeTestSteps: - - task: DownloadPipelineArtifact@2 - inputs: - artifactName: 'packages_windows' - targetPath: $(Build.ArtifactStagingDirectory)/packages - - - task: DownloadPipelineArtifact@2 - inputs: - artifactName: 'packages_mac' - targetPath: $(Build.ArtifactStagingDirectory)/packages - - - task: DownloadPipelineArtifact@2 - inputs: - artifactName: 'packages_linux' - targetPath: $(Build.ArtifactStagingDirectory)/packages + - template: /eng/pipelines/templates/steps/download-package-artifacts.yml - template: ../steps/set-dev-build.yml parameters: diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index 8db954259ad5..0cae4afb350d 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -132,20 +132,7 @@ jobs: os: linux steps: - - task: DownloadPipelineArtifact@2 - inputs: - artifactName: 'packages_windows' - targetPath: $(Build.ArtifactStagingDirectory)/packages - - - task: DownloadPipelineArtifact@2 - inputs: - artifactName: 'packages_mac' - targetPath: $(Build.ArtifactStagingDirectory)/packages - - - task: DownloadPipelineArtifact@2 - inputs: - artifactName: 'packages_linux' - targetPath: $(Build.ArtifactStagingDirectory)/packages + - template: /eng/pipelines/templates/steps/download-package-artifacts.yml - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml parameters: diff --git a/eng/pipelines/templates/steps/download-package-artifacts.yml b/eng/pipelines/templates/steps/download-package-artifacts.yml new file mode 100644 index 000000000000..01a0361af845 --- /dev/null +++ b/eng/pipelines/templates/steps/download-package-artifacts.yml @@ -0,0 +1,15 @@ +steps: + - task: DownloadPipelineArtifact@2 + inputs: + artifactName: 'packages_windows' + targetPath: $(Build.ArtifactStagingDirectory) + + - task: DownloadPipelineArtifact@2 + inputs: + artifactName: 'packages_mac' + targetPath: $(Build.ArtifactStagingDirectory) + + - task: DownloadPipelineArtifact@2 + inputs: + artifactName: 'packages_linux' + targetPath: $(Build.ArtifactStagingDirectory) From 871e66225c8c57f40ae8163e1c6f783fde484cb2 Mon Sep 17 00:00:00 2001 From: Scott Beddall <45376673+scbedd@users.noreply.github.com> Date: Tue, 23 Jul 2024 13:07:09 -0700 Subject: [PATCH 011/119] Update eng/pipelines/templates/steps/build.yml Co-authored-by: Wes Haggard --- eng/pipelines/templates/steps/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/steps/build.yml b/eng/pipelines/templates/steps/build.yml index f43911bf543c..6dd0391d6155 100644 --- a/eng/pipelines/templates/steps/build.yml +++ b/eng/pipelines/templates/steps/build.yml @@ -27,7 +27,7 @@ steps: parameters: ServiceDirectory: ${{ parameters.ServiceDirectory }} - - template: ../steps/build-package-artifacts.yml + - template: /eng/pipelines/templates/steps/build-package-artifacts.yml parameters: ServiceDirectory: ${{ parameters.ServiceDirectory }} BeforePublishSteps: ${{ parameters.BeforePublishSteps }} From 0c0f06da5024a483e0393f0a01bbc6e5e336341a Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 23 Jul 2024 15:14:23 -0700 Subject: [PATCH 012/119] update the conditions in build-package-artifacts. the intent is to save effort where possible --- .../steps/build-package-artifacts.yml | 22 +++++++++---------- .../templates/steps/set-dev-build.yml | 5 +++-- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/eng/pipelines/templates/steps/build-package-artifacts.yml b/eng/pipelines/templates/steps/build-package-artifacts.yml index e875d064af81..9750b10b49a6 100644 --- a/eng/pipelines/templates/steps/build-package-artifacts.yml +++ b/eng/pipelines/templates/steps/build-package-artifacts.yml @@ -16,25 +16,19 @@ parameters: default: 'public/azure-sdk-for-python' - name: ArtifactSuffix type: string - default: '' + default: 'linux' -# The variable TargetingString is set by template `eng/pipelines/templates/steps/targeting-string-resolve.yml`. This template is invoked from yml files: -# eng/pipelines/templates/jobs/ci.tests.yml -# eng/pipelines/templates/jobs/ci.yml -# eng/pipelines/templates/jobs/live.test.yml +# The variable TargetingString is set by template `eng/pipelines/templates/steps/targeting-string-resolve.yml`. This template is invoked wherever +# package targeting needs to be resolved prior to doing any actual work, whether it be build, static analysis, or test. # Please use `$(TargetingString)` to refer to the python packages glob string. This was previously `${{ parameters.BuildTargetingString }}`. This variable # will be updated by PR diff detection as well. -# The parameter 'ArtifactSuffix' is set at compile time by the strategy for the `Build` job. -# Specifically, one of 3 values `linux`, `mac`, or `windows` will always be populated when invoked from `jobs/ci.yml`. - -# However, please note that this variable will not ALWAYS be set. If using `build-artifacts.yml`, ensure that this variable is set AT LEAST -# to "linux", otherwise the primary tasks will not be run as expected. APIStub, Docs, etc are NOT run on machines that aren't linux. - +# In addition, the variable ENABLE_EXTENSION_BUILD is honored to enable builds on windows and mac if necessary. steps: - task: UsePythonVersion@0 displayName: 'Use Python $(PythonVersion)' + condition: and(succeeded(), or(eq(variables['ENABLE_EXTENSION_BUILD', 'true'), eq('${{ parameters.ArtifactSuffix }}', 'linux'))) inputs: versionSpec: $(PythonVersion) @@ -55,6 +49,7 @@ steps: parameters: VirtualEnvironmentName: "venv" Activate: false + Condition: and(succeeded(), or(eq(variables['ENABLE_EXTENSION_BUILD', 'true'), eq('${{ parameters.ArtifactSuffix }}', 'linux'))) - pwsh: | $(VENV_ACTIVATION_SCRIPT) @@ -62,10 +57,12 @@ steps: python -m pip install --force -r eng/ci_tools.txt python -m pip freeze --all displayName: 'Prep Environment' + condition: and(succeeded(), or(eq(variables['ENABLE_EXTENSION_BUILD', 'true'), eq('${{ parameters.ArtifactSuffix }}', 'linux'))) - template: set-dev-build.yml@self parameters: ServiceDirectory: ${{ parameters.ServiceDirectory }} + CustomCondition: and(succeeded(), or(eq(variables['ENABLE_EXTENSION_BUILD', 'true'), eq('${{ parameters.ArtifactSuffix }}', 'linux'))) - task: Powershell@2 inputs: @@ -77,7 +74,7 @@ steps: pwsh: true workingDirectory: $(Pipeline.Workspace) displayName: Update package properties with dev version - condition: and(succeeded(), eq(variables['SetDevVersion'],'true')) + condition: and(succeeded(), eq(variables['SetDevVersion'],'true'), or(eq(variables['ENABLE_EXTENSION_BUILD', 'true'), eq('${{ parameters.ArtifactSuffix }}', 'linux'))) - script: | sudo apt-get update @@ -91,6 +88,7 @@ steps: which python sdk_build -d "$(Build.ArtifactStagingDirectory)" "$(TargetingString)" --service=${{parameters.ServiceDirectory}} --inactive displayName: 'Generate Packages' + condition: and(succeeded(), or(eq(variables['ENABLE_EXTENSION_BUILD', 'true'), eq('${{ parameters.ArtifactSuffix }}', 'linux'))) timeoutInMinutes: 80 env: CIBW_BUILD_VERBOSITY: 3 diff --git a/eng/pipelines/templates/steps/set-dev-build.yml b/eng/pipelines/templates/steps/set-dev-build.yml index 3a5761b4c6e7..be9cdd08d85e 100644 --- a/eng/pipelines/templates/steps/set-dev-build.yml +++ b/eng/pipelines/templates/steps/set-dev-build.yml @@ -1,5 +1,6 @@ parameters: ServiceDirectory: '' + CustomCondition: 'succeededOrFailed()' # The variable TargetingString is set by template `eng/pipelines/templates/steps/targeting-string-resolve.yml`. This template is invoked from yml files: # eng/pipelines/templates/jobs/ci.tests.yml @@ -11,10 +12,10 @@ steps: - template: /eng/common/pipelines/templates/steps/daily-dev-build-variable.yml parameters: ServiceDirectory: ${{ parameters.ServiceDirectory }} - Condition: succeededOrFailed() + Condition: ${{ parameters.CustomCondition }} - pwsh: | python -m pip install "tools/azure-sdk-tools[build]" sdk_set_dev_version "$(TargetingString)" --service="${{parameters.ServiceDirectory}}" --build-id="$(Build.BuildNumber)" displayName: "Update package versions for dev build" - condition: and(succeededOrFailed(), eq(variables['SetDevVersion'],'true')) + condition: and(succeededOrFailed(), eq(variables['SetDevVersion'],'true'), {{ parameters.CustomCondition }}) From b3584298136704f964ec80850028c2b2592adbd1 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 23 Jul 2024 15:27:01 -0700 Subject: [PATCH 013/119] correct all the variable references --- .../templates/steps/build-package-artifacts.yml | 12 ++++++------ eng/pipelines/templates/steps/set-dev-build.yml | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/eng/pipelines/templates/steps/build-package-artifacts.yml b/eng/pipelines/templates/steps/build-package-artifacts.yml index 9750b10b49a6..257c91eaa066 100644 --- a/eng/pipelines/templates/steps/build-package-artifacts.yml +++ b/eng/pipelines/templates/steps/build-package-artifacts.yml @@ -28,7 +28,7 @@ parameters: steps: - task: UsePythonVersion@0 displayName: 'Use Python $(PythonVersion)' - condition: and(succeeded(), or(eq(variables['ENABLE_EXTENSION_BUILD', 'true'), eq('${{ parameters.ArtifactSuffix }}', 'linux'))) + condition: and(succeeded(), or(eq(variables['ENABLE_EXTENSION_BUILD'], 'true'), eq('${{ parameters.ArtifactSuffix }}', 'linux'))) inputs: versionSpec: $(PythonVersion) @@ -49,7 +49,7 @@ steps: parameters: VirtualEnvironmentName: "venv" Activate: false - Condition: and(succeeded(), or(eq(variables['ENABLE_EXTENSION_BUILD', 'true'), eq('${{ parameters.ArtifactSuffix }}', 'linux'))) + Condition: and(succeeded(), or(eq(variables['ENABLE_EXTENSION_BUILD'], 'true'), eq('${{ parameters.ArtifactSuffix }}', 'linux'))) - pwsh: | $(VENV_ACTIVATION_SCRIPT) @@ -57,12 +57,12 @@ steps: python -m pip install --force -r eng/ci_tools.txt python -m pip freeze --all displayName: 'Prep Environment' - condition: and(succeeded(), or(eq(variables['ENABLE_EXTENSION_BUILD', 'true'), eq('${{ parameters.ArtifactSuffix }}', 'linux'))) + condition: and(succeeded(), or(eq(variables['ENABLE_EXTENSION_BUILD'], 'true'), eq('${{ parameters.ArtifactSuffix }}', 'linux'))) - template: set-dev-build.yml@self parameters: ServiceDirectory: ${{ parameters.ServiceDirectory }} - CustomCondition: and(succeeded(), or(eq(variables['ENABLE_EXTENSION_BUILD', 'true'), eq('${{ parameters.ArtifactSuffix }}', 'linux'))) + Condition: and(succeeded(), or(eq(variables['ENABLE_EXTENSION_BUILD'], 'true'), eq('${{ parameters.ArtifactSuffix }}', 'linux'))) - task: Powershell@2 inputs: @@ -74,7 +74,7 @@ steps: pwsh: true workingDirectory: $(Pipeline.Workspace) displayName: Update package properties with dev version - condition: and(succeeded(), eq(variables['SetDevVersion'],'true'), or(eq(variables['ENABLE_EXTENSION_BUILD', 'true'), eq('${{ parameters.ArtifactSuffix }}', 'linux'))) + condition: and(succeeded(), eq(variables['SetDevVersion'],'true'), or(eq(variables['ENABLE_EXTENSION_BUILD'], 'true'), eq('${{ parameters.ArtifactSuffix }}', 'linux'))) - script: | sudo apt-get update @@ -88,7 +88,7 @@ steps: which python sdk_build -d "$(Build.ArtifactStagingDirectory)" "$(TargetingString)" --service=${{parameters.ServiceDirectory}} --inactive displayName: 'Generate Packages' - condition: and(succeeded(), or(eq(variables['ENABLE_EXTENSION_BUILD', 'true'), eq('${{ parameters.ArtifactSuffix }}', 'linux'))) + condition: and(succeeded(), or(eq(variables['ENABLE_EXTENSION_BUILD'], 'true'), eq('${{ parameters.ArtifactSuffix }}', 'linux'))) timeoutInMinutes: 80 env: CIBW_BUILD_VERBOSITY: 3 diff --git a/eng/pipelines/templates/steps/set-dev-build.yml b/eng/pipelines/templates/steps/set-dev-build.yml index be9cdd08d85e..241e9137a760 100644 --- a/eng/pipelines/templates/steps/set-dev-build.yml +++ b/eng/pipelines/templates/steps/set-dev-build.yml @@ -1,6 +1,6 @@ parameters: ServiceDirectory: '' - CustomCondition: 'succeededOrFailed()' + Condition: 'succeededOrFailed()' # The variable TargetingString is set by template `eng/pipelines/templates/steps/targeting-string-resolve.yml`. This template is invoked from yml files: # eng/pipelines/templates/jobs/ci.tests.yml @@ -12,10 +12,10 @@ steps: - template: /eng/common/pipelines/templates/steps/daily-dev-build-variable.yml parameters: ServiceDirectory: ${{ parameters.ServiceDirectory }} - Condition: ${{ parameters.CustomCondition }} + Condition: ${{ parameters.Condition }} - pwsh: | python -m pip install "tools/azure-sdk-tools[build]" sdk_set_dev_version "$(TargetingString)" --service="${{parameters.ServiceDirectory}}" --build-id="$(Build.BuildNumber)" displayName: "Update package versions for dev build" - condition: and(succeededOrFailed(), eq(variables['SetDevVersion'],'true'), {{ parameters.CustomCondition }}) + condition: and(succeededOrFailed(), eq(variables['SetDevVersion'],'true'), {{ parameters.Condition }}) From 79f95f7dabde6d76493b82eec4ee9785dfea92bb Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 23 Jul 2024 15:29:23 -0700 Subject: [PATCH 014/119] correct condition in set dev build --- eng/pipelines/templates/steps/set-dev-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/steps/set-dev-build.yml b/eng/pipelines/templates/steps/set-dev-build.yml index 241e9137a760..07de80a0bf78 100644 --- a/eng/pipelines/templates/steps/set-dev-build.yml +++ b/eng/pipelines/templates/steps/set-dev-build.yml @@ -18,4 +18,4 @@ steps: python -m pip install "tools/azure-sdk-tools[build]" sdk_set_dev_version "$(TargetingString)" --service="${{parameters.ServiceDirectory}}" --build-id="$(Build.BuildNumber)" displayName: "Update package versions for dev build" - condition: and(succeededOrFailed(), eq(variables['SetDevVersion'],'true'), {{ parameters.Condition }}) + condition: and(succeededOrFailed(), eq(variables['SetDevVersion'],'true'), ${{ parameters.Condition }}) From 9753eb802ec3be770c53888999c5eeb585b83bc5 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 23 Jul 2024 16:08:49 -0700 Subject: [PATCH 015/119] simplify the conditional --- eng/pipelines/templates/steps/build-package-artifacts.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/eng/pipelines/templates/steps/build-package-artifacts.yml b/eng/pipelines/templates/steps/build-package-artifacts.yml index 257c91eaa066..41c7f4186327 100644 --- a/eng/pipelines/templates/steps/build-package-artifacts.yml +++ b/eng/pipelines/templates/steps/build-package-artifacts.yml @@ -102,6 +102,13 @@ steps: - ${{ parameters.BeforePublishSteps }} + # we need to publish an empty artifact. when publishing an empty artifact, let's ensure that there isn't anything to + # cause CG or SBOM generation to have to do actual work. + - pwsh: | + Remove-Item -Force -Recurse $(Build.SourcesDirectory) + displayName: Clean up repo + condition: and(succeeded(), ne(variables['ENABLE_EXTENSION_BUILD'], 'true'), ne('${{ parameters.ArtifactSuffix }}', 'linux')) + - ${{ if eq(parameters.ArtifactSuffix, '') }}: - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml parameters: From 7ff3b894f4b7686d5c2c27a7e807fdc0bfa7ab5c Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 23 Jul 2024 16:17:45 -0700 Subject: [PATCH 016/119] cant remove a directory we're operating in --- eng/pipelines/templates/steps/build-package-artifacts.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/steps/build-package-artifacts.yml b/eng/pipelines/templates/steps/build-package-artifacts.yml index 41c7f4186327..1207cbca2b6b 100644 --- a/eng/pipelines/templates/steps/build-package-artifacts.yml +++ b/eng/pipelines/templates/steps/build-package-artifacts.yml @@ -105,7 +105,7 @@ steps: # we need to publish an empty artifact. when publishing an empty artifact, let's ensure that there isn't anything to # cause CG or SBOM generation to have to do actual work. - pwsh: | - Remove-Item -Force -Recurse $(Build.SourcesDirectory) + Remove-Item -Force -Recurse $(Build.SourcesDirectory)/* displayName: Clean up repo condition: and(succeeded(), ne(variables['ENABLE_EXTENSION_BUILD'], 'true'), ne('${{ parameters.ArtifactSuffix }}', 'linux')) From 105edfe01415c968f78fafd8ca96525f8a87dff0 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 23 Jul 2024 16:27:32 -0700 Subject: [PATCH 017/119] leave behind the credscan suppression file though --- eng/pipelines/templates/steps/build-package-artifacts.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/eng/pipelines/templates/steps/build-package-artifacts.yml b/eng/pipelines/templates/steps/build-package-artifacts.yml index 1207cbca2b6b..21ac1ce06bdf 100644 --- a/eng/pipelines/templates/steps/build-package-artifacts.yml +++ b/eng/pipelines/templates/steps/build-package-artifacts.yml @@ -105,7 +105,9 @@ steps: # we need to publish an empty artifact. when publishing an empty artifact, let's ensure that there isn't anything to # cause CG or SBOM generation to have to do actual work. - pwsh: | - Remove-Item -Force -Recurse $(Build.SourcesDirectory)/* + Get-ChildItem -Path "$(Build.SourcesDirectory)" -Recurse ` + | Where-Object { $_.FullName -ne "$(Build.SourcesDirectory)/eng/CredScanSuppression.json" } ` + | Remove-Item -Force -Recurse displayName: Clean up repo condition: and(succeeded(), ne(variables['ENABLE_EXTENSION_BUILD'], 'true'), ne('${{ parameters.ArtifactSuffix }}', 'linux')) From 2ba740d5137a1eaa0b5f4d260f395fb848cad53a Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 23 Jul 2024 16:42:46 -0700 Subject: [PATCH 018/119] clean up all the files in the repo --- .../templates/steps/build-package-artifacts.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/eng/pipelines/templates/steps/build-package-artifacts.yml b/eng/pipelines/templates/steps/build-package-artifacts.yml index 21ac1ce06bdf..5a0fe9d80222 100644 --- a/eng/pipelines/templates/steps/build-package-artifacts.yml +++ b/eng/pipelines/templates/steps/build-package-artifacts.yml @@ -105,9 +105,17 @@ steps: # we need to publish an empty artifact. when publishing an empty artifact, let's ensure that there isn't anything to # cause CG or SBOM generation to have to do actual work. - pwsh: | + Write-Host "BEFORE" + + Get-ChildItem -R $(Build.SourcesDirectory) | % { Write-Host $_.FullName } + Get-ChildItem -Path "$(Build.SourcesDirectory)" -Recurse ` - | Where-Object { $_.FullName -ne "$(Build.SourcesDirectory)/eng/CredScanSuppression.json" } ` + | Where-Object { !($_.Name.EndsWith("CredScanSuppression.json")) -and ($_.PSIsContainer -ne $true) } ` | Remove-Item -Force -Recurse + + Write-Host "AFTER" + + Get-ChildItem -R $(Build.SourcesDirectory) | % { Write-Host $_.FullName } displayName: Clean up repo condition: and(succeeded(), ne(variables['ENABLE_EXTENSION_BUILD'], 'true'), ne('${{ parameters.ArtifactSuffix }}', 'linux')) From c205cb69070178b97f746892b8ee5f6798081f97 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 23 Jul 2024 17:06:19 -0700 Subject: [PATCH 019/119] buncha updates! --- eng/pipelines/templates/jobs/regression.yml | 7 ++----- .../stages/archetype-python-release.yml | 19 +++++++++++++------ .../steps/download-package-artifacts.yml | 9 ++++++--- 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/eng/pipelines/templates/jobs/regression.yml b/eng/pipelines/templates/jobs/regression.yml index 2fbf460bd7e7..8c3b559b48c7 100644 --- a/eng/pipelines/templates/jobs/regression.yml +++ b/eng/pipelines/templates/jobs/regression.yml @@ -79,13 +79,10 @@ jobs: displayName: 'Use Python 3.9' inputs: versionSpec: '3.9' - + - template: /eng/pipelines/templates/steps/use-venv.yml - - task: DownloadPipelineArtifact@2 - inputs: - artifactName: 'packages' - targetPath: $(Build.ArtifactStagingDirectory) + - template: /eng/pipelines/templates/steps/download-package-artifacts.yml - pwsh: | mkdir -p $(TEST_PROXY_FOLDER) diff --git a/eng/pipelines/templates/stages/archetype-python-release.yml b/eng/pipelines/templates/stages/archetype-python-release.yml index b45f0d861803..4fcf014658a2 100644 --- a/eng/pipelines/templates/stages/archetype-python-release.yml +++ b/eng/pipelines/templates/stages/archetype-python-release.yml @@ -106,9 +106,10 @@ stages: deploy: steps: - checkout: self - - download: current - artifact: ${{parameters.ArtifactName}} - timeoutInMinutes: 5 + + - template: /eng/pipelines/templates/steps/download-package-artifacts.yml + parameters: + TargetPath: $(Pipeline.Workspace) - task: UsePythonVersion@0 inputs: @@ -229,7 +230,9 @@ stages: Paths: - sdk/**/*.md - .github/CODEOWNERS - - download: current + - template: /eng/pipelines/templates/steps/download-package-artifacts.yml + parameters: + TargetPath: $(Pipeline.Workspace) - template: /eng/common/pipelines/templates/steps/update-docsms-metadata.yml parameters: PackageInfoLocations: @@ -295,7 +298,9 @@ stages: os: linux steps: - checkout: none - - download: current + - template: /eng/pipelines/templates/steps/download-package-artifacts.yml + parameters: + TargetPath: $(Pipeline.Workspace) artifact: ${{parameters.ArtifactName}} timeoutInMinutes: 5 - task: UsePythonVersion@0 @@ -338,7 +343,9 @@ stages: Paths: - sdk/**/*.md - .github/CODEOWNERS - - download: current + - template: /eng/pipelines/templates/steps/download-package-artifacts.yml + parameters: + TargetPath: $(Pipeline.Workspace) - pwsh: | Get-ChildItem -Recurse $(Pipeline.Workspace)/${{parameters.ArtifactName}}/ displayName: Show visible artifacts diff --git a/eng/pipelines/templates/steps/download-package-artifacts.yml b/eng/pipelines/templates/steps/download-package-artifacts.yml index 01a0361af845..c9d46d35bae8 100644 --- a/eng/pipelines/templates/steps/download-package-artifacts.yml +++ b/eng/pipelines/templates/steps/download-package-artifacts.yml @@ -1,15 +1,18 @@ +parameters: + TargetPath: $(Build.ArtifactStagingDirectory) + steps: - task: DownloadPipelineArtifact@2 inputs: artifactName: 'packages_windows' - targetPath: $(Build.ArtifactStagingDirectory) + targetPath: ${{ parameters.TargetPath }} - task: DownloadPipelineArtifact@2 inputs: artifactName: 'packages_mac' - targetPath: $(Build.ArtifactStagingDirectory) + targetPath: ${{ parameters.TargetPath }} - task: DownloadPipelineArtifact@2 inputs: artifactName: 'packages_linux' - targetPath: $(Build.ArtifactStagingDirectory) + targetPath: ${{ parameters.TargetPath }} From 873ffd6b41d4cfb1d0d2c62b484bdf48b6f1594d Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 23 Jul 2024 17:57:08 -0700 Subject: [PATCH 020/119] clean up the outputs in build-package-artifacts --- eng/pipelines/templates/steps/build-package-artifacts.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/eng/pipelines/templates/steps/build-package-artifacts.yml b/eng/pipelines/templates/steps/build-package-artifacts.yml index 5a0fe9d80222..4f0fd376c3be 100644 --- a/eng/pipelines/templates/steps/build-package-artifacts.yml +++ b/eng/pipelines/templates/steps/build-package-artifacts.yml @@ -105,17 +105,9 @@ steps: # we need to publish an empty artifact. when publishing an empty artifact, let's ensure that there isn't anything to # cause CG or SBOM generation to have to do actual work. - pwsh: | - Write-Host "BEFORE" - - Get-ChildItem -R $(Build.SourcesDirectory) | % { Write-Host $_.FullName } - Get-ChildItem -Path "$(Build.SourcesDirectory)" -Recurse ` | Where-Object { !($_.Name.EndsWith("CredScanSuppression.json")) -and ($_.PSIsContainer -ne $true) } ` | Remove-Item -Force -Recurse - - Write-Host "AFTER" - - Get-ChildItem -R $(Build.SourcesDirectory) | % { Write-Host $_.FullName } displayName: Clean up repo condition: and(succeeded(), ne(variables['ENABLE_EXTENSION_BUILD'], 'true'), ne('${{ parameters.ArtifactSuffix }}', 'linux')) From 56ec6834b766dbdccaff041c92f6977aa01b0925 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 23 Jul 2024 18:25:47 -0700 Subject: [PATCH 021/119] fix the artifact download call --- eng/pipelines/templates/stages/archetype-python-release.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/eng/pipelines/templates/stages/archetype-python-release.yml b/eng/pipelines/templates/stages/archetype-python-release.yml index 4fcf014658a2..06711cceccd5 100644 --- a/eng/pipelines/templates/stages/archetype-python-release.yml +++ b/eng/pipelines/templates/stages/archetype-python-release.yml @@ -301,8 +301,6 @@ stages: - template: /eng/pipelines/templates/steps/download-package-artifacts.yml parameters: TargetPath: $(Pipeline.Workspace) - artifact: ${{parameters.ArtifactName}} - timeoutInMinutes: 5 - task: UsePythonVersion@0 - script: | set -e From 387d3d078aa8a612da70e379339d54fc5a73cdca Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Wed, 24 Jul 2024 10:58:17 -0700 Subject: [PATCH 022/119] restore the python release stages to what they were. my changes were not required --- .../stages/archetype-python-release.yml | 21 +++++++------------ .../steps/build-extended-artifacts.yml | 3 --- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/eng/pipelines/templates/stages/archetype-python-release.yml b/eng/pipelines/templates/stages/archetype-python-release.yml index 06711cceccd5..3009acce29cd 100644 --- a/eng/pipelines/templates/stages/archetype-python-release.yml +++ b/eng/pipelines/templates/stages/archetype-python-release.yml @@ -107,9 +107,9 @@ stages: steps: - checkout: self - - template: /eng/pipelines/templates/steps/download-package-artifacts.yml - parameters: - TargetPath: $(Pipeline.Workspace) + - download: current + artifact: ${{parameters.ArtifactName}} + timeoutInMinutes: 5 - task: UsePythonVersion@0 inputs: @@ -230,9 +230,7 @@ stages: Paths: - sdk/**/*.md - .github/CODEOWNERS - - template: /eng/pipelines/templates/steps/download-package-artifacts.yml - parameters: - TargetPath: $(Pipeline.Workspace) + - download: current - template: /eng/common/pipelines/templates/steps/update-docsms-metadata.yml parameters: PackageInfoLocations: @@ -297,10 +295,9 @@ stages: name: azsdk-pool-mms-ubuntu-2004-general os: linux steps: - - checkout: none - - template: /eng/pipelines/templates/steps/download-package-artifacts.yml - parameters: - TargetPath: $(Pipeline.Workspace) + - download: current + artifact: ${{parameters.ArtifactName}} + timeoutInMinutes: 5 - task: UsePythonVersion@0 - script: | set -e @@ -341,9 +338,7 @@ stages: Paths: - sdk/**/*.md - .github/CODEOWNERS - - template: /eng/pipelines/templates/steps/download-package-artifacts.yml - parameters: - TargetPath: $(Pipeline.Workspace) + - download: current - pwsh: | Get-ChildItem -Recurse $(Pipeline.Workspace)/${{parameters.ArtifactName}}/ displayName: Show visible artifacts diff --git a/eng/pipelines/templates/steps/build-extended-artifacts.yml b/eng/pipelines/templates/steps/build-extended-artifacts.yml index db3d2634e12c..5cfdb56e2a48 100644 --- a/eng/pipelines/templates/steps/build-extended-artifacts.yml +++ b/eng/pipelines/templates/steps/build-extended-artifacts.yml @@ -57,9 +57,6 @@ steps: - template: auth-dev-feed.yml parameters: DevFeedName: ${{ parameters.DevFeedName }} - - pwsh: | - Get-ChildItem -R "$(Build.ArtifactStagingDirectory)" | % { Write-Host $_.FullName } - displayName: display visible files - task: PythonScript@0 displayName: 'Generate Docs' From c3ac400498a9baaf86d922b1d5783f15edd9e180 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Wed, 24 Jul 2024 18:54:11 -0700 Subject: [PATCH 023/119] rationalizing our pipeline v3 work into the new simplified build. very close to being ready to go. got a lot of simplifying to work through --- eng/pipelines/templates/jobs/ci.tests.yml | 17 +++-- eng/pipelines/templates/jobs/ci.yml | 5 ++ .../steps/build-extended-artifacts.yml | 2 - .../steps/build-package-artifacts.yml | 75 +++++++++++++++++-- eng/pipelines/templates/steps/build-test.yml | 6 -- eng/pipelines/templates/steps/build.yml | 9 +-- .../steps/resolve-package-targeting.yml | 38 ++++++++++ .../templates/steps/set-dev-build.yml | 14 +--- .../steps/targeting-string-resolve.yml | 65 ++++++++++++---- eng/scripts/get_package_properties.py | 46 ++++++++---- .../calculate_targeted_packages.py | 49 ++++++++++++ scripts/devops_tasks/dispatch_tox.py | 4 +- sdk/pr.yml | 22 ++++++ tools/azure-sdk-tools/ci_tools/build.py | 2 +- 14 files changed, 283 insertions(+), 71 deletions(-) create mode 100644 eng/pipelines/templates/steps/resolve-package-targeting.yml create mode 100644 scripts/devops_tasks/calculate_targeted_packages.py create mode 100644 sdk/pr.yml diff --git a/eng/pipelines/templates/jobs/ci.tests.yml b/eng/pipelines/templates/jobs/ci.tests.yml index 7219acb71eb3..02debcc518e0 100644 --- a/eng/pipelines/templates/jobs/ci.tests.yml +++ b/eng/pipelines/templates/jobs/ci.tests.yml @@ -79,10 +79,21 @@ jobs: value: ${{ parameters.InjectedPackages }} steps: + - template: /eng/pipelines/templates/steps/download-package-artifacts.yml + - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml parameters: BuildTargetingString: ${{ parameters.BuildTargetingString }} + - template: /eng/pipelines/templates/steps/resolve-package-targeting.yml + parameters: + BuildTargetingParameter: ${{ parameters.BuildTargetingString }} + PackagePropertiesFolder: $(Build.ArtifactStagingDirectory)/PackageInfo + + - template: ../steps/set-dev-build.yml + parameters: + ServiceDirectory: ${{ parameters.ServiceDirectory }} + - template: /eng/common/pipelines/templates/steps/verify-agent-os.yml parameters: AgentImage: ${{ parameters.OSName }} @@ -122,12 +133,6 @@ jobs: InjectedPackages: $(InjectedPackages) TestProxy: ${{ parameters.TestProxy }} BeforeTestSteps: - - template: /eng/pipelines/templates/steps/download-package-artifacts.yml - - - template: ../steps/set-dev-build.yml - parameters: - ServiceDirectory: ${{ parameters.ServiceDirectory }} - - ${{ each step in parameters.BeforeTestSteps }}: - ${{ step }} AfterTestSteps: ${{ parameters.AfterTestSteps }} diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index 0cae4afb350d..5481e917a7c0 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -138,6 +138,11 @@ jobs: parameters: BuildTargetingString: ${{ parameters.BuildTargetingString }} + - template: /eng/pipelines/templates/steps/resolve-package-targeting.yml + parameters: + BuildTargetingParameter: ${{ parameters.BuildTargetingString }} + PackagePropertiesFolder: $(Build.ArtifactStagingDirectory)/PackageInfo + - template: ../steps/build-extended-artifacts.yml parameters: ServiceDirectory: ${{ parameters.ServiceDirectory }} diff --git a/eng/pipelines/templates/steps/build-extended-artifacts.yml b/eng/pipelines/templates/steps/build-extended-artifacts.yml index 5cfdb56e2a48..ad7ef177823d 100644 --- a/eng/pipelines/templates/steps/build-extended-artifacts.yml +++ b/eng/pipelines/templates/steps/build-extended-artifacts.yml @@ -50,8 +50,6 @@ steps: displayName: 'Prep Environment' - template: set-dev-build.yml - parameters: - ServiceDirectory: ${{ parameters.ServiceDirectory }} - ${{if eq(variables['System.TeamProject'], 'internal') }}: - template: auth-dev-feed.yml diff --git a/eng/pipelines/templates/steps/build-package-artifacts.yml b/eng/pipelines/templates/steps/build-package-artifacts.yml index 4f0fd376c3be..2aa6e5ee0c0e 100644 --- a/eng/pipelines/templates/steps/build-package-artifacts.yml +++ b/eng/pipelines/templates/steps/build-package-artifacts.yml @@ -17,6 +17,10 @@ parameters: - name: ArtifactSuffix type: string default: 'linux' + - name: BuildTargetingString + type: string + default: '*' + # The variable TargetingString is set by template `eng/pipelines/templates/steps/targeting-string-resolve.yml`. This template is invoked wherever # package targeting needs to be resolved prior to doing any actual work, whether it be build, static analysis, or test. @@ -45,6 +49,72 @@ steps: displayName: 'Tag scheduled builds' condition: and(eq(variables['Build.SourceBranchName'], variables['DefaultBranch']), eq(variables['Build.Reason'],'Schedule')) + # set dev build calls SavePackageProperties with dev version if necessary. this will never happen in a PR build + # however, it will set the variable SetDevVersion as appropriate for normal nightly runs + - template: set-dev-build.yml + parameters: + ServiceDirectory: ${{ parameters.ServiceDirectory }} + Condition: and(succeeded(), or(eq(variables['ENABLE_EXTENSION_BUILD'], 'true'), eq('${{ parameters.ArtifactSuffix }}', 'linux'))) + + # now we need to call Save-Package-Properties so that we can filter on it + - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: + - pwsh: | + mkdir -p $(Build.ArtifactStagingDirectory)/diff + displayName: Create PR Diff Folder + - task: PowerShell@2 + inputs: + filePath: 'eng/common/scripts/Generate-PR-Diff.ps1' + arguments: '-TargetPath "$(Build.SourcesDirectory)" -ArtifactPath $(Build.ArtifactStagingDirectory)/diff' + displayName: 'Generate PR Diff' + + - pwsh: | + Write-Host "We freshly generated the PR diff, and this is what it sees!" + Get-ChildItem -R -Force $(Build.ArtifactStagingDirectory)/diff | % { $_.FullName } + cat $(Build.ArtifactStagingDirectory)/diff/diff.json + displayName: Dump PR Diff + - task: Powershell@2 + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/scripts/Save-Package-Properties.ps1 + arguments: > + -PrDiff $(Build.ArtifactStagingDirectory)/diff/diff.json + -OutDirectory $(Build.ArtifactStagingDirectory)/PackageInfo + pwsh: true + workingDirectory: $(Pipeline.Workspace) + displayName: Save package properties filtered for PR + - ${{ else }}: + # todo add a variable to make -AddDevVersion optionally show up, versus calling this twice. + - task: Powershell@2 + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/scripts/Save-Package-Properties.ps1 + arguments: > + -ServiceDirectory ${{parameters.ServiceDirectory}} + -OutDirectory $(Build.ArtifactStagingDirectory)/PackageInfo + pwsh: true + workingDirectory: $(Pipeline.Workspace) + displayName: Save package properties for service + - task: Powershell@2 + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/scripts/Save-Package-Properties.ps1 + arguments: > + -ServiceDirectory ${{parameters.ServiceDirectory}} + -OutDirectory $(Build.ArtifactStagingDirectory)/PackageInfo + -AddDevVersion + pwsh: true + workingDirectory: $(Pipeline.Workspace) + displayName: Update package properties with dev version + condition: and(succeeded(),eq(variables['SetDevVersion'],'true')) + + + - template: /eng/pipelines/templates/steps/resolve-package-targeting.yml + parameters: + BuildTargetingParameter: ${{ parameters.BuildTargetingString }} + PackagePropertiesFolder: $(Build.ArtifactStagingDirectory)/PackageInfo + TargetRepo: $(Build.SourcesDirectory) + + - template: /eng/pipelines/templates/steps/resolve-build-platforms.yml + parameters: + ServiceDirectory: ${{ parameters.ServiceDirectory }} + - template: /eng/pipelines/templates/steps/use-venv.yml parameters: VirtualEnvironmentName: "venv" @@ -59,11 +129,6 @@ steps: displayName: 'Prep Environment' condition: and(succeeded(), or(eq(variables['ENABLE_EXTENSION_BUILD'], 'true'), eq('${{ parameters.ArtifactSuffix }}', 'linux'))) - - template: set-dev-build.yml@self - parameters: - ServiceDirectory: ${{ parameters.ServiceDirectory }} - Condition: and(succeeded(), or(eq(variables['ENABLE_EXTENSION_BUILD'], 'true'), eq('${{ parameters.ArtifactSuffix }}', 'linux'))) - - task: Powershell@2 inputs: filePath: $(Build.SourcesDirectory)/eng/common/scripts/Save-Package-Properties.ps1 diff --git a/eng/pipelines/templates/steps/build-test.yml b/eng/pipelines/templates/steps/build-test.yml index c9110a5b7ab2..09d7054d20fd 100644 --- a/eng/pipelines/templates/steps/build-test.yml +++ b/eng/pipelines/templates/steps/build-test.yml @@ -25,10 +25,6 @@ parameters: # Please use `$(TargetingString)` to refer to the python packages glob string. This was previously `${{ parameters.BuildTargetingString }}`. steps: - - pwsh: | - gci -r $(Build.ArtifactStagingDirectory) - displayName: Dump Artifact Directory - - template: /eng/pipelines/templates/steps/use-python-version.yml parameters: versionSpec: '${{ parameters.PythonVersion }}' @@ -38,8 +34,6 @@ steps: Activate: false - template: set-dev-build.yml - parameters: - ServiceDirectory: ${{ parameters.ServiceDirectory }} - pwsh: | if ($IsWindows) { diff --git a/eng/pipelines/templates/steps/build.yml b/eng/pipelines/templates/steps/build.yml index 6dd0391d6155..fea6fc496053 100644 --- a/eng/pipelines/templates/steps/build.yml +++ b/eng/pipelines/templates/steps/build.yml @@ -19,14 +19,6 @@ parameters: default: [] steps: - - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml - parameters: - BuildTargetingString: ${{ parameters.BuildTargetingString }} - - - template: /eng/pipelines/templates/steps/resolve-build-platforms.yml - parameters: - ServiceDirectory: ${{ parameters.ServiceDirectory }} - - template: /eng/pipelines/templates/steps/build-package-artifacts.yml parameters: ServiceDirectory: ${{ parameters.ServiceDirectory }} @@ -34,3 +26,4 @@ steps: TestPipeline: ${{ parameters.TestPipeline }} Artifacts: ${{ parameters.Artifacts }} ArtifactSuffix: ${{ parameters.ArtifactSuffix }} + BuildTargetingString: ${{ parameters.BuildTargetingString }} diff --git a/eng/pipelines/templates/steps/resolve-package-targeting.yml b/eng/pipelines/templates/steps/resolve-package-targeting.yml new file mode 100644 index 000000000000..e98910faf37f --- /dev/null +++ b/eng/pipelines/templates/steps/resolve-package-targeting.yml @@ -0,0 +1,38 @@ +parameters: + - name: BuildTargetingParameter + type: string + default: 'azure-*' + - name: TargetRepo + type: string + default: $(Build.SourcesDirectory) + - name: PackagePropertiesFolder + type: string + +steps: + - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: + # if we are in a PR, utilize the fact that the package properties folder will already be downloaded in the Packages folder + # any json file that exists within the PackagePropertiesFolder will be added to the targeting string + - pwsh: | + $packageProperties = Get-ChildItem -Recurse -Force "${{ parameters.PackagePropertiesFolder }}" ` + | Where-Object { $_.Extension -eq '.json' } ` + | Foreach-Object { $_.Name } ` + | ForEach-Object { $_.Replace(".json", "") } + $setting = $packageProperties -join "," + Write-Host "Setting TargetingString to $setting" + Write-Host "##vso[task.setvariable variable=TargetingString;]$setting" + displayName: Check override of targeting string by variable for PR builds + - ${{ else }}: + # if not triggered for a PR, simply use the targeting string from the parameter and the variable + - pwsh: | + $setting = "${{ parameters.BuildTargetingParameter }}" + # if the variable is not set, it'll just come back as the variable name. otherwise it's set. + if ('$(BuildTargetingString)' -ne ('$' + '(BuildTargetingString)')) { + Write-Host "The variable named BuildTargetingString is set to $(BuildTargetingString)" + $setting = "$(BuildTargetingString)" + } + else { + Write-Host "We are falling back to the parameter value ${{ parameters.BuildTargetingParameter }}" + } + Write-Host "Setting TargetingString to $setting" + Write-Host "##vso[task.setvariable variable=TargetingString;]$setting" + displayName: Check override of targeting string by variable for individual build \ No newline at end of file diff --git a/eng/pipelines/templates/steps/set-dev-build.yml b/eng/pipelines/templates/steps/set-dev-build.yml index 07de80a0bf78..9928fbdf2d11 100644 --- a/eng/pipelines/templates/steps/set-dev-build.yml +++ b/eng/pipelines/templates/steps/set-dev-build.yml @@ -2,20 +2,14 @@ parameters: ServiceDirectory: '' Condition: 'succeededOrFailed()' -# The variable TargetingString is set by template `eng/pipelines/templates/steps/targeting-string-resolve.yml`. This template is invoked from yml files: -# eng/pipelines/templates/jobs/ci.tests.yml -# eng/pipelines/templates/jobs/ci.yml -# eng/pipelines/templates/jobs/live.test.yml - -# Please use `$(TargetingString)` to refer to the python packages glob string. This was previously `${{ parameters.BuildTargetingString }}`. steps: + # this will never call save Save-Package-Properties directly as ServiceDirectory is not present + # it will however set an indicator for whether or not we're in a nightly build though. + # we will handle the Save-Package-Properties call ourselves - template: /eng/common/pipelines/templates/steps/daily-dev-build-variable.yml - parameters: - ServiceDirectory: ${{ parameters.ServiceDirectory }} - Condition: ${{ parameters.Condition }} - pwsh: | python -m pip install "tools/azure-sdk-tools[build]" - sdk_set_dev_version "$(TargetingString)" --service="${{parameters.ServiceDirectory}}" --build-id="$(Build.BuildNumber)" + sdk_set_dev_version "azure*" --build-id="$(Build.BuildNumber)" displayName: "Update package versions for dev build" condition: and(succeededOrFailed(), eq(variables['SetDevVersion'],'true'), ${{ parameters.Condition }}) diff --git a/eng/pipelines/templates/steps/targeting-string-resolve.yml b/eng/pipelines/templates/steps/targeting-string-resolve.yml index 268d6d92c3e7..a6d8687a1385 100644 --- a/eng/pipelines/templates/steps/targeting-string-resolve.yml +++ b/eng/pipelines/templates/steps/targeting-string-resolve.yml @@ -2,21 +2,56 @@ parameters: - name: BuildTargetingString type: string default: 'azure-*' + - name: GeneratePRDiff + type: boolean + default: false + - name: TargetRepo + type: string + default: $(Build.SourcesDirectory) + - name: PackagePropertiesFolder + type: string steps: - - pwsh: | - $setting = "${{ parameters.BuildTargetingString }}" - - # if the variable is not set, it'll just come back as the variable name. otherwise it's set. - if ('$(BuildTargetingString)' -ne ('$' + '(BuildTargetingString)')) { - Write-Host "The variable named BuildTargetingString is set to $(BuildTargetingString)" - $setting = "$(BuildTargetingString)" - } - else { - Write-Host "We are falling back to the parameter value ${{ parameters.BuildTargetingString }}" - } + # if triggered for a PR, handle PR diff side + - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: + - ${{ if parameters.GeneratePRDiff }}: + - pwsh: | + mkdir -p ${{ parameters.PRDiffFolder }} + displayName: Create PR Diff Folder + - task: PowerShell@2 + inputs: + filePath: 'eng/common/scripts/Generate-PR-Diff.ps1' + arguments: '-TargetPath ${{ parameters.TargetRepo }} -ArtifactPath ${{ parameters.PRDiffFolder }}' + displayName: 'Compare PR changes to targeting string' - Write-Host "Setting TargetingString to $setting" - Write-Host "##vso[task.setvariable variable=TargetingString]$setting" - displayName: Check override of targeting string - condition: and(succeededOrFailed(), eq(variables['TargetingString'],'')) + - pwsh: | + Write-Host "We freshly generated the PR diff, and this is what it sees!" + Get-ChildItem -R -Force ${{ parameters.PRDiffFolder }} | % { $_.FullName } + cat ${{ parameters.PRDiffFolder }}/diff.json + Write-Host "##vso[task.setvariable variable=PRDiffJson;]${{ parameters.PRDiffFolder }}/diff.json" + displayName: Coalesce targeting string for PR Builds + - ${{ else }}: + - pwsh: | + Write-Host "We are in a PR, but a job that doesn't generate the pr diff itself. This is what we see!" + Get-ChildItem -Recurse -Force "$(Build.ArtifactStagingDirectory)" | % { $_.FullName } + Write-Host "##vso[task.setvariable variable=PRDiffJson;]${{ parameters.PRDiffFolder }}/diff.json" + displayName: Coalesce targeting string for PR Builds + - pwsh: | + # code this shit to honor look for package properties files and then invoke checks for any one that exists + displayName: Resolve Targeted Packages + # if not triggered for a PR, simply attempt to locate the targeted packages from the parameter and variable. (standard for internal builds) + - ${{ else }}: + - pwsh: | + $setting = "${{ parameters.BuildTargetingString }}" + # if the variable is not set, it'll just come back as the variable name. otherwise it's set. + if ('$(BuildTargetingString)' -ne ('$' + '(BuildTargetingString)')) { + Write-Host "The variable named BuildTargetingString is set to $(BuildTargetingString)" + $setting = "$(BuildTargetingString)" + } + else { + Write-Host "We are falling back to the parameter value ${{ parameters.BuildTargetingString }}" + } + Write-Host "Setting TargetingString to $setting" + Write-Host "##vso[task.setvariable variable=TargetingString;]$setting" + displayName: Check override of targeting string by variable for individual build + condition: and(succeededOrFailed(), eq(variables['TargetingString'],'')) \ No newline at end of file diff --git a/eng/scripts/get_package_properties.py b/eng/scripts/get_package_properties.py index c0bea2a83592..a1d805e78086 100644 --- a/eng/scripts/get_package_properties.py +++ b/eng/scripts/get_package_properties.py @@ -1,28 +1,42 @@ import argparse -import sys import glob import os import re from ci_tools.parsing import ParsedSetup +root_dir = os.path.abspath(os.path.join(os.path.abspath(__file__), "..", "..", "..")) + +def dump_setup(path: str) -> None: + try: + parsed = ParsedSetup.from_path(path) + print( + "{0} {1} {2} {3}".format( + parsed.name, parsed.version, parsed.is_new_sdk, os.path.dirname(parsed.setup_filename) + ) + ) + except: + pass + if __name__ == "__main__": parser = argparse.ArgumentParser(description="Get package version details from the repo") - parser.add_argument("-s", "--search_path", required=True, help="The scope of the search") + parser.add_argument("-s", "--search_path", required=False, help="The scope of the search. Usually a service directory path sdk/X.") + parser.add_argument("-p", "--packages", required=False, help="The specific package names to search for. Comma separated list.") args = parser.parse_args() - # Use abspath for the os.walk because if setup parsing fails it often changes cwd which throws off the relative walk - for root, dirs, files in os.walk(os.path.abspath(args.search_path)): - if re.search(r"sdk[\\/][^\\/]+[\\/][^\\/]+$", root): - if "setup.py" in files: - try: - parsed = ParsedSetup.from_path(root) + if args.search_path: + # Use abspath for the os.walk because if setup parsing fails it often changes cwd which throws off the relative walk + for root, dirs, files in os.walk(os.path.abspath(args.search_path)): + if re.search(r"sdk[\\/][^\\/]+[\\/][^\\/]+$", root): + if "setup.py" in files: + dump_setup(root) + elif args.packages: + for pkg in [pkg.strip() for pkg in args.packages.split(",")]: + pkg_search = os.path.join(root_dir, "sdk", "*", pkg, "setup.py") + globbed = glob.glob(pkg_search) - print( - "{0} {1} {2} {3}".format( - parsed.name, parsed.version, parsed.is_new_sdk, os.path.dirname(parsed.setup_filename) - ) - ) - except: - # Skip setup.py if the package cannot be parsed - pass + if globbed: + for match in globbed: + dump_setup(match) + else: + raise ValueError("Must provide either a search path or a package name list.") \ No newline at end of file diff --git a/scripts/devops_tasks/calculate_targeted_packages.py b/scripts/devops_tasks/calculate_targeted_packages.py new file mode 100644 index 000000000000..f75ce345c39d --- /dev/null +++ b/scripts/devops_tasks/calculate_targeted_packages.py @@ -0,0 +1,49 @@ +import argparse +import json +import os + +from ci_tools.functions import discover_targeted_packages +from typing import Dict + +root_dir = os.path.abspath(os.path.join(os.path.abspath(__file__), "..", "..", "..")) + +def get_package_folder_from_path(file: str) -> str: + parts = file.split("/") + + if len(parts) < 3: + return "" + return os.sep.join(parts[:3]) + +def register(idx: Dict[str, str], key: str, value: str) -> None: + if key in idx: + idx[key].append(value) + else: + idx[key] = [value] + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Calculate the targeted packages') + parser.add_argument('input_json', type=str, help='Location of the diff.json file') + + args = parser.parse_args() + + + with open(args.input_json, 'r') as f: + diff = json.load(f) + + targeted_files = diff['ChangedFiles'] + + all_packages = {} # {package: [file]} + + for file in targeted_files: + # register(all_packages, , file) + package = get_package_folder_from_path(file) + if package: + package_directory = os.path.join(root_dir, package) + package_name = os.path.basename(package_directory) + + if os.path.exists(os.path.join(package_directory, "pyproject.toml")) or os.path.exists(os.path.join(package_directory, "setup.py")): + register(all_packages, package_name, file) + + packages = ",".join(all_packages.keys()) + print(f"This script sees the following packages [{packages}].") + print(f"##vso[task.setvariable variable=TargetingString;]{packages}") \ No newline at end of file diff --git a/scripts/devops_tasks/dispatch_tox.py b/scripts/devops_tasks/dispatch_tox.py index 03c2dc76e341..4b9dd3196ffa 100644 --- a/scripts/devops_tasks/dispatch_tox.py +++ b/scripts/devops_tasks/dispatch_tox.py @@ -26,7 +26,7 @@ if __name__ == "__main__": parser = argparse.ArgumentParser( description=""" -This script is the single point for all checks invoked by CI within this repo. It works in two phases. +This script is the single point for all checks invoked by CI within this repo. It works in two phases. 1. Identify which packages in the repo are in scope for this script invocation, based on a glob string and a service directory. 2. Invoke one or multiple `tox` environments for each package identified as in scope. @@ -116,7 +116,7 @@ # We need to support both CI builds of everything and individual service # folders. This logic allows us to do both. - if args.service: + if args.service and args.service != "auto": service_dir = os.path.join("sdk", args.service) target_dir = os.path.join(root_dir, service_dir) else: diff --git a/sdk/pr.yml b/sdk/pr.yml new file mode 100644 index 000000000000..5fcf42f176fc --- /dev/null +++ b/sdk/pr.yml @@ -0,0 +1,22 @@ +trigger: + branches: + exclude: + - pipelinev3-v2 + +pr: + branches: + include: + - pipelinev3* + +parameters: + - name: Service + type: string + default: auto + +extends: + template: ../eng/pipelines/templates/stages/archetype-sdk-client.yml + parameters: + ServiceDirectory: ${{ parameters.Service }} + BuildTargetingString: "*" + ValidateFormatting: true + TestProxy: true \ No newline at end of file diff --git a/tools/azure-sdk-tools/ci_tools/build.py b/tools/azure-sdk-tools/ci_tools/build.py index dabdb11cb021..8ec07d03dd4a 100644 --- a/tools/azure-sdk-tools/ci_tools/build.py +++ b/tools/azure-sdk-tools/ci_tools/build.py @@ -100,7 +100,7 @@ def build() -> None: repo_root = discover_repo_root(args.repo) - if args.service: + if args.service and args.service != "auto": service_dir = os.path.join("sdk", args.service) target_dir = os.path.join(repo_root, service_dir) else: From 393d610dbb25c5f31c2b051b113c8fe6b40278e7 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Thu, 25 Jul 2024 11:40:40 -0700 Subject: [PATCH 024/119] update usage of resolve-package-targeting everywhere which will honor the packageproperties folder --- eng/pipelines/templates/jobs/ci.tests.yml | 8 ++----- eng/pipelines/templates/jobs/ci.yml | 23 ++++++++----------- eng/pipelines/templates/jobs/live.tests.yml | 2 +- eng/pipelines/templates/jobs/regression.yml | 2 +- .../jobs/trigger-ml-sample-pipeline.yml | 4 ++-- eng/pipelines/templates/steps/analyze.yml | 7 +----- .../steps/build-extended-artifacts.yml | 7 ++---- .../steps/build-package-artifacts.yml | 3 +-- eng/pipelines/templates/steps/build-test.yml | 6 +---- .../steps/resolve-build-platforms.yml | 2 +- .../steps/resolve-package-targeting.yml | 5 +--- eng/pipelines/templates/steps/run_apistub.yml | 12 +++------- eng/pipelines/templates/steps/run_bandit.yml | 13 ++++------- .../templates/steps/run_breaking_changes.yml | 7 +----- eng/pipelines/templates/steps/run_mypy.yml | 7 +----- eng/pipelines/templates/steps/run_pylint.yml | 7 +----- eng/pipelines/templates/steps/run_pyright.yml | 7 +----- 17 files changed, 34 insertions(+), 88 deletions(-) diff --git a/eng/pipelines/templates/jobs/ci.tests.yml b/eng/pipelines/templates/jobs/ci.tests.yml index 02debcc518e0..84ce8365e532 100644 --- a/eng/pipelines/templates/jobs/ci.tests.yml +++ b/eng/pipelines/templates/jobs/ci.tests.yml @@ -81,19 +81,15 @@ jobs: steps: - template: /eng/pipelines/templates/steps/download-package-artifacts.yml - - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml + - template: ../steps/set-dev-build.yml parameters: - BuildTargetingString: ${{ parameters.BuildTargetingString }} + ServiceDirectory: ${{ parameters.ServiceDirectory }} - template: /eng/pipelines/templates/steps/resolve-package-targeting.yml parameters: BuildTargetingParameter: ${{ parameters.BuildTargetingString }} PackagePropertiesFolder: $(Build.ArtifactStagingDirectory)/PackageInfo - - template: ../steps/set-dev-build.yml - parameters: - ServiceDirectory: ${{ parameters.ServiceDirectory }} - - template: /eng/common/pipelines/templates/steps/verify-agent-os.yml parameters: AgentImage: ${{ parameters.OSName }} diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index d7c35c410e59..ac1a547a7b74 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -78,7 +78,7 @@ jobs: BeforePublishSteps: ${{ parameters.BeforePublishSteps }} TestPipeline: ${{ parameters.TestPipeline }} Artifacts: ${{ parameters.Artifacts }} - ArtifactSuffix: ${{ parameters.ArtifactSuffix }} + ArtifactSuffix: linux BuildTargetingString: ${{ parameters.BuildTargetingString }} - job: 'Build_Windows' @@ -96,7 +96,7 @@ jobs: BeforePublishSteps: ${{ parameters.BeforePublishSteps }} TestPipeline: ${{ parameters.TestPipeline }} Artifacts: ${{ parameters.Artifacts }} - ArtifactSuffix: ${{ parameters.ArtifactSuffix }} + ArtifactSuffix: windows BuildTargetingString: ${{ parameters.BuildTargetingString }} - job: 'Build_MacOS' @@ -114,7 +114,7 @@ jobs: BeforePublishSteps: ${{ parameters.BeforePublishSteps }} TestPipeline: ${{ parameters.TestPipeline }} Artifacts: ${{ parameters.Artifacts }} - ArtifactSuffix: ${{ parameters.ArtifactSuffix }} + ArtifactSuffix: mac BuildTargetingString: ${{ parameters.BuildTargetingString }} - job: 'Build_Extended' @@ -134,13 +134,9 @@ jobs: steps: - template: /eng/pipelines/templates/steps/download-package-artifacts.yml - - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml - parameters: - BuildTargetingString: ${{ parameters.BuildTargetingString }} - - template: /eng/pipelines/templates/steps/resolve-package-targeting.yml parameters: - BuildTargetingParameter: ${{ parameters.BuildTargetingString }} + BuildTargetingString: ${{ parameters.BuildTargetingString }} PackagePropertiesFolder: $(Build.ArtifactStagingDirectory)/PackageInfo - template: ../steps/build-extended-artifacts.yml @@ -165,10 +161,6 @@ jobs: os: linux steps: - - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml - parameters: - BuildTargetingString: ${{ parameters.BuildTargetingString }} - - template: /eng/common/pipelines/templates/steps/check-spelling.yml parameters: ContinueOnError: false @@ -189,6 +181,11 @@ jobs: artifactName: 'packages_extended' targetPath: $(Build.ArtifactStagingDirectory) + - template: /eng/pipelines/templates/steps/resolve-package-targeting.yml + parameters: + BuildTargetingString: ${{ parameters.BuildTargetingString }} + PackagePropertiesFolder: $(Build.ArtifactStagingDirectory)/PackageInfo + - template: ../steps/analyze.yml parameters: ServiceDirectory: ${{ parameters.ServiceDirectory }} @@ -246,7 +243,7 @@ jobs: - pwsh: | python -m pip install "./tools/azure-sdk-tools[build]" displayName: 'Prep Environment' - - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml + - template: /eng/pipelines/templates/steps/resolve-package-targeting.yml parameters: BuildTargetingString: ${{ parameters.BuildTargetingString }} - task: PythonScript@0 diff --git a/eng/pipelines/templates/jobs/live.tests.yml b/eng/pipelines/templates/jobs/live.tests.yml index 53ec04999e62..eb6a2088bb63 100644 --- a/eng/pipelines/templates/jobs/live.tests.yml +++ b/eng/pipelines/templates/jobs/live.tests.yml @@ -110,7 +110,7 @@ jobs: parameters: AgentImage: ${{ parameters.OSName }} - - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml + - template: /eng/pipelines/templates/steps/resolve-package-targeting.yml parameters: BuildTargetingString: ${{ parameters.BuildTargetingString }} diff --git a/eng/pipelines/templates/jobs/regression.yml b/eng/pipelines/templates/jobs/regression.yml index 8c3b559b48c7..4fa0629ed1be 100644 --- a/eng/pipelines/templates/jobs/regression.yml +++ b/eng/pipelines/templates/jobs/regression.yml @@ -71,7 +71,7 @@ jobs: # Please use `$(TargetingString)` to refer to the python packages glob string. This was previously `${{ parameters.BuildTargetingString }}`. steps: - - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml + - template: /eng/pipelines/templates/steps/resolve-package-targeting.yml parameters: BuildTargetingString: ${{ parameters.BuildTargetingString }} diff --git a/eng/pipelines/templates/jobs/trigger-ml-sample-pipeline.yml b/eng/pipelines/templates/jobs/trigger-ml-sample-pipeline.yml index 9b2e190d6094..304a4e124a50 100644 --- a/eng/pipelines/templates/jobs/trigger-ml-sample-pipeline.yml +++ b/eng/pipelines/templates/jobs/trigger-ml-sample-pipeline.yml @@ -4,7 +4,7 @@ parameters: default: 'ml' - name: Artifacts type: object - default: + default: - name: azure-ai-ml safeName: azureaiml skipVerifyChangeLog: true @@ -24,7 +24,7 @@ jobs: vmImage: MMSUbuntu20.04 steps: - - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml + - template: /eng/pipelines/templates/steps/resolve-package-targeting.yml parameters: BuildTargetingString: "azure-*" diff --git a/eng/pipelines/templates/steps/analyze.yml b/eng/pipelines/templates/steps/analyze.yml index f1ff969ec87b..8e82325daf90 100644 --- a/eng/pipelines/templates/steps/analyze.yml +++ b/eng/pipelines/templates/steps/analyze.yml @@ -8,12 +8,7 @@ parameters: ValidateFormatting: false GenerateApiReviewForManualOnly: false -# The variable TargetingString is set by template `eng/pipelines/templates/steps/targeting-string-resolve.yml`. This template is invoked from yml files: -# eng/pipelines/templates/jobs/ci.tests.yml -# eng/pipelines/templates/jobs/ci.yml -# eng/pipelines/templates/jobs/live.test.yml - -# Please use `$(TargetingString)` to refer to the python packages glob string. This was previously `${{ parameters.BuildTargetingString }}`. +# Please use `$(TargetingString)` to refer to the python packages glob string. This variable is set from resolve-package-targeting.yml. steps: - template: /eng/pipelines/templates/steps/analyze_dependency.yml parameters: diff --git a/eng/pipelines/templates/steps/build-extended-artifacts.yml b/eng/pipelines/templates/steps/build-extended-artifacts.yml index ad7ef177823d..79f7c1f96fca 100644 --- a/eng/pipelines/templates/steps/build-extended-artifacts.yml +++ b/eng/pipelines/templates/steps/build-extended-artifacts.yml @@ -17,12 +17,9 @@ parameters: - name: DevFeedName type: string default: 'public/azure-sdk-for-python' -# The variable TargetingString is set by template `eng/pipelines/templates/steps/targeting-string-resolve.yml`. This template is invoked from yml files: -# eng/pipelines/templates/jobs/ci.tests.yml -# eng/pipelines/templates/jobs/ci.yml -# eng/pipelines/templates/jobs/live.test.yml -# Please use `$(TargetingString)` to refer to the python packages glob string. This was previously `${{ parameters.BuildTargetingString }}`. + +# Please use `$(TargetingString)` to refer to the python packages glob string. This variable is set from resolve-package-targeting.yml. steps: - task: UsePythonVersion@0 displayName: 'Use Python 3.11' diff --git a/eng/pipelines/templates/steps/build-package-artifacts.yml b/eng/pipelines/templates/steps/build-package-artifacts.yml index acac1d58066d..4482471adeef 100644 --- a/eng/pipelines/templates/steps/build-package-artifacts.yml +++ b/eng/pipelines/templates/steps/build-package-artifacts.yml @@ -102,9 +102,8 @@ steps: - template: /eng/pipelines/templates/steps/resolve-package-targeting.yml parameters: - BuildTargetingParameter: ${{ parameters.BuildTargetingString }} + BuildTargetingString: ${{ parameters.BuildTargetingString }} PackagePropertiesFolder: $(Build.ArtifactStagingDirectory)/PackageInfo - TargetRepo: $(Build.SourcesDirectory) - template: /eng/pipelines/templates/steps/resolve-build-platforms.yml parameters: diff --git a/eng/pipelines/templates/steps/build-test.yml b/eng/pipelines/templates/steps/build-test.yml index 09d7054d20fd..030af919715c 100644 --- a/eng/pipelines/templates/steps/build-test.yml +++ b/eng/pipelines/templates/steps/build-test.yml @@ -18,12 +18,8 @@ parameters: UseFederatedAuth: false ServiceConnection: '' -# The variable TargetingString is set by template `eng/pipelines/templates/steps/targeting-string-resolve.yml`. This template is invoked from yml files: -# eng/pipelines/templates/jobs/ci.tests.yml -# eng/pipelines/templates/jobs/ci.yml -# eng/pipelines/templates/jobs/live.test.yml -# Please use `$(TargetingString)` to refer to the python packages glob string. This was previously `${{ parameters.BuildTargetingString }}`. +# Please use `$(TargetingString)` to refer to the python packages glob string. This variable is set from resolve-package-targeting.yml. steps: - template: /eng/pipelines/templates/steps/use-python-version.yml parameters: diff --git a/eng/pipelines/templates/steps/resolve-build-platforms.yml b/eng/pipelines/templates/steps/resolve-build-platforms.yml index 569a80cdccdb..2d940d171d42 100644 --- a/eng/pipelines/templates/steps/resolve-build-platforms.yml +++ b/eng/pipelines/templates/steps/resolve-build-platforms.yml @@ -5,7 +5,7 @@ parameters: steps: # when we merge pipeline v3, this check will change to examining the targeting string $(TargetingString) - # as the generate-pr-diff call + resolution will be present in targeting-string-resolve.yml. + # as the generate-pr-diff call + resolution will be present in resolve-package-targeting.yml. # until then, we simply check to see if we're targeting storage service directory - pwsh: | if ("${{ parameters.ServiceDirectory }}" -eq "storage") { diff --git a/eng/pipelines/templates/steps/resolve-package-targeting.yml b/eng/pipelines/templates/steps/resolve-package-targeting.yml index 7924eeac0f9e..a88ea35f7bf8 100644 --- a/eng/pipelines/templates/steps/resolve-package-targeting.yml +++ b/eng/pipelines/templates/steps/resolve-package-targeting.yml @@ -2,15 +2,12 @@ parameters: - name: BuildTargetingString type: string default: 'azure-*' - - name: TargetRepo - type: string - default: $(Build.SourcesDirectory) - name: PackagePropertiesFolder type: string steps: # if triggered for a PR, handle PR diff side - - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: + - ${{ if and(eq(variables['Build.Reason'], 'PullRequest'), ne('${{ parameters.PackagePropertiesFolder }}','') }}: - pwsh: | $packageProperties = Get-ChildItem -Recurse -Force "${{ parameters.PackagePropertiesFolder }}" ` | Where-Object { $_.Extension -eq '.json' } ` diff --git a/eng/pipelines/templates/steps/run_apistub.yml b/eng/pipelines/templates/steps/run_apistub.yml index 57c7066d0832..d18e0f2752be 100644 --- a/eng/pipelines/templates/steps/run_apistub.yml +++ b/eng/pipelines/templates/steps/run_apistub.yml @@ -2,13 +2,7 @@ parameters: ServiceDirectory: '' AdditionalTestArgs: '' - -# The variable TargetingString is set by template `eng/pipelines/templates/steps/targeting-string-resolve.yml`. This template is invoked from yml files: -# eng/pipelines/templates/jobs/ci.tests.yml -# eng/pipelines/templates/jobs/ci.yml -# eng/pipelines/templates/jobs/live.test.yml - -# Please use `$(TargetingString)` to refer to the python packages glob string. This was previously `${{ parameters.BuildTargetingString }}`. +# Please use `$(TargetingString)` to refer to the python packages glob string. This variable is set from resolve-package-targeting.yml. steps: - task: UsePythonVersion@0 displayName: 'Use Python 3.10' @@ -30,8 +24,8 @@ steps: "$(TargetingString)" ${{ parameters.AdditionalTestArgs }} -d "$(Build.ArtifactStagingDirectory)" - --mark_arg="${{ parameters.TestMarkArgument }}" - --service="${{ parameters.ServiceDirectory }}" + --mark_arg="${{ parameters.TestMarkArgument }}" + --service="${{ parameters.ServiceDirectory }}" --toxenv="apistub" --disablecov --filter-type="Omit_management" diff --git a/eng/pipelines/templates/steps/run_bandit.yml b/eng/pipelines/templates/steps/run_bandit.yml index 4bb8c374d11e..a57cfdb27202 100644 --- a/eng/pipelines/templates/steps/run_bandit.yml +++ b/eng/pipelines/templates/steps/run_bandit.yml @@ -3,12 +3,7 @@ parameters: TestMarkArgument: '' EnvVars: {} -# The variable TargetingString is set by template `eng/pipelines/templates/steps/targeting-string-resolve.yml`. This template is invoked from yml files: -# eng/pipelines/templates/jobs/ci.tests.yml -# eng/pipelines/templates/jobs/ci.yml -# eng/pipelines/templates/jobs/live.test.yml - -# Please use `$(TargetingString)` to refer to the python packages glob string. This was previously `${{ parameters.BuildTargetingString }}`. +# Please use `$(TargetingString)` to refer to the python packages glob string. This variable is set from resolve-package-targeting.yml. steps: - script: | python -m pip install -r eng/ci_tools.txt @@ -20,9 +15,9 @@ steps: inputs: scriptPath: 'scripts/devops_tasks/dispatch_tox.py' arguments: >- - "$(TargetingString)" - --mark_arg="${{ parameters.TestMarkArgument }}" - --service="${{ parameters.ServiceDirectory }}" + "$(TargetingString)" + --mark_arg="${{ parameters.TestMarkArgument }}" + --service="${{ parameters.ServiceDirectory }}" --toxenv="bandit" --disablecov --filter-type="Omit_management" diff --git a/eng/pipelines/templates/steps/run_breaking_changes.yml b/eng/pipelines/templates/steps/run_breaking_changes.yml index 499ee3e51081..31796638c800 100644 --- a/eng/pipelines/templates/steps/run_breaking_changes.yml +++ b/eng/pipelines/templates/steps/run_breaking_changes.yml @@ -3,12 +3,7 @@ parameters: TestMarkArgument: '' EnvVars: {} -# The variable TargetingString is set by template `eng/pipelines/templates/steps/targeting-string-resolve.yml`. This template is invoked from yml files: -# eng/pipelines/templates/jobs/ci.tests.yml -# eng/pipelines/templates/jobs/ci.yml -# eng/pipelines/templates/jobs/live.test.yml - -# Please use `$(TargetingString)` to refer to the python packages glob string. This was previously `${{ parameters.BuildTargetingString }}`. +# Please use `$(TargetingString)` to refer to the python packages glob string. This variable is set from resolve-package-targeting.yml. steps: - task: PythonScript@0 displayName: 'Run Breaking Changes' diff --git a/eng/pipelines/templates/steps/run_mypy.yml b/eng/pipelines/templates/steps/run_mypy.yml index e05e4ca65467..7a520b91132b 100644 --- a/eng/pipelines/templates/steps/run_mypy.yml +++ b/eng/pipelines/templates/steps/run_mypy.yml @@ -5,12 +5,7 @@ parameters: EnvVars: {} AdditionalTestArgs: '' -# The variable TargetingString is set by template `eng/pipelines/templates/steps/targeting-string-resolve.yml`. This template is invoked from yml files: -# eng/pipelines/templates/jobs/ci.tests.yml -# eng/pipelines/templates/jobs/ci.yml -# eng/pipelines/templates/jobs/live.test.yml - -# Please use `$(TargetingString)` to refer to the python packages glob string. This was previously `${{ parameters.BuildTargetingString }}`. +# Please use `$(TargetingString)` to refer to the python packages glob string. This variable is set from resolve-package-targeting.yml. steps: - task: UsePythonVersion@0 displayName: 'Use Python 3.8' diff --git a/eng/pipelines/templates/steps/run_pylint.yml b/eng/pipelines/templates/steps/run_pylint.yml index 9a39873d3bdd..7087cbc88b22 100644 --- a/eng/pipelines/templates/steps/run_pylint.yml +++ b/eng/pipelines/templates/steps/run_pylint.yml @@ -4,12 +4,7 @@ parameters: EnvVars: {} AdditionalTestArgs: '' -# The variable TargetingString is set by template `eng/pipelines/templates/steps/targeting-string-resolve.yml`. This template is invoked from yml files: -# eng/pipelines/templates/jobs/ci.tests.yml -# eng/pipelines/templates/jobs/ci.yml -# eng/pipelines/templates/jobs/live.test.yml - -# Please use `$(TargetingString)` to refer to the python packages glob string. This was previously `${{ parameters.BuildTargetingString }}`. +# Please use `$(TargetingString)` to refer to the python packages glob string. This variable is set from resolve-package-targeting.yml. steps: - task: UsePythonVersion@0 displayName: 'Use Python 3.8' diff --git a/eng/pipelines/templates/steps/run_pyright.yml b/eng/pipelines/templates/steps/run_pyright.yml index 9ac8ced7bb00..44d262e98d2c 100644 --- a/eng/pipelines/templates/steps/run_pyright.yml +++ b/eng/pipelines/templates/steps/run_pyright.yml @@ -4,12 +4,7 @@ parameters: TestMarkArgument: '' AdditionalTestArgs: '' -# The variable TargetingString is set by template `eng/pipelines/templates/steps/targeting-string-resolve.yml`. This template is invoked from yml files: -# eng/pipelines/templates/jobs/ci.tests.yml -# eng/pipelines/templates/jobs/ci.yml -# eng/pipelines/templates/jobs/live.test.yml - -# Please use `$(TargetingString)` to refer to the python packages glob string. This was previously `${{ parameters.BuildTargetingString }}`. +# Please use `$(TargetingString)` to refer to the python packages glob string. This variable is set from resolve-package-targeting.yml. steps: - task: UsePythonVersion@0 displayName: 'Use Python 3.8' From cca4e9fcb32c5b5487a321596370eaf0b1f1bb73 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Thu, 25 Jul 2024 11:45:58 -0700 Subject: [PATCH 025/119] update our commit reference --- eng/pipelines/templates/steps/resolve-package-targeting.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/templates/steps/resolve-package-targeting.yml b/eng/pipelines/templates/steps/resolve-package-targeting.yml index a88ea35f7bf8..8732be1a4482 100644 --- a/eng/pipelines/templates/steps/resolve-package-targeting.yml +++ b/eng/pipelines/templates/steps/resolve-package-targeting.yml @@ -6,8 +6,9 @@ parameters: type: string steps: - # if triggered for a PR, handle PR diff side - - ${{ if and(eq(variables['Build.Reason'], 'PullRequest'), ne('${{ parameters.PackagePropertiesFolder }}','') }}: + # if triggered for a PR, handle PR diff side adding the default for packageproperties folder will mean this will always act + # as a non-pr resolution + - ${{ if and(eq(variables['Build.Reason'], 'PullRequest'), ne(parameters.PackagePropertiesFolder, '') }}: - pwsh: | $packageProperties = Get-ChildItem -Recurse -Force "${{ parameters.PackagePropertiesFolder }}" ` | Where-Object { $_.Extension -eq '.json' } ` From 3543235f53d2d978769d99a108802bb36692cdf3 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Thu, 25 Jul 2024 11:49:10 -0700 Subject: [PATCH 026/119] save default parameter --- eng/pipelines/templates/steps/resolve-package-targeting.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eng/pipelines/templates/steps/resolve-package-targeting.yml b/eng/pipelines/templates/steps/resolve-package-targeting.yml index 8732be1a4482..7fcd0ef256a6 100644 --- a/eng/pipelines/templates/steps/resolve-package-targeting.yml +++ b/eng/pipelines/templates/steps/resolve-package-targeting.yml @@ -4,6 +4,7 @@ parameters: default: 'azure-*' - name: PackagePropertiesFolder type: string + default: '' steps: # if triggered for a PR, handle PR diff side adding the default for packageproperties folder will mean this will always act From fb92c74750d6b390e3f177350a4eaf8f2a7d059f Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Thu, 25 Jul 2024 11:50:28 -0700 Subject: [PATCH 027/119] correct conditional --- eng/pipelines/templates/steps/resolve-package-targeting.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/steps/resolve-package-targeting.yml b/eng/pipelines/templates/steps/resolve-package-targeting.yml index 7fcd0ef256a6..e6346e21449b 100644 --- a/eng/pipelines/templates/steps/resolve-package-targeting.yml +++ b/eng/pipelines/templates/steps/resolve-package-targeting.yml @@ -9,7 +9,7 @@ parameters: steps: # if triggered for a PR, handle PR diff side adding the default for packageproperties folder will mean this will always act # as a non-pr resolution - - ${{ if and(eq(variables['Build.Reason'], 'PullRequest'), ne(parameters.PackagePropertiesFolder, '') }}: + - ${{ if and(eq(variables['Build.Reason'], 'PullRequest'), ne(parameters.PackagePropertiesFolder, '')) }}: - pwsh: | $packageProperties = Get-ChildItem -Recurse -Force "${{ parameters.PackagePropertiesFolder }}" ` | Where-Object { $_.Extension -eq '.json' } ` From c1135d70395701d299b10cc8c1a4c856ca0dd212 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Thu, 25 Jul 2024 11:52:23 -0700 Subject: [PATCH 028/119] BuildTargetingParameter -> BuildTargetingString --- eng/pipelines/templates/jobs/ci.tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/jobs/ci.tests.yml b/eng/pipelines/templates/jobs/ci.tests.yml index 84ce8365e532..2acfd6ad329e 100644 --- a/eng/pipelines/templates/jobs/ci.tests.yml +++ b/eng/pipelines/templates/jobs/ci.tests.yml @@ -87,7 +87,7 @@ jobs: - template: /eng/pipelines/templates/steps/resolve-package-targeting.yml parameters: - BuildTargetingParameter: ${{ parameters.BuildTargetingString }} + BuildTargetingString: ${{ parameters.BuildTargetingString }} PackagePropertiesFolder: $(Build.ArtifactStagingDirectory)/PackageInfo - template: /eng/common/pipelines/templates/steps/verify-agent-os.yml From 29b729c863e2b7a29f8601f630506c9bf44cd261 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Thu, 25 Jul 2024 12:21:16 -0700 Subject: [PATCH 029/119] change ordering of savepackageproperties --- .../templates/steps/build-package-artifacts.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/eng/pipelines/templates/steps/build-package-artifacts.yml b/eng/pipelines/templates/steps/build-package-artifacts.yml index 4482471adeef..ae75258c3984 100644 --- a/eng/pipelines/templates/steps/build-package-artifacts.yml +++ b/eng/pipelines/templates/steps/build-package-artifacts.yml @@ -78,27 +78,26 @@ steps: - ${{ else }}: - task: Powershell@2 - condition: and(succeeded(), ne(variables['SetDevVersion'],'true')) + condition: and(succeeded(), eq(variables['SetDevVersion'],'true')) inputs: filePath: $(Build.SourcesDirectory)/eng/common/scripts/Save-Package-Properties.ps1 arguments: > -ServiceDirectory ${{parameters.ServiceDirectory}} -OutDirectory $(Build.ArtifactStagingDirectory)/PackageInfo + -AddDevVersion pwsh: true workingDirectory: $(Pipeline.Workspace) - displayName: Save package properties for service - + displayName: Save package properties with dev version - task: Powershell@2 - condition: and(succeeded(), eq(variables['SetDevVersion'],'true')) + condition: and(succeeded(), ne(variables['SetDevVersion'],'true')) inputs: filePath: $(Build.SourcesDirectory)/eng/common/scripts/Save-Package-Properties.ps1 arguments: > -ServiceDirectory ${{parameters.ServiceDirectory}} -OutDirectory $(Build.ArtifactStagingDirectory)/PackageInfo - -AddDevVersion pwsh: true workingDirectory: $(Pipeline.Workspace) - displayName: Update package properties with dev version + displayName: Save package properties for service - template: /eng/pipelines/templates/steps/resolve-package-targeting.yml parameters: From 0340473b08c8afb1f9abb06aea0fc5b0c3d39e1a Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Thu, 25 Jul 2024 12:24:52 -0700 Subject: [PATCH 030/119] touch core so that the core PR has something to work against --- sdk/core/azure-core/perf.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/core/azure-core/perf.yml b/sdk/core/azure-core/perf.yml index a7774ec06b02..97e45420ba93 100644 --- a/sdk/core/azure-core/perf.yml +++ b/sdk/core/azure-core/perf.yml @@ -34,4 +34,4 @@ extends: Tests: ${{ parameters.Tests }} Arguments: ${{ parameters.Arguments }} Iterations: ${{ parameters.Iterations }} - AdditionalArguments: ${{ parameters.AdditionalArguments }} \ No newline at end of file + AdditionalArguments: ${{ parameters.AdditionalArguments }} From ba9ad00b247e031f727e8bed48db09d3d4369bed Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Thu, 25 Jul 2024 12:57:33 -0700 Subject: [PATCH 031/119] going to need to add this to an eng/common update --- eng/common/scripts/Generate-PR-Diff.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/common/scripts/Generate-PR-Diff.ps1 b/eng/common/scripts/Generate-PR-Diff.ps1 index d84f9e15ca7c..7a31456b36ca 100644 --- a/eng/common/scripts/Generate-PR-Diff.ps1 +++ b/eng/common/scripts/Generate-PR-Diff.ps1 @@ -19,14 +19,14 @@ Param ( [string] $TargetPath ) -. (Join-Path $PSScriptRoot "Helpers" git-helpers.ps1) +. (Join-Path $PSScriptRoot "Helpers" "git-helpers.ps1") function Get-ChangedServices { Param ( [Parameter(Mandatory=$True)] [string[]] $ChangedFiles ) - + $changedServices = $ChangedFiles | Foreach-Object { if ($_ -match "sdk/([^/]+)") { $matches[1] } } | Sort-Object -Unique return $changedServices From 8c1df15c2286ac4078be12d8a21098183940f3ba Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Thu, 25 Jul 2024 13:21:55 -0700 Subject: [PATCH 032/119] dot include the path --- eng/common/scripts/Generate-PR-Diff.ps1 | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/eng/common/scripts/Generate-PR-Diff.ps1 b/eng/common/scripts/Generate-PR-Diff.ps1 index 7a31456b36ca..55c10f2425a7 100644 --- a/eng/common/scripts/Generate-PR-Diff.ps1 +++ b/eng/common/scripts/Generate-PR-Diff.ps1 @@ -19,7 +19,12 @@ Param ( [string] $TargetPath ) -. (Join-Path $PSScriptRoot "Helpers" "git-helpers.ps1") + +$path = (Join-Path $PSScriptRoot "Helpers" "git-helpers.ps1") + +Write-Host "This is what we're trying to dot include $path" + +. $path function Get-ChangedServices { Param ( From cad0a2f1ddf977dd116e6e5b1fa7819506440130 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Thu, 25 Jul 2024 13:46:01 -0700 Subject: [PATCH 033/119] generate pr diff with platform specific handlers --- .../templates/steps/build-package-artifacts.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/eng/pipelines/templates/steps/build-package-artifacts.yml b/eng/pipelines/templates/steps/build-package-artifacts.yml index ae75258c3984..8644e7d3bae6 100644 --- a/eng/pipelines/templates/steps/build-package-artifacts.yml +++ b/eng/pipelines/templates/steps/build-package-artifacts.yml @@ -54,10 +54,17 @@ steps: mkdir -p $(Build.ArtifactStagingDirectory)/diff displayName: Create PR Diff Folder + - pwsh: | + $location = Join-Path "$(Build.ArtifactStagingDirectory)" "diff" + + Write-Host "./eng/common/scripts/Generate-PR-Diff.ps1 -TargetPath `"$(Build.SourcesDirectory)`" -ArtifactPath `"$location`"" + ./eng/common/scripts/Generate-PR-Diff.ps1 -TargetPath "$(Build.SourcesDirectory)" -ArtifactPath "$location" + displayName: Generate PR Diff + - task: PowerShell@2 inputs: filePath: 'eng/common/scripts/Generate-PR-Diff.ps1' - arguments: '-TargetPath "$(Build.SourcesDirectory)" -ArtifactPath $(Build.ArtifactStagingDirectory)/diff' + arguments: '' displayName: 'Generate PR Diff' - pwsh: | From 32b775bf6958239287c02fa53281e5e63fde22ef Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Thu, 25 Jul 2024 14:00:53 -0700 Subject: [PATCH 034/119] remove bad call to generate-pr-diff --- eng/pipelines/templates/steps/build-package-artifacts.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/eng/pipelines/templates/steps/build-package-artifacts.yml b/eng/pipelines/templates/steps/build-package-artifacts.yml index 8644e7d3bae6..6e7f7b3dc3f9 100644 --- a/eng/pipelines/templates/steps/build-package-artifacts.yml +++ b/eng/pipelines/templates/steps/build-package-artifacts.yml @@ -61,12 +61,6 @@ steps: ./eng/common/scripts/Generate-PR-Diff.ps1 -TargetPath "$(Build.SourcesDirectory)" -ArtifactPath "$location" displayName: Generate PR Diff - - task: PowerShell@2 - inputs: - filePath: 'eng/common/scripts/Generate-PR-Diff.ps1' - arguments: '' - displayName: 'Generate PR Diff' - - pwsh: | Write-Host "We freshly generated the PR diff, and this is what it sees!" Get-ChildItem -R -Force $(Build.ArtifactStagingDirectory)/diff | % { $_.FullName } From f192370d8325df732cc26c9a063f71572de77746 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Fri, 26 Jul 2024 18:11:20 -0700 Subject: [PATCH 035/119] adjustments to eng/common --- .../templates/steps/verify-changelogs.yml | 17 ++++++++++ eng/common/scripts/Verify-ChangeLogs.ps1 | 32 +++++++++++++++++++ eng/pipelines/templates/steps/analyze.yml | 22 ++++++++----- 3 files changed, 63 insertions(+), 8 deletions(-) create mode 100644 eng/common/pipelines/templates/steps/verify-changelogs.yml create mode 100644 eng/common/scripts/Verify-ChangeLogs.ps1 diff --git a/eng/common/pipelines/templates/steps/verify-changelogs.yml b/eng/common/pipelines/templates/steps/verify-changelogs.yml new file mode 100644 index 000000000000..08119f0fd8d5 --- /dev/null +++ b/eng/common/pipelines/templates/steps/verify-changelogs.yml @@ -0,0 +1,17 @@ +parameters: +- name: PackageInfoDirectory + type: string +- name: Condition + type: string + default: succeeded() + +steps: + - task: Powershell@2 + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/scripts/Verify-ChangeLogs.ps1 + arguments: > + -PackageInfoDirectory '${{ parameters.PackageInfoDirectory }}' + pwsh: true + displayName: Verify ChangeLogEntry for ${{ parameters.PackageName }} + condition: ${{ parameters.Condition }} + continueOnError: false diff --git a/eng/common/scripts/Verify-ChangeLogs.ps1 b/eng/common/scripts/Verify-ChangeLogs.ps1 new file mode 100644 index 000000000000..537a5fc98599 --- /dev/null +++ b/eng/common/scripts/Verify-ChangeLogs.ps1 @@ -0,0 +1,32 @@ +# Wrapper Script for ChangeLog Verification in a PR +[CmdletBinding()] +param ( + [String]$PackagePropertiesFolder +) +Set-StrictMode -Version 3 + +. (Join-Path $PSScriptRoot common.ps1) + +# find which packages we need to confirm the changelog for +$packageProperties = Get-ChildItem -Recurse -Force "$PackagePropertiesFolder" ` + | Where-Object { $_.Extension -eq '.json' } + +# grab the json file, then confirm the changelog entry for it +$allPassing = $true +foreach($propertiesFile in $packageProperties) { + $PackageProp = Get-Content -Path $propertiesFile | ConvertFrom-Json + + $validChangeLog = Confirm-ChangeLogEntry -ChangeLogLocation $PackageProp.ChangeLogPath -VersionString $PackageProp.Version -ForRelease $false + + if (-not $validChangeLog) { + $allPassing = $false + } +} + + +if (!$allPassing) +{ + exit 1 +} + +exit 0 \ No newline at end of file diff --git a/eng/pipelines/templates/steps/analyze.yml b/eng/pipelines/templates/steps/analyze.yml index 8e82325daf90..38a61b2dac90 100644 --- a/eng/pipelines/templates/steps/analyze.yml +++ b/eng/pipelines/templates/steps/analyze.yml @@ -27,14 +27,20 @@ steps: scriptPath: 'scripts/devops_tasks/set_tox_environment.py' arguments: '"$(TargetingString)" --team-project="$(System.TeamProject)" --service="${{ parameters.ServiceDirectory }}"' - - ${{ each artifact in parameters.Artifacts }}: - - ${{if ne(artifact.skipVerifyChangeLog, 'true')}}: - - template: /eng/common/pipelines/templates/steps/verify-changelog.yml - parameters: - PackageName: ${{artifact.name}} - ServiceName: ${{parameters.ServiceDirectory}} - ForRelease: false - Condition: succeededOrFailed() + - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: + - template: /eng/common/pipelines/templates/steps/verify-changelogs.yml + parameters: + PackagePropertiesFolder: $(Build.ArtifactStagingDirectory)/PackageInfo + Condition: succeededOrFailed() + - ${{ else }}: + - ${{ each artifact in parameters.Artifacts }}: + - ${{if ne(artifact.skipVerifyChangeLog, 'true')}}: + - template: /eng/common/pipelines/templates/steps/verify-changelog.yml + parameters: + PackageName: ${{artifact.name}} + ServiceName: ${{parameters.ServiceDirectory}} + ForRelease: false + Condition: succeededOrFailed() - script: | python -m pip install "./tools/azure-sdk-tools[build]" -q -I From 6699f69875cf9ecca483db2767b19f9058939e35 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Fri, 26 Jul 2024 18:18:37 -0700 Subject: [PATCH 036/119] PackageInfoDirectory -> PackagePropertiesFolder --- eng/common/pipelines/templates/steps/verify-changelogs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/pipelines/templates/steps/verify-changelogs.yml b/eng/common/pipelines/templates/steps/verify-changelogs.yml index 08119f0fd8d5..54a423b5e3b4 100644 --- a/eng/common/pipelines/templates/steps/verify-changelogs.yml +++ b/eng/common/pipelines/templates/steps/verify-changelogs.yml @@ -1,5 +1,5 @@ parameters: -- name: PackageInfoDirectory +- name: PackagePropertiesFolder type: string - name: Condition type: string From 3f0feaadd4c7b48fe5880876fc6aba6ec2cf4c4d Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Fri, 26 Jul 2024 18:18:53 -0700 Subject: [PATCH 037/119] update usage --- eng/common/pipelines/templates/steps/verify-changelogs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/pipelines/templates/steps/verify-changelogs.yml b/eng/common/pipelines/templates/steps/verify-changelogs.yml index 54a423b5e3b4..599e5f917e4a 100644 --- a/eng/common/pipelines/templates/steps/verify-changelogs.yml +++ b/eng/common/pipelines/templates/steps/verify-changelogs.yml @@ -10,7 +10,7 @@ steps: inputs: filePath: $(Build.SourcesDirectory)/eng/common/scripts/Verify-ChangeLogs.ps1 arguments: > - -PackageInfoDirectory '${{ parameters.PackageInfoDirectory }}' + -PackageInfoDirectory '${{ parameters.PackagePropertiesFolder }}' pwsh: true displayName: Verify ChangeLogEntry for ${{ parameters.PackageName }} condition: ${{ parameters.Condition }} From e854dd08ee221695be7a4d053561e84c917279e1 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Fri, 26 Jul 2024 18:29:47 -0700 Subject: [PATCH 038/119] update --- eng/common/pipelines/templates/steps/verify-changelogs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/pipelines/templates/steps/verify-changelogs.yml b/eng/common/pipelines/templates/steps/verify-changelogs.yml index 599e5f917e4a..7870f30cdc62 100644 --- a/eng/common/pipelines/templates/steps/verify-changelogs.yml +++ b/eng/common/pipelines/templates/steps/verify-changelogs.yml @@ -12,6 +12,6 @@ steps: arguments: > -PackageInfoDirectory '${{ parameters.PackagePropertiesFolder }}' pwsh: true - displayName: Verify ChangeLogEntry for ${{ parameters.PackageName }} + displayName: Verify ChangeLogEntries condition: ${{ parameters.Condition }} continueOnError: false From e3e6b508c247b7f824cf046c788db14a1cc1d210 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Mon, 29 Jul 2024 17:49:27 -0700 Subject: [PATCH 039/119] verify changelogs should work now --- eng/common/pipelines/templates/steps/verify-changelogs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/pipelines/templates/steps/verify-changelogs.yml b/eng/common/pipelines/templates/steps/verify-changelogs.yml index 7870f30cdc62..38a6cc354d28 100644 --- a/eng/common/pipelines/templates/steps/verify-changelogs.yml +++ b/eng/common/pipelines/templates/steps/verify-changelogs.yml @@ -10,7 +10,7 @@ steps: inputs: filePath: $(Build.SourcesDirectory)/eng/common/scripts/Verify-ChangeLogs.ps1 arguments: > - -PackageInfoDirectory '${{ parameters.PackagePropertiesFolder }}' + -PackagePropertiesFolder '${{ parameters.PackagePropertiesFolder }}' pwsh: true displayName: Verify ChangeLogEntries condition: ${{ parameters.Condition }} From 7d00fbbb6b8afffe508136303d2af6952481f75e Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 30 Jul 2024 11:36:16 -0700 Subject: [PATCH 040/119] updating how we handle readme verify --- .../templates/steps/verify-readmes.yml | 45 +++++++++++++++++++ .../templates/steps/analyze_dependency.yml | 19 +++++--- 2 files changed, 59 insertions(+), 5 deletions(-) create mode 100644 eng/common/pipelines/templates/steps/verify-readmes.yml diff --git a/eng/common/pipelines/templates/steps/verify-readmes.yml b/eng/common/pipelines/templates/steps/verify-readmes.yml new file mode 100644 index 000000000000..2dc77b2fd356 --- /dev/null +++ b/eng/common/pipelines/templates/steps/verify-readmes.yml @@ -0,0 +1,45 @@ +parameters: +- name: PackagePropertiesFolder + type: string +- name: RepoRoot + type: string + default: $(Build.SourcesDirectory) +- name: SettingsPath + type: string + default: '$(Build.SourcesDirectory)/eng/.docsettings.yml' +- name: DocWardenVersion + type: string + default: '' +- name: Condition + type: string + default: succeeded() + +steps: +- pwsh: | + # find which packages we need to confirm the changelog for + $packageProperties = Get-ChildItem -Recurse -Force "$PackagePropertiesFolder" ` + | Where-Object { $_.Extension -eq '.json' } + + $paths = @() + + foreach($propertiesFile in $packageProperties) { + $PackageProp = Get-Content -Path $propertiesFile | ConvertFrom-Json + + $paths += $PackageProp.DirectoryPath + } + + $scanPaths = $paths -join "," + Write-Host "##vso[task.setvariable variable=ScanPathArgument;]$scanPaths" + displayName: Populate Scan Paths + +- task: PowerShell@2 + displayName: "Verify Readmes" + condition: ${{ parameters.Condition }} + inputs: + filePath: "eng/common/scripts/Verify-Readme.ps1" + arguments: > + -DocWardenVersion '${{ parameters.DocWardenVersion }}' + -ScanPaths '$(ScanPathArgument)' + -RepoRoot ${{ parameters.RepoRoot }} + -SettingsPath ${{ parameters.SettingsPath }} + pwsh: true \ No newline at end of file diff --git a/eng/pipelines/templates/steps/analyze_dependency.yml b/eng/pipelines/templates/steps/analyze_dependency.yml index 9b4988c25156..8fafdcdc9581 100644 --- a/eng/pipelines/templates/steps/analyze_dependency.yml +++ b/eng/pipelines/templates/steps/analyze_dependency.yml @@ -9,15 +9,24 @@ steps: versionSpec: '$(PythonVersion)' - pwsh: | - python -m pip install setuptools==58.3.0 python -m pip install -r eng/ci_tools.txt displayName: 'Install Python Tools' condition: succeededOrFailed() - - template: /eng/common/pipelines/templates/steps/verify-readme.yml - parameters: - ScanPath: ${{ parameters.ScanPath }} - Condition: succeededOrFailed() + - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: + - template: /eng/common/pipelines/templates/steps/verify-changelogs.yml + parameters: + PackagePropertiesFolder: $(Build.ArtifactStagingDirectory)/PackageInfo + Condition: succeededOrFailed() + - template: /eng/common/pipelines/templates/steps/verify-readme.yml + parameters: + PackagePropertiesFolder: $(Build.ArtifactStagingDirectory)/PackageInfo + Condition: succeededOrFailed() + - ${{ else }}: + - template: /eng/common/pipelines/templates/steps/verify-readme.yml + parameters: + ScanPath: ${{ parameters.ScanPath }} + Condition: succeededOrFailed() - pwsh: | sdk_analyze_deps --verbose --out "$(Build.ArtifactStagingDirectory)/reports/dependencies.html" --dump "$(Build.ArtifactStagingDirectory)/reports" From da6ad20e0abffd3b9e5e8ee243436cbeca51154a Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 30 Jul 2024 11:50:07 -0700 Subject: [PATCH 041/119] resolve problem with the argument not being honored because the wrong file was being invoked --- eng/pipelines/templates/steps/analyze_dependency.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/eng/pipelines/templates/steps/analyze_dependency.yml b/eng/pipelines/templates/steps/analyze_dependency.yml index 8fafdcdc9581..93d149cdf46f 100644 --- a/eng/pipelines/templates/steps/analyze_dependency.yml +++ b/eng/pipelines/templates/steps/analyze_dependency.yml @@ -14,11 +14,7 @@ steps: condition: succeededOrFailed() - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: - - template: /eng/common/pipelines/templates/steps/verify-changelogs.yml - parameters: - PackagePropertiesFolder: $(Build.ArtifactStagingDirectory)/PackageInfo - Condition: succeededOrFailed() - - template: /eng/common/pipelines/templates/steps/verify-readme.yml + - template: /eng/common/pipelines/templates/steps/verify-readmes.yml parameters: PackagePropertiesFolder: $(Build.ArtifactStagingDirectory)/PackageInfo Condition: succeededOrFailed() From fd04ac93206046177634434b7c47b6927a891726 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 30 Jul 2024 12:37:40 -0700 Subject: [PATCH 042/119] we need to pass a parameter not a test value that will always be unpopulated in CI --- eng/common/pipelines/templates/steps/verify-readmes.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/eng/common/pipelines/templates/steps/verify-readmes.yml b/eng/common/pipelines/templates/steps/verify-readmes.yml index 2dc77b2fd356..08f8076725f8 100644 --- a/eng/common/pipelines/templates/steps/verify-readmes.yml +++ b/eng/common/pipelines/templates/steps/verify-readmes.yml @@ -16,9 +16,8 @@ parameters: steps: - pwsh: | - # find which packages we need to confirm the changelog for - $packageProperties = Get-ChildItem -Recurse -Force "$PackagePropertiesFolder" ` - | Where-Object { $_.Extension -eq '.json' } + $packageProperties = Get-ChildItem -Recurse -Force "${{ parameters.PackagePropertiesFolder }}" ` + | Where-Object { $_.Extension -eq '.json' } ` $paths = @() From c4507583c9e8ac56491e996c64195f2ddb33c4ff Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 30 Jul 2024 19:10:06 -0700 Subject: [PATCH 043/119] further progress for the evening --- .../templates/steps/detect-pr-api-changes.yml | 30 +++++ eng/common/scripts/Package-Properties.ps1 | 7 +- eng/scripts/Language-Settings.ps1 | 7 ++ eng/scripts/get_package_properties.py | 117 +++++++++++++----- 4 files changed, 130 insertions(+), 31 deletions(-) create mode 100644 eng/common/pipelines/templates/steps/detect-pr-api-changes.yml diff --git a/eng/common/pipelines/templates/steps/detect-pr-api-changes.yml b/eng/common/pipelines/templates/steps/detect-pr-api-changes.yml new file mode 100644 index 000000000000..e6a19c7a5ad6 --- /dev/null +++ b/eng/common/pipelines/templates/steps/detect-pr-api-changes.yml @@ -0,0 +1,30 @@ +parameters: + - name: PackagePropertiesFolder + type: string + - name: ArtifactPath + type: string + default: $(Build.ArtifactStagingDirectory) + +steps: + - pwsh: | + $apiChangeDetectRequestUrl = "https://apiview.dev/PullRequest/DetectApiChanges" + echo "##vso[task.setvariable variable=ApiChangeDetectRequestUrl]$apiChangeDetectRequestUrl" + displayName: "Set API change detect request URL" + condition: eq(variables['ApiChangeDetectRequestUrl'], '') + + - task: Powershell@2 + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/scripts/Detect-Api-Changes.ps1 + arguments: > + -ArtifactList ('${{ convertToJson(parameters.Artifacts) }}' | ConvertFrom-Json | Select-Object Name) + -ArtifactPath ${{parameters.ArtifactPath}} + -CommitSha '$(Build.SourceVersion)' + -BuildId $(Build.BuildId) + -PullRequestNumber $(System.PullRequest.PullRequestNumber) + -RepoFullName $(Build.Repository.Name) + -APIViewUri $(ApiChangeDetectRequestUrl) + -ArtifactName ${{ parameters.ArtifactName }} + -DevopsProject $(System.TeamProject) + pwsh: true + displayName: Detect API changes + condition: and(succeededOrFailed(), eq(variables['Build.Reason'],'PullRequest')) diff --git a/eng/common/scripts/Package-Properties.ps1 b/eng/common/scripts/Package-Properties.ps1 index 0e703a9d4374..9c47d82d53ad 100644 --- a/eng/common/scripts/Package-Properties.ps1 +++ b/eng/common/scripts/Package-Properties.ps1 @@ -15,6 +15,7 @@ class PackageProps [boolean]$IsNewSdk [string]$ArtifactName [string]$ReleaseStatus + [string[]]$DependentPackages PackageProps([string]$name, [string]$version, [string]$directoryPath, [string]$serviceDirectory) { @@ -55,7 +56,7 @@ class PackageProps if ($changeLogEntry -and $changeLogEntry.ReleaseStatus) { $this.ReleaseStatus = $changeLogEntry.ReleaseStatus.Trim().Trim("()") - } + } } else { @@ -101,7 +102,7 @@ function Get-PkgProperties return $pkgProps[0] } - LogError "Failed to retrive Properties for [$PackageName]" + LogError "Failed to retrieve Properties for [$PackageName]" return $null } @@ -112,6 +113,8 @@ function Get-PrPkgProperties([string]$InputDiffJson) { $diff = Get-Content $InputDiffJson | ConvertFrom-Json $targetedFiles = $diff.ChangedFiles + $dependentPackagesForInclusion = @() + foreach($pkg in $allPackageProperties) { $pkgDirectory = Resolve-Path "$($pkg.DirectoryPath)" diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 index 092fbd739e8b..331d3896503f 100644 --- a/eng/scripts/Language-Settings.ps1 +++ b/eng/scripts/Language-Settings.ps1 @@ -35,13 +35,19 @@ function Get-AllPackageInfoFromRepo ($serviceDirectory) Pop-Location } + foreach ($line in $allPkgPropLines) { + Write-Host $line + $pkgInfo = ($line -Split " ") $packageName = $pkgInfo[0] $packageVersion = $pkgInfo[1] $isNewSdk = ($pkgInfo[2] -eq "True") $pkgDirectoryPath = $pkgInfo[3] + + $additionalDependentPackages = $pkgInfo[4] -Split "," + $serviceDirectoryName = Split-Path (Split-Path -Path $pkgDirectoryPath -Parent) -Leaf if ($packageName -match "mgmt") { @@ -55,6 +61,7 @@ function Get-AllPackageInfoFromRepo ($serviceDirectory) $pkgProp.IsNewSdk = $isNewSdk $pkgProp.SdkType = $sdkType $pkgProp.ArtifactName = $packageName + $pkgProp.DependentPackages = $additionalDependentPackages $allPackageProps += $pkgProp } return $allPackageProps diff --git a/eng/scripts/get_package_properties.py b/eng/scripts/get_package_properties.py index a1d805e78086..71c768c6bc62 100644 --- a/eng/scripts/get_package_properties.py +++ b/eng/scripts/get_package_properties.py @@ -1,42 +1,101 @@ import argparse +import sys import glob import os import re +from typing import Dict, List + from ci_tools.parsing import ParsedSetup -root_dir = os.path.abspath(os.path.join(os.path.abspath(__file__), "..", "..", "..")) +additional_pr_triggers: Dict[str, List[str]] = { + "azure-core":[ + "sdk/storage/azure-storage-blob", + "sdk/servicebus/azure-servicebus", + "sdk/eventhub/azure-eventhub", + "sdk/tables/azure-data-table", + "sdk/appconfig/azure-appconfig", + "sdk/keyvault/azure-keyvault-keys", + "sdk/identity/azure-identity", + "sdk/core/azure-mgmt-core", + "sdk/core/azure-core-experimental", + "sdk/core/azure-core-tracing-opentelemetry", + "sdk/core/azure-core-tracing-opencensus", + "sdk/cosmos/azure-cosmos", + "sdk/ml/azure-ai-ml", + "sdk/ai/azure-ai-documentintelligence", + "sdk/ai/azure-ai-inference", + "sdk/ai/azure-ai-textanalytics", + "sdk/ai/azure-ai-doctranslation", + "sdk/compute/azure-mgmt-compute", + "sdk/communication/azure-communication-chat", + "sdk/communication/azure-communication-identity", + ], + "azure-mgmt-core": [ + "sdk/compute/azure-mgmt-compute", + "sdk/network/azure-mgmt-network", + "sdk/resource/azure-mgmt-resource", + "sdk/keyvault/azure-mgmt-keyvault", + ] +} + +# todo triggers based on paths and not files +# tools/ +# azure-storage-blob +# azure-servicebus +# azure-eventhub +# azure-data-table +# azure-appconfig +# azure-keyvault-keys +# azure-identity +# azure-mgmt-core +# azure-core-experimental +# azure-core-tracing-opentelemetry +# azure-core-tracing-opencensus +# azure-cosmos +# azure-ai-documentintelligence +# azure-ai-ml +# azure-ai-inference +# azure-ai-textanalytics +# azure-ai-doctranslation +# azure-mgmt-compute +# azure-communication-chat +# azure-communication-identity + +# eng/ +# azure-template +# azure-core + +# scripts/, doc/, common/, conda/ +# azure-template +# azure-core -def dump_setup(path: str) -> None: - try: - parsed = ParsedSetup.from_path(path) - print( - "{0} {1} {2} {3}".format( - parsed.name, parsed.version, parsed.is_new_sdk, os.path.dirname(parsed.setup_filename) - ) - ) - except: - pass if __name__ == "__main__": parser = argparse.ArgumentParser(description="Get package version details from the repo") - parser.add_argument("-s", "--search_path", required=False, help="The scope of the search. Usually a service directory path sdk/X.") - parser.add_argument("-p", "--packages", required=False, help="The specific package names to search for. Comma separated list.") + parser.add_argument("-s", "--search_path", required=True, help="The scope of the search") args = parser.parse_args() - if args.search_path: - # Use abspath for the os.walk because if setup parsing fails it often changes cwd which throws off the relative walk - for root, dirs, files in os.walk(os.path.abspath(args.search_path)): - if re.search(r"sdk[\\/][^\\/]+[\\/][^\\/]+$", root): - if "setup.py" in files: - dump_setup(root) - elif args.packages: - for pkg in [pkg.strip() for pkg in args.packages.split(",")]: - pkg_search = os.path.join(root_dir, "sdk", "*", pkg, "setup.py") - globbed = glob.glob(pkg_search) - - if globbed: - for match in globbed: - dump_setup(match) - else: - raise ValueError("Must provide either a search path or a package name list.") \ No newline at end of file + # Use abspath for the os.walk because if setup parsing fails it often changes cwd which throws off the relative walk + for root, dirs, files in os.walk(os.path.abspath(args.search_path)): + if re.search(r"sdk[\\/][^\\/]+[\\/][^\\/]+$", root): + if "setup.py" in files: + try: + parsed = ParsedSetup.from_path(root) + + dependent_packages = "" + if parsed.name in additional_pr_triggers: + dependent_packages = ",".join(additional_pr_triggers[parsed.name]) + + print( + "{0} {1} {2} {3} {4}".format( + parsed.name, + parsed.version, + parsed.is_new_sdk, + os.path.dirname(parsed.setup_filename), + dependent_packages + ) + ) + except: + # Skip setup.py if the package cannot be parsed + pass \ No newline at end of file From c65a6d965a5c11afd97a12b55e8a82215b9b6117 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Wed, 31 Jul 2024 17:46:11 -0700 Subject: [PATCH 044/119] ok so now we expand our targeted packages per the language settings --- eng/common/scripts/Package-Properties.ps1 | 20 ++++++++- eng/scripts/Language-Settings.ps1 | 2 - eng/scripts/get_package_properties.py | 50 +++++++++++------------ 3 files changed, 42 insertions(+), 30 deletions(-) diff --git a/eng/common/scripts/Package-Properties.ps1 b/eng/common/scripts/Package-Properties.ps1 index 9c47d82d53ad..7e2338c53679 100644 --- a/eng/common/scripts/Package-Properties.ps1 +++ b/eng/common/scripts/Package-Properties.ps1 @@ -114,21 +114,37 @@ function Get-PrPkgProperties([string]$InputDiffJson) { $targetedFiles = $diff.ChangedFiles $dependentPackagesForInclusion = @() + $lookup = @{} - foreach($pkg in $allPackageProperties) + foreach ($pkg in $allPackageProperties) { $pkgDirectory = Resolve-Path "$($pkg.DirectoryPath)" + $lookupKey = ($pkg.DirectoryPath).Replace($RepoRoot, "").SubString(1) + $lookup[$lookupKey] = $pkg + Write-Host $lookupKey - foreach($file in $targetedFiles) + foreach ($file in $targetedFiles) { $filePath = Resolve-Path (Join-Path $RepoRoot $file) $shouldInclude = $filePath -like "$pkgDirectory*" if ($shouldInclude) { $packagesWithChanges += $pkg + + if ($pkg.DependentPackages) { + $dependentPackagesForInclusion += $pkg.DependentPackages + } } } } + foreach ($addition in $dependentPackagesForInclusion) { + Write-Host $addition + + if ($lookup[$addition]) { + $packagesWithChanges += $lookup[$addition] + } + } + return $packagesWithChanges } diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 index 331d3896503f..91b13e924f69 100644 --- a/eng/scripts/Language-Settings.ps1 +++ b/eng/scripts/Language-Settings.ps1 @@ -38,8 +38,6 @@ function Get-AllPackageInfoFromRepo ($serviceDirectory) foreach ($line in $allPkgPropLines) { - Write-Host $line - $pkgInfo = ($line -Split " ") $packageName = $pkgInfo[0] $packageVersion = $pkgInfo[1] diff --git a/eng/scripts/get_package_properties.py b/eng/scripts/get_package_properties.py index 71c768c6bc62..8598fc61a037 100644 --- a/eng/scripts/get_package_properties.py +++ b/eng/scripts/get_package_properties.py @@ -1,6 +1,4 @@ import argparse -import sys -import glob import os import re @@ -10,32 +8,32 @@ additional_pr_triggers: Dict[str, List[str]] = { "azure-core":[ - "sdk/storage/azure-storage-blob", - "sdk/servicebus/azure-servicebus", - "sdk/eventhub/azure-eventhub", - "sdk/tables/azure-data-table", - "sdk/appconfig/azure-appconfig", - "sdk/keyvault/azure-keyvault-keys", - "sdk/identity/azure-identity", - "sdk/core/azure-mgmt-core", - "sdk/core/azure-core-experimental", - "sdk/core/azure-core-tracing-opentelemetry", - "sdk/core/azure-core-tracing-opencensus", - "sdk/cosmos/azure-cosmos", - "sdk/ml/azure-ai-ml", - "sdk/ai/azure-ai-documentintelligence", - "sdk/ai/azure-ai-inference", - "sdk/ai/azure-ai-textanalytics", - "sdk/ai/azure-ai-doctranslation", - "sdk/compute/azure-mgmt-compute", - "sdk/communication/azure-communication-chat", - "sdk/communication/azure-communication-identity", + os.path.join("sdk", "storage", "azure-storage-blob"), + os.path.join("sdk", "servicebus", "azure-servicebus"), + os.path.join("sdk", "eventhub", "azure-eventhub"), + os.path.join("sdk", "tables", "azure-data-table"), + os.path.join("sdk", "appconfig", "azure-appconfig"), + os.path.join("sdk", "keyvault", "azure-keyvault-keys"), + os.path.join("sdk", "identity", "azure-identity"), + os.path.join("sdk", "core", "azure-mgmt-core"), + os.path.join("sdk", "core", "azure-core-experimental"), + os.path.join("sdk", "core", "azure-core-tracing-opentelemetry"), + os.path.join("sdk", "core", "azure-core-tracing-opencensus"), + os.path.join("sdk", "cosmos", "azure-cosmos"), + os.path.join("sdk", "ml", "azure-ai-ml"), + os.path.join("sdk", "ai", "azure-ai-documentintelligence"), + os.path.join("sdk", "ai", "azure-ai-inference"), + os.path.join("sdk", "ai", "azure-ai-textanalytics"), + os.path.join("sdk", "ai", "azure-ai-doctranslation"), + os.path.join("sdk", "compute", "azure-mgmt-compute"), + os.path.join("sdk", "communication", "azure-communication-chat"), + os.path.join("sdk", "communication", "azure-communication-identity") ], "azure-mgmt-core": [ - "sdk/compute/azure-mgmt-compute", - "sdk/network/azure-mgmt-network", - "sdk/resource/azure-mgmt-resource", - "sdk/keyvault/azure-mgmt-keyvault", + os.path.join("sdk", "compute", "azure-mgmt-compute"), + os.path.join("sdk", "network", "azure-mgmt-network"), + os.path.join("sdk", "resource", "azure-mgmt-resource"), + os.path.join("sdk", "keyvault", "azure-mgmt-keyvault") ] } From 71503875deb42ecacf271d5e79d67b1a7a8e5849 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Wed, 31 Jul 2024 18:31:12 -0700 Subject: [PATCH 045/119] handle changes to detectApiChanges --- .../templates/steps/detect-api-changes.yml | 2 -- .../templates/steps/verify-readmes.yml | 2 ++ eng/common/scripts/Detect-Api-Changes.ps1 | 20 ++++++++++++------- eng/common/scripts/Package-Properties.ps1 | 4 +--- 4 files changed, 16 insertions(+), 12 deletions(-) diff --git a/eng/common/pipelines/templates/steps/detect-api-changes.yml b/eng/common/pipelines/templates/steps/detect-api-changes.yml index f39a88eaa3a5..8d605d8694b1 100644 --- a/eng/common/pipelines/templates/steps/detect-api-changes.yml +++ b/eng/common/pipelines/templates/steps/detect-api-changes.yml @@ -1,6 +1,5 @@ parameters: ArtifactPath: $(Build.ArtifactStagingDirectory) - Artifacts: [] ArtifactName: 'packages' steps: @@ -14,7 +13,6 @@ steps: inputs: filePath: $(Build.SourcesDirectory)/eng/common/scripts/Detect-Api-Changes.ps1 arguments: > - -ArtifactList ('${{ convertToJson(parameters.Artifacts) }}' | ConvertFrom-Json | Select-Object Name) -ArtifactPath ${{parameters.ArtifactPath}} -CommitSha '$(Build.SourceVersion)' -BuildId $(Build.BuildId) diff --git a/eng/common/pipelines/templates/steps/verify-readmes.yml b/eng/common/pipelines/templates/steps/verify-readmes.yml index 08f8076725f8..949e6eddb31f 100644 --- a/eng/common/pipelines/templates/steps/verify-readmes.yml +++ b/eng/common/pipelines/templates/steps/verify-readmes.yml @@ -34,6 +34,8 @@ steps: - task: PowerShell@2 displayName: "Verify Readmes" condition: ${{ parameters.Condition }} + # todo: fix this continueOnError + continueOnError: true inputs: filePath: "eng/common/scripts/Verify-Readme.ps1" arguments: > diff --git a/eng/common/scripts/Detect-Api-Changes.ps1 b/eng/common/scripts/Detect-Api-Changes.ps1 index 522346502332..550f535a7ee4 100644 --- a/eng/common/scripts/Detect-Api-Changes.ps1 +++ b/eng/common/scripts/Detect-Api-Changes.ps1 @@ -10,8 +10,6 @@ Param ( [string] $BuildId, [Parameter(Mandatory=$True)] [string] $CommitSha, - [Parameter(Mandatory=$True)] - [array] $ArtifactList, [string] $APIViewUri, [string] $RepoFullName = "", [string] $ArtifactName = "packages", @@ -21,6 +19,8 @@ Param ( . (Join-Path $PSScriptRoot common.ps1) +$configFileDir = Join-Path -Path $ArtifactPath "PackageInfo" + # Submit API review request and return status whether current revision is approved or pending or failed to create review function Submit-Request($filePath, $packageName) { @@ -64,7 +64,6 @@ function Submit-Request($filePath, $packageName) function Should-Process-Package($pkgPath, $packageName) { $pkg = Split-Path -Leaf $pkgPath - $configFileDir = Join-Path -Path $ArtifactPath "PackageInfo" $pkgPropPath = Join-Path -Path $configFileDir "$packageName.json" if (!(Test-Path $pkgPropPath)) { @@ -103,15 +102,22 @@ if (!($FindArtifactForApiReviewFn -and (Test-Path "Function:$FindArtifactForApiR } $responses = @{} -foreach ($artifact in $ArtifactList) + +$packageProperties = Get-ChildItem -Recurse -Force "$configFileDir" ` + | Where-Object { $_.Extension -eq '.json' } + +foreach ($packagePropFile in $packageProperties) { - Write-Host "Processing $($artifact.name)" - $packages = &$FindArtifactForApiReviewFn $ArtifactPath $artifact.name + $packageMetadata = Get-Content $packagePropFile | ConvertFrom-Json + Write-Host "Processing $($packageMetadata.Name)" + + $packages = &$FindArtifactForApiReviewFn $ArtifactPath $packageMetadata.Name + if ($packages) { $pkgPath = $packages.Values[0] $isRequired = Should-Process-Package -pkgPath $pkgPath -packageName $artifact.name - Write-Host "Is API change detect required for $($artifact.name):$($isRequired)" + Write-Host "Is API change detect required for $($packages.Name):$($isRequired)" if ($isRequired -eq $True) { $filePath = $pkgPath.Replace($ArtifactPath , "").Replace("\", "/") diff --git a/eng/common/scripts/Package-Properties.ps1 b/eng/common/scripts/Package-Properties.ps1 index 7e2338c53679..7ea17de44ccb 100644 --- a/eng/common/scripts/Package-Properties.ps1 +++ b/eng/common/scripts/Package-Properties.ps1 @@ -16,6 +16,7 @@ class PackageProps [string]$ArtifactName [string]$ReleaseStatus [string[]]$DependentPackages + [string]$IncludedByFile PackageProps([string]$name, [string]$version, [string]$directoryPath, [string]$serviceDirectory) { @@ -121,7 +122,6 @@ function Get-PrPkgProperties([string]$InputDiffJson) { $pkgDirectory = Resolve-Path "$($pkg.DirectoryPath)" $lookupKey = ($pkg.DirectoryPath).Replace($RepoRoot, "").SubString(1) $lookup[$lookupKey] = $pkg - Write-Host $lookupKey foreach ($file in $targetedFiles) { @@ -138,8 +138,6 @@ function Get-PrPkgProperties([string]$InputDiffJson) { } foreach ($addition in $dependentPackagesForInclusion) { - Write-Host $addition - if ($lookup[$addition]) { $packagesWithChanges += $lookup[$addition] } From 52b5e010a9e8f59dac862747b769706daa1decbc Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Wed, 31 Jul 2024 18:43:22 -0700 Subject: [PATCH 046/119] debug output so we can understand why this isn't matching --- eng/common/scripts/Package-Properties.ps1 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eng/common/scripts/Package-Properties.ps1 b/eng/common/scripts/Package-Properties.ps1 index 7ea17de44ccb..fca09133acbe 100644 --- a/eng/common/scripts/Package-Properties.ps1 +++ b/eng/common/scripts/Package-Properties.ps1 @@ -121,6 +121,7 @@ function Get-PrPkgProperties([string]$InputDiffJson) { { $pkgDirectory = Resolve-Path "$($pkg.DirectoryPath)" $lookupKey = ($pkg.DirectoryPath).Replace($RepoRoot, "").SubString(1) + Write-Host "Storing $($pkg.Name) in $lookupKey" $lookup[$lookupKey] = $pkg foreach ($file in $targetedFiles) @@ -138,6 +139,7 @@ function Get-PrPkgProperties([string]$InputDiffJson) { } foreach ($addition in $dependentPackagesForInclusion) { + Write-Host "Examining $addition to see if it shows up in the complete package list" if ($lookup[$addition]) { $packagesWithChanges += $lookup[$addition] } From 27fd2c457dbb9217ddd91162389ac4a8cdb92bba Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Wed, 31 Jul 2024 19:15:35 -0700 Subject: [PATCH 047/119] even more debug output! trying to understand why dependent packages aren't working properly in CI, but are fine locally --- eng/common/scripts/Package-Properties.ps1 | 1 + 1 file changed, 1 insertion(+) diff --git a/eng/common/scripts/Package-Properties.ps1 b/eng/common/scripts/Package-Properties.ps1 index fca09133acbe..3659940eb729 100644 --- a/eng/common/scripts/Package-Properties.ps1 +++ b/eng/common/scripts/Package-Properties.ps1 @@ -131,6 +131,7 @@ function Get-PrPkgProperties([string]$InputDiffJson) { if ($shouldInclude) { $packagesWithChanges += $pkg + Write-Host "While examining $($pkg.Name) I saw the follinwg dependent packages $($pkg.DependentPackages)" if ($pkg.DependentPackages) { $dependentPackagesForInclusion += $pkg.DependentPackages } From d3518bfdb85bb809fefd5cef617e7ed96c742616 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Thu, 1 Aug 2024 10:39:27 -0700 Subject: [PATCH 048/119] why is that not outputting? --- eng/common/scripts/Package-Properties.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/scripts/Package-Properties.ps1 b/eng/common/scripts/Package-Properties.ps1 index 3659940eb729..d2148456e2c3 100644 --- a/eng/common/scripts/Package-Properties.ps1 +++ b/eng/common/scripts/Package-Properties.ps1 @@ -131,7 +131,7 @@ function Get-PrPkgProperties([string]$InputDiffJson) { if ($shouldInclude) { $packagesWithChanges += $pkg - Write-Host "While examining $($pkg.Name) I saw the follinwg dependent packages $($pkg.DependentPackages)" + Write-Host "While examining $($pkg.Name) I saw the following dependent packages $($pkg.DependentPackages)" if ($pkg.DependentPackages) { $dependentPackagesForInclusion += $pkg.DependentPackages } From ec296172ee14c28f7bceb7c555212129985286b2 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Thu, 1 Aug 2024 11:02:17 -0700 Subject: [PATCH 049/119] resolve issue with detect api changes --- .../templates/steps/detect-pr-api-changes.yml | 30 ------------------- eng/common/scripts/Detect-Api-Changes.ps1 | 10 +++---- 2 files changed, 5 insertions(+), 35 deletions(-) delete mode 100644 eng/common/pipelines/templates/steps/detect-pr-api-changes.yml diff --git a/eng/common/pipelines/templates/steps/detect-pr-api-changes.yml b/eng/common/pipelines/templates/steps/detect-pr-api-changes.yml deleted file mode 100644 index e6a19c7a5ad6..000000000000 --- a/eng/common/pipelines/templates/steps/detect-pr-api-changes.yml +++ /dev/null @@ -1,30 +0,0 @@ -parameters: - - name: PackagePropertiesFolder - type: string - - name: ArtifactPath - type: string - default: $(Build.ArtifactStagingDirectory) - -steps: - - pwsh: | - $apiChangeDetectRequestUrl = "https://apiview.dev/PullRequest/DetectApiChanges" - echo "##vso[task.setvariable variable=ApiChangeDetectRequestUrl]$apiChangeDetectRequestUrl" - displayName: "Set API change detect request URL" - condition: eq(variables['ApiChangeDetectRequestUrl'], '') - - - task: Powershell@2 - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/scripts/Detect-Api-Changes.ps1 - arguments: > - -ArtifactList ('${{ convertToJson(parameters.Artifacts) }}' | ConvertFrom-Json | Select-Object Name) - -ArtifactPath ${{parameters.ArtifactPath}} - -CommitSha '$(Build.SourceVersion)' - -BuildId $(Build.BuildId) - -PullRequestNumber $(System.PullRequest.PullRequestNumber) - -RepoFullName $(Build.Repository.Name) - -APIViewUri $(ApiChangeDetectRequestUrl) - -ArtifactName ${{ parameters.ArtifactName }} - -DevopsProject $(System.TeamProject) - pwsh: true - displayName: Detect API changes - condition: and(succeededOrFailed(), eq(variables['Build.Reason'],'PullRequest')) diff --git a/eng/common/scripts/Detect-Api-Changes.ps1 b/eng/common/scripts/Detect-Api-Changes.ps1 index 550f535a7ee4..e0c269b13ec1 100644 --- a/eng/common/scripts/Detect-Api-Changes.ps1 +++ b/eng/common/scripts/Detect-Api-Changes.ps1 @@ -116,25 +116,25 @@ foreach ($packagePropFile in $packageProperties) if ($packages) { $pkgPath = $packages.Values[0] - $isRequired = Should-Process-Package -pkgPath $pkgPath -packageName $artifact.name + $isRequired = Should-Process-Package -pkgPath $pkgPath -packageName $($packageMetadata.Name) Write-Host "Is API change detect required for $($packages.Name):$($isRequired)" if ($isRequired -eq $True) { $filePath = $pkgPath.Replace($ArtifactPath , "").Replace("\", "/") - $respCode = Submit-Request -filePath $filePath -packageName $artifact.name + $respCode = Submit-Request -filePath $filePath -packageName $($packageMetadata.Name) if ($respCode -ne '200') { - $responses[$artifact.name] = $respCode + $responses[$($packageMetadata.Name)] = $respCode } } else { - Write-Host "Pull request does not have any change for $($artifact.name). Skipping API change detect." + Write-Host "Pull request does not have any change for $($packageMetadata.Name)). Skipping API change detect." } } else { - Write-Host "No package is found in artifact path to find API changes for $($artifact.name)" + Write-Host "No package is found in artifact path to find API changes for $($packageMetadata.Name)" } } From 18cbb11cf989fd67c786972bfe72049031b80a4f Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Thu, 1 Aug 2024 11:22:51 -0700 Subject: [PATCH 050/119] to fix the readme issues, we just need to ensure that the path passed in includes the repo root path --- eng/common/pipelines/templates/steps/verify-readmes.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/pipelines/templates/steps/verify-readmes.yml b/eng/common/pipelines/templates/steps/verify-readmes.yml index 949e6eddb31f..0e288ce1d1be 100644 --- a/eng/common/pipelines/templates/steps/verify-readmes.yml +++ b/eng/common/pipelines/templates/steps/verify-readmes.yml @@ -24,7 +24,7 @@ steps: foreach($propertiesFile in $packageProperties) { $PackageProp = Get-Content -Path $propertiesFile | ConvertFrom-Json - $paths += $PackageProp.DirectoryPath + $paths += (Join-Path "$(Build.SourcesDirectory)" $PackageProp.DirectoryPath) } $scanPaths = $paths -join "," From e9fda5eaacd6b0f9192aff27a7a6cd3b4f67a0a1 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Thu, 1 Aug 2024 11:53:40 -0700 Subject: [PATCH 051/119] remove debug output --- eng/common/scripts/Generate-PR-Diff.ps1 | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/eng/common/scripts/Generate-PR-Diff.ps1 b/eng/common/scripts/Generate-PR-Diff.ps1 index 55c10f2425a7..c0ea3d1493d0 100644 --- a/eng/common/scripts/Generate-PR-Diff.ps1 +++ b/eng/common/scripts/Generate-PR-Diff.ps1 @@ -20,11 +20,7 @@ Param ( ) -$path = (Join-Path $PSScriptRoot "Helpers" "git-helpers.ps1") - -Write-Host "This is what we're trying to dot include $path" - -. $path +. (Join-Path $PSScriptRoot "Helpers" "git-helpers.ps1") function Get-ChangedServices { Param ( From 949e324c4036a69b427437658c51947fc88012fe Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Thu, 1 Aug 2024 12:02:05 -0700 Subject: [PATCH 052/119] remove additional debug statements --- eng/common/scripts/Package-Properties.ps1 | 4 ---- 1 file changed, 4 deletions(-) diff --git a/eng/common/scripts/Package-Properties.ps1 b/eng/common/scripts/Package-Properties.ps1 index d2148456e2c3..2250b4f80ba6 100644 --- a/eng/common/scripts/Package-Properties.ps1 +++ b/eng/common/scripts/Package-Properties.ps1 @@ -16,7 +16,6 @@ class PackageProps [string]$ArtifactName [string]$ReleaseStatus [string[]]$DependentPackages - [string]$IncludedByFile PackageProps([string]$name, [string]$version, [string]$directoryPath, [string]$serviceDirectory) { @@ -121,7 +120,6 @@ function Get-PrPkgProperties([string]$InputDiffJson) { { $pkgDirectory = Resolve-Path "$($pkg.DirectoryPath)" $lookupKey = ($pkg.DirectoryPath).Replace($RepoRoot, "").SubString(1) - Write-Host "Storing $($pkg.Name) in $lookupKey" $lookup[$lookupKey] = $pkg foreach ($file in $targetedFiles) @@ -131,7 +129,6 @@ function Get-PrPkgProperties([string]$InputDiffJson) { if ($shouldInclude) { $packagesWithChanges += $pkg - Write-Host "While examining $($pkg.Name) I saw the following dependent packages $($pkg.DependentPackages)" if ($pkg.DependentPackages) { $dependentPackagesForInclusion += $pkg.DependentPackages } @@ -140,7 +137,6 @@ function Get-PrPkgProperties([string]$InputDiffJson) { } foreach ($addition in $dependentPackagesForInclusion) { - Write-Host "Examining $addition to see if it shows up in the complete package list" if ($lookup[$addition]) { $packagesWithChanges += $lookup[$addition] } From a5c76d0331e6ae28905e5c313e010b4b5d716486 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Thu, 1 Aug 2024 12:03:25 -0700 Subject: [PATCH 053/119] update the test timeout to 120 minutes for the pr build --- sdk/pr.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sdk/pr.yml b/sdk/pr.yml index 5fcf42f176fc..3b5c01503667 100644 --- a/sdk/pr.yml +++ b/sdk/pr.yml @@ -19,4 +19,5 @@ extends: ServiceDirectory: ${{ parameters.Service }} BuildTargetingString: "*" ValidateFormatting: true - TestProxy: true \ No newline at end of file + TestProxy: true + TestTimeOutInMinutes: 120 \ No newline at end of file From 3c33a47a300d5a70a3e12722ef5b0f43e62c8227 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 6 Aug 2024 15:16:06 -0700 Subject: [PATCH 054/119] extend timeout even longer --- sdk/pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/pr.yml b/sdk/pr.yml index 3b5c01503667..f7ae0940527c 100644 --- a/sdk/pr.yml +++ b/sdk/pr.yml @@ -20,4 +20,4 @@ extends: BuildTargetingString: "*" ValidateFormatting: true TestProxy: true - TestTimeOutInMinutes: 120 \ No newline at end of file + TestTimeOutInMinutes: 180 \ No newline at end of file From b1778d34d8af0e47b7fa3e49c137ee183f91eba9 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 6 Aug 2024 16:25:29 -0700 Subject: [PATCH 055/119] update from Name to ArtifactName wthin Detect-Api-Changes, as js needs that specific property and python isn't negatively affected --- eng/common/scripts/Detect-Api-Changes.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/scripts/Detect-Api-Changes.ps1 b/eng/common/scripts/Detect-Api-Changes.ps1 index e0c269b13ec1..b25727f141d9 100644 --- a/eng/common/scripts/Detect-Api-Changes.ps1 +++ b/eng/common/scripts/Detect-Api-Changes.ps1 @@ -116,7 +116,7 @@ foreach ($packagePropFile in $packageProperties) if ($packages) { $pkgPath = $packages.Values[0] - $isRequired = Should-Process-Package -pkgPath $pkgPath -packageName $($packageMetadata.Name) + $isRequired = Should-Process-Package -pkgPath $pkgPath -packageName $($packageMetadata.ArtifactName) Write-Host "Is API change detect required for $($packages.Name):$($isRequired)" if ($isRequired -eq $True) { From c23b1cf9c962c9cecf898e2f290737380a85ac24 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 6 Aug 2024 16:28:13 -0700 Subject: [PATCH 056/119] actually handle the rename properly --- eng/common/scripts/Detect-Api-Changes.ps1 | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/eng/common/scripts/Detect-Api-Changes.ps1 b/eng/common/scripts/Detect-Api-Changes.ps1 index b25727f141d9..d590ce462406 100644 --- a/eng/common/scripts/Detect-Api-Changes.ps1 +++ b/eng/common/scripts/Detect-Api-Changes.ps1 @@ -109,7 +109,7 @@ $packageProperties = Get-ChildItem -Recurse -Force "$configFileDir" ` foreach ($packagePropFile in $packageProperties) { $packageMetadata = Get-Content $packagePropFile | ConvertFrom-Json - Write-Host "Processing $($packageMetadata.Name)" + Write-Host "Processing $($packageMetadata.ArtifactName)" $packages = &$FindArtifactForApiReviewFn $ArtifactPath $packageMetadata.Name @@ -121,20 +121,20 @@ foreach ($packagePropFile in $packageProperties) if ($isRequired -eq $True) { $filePath = $pkgPath.Replace($ArtifactPath , "").Replace("\", "/") - $respCode = Submit-Request -filePath $filePath -packageName $($packageMetadata.Name) + $respCode = Submit-Request -filePath $filePath -packageName $($packageMetadata.ArtifactName) if ($respCode -ne '200') { - $responses[$($packageMetadata.Name)] = $respCode + $responses[$($packageMetadata.ArtifactName)] = $respCode } } else { - Write-Host "Pull request does not have any change for $($packageMetadata.Name)). Skipping API change detect." + Write-Host "Pull request does not have any change for $($packageMetadata.ArtifactName)). Skipping API change detect." } } else { - Write-Host "No package is found in artifact path to find API changes for $($packageMetadata.Name)" + Write-Host "No package is found in artifact path to find API changes for $($packageMetadata.ArtifactName)" } } From e0884a473acedd349af31185cded5ab1443a0219 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Thu, 8 Aug 2024 13:20:32 -0700 Subject: [PATCH 057/119] make this thing totally ignore service directory when testing. so that build targeting string can properly win out --- eng/pipelines/templates/jobs/ci.tests.yml | 1 - .../steps/resolve-package-targeting.yml | 37 ++++++++----------- 2 files changed, 16 insertions(+), 22 deletions(-) diff --git a/eng/pipelines/templates/jobs/ci.tests.yml b/eng/pipelines/templates/jobs/ci.tests.yml index 2acfd6ad329e..4df0094c79cf 100644 --- a/eng/pipelines/templates/jobs/ci.tests.yml +++ b/eng/pipelines/templates/jobs/ci.tests.yml @@ -119,7 +119,6 @@ jobs: - template: ../steps/build-test.yml parameters: - ServiceDirectory: ${{ parameters.ServiceDirectory }} TestMarkArgument: ${{ parameters.TestMarkArgument }} AdditionalTestArgs: '--wheel_dir="$(Build.ArtifactStagingDirectory)"' CoverageArg: $(CoverageArg) diff --git a/eng/pipelines/templates/steps/resolve-package-targeting.yml b/eng/pipelines/templates/steps/resolve-package-targeting.yml index e6346e21449b..d3d6e84caa47 100644 --- a/eng/pipelines/templates/steps/resolve-package-targeting.yml +++ b/eng/pipelines/templates/steps/resolve-package-targeting.yml @@ -7,10 +7,17 @@ parameters: default: '' steps: - # if triggered for a PR, handle PR diff side adding the default for packageproperties folder will mean this will always act - # as a non-pr resolution - - ${{ if and(eq(variables['Build.Reason'], 'PullRequest'), ne(parameters.PackagePropertiesFolder, '')) }}: - - pwsh: | + # whether we are running within a PR build or not, we need to walk the package properties and save them to + # variable $(TargetingString) so that every other check and accept it as an argument. + - pwsh: | + $setting = "${{ parameters.BuildTargetingString }}" + + # if the variable is not set, it'll just come back as the variable name. otherwise it's set. + if ('$(BuildTargetingString)' -ne ('$' + '(BuildTargetingString)')) { + Write-Host "The variable named BuildTargetingString is set to $(BuildTargetingString)" + $setting = "$(BuildTargetingString)" + } + else { $packageProperties = Get-ChildItem -Recurse -Force "${{ parameters.PackagePropertiesFolder }}" ` | Where-Object { $_.Extension -eq '.json' } ` | Foreach-Object { $_.Name } ` @@ -18,20 +25,8 @@ steps: $setting = $packageProperties -join "," Write-Host "Setting TargetingString to $setting" Write-Host "##vso[task.setvariable variable=TargetingString;]$setting" - displayName: Resolve PR Package Targets - # if not triggered for a PR, simply attempt to locate the targeted packages from the service directory combined with runtime variable. - - ${{ else }}: - - pwsh: | - $setting = "${{ parameters.BuildTargetingString }}" - # if the variable is not set, it'll just come back as the variable name. otherwise it's set. - if ('$(BuildTargetingString)' -ne ('$' + '(BuildTargetingString)')) { - Write-Host "The variable named BuildTargetingString is set to $(BuildTargetingString)" - $setting = "$(BuildTargetingString)" - } - else { - Write-Host "We are falling back to the parameter value ${{ parameters.BuildTargetingString }}" - } - Write-Host "Setting TargetingString to $setting" - Write-Host "##vso[task.setvariable variable=TargetingString;]$setting" - displayName: Check override of targeting string by variable for individual build - condition: and(succeededOrFailed(), eq(variables['TargetingString'],'')) \ No newline at end of file + } + + Write-Host "Setting TargetingString to $setting" + Write-Host "##vso[task.setvariable variable=TargetingString;]$setting" + displayName: Resolve targeted packages \ No newline at end of file From 3ced7a732acf390a43f1990a2c88e6364b3707d5 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Thu, 8 Aug 2024 13:38:17 -0700 Subject: [PATCH 058/119] don't need double output on resolve package targeting --- eng/pipelines/templates/steps/resolve-package-targeting.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/eng/pipelines/templates/steps/resolve-package-targeting.yml b/eng/pipelines/templates/steps/resolve-package-targeting.yml index d3d6e84caa47..d6bb5aee179d 100644 --- a/eng/pipelines/templates/steps/resolve-package-targeting.yml +++ b/eng/pipelines/templates/steps/resolve-package-targeting.yml @@ -23,8 +23,6 @@ steps: | Foreach-Object { $_.Name } ` | ForEach-Object { $_.Replace(".json", "") } $setting = $packageProperties -join "," - Write-Host "Setting TargetingString to $setting" - Write-Host "##vso[task.setvariable variable=TargetingString;]$setting" } Write-Host "Setting TargetingString to $setting" From fd5816e296ffdb0d7a5fc8f6c6b9b45a6709f04b Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Thu, 8 Aug 2024 14:54:31 -0700 Subject: [PATCH 059/119] resolve issue with not generating the packages we expect. update resolve-build-platforms to look at specific package --- eng/pipelines/templates/steps/build-package-artifacts.yml | 6 ++++-- eng/pipelines/templates/steps/resolve-build-platforms.yml | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/eng/pipelines/templates/steps/build-package-artifacts.yml b/eng/pipelines/templates/steps/build-package-artifacts.yml index 6e7f7b3dc3f9..8a3ab5c522ad 100644 --- a/eng/pipelines/templates/steps/build-package-artifacts.yml +++ b/eng/pipelines/templates/steps/build-package-artifacts.yml @@ -105,9 +105,11 @@ steps: BuildTargetingString: ${{ parameters.BuildTargetingString }} PackagePropertiesFolder: $(Build.ArtifactStagingDirectory)/PackageInfo + # todo, walk the artifacts and ensure that one which includes an extension package is present + # if not, we only need to build on linux. if so, we need to build on all platforms - template: /eng/pipelines/templates/steps/resolve-build-platforms.yml parameters: - ServiceDirectory: ${{ parameters.ServiceDirectory }} + PackagePropertiesFolder: $(Build.ArtifactStagingDirectory)/PackageInfo - template: /eng/pipelines/templates/steps/use-venv.yml parameters: @@ -145,7 +147,7 @@ steps: - pwsh: | $(VENV_ACTIVATION_SCRIPT) which python - sdk_build -d "$(Build.ArtifactStagingDirectory)" "$(TargetingString)" --service=${{parameters.ServiceDirectory}} --inactive + sdk_build -d "$(Build.ArtifactStagingDirectory)" "$(TargetingString)" --inactive displayName: 'Generate Packages' condition: and(succeeded(), or(eq(variables['ENABLE_EXTENSION_BUILD'], 'true'), eq('${{ parameters.ArtifactSuffix }}', 'linux'))) timeoutInMinutes: 80 diff --git a/eng/pipelines/templates/steps/resolve-build-platforms.yml b/eng/pipelines/templates/steps/resolve-build-platforms.yml index 2d940d171d42..57d84e2b5c31 100644 --- a/eng/pipelines/templates/steps/resolve-build-platforms.yml +++ b/eng/pipelines/templates/steps/resolve-build-platforms.yml @@ -1,5 +1,5 @@ parameters: - - name: ServiceDirectory + - name: PackagePropertiesFolder type: string default: '' @@ -8,7 +8,7 @@ steps: # as the generate-pr-diff call + resolution will be present in resolve-package-targeting.yml. # until then, we simply check to see if we're targeting storage service directory - pwsh: | - if ("${{ parameters.ServiceDirectory }}" -eq "storage") { + if ("${{ parameters.PackagePropertiesFolder }}".Contains("azure-storage-extension")) { Write-Host "Targeting storage, enabling extension build." Write-Host "##vso[task.setvariable variable=ENABLE_EXTENSION_BUILD]true" } From 4a674973b6da4cbb818ca2deababb1cbb41075e3 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Fri, 9 Aug 2024 11:51:40 -0700 Subject: [PATCH 060/119] resolve the problem with verify readmes failing for internal run. expand runtime a bit for forgiveness. condition targetingSTring on it not being defined already. --- .../templates/steps/analyze_dependency.yml | 14 ++++---------- eng/pipelines/templates/steps/build-test.yml | 7 +++---- .../templates/steps/resolve-package-targeting.yml | 4 +++- sdk/core/ci.yml | 1 + 4 files changed, 11 insertions(+), 15 deletions(-) diff --git a/eng/pipelines/templates/steps/analyze_dependency.yml b/eng/pipelines/templates/steps/analyze_dependency.yml index 93d149cdf46f..37da2dce5aef 100644 --- a/eng/pipelines/templates/steps/analyze_dependency.yml +++ b/eng/pipelines/templates/steps/analyze_dependency.yml @@ -13,16 +13,10 @@ steps: displayName: 'Install Python Tools' condition: succeededOrFailed() - - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: - - template: /eng/common/pipelines/templates/steps/verify-readmes.yml - parameters: - PackagePropertiesFolder: $(Build.ArtifactStagingDirectory)/PackageInfo - Condition: succeededOrFailed() - - ${{ else }}: - - template: /eng/common/pipelines/templates/steps/verify-readme.yml - parameters: - ScanPath: ${{ parameters.ScanPath }} - Condition: succeededOrFailed() + - template: /eng/common/pipelines/templates/steps/verify-readmes.yml + parameters: + PackagePropertiesFolder: $(Build.ArtifactStagingDirectory)/PackageInfo + Condition: succeededOrFailed() - pwsh: | sdk_analyze_deps --verbose --out "$(Build.ArtifactStagingDirectory)/reports/dependencies.html" --dump "$(Build.ArtifactStagingDirectory)/reports" diff --git a/eng/pipelines/templates/steps/build-test.yml b/eng/pipelines/templates/steps/build-test.yml index 030af919715c..a3c472294067 100644 --- a/eng/pipelines/templates/steps/build-test.yml +++ b/eng/pipelines/templates/steps/build-test.yml @@ -45,10 +45,9 @@ steps: Write-Host (Get-Command python).Source displayName: 'Prep Environment' - - ${{if eq(parameters.TestProxy, true) }}: - - template: /eng/common/testproxy/test-proxy-tool.yml - parameters: - runProxy: false + - template: /eng/common/testproxy/test-proxy-tool.yml + parameters: + runProxy: false - ${{if eq(variables['System.TeamProject'], 'internal') }}: - template: ../steps/auth-dev-feed.yml diff --git a/eng/pipelines/templates/steps/resolve-package-targeting.yml b/eng/pipelines/templates/steps/resolve-package-targeting.yml index d6bb5aee179d..8b05b23b6c6f 100644 --- a/eng/pipelines/templates/steps/resolve-package-targeting.yml +++ b/eng/pipelines/templates/steps/resolve-package-targeting.yml @@ -27,4 +27,6 @@ steps: Write-Host "Setting TargetingString to $setting" Write-Host "##vso[task.setvariable variable=TargetingString;]$setting" - displayName: Resolve targeted packages \ No newline at end of file + displayName: Resolve targeted packages + # if targeting has been set by matrix, this value will already be populated + condition: eq(variables['TargetingString]'], '') diff --git a/sdk/core/ci.yml b/sdk/core/ci.yml index 2f486f6901bb..302a65b9bc6d 100644 --- a/sdk/core/ci.yml +++ b/sdk/core/ci.yml @@ -38,6 +38,7 @@ extends: BuildTargetingString: "*" ValidateFormatting: true TestProxy: false + TimeoutInMinutes: 120 Artifacts: - name: azure-core safeName: azurecore From 92f979213993c0684e5be4ed00c112fc59bf0536 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Fri, 9 Aug 2024 12:32:41 -0700 Subject: [PATCH 061/119] TimeoutInMinutes -> TestTimeoutInMinutes --- sdk/core/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/core/ci.yml b/sdk/core/ci.yml index 302a65b9bc6d..d358db016420 100644 --- a/sdk/core/ci.yml +++ b/sdk/core/ci.yml @@ -38,7 +38,7 @@ extends: BuildTargetingString: "*" ValidateFormatting: true TestProxy: false - TimeoutInMinutes: 120 + TestTimeoutInMinutes: 120 Artifacts: - name: azure-core safeName: azurecore From c25fda2dda4e946bd89a3f3563cdbb7de3dd3956 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Fri, 9 Aug 2024 14:55:58 -0700 Subject: [PATCH 062/119] update breaking samples by popping the bad configs --- sdk/core/azure-core/samples/test_example_async.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/sdk/core/azure-core/samples/test_example_async.py b/sdk/core/azure-core/samples/test_example_async.py index a94b1331f6bf..499775b0cfe2 100644 --- a/sdk/core/azure-core/samples/test_example_async.py +++ b/sdk/core/azure-core/samples/test_example_async.py @@ -23,6 +23,7 @@ # THE SOFTWARE. # # -------------------------------------------------------------------------- +import os from typing import Iterable, MutableSequence, Union import pytest from azure.core.pipeline import AsyncPipeline @@ -70,6 +71,8 @@ async def test_example_asyncio(): @pytest.mark.asyncio async def test_example_aiohttp(): + os.environ.pop("SSL_CERT_DIR", None) + os.environ.pop("REQUESTS_CA_BUNDLE", None) request = HttpRequest("GET", "https://bing.com") policies: Iterable[Union[AsyncHTTPPolicy, SansIOHTTPPolicy]] = [ UserAgentPolicy("myuseragent"), @@ -86,6 +89,8 @@ async def test_example_aiohttp(): @pytest.mark.asyncio async def test_example_async_pipeline(): + os.environ.pop("SSL_CERT_DIR", None) + os.environ.pop("REQUESTS_CA_BUNDLE", None) # [START build_async_pipeline] from azure.core.pipeline import AsyncPipeline from azure.core.pipeline.policies import AsyncRedirectPolicy, UserAgentPolicy @@ -108,6 +113,8 @@ async def test_example_async_pipeline(): @pytest.mark.asyncio async def test_example_async_pipeline_client(): + os.environ.pop("SSL_CERT_DIR", None) + os.environ.pop("REQUESTS_CA_BUNDLE", None) url = "https://bing.com" # [START build_async_pipeline_client] @@ -131,6 +138,8 @@ async def test_example_async_pipeline_client(): @pytest.mark.asyncio async def test_example_async_redirect_policy(): + os.environ.pop("SSL_CERT_DIR", None) + os.environ.pop("REQUESTS_CA_BUNDLE", None) url = "https://bing.com" request = HttpRequest("GET", url) @@ -159,6 +168,8 @@ async def test_example_async_redirect_policy(): @pytest.mark.asyncio async def test_example_async_retry_policy(): + os.environ.pop("SSL_CERT_DIR", None) + os.environ.pop("REQUESTS_CA_BUNDLE", None) url = "https://bing.com" request = HttpRequest("GET", "https://bing.com") policies: MutableSequence[Union[AsyncHTTPPolicy, SansIOHTTPPolicy]] = [ From 4b26f7eb57aac546e1bf46a6e06ca36ea4099c32 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Fri, 9 Aug 2024 15:04:48 -0700 Subject: [PATCH 063/119] remove an extra call to setdevbuild that is unnecessary and ends up using the default python version which crashes on us --- eng/pipelines/templates/jobs/ci.tests.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/eng/pipelines/templates/jobs/ci.tests.yml b/eng/pipelines/templates/jobs/ci.tests.yml index 4df0094c79cf..0379f372425b 100644 --- a/eng/pipelines/templates/jobs/ci.tests.yml +++ b/eng/pipelines/templates/jobs/ci.tests.yml @@ -81,10 +81,6 @@ jobs: steps: - template: /eng/pipelines/templates/steps/download-package-artifacts.yml - - template: ../steps/set-dev-build.yml - parameters: - ServiceDirectory: ${{ parameters.ServiceDirectory }} - - template: /eng/pipelines/templates/steps/resolve-package-targeting.yml parameters: BuildTargetingString: ${{ parameters.BuildTargetingString }} From 82fe6ebc49c3c77f439ff670344f662c862c64ea Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 13 Aug 2024 09:55:28 -0700 Subject: [PATCH 064/119] commit progress towards rebalancing the matrix --- eng/pipelines/templates/jobs/ci.yml | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index ac1a547a7b74..bcbd5f5c5121 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -209,6 +209,27 @@ jobs: MatrixConfigs: ${{ parameters.MatrixConfigs }} MatrixFilters: ${{ parameters.MatrixFilters }} MatrixReplace: ${{ parameters.MatrixReplace }} + PreGenerationSteps: + + - template: /eng/pipelines/templates/steps/download-package-artifacts.yml + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.12' + conditon: eq(variables['Build.Reason'], 'PullRequest') + - pwsh: | + python -m pip install "./tools/azure-sdk-tools[build]" + displayName: 'Prep Environment' + conditon: eq(variables['Build.Reason'], 'PullRequest') + - task: PythonScript@0 + displayName: 'Ensure service coverage' + conditon: eq(variables['Build.Reason'], 'PullRequest') + inputs: + scriptPath: '$(Build.SourcesDirectory)/scripts/devops_tasks/batch_pr_tests.py' + arguments: >- + --package_info="$(Build.ArtifactStagingDirectory)/PackageInfo" + --json=$(Build.SourcesDirectory)/eng/pipelines/templates/stages/platform-matrix.json + + CloudConfig: Cloud: Public AdditionalParameters: @@ -240,12 +261,11 @@ jobs: - task: UsePythonVersion@0 inputs: versionSpec: '3.12' + condition: Build - pwsh: | python -m pip install "./tools/azure-sdk-tools[build]" displayName: 'Prep Environment' - - template: /eng/pipelines/templates/steps/resolve-package-targeting.yml - parameters: - BuildTargetingString: ${{ parameters.BuildTargetingString }} + - template: /eng/pipelines/templates/steps/download-package-artifacts.yml - task: PythonScript@0 displayName: 'Ensure service coverage' inputs: From 1c12314dadd30c9d6de10d3bca51c0f59104f268 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 13 Aug 2024 13:03:58 -0700 Subject: [PATCH 065/119] handling more and less than batchcount worth of packages --- eng/pipelines/templates/jobs/ci.yml | 22 ++--- .../templates/stages/platform-matrix.json | 9 +- eng/scripts/distribute-packages-to-matrix.ps1 | 86 +++++++++++++++++++ 3 files changed, 99 insertions(+), 18 deletions(-) create mode 100644 eng/scripts/distribute-packages-to-matrix.ps1 diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index bcbd5f5c5121..4f44a7049ba6 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -212,23 +212,15 @@ jobs: PreGenerationSteps: - template: /eng/pipelines/templates/steps/download-package-artifacts.yml - - task: UsePythonVersion@0 + + - task: PowerShell@5 inputs: - versionSpec: '3.12' - conditon: eq(variables['Build.Reason'], 'PullRequest') - - pwsh: | - python -m pip install "./tools/azure-sdk-tools[build]" - displayName: 'Prep Environment' + targetType: 'filePath' + pwsh: true + filePath: '/eng/scripts/distribute-packages-to-matrix.ps1' + arguments: '-PackageInfo "$(Build.ArtifactStagingDirectory)/PackageInfo" -PlatformJson $(Build.SourcesDirectory)/eng/pipelines/templates/stages/platform-matrix.json' + displayName: 'Distribute Packages to Matrix' conditon: eq(variables['Build.Reason'], 'PullRequest') - - task: PythonScript@0 - displayName: 'Ensure service coverage' - conditon: eq(variables['Build.Reason'], 'PullRequest') - inputs: - scriptPath: '$(Build.SourcesDirectory)/scripts/devops_tasks/batch_pr_tests.py' - arguments: >- - --package_info="$(Build.ArtifactStagingDirectory)/PackageInfo" - --json=$(Build.SourcesDirectory)/eng/pipelines/templates/stages/platform-matrix.json - CloudConfig: Cloud: Public diff --git a/eng/pipelines/templates/stages/platform-matrix.json b/eng/pipelines/templates/stages/platform-matrix.json index fcda14bcd8f8..8f089f6920f2 100644 --- a/eng/pipelines/templates/stages/platform-matrix.json +++ b/eng/pipelines/templates/stages/platform-matrix.json @@ -12,7 +12,8 @@ }, "PythonVersion": [ "3.8", "pypy3.9", "3.11", "3.10" ], "CoverageArg": "--disablecov", - "TestSamples": "false" + "TestSamples": "false", + "TargetingString": "" }, "include": [ { @@ -22,7 +23,8 @@ "Pool": "env:LINUXPOOL", "PythonVersion": "3.9", "CoverageArg": "", - "TestSamples": "false" + "TestSamples": "false", + "TargetingString": "" } } }, @@ -33,7 +35,8 @@ "Pool": "env:LINUXPOOL", "PythonVersion": "3.12", "CoverageArg": "--disablecov", - "TestSamples": "false" + "TestSamples": "false", + "TargetingString": "" } } } diff --git a/eng/scripts/distribute-packages-to-matrix.ps1 b/eng/scripts/distribute-packages-to-matrix.ps1 new file mode 100644 index 000000000000..92ab5f5275dc --- /dev/null +++ b/eng/scripts/distribute-packages-to-matrix.ps1 @@ -0,0 +1,86 @@ +<# +.SYNOPSIS +Distributes a set of packages to a platform matrix file by adding TargetingString property dynamically. +If an unexpected situation is encountered, the script will make no changes to the input file. + +.DESCRIPTION +Because of the way the platform matrix file is structured, we need to distribute the packages in a way that +honors the Include packages. We have these included this way because want to ensure that these python versions +run on that platform. This is aware of these additional configurations and will set TargetingString for them as well. +#> +param( + [parameter(Mandatory=$true)] + [string]$PackageInfoFolder, + [parameter(Mandatory=$true)] + [string]$PlatformMatrix +) + +Set-StrictMode -Version 4 + +if (!(Test-Path $PackageInfoFolder)) { + Write-Error "PackageInfo folder file not found: $PackageInfoFolder" + exit 1 +} + +if (!(Test-Path $PlatformMatrix)) { + Write-Error "Platform matrix file not found: $PlatformMatrix" + exit 1 +} + +$packageProperties = Get-ChildItem -Recurse "$PackageInfoFolder" *.json +$platformMatrixFile = Get-Content -Path $PlatformMatrix | ConvertFrom-Json + +# determine batch dize +$versionCount = $matrix.matrix.PythonVersion.Count +$includeCount = $matrix.include.Count +$batchCount = $versionCount + $includeCount +if ($batchCount -eq 0) { + Write-Error "No batches detected, skipping without updating platform matrix file $PlatformMatrix" + exit 0 +} +$batchSize = $packageProperties.Count % $batchCount +$batches = @{} +for ($i = 0; $i -lt $batchCount; $i++) { + $batches[$i] = @() +} + +# what situations can we have with the calculated batch size of 6? (number of platforms + include) +# 1. < 6 packages we simply exit. default distribution will be fine +# 2. >= 6 packages we rotate and assign 1 to each batch until all work is exhausted +if ($packageProperties.Count -lt $batchCount) { + Write-Error "Not enough packages to distribute to all batches, skipping without updating platform matrix file $PlatformMatrix" + exit 0 +} + +$batchCounter = 0 +$batchValues = @() + +# regular case is when we have more packages than batches +if ($packageProperties.Count -ge $batchCount) { + for ($i = 0; $i -lt $packageProperties.Count; $i++) { + $batches[$batchCounter] += $packageProperties[$i].Name.Replace(".json", "") + $batchCounter = ($batchCounter + 1) % $batchCount + } +} +# the case where we have fewer packages than batches, we instead walk the batches +# and assign packages to them, even if we end up assigning the same package twice +# that's ok, we just don't want to have dead platforms +else { + for ($i = 0; $i -lt $batchCount; $i++) { + $batches[$i] += $packageProperties[$i % $packageProperties.Count].Name.Replace(".json", "") + } +} + +# simplify the batches down +foreach($key in $batches.Keys) { + $batchValues += $batches[$key] -join "," +} + +# set the values for the matrix +$platformMatrixFile.matrix.TargetingString = $batchValues[0..($versionCount - 1)] + +# set the values for include +$platformMatrixFile.include[0].CoverageConfig.ubuntu2004_39_coverage.TargetingString = $batchValues[$versionCount] +$platformMatrixFile.include[1].Config.Ubuntu2004_312.TargetingString = $batchValues[$versionCount+1] + +$platformMatrixFile | ConvertTo-Json -Depth 100 | Set-Content -Path $PlatformMatrix From 32a5b1f9b9786f883658eae8759dc7b251eeb511 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 13 Aug 2024 13:08:30 -0700 Subject: [PATCH 066/119] we now properly handle smaller builds --- eng/scripts/distribute-packages-to-matrix.ps1 | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/eng/scripts/distribute-packages-to-matrix.ps1 b/eng/scripts/distribute-packages-to-matrix.ps1 index 92ab5f5275dc..288345df4f47 100644 --- a/eng/scripts/distribute-packages-to-matrix.ps1 +++ b/eng/scripts/distribute-packages-to-matrix.ps1 @@ -30,7 +30,7 @@ if (!(Test-Path $PlatformMatrix)) { $packageProperties = Get-ChildItem -Recurse "$PackageInfoFolder" *.json $platformMatrixFile = Get-Content -Path $PlatformMatrix | ConvertFrom-Json -# determine batch dize +# determine batch size and initialize batches $versionCount = $matrix.matrix.PythonVersion.Count $includeCount = $matrix.include.Count $batchCount = $versionCount + $includeCount @@ -39,6 +39,9 @@ if ($batchCount -eq 0) { exit 0 } $batchSize = $packageProperties.Count % $batchCount +# batches is a hashtable because you cannot instantiate an array of empty arrays +# the add method will merely invoke and silently NOT add a new item +# using a hashtable bypasses this limitation $batches = @{} for ($i = 0; $i -lt $batchCount; $i++) { $batches[$i] = @() @@ -47,11 +50,6 @@ for ($i = 0; $i -lt $batchCount; $i++) { # what situations can we have with the calculated batch size of 6? (number of platforms + include) # 1. < 6 packages we simply exit. default distribution will be fine # 2. >= 6 packages we rotate and assign 1 to each batch until all work is exhausted -if ($packageProperties.Count -lt $batchCount) { - Write-Error "Not enough packages to distribute to all batches, skipping without updating platform matrix file $PlatformMatrix" - exit 0 -} - $batchCounter = 0 $batchValues = @() @@ -71,7 +69,7 @@ else { } } -# simplify the batches down +# simplify the batches down to single values that will be added to the matrix foreach($key in $batches.Keys) { $batchValues += $batches[$key] -join "," } From 8e4b198dd0bca0c65f3152ce6b5436c947a0735a Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 13 Aug 2024 13:12:02 -0700 Subject: [PATCH 067/119] conditon -> condition --- eng/pipelines/templates/jobs/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index 4f44a7049ba6..592bb4d5f153 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -220,7 +220,7 @@ jobs: filePath: '/eng/scripts/distribute-packages-to-matrix.ps1' arguments: '-PackageInfo "$(Build.ArtifactStagingDirectory)/PackageInfo" -PlatformJson $(Build.SourcesDirectory)/eng/pipelines/templates/stages/platform-matrix.json' displayName: 'Distribute Packages to Matrix' - conditon: eq(variables['Build.Reason'], 'PullRequest') + condition: eq(variables['Build.Reason'], 'PullRequest') CloudConfig: Cloud: Public From 3e0dbc1d07ddc27d940224a64312aec6d1461784 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 13 Aug 2024 13:13:20 -0700 Subject: [PATCH 068/119] task: PowerShell -> task: AzurePowerShell --- eng/pipelines/templates/jobs/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index 592bb4d5f153..5c7dfb585274 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -213,7 +213,7 @@ jobs: - template: /eng/pipelines/templates/steps/download-package-artifacts.yml - - task: PowerShell@5 + - task: AzurePowerShell@5 inputs: targetType: 'filePath' pwsh: true From 4081775f4abb2f7577763e924280103cf908826a Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 13 Aug 2024 13:38:39 -0700 Subject: [PATCH 069/119] call the script simpler --- eng/pipelines/templates/jobs/ci.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index 5c7dfb585274..476995c64602 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -213,12 +213,10 @@ jobs: - template: /eng/pipelines/templates/steps/download-package-artifacts.yml - - task: AzurePowerShell@5 - inputs: - targetType: 'filePath' - pwsh: true - filePath: '/eng/scripts/distribute-packages-to-matrix.ps1' - arguments: '-PackageInfo "$(Build.ArtifactStagingDirectory)/PackageInfo" -PlatformJson $(Build.SourcesDirectory)/eng/pipelines/templates/stages/platform-matrix.json' + - pwsh: | + ./eng/scripts/distribute-packages-to-matrix.ps1 ` + -PackageInfo "$(Build.ArtifactStagingDirectory)/PackageInfo" ` + -PlatformJson $(Build.SourcesDirectory)/eng/pipelines/templates/stages/platform-matrix.json displayName: 'Distribute Packages to Matrix' condition: eq(variables['Build.Reason'], 'PullRequest') From 42054dfa919c719cc94ba6a4498ebb9bbcf82148 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 13 Aug 2024 13:54:05 -0700 Subject: [PATCH 070/119] PlatformJson -> PlatformMatrix --- eng/pipelines/templates/jobs/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index 476995c64602..486102460c60 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -216,7 +216,7 @@ jobs: - pwsh: | ./eng/scripts/distribute-packages-to-matrix.ps1 ` -PackageInfo "$(Build.ArtifactStagingDirectory)/PackageInfo" ` - -PlatformJson $(Build.SourcesDirectory)/eng/pipelines/templates/stages/platform-matrix.json + -PlatformMatrix $(Build.SourcesDirectory)/eng/pipelines/templates/stages/platform-matrix.json displayName: 'Distribute Packages to Matrix' condition: eq(variables['Build.Reason'], 'PullRequest') From 92e7f661f36d161781680f4b84221ee4185bff0a Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 13 Aug 2024 14:05:26 -0700 Subject: [PATCH 071/119] ensure that corehttp builds nightly --- eng/pipelines/templates/steps/set-dev-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/steps/set-dev-build.yml b/eng/pipelines/templates/steps/set-dev-build.yml index 9928fbdf2d11..26233110f3c8 100644 --- a/eng/pipelines/templates/steps/set-dev-build.yml +++ b/eng/pipelines/templates/steps/set-dev-build.yml @@ -10,6 +10,6 @@ steps: - pwsh: | python -m pip install "tools/azure-sdk-tools[build]" - sdk_set_dev_version "azure*" --build-id="$(Build.BuildNumber)" + sdk_set_dev_version "*" --build-id="$(Build.BuildNumber)" displayName: "Update package versions for dev build" condition: and(succeededOrFailed(), eq(variables['SetDevVersion'],'true'), ${{ parameters.Condition }}) From 0f122b7e26020a963a9e70c1d3cc18ff7a053a16 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 13 Aug 2024 14:17:00 -0700 Subject: [PATCH 072/119] add our pipeline to the sparse checkout path --- eng/pipelines/templates/jobs/ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index 486102460c60..9f7f6356a977 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -202,6 +202,7 @@ jobs: JobTemplatePath: /eng/pipelines/templates/jobs/ci.tests.yml OsVmImage: azsdk-pool-mms-ubuntu-2004-1espt Pool: azsdk-pool-mms-ubuntu-2004-general + SparseCheckoutPaths: [ "eng/" ] DependsOn: - 'Build_Linux' - 'Build_Windows' @@ -213,6 +214,10 @@ jobs: - template: /eng/pipelines/templates/steps/download-package-artifacts.yml + - pwsh: | + Get-ChildItem -R $(Build.SourcesDirectory) | % { Write-Host $_.FullName } + displayName: Dump Visible Artifacts + - pwsh: | ./eng/scripts/distribute-packages-to-matrix.ps1 ` -PackageInfo "$(Build.ArtifactStagingDirectory)/PackageInfo" ` From 4c4ec6908a6340518295a1cb74f32573d185f3bc Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 13 Aug 2024 14:34:41 -0700 Subject: [PATCH 073/119] update core test, remove sparsecheckoutpath argument eng/ should be present --- eng/pipelines/templates/jobs/ci.yml | 1 - sdk/core/corehttp/tests/async_tests/test_pipeline_async.py | 3 +++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index 9f7f6356a977..2c3c566665ef 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -202,7 +202,6 @@ jobs: JobTemplatePath: /eng/pipelines/templates/jobs/ci.tests.yml OsVmImage: azsdk-pool-mms-ubuntu-2004-1espt Pool: azsdk-pool-mms-ubuntu-2004-general - SparseCheckoutPaths: [ "eng/" ] DependsOn: - 'Build_Linux' - 'Build_Windows' diff --git a/sdk/core/corehttp/tests/async_tests/test_pipeline_async.py b/sdk/core/corehttp/tests/async_tests/test_pipeline_async.py index 007ef094d667..3e481311c146 100644 --- a/sdk/core/corehttp/tests/async_tests/test_pipeline_async.py +++ b/sdk/core/corehttp/tests/async_tests/test_pipeline_async.py @@ -22,6 +22,7 @@ import aiohttp import httpx import pytest +import os from utils import ASYNC_TRANSPORTS @@ -85,6 +86,8 @@ def on_response(self, request, response): @pytest.mark.asyncio @pytest.mark.parametrize("transport", ASYNC_TRANSPORTS) async def test_transport_socket_timeout(transport): + os.environ.pop("SSL_CERT_DIR", None) + os.environ.pop("REQUESTS_CA_BUNDLE", None) request = HttpRequest("GET", "https://bing.com") policies = [UserAgentPolicy("myusergant")] # Sometimes this will raise a read timeout, sometimes a socket timeout depending on timing. From 548dc36b495c3d78038ff5472616f77a0261b823 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 13 Aug 2024 14:54:55 -0700 Subject: [PATCH 074/119] repair issues with regression matrix condition --- eng/pipelines/templates/jobs/ci.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index 2c3c566665ef..5b5b84fccc10 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -255,17 +255,15 @@ jobs: - task: UsePythonVersion@0 inputs: versionSpec: '3.12' - condition: Build - pwsh: | python -m pip install "./tools/azure-sdk-tools[build]" displayName: 'Prep Environment' - - template: /eng/pipelines/templates/steps/download-package-artifacts.yml - task: PythonScript@0 displayName: 'Ensure service coverage' inputs: scriptPath: '$(Build.SourcesDirectory)/scripts/devops_tasks/update_regression_services.py' arguments: >- - "$(TargetingString)" + "azure*" --service="${{ parameters.ServiceDirectory }}" --json=$(Build.SourcesDirectory)/eng/pipelines/templates/stages/regression-job-matrix.json CloudConfig: From 698b0fdd73b7e4af28fa9cb114682522110cafe4 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 13 Aug 2024 14:59:17 -0700 Subject: [PATCH 075/119] call into the script in hopefully the proper fashion --- eng/pipelines/templates/jobs/ci.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index 5b5b84fccc10..eab23485271a 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -217,10 +217,14 @@ jobs: Get-ChildItem -R $(Build.SourcesDirectory) | % { Write-Host $_.FullName } displayName: Dump Visible Artifacts - - pwsh: | - ./eng/scripts/distribute-packages-to-matrix.ps1 ` - -PackageInfo "$(Build.ArtifactStagingDirectory)/PackageInfo" ` - -PlatformMatrix $(Build.SourcesDirectory)/eng/pipelines/templates/stages/platform-matrix.json + - task: Powershell@2 + inputs: + pwsh: true + filePath: eng/scripts/distribute-packages-to-matrix.ps1 + arguments: + arguments: >- + -PackageInfoFolder "$(Build.ArtifactStagingDirectory)/PackageInfo" + -PlatformMatrix "$(Build.SourcesDirectory)/eng/pipelines/templates/stages/platform-matrix.json displayName: 'Distribute Packages to Matrix' condition: eq(variables['Build.Reason'], 'PullRequest') From dba37d13592dcdb15ffb1401090bb5da650b3ee4 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 13 Aug 2024 15:46:26 -0700 Subject: [PATCH 076/119] double arguments param --- eng/pipelines/templates/jobs/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index eab23485271a..c58fb53d1ebf 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -221,7 +221,6 @@ jobs: inputs: pwsh: true filePath: eng/scripts/distribute-packages-to-matrix.ps1 - arguments: arguments: >- -PackageInfoFolder "$(Build.ArtifactStagingDirectory)/PackageInfo" -PlatformMatrix "$(Build.SourcesDirectory)/eng/pipelines/templates/stages/platform-matrix.json From f575e0304fd6241c3f40da21aac59d46c62f92a8 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 13 Aug 2024 16:08:49 -0700 Subject: [PATCH 077/119] i am blind --- eng/pipelines/templates/jobs/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index c58fb53d1ebf..52e03eee9328 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -223,7 +223,7 @@ jobs: filePath: eng/scripts/distribute-packages-to-matrix.ps1 arguments: >- -PackageInfoFolder "$(Build.ArtifactStagingDirectory)/PackageInfo" - -PlatformMatrix "$(Build.SourcesDirectory)/eng/pipelines/templates/stages/platform-matrix.json + -PlatformMatrix "$(Build.SourcesDirectory)/eng/pipelines/templates/stages/platform-matrix.json" displayName: 'Distribute Packages to Matrix' condition: eq(variables['Build.Reason'], 'PullRequest') From 809bf78b023f0ec3e4a3dcc1a72305e1c3e02f26 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 13 Aug 2024 16:26:28 -0700 Subject: [PATCH 078/119] finally fix the real problem. we weren't referencing a variable that existed --- eng/scripts/distribute-packages-to-matrix.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/scripts/distribute-packages-to-matrix.ps1 b/eng/scripts/distribute-packages-to-matrix.ps1 index 288345df4f47..3f1b57ce3c2e 100644 --- a/eng/scripts/distribute-packages-to-matrix.ps1 +++ b/eng/scripts/distribute-packages-to-matrix.ps1 @@ -28,7 +28,7 @@ if (!(Test-Path $PlatformMatrix)) { } $packageProperties = Get-ChildItem -Recurse "$PackageInfoFolder" *.json -$platformMatrixFile = Get-Content -Path $PlatformMatrix | ConvertFrom-Json +$matrix = Get-Content -Path $PlatformMatrix | ConvertFrom-Json # determine batch size and initialize batches $versionCount = $matrix.matrix.PythonVersion.Count From 03943c18c4de234ec4067eb7f9cda26424033977 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 13 Aug 2024 16:49:23 -0700 Subject: [PATCH 079/119] why am I having such an issue working through these? --- eng/scripts/distribute-packages-to-matrix.ps1 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/eng/scripts/distribute-packages-to-matrix.ps1 b/eng/scripts/distribute-packages-to-matrix.ps1 index 3f1b57ce3c2e..1f71f297d28c 100644 --- a/eng/scripts/distribute-packages-to-matrix.ps1 +++ b/eng/scripts/distribute-packages-to-matrix.ps1 @@ -75,10 +75,10 @@ foreach($key in $batches.Keys) { } # set the values for the matrix -$platformMatrixFile.matrix.TargetingString = $batchValues[0..($versionCount - 1)] +$matrix.matrix.TargetingString = $batchValues[0..($versionCount - 1)] # set the values for include -$platformMatrixFile.include[0].CoverageConfig.ubuntu2004_39_coverage.TargetingString = $batchValues[$versionCount] -$platformMatrixFile.include[1].Config.Ubuntu2004_312.TargetingString = $batchValues[$versionCount+1] +$matrix.include[0].CoverageConfig.ubuntu2004_39_coverage.TargetingString = $batchValues[$versionCount] +$matrix.include[1].Config.Ubuntu2004_312.TargetingString = $batchValues[$versionCount+1] -$platformMatrixFile | ConvertTo-Json -Depth 100 | Set-Content -Path $PlatformMatrix +$matrix | ConvertTo-Json -Depth 100 | Set-Content -Path $PlatformMatrix From d7c7208e7ac19e370d2e51f303d994c87ed977d4 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 13 Aug 2024 17:07:39 -0700 Subject: [PATCH 080/119] checking the wrong variable name --- eng/pipelines/templates/steps/resolve-package-targeting.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/steps/resolve-package-targeting.yml b/eng/pipelines/templates/steps/resolve-package-targeting.yml index 8b05b23b6c6f..e79cc63c95d0 100644 --- a/eng/pipelines/templates/steps/resolve-package-targeting.yml +++ b/eng/pipelines/templates/steps/resolve-package-targeting.yml @@ -29,4 +29,4 @@ steps: Write-Host "##vso[task.setvariable variable=TargetingString;]$setting" displayName: Resolve targeted packages # if targeting has been set by matrix, this value will already be populated - condition: eq(variables['TargetingString]'], '') + condition: eq(variables['TargetingString'], '') From f3934ccf31d6d3376613161c05a87db946ba8099 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Wed, 14 Aug 2024 11:25:51 -0700 Subject: [PATCH 081/119] add the default variable so that it can be set later --- eng/pipelines/templates/jobs/ci.yml | 4 ---- eng/pipelines/templates/variables/globals.yml | 3 ++- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index 52e03eee9328..857acc917c5a 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -213,10 +213,6 @@ jobs: - template: /eng/pipelines/templates/steps/download-package-artifacts.yml - - pwsh: | - Get-ChildItem -R $(Build.SourcesDirectory) | % { Write-Host $_.FullName } - displayName: Dump Visible Artifacts - - task: Powershell@2 inputs: pwsh: true diff --git a/eng/pipelines/templates/variables/globals.yml b/eng/pipelines/templates/variables/globals.yml index 065d73d83102..a4e5375cb507 100644 --- a/eng/pipelines/templates/variables/globals.yml +++ b/eng/pipelines/templates/variables/globals.yml @@ -10,4 +10,5 @@ variables: BuildId: $(Build.BuildId) REPOROOT: $(Build.SourcesDirectory) WINDOWS_OUTPUTROOT: $(REPOROOT)\out - WindowsContainerImage: 'onebranch.azurecr.io/windows/ltsc2019/vse2022:latest' \ No newline at end of file + WindowsContainerImage: 'onebranch.azurecr.io/windows/ltsc2019/vse2022:latest' + TargetingString: '' From 12a5906c029a900bbb256b133632405c4ff322b6 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Wed, 14 Aug 2024 12:18:22 -0700 Subject: [PATCH 082/119] encode the state of the world for ml so that core builds aren't failed by it. remove azure-cosmos due to dependency on windows and emulator presence. --- eng/pipelines/templates/variables/globals.yml | 1 - eng/scripts/distribute-packages-to-matrix.ps1 | 6 +++--- eng/scripts/get_package_properties.py | 4 +++- sdk/ml/azure-ai-ml/pyproject.toml | 8 ++++---- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/eng/pipelines/templates/variables/globals.yml b/eng/pipelines/templates/variables/globals.yml index a4e5375cb507..63309a3390f0 100644 --- a/eng/pipelines/templates/variables/globals.yml +++ b/eng/pipelines/templates/variables/globals.yml @@ -11,4 +11,3 @@ variables: REPOROOT: $(Build.SourcesDirectory) WINDOWS_OUTPUTROOT: $(REPOROOT)\out WindowsContainerImage: 'onebranch.azurecr.io/windows/ltsc2019/vse2022:latest' - TargetingString: '' diff --git a/eng/scripts/distribute-packages-to-matrix.ps1 b/eng/scripts/distribute-packages-to-matrix.ps1 index 1f71f297d28c..7c7b9f4a1665 100644 --- a/eng/scripts/distribute-packages-to-matrix.ps1 +++ b/eng/scripts/distribute-packages-to-matrix.ps1 @@ -75,10 +75,10 @@ foreach($key in $batches.Keys) { } # set the values for the matrix -$matrix.matrix.TargetingString = $batchValues[0..($versionCount - 1)] +$matrix.matrix | Add-Member -MemberType NoteProperty -Name TargetingString -Value $batchValues[0..($versionCount - 1)] # set the values for include -$matrix.include[0].CoverageConfig.ubuntu2004_39_coverage.TargetingString = $batchValues[$versionCount] -$matrix.include[1].Config.Ubuntu2004_312.TargetingString = $batchValues[$versionCount+1] +$matrix.include[0].CoverageConfig.ubuntu2004_39_coverage | Add-Member -MemberType NoteProperty -Name TargetingString -Value $batchValues[$versionCount] +$matrix.include[1].Config.Ubuntu2004_312 | Add-Member -MemberType NoteProperty -Name TargetingString -Value $batchValues[$versionCount+1] $matrix | ConvertTo-Json -Depth 100 | Set-Content -Path $PlatformMatrix diff --git a/eng/scripts/get_package_properties.py b/eng/scripts/get_package_properties.py index 8598fc61a037..2f090c5b1f84 100644 --- a/eng/scripts/get_package_properties.py +++ b/eng/scripts/get_package_properties.py @@ -19,7 +19,9 @@ os.path.join("sdk", "core", "azure-core-experimental"), os.path.join("sdk", "core", "azure-core-tracing-opentelemetry"), os.path.join("sdk", "core", "azure-core-tracing-opencensus"), - os.path.join("sdk", "cosmos", "azure-cosmos"), + # todo: this currently won't be included, as azure-cosmos needs some special construction + # related to windows only + emulator + # os.path.join("sdk", "cosmos", "azure-cosmos"), os.path.join("sdk", "ml", "azure-ai-ml"), os.path.join("sdk", "ai", "azure-ai-documentintelligence"), os.path.join("sdk", "ai", "azure-ai-inference"), diff --git a/sdk/ml/azure-ai-ml/pyproject.toml b/sdk/ml/azure-ai-ml/pyproject.toml index 5c9be4ca8aba..6794f524e808 100644 --- a/sdk/ml/azure-ai-ml/pyproject.toml +++ b/sdk/ml/azure-ai-ml/pyproject.toml @@ -4,7 +4,7 @@ pyright = false type_check_samples = false verifytypes = false pylint = true - +mindependency = false [tool.isort] profile = "black" @@ -19,7 +19,7 @@ extend_skip_glob = [ "*/.tox/*", ] -[tool.mypy] +[tool.mypy] python_version = "3.10" exclude = [ "azure/ai/ml/_restclient", @@ -33,8 +33,8 @@ exclude = [ "azure/ai/ml/_utils", "azure/ai/ml/exceptions.py", "azure/ai/ml/_exception_helper.py", -] -warn_unused_configs = true +] +warn_unused_configs = true follow_imports = "skip" ignore_missing_imports = true follow_imports_for_stubs = false From 8f13dcdfcd84a4cf2f09caf13e0fdb42382ed78f Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Wed, 14 Aug 2024 12:33:56 -0700 Subject: [PATCH 083/119] when I reverted the check, I also reverted the removal of the properties --- eng/pipelines/templates/stages/platform-matrix.json | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/eng/pipelines/templates/stages/platform-matrix.json b/eng/pipelines/templates/stages/platform-matrix.json index 8f089f6920f2..fcda14bcd8f8 100644 --- a/eng/pipelines/templates/stages/platform-matrix.json +++ b/eng/pipelines/templates/stages/platform-matrix.json @@ -12,8 +12,7 @@ }, "PythonVersion": [ "3.8", "pypy3.9", "3.11", "3.10" ], "CoverageArg": "--disablecov", - "TestSamples": "false", - "TargetingString": "" + "TestSamples": "false" }, "include": [ { @@ -23,8 +22,7 @@ "Pool": "env:LINUXPOOL", "PythonVersion": "3.9", "CoverageArg": "", - "TestSamples": "false", - "TargetingString": "" + "TestSamples": "false" } } }, @@ -35,8 +33,7 @@ "Pool": "env:LINUXPOOL", "PythonVersion": "3.12", "CoverageArg": "--disablecov", - "TestSamples": "false", - "TargetingString": "" + "TestSamples": "false" } } } From 1eef1de7f36431a155b1bbb24cc48e65c039da17 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Wed, 14 Aug 2024 13:50:02 -0700 Subject: [PATCH 084/119] re-balance the dependent packages so we don't assign azure-storage-blob to pypy39. disable latestdependency for azure-ai-ml because they have that overridden on the job level --- eng/scripts/get_package_properties.py | 4 ++-- sdk/ml/azure-ai-ml/pyproject.toml | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/eng/scripts/get_package_properties.py b/eng/scripts/get_package_properties.py index 2f090c5b1f84..6406ccca74c1 100644 --- a/eng/scripts/get_package_properties.py +++ b/eng/scripts/get_package_properties.py @@ -8,7 +8,6 @@ additional_pr_triggers: Dict[str, List[str]] = { "azure-core":[ - os.path.join("sdk", "storage", "azure-storage-blob"), os.path.join("sdk", "servicebus", "azure-servicebus"), os.path.join("sdk", "eventhub", "azure-eventhub"), os.path.join("sdk", "tables", "azure-data-table"), @@ -29,7 +28,8 @@ os.path.join("sdk", "ai", "azure-ai-doctranslation"), os.path.join("sdk", "compute", "azure-mgmt-compute"), os.path.join("sdk", "communication", "azure-communication-chat"), - os.path.join("sdk", "communication", "azure-communication-identity") + os.path.join("sdk", "communication", "azure-communication-identity"), + os.path.join("sdk", "storage", "azure-storage-blob") ], "azure-mgmt-core": [ os.path.join("sdk", "compute", "azure-mgmt-compute"), diff --git a/sdk/ml/azure-ai-ml/pyproject.toml b/sdk/ml/azure-ai-ml/pyproject.toml index 6794f524e808..a5a8d973ebe5 100644 --- a/sdk/ml/azure-ai-ml/pyproject.toml +++ b/sdk/ml/azure-ai-ml/pyproject.toml @@ -5,6 +5,7 @@ type_check_samples = false verifytypes = false pylint = true mindependency = false +latestdependency = false [tool.isort] profile = "black" From a9fac620a47403a20ed524dde6982affd01a586f Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Wed, 14 Aug 2024 14:13:07 -0700 Subject: [PATCH 085/119] rebalance so that pypy39 doesn't get assigned storage --- eng/scripts/distribute-packages-to-matrix.ps1 | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/eng/scripts/distribute-packages-to-matrix.ps1 b/eng/scripts/distribute-packages-to-matrix.ps1 index 7c7b9f4a1665..0d2baf743c73 100644 --- a/eng/scripts/distribute-packages-to-matrix.ps1 +++ b/eng/scripts/distribute-packages-to-matrix.ps1 @@ -56,13 +56,14 @@ $batchValues = @() # regular case is when we have more packages than batches if ($packageProperties.Count -ge $batchCount) { for ($i = 0; $i -lt $packageProperties.Count; $i++) { + Write-Host "Assigning package $($packageProperties[$i].Name) to batch $batchCounter" $batches[$batchCounter] += $packageProperties[$i].Name.Replace(".json", "") $batchCounter = ($batchCounter + 1) % $batchCount } } # the case where we have fewer packages than batches, we instead walk the batches # and assign packages to them, even if we end up assigning the same package twice -# that's ok, we just don't want to have dead platforms +# that's ok, we just don't want to have dead platforms running no tests else { for ($i = 0; $i -lt $batchCount; $i++) { $batches[$i] += $packageProperties[$i % $packageProperties.Count].Name.Replace(".json", "") @@ -74,11 +75,11 @@ foreach($key in $batches.Keys) { $batchValues += $batches[$key] -join "," } -# set the values for the matrix -$matrix.matrix | Add-Member -MemberType NoteProperty -Name TargetingString -Value $batchValues[0..($versionCount - 1)] - # set the values for include -$matrix.include[0].CoverageConfig.ubuntu2004_39_coverage | Add-Member -MemberType NoteProperty -Name TargetingString -Value $batchValues[$versionCount] -$matrix.include[1].Config.Ubuntu2004_312 | Add-Member -MemberType NoteProperty -Name TargetingString -Value $batchValues[$versionCount+1] +$matrix.include[0].CoverageConfig.ubuntu2004_39_coverage | Add-Member -Force -MemberType NoteProperty -Name TargetingString -Value $batchValues[0] +$matrix.include[1].Config.Ubuntu2004_312 | Add-Member -Force -MemberType NoteProperty -Name TargetingString -Value $batchValues[1] + +# set the values for the matrix +$matrix.matrix | Add-Member -Force -MemberType NoteProperty -Name TargetingString -Value $batchValues[2..($batchValues.Length-1)] $matrix | ConvertTo-Json -Depth 100 | Set-Content -Path $PlatformMatrix From a8c9fd0bb450ced7cef7d3a0e301b841bf7553b7 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Wed, 14 Aug 2024 15:01:35 -0700 Subject: [PATCH 086/119] easy visibility as to what is being built --- .../templates/steps/resolve-package-targeting.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/templates/steps/resolve-package-targeting.yml b/eng/pipelines/templates/steps/resolve-package-targeting.yml index e79cc63c95d0..da6e35d3ea04 100644 --- a/eng/pipelines/templates/steps/resolve-package-targeting.yml +++ b/eng/pipelines/templates/steps/resolve-package-targeting.yml @@ -25,8 +25,12 @@ steps: $setting = $packageProperties -join "," } - Write-Host "Setting TargetingString to $setting" + Write-Host "Setting TargetingString to: `n$setting" Write-Host "##vso[task.setvariable variable=TargetingString;]$setting" - displayName: Resolve targeted packages + displayName: Resolve Targeted Packages # if targeting has been set by matrix, this value will already be populated condition: eq(variables['TargetingString'], '') + - pwsh: | + Write-Host "This run is targeting: `n$(TargetingString)" + displayName: Resolve Targeted Packages + condition: ne(variables['TargetingString'], '') \ No newline at end of file From d9acae6e3a664c5263ad97912ee6720ffb342834 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Mon, 26 Aug 2024 11:56:15 -0700 Subject: [PATCH 087/119] generate a parallel invocation for sphinx. need to take this over to the new pipeline --- .../steps/build-extended-artifacts.yml | 15 ++--- eng/pipelines/templates/steps/run_sphinx.yml | 14 +++++ eng/scripts/invoke-sphinx-parallel.ps1 | 59 +++++++++++++++++++ 3 files changed, 77 insertions(+), 11 deletions(-) create mode 100644 eng/pipelines/templates/steps/run_sphinx.yml create mode 100644 eng/scripts/invoke-sphinx-parallel.ps1 diff --git a/eng/pipelines/templates/steps/build-extended-artifacts.yml b/eng/pipelines/templates/steps/build-extended-artifacts.yml index 79f7c1f96fca..d5cd069ea77f 100644 --- a/eng/pipelines/templates/steps/build-extended-artifacts.yml +++ b/eng/pipelines/templates/steps/build-extended-artifacts.yml @@ -42,7 +42,6 @@ steps: - template: /eng/pipelines/templates/steps/use-venv.yml - script: | - python -m pip install setuptools==58.3.0 python -m pip install -r eng/ci_tools.txt displayName: 'Prep Environment' @@ -53,16 +52,10 @@ steps: parameters: DevFeedName: ${{ parameters.DevFeedName }} - - task: PythonScript@0 - displayName: 'Generate Docs' - condition: and(succeededOrFailed(), ${{parameters.BuildDocs}}) - inputs: - scriptPath: 'scripts/devops_tasks/dispatch_tox.py' - arguments: >- - "$(TargetingString)" - --service="${{ parameters.ServiceDirectory }}" - --toxenv=sphinx - --wheel_dir="$(Build.ArtifactStagingDirectory)" + - template: /eng/pipelines/templates/steps/run_sphinx.yml + parameters: + ServiceDirectory: "${{ parameters.ServiceDirectory }}" + WheelDirectory: "$(Build.ArtifactStagingDirectory)" - pwsh: | Write-Host "##vso[task.setvariable variable=PIP_INDEX_URL]https://pypi.python.org/simple" diff --git a/eng/pipelines/templates/steps/run_sphinx.yml b/eng/pipelines/templates/steps/run_sphinx.yml new file mode 100644 index 000000000000..edba67d84ba5 --- /dev/null +++ b/eng/pipelines/templates/steps/run_sphinx.yml @@ -0,0 +1,14 @@ +parameters: + ServiceDirectory: '' + WheelDirectory: '' + +# `$(TargetingString)` is set by a prior resolution step. +steps: + - pwsh: | + ./eng/scripts/invoke-sphinx-parallel.ps1 ` + -TargetingString "$(TargetingString)" ` + -RepoRoot "$(Build.Sourcesdirectory)" ` + -ServiceDirectory "${{ parameters.ServiceDirectory }}" ` + -WheelDirectory "${{ parameters.WheelDirectory }}" + Write-Host $LASTEXITCODE + displayName: 'Run Sphinx Environments' diff --git a/eng/scripts/invoke-sphinx-parallel.ps1 b/eng/scripts/invoke-sphinx-parallel.ps1 new file mode 100644 index 000000000000..a46c6bfbcec5 --- /dev/null +++ b/eng/scripts/invoke-sphinx-parallel.ps1 @@ -0,0 +1,59 @@ + + +<# +.DESCRIPTION +This script is used to dispatch tox in parallel to multiple target packages. + +Because the sphinx invocation is IO-bound and do not influence each other, we are doing just simply to speed up +the total runtime. +#> + +param( + $TargetingString, + $RepoRoot, + $ServiceDirectory = "", + $WheelDirectory = "" +) + +$pythonScript = Join-Path "$RepoRoot" "scripts" "devops_tasks" "dispatch_tox.py" + +# Create an array to hold the jobs +$jobs = @() + +$packages = $TargetingString -split " " + +$optionalParams = "" + +if ($ServiceDirectory) { + $optionalParams += "--service $ServiceDirectory" +} + +if ($WheelDirectory) { + $optionalParams += " --wheel_dir $WheelDirectory" +} + +# Start 21 parallel invocations of the Python script +foreach ($package in $packages) { + $jobs += Start-Job -ScriptBlock { + param($Pkg, $ScriptPath, $RepoRoot, $Params) + $log = Join-Path $RepoRoot "sphinx-$Pkg.log" + & python $ScriptPath $Pkg --toxenv="sphinx" $Params --disablecov 2>&1 >> $log + return $LASTEXITCODE + } -ArgumentList $package, $pythonScript, $RepoRoot, $optionalParams +} + +$jobs | ForEach-Object { $_ | Wait-Job } + +$exitCodes = $jobs | ForEach-Object { $_ | Receive-Job } +$nonZeroExit = $exitCodes | Where-Object { $_ -ne 0 } +$jobs | ForEach-Object { $_ | Remove-Job } + +if ($nonZeroExit) { + Write-Host "One or more scripts failed with a non-zero exit code." + exit 1 +} else { + Write-Host "All scripts completed successfully." + exit 0 +} + +Get-ChildItem $RepoRoot -Filter "sphinx-*.log" | ForEach-Object { Get-Content $_.FullName } \ No newline at end of file From 85ead190265279ec5f6adeb61e9c52c7f120d24b Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Mon, 26 Aug 2024 13:53:50 -0700 Subject: [PATCH 088/119] targeting string is split by comma not space --- eng/scripts/invoke-sphinx-parallel.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/scripts/invoke-sphinx-parallel.ps1 b/eng/scripts/invoke-sphinx-parallel.ps1 index a46c6bfbcec5..573208366323 100644 --- a/eng/scripts/invoke-sphinx-parallel.ps1 +++ b/eng/scripts/invoke-sphinx-parallel.ps1 @@ -20,7 +20,7 @@ $pythonScript = Join-Path "$RepoRoot" "scripts" "devops_tasks" "dispatch_tox.py" # Create an array to hold the jobs $jobs = @() -$packages = $TargetingString -split " " +$packages = $TargetingString -split "," $optionalParams = "" From c969039d60105a36d0373d6efbba437d18656563 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Mon, 26 Aug 2024 14:19:58 -0700 Subject: [PATCH 089/119] ensure that we get the sphinx log contents --- eng/scripts/invoke-sphinx-parallel.ps1 | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/eng/scripts/invoke-sphinx-parallel.ps1 b/eng/scripts/invoke-sphinx-parallel.ps1 index 573208366323..d057e5472780 100644 --- a/eng/scripts/invoke-sphinx-parallel.ps1 +++ b/eng/scripts/invoke-sphinx-parallel.ps1 @@ -50,10 +50,9 @@ $jobs | ForEach-Object { $_ | Remove-Job } if ($nonZeroExit) { Write-Host "One or more scripts failed with a non-zero exit code." + Get-ChildItem $RepoRoot -Filter "sphinx-*.log" | ForEach-Object { Get-Content $_.FullName } exit 1 } else { Write-Host "All scripts completed successfully." exit 0 } - -Get-ChildItem $RepoRoot -Filter "sphinx-*.log" | ForEach-Object { Get-Content $_.FullName } \ No newline at end of file From fcc1dcffaa66f9003549df2fc57b8878cc4cfdee Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Mon, 26 Aug 2024 14:42:35 -0700 Subject: [PATCH 090/119] change the formatting of the arguments --- eng/scripts/invoke-sphinx-parallel.ps1 | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/eng/scripts/invoke-sphinx-parallel.ps1 b/eng/scripts/invoke-sphinx-parallel.ps1 index d057e5472780..ef3e5e5d4bf7 100644 --- a/eng/scripts/invoke-sphinx-parallel.ps1 +++ b/eng/scripts/invoke-sphinx-parallel.ps1 @@ -16,27 +16,23 @@ param( ) $pythonScript = Join-Path "$RepoRoot" "scripts" "devops_tasks" "dispatch_tox.py" - -# Create an array to hold the jobs $jobs = @() - $packages = $TargetingString -split "," - $optionalParams = "" if ($ServiceDirectory) { - $optionalParams += "--service $ServiceDirectory" + $optionalParams += "--service=$ServiceDirectory" } if ($WheelDirectory) { - $optionalParams += " --wheel_dir $WheelDirectory" + $optionalParams += " --wheel_dir=$WheelDirectory" } -# Start 21 parallel invocations of the Python script foreach ($package in $packages) { $jobs += Start-Job -ScriptBlock { param($Pkg, $ScriptPath, $RepoRoot, $Params) $log = Join-Path $RepoRoot "sphinx-$Pkg.log" + Write-Host "& python $ScriptPath $Pkg --toxenv="sphinx" $Params --disablecov 2>&1 >> $log" & python $ScriptPath $Pkg --toxenv="sphinx" $Params --disablecov 2>&1 >> $log return $LASTEXITCODE } -ArgumentList $package, $pythonScript, $RepoRoot, $optionalParams From 1826bfab5616fcbba2db0c1665040c08443371a6 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Mon, 26 Aug 2024 14:58:02 -0700 Subject: [PATCH 091/119] dump the outputs even in the case of success --- eng/scripts/invoke-sphinx-parallel.ps1 | 1 + 1 file changed, 1 insertion(+) diff --git a/eng/scripts/invoke-sphinx-parallel.ps1 b/eng/scripts/invoke-sphinx-parallel.ps1 index ef3e5e5d4bf7..924648dbfc5f 100644 --- a/eng/scripts/invoke-sphinx-parallel.ps1 +++ b/eng/scripts/invoke-sphinx-parallel.ps1 @@ -50,5 +50,6 @@ if ($nonZeroExit) { exit 1 } else { Write-Host "All scripts completed successfully." + Get-ChildItem $RepoRoot -Filter "sphinx-*.log" | ForEach-Object { Get-Content $_.FullName } exit 0 } From f058dbce1ca7325e321f2dd4316f4dc97f5bb4e6 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Mon, 26 Aug 2024 15:19:52 -0700 Subject: [PATCH 092/119] some additional debugging. why isn't this picking up actual packages? --- eng/scripts/invoke-sphinx-parallel.ps1 | 4 ++-- scripts/devops_tasks/dispatch_tox.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/scripts/invoke-sphinx-parallel.ps1 b/eng/scripts/invoke-sphinx-parallel.ps1 index 924648dbfc5f..de2084015a63 100644 --- a/eng/scripts/invoke-sphinx-parallel.ps1 +++ b/eng/scripts/invoke-sphinx-parallel.ps1 @@ -32,8 +32,8 @@ foreach ($package in $packages) { $jobs += Start-Job -ScriptBlock { param($Pkg, $ScriptPath, $RepoRoot, $Params) $log = Join-Path $RepoRoot "sphinx-$Pkg.log" - Write-Host "& python $ScriptPath $Pkg --toxenv="sphinx" $Params --disablecov 2>&1 >> $log" - & python $ScriptPath $Pkg --toxenv="sphinx" $Params --disablecov 2>&1 >> $log + Write-Host "& python $ScriptPath $Pkg --toxenv=sphinx $Params --disablecov 2>&1 >> $log" + & python $ScriptPath $Pkg --toxenv=sphinx $Params --disablecov 2>&1 >> $log return $LASTEXITCODE } -ArgumentList $package, $pythonScript, $RepoRoot, $optionalParams } diff --git a/scripts/devops_tasks/dispatch_tox.py b/scripts/devops_tasks/dispatch_tox.py index 4b9dd3196ffa..7eaafbf80c8f 100644 --- a/scripts/devops_tasks/dispatch_tox.py +++ b/scripts/devops_tasks/dispatch_tox.py @@ -133,7 +133,7 @@ ) if len(targeted_packages) == 0: - logging.info("No packages collected. Exit 0.") + logging.info("No packages collected for targeting string {args.glob_string}. Exit 0.") exit(0) logging.info(f"Executing prep_and_run_tox with the executable {sys.executable}.") From 9e45d4df8e798980638b6222e8705184901e73ae Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Mon, 26 Aug 2024 15:38:47 -0700 Subject: [PATCH 093/119] additional debugging. why? --- scripts/devops_tasks/dispatch_tox.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/devops_tasks/dispatch_tox.py b/scripts/devops_tasks/dispatch_tox.py index 7eaafbf80c8f..53d2c76bb55e 100644 --- a/scripts/devops_tasks/dispatch_tox.py +++ b/scripts/devops_tasks/dispatch_tox.py @@ -133,7 +133,7 @@ ) if len(targeted_packages) == 0: - logging.info("No packages collected for targeting string {args.glob_string}. Exit 0.") + logging.info(f"No packages collected for targeting string {args.glob_string} and root dir {root_dir}. Exit 0.") exit(0) logging.info(f"Executing prep_and_run_tox with the executable {sys.executable}.") From 532539f7b64098a183fa2c926db0c363961abcdc Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Mon, 26 Aug 2024 16:15:55 -0700 Subject: [PATCH 094/119] logging update --- tools/azure-sdk-tools/ci_tools/functions.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tools/azure-sdk-tools/ci_tools/functions.py b/tools/azure-sdk-tools/ci_tools/functions.py index 26b53ee6e222..609d34e1bce6 100644 --- a/tools/azure-sdk-tools/ci_tools/functions.py +++ b/tools/azure-sdk-tools/ci_tools/functions.py @@ -201,21 +201,24 @@ def discover_targeted_packages( # glob the starting package set collected_packages = glob_packages(glob_string, target_root_dir) + logging.info(f"Results for glob_string \"{glob_string}\" and root directory \"{target_root_dir}\" are: {collected_packages}") # apply the additional contains filter collected_packages = [pkg for pkg in collected_packages if additional_contains_filter in pkg] + logging.info(f"Results after additional contains filter: \"{additional_contains_filter}\" {collected_packages}") # filter for compatibility, this means excluding a package that doesn't support py36 when we are running a py36 executable if compatibility_filter: collected_packages = apply_compatibility_filter(collected_packages) + logging.info(f"Results after compatibility filter: {collected_packages}") - # apply package-specific exclusions only if we have gotten more than one - if len(collected_packages) > 1: - if not include_inactive: - collected_packages = apply_inactive_filter(collected_packages) + if not include_inactive: + collected_packages = apply_inactive_filter(collected_packages) + logging.info(f"Results after inactive filter: {collected_packages}") # Apply filter based on filter type. for e.g. Docs, Regression, Management collected_packages = apply_business_filter(collected_packages, filter_type) + logging.info(f"Results after business filter: {collected_packages}") return sorted(collected_packages) From 0ba34bbec3644160132acf6b09a5a67b8b78b8ad Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Mon, 26 Aug 2024 16:43:34 -0700 Subject: [PATCH 095/119] more logging updates --- eng/scripts/invoke-sphinx-parallel.ps1 | 6 +++--- scripts/devops_tasks/dispatch_tox.py | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/eng/scripts/invoke-sphinx-parallel.ps1 b/eng/scripts/invoke-sphinx-parallel.ps1 index de2084015a63..bc5730d3294d 100644 --- a/eng/scripts/invoke-sphinx-parallel.ps1 +++ b/eng/scripts/invoke-sphinx-parallel.ps1 @@ -21,18 +21,18 @@ $packages = $TargetingString -split "," $optionalParams = "" if ($ServiceDirectory) { - $optionalParams += "--service=$ServiceDirectory" + $optionalParams += "--service $ServiceDirectory" } if ($WheelDirectory) { - $optionalParams += " --wheel_dir=$WheelDirectory" + $optionalParams += " -w $WheelDirectory" } foreach ($package in $packages) { $jobs += Start-Job -ScriptBlock { param($Pkg, $ScriptPath, $RepoRoot, $Params) $log = Join-Path $RepoRoot "sphinx-$Pkg.log" - Write-Host "& python $ScriptPath $Pkg --toxenv=sphinx $Params --disablecov 2>&1 >> $log" + Write-Host "& python $ScriptPath $Pkg --toxenv sphinx $Params --disablecov 2>&1 >> $log" & python $ScriptPath $Pkg --toxenv=sphinx $Params --disablecov 2>&1 >> $log return $LASTEXITCODE } -ArgumentList $package, $pythonScript, $RepoRoot, $optionalParams diff --git a/scripts/devops_tasks/dispatch_tox.py b/scripts/devops_tasks/dispatch_tox.py index 53d2c76bb55e..0317c18736ff 100644 --- a/scripts/devops_tasks/dispatch_tox.py +++ b/scripts/devops_tasks/dispatch_tox.py @@ -122,6 +122,8 @@ else: target_dir = root_dir + logging.info(f"Beginning discovery for {args.service} and root dir {root_dir}. Resolving to {target_dir}.") + if args.filter_type == "None": args.filter_type = "Build" compatibility_filter = False From cd6cf1ed1821c065009f71cc3decd0d7e66fcbcd Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Mon, 26 Aug 2024 17:13:04 -0700 Subject: [PATCH 096/119] save some simple param updates --- eng/scripts/invoke-sphinx-parallel.ps1 | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/eng/scripts/invoke-sphinx-parallel.ps1 b/eng/scripts/invoke-sphinx-parallel.ps1 index bc5730d3294d..b3dbe121bda0 100644 --- a/eng/scripts/invoke-sphinx-parallel.ps1 +++ b/eng/scripts/invoke-sphinx-parallel.ps1 @@ -18,24 +18,26 @@ param( $pythonScript = Join-Path "$RepoRoot" "scripts" "devops_tasks" "dispatch_tox.py" $jobs = @() $packages = $TargetingString -split "," -$optionalParams = "" + +$serviceParam = "" +$wheelParam = "" if ($ServiceDirectory) { - $optionalParams += "--service $ServiceDirectory" + $serviceParam = "--service=$ServiceDirectory" } if ($WheelDirectory) { - $optionalParams += " -w $WheelDirectory" + $wheelParam = " -w $WheelDirectory" } foreach ($package in $packages) { $jobs += Start-Job -ScriptBlock { - param($Pkg, $ScriptPath, $RepoRoot, $Params) + param($Pkg, $ScriptPath, $RepoRoot, $ServiceParam, $WheelParam) $log = Join-Path $RepoRoot "sphinx-$Pkg.log" - Write-Host "& python $ScriptPath $Pkg --toxenv sphinx $Params --disablecov 2>&1 >> $log" - & python $ScriptPath $Pkg --toxenv=sphinx $Params --disablecov 2>&1 >> $log + Write-Host "& python $ScriptPath $Pkg --toxenv=sphinx $ServiceParam $WheelParam --disablecov 2>&1 >> $log" + & python $ScriptPath $Pkg --toxenv=sphinx $WheelParam $ServiceParam $WheelParam --disablecov 2>&1 >> $log return $LASTEXITCODE - } -ArgumentList $package, $pythonScript, $RepoRoot, $optionalParams + } -ArgumentList $package, $pythonScript, $RepoRoot, $serviceParam, $wheelParam } $jobs | ForEach-Object { $_ | Wait-Job } From 6fb61a497005602d329dff1c5285b0abe7f55e3e Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Mon, 26 Aug 2024 17:49:37 -0700 Subject: [PATCH 097/119] simplify some loggers so we can't stomp on ourselves --- eng/scripts/invoke-sphinx-parallel.ps1 | 17 +++-------------- tools/azure-sdk-tools/ci_tools/build.py | 9 +++------ 2 files changed, 6 insertions(+), 20 deletions(-) diff --git a/eng/scripts/invoke-sphinx-parallel.ps1 b/eng/scripts/invoke-sphinx-parallel.ps1 index b3dbe121bda0..99e2ed52ce10 100644 --- a/eng/scripts/invoke-sphinx-parallel.ps1 +++ b/eng/scripts/invoke-sphinx-parallel.ps1 @@ -19,25 +19,14 @@ $pythonScript = Join-Path "$RepoRoot" "scripts" "devops_tasks" "dispatch_tox.py" $jobs = @() $packages = $TargetingString -split "," -$serviceParam = "" -$wheelParam = "" - -if ($ServiceDirectory) { - $serviceParam = "--service=$ServiceDirectory" -} - -if ($WheelDirectory) { - $wheelParam = " -w $WheelDirectory" -} - foreach ($package in $packages) { $jobs += Start-Job -ScriptBlock { param($Pkg, $ScriptPath, $RepoRoot, $ServiceParam, $WheelParam) $log = Join-Path $RepoRoot "sphinx-$Pkg.log" - Write-Host "& python $ScriptPath $Pkg --toxenv=sphinx $ServiceParam $WheelParam --disablecov 2>&1 >> $log" - & python $ScriptPath $Pkg --toxenv=sphinx $WheelParam $ServiceParam $WheelParam --disablecov 2>&1 >> $log + Write-Host "& python $ScriptPath $Pkg --toxenv=sphinx --service `"$ServiceParam`" -w `"$WheelParam`" --disablecov 2>&1 >> $log" + & python $ScriptPath $Pkg --toxenv=sphinx --service "$ServiceParam" -w "$WheelParam" --disablecov 2>&1 >> $log return $LASTEXITCODE - } -ArgumentList $package, $pythonScript, $RepoRoot, $serviceParam, $wheelParam + } -ArgumentList $package, $pythonScript, $RepoRoot, $ServiceDirectory, $WheelDirectory } $jobs | ForEach-Object { $_ | Wait-Job } diff --git a/tools/azure-sdk-tools/ci_tools/build.py b/tools/azure-sdk-tools/ci_tools/build.py index 8ec07d03dd4a..be83cfe1eb6a 100644 --- a/tools/azure-sdk-tools/ci_tools/build.py +++ b/tools/azure-sdk-tools/ci_tools/build.py @@ -9,9 +9,6 @@ from ci_tools.variables import discover_repo_root, get_artifact_directory from ci_tools.versioning.version_shared import set_version_py, set_dev_classifier from ci_tools.versioning.version_set_dev import get_dev_version, format_build_id -from ci_tools.logging import initialize_logger, run_logged - -logger = initialize_logger("build.py") def build() -> None: parser = argparse.ArgumentParser( @@ -106,7 +103,7 @@ def build() -> None: else: target_dir = repo_root - logger.debug(f"Searching for packages starting from {target_dir} with glob string {args.glob_string} and package filter {args.package_filter_string}") + logging.debug(f"Searching for packages starting from {target_dir} with glob string {args.glob_string} and package filter {args.package_filter_string}") targeted_packages = discover_targeted_packages( args.glob_string, @@ -150,7 +147,7 @@ def build_packages( build_apiview_artifact: bool = False, build_id: str = "", ): - logger.log(level=logging.INFO, msg=f"Generating {targeted_packages} using python{sys.version}") + logging.log(level=logging.INFO, msg=f"Generating {targeted_packages} using python{sys.version}") for package_root in targeted_packages: setup_parsed = ParsedSetup.from_path(package_root) @@ -162,7 +159,7 @@ def build_packages( new_version = get_dev_version(setup_parsed.version, build_id) - logger.log(level=logging.DEBUG, msg=f"{setup_parsed.name}: {setup_parsed.version} -> {new_version}") + logging.log(level=logging.DEBUG, msg=f"{setup_parsed.name}: {setup_parsed.version} -> {new_version}") set_version_py(setup_parsed.setup_filename, new_version) set_dev_classifier(setup_parsed.setup_filename, new_version) From 5db6fb49012ec51d850b191e4157cd301fdf98ec Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Thu, 29 Aug 2024 17:00:38 -0700 Subject: [PATCH 098/119] merge main, then update language-settings to reflect the new property name --- eng/scripts/Language-Settings.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 index 322d9e38d327..52b6dbeb9e6b 100644 --- a/eng/scripts/Language-Settings.ps1 +++ b/eng/scripts/Language-Settings.ps1 @@ -44,7 +44,7 @@ function Get-AllPackageInfoFromRepo ($serviceDirectory) $isNewSdk = ($pkgInfo[2] -eq "True") $pkgDirectoryPath = $pkgInfo[3] - $additionalDependentPackages = $pkgInfo[4] -Split "," + $additionalValidationPackages = $pkgInfo[4] -Split "," $serviceDirectoryName = Split-Path (Split-Path -Path $pkgDirectoryPath -Parent) -Leaf if ($packageName -match "mgmt") @@ -59,7 +59,7 @@ function Get-AllPackageInfoFromRepo ($serviceDirectory) $pkgProp.IsNewSdk = $isNewSdk $pkgProp.SdkType = $sdkType $pkgProp.ArtifactName = $packageName - $pkgProp.DependentPackages = $additionalDependentPackages + $pkgProp.AdditionalValidationPackages = $additionalValidationPackages $allPackageProps += $pkgProp } return $allPackageProps From 52fe2a2ba878079bd1bdce942a23bc7c0d149dae Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Thu, 29 Aug 2024 18:29:18 -0700 Subject: [PATCH 099/119] I think the issue is the replacement of the dev reqs stomping on each other. --- eng/pipelines/templates/steps/run_sphinx.yml | 4 ++ scripts/devops_tasks/replace_requirements.py | 65 ++++++++++++++++++++ 2 files changed, 69 insertions(+) create mode 100644 scripts/devops_tasks/replace_requirements.py diff --git a/eng/pipelines/templates/steps/run_sphinx.yml b/eng/pipelines/templates/steps/run_sphinx.yml index edba67d84ba5..b63ae64b83c9 100644 --- a/eng/pipelines/templates/steps/run_sphinx.yml +++ b/eng/pipelines/templates/steps/run_sphinx.yml @@ -4,6 +4,10 @@ parameters: # `$(TargetingString)` is set by a prior resolution step. steps: + - pwsh: | + python ./scripts/devops_tasks/replace_requirements.py --service "${{ parameters.ServiceDirectory }}" -w "${{ parameters.WheelDirectory }}" $(TargetingString) + displayName: Pre-Build Dev Requirements + - pwsh: | ./eng/scripts/invoke-sphinx-parallel.ps1 ` -TargetingString "$(TargetingString)" ` diff --git a/scripts/devops_tasks/replace_requirements.py b/scripts/devops_tasks/replace_requirements.py new file mode 100644 index 000000000000..60cf55d1e666 --- /dev/null +++ b/scripts/devops_tasks/replace_requirements.py @@ -0,0 +1,65 @@ +import os +import argparse +import logging + +root_dir = os.path.abspath(os.path.join(os.path.abspath(__file__), "..", "..", "..")) +test_tools_path = os.path.join(root_dir, "eng", "test_tools.txt") +dependency_tools_path = os.path.join(root_dir, "eng", "dependency_tools.txt") + +from ci_tools.functions import discover_targeted_packages +from ci_tools.scenario.generation import replace_dev_reqs + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description=""" +This script is a single point of access to replace relative requirements within the repo with prebuilt versions. +""" + ) + + parser.add_argument( + "glob_string", + nargs="?", + help=( + "A comma separated list of glob strings that will target the top level directories that contain packages." + 'Examples: All = "azure-*", Single = "azure-keyvault", Targeted Multiple = "azure-keyvault,azure-mgmt-resource"' + ), + ) + + parser.add_argument( + "--service", + help=("Name of service directory (under sdk/) to test. Example: --service applicationinsights"), + ) + + parser.add_argument( + "-w", + "--wheel_dir", + dest="wheel_dir", + help="Location for prebuilt artifacts (if any)", + ) + + args = parser.parse_args() + + # We need to support both CI builds of everything and individual service + # folders. This logic allows us to do both. + if args.service and args.service != "auto": + service_dir = os.path.join("sdk", args.service) + target_dir = os.path.join(root_dir, service_dir) + else: + target_dir = root_dir + + logging.info(f"Replacing local relative requirements in {test_tools_path} and {dependency_tools_path}.") + + logging.info(f"Beginning discovery for {args.service} and root dir {root_dir}. Resolving to {target_dir}.") + + targeted_packages = discover_targeted_packages( + args.glob_string, target_dir, "", "Build", False + ) + + for package in targeted_packages: + dev_requirements_path = os.path.join(package, "dev_requirements.txt") + + breakpoint() + replace_dev_reqs(test_tools_path, package, args.wheel_dir) + replace_dev_reqs(dependency_tools_path, package, args.wheel_dir) + replace_dev_reqs(dev_requirements_path, package, args.wheel_dir) From d3d89ddf968582d9c7a918f1e1f91f932a07ec91 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Thu, 29 Aug 2024 19:03:08 -0700 Subject: [PATCH 100/119] I left a breakpoint behind --- scripts/devops_tasks/replace_requirements.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/devops_tasks/replace_requirements.py b/scripts/devops_tasks/replace_requirements.py index 60cf55d1e666..952a5db0e43c 100644 --- a/scripts/devops_tasks/replace_requirements.py +++ b/scripts/devops_tasks/replace_requirements.py @@ -59,7 +59,6 @@ for package in targeted_packages: dev_requirements_path = os.path.join(package, "dev_requirements.txt") - breakpoint() replace_dev_reqs(test_tools_path, package, args.wheel_dir) replace_dev_reqs(dependency_tools_path, package, args.wheel_dir) replace_dev_reqs(dev_requirements_path, package, args.wheel_dir) From 3b38df41ab0a62fc907c91e1ce418b14c019424e Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Fri, 30 Aug 2024 11:39:43 -0700 Subject: [PATCH 101/119] repair namespace update that doesn't get an artifact list anymore for PRs --- .../steps/build-extended-artifacts.yml | 1 - .../Save-Package-Namespaces-Property.ps1 | 18 ++++++------------ 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/eng/pipelines/templates/steps/build-extended-artifacts.yml b/eng/pipelines/templates/steps/build-extended-artifacts.yml index 0e34b5ece86c..d0aed16f77ad 100644 --- a/eng/pipelines/templates/steps/build-extended-artifacts.yml +++ b/eng/pipelines/templates/steps/build-extended-artifacts.yml @@ -73,7 +73,6 @@ steps: filePath: $(Build.SourcesDirectory)/eng/scripts/Save-Package-Namespaces-Property.ps1 arguments: > -ArtifactStagingDirectory "$(Build.ArtifactStagingDirectory)" - -ArtifactsList ('${{ convertToJson(parameters.Artifacts) }}' | ConvertFrom-Json) pwsh: true workingDirectory: $(Pipeline.Workspace) displayName: Update package properties with namespaces diff --git a/eng/scripts/Save-Package-Namespaces-Property.ps1 b/eng/scripts/Save-Package-Namespaces-Property.ps1 index 3c305d23c5f7..20a7cf681e4b 100644 --- a/eng/scripts/Save-Package-Namespaces-Property.ps1 +++ b/eng/scripts/Save-Package-Namespaces-Property.ps1 @@ -24,8 +24,6 @@ producing docs. Param ( [Parameter(Mandatory = $True)] [string] $ArtifactStagingDirectory, - [Parameter(Mandatory=$true)] - [array] $ArtifactsList ) $ArtifactsList = $ArtifactsList | Where-Object -Not "skipPublishDocMs" @@ -43,19 +41,15 @@ if (-not (Test-Path -Path $ArtifactStagingDirectory)) { exit 1 } -Write-Host "" -Write-Host "ArtifactsList:" -$ArtifactsList | Format-Table -Property Name | Out-String | Write-Host - $packageInfoDirectory = Join-Path $ArtifactStagingDirectory "PackageInfo" - $foundError = $false -# At this point the packageInfo files should have been already been created. -# The only thing being done here is adding or updating namespaces for libraries -# that will be producing docs. -foreach($artifact in $ArtifactsList) { +$artifacts = Get-ChildItem -Path $packageInfoDirectory -File -Filter "*.json" +$artifacts | Format-Table -Property Name | Out-String | Write-Host + +# by this point, the PackageInfo folder will have a json file for each artifact +# we simply need to read each file, get the appropriate metadata, and add the namespaces if necessary +foreach($packageInfoFile in $artifacts) { # Get the version from the packageInfo file - $packageInfoFile = Join-Path $packageInfoDirectory "$($artifact.Name).json" Write-Host "processing $($packageInfoFile.FullName)" $packageInfo = ConvertFrom-Json (Get-Content $packageInfoFile -Raw) $version = $packageInfo.Version From 95fb1eab0fc2c9c41db6e9399a69c148724a292c Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Fri, 30 Aug 2024 12:04:07 -0700 Subject: [PATCH 102/119] remove extra comma --- eng/scripts/Save-Package-Namespaces-Property.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/scripts/Save-Package-Namespaces-Property.ps1 b/eng/scripts/Save-Package-Namespaces-Property.ps1 index 20a7cf681e4b..22d576e61992 100644 --- a/eng/scripts/Save-Package-Namespaces-Property.ps1 +++ b/eng/scripts/Save-Package-Namespaces-Property.ps1 @@ -23,7 +23,7 @@ producing docs. [CmdletBinding()] Param ( [Parameter(Mandatory = $True)] - [string] $ArtifactStagingDirectory, + [string] $ArtifactStagingDirectory ) $ArtifactsList = $ArtifactsList | Where-Object -Not "skipPublishDocMs" From 7d51ba435dace93b11123858ed6045426eecc0db Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Fri, 30 Aug 2024 14:24:41 -0700 Subject: [PATCH 103/119] dealing with some output issues. --- .../steps/resolve-package-targeting.yml | 4 ++-- ...x-parallel.ps1 => invoke-tox-parallel.ps1} | 19 ++++++++++++------- scripts/devops_tasks/set_tox_environment.py | 2 +- 3 files changed, 15 insertions(+), 10 deletions(-) rename eng/scripts/{invoke-sphinx-parallel.ps1 => invoke-tox-parallel.ps1} (58%) diff --git a/eng/pipelines/templates/steps/resolve-package-targeting.yml b/eng/pipelines/templates/steps/resolve-package-targeting.yml index da6e35d3ea04..e2a612126ced 100644 --- a/eng/pipelines/templates/steps/resolve-package-targeting.yml +++ b/eng/pipelines/templates/steps/resolve-package-targeting.yml @@ -25,12 +25,12 @@ steps: $setting = $packageProperties -join "," } - Write-Host "Setting TargetingString to: `n$setting" Write-Host "##vso[task.setvariable variable=TargetingString;]$setting" displayName: Resolve Targeted Packages # if targeting has been set by matrix, this value will already be populated condition: eq(variables['TargetingString'], '') + - pwsh: | Write-Host "This run is targeting: `n$(TargetingString)" - displayName: Resolve Targeted Packages + displayName: Display Targeted Packages condition: ne(variables['TargetingString'], '') \ No newline at end of file diff --git a/eng/scripts/invoke-sphinx-parallel.ps1 b/eng/scripts/invoke-tox-parallel.ps1 similarity index 58% rename from eng/scripts/invoke-sphinx-parallel.ps1 rename to eng/scripts/invoke-tox-parallel.ps1 index 99e2ed52ce10..63406a07a18c 100644 --- a/eng/scripts/invoke-sphinx-parallel.ps1 +++ b/eng/scripts/invoke-tox-parallel.ps1 @@ -11,6 +11,7 @@ the total runtime. param( $TargetingString, $RepoRoot, + $Check = "sphinx" $ServiceDirectory = "", $WheelDirectory = "" ) @@ -21,12 +22,12 @@ $packages = $TargetingString -split "," foreach ($package in $packages) { $jobs += Start-Job -ScriptBlock { - param($Pkg, $ScriptPath, $RepoRoot, $ServiceParam, $WheelParam) - $log = Join-Path $RepoRoot "sphinx-$Pkg.log" - Write-Host "& python $ScriptPath $Pkg --toxenv=sphinx --service `"$ServiceParam`" -w `"$WheelParam`" --disablecov 2>&1 >> $log" - & python $ScriptPath $Pkg --toxenv=sphinx --service "$ServiceParam" -w "$WheelParam" --disablecov 2>&1 >> $log + param($Pkg, $ScriptPath, $RepoRoot, $Toxenv, $ServiceParam, $WheelParam) + $log = Join-Path $RepoRoot "$Toxenv-$Pkg.log" + Write-Host "& python $ScriptPath $Pkg --toxenv=$Toxenv --service `"$ServiceParam`" -w `"$WheelParam`" --disablecov 2>&1 >> $log" + & python $ScriptPath $Pkg --toxenv=$Toxenv --service "$ServiceParam" -w "$WheelParam" --disablecov 2>&1 >> $log return $LASTEXITCODE - } -ArgumentList $package, $pythonScript, $RepoRoot, $ServiceDirectory, $WheelDirectory + } -ArgumentList $package, $pythonScript, $RepoRoot, $Check, $ServiceDirectory, $WheelDirectory } $jobs | ForEach-Object { $_ | Wait-Job } @@ -35,12 +36,16 @@ $exitCodes = $jobs | ForEach-Object { $_ | Receive-Job } $nonZeroExit = $exitCodes | Where-Object { $_ -ne 0 } $jobs | ForEach-Object { $_ | Remove-Job } +Get-ChildItem $RepoRoot -Filter "$Toxenv-*.log" ` + | ForEach-Object { + Write-Host "Output for $($_.Name)" + Get-Content $_.FullName + } + if ($nonZeroExit) { Write-Host "One or more scripts failed with a non-zero exit code." - Get-ChildItem $RepoRoot -Filter "sphinx-*.log" | ForEach-Object { Get-Content $_.FullName } exit 1 } else { Write-Host "All scripts completed successfully." - Get-ChildItem $RepoRoot -Filter "sphinx-*.log" | ForEach-Object { Get-Content $_.FullName } exit 0 } diff --git a/scripts/devops_tasks/set_tox_environment.py b/scripts/devops_tasks/set_tox_environment.py index 1dbe8140897f..fa689640ee55 100644 --- a/scripts/devops_tasks/set_tox_environment.py +++ b/scripts/devops_tasks/set_tox_environment.py @@ -46,7 +46,7 @@ def process_ci_skips(glob_string: str, service: str ) -> None: checks_with_global_skip = ["pylint", "verifywhl", "verifysdist" "bandit", "mypy", "pyright", "verifytypes"] root_dir = os.path.abspath(os.path.join(os.path.abspath(__file__), "..", "..", "..")) - if service: + if service and service != "auto": target_dir = os.path.join(root_dir, "sdk", service) else: target_dir = root_dir From 0e55d61dd24f64a1da0fb23b999e0818ed488648 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Fri, 30 Aug 2024 14:30:33 -0700 Subject: [PATCH 104/119] force save-package-properties commit into proper format --- eng/common/scripts/Package-Properties.ps1 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/eng/common/scripts/Package-Properties.ps1 b/eng/common/scripts/Package-Properties.ps1 index 6a70070beb56..dc3090ea63f7 100644 --- a/eng/common/scripts/Package-Properties.ps1 +++ b/eng/common/scripts/Package-Properties.ps1 @@ -121,6 +121,7 @@ function Get-PrPkgProperties([string]$InputDiffJson) { $pkgDirectory = Resolve-Path "$($pkg.DirectoryPath)" $lookupKey = ($pkg.DirectoryPath).Replace($RepoRoot, "").SubString(1) $lookup[$lookupKey] = $pkg + Write-Host "Saving lookup key $lookupKey for $pkg" foreach ($file in $targetedFiles) { @@ -130,6 +131,7 @@ function Get-PrPkgProperties([string]$InputDiffJson) { $packagesWithChanges += $pkg if ($pkg.AdditionalValidationPackages) { + Write-Host ($pkg.AdditionalValidationPackages -join ",") $additionalValidationPackages += $pkg.AdditionalValidationPackages } } @@ -138,6 +140,7 @@ function Get-PrPkgProperties([string]$InputDiffJson) { foreach ($addition in $additionalValidationPackages) { $key = $addition.Replace($RepoRoot, "").SubString(1) + Write-Host "Processing $addition to see if we have metadata about it: $key" if ($lookup[$key]) { $packagesWithChanges += $lookup[$key] From 9ad9e9b834a38cbe31a2a73d05482ff292b6d1a1 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Fri, 30 Aug 2024 14:40:16 -0700 Subject: [PATCH 105/119] Revert "force save-package-properties commit into proper format" This reverts commit 0e55d61dd24f64a1da0fb23b999e0818ed488648. --- eng/common/scripts/Package-Properties.ps1 | 3 --- 1 file changed, 3 deletions(-) diff --git a/eng/common/scripts/Package-Properties.ps1 b/eng/common/scripts/Package-Properties.ps1 index dc3090ea63f7..6a70070beb56 100644 --- a/eng/common/scripts/Package-Properties.ps1 +++ b/eng/common/scripts/Package-Properties.ps1 @@ -121,7 +121,6 @@ function Get-PrPkgProperties([string]$InputDiffJson) { $pkgDirectory = Resolve-Path "$($pkg.DirectoryPath)" $lookupKey = ($pkg.DirectoryPath).Replace($RepoRoot, "").SubString(1) $lookup[$lookupKey] = $pkg - Write-Host "Saving lookup key $lookupKey for $pkg" foreach ($file in $targetedFiles) { @@ -131,7 +130,6 @@ function Get-PrPkgProperties([string]$InputDiffJson) { $packagesWithChanges += $pkg if ($pkg.AdditionalValidationPackages) { - Write-Host ($pkg.AdditionalValidationPackages -join ",") $additionalValidationPackages += $pkg.AdditionalValidationPackages } } @@ -140,7 +138,6 @@ function Get-PrPkgProperties([string]$InputDiffJson) { foreach ($addition in $additionalValidationPackages) { $key = $addition.Replace($RepoRoot, "").SubString(1) - Write-Host "Processing $addition to see if we have metadata about it: $key" if ($lookup[$key]) { $packagesWithChanges += $lookup[$key] From 25951f72600a898da5a734ba51b8f2894ed0c3c5 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Fri, 30 Aug 2024 14:45:13 -0700 Subject: [PATCH 106/119] update save-package-properties --- eng/pipelines/templates/steps/run_sphinx.yml | 2 +- eng/scripts/get_package_properties.py | 48 ++++++++++---------- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/eng/pipelines/templates/steps/run_sphinx.yml b/eng/pipelines/templates/steps/run_sphinx.yml index b63ae64b83c9..168f080212bf 100644 --- a/eng/pipelines/templates/steps/run_sphinx.yml +++ b/eng/pipelines/templates/steps/run_sphinx.yml @@ -9,7 +9,7 @@ steps: displayName: Pre-Build Dev Requirements - pwsh: | - ./eng/scripts/invoke-sphinx-parallel.ps1 ` + ./eng/scripts/invoke-tox-parallel.ps1 ` -TargetingString "$(TargetingString)" ` -RepoRoot "$(Build.Sourcesdirectory)" ` -ServiceDirectory "${{ parameters.ServiceDirectory }}" ` diff --git a/eng/scripts/get_package_properties.py b/eng/scripts/get_package_properties.py index 6406ccca74c1..99ed62fc878d 100644 --- a/eng/scripts/get_package_properties.py +++ b/eng/scripts/get_package_properties.py @@ -8,34 +8,34 @@ additional_pr_triggers: Dict[str, List[str]] = { "azure-core":[ - os.path.join("sdk", "servicebus", "azure-servicebus"), - os.path.join("sdk", "eventhub", "azure-eventhub"), - os.path.join("sdk", "tables", "azure-data-table"), - os.path.join("sdk", "appconfig", "azure-appconfig"), - os.path.join("sdk", "keyvault", "azure-keyvault-keys"), - os.path.join("sdk", "identity", "azure-identity"), - os.path.join("sdk", "core", "azure-mgmt-core"), - os.path.join("sdk", "core", "azure-core-experimental"), - os.path.join("sdk", "core", "azure-core-tracing-opentelemetry"), - os.path.join("sdk", "core", "azure-core-tracing-opencensus"), + os.path.join("/sdk", "servicebus", "azure-servicebus"), + os.path.join("/sdk", "eventhub", "azure-eventhub"), + os.path.join("/sdk", "tables", "azure-data-table"), + os.path.join("/sdk", "appconfig", "azure-appconfig"), + os.path.join("/sdk", "keyvault", "azure-keyvault-keys"), + os.path.join("/sdk", "identity", "azure-identity"), + os.path.join("/sdk", "core", "azure-mgmt-core"), + os.path.join("/sdk", "core", "azure-core-experimental"), + os.path.join("/sdk", "core", "azure-core-tracing-opentelemetry"), + os.path.join("/sdk", "core", "azure-core-tracing-opencensus"), # todo: this currently won't be included, as azure-cosmos needs some special construction # related to windows only + emulator - # os.path.join("sdk", "cosmos", "azure-cosmos"), - os.path.join("sdk", "ml", "azure-ai-ml"), - os.path.join("sdk", "ai", "azure-ai-documentintelligence"), - os.path.join("sdk", "ai", "azure-ai-inference"), - os.path.join("sdk", "ai", "azure-ai-textanalytics"), - os.path.join("sdk", "ai", "azure-ai-doctranslation"), - os.path.join("sdk", "compute", "azure-mgmt-compute"), - os.path.join("sdk", "communication", "azure-communication-chat"), - os.path.join("sdk", "communication", "azure-communication-identity"), - os.path.join("sdk", "storage", "azure-storage-blob") + # os.path.join("/sdk", "cosmos", "azure-cosmos"), + os.path.join("/sdk", "ml", "azure-ai-ml"), + os.path.join("/sdk", "ai", "azure-ai-documentintelligence"), + os.path.join("/sdk", "ai", "azure-ai-inference"), + os.path.join("/sdk", "ai", "azure-ai-textanalytics"), + os.path.join("/sdk", "ai", "azure-ai-doctranslation"), + os.path.join("/sdk", "compute", "azure-mgmt-compute"), + os.path.join("/sdk", "communication", "azure-communication-chat"), + os.path.join("/sdk", "communication", "azure-communication-identity"), + os.path.join("/sdk", "storage", "azure-storage-blob") ], "azure-mgmt-core": [ - os.path.join("sdk", "compute", "azure-mgmt-compute"), - os.path.join("sdk", "network", "azure-mgmt-network"), - os.path.join("sdk", "resource", "azure-mgmt-resource"), - os.path.join("sdk", "keyvault", "azure-mgmt-keyvault") + os.path.join("/sdk", "compute", "azure-mgmt-compute"), + os.path.join("/sdk", "network", "azure-mgmt-network"), + os.path.join("/sdk", "resource", "azure-mgmt-resource"), + os.path.join("/sdk", "keyvault", "azure-mgmt-keyvault") ] } From e5e5464778b4619c9eda50d26ddc57c0fac59378 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Fri, 30 Aug 2024 14:58:28 -0700 Subject: [PATCH 107/119] how do I keep missing these little issues --- eng/scripts/invoke-tox-parallel.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/scripts/invoke-tox-parallel.ps1 b/eng/scripts/invoke-tox-parallel.ps1 index 63406a07a18c..5724b4f0dbb3 100644 --- a/eng/scripts/invoke-tox-parallel.ps1 +++ b/eng/scripts/invoke-tox-parallel.ps1 @@ -11,7 +11,7 @@ the total runtime. param( $TargetingString, $RepoRoot, - $Check = "sphinx" + $Check = "sphinx", $ServiceDirectory = "", $WheelDirectory = "" ) From 9c458b13cecad1dac7129faceaf5f80f9ecaca01 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Fri, 30 Aug 2024 15:27:23 -0700 Subject: [PATCH 108/119] add a timeout to the sphinx build job, add dump of logs so we can figure out what went wrong --- eng/pipelines/templates/steps/run_sphinx.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/eng/pipelines/templates/steps/run_sphinx.yml b/eng/pipelines/templates/steps/run_sphinx.yml index 168f080212bf..4a45868a5e8a 100644 --- a/eng/pipelines/templates/steps/run_sphinx.yml +++ b/eng/pipelines/templates/steps/run_sphinx.yml @@ -7,6 +7,12 @@ steps: - pwsh: | python ./scripts/devops_tasks/replace_requirements.py --service "${{ parameters.ServiceDirectory }}" -w "${{ parameters.WheelDirectory }}" $(TargetingString) displayName: Pre-Build Dev Requirements + timeoutInMinutes: 15 + + - pwsh: | + Get-ChildItem $RepoRoot -Filter "$Toxenv-*.log" | ForEach-Object { Write-Host "Output for $($_.Name)"; Get-Content $_.FullName } + display: Dump Sphinx Logs + condition: failed() - pwsh: | ./eng/scripts/invoke-tox-parallel.ps1 ` From ba0a0ad95d4834695f30c0f427cdeb80b9e22692 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Fri, 30 Aug 2024 15:48:15 -0700 Subject: [PATCH 109/119] display -> displayNAme --- eng/pipelines/templates/steps/run_sphinx.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/steps/run_sphinx.yml b/eng/pipelines/templates/steps/run_sphinx.yml index 4a45868a5e8a..babdfbcaa696 100644 --- a/eng/pipelines/templates/steps/run_sphinx.yml +++ b/eng/pipelines/templates/steps/run_sphinx.yml @@ -11,7 +11,7 @@ steps: - pwsh: | Get-ChildItem $RepoRoot -Filter "$Toxenv-*.log" | ForEach-Object { Write-Host "Output for $($_.Name)"; Get-Content $_.FullName } - display: Dump Sphinx Logs + displayName: Dump Sphinx Logs condition: failed() - pwsh: | From ee38ac03e50af5f0105592db36d4cc89c79e9bc6 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 3 Sep 2024 10:44:50 -0700 Subject: [PATCH 110/119] section the documentation build away, as nothing other than the release directly depends on it. build_extended will still produce APIView and packages_extended. --- eng/pipelines/templates/jobs/ci.yml | 32 ++++++++ .../steps/build-extended-artifacts.yml | 78 ++++++++++--------- eng/pipelines/templates/steps/run_apistub.yml | 6 ++ 3 files changed, 78 insertions(+), 38 deletions(-) diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index 857acc917c5a..0a1ee8db8bae 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -134,6 +134,37 @@ jobs: steps: - template: /eng/pipelines/templates/steps/download-package-artifacts.yml + - template: /eng/pipelines/templates/steps/resolve-package-targeting.yml + parameters: + BuildTargetingString: ${{ parameters.BuildTargetingString }} + PackagePropertiesFolder: $(Build.ArtifactStagingDirectory)/PackageInfo + + - template: ../steps/build-extended-artifacts.yml + parameters: + ServiceDirectory: ${{ parameters.ServiceDirectory }} + BeforePublishSteps: ${{ parameters.BeforePublishSteps }} + BuildDocs: false + RunApiStubGen: true + TestPipeline: ${{ parameters.TestPipeline }} + Artifacts: ${{ parameters.Artifacts }} + + - job: 'Build_Documentation' + displayName: Build Docs + dependsOn: + - 'Build_Linux' + - 'Build_Windows' + - 'Build_MacOS' + + timeoutInMinutes: 90 + + pool: + name: $(LINUXPOOL) + image: $(LINUXVMIMAGE) + os: linux + + steps: + - template: /eng/pipelines/templates/steps/download-package-artifacts.yml + - template: /eng/pipelines/templates/steps/resolve-package-targeting.yml parameters: BuildTargetingString: ${{ parameters.BuildTargetingString }} @@ -146,6 +177,7 @@ jobs: BuildDocs: ${{ parameters.BuildDocs }} TestPipeline: ${{ parameters.TestPipeline }} Artifacts: ${{ parameters.Artifacts }} + RunApiStubGen: false - job: 'Analyze' condition: and(succeededOrFailed(), ne(variables['Skip.Analyze'], 'true')) diff --git a/eng/pipelines/templates/steps/build-extended-artifacts.yml b/eng/pipelines/templates/steps/build-extended-artifacts.yml index d0aed16f77ad..30e22f306be2 100644 --- a/eng/pipelines/templates/steps/build-extended-artifacts.yml +++ b/eng/pipelines/templates/steps/build-extended-artifacts.yml @@ -8,16 +8,18 @@ parameters: - name: ServiceDirectory type: string default: '' - - name: BuildDocs - type: boolean - default: true - name: Artifacts type: object default: [] - name: DevFeedName type: string default: 'public/azure-sdk-for-python' - + - name: BuildDocs + type: boolean + default: true + - name: RunApiStubGen + type: boolean + default: true # Please use `$(TargetingString)` to refer to the python packages glob string. This variable is set from resolve-package-targeting.yml. steps: @@ -52,43 +54,43 @@ steps: parameters: DevFeedName: ${{ parameters.DevFeedName }} - - template: /eng/pipelines/templates/steps/run_sphinx.yml - parameters: - ServiceDirectory: "${{ parameters.ServiceDirectory }}" - WheelDirectory: "$(Build.ArtifactStagingDirectory)" - - - pwsh: | - Write-Host "##vso[task.setvariable variable=PIP_INDEX_URL]https://pypi.python.org/simple" - displayName: Reset PIP Index For APIStubGen + - ${{ if eq(parameters.BuildDocs, 'true') }}: + - template: /eng/pipelines/templates/steps/run_sphinx.yml + parameters: + ServiceDirectory: "${{ parameters.ServiceDirectory }}" + WheelDirectory: "$(Build.ArtifactStagingDirectory)" - - template: /eng/pipelines/templates/steps/run_apistub.yml - parameters: - ServiceDirectory: ${{ parameters.ServiceDirectory }} - AdditionalTestArgs: '--wheel_dir="$(Build.ArtifactStagingDirectory)"' + - ${{ if eq(parameters.RunApiStubGen, 'true') }}: + - template: /eng/pipelines/templates/steps/run_apistub.yml + parameters: + ServiceDirectory: ${{ parameters.ServiceDirectory }} + AdditionalTestArgs: '--wheel_dir="$(Build.ArtifactStagingDirectory)"' - ${{ parameters.BeforePublishSteps }} - - task: Powershell@2 - inputs: - filePath: $(Build.SourcesDirectory)/eng/scripts/Save-Package-Namespaces-Property.ps1 - arguments: > - -ArtifactStagingDirectory "$(Build.ArtifactStagingDirectory)" - pwsh: true - workingDirectory: $(Pipeline.Workspace) - displayName: Update package properties with namespaces - - - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml - parameters: - ArtifactPath: '$(Build.ArtifactStagingDirectory)' - ArtifactName: 'packages_extended' + - ${{ if eq(parameters.RunApiStubGen, 'true') }}: + - task: Powershell@2 + inputs: + filePath: $(Build.SourcesDirectory)/eng/scripts/Save-Package-Namespaces-Property.ps1 + arguments: > + -ArtifactStagingDirectory "$(Build.ArtifactStagingDirectory)" + pwsh: true + workingDirectory: $(Pipeline.Workspace) + displayName: Update package properties with namespaces + + - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml + parameters: + ArtifactPath: '$(Build.ArtifactStagingDirectory)' + ArtifactName: 'packages_extended' - - pwsh: | - $directoryExists = Test-Path -Path "$(Build.SourcesDirectory)/_docs" - Write-Output "##vso[task.setvariable variable=DirectoryExists]$directoryExists" - displayName: Check if docs directory exists + - ${{ if eq(parameters.BuildDocs, 'true') }}: + - pwsh: | + $directoryExists = Test-Path -Path "$(Build.SourcesDirectory)/_docs" + Write-Output "##vso[task.setvariable variable=DirectoryExists]$directoryExists" + displayName: Check if docs directory exists - - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml - parameters: - ArtifactPath: '$(Build.SourcesDirectory)/_docs' - CustomCondition: and(${{ parameters.BuildDocs }}, eq(variables['DirectoryExists'], 'True')) - ArtifactName: 'documentation' + - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml + parameters: + ArtifactPath: '$(Build.SourcesDirectory)/_docs' + CustomCondition: eq(variables['DirectoryExists'], 'True') + ArtifactName: 'documentation' diff --git a/eng/pipelines/templates/steps/run_apistub.yml b/eng/pipelines/templates/steps/run_apistub.yml index d18e0f2752be..7928b0d22f74 100644 --- a/eng/pipelines/templates/steps/run_apistub.yml +++ b/eng/pipelines/templates/steps/run_apistub.yml @@ -1,9 +1,15 @@ parameters: ServiceDirectory: '' AdditionalTestArgs: '' + RunApiStub: true # Please use `$(TargetingString)` to refer to the python packages glob string. This variable is set from resolve-package-targeting.yml. steps: + - pwsh: | + Write-Host "##vso[task.setvariable variable=PIP_INDEX_URL]https://pypi.python.org/simple" + displayName: Reset PIP Index For APIStubGen + condition: and(succeededOrFailed(), eq(variables['Skip.ApiStubGen'],'true')) + - task: UsePythonVersion@0 displayName: 'Use Python 3.10' inputs: From f238f60808f7b8a28beb0ae89b0ff664d9320691 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 3 Sep 2024 14:57:54 -0700 Subject: [PATCH 111/119] save progress such that azure-storage-blob skips on pypy distribution --- tools/azure-sdk-tools/ci_tools/functions.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tools/azure-sdk-tools/ci_tools/functions.py b/tools/azure-sdk-tools/ci_tools/functions.py index 8ad3bb114900..a1412b43f083 100644 --- a/tools/azure-sdk-tools/ci_tools/functions.py +++ b/tools/azure-sdk-tools/ci_tools/functions.py @@ -42,6 +42,9 @@ ] TEST_COMPATIBILITY_MAP = {} +TEST_PYTHON_DISTRO_INCOMPATIBILITY_MAP = { + "azure-storage-blob": "pypy" +} omit_regression = ( lambda x: "nspkg" not in x @@ -104,7 +107,12 @@ def apply_compatibility_filter(package_set: List[str]) -> List[str]: if pkg_specs_override: spec_set = SpecifierSet(pkg_specs_override) - if running_major_version in spec_set: + distro_compat = True + distro_incompat = TEST_PYTHON_DISTRO_INCOMPATIBILITY_MAP.get(os.path.basename(pkg), None) + if distro_incompat and distro_incompat in platform.python_implementation().lower(): + distro_compat = False + + if running_major_version in spec_set and distro_compat: collected_packages.append(pkg) logging.debug("Target packages after applying compatibility filter: {}".format(collected_packages)) From 59d93ee403d20b64811aa0d925c4ea5301e932b9 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Wed, 11 Sep 2024 15:25:58 -0700 Subject: [PATCH 112/119] allow us to diff when a package is included vs not --- eng/common/scripts/Package-Properties.ps1 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/eng/common/scripts/Package-Properties.ps1 b/eng/common/scripts/Package-Properties.ps1 index 6a70070beb56..545d04890035 100644 --- a/eng/common/scripts/Package-Properties.ps1 +++ b/eng/common/scripts/Package-Properties.ps1 @@ -15,6 +15,7 @@ class PackageProps [boolean]$IsNewSdk [string]$ArtifactName [string]$ReleaseStatus + [boolean]$IncludedForValidation [string[]]$AdditionalValidationPackages PackageProps([string]$name, [string]$version, [string]$directoryPath, [string]$serviceDirectory) @@ -38,6 +39,7 @@ class PackageProps $this.Version = $version $this.DirectoryPath = $directoryPath $this.ServiceDirectory = $serviceDirectory + $this.IncludedForValidation = $false if (Test-Path (Join-Path $directoryPath "README.md")) { @@ -140,6 +142,8 @@ function Get-PrPkgProperties([string]$InputDiffJson) { $key = $addition.Replace($RepoRoot, "").SubString(1) if ($lookup[$key]) { + $lookup[$key].IncludedForValidation = $true + Write-Host $lookup[$key].IncludedForValidation $packagesWithChanges += $lookup[$key] } } From a69dadb9f983d42fd3cb053326008e1913d3a336 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Wed, 11 Sep 2024 17:43:35 -0700 Subject: [PATCH 113/119] commit my progress so I have a place to revert to, don't push this --- eng/pipelines/templates/jobs/ci.tests.yml | 1 - eng/pipelines/templates/jobs/ci.yml | 3 +- .../templates/stages/platform-matrix.json | 29 +----- eng/pipelines/templates/steps/build-test.yml | 8 +- eng/scripts/distribute-packages-to-matrix.ps1 | 88 ++++++++++++------- 5 files changed, 64 insertions(+), 65 deletions(-) diff --git a/eng/pipelines/templates/jobs/ci.tests.yml b/eng/pipelines/templates/jobs/ci.tests.yml index 0379f372425b..e89c7ab3e5a5 100644 --- a/eng/pipelines/templates/jobs/ci.tests.yml +++ b/eng/pipelines/templates/jobs/ci.tests.yml @@ -117,7 +117,6 @@ jobs: parameters: TestMarkArgument: ${{ parameters.TestMarkArgument }} AdditionalTestArgs: '--wheel_dir="$(Build.ArtifactStagingDirectory)"' - CoverageArg: $(CoverageArg) PythonVersion: $(PythonVersion) ToxTestEnv: $(toxenv) ToxEnvParallel: ${{ parameters.ToxEnvParallel }} diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index 0a1ee8db8bae..448471caae40 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -244,6 +244,7 @@ jobs: PreGenerationSteps: - template: /eng/pipelines/templates/steps/download-package-artifacts.yml + condition: eq(variables['Build.Reason'], 'PullRequest') - task: Powershell@2 inputs: @@ -251,7 +252,7 @@ jobs: filePath: eng/scripts/distribute-packages-to-matrix.ps1 arguments: >- -PackageInfoFolder "$(Build.ArtifactStagingDirectory)/PackageInfo" - -PlatformMatrix "$(Build.SourcesDirectory)/eng/pipelines/templates/stages/platform-matrix.json" + -PlatformMatrix "${{ parameters.MatrixConfigs[0].Path }}" displayName: 'Distribute Packages to Matrix' condition: eq(variables['Build.Reason'], 'PullRequest') diff --git a/eng/pipelines/templates/stages/platform-matrix.json b/eng/pipelines/templates/stages/platform-matrix.json index fcda14bcd8f8..1311e1ebfbab 100644 --- a/eng/pipelines/templates/stages/platform-matrix.json +++ b/eng/pipelines/templates/stages/platform-matrix.json @@ -10,32 +10,7 @@ "ubuntu-20.04": { "OSVmImage": "env:LINUXVMIMAGE", "Pool": "env:LINUXPOOL" }, "macos-latest": { "OSVmImage": "env:MACVMIMAGE", "Pool": "env:MACPOOL" } }, - "PythonVersion": [ "3.8", "pypy3.9", "3.11", "3.10" ], - "CoverageArg": "--disablecov", + "PythonVersion": [ "3.8", "pypy3.9", "3.9", "3.11", "3.12", "3.10" ], "TestSamples": "false" - }, - "include": [ - { - "CoverageConfig": { - "ubuntu2004_39_coverage": { - "OSVmImage": "env:LINUXVMIMAGE", - "Pool": "env:LINUXPOOL", - "PythonVersion": "3.9", - "CoverageArg": "", - "TestSamples": "false" - } - } - }, - { - "Config": { - "Ubuntu2004_312": { - "OSVmImage": "env:LINUXVMIMAGE", - "Pool": "env:LINUXPOOL", - "PythonVersion": "3.12", - "CoverageArg": "--disablecov", - "TestSamples": "false" - } - } - } - ] + } } diff --git a/eng/pipelines/templates/steps/build-test.yml b/eng/pipelines/templates/steps/build-test.yml index a3c472294067..b9f04a19dada 100644 --- a/eng/pipelines/templates/steps/build-test.yml +++ b/eng/pipelines/templates/steps/build-test.yml @@ -8,9 +8,7 @@ parameters: OSVmImage: '' BeforeTestSteps: [] AfterTestSteps: [] - CoverageArg: '' ToxTestEnv: "" - RunCoverage: ne(variables['CoverageArg'], '--disablecov') ToxEnvParallel: '' InjectedPackages: '' DevFeedName: 'public/azure-sdk-for-python' @@ -31,6 +29,12 @@ steps: - template: set-dev-build.yml + - pwsh: | + if ("$(PythonVersion)" -ne '3.9') { + Write-Host "##vso[task.setvariable variable=CoverageArg]--disableCov" + } + displayName: "Set Coverage Arg" + - pwsh: | if ($IsWindows) { . $(VENV_LOCATION)/Scripts/Activate.ps1 diff --git a/eng/scripts/distribute-packages-to-matrix.ps1 b/eng/scripts/distribute-packages-to-matrix.ps1 index 0d2baf743c73..30e00fd2a058 100644 --- a/eng/scripts/distribute-packages-to-matrix.ps1 +++ b/eng/scripts/distribute-packages-to-matrix.ps1 @@ -27,18 +27,22 @@ if (!(Test-Path $PlatformMatrix)) { exit 1 } -$packageProperties = Get-ChildItem -Recurse "$PackageInfoFolder" *.json +$packageProperties = Get-ChildItem -Recurse "$PackageInfoFolder" *.json | % { Get-Content -Path $_.FullName | ConvertFrom-Json } $matrix = Get-Content -Path $PlatformMatrix | ConvertFrom-Json -# determine batch size and initialize batches $versionCount = $matrix.matrix.PythonVersion.Count -$includeCount = $matrix.include.Count +$includeCount = 0 +if ($matrix.PSObject.Properties.Name -contains "include") { + $includeCount = $matrix.include.Count +} $batchCount = $versionCount + $includeCount if ($batchCount -eq 0) { Write-Error "No batches detected, skipping without updating platform matrix file $PlatformMatrix" - exit 0 + exit 1 } -$batchSize = $packageProperties.Count % $batchCount + +$batchSize = 5 + # batches is a hashtable because you cannot instantiate an array of empty arrays # the add method will merely invoke and silently NOT add a new item # using a hashtable bypasses this limitation @@ -47,39 +51,55 @@ for ($i = 0; $i -lt $batchCount; $i++) { $batches[$i] = @() } -# what situations can we have with the calculated batch size of 6? (number of platforms + include) -# 1. < 6 packages we simply exit. default distribution will be fine -# 2. >= 6 packages we rotate and assign 1 to each batch until all work is exhausted $batchCounter = 0 $batchValues = @() -# regular case is when we have more packages than batches -if ($packageProperties.Count -ge $batchCount) { - for ($i = 0; $i -lt $packageProperties.Count; $i++) { - Write-Host "Assigning package $($packageProperties[$i].Name) to batch $batchCounter" - $batches[$batchCounter] += $packageProperties[$i].Name.Replace(".json", "") - $batchCounter = ($batchCounter + 1) % $batchCount - } -} -# the case where we have fewer packages than batches, we instead walk the batches -# and assign packages to them, even if we end up assigning the same package twice -# that's ok, we just don't want to have dead platforms running no tests -else { - for ($i = 0; $i -lt $batchCount; $i++) { - $batches[$i] += $packageProperties[$i % $packageProperties.Count].Name.Replace(".json", "") - } -} +# if there is an IMPORT, we should import that first and use THAT. we have a clear edge case where a nested include will NOT work. +# I am accepting that in interests of making this work with all our platform matrix json files throughout the repo +# TODO: implement grabbing the matrix from the import if it exists -# simplify the batches down to single values that will be added to the matrix -foreach($key in $batches.Keys) { - $batchValues += $batches[$key] -join "," -} +$directIncludedPackages = $packageProperties | Where-Object { $_.IncludedForValidation -eq $false } +$indirectIncludedPackages = $packageProperties | Where-Object { $_.IncludedForValidation -eq $true } + +# for python, for fast tests, I want the sets of packages to be broken up into sets of five. + +# I will assign all the direct included packages first. our goal is to get complete coverage of the direct included packages +# then, for the indirect packages, we will ADD them as extra TargetingString bundles to the matrix. +# this means that these "extra" packages will never run on any platform that's not present in INCLUDE, but frankly +# that's nbd IMO + +# all directly included packages should have their tests invoked in full, so we just need to generate the batches, multiplex them times the number of packages, +# and evenly assign them to the batches + direct to "include" members + + + +# # regular case is when we have more packages than batches +# if ($packageProperties.Count -ge $batchCount) { +# for ($i = 0; $i -lt $packageProperties.Count; $i++) { +# Write-Host "Assigning package $($packageProperties[$i].Name) to batch $batchCounter" +# $batches[$batchCounter] += $packageProperties[$i].Name.Replace(".json", "") +# $batchCounter = ($batchCounter + 1) % $batchCount +# } +# } +# # the case where we have fewer packages than batches, we instead walk the batches +# # and assign packages to them, even if we end up assigning the same package twice +# # that's ok, we just don't want to have dead platforms running no tests +# else { +# for ($i = 0; $i -lt $batchCount; $i++) { +# $batches[$i] += $packageProperties[$i % $packageProperties.Count].Name.Replace(".json", "") +# } +# } + +# # simplify the batches down to single values that will be added to the matrix +# foreach($key in $batches.Keys) { +# $batchValues += $batches[$key] -join "," +# } -# set the values for include -$matrix.include[0].CoverageConfig.ubuntu2004_39_coverage | Add-Member -Force -MemberType NoteProperty -Name TargetingString -Value $batchValues[0] -$matrix.include[1].Config.Ubuntu2004_312 | Add-Member -Force -MemberType NoteProperty -Name TargetingString -Value $batchValues[1] +# # set the values for include +# $matrix.include[0].CoverageConfig.ubuntu2004_39_coverage | Add-Member -Force -MemberType NoteProperty -Name TargetingString -Value $batchValues[0] +# $matrix.include[1].Config.Ubuntu2004_312 | Add-Member -Force -MemberType NoteProperty -Name TargetingString -Value $batchValues[1] -# set the values for the matrix -$matrix.matrix | Add-Member -Force -MemberType NoteProperty -Name TargetingString -Value $batchValues[2..($batchValues.Length-1)] +# # set the values for the matrix +# $matrix.matrix | Add-Member -Force -MemberType NoteProperty -Name TargetingString -Value $batchValues[2..($batchValues.Length-1)] -$matrix | ConvertTo-Json -Depth 100 | Set-Content -Path $PlatformMatrix +# $matrix | ConvertTo-Json -Depth 100 | Set-Content -Path $PlatformMatrix From bf633efb5337708d913186833e50f1f1b97e5230 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Wed, 11 Sep 2024 19:16:22 -0700 Subject: [PATCH 114/119] commit platform matrix so we can easily rerun modification script --- eng/pipelines/templates/stages/platform-matrix.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eng/pipelines/templates/stages/platform-matrix.json b/eng/pipelines/templates/stages/platform-matrix.json index 1311e1ebfbab..596c51addd40 100644 --- a/eng/pipelines/templates/stages/platform-matrix.json +++ b/eng/pipelines/templates/stages/platform-matrix.json @@ -11,6 +11,7 @@ "macos-latest": { "OSVmImage": "env:MACVMIMAGE", "Pool": "env:MACPOOL" } }, "PythonVersion": [ "3.8", "pypy3.9", "3.9", "3.11", "3.12", "3.10" ], - "TestSamples": "false" + "TestSamples": "false", + "TargetingString": "azure*" } } From 65253ea80db984d6b54ab25cc9ccf09b855f84ef Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Wed, 11 Sep 2024 19:29:16 -0700 Subject: [PATCH 115/119] save progress for the evening --- eng/scripts/distribute-packages-to-matrix.ps1 | 72 ++++++++++++++++++- 1 file changed, 69 insertions(+), 3 deletions(-) diff --git a/eng/scripts/distribute-packages-to-matrix.ps1 b/eng/scripts/distribute-packages-to-matrix.ps1 index 30e00fd2a058..98909463dec9 100644 --- a/eng/scripts/distribute-packages-to-matrix.ps1 +++ b/eng/scripts/distribute-packages-to-matrix.ps1 @@ -27,11 +27,36 @@ if (!(Test-Path $PlatformMatrix)) { exit 1 } + +function Split-ArrayIntoBatches { + param ( + [Parameter(Mandatory=$true)] + [object[]]$InputArray, + + [int]$BatchSize = 5 + ) + + # Initialize an empty array to hold the batches + $batches = @() + + # Loop through the input array in increments of the batch size + for ($i = 0; $i -lt $InputArray.Count; $i += $BatchSize) { + # Create a batch containing up to $BatchSize elements + $batch = $InputArray[$i..[math]::Min($i + $BatchSize - 1, $InputArray.Count - 1)] + + # Add the batch to the list of batches + $batches += ,$batch + } + + return ,$batches +} + $packageProperties = Get-ChildItem -Recurse "$PackageInfoFolder" *.json | % { Get-Content -Path $_.FullName | ConvertFrom-Json } $matrix = Get-Content -Path $PlatformMatrix | ConvertFrom-Json $versionCount = $matrix.matrix.PythonVersion.Count $includeCount = 0 +$includeObject = $null if ($matrix.PSObject.Properties.Name -contains "include") { $includeCount = $matrix.include.Count } @@ -61,6 +86,9 @@ $batchValues = @() $directIncludedPackages = $packageProperties | Where-Object { $_.IncludedForValidation -eq $false } $indirectIncludedPackages = $packageProperties | Where-Object { $_.IncludedForValidation -eq $true } +Write-Host "Direct Included Length = $($directIncludedPackages.Length)" +Write-Host "IndirectIncludedPackages Included Length = $($indirectIncludedPackages.Length)" + # for python, for fast tests, I want the sets of packages to be broken up into sets of five. # I will assign all the direct included packages first. our goal is to get complete coverage of the direct included packages @@ -68,10 +96,48 @@ $indirectIncludedPackages = $packageProperties | Where-Object { $_.IncludedForVa # this means that these "extra" packages will never run on any platform that's not present in INCLUDE, but frankly # that's nbd IMO -# all directly included packages should have their tests invoked in full, so we just need to generate the batches, multiplex them times the number of packages, -# and evenly assign them to the batches + direct to "include" members +$directBatches = Split-ArrayIntoBatches -InputArray $directIncludedPackages -BatchSize $batchSize +Write-Host $directBatches.Length +Write-Host $directBatches[0].Length + +# all directly included packages should have their tests invoked in full, so, lets use a basic storage service discovery. We'll have direct package batches of +# ["azure-storage-blob", "azure-storage-blob-changefeed", "azure-storage-extensions", "azure-storage-file-datalake", "azure-storage-file-share"] +# ["azure-storage-queue"] + +# any other packages will be added to the matrix as additional targetingstring batches. it'll be sparse in that there won't be full matrix coverage + +# a basic platform matrix has the following: +# "matrix" -> "PythonVersion" = ["3.6", "3.7", "3.8", "3.9", "3.10"] +# "include" -> "ConfigName" -> "ActualJobName" -> PythonVersion=3.9 +# so what we need to do is for each batch + # 1. assign the batch * matrix size to the matrix as "TargetingString" + # 2. assign the batch to each include with the batch being the value for TargetingString for the include + # the easiest way to do this is to take a copy of the `Include` object at the beginning of this script, + # for each batch, assign the batch to the include object, and then add the include object to the matrix object + # this will have the result of multiplexing our includes, so we will need to update the name there + +if ($directBatches) { + $matrix.matrix.TargetingString = @() +} + + +foreach($batch in $directBatches) { + $targetingString = ($batch | Select-Object -ExpandProperty Name) -join "," + + $targetingStringArray = @($targetingString) * $versionCount + Write-Host "Returning batch:" + foreach($item in $targetingStringArray) { + Write-Host "`t$item" + } + + $matrix.matrix.TargetingString += $targetingStringArray + + # now we need to add the batch to the includes if they exist + if () +} +$matrix | ConvertTo-Json -Depth 100 # # regular case is when we have more packages than batches # if ($packageProperties.Count -ge $batchCount) { @@ -102,4 +168,4 @@ $indirectIncludedPackages = $packageProperties | Where-Object { $_.IncludedForVa # # set the values for the matrix # $matrix.matrix | Add-Member -Force -MemberType NoteProperty -Name TargetingString -Value $batchValues[2..($batchValues.Length-1)] -# $matrix | ConvertTo-Json -Depth 100 | Set-Content -Path $PlatformMatrix +$matrix | ConvertTo-Json -Depth 100 | Set-Content -Path $PlatformMatrix From cd5bd085a8fb48c103d12ff8bc05885ee2de3c46 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Thu, 12 Sep 2024 11:42:58 -0700 Subject: [PATCH 116/119] ready to handle include duplication --- eng/scripts/distribute-packages-to-matrix.ps1 | 26 ++++++++++++++----- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/eng/scripts/distribute-packages-to-matrix.ps1 b/eng/scripts/distribute-packages-to-matrix.ps1 index 98909463dec9..8c292b11e334 100644 --- a/eng/scripts/distribute-packages-to-matrix.ps1 +++ b/eng/scripts/distribute-packages-to-matrix.ps1 @@ -59,6 +59,8 @@ $includeCount = 0 $includeObject = $null if ($matrix.PSObject.Properties.Name -contains "include") { $includeCount = $matrix.include.Count + $originalInclude = $matrix.include + Write-Host "Original include type " + $originalInclude.GetType() } $batchCount = $versionCount + $includeCount if ($batchCount -eq 0) { @@ -75,14 +77,12 @@ $batches = @{} for ($i = 0; $i -lt $batchCount; $i++) { $batches[$i] = @() } - $batchCounter = 0 $batchValues = @() # if there is an IMPORT, we should import that first and use THAT. we have a clear edge case where a nested include will NOT work. # I am accepting that in interests of making this work with all our platform matrix json files throughout the repo # TODO: implement grabbing the matrix from the import if it exists - $directIncludedPackages = $packageProperties | Where-Object { $_.IncludedForValidation -eq $false } $indirectIncludedPackages = $packageProperties | Where-Object { $_.IncludedForValidation -eq $true } @@ -99,7 +99,6 @@ Write-Host "IndirectIncludedPackages Included Length = $($indirectIncludedPackag $directBatches = Split-ArrayIntoBatches -InputArray $directIncludedPackages -BatchSize $batchSize Write-Host $directBatches.Length Write-Host $directBatches[0].Length - # all directly included packages should have their tests invoked in full, so, lets use a basic storage service discovery. We'll have direct package batches of # ["azure-storage-blob", "azure-storage-blob-changefeed", "azure-storage-extensions", "azure-storage-file-datalake", "azure-storage-file-share"] @@ -118,13 +117,21 @@ Write-Host $directBatches[0].Length # this will have the result of multiplexing our includes, so we will need to update the name there if ($directBatches) { - $matrix.matrix.TargetingString = @() + # we need to ensure the presence of TargetingString in the matrix object + if (-not $matrix.matrix.PSObject.Properties["TargetingString"]) { + $matrix.matrix | Add-Member -Force -MemberType NoteProperty -Name TargetingString -Value @() + } + else { + $matrix.matrix.TargetingString = @() + } } - foreach($batch in $directBatches) { $targetingString = ($batch | Select-Object -ExpandProperty Name) -join "," + # we need to equal the number of python versions in the matrix (to ensure we get complete coverage) + # so we need to multiply the targeting string by the number of python versions, that'll ensure that + # across all python versions we see the same packages $targetingStringArray = @($targetingString) * $versionCount Write-Host "Returning batch:" foreach($item in $targetingStringArray) { @@ -133,8 +140,13 @@ foreach($batch in $directBatches) { $matrix.matrix.TargetingString += $targetingStringArray - # now we need to add the batch to the includes if they exist - if () + # if there were any include objects, we need to duplicate them exactly and add the targeting string to each + # this means that the number of includes at the end of this operation will be incoming # of includes * the number of batches + # if ($includeCount -gt 0) { + # foreach($include in $matrix.include) { + # $include | Add-Member -Force -MemberType NoteProperty -Name TargetingString -Value $targetingString + # } + # } } $matrix | ConvertTo-Json -Depth 100 From 71f28e1219abc07b1b38eaf2d5f009566c98d2f3 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Thu, 12 Sep 2024 13:06:02 -0700 Subject: [PATCH 117/119] right before adding include update --- eng/scripts/distribute-packages-to-matrix.ps1 | 56 +++++++++++++++++-- 1 file changed, 51 insertions(+), 5 deletions(-) diff --git a/eng/scripts/distribute-packages-to-matrix.ps1 b/eng/scripts/distribute-packages-to-matrix.ps1 index 8c292b11e334..1d82398dc659 100644 --- a/eng/scripts/distribute-packages-to-matrix.ps1 +++ b/eng/scripts/distribute-packages-to-matrix.ps1 @@ -60,6 +60,8 @@ $includeObject = $null if ($matrix.PSObject.Properties.Name -contains "include") { $includeCount = $matrix.include.Count $originalInclude = $matrix.include + # todo: reenable this + $matrix.include = [] Write-Host "Original include type " + $originalInclude.GetType() } $batchCount = $versionCount + $includeCount @@ -142,11 +144,55 @@ foreach($batch in $directBatches) { # if there were any include objects, we need to duplicate them exactly and add the targeting string to each # this means that the number of includes at the end of this operation will be incoming # of includes * the number of batches - # if ($includeCount -gt 0) { - # foreach($include in $matrix.include) { - # $include | Add-Member -Force -MemberType NoteProperty -Name TargetingString -Value $targetingString - # } - # } + if ($includeCount -gt 0) { + Write-Host "Walking include objects for $targetingString" + $includeCopy = $originalInclude | ConvertTo-Json -Depth 100 | ConvertFrom-Json + foreach ($configElement in $includeCopy) { + if ($configElement.PSObject.Properties) { + # an include looks like this + # access by .[ConfigElement] + # { + # "CoverageConfig": { + # "ubuntu2004_39_coverage": { + # "OSVmImage": "env:LINUXVMIMAGE", + # "Pool": "env:LINUXPOOL", + # "PythonVersion": "3.9", + # "CoverageArg": "", + # "TestSamples": "false" + # } + # } + # } + $topLevelPropertyName = $configElement.PSObject.Properties.Name + $topLevelPropertyValue = $configElement.PSObject.Properties[$topLevelPropertyName].Value + + if ($topLevelPropertyValue.PSObject.Properties) { + + $secondLevelPropertyName = $topLevelPropertyValue.PSObject.Properties.Name + $secondLevelPropertyValue = $topLevelPropertyValue.PSObject.Properties[$secondLevelPropertyName].Value + + Write-Host "Top-level: $topLevelPropertyName" + Write-Host "Second-level: $secondLevelPropertyName" + + # Example of updating the second-level property name (optional) + # This will rename the property `ubuntu2004_39_coverage` to something else + $newSecondLevelName = "$secondLevelPropertyName_$targetingString" + + $topLevelPropertyValue.PSObject.Properties.Remove($secondLevelPropertyName) + $topLevelPropertyValue | Add-Member -MemberType NoteProperty -Name $newSecondLevelName -Value $secondLevelPropertyValue + + # add the targeting string property if it doesn't already exist + if (-not $topLevelPropertyValue.$newSecondLevelName.PSObject.Properties["TargetingString"]) { + $topLevelPropertyValue.$newSecondLevelName | Add-Member -Force -MemberType NoteProperty -Name TargetingString -Value @() + } + + # set the targeting string + $topLevelPropertyValue.$newSecondLevelName.TargetingString = $targetingString + } + } + + $matrix.include += $configElement + } + } } $matrix | ConvertTo-Json -Depth 100 From 44f06741835355c7651b6d954c801ef02ba2899c Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Thu, 12 Sep 2024 13:07:04 -0700 Subject: [PATCH 118/119] it works! we're now autogenerating new include patterns! --- eng/scripts/distribute-packages-to-matrix.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/scripts/distribute-packages-to-matrix.ps1 b/eng/scripts/distribute-packages-to-matrix.ps1 index 1d82398dc659..b11f43c516ac 100644 --- a/eng/scripts/distribute-packages-to-matrix.ps1 +++ b/eng/scripts/distribute-packages-to-matrix.ps1 @@ -61,7 +61,7 @@ if ($matrix.PSObject.Properties.Name -contains "include") { $includeCount = $matrix.include.Count $originalInclude = $matrix.include # todo: reenable this - $matrix.include = [] + $matrix.include = @() Write-Host "Original include type " + $originalInclude.GetType() } $batchCount = $versionCount + $includeCount @@ -175,7 +175,7 @@ foreach($batch in $directBatches) { # Example of updating the second-level property name (optional) # This will rename the property `ubuntu2004_39_coverage` to something else - $newSecondLevelName = "$secondLevelPropertyName_$targetingString" + $newSecondLevelName = "$secondLevelPropertyName" + "$targetingString" $topLevelPropertyValue.PSObject.Properties.Remove($secondLevelPropertyName) $topLevelPropertyValue | Add-Member -MemberType NoteProperty -Name $newSecondLevelName -Value $secondLevelPropertyValue From 76c50102cd56b5306b9ab7e5a16d523fa28d10b1 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Thu, 12 Sep 2024 13:52:57 -0700 Subject: [PATCH 119/119] this will now work for various platform matrix jsons. going to restore stages/platform-matrix.json back to what it was before --- eng/scripts/distribute-packages-to-matrix.ps1 | 52 +++---------------- 1 file changed, 8 insertions(+), 44 deletions(-) diff --git a/eng/scripts/distribute-packages-to-matrix.ps1 b/eng/scripts/distribute-packages-to-matrix.ps1 index b11f43c516ac..b6f8d9bdb3b6 100644 --- a/eng/scripts/distribute-packages-to-matrix.ps1 +++ b/eng/scripts/distribute-packages-to-matrix.ps1 @@ -150,15 +150,16 @@ foreach($batch in $directBatches) { foreach ($configElement in $includeCopy) { if ($configElement.PSObject.Properties) { # an include looks like this - # access by .[ConfigElement] # { + # # "CoverageConfig": { + # # "ubuntu2004_39_coverage": { - # "OSVmImage": "env:LINUXVMIMAGE", - # "Pool": "env:LINUXPOOL", - # "PythonVersion": "3.9", - # "CoverageArg": "", - # "TestSamples": "false" + # "OSVmImage": "env:LINUXVMIMAGE", + # "Pool": "env:LINUXPOOL", + # "PythonVersion": "3.9", + # "CoverageArg": "", + # "TestSamples": "false" # } # } # } @@ -166,15 +167,9 @@ foreach($batch in $directBatches) { $topLevelPropertyValue = $configElement.PSObject.Properties[$topLevelPropertyName].Value if ($topLevelPropertyValue.PSObject.Properties) { - $secondLevelPropertyName = $topLevelPropertyValue.PSObject.Properties.Name $secondLevelPropertyValue = $topLevelPropertyValue.PSObject.Properties[$secondLevelPropertyName].Value - Write-Host "Top-level: $topLevelPropertyName" - Write-Host "Second-level: $secondLevelPropertyName" - - # Example of updating the second-level property name (optional) - # This will rename the property `ubuntu2004_39_coverage` to something else $newSecondLevelName = "$secondLevelPropertyName" + "$targetingString" $topLevelPropertyValue.PSObject.Properties.Remove($secondLevelPropertyName) @@ -182,7 +177,7 @@ foreach($batch in $directBatches) { # add the targeting string property if it doesn't already exist if (-not $topLevelPropertyValue.$newSecondLevelName.PSObject.Properties["TargetingString"]) { - $topLevelPropertyValue.$newSecondLevelName | Add-Member -Force -MemberType NoteProperty -Name TargetingString -Value @() + $topLevelPropertyValue.$newSecondLevelName | Add-Member -Force -MemberType NoteProperty -Name TargetingString -Value "" } # set the targeting string @@ -195,35 +190,4 @@ foreach($batch in $directBatches) { } } -$matrix | ConvertTo-Json -Depth 100 - -# # regular case is when we have more packages than batches -# if ($packageProperties.Count -ge $batchCount) { -# for ($i = 0; $i -lt $packageProperties.Count; $i++) { -# Write-Host "Assigning package $($packageProperties[$i].Name) to batch $batchCounter" -# $batches[$batchCounter] += $packageProperties[$i].Name.Replace(".json", "") -# $batchCounter = ($batchCounter + 1) % $batchCount -# } -# } -# # the case where we have fewer packages than batches, we instead walk the batches -# # and assign packages to them, even if we end up assigning the same package twice -# # that's ok, we just don't want to have dead platforms running no tests -# else { -# for ($i = 0; $i -lt $batchCount; $i++) { -# $batches[$i] += $packageProperties[$i % $packageProperties.Count].Name.Replace(".json", "") -# } -# } - -# # simplify the batches down to single values that will be added to the matrix -# foreach($key in $batches.Keys) { -# $batchValues += $batches[$key] -join "," -# } - -# # set the values for include -# $matrix.include[0].CoverageConfig.ubuntu2004_39_coverage | Add-Member -Force -MemberType NoteProperty -Name TargetingString -Value $batchValues[0] -# $matrix.include[1].Config.Ubuntu2004_312 | Add-Member -Force -MemberType NoteProperty -Name TargetingString -Value $batchValues[1] - -# # set the values for the matrix -# $matrix.matrix | Add-Member -Force -MemberType NoteProperty -Name TargetingString -Value $batchValues[2..($batchValues.Length-1)] - $matrix | ConvertTo-Json -Depth 100 | Set-Content -Path $PlatformMatrix