diff --git a/eng/common/scripts/Package-Properties.ps1 b/eng/common/scripts/Package-Properties.ps1 index e3cea7d392eb..d8fd8d10aac9 100644 --- a/eng/common/scripts/Package-Properties.ps1 +++ b/eng/common/scripts/Package-Properties.ps1 @@ -15,6 +15,7 @@ class PackageProps [boolean]$IsNewSdk [string]$ArtifactName [string]$ReleaseStatus + [boolean]$IncludedForValidation [string[]]$AdditionalValidationPackages PackageProps([string]$name, [string]$version, [string]$directoryPath, [string]$serviceDirectory) @@ -38,6 +39,7 @@ class PackageProps $this.Version = $version $this.DirectoryPath = $directoryPath $this.ServiceDirectory = $serviceDirectory + $this.IncludedForValidation = $false if (Test-Path (Join-Path $directoryPath "README.md")) { @@ -143,6 +145,8 @@ function Get-PrPkgProperties([string]$InputDiffJson) { $key = $addition.Replace($RepoRoot, "").TrimStart('\/') if ($lookup[$key]) { + $lookup[$key].IncludedForValidation = $true + Write-Host $lookup[$key].IncludedForValidation $packagesWithChanges += $lookup[$key] } } diff --git a/eng/pipelines/templates/jobs/ci.tests.yml b/eng/pipelines/templates/jobs/ci.tests.yml index 7219acb71eb3..e89c7ab3e5a5 100644 --- a/eng/pipelines/templates/jobs/ci.tests.yml +++ b/eng/pipelines/templates/jobs/ci.tests.yml @@ -79,9 +79,12 @@ jobs: value: ${{ parameters.InjectedPackages }} steps: - - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml + - template: /eng/pipelines/templates/steps/download-package-artifacts.yml + + - template: /eng/pipelines/templates/steps/resolve-package-targeting.yml parameters: BuildTargetingString: ${{ parameters.BuildTargetingString }} + PackagePropertiesFolder: $(Build.ArtifactStagingDirectory)/PackageInfo - template: /eng/common/pipelines/templates/steps/verify-agent-os.yml parameters: @@ -112,22 +115,14 @@ jobs: - template: ../steps/build-test.yml parameters: - ServiceDirectory: ${{ parameters.ServiceDirectory }} TestMarkArgument: ${{ parameters.TestMarkArgument }} AdditionalTestArgs: '--wheel_dir="$(Build.ArtifactStagingDirectory)"' - CoverageArg: $(CoverageArg) PythonVersion: $(PythonVersion) ToxTestEnv: $(toxenv) ToxEnvParallel: ${{ parameters.ToxEnvParallel }} InjectedPackages: $(InjectedPackages) TestProxy: ${{ parameters.TestProxy }} BeforeTestSteps: - - template: /eng/pipelines/templates/steps/download-package-artifacts.yml - - - template: ../steps/set-dev-build.yml - parameters: - ServiceDirectory: ${{ parameters.ServiceDirectory }} - - ${{ each step in parameters.BeforeTestSteps }}: - ${{ step }} AfterTestSteps: ${{ parameters.AfterTestSteps }} diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index 66d02fe9396c..016b04618c58 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -75,14 +75,14 @@ jobs: ComponentDetection.SourcePath: '$(Build.SourcesDirectory)/sdk/${{ parameters.ServiceDirectory }}' steps: - - template: /eng/pipelines/templates/steps/build.yml + - template: /eng/pipelines/templates/steps/build-package-artifacts.yml parameters: ServiceDirectory: ${{ parameters.ServiceDirectory }} - BuildTargetingString: ${{ parameters.BuildTargetingString }} BeforePublishSteps: ${{ parameters.BeforePublishSteps }} TestPipeline: ${{ parameters.TestPipeline }} - ArtifactSuffix: linux Artifacts: ${{ parameters.Artifacts }} + ArtifactSuffix: linux + BuildTargetingString: ${{ parameters.BuildTargetingString }} - job: 'Build_Windows' timeoutInMinutes: 90 @@ -96,14 +96,14 @@ jobs: ComponentDetection.SourcePath: '$(Build.SourcesDirectory)/sdk/${{ parameters.ServiceDirectory }}' steps: - - template: /eng/pipelines/templates/steps/build.yml + - template: /eng/pipelines/templates/steps/build-package-artifacts.yml parameters: ServiceDirectory: ${{ parameters.ServiceDirectory }} - BuildTargetingString: ${{ parameters.BuildTargetingString }} BeforePublishSteps: ${{ parameters.BeforePublishSteps }} TestPipeline: ${{ parameters.TestPipeline }} - ArtifactSuffix: windows Artifacts: ${{ parameters.Artifacts }} + ArtifactSuffix: windows + BuildTargetingString: ${{ parameters.BuildTargetingString }} - job: 'Build_MacOS' timeoutInMinutes: 90 @@ -117,14 +117,14 @@ jobs: ComponentDetection.SourcePath: '$(Build.SourcesDirectory)/sdk/${{ parameters.ServiceDirectory }}' steps: - - template: /eng/pipelines/templates/steps/build.yml + - template: /eng/pipelines/templates/steps/build-package-artifacts.yml parameters: ServiceDirectory: ${{ parameters.ServiceDirectory }} - BuildTargetingString: ${{ parameters.BuildTargetingString }} BeforePublishSteps: ${{ parameters.BeforePublishSteps }} TestPipeline: ${{ parameters.TestPipeline }} - ArtifactSuffix: mac Artifacts: ${{ parameters.Artifacts }} + ArtifactSuffix: mac + BuildTargetingString: ${{ parameters.BuildTargetingString }} - job: 'Build_Extended' displayName: Build Extended @@ -146,9 +146,41 @@ jobs: steps: - template: /eng/pipelines/templates/steps/download-package-artifacts.yml - - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml + - template: /eng/pipelines/templates/steps/resolve-package-targeting.yml parameters: BuildTargetingString: ${{ parameters.BuildTargetingString }} + PackagePropertiesFolder: $(Build.ArtifactStagingDirectory)/PackageInfo + + - template: ../steps/build-extended-artifacts.yml + parameters: + ServiceDirectory: ${{ parameters.ServiceDirectory }} + BeforePublishSteps: ${{ parameters.BeforePublishSteps }} + BuildDocs: false + RunApiStubGen: true + TestPipeline: ${{ parameters.TestPipeline }} + Artifacts: ${{ parameters.Artifacts }} + + - job: 'Build_Documentation' + displayName: Build Docs + dependsOn: + - 'Build_Linux' + - 'Build_Windows' + - 'Build_MacOS' + + timeoutInMinutes: 90 + + pool: + name: $(LINUXPOOL) + image: $(LINUXVMIMAGE) + os: linux + + steps: + - template: /eng/pipelines/templates/steps/download-package-artifacts.yml + + - template: /eng/pipelines/templates/steps/resolve-package-targeting.yml + parameters: + BuildTargetingString: ${{ parameters.BuildTargetingString }} + PackagePropertiesFolder: $(Build.ArtifactStagingDirectory)/PackageInfo - template: ../steps/build-extended-artifacts.yml parameters: @@ -157,6 +189,7 @@ jobs: BuildDocs: ${{ parameters.BuildDocs }} TestPipeline: ${{ parameters.TestPipeline }} Artifacts: ${{ parameters.Artifacts }} + RunApiStubGen: false - job: 'Analyze' condition: and(succeeded(), ne(variables['Skip.Analyze'], 'true')) @@ -172,10 +205,6 @@ jobs: os: linux steps: - - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml - parameters: - BuildTargetingString: ${{ parameters.BuildTargetingString }} - - template: /eng/common/pipelines/templates/steps/check-spelling.yml parameters: ContinueOnError: false @@ -196,6 +225,11 @@ jobs: artifactName: 'packages_extended' targetPath: $(Build.ArtifactStagingDirectory) + - template: /eng/pipelines/templates/steps/resolve-package-targeting.yml + parameters: + BuildTargetingString: ${{ parameters.BuildTargetingString }} + PackagePropertiesFolder: $(Build.ArtifactStagingDirectory)/PackageInfo + - template: ../steps/analyze.yml parameters: ServiceDirectory: ${{ parameters.ServiceDirectory }} @@ -219,6 +253,21 @@ jobs: MatrixConfigs: ${{ parameters.MatrixConfigs }} MatrixFilters: ${{ parameters.MatrixFilters }} MatrixReplace: ${{ parameters.MatrixReplace }} + PreGenerationSteps: + + - template: /eng/pipelines/templates/steps/download-package-artifacts.yml + condition: eq(variables['Build.Reason'], 'PullRequest') + + - task: Powershell@2 + inputs: + pwsh: true + filePath: eng/scripts/distribute-packages-to-matrix.ps1 + arguments: >- + -PackageInfoFolder "$(Build.ArtifactStagingDirectory)/PackageInfo" + -PlatformMatrix "${{ parameters.MatrixConfigs[0].Path }}" + displayName: 'Distribute Packages to Matrix' + condition: eq(variables['Build.Reason'], 'PullRequest') + CloudConfig: Cloud: Public AdditionalParameters: @@ -253,15 +302,12 @@ jobs: - pwsh: | python -m pip install "./tools/azure-sdk-tools[build]" displayName: 'Prep Environment' - - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml - parameters: - BuildTargetingString: ${{ parameters.BuildTargetingString }} - task: PythonScript@0 displayName: 'Ensure service coverage' inputs: scriptPath: '$(Build.SourcesDirectory)/scripts/devops_tasks/update_regression_services.py' arguments: >- - "$(TargetingString)" + "azure*" --service="${{ parameters.ServiceDirectory }}" --json=$(Build.SourcesDirectory)/eng/pipelines/templates/stages/regression-job-matrix.json CloudConfig: diff --git a/eng/pipelines/templates/jobs/live.tests.yml b/eng/pipelines/templates/jobs/live.tests.yml index 53ec04999e62..eb6a2088bb63 100644 --- a/eng/pipelines/templates/jobs/live.tests.yml +++ b/eng/pipelines/templates/jobs/live.tests.yml @@ -110,7 +110,7 @@ jobs: parameters: AgentImage: ${{ parameters.OSName }} - - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml + - template: /eng/pipelines/templates/steps/resolve-package-targeting.yml parameters: BuildTargetingString: ${{ parameters.BuildTargetingString }} diff --git a/eng/pipelines/templates/jobs/regression.yml b/eng/pipelines/templates/jobs/regression.yml index 8c3b559b48c7..4fa0629ed1be 100644 --- a/eng/pipelines/templates/jobs/regression.yml +++ b/eng/pipelines/templates/jobs/regression.yml @@ -71,7 +71,7 @@ jobs: # Please use `$(TargetingString)` to refer to the python packages glob string. This was previously `${{ parameters.BuildTargetingString }}`. steps: - - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml + - template: /eng/pipelines/templates/steps/resolve-package-targeting.yml parameters: BuildTargetingString: ${{ parameters.BuildTargetingString }} diff --git a/eng/pipelines/templates/stages/platform-matrix.json b/eng/pipelines/templates/stages/platform-matrix.json index fcda14bcd8f8..596c51addd40 100644 --- a/eng/pipelines/templates/stages/platform-matrix.json +++ b/eng/pipelines/templates/stages/platform-matrix.json @@ -10,32 +10,8 @@ "ubuntu-20.04": { "OSVmImage": "env:LINUXVMIMAGE", "Pool": "env:LINUXPOOL" }, "macos-latest": { "OSVmImage": "env:MACVMIMAGE", "Pool": "env:MACPOOL" } }, - "PythonVersion": [ "3.8", "pypy3.9", "3.11", "3.10" ], - "CoverageArg": "--disablecov", - "TestSamples": "false" - }, - "include": [ - { - "CoverageConfig": { - "ubuntu2004_39_coverage": { - "OSVmImage": "env:LINUXVMIMAGE", - "Pool": "env:LINUXPOOL", - "PythonVersion": "3.9", - "CoverageArg": "", - "TestSamples": "false" - } - } - }, - { - "Config": { - "Ubuntu2004_312": { - "OSVmImage": "env:LINUXVMIMAGE", - "Pool": "env:LINUXPOOL", - "PythonVersion": "3.12", - "CoverageArg": "--disablecov", - "TestSamples": "false" - } - } - } - ] + "PythonVersion": [ "3.8", "pypy3.9", "3.9", "3.11", "3.12", "3.10" ], + "TestSamples": "false", + "TargetingString": "azure*" + } } diff --git a/eng/pipelines/templates/steps/analyze.yml b/eng/pipelines/templates/steps/analyze.yml index f1ff969ec87b..38a61b2dac90 100644 --- a/eng/pipelines/templates/steps/analyze.yml +++ b/eng/pipelines/templates/steps/analyze.yml @@ -8,12 +8,7 @@ parameters: ValidateFormatting: false GenerateApiReviewForManualOnly: false -# The variable TargetingString is set by template `eng/pipelines/templates/steps/targeting-string-resolve.yml`. This template is invoked from yml files: -# eng/pipelines/templates/jobs/ci.tests.yml -# eng/pipelines/templates/jobs/ci.yml -# eng/pipelines/templates/jobs/live.test.yml - -# Please use `$(TargetingString)` to refer to the python packages glob string. This was previously `${{ parameters.BuildTargetingString }}`. +# Please use `$(TargetingString)` to refer to the python packages glob string. This variable is set from resolve-package-targeting.yml. steps: - template: /eng/pipelines/templates/steps/analyze_dependency.yml parameters: @@ -32,14 +27,20 @@ steps: scriptPath: 'scripts/devops_tasks/set_tox_environment.py' arguments: '"$(TargetingString)" --team-project="$(System.TeamProject)" --service="${{ parameters.ServiceDirectory }}"' - - ${{ each artifact in parameters.Artifacts }}: - - ${{if ne(artifact.skipVerifyChangeLog, 'true')}}: - - template: /eng/common/pipelines/templates/steps/verify-changelog.yml - parameters: - PackageName: ${{artifact.name}} - ServiceName: ${{parameters.ServiceDirectory}} - ForRelease: false - Condition: succeededOrFailed() + - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: + - template: /eng/common/pipelines/templates/steps/verify-changelogs.yml + parameters: + PackagePropertiesFolder: $(Build.ArtifactStagingDirectory)/PackageInfo + Condition: succeededOrFailed() + - ${{ else }}: + - ${{ each artifact in parameters.Artifacts }}: + - ${{if ne(artifact.skipVerifyChangeLog, 'true')}}: + - template: /eng/common/pipelines/templates/steps/verify-changelog.yml + parameters: + PackageName: ${{artifact.name}} + ServiceName: ${{parameters.ServiceDirectory}} + ForRelease: false + Condition: succeededOrFailed() - script: | python -m pip install "./tools/azure-sdk-tools[build]" -q -I diff --git a/eng/pipelines/templates/steps/analyze_dependency.yml b/eng/pipelines/templates/steps/analyze_dependency.yml index f52d7f70c31b..37da2dce5aef 100644 --- a/eng/pipelines/templates/steps/analyze_dependency.yml +++ b/eng/pipelines/templates/steps/analyze_dependency.yml @@ -13,9 +13,9 @@ steps: displayName: 'Install Python Tools' condition: succeededOrFailed() - - template: /eng/common/pipelines/templates/steps/verify-readme.yml + - template: /eng/common/pipelines/templates/steps/verify-readmes.yml parameters: - ScanPath: ${{ parameters.ScanPath }} + PackagePropertiesFolder: $(Build.ArtifactStagingDirectory)/PackageInfo Condition: succeededOrFailed() - pwsh: | diff --git a/eng/pipelines/templates/steps/build-extended-artifacts.yml b/eng/pipelines/templates/steps/build-extended-artifacts.yml index 8eb567c80042..6cd8fd768ecf 100644 --- a/eng/pipelines/templates/steps/build-extended-artifacts.yml +++ b/eng/pipelines/templates/steps/build-extended-artifacts.yml @@ -8,21 +8,20 @@ parameters: - name: ServiceDirectory type: string default: '' - - name: BuildDocs - type: boolean - default: true - name: Artifacts type: object default: [] - name: DevFeedName type: string default: 'public/azure-sdk-for-python' -# The variable TargetingString is set by template `eng/pipelines/templates/steps/targeting-string-resolve.yml`. This template is invoked from yml files: -# eng/pipelines/templates/jobs/ci.tests.yml -# eng/pipelines/templates/jobs/ci.yml -# eng/pipelines/templates/jobs/live.test.yml + - name: BuildDocs + type: boolean + default: true + - name: RunApiStubGen + type: boolean + default: true -# Please use `$(TargetingString)` to refer to the python packages glob string. This was previously `${{ parameters.BuildTargetingString }}`. +# Please use `$(TargetingString)` to refer to the python packages glob string. This variable is set from resolve-package-targeting.yml. steps: - task: UsePythonVersion@0 displayName: 'Use Python 3.11' @@ -54,40 +53,30 @@ steps: displayName: 'Prep Environment' - template: set-dev-build.yml - parameters: - ServiceDirectory: ${{ parameters.ServiceDirectory }} - - - task: PythonScript@0 - displayName: 'Generate Docs' - condition: and(succeededOrFailed(), ${{parameters.BuildDocs}}) - inputs: - scriptPath: 'scripts/devops_tasks/dispatch_tox.py' - arguments: >- - "$(TargetingString)" - --service="${{ parameters.ServiceDirectory }}" - --toxenv=sphinx - --wheel_dir="$(Build.ArtifactStagingDirectory)" - - pwsh: | - Write-Host "##vso[task.setvariable variable=PIP_INDEX_URL]https://pypi.python.org/simple" - displayName: Reset PIP Index For APIStubGen + - ${{ if eq(parameters.BuildDocs, 'true') }}: + - template: /eng/pipelines/templates/steps/run_sphinx.yml + parameters: + ServiceDirectory: "${{ parameters.ServiceDirectory }}" + WheelDirectory: "$(Build.ArtifactStagingDirectory)" - - template: /eng/pipelines/templates/steps/run_apistub.yml - parameters: - ServiceDirectory: ${{ parameters.ServiceDirectory }} - AdditionalTestArgs: '--wheel_dir="$(Build.ArtifactStagingDirectory)"' + - ${{ if eq(parameters.RunApiStubGen, 'true') }}: + - template: /eng/pipelines/templates/steps/run_apistub.yml + parameters: + ServiceDirectory: ${{ parameters.ServiceDirectory }} + AdditionalTestArgs: '--wheel_dir="$(Build.ArtifactStagingDirectory)"' - ${{ parameters.BeforePublishSteps }} - - task: Powershell@2 - inputs: - filePath: $(Build.SourcesDirectory)/eng/scripts/Save-Package-Namespaces-Property.ps1 - arguments: > - -ArtifactStagingDirectory "$(Build.ArtifactStagingDirectory)" - -ArtifactsList ('${{ convertToJson(parameters.Artifacts) }}' | ConvertFrom-Json) - pwsh: true - workingDirectory: $(Pipeline.Workspace) - displayName: Update package properties with namespaces + - ${{ if eq(parameters.RunApiStubGen, 'true') }}: + - task: Powershell@2 + inputs: + filePath: $(Build.SourcesDirectory)/eng/scripts/Save-Package-Namespaces-Property.ps1 + arguments: > + -ArtifactStagingDirectory "$(Build.ArtifactStagingDirectory)" + pwsh: true + workingDirectory: $(Pipeline.Workspace) + displayName: Update package properties with namespaces # Component governance collates discovered dependencies from a combination of # manifest files, splatted json output from CG detections, setup.py files, and other sources. @@ -111,13 +100,14 @@ steps: ArtifactPath: '$(Build.ArtifactStagingDirectory)' ArtifactName: 'packages_extended' - - pwsh: | - $directoryExists = Test-Path -Path "$(Build.SourcesDirectory)/_docs" - Write-Output "##vso[task.setvariable variable=DirectoryExists]$directoryExists" - displayName: Check if docs directory exists + - ${{ if eq(parameters.BuildDocs, 'true') }}: + - pwsh: | + $directoryExists = Test-Path -Path "$(Build.SourcesDirectory)/_docs" + Write-Output "##vso[task.setvariable variable=DirectoryExists]$directoryExists" + displayName: Check if docs directory exists - - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml - parameters: - ArtifactPath: '$(Build.SourcesDirectory)/_docs' - CustomCondition: and(${{ parameters.BuildDocs }}, eq(variables['DirectoryExists'], 'True')) - ArtifactName: 'documentation' + - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml + parameters: + ArtifactPath: '$(Build.SourcesDirectory)/_docs' + CustomCondition: eq(variables['DirectoryExists'], 'True') + ArtifactName: 'documentation' diff --git a/eng/pipelines/templates/steps/build-package-artifacts.yml b/eng/pipelines/templates/steps/build-package-artifacts.yml index 89896afd06e3..dfc039321b77 100644 --- a/eng/pipelines/templates/steps/build-package-artifacts.yml +++ b/eng/pipelines/templates/steps/build-package-artifacts.yml @@ -17,14 +17,10 @@ parameters: - name: ArtifactSuffix type: string default: 'linux' + - name: BuildTargetingString + type: string + default: '*' -# The variable TargetingString is set by template `eng/pipelines/templates/steps/targeting-string-resolve.yml`. This template is invoked wherever -# package targeting needs to be resolved prior to doing any actual work, whether it be build, static analysis, or test. - -# Please use `$(TargetingString)` to refer to the python packages glob string. This was previously `${{ parameters.BuildTargetingString }}`. This variable -# will be updated by PR diff detection as well. - -# In addition, the variable ENABLE_EXTENSION_BUILD is honored to enable builds on windows and mac if necessary. steps: - task: UsePythonVersion@0 displayName: 'Use Python $(PythonVersion)' @@ -40,15 +36,84 @@ steps: - template: /eng/common/pipelines/templates/steps/set-default-branch.yml@self + - ${{if eq(variables['System.TeamProject'], 'internal') }}: + - template: auth-dev-feed.yml + parameters: + DevFeedName: ${{ parameters.DevFeedName }} + - script: | echo "##vso[build.addbuildtag]Scheduled" displayName: 'Tag scheduled builds' condition: and(eq(variables['Build.SourceBranchName'], variables['DefaultBranch']), eq(variables['Build.Reason'],'Schedule')) - - ${{if eq(variables['System.TeamProject'], 'internal') }}: - - template: auth-dev-feed.yml - parameters: - DevFeedName: ${{ parameters.DevFeedName }} + # set dev build calls SavePackageProperties with dev version if necessary. this will never happen in a PR build + # however, it will set the variable SetDevVersion as appropriate for normal nightly runs + - template: set-dev-build.yml + parameters: + ServiceDirectory: ${{ parameters.ServiceDirectory }} + + # now we need to call Save-Package-Properties so that we can filter on it + - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: + - pwsh: | + mkdir -p $(Build.ArtifactStagingDirectory)/diff + displayName: Create PR Diff Folder + + - pwsh: | + $location = Join-Path "$(Build.ArtifactStagingDirectory)" "diff" + + Write-Host "./eng/common/scripts/Generate-PR-Diff.ps1 -TargetPath `"$(Build.SourcesDirectory)`" -ArtifactPath `"$location`"" + ./eng/common/scripts/Generate-PR-Diff.ps1 -TargetPath "$(Build.SourcesDirectory)" -ArtifactPath "$location" + displayName: Generate PR Diff + + - pwsh: | + Write-Host "We freshly generated the PR diff, and this is what it sees!" + Get-ChildItem -R -Force $(Build.ArtifactStagingDirectory)/diff | % { $_.FullName } + cat $(Build.ArtifactStagingDirectory)/diff/diff.json + displayName: Dump PR Diff + + - task: Powershell@2 + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/scripts/Save-Package-Properties.ps1 + arguments: > + -PrDiff $(Build.ArtifactStagingDirectory)/diff/diff.json + -OutDirectory $(Build.ArtifactStagingDirectory)/PackageInfo + pwsh: true + workingDirectory: $(Pipeline.Workspace) + displayName: Save package properties filtered for PR + + - ${{ else }}: + - task: Powershell@2 + condition: and(succeeded(), eq(variables['SetDevVersion'],'true')) + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/scripts/Save-Package-Properties.ps1 + arguments: > + -ServiceDirectory ${{parameters.ServiceDirectory}} + -OutDirectory $(Build.ArtifactStagingDirectory)/PackageInfo + -AddDevVersion + pwsh: true + workingDirectory: $(Pipeline.Workspace) + displayName: Save package properties with dev version + - task: Powershell@2 + condition: and(succeeded(), ne(variables['SetDevVersion'],'true')) + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/scripts/Save-Package-Properties.ps1 + arguments: > + -ServiceDirectory ${{parameters.ServiceDirectory}} + -OutDirectory $(Build.ArtifactStagingDirectory)/PackageInfo + pwsh: true + workingDirectory: $(Pipeline.Workspace) + displayName: Save package properties for service + + - template: /eng/pipelines/templates/steps/resolve-package-targeting.yml + parameters: + BuildTargetingString: ${{ parameters.BuildTargetingString }} + PackagePropertiesFolder: $(Build.ArtifactStagingDirectory)/PackageInfo + + # todo, walk the artifacts and ensure that one which includes an extension package is present + # if not, we only need to build on linux. if so, we need to build on all platforms + - template: /eng/pipelines/templates/steps/resolve-build-platforms.yml + parameters: + PackagePropertiesFolder: $(Build.ArtifactStagingDirectory)/PackageInfo - template: /eng/pipelines/templates/steps/use-venv.yml parameters: @@ -64,11 +129,6 @@ steps: displayName: 'Prep Environment' condition: and(succeeded(), or(eq(variables['ENABLE_EXTENSION_BUILD'], 'true'), eq('${{ parameters.ArtifactSuffix }}', 'linux'))) - - template: set-dev-build.yml@self - parameters: - ServiceDirectory: ${{ parameters.ServiceDirectory }} - Condition: and(succeeded(), or(eq(variables['ENABLE_EXTENSION_BUILD'], 'true'), eq('${{ parameters.ArtifactSuffix }}', 'linux'))) - - task: Powershell@2 inputs: filePath: $(Build.SourcesDirectory)/eng/common/scripts/Save-Package-Properties.ps1 @@ -91,7 +151,7 @@ steps: - pwsh: | $(VENV_ACTIVATION_SCRIPT) which python - sdk_build -d "$(Build.ArtifactStagingDirectory)" "$(TargetingString)" --service=${{parameters.ServiceDirectory}} --inactive + sdk_build -d "$(Build.ArtifactStagingDirectory)" "$(TargetingString)" --inactive displayName: 'Generate Packages' condition: and(succeeded(), or(eq(variables['ENABLE_EXTENSION_BUILD'], 'true'), eq('${{ parameters.ArtifactSuffix }}', 'linux'))) timeoutInMinutes: 80 diff --git a/eng/pipelines/templates/steps/build-test.yml b/eng/pipelines/templates/steps/build-test.yml index c9110a5b7ab2..b9f04a19dada 100644 --- a/eng/pipelines/templates/steps/build-test.yml +++ b/eng/pipelines/templates/steps/build-test.yml @@ -8,9 +8,7 @@ parameters: OSVmImage: '' BeforeTestSteps: [] AfterTestSteps: [] - CoverageArg: '' ToxTestEnv: "" - RunCoverage: ne(variables['CoverageArg'], '--disablecov') ToxEnvParallel: '' InjectedPackages: '' DevFeedName: 'public/azure-sdk-for-python' @@ -18,17 +16,9 @@ parameters: UseFederatedAuth: false ServiceConnection: '' -# The variable TargetingString is set by template `eng/pipelines/templates/steps/targeting-string-resolve.yml`. This template is invoked from yml files: -# eng/pipelines/templates/jobs/ci.tests.yml -# eng/pipelines/templates/jobs/ci.yml -# eng/pipelines/templates/jobs/live.test.yml -# Please use `$(TargetingString)` to refer to the python packages glob string. This was previously `${{ parameters.BuildTargetingString }}`. +# Please use `$(TargetingString)` to refer to the python packages glob string. This variable is set from resolve-package-targeting.yml. steps: - - pwsh: | - gci -r $(Build.ArtifactStagingDirectory) - displayName: Dump Artifact Directory - - template: /eng/pipelines/templates/steps/use-python-version.yml parameters: versionSpec: '${{ parameters.PythonVersion }}' @@ -38,8 +28,12 @@ steps: Activate: false - template: set-dev-build.yml - parameters: - ServiceDirectory: ${{ parameters.ServiceDirectory }} + + - pwsh: | + if ("$(PythonVersion)" -ne '3.9') { + Write-Host "##vso[task.setvariable variable=CoverageArg]--disableCov" + } + displayName: "Set Coverage Arg" - pwsh: | if ($IsWindows) { @@ -55,10 +49,9 @@ steps: Write-Host (Get-Command python).Source displayName: 'Prep Environment' - - ${{if eq(parameters.TestProxy, true) }}: - - template: /eng/common/testproxy/test-proxy-tool.yml - parameters: - runProxy: false + - template: /eng/common/testproxy/test-proxy-tool.yml + parameters: + runProxy: false - ${{if eq(variables['System.TeamProject'], 'internal') }}: - template: ../steps/auth-dev-feed.yml diff --git a/eng/pipelines/templates/steps/build.yml b/eng/pipelines/templates/steps/build.yml deleted file mode 100644 index 6dd0391d6155..000000000000 --- a/eng/pipelines/templates/steps/build.yml +++ /dev/null @@ -1,36 +0,0 @@ -parameters: - - name: ServiceDirectory - type: string - default: '' - - name: BuildTargetingString - type: string - default: 'azure-*' - - name: BeforePublishSteps - type: object - default: [] - - name: TestPipeline - type: boolean - default: false - - name: ArtifactSuffix - type: string - default: '' - - name: Artifacts - type: object - default: [] - -steps: - - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml - parameters: - BuildTargetingString: ${{ parameters.BuildTargetingString }} - - - template: /eng/pipelines/templates/steps/resolve-build-platforms.yml - parameters: - ServiceDirectory: ${{ parameters.ServiceDirectory }} - - - template: /eng/pipelines/templates/steps/build-package-artifacts.yml - parameters: - ServiceDirectory: ${{ parameters.ServiceDirectory }} - BeforePublishSteps: ${{ parameters.BeforePublishSteps }} - TestPipeline: ${{ parameters.TestPipeline }} - Artifacts: ${{ parameters.Artifacts }} - ArtifactSuffix: ${{ parameters.ArtifactSuffix }} diff --git a/eng/pipelines/templates/steps/resolve-build-platforms.yml b/eng/pipelines/templates/steps/resolve-build-platforms.yml index 569a80cdccdb..57d84e2b5c31 100644 --- a/eng/pipelines/templates/steps/resolve-build-platforms.yml +++ b/eng/pipelines/templates/steps/resolve-build-platforms.yml @@ -1,14 +1,14 @@ parameters: - - name: ServiceDirectory + - name: PackagePropertiesFolder type: string default: '' steps: # when we merge pipeline v3, this check will change to examining the targeting string $(TargetingString) - # as the generate-pr-diff call + resolution will be present in targeting-string-resolve.yml. + # as the generate-pr-diff call + resolution will be present in resolve-package-targeting.yml. # until then, we simply check to see if we're targeting storage service directory - pwsh: | - if ("${{ parameters.ServiceDirectory }}" -eq "storage") { + if ("${{ parameters.PackagePropertiesFolder }}".Contains("azure-storage-extension")) { Write-Host "Targeting storage, enabling extension build." Write-Host "##vso[task.setvariable variable=ENABLE_EXTENSION_BUILD]true" } diff --git a/eng/pipelines/templates/steps/resolve-package-targeting.yml b/eng/pipelines/templates/steps/resolve-package-targeting.yml new file mode 100644 index 000000000000..e2a612126ced --- /dev/null +++ b/eng/pipelines/templates/steps/resolve-package-targeting.yml @@ -0,0 +1,36 @@ +parameters: + - name: BuildTargetingString + type: string + default: 'azure-*' + - name: PackagePropertiesFolder + type: string + default: '' + +steps: + # whether we are running within a PR build or not, we need to walk the package properties and save them to + # variable $(TargetingString) so that every other check and accept it as an argument. + - pwsh: | + $setting = "${{ parameters.BuildTargetingString }}" + + # if the variable is not set, it'll just come back as the variable name. otherwise it's set. + if ('$(BuildTargetingString)' -ne ('$' + '(BuildTargetingString)')) { + Write-Host "The variable named BuildTargetingString is set to $(BuildTargetingString)" + $setting = "$(BuildTargetingString)" + } + else { + $packageProperties = Get-ChildItem -Recurse -Force "${{ parameters.PackagePropertiesFolder }}" ` + | Where-Object { $_.Extension -eq '.json' } ` + | Foreach-Object { $_.Name } ` + | ForEach-Object { $_.Replace(".json", "") } + $setting = $packageProperties -join "," + } + + Write-Host "##vso[task.setvariable variable=TargetingString;]$setting" + displayName: Resolve Targeted Packages + # if targeting has been set by matrix, this value will already be populated + condition: eq(variables['TargetingString'], '') + + - pwsh: | + Write-Host "This run is targeting: `n$(TargetingString)" + displayName: Display Targeted Packages + condition: ne(variables['TargetingString'], '') \ No newline at end of file diff --git a/eng/pipelines/templates/steps/run_apistub.yml b/eng/pipelines/templates/steps/run_apistub.yml index 57c7066d0832..7928b0d22f74 100644 --- a/eng/pipelines/templates/steps/run_apistub.yml +++ b/eng/pipelines/templates/steps/run_apistub.yml @@ -1,15 +1,15 @@ parameters: ServiceDirectory: '' AdditionalTestArgs: '' + RunApiStub: true - -# The variable TargetingString is set by template `eng/pipelines/templates/steps/targeting-string-resolve.yml`. This template is invoked from yml files: -# eng/pipelines/templates/jobs/ci.tests.yml -# eng/pipelines/templates/jobs/ci.yml -# eng/pipelines/templates/jobs/live.test.yml - -# Please use `$(TargetingString)` to refer to the python packages glob string. This was previously `${{ parameters.BuildTargetingString }}`. +# Please use `$(TargetingString)` to refer to the python packages glob string. This variable is set from resolve-package-targeting.yml. steps: + - pwsh: | + Write-Host "##vso[task.setvariable variable=PIP_INDEX_URL]https://pypi.python.org/simple" + displayName: Reset PIP Index For APIStubGen + condition: and(succeededOrFailed(), eq(variables['Skip.ApiStubGen'],'true')) + - task: UsePythonVersion@0 displayName: 'Use Python 3.10' inputs: @@ -30,8 +30,8 @@ steps: "$(TargetingString)" ${{ parameters.AdditionalTestArgs }} -d "$(Build.ArtifactStagingDirectory)" - --mark_arg="${{ parameters.TestMarkArgument }}" - --service="${{ parameters.ServiceDirectory }}" + --mark_arg="${{ parameters.TestMarkArgument }}" + --service="${{ parameters.ServiceDirectory }}" --toxenv="apistub" --disablecov --filter-type="Omit_management" diff --git a/eng/pipelines/templates/steps/run_bandit.yml b/eng/pipelines/templates/steps/run_bandit.yml index 4bb8c374d11e..a57cfdb27202 100644 --- a/eng/pipelines/templates/steps/run_bandit.yml +++ b/eng/pipelines/templates/steps/run_bandit.yml @@ -3,12 +3,7 @@ parameters: TestMarkArgument: '' EnvVars: {} -# The variable TargetingString is set by template `eng/pipelines/templates/steps/targeting-string-resolve.yml`. This template is invoked from yml files: -# eng/pipelines/templates/jobs/ci.tests.yml -# eng/pipelines/templates/jobs/ci.yml -# eng/pipelines/templates/jobs/live.test.yml - -# Please use `$(TargetingString)` to refer to the python packages glob string. This was previously `${{ parameters.BuildTargetingString }}`. +# Please use `$(TargetingString)` to refer to the python packages glob string. This variable is set from resolve-package-targeting.yml. steps: - script: | python -m pip install -r eng/ci_tools.txt @@ -20,9 +15,9 @@ steps: inputs: scriptPath: 'scripts/devops_tasks/dispatch_tox.py' arguments: >- - "$(TargetingString)" - --mark_arg="${{ parameters.TestMarkArgument }}" - --service="${{ parameters.ServiceDirectory }}" + "$(TargetingString)" + --mark_arg="${{ parameters.TestMarkArgument }}" + --service="${{ parameters.ServiceDirectory }}" --toxenv="bandit" --disablecov --filter-type="Omit_management" diff --git a/eng/pipelines/templates/steps/run_breaking_changes.yml b/eng/pipelines/templates/steps/run_breaking_changes.yml index 499ee3e51081..31796638c800 100644 --- a/eng/pipelines/templates/steps/run_breaking_changes.yml +++ b/eng/pipelines/templates/steps/run_breaking_changes.yml @@ -3,12 +3,7 @@ parameters: TestMarkArgument: '' EnvVars: {} -# The variable TargetingString is set by template `eng/pipelines/templates/steps/targeting-string-resolve.yml`. This template is invoked from yml files: -# eng/pipelines/templates/jobs/ci.tests.yml -# eng/pipelines/templates/jobs/ci.yml -# eng/pipelines/templates/jobs/live.test.yml - -# Please use `$(TargetingString)` to refer to the python packages glob string. This was previously `${{ parameters.BuildTargetingString }}`. +# Please use `$(TargetingString)` to refer to the python packages glob string. This variable is set from resolve-package-targeting.yml. steps: - task: PythonScript@0 displayName: 'Run Breaking Changes' diff --git a/eng/pipelines/templates/steps/run_mypy.yml b/eng/pipelines/templates/steps/run_mypy.yml index e05e4ca65467..7a520b91132b 100644 --- a/eng/pipelines/templates/steps/run_mypy.yml +++ b/eng/pipelines/templates/steps/run_mypy.yml @@ -5,12 +5,7 @@ parameters: EnvVars: {} AdditionalTestArgs: '' -# The variable TargetingString is set by template `eng/pipelines/templates/steps/targeting-string-resolve.yml`. This template is invoked from yml files: -# eng/pipelines/templates/jobs/ci.tests.yml -# eng/pipelines/templates/jobs/ci.yml -# eng/pipelines/templates/jobs/live.test.yml - -# Please use `$(TargetingString)` to refer to the python packages glob string. This was previously `${{ parameters.BuildTargetingString }}`. +# Please use `$(TargetingString)` to refer to the python packages glob string. This variable is set from resolve-package-targeting.yml. steps: - task: UsePythonVersion@0 displayName: 'Use Python 3.8' diff --git a/eng/pipelines/templates/steps/run_pylint.yml b/eng/pipelines/templates/steps/run_pylint.yml index 9a39873d3bdd..7087cbc88b22 100644 --- a/eng/pipelines/templates/steps/run_pylint.yml +++ b/eng/pipelines/templates/steps/run_pylint.yml @@ -4,12 +4,7 @@ parameters: EnvVars: {} AdditionalTestArgs: '' -# The variable TargetingString is set by template `eng/pipelines/templates/steps/targeting-string-resolve.yml`. This template is invoked from yml files: -# eng/pipelines/templates/jobs/ci.tests.yml -# eng/pipelines/templates/jobs/ci.yml -# eng/pipelines/templates/jobs/live.test.yml - -# Please use `$(TargetingString)` to refer to the python packages glob string. This was previously `${{ parameters.BuildTargetingString }}`. +# Please use `$(TargetingString)` to refer to the python packages glob string. This variable is set from resolve-package-targeting.yml. steps: - task: UsePythonVersion@0 displayName: 'Use Python 3.8' diff --git a/eng/pipelines/templates/steps/run_pyright.yml b/eng/pipelines/templates/steps/run_pyright.yml index 9ac8ced7bb00..44d262e98d2c 100644 --- a/eng/pipelines/templates/steps/run_pyright.yml +++ b/eng/pipelines/templates/steps/run_pyright.yml @@ -4,12 +4,7 @@ parameters: TestMarkArgument: '' AdditionalTestArgs: '' -# The variable TargetingString is set by template `eng/pipelines/templates/steps/targeting-string-resolve.yml`. This template is invoked from yml files: -# eng/pipelines/templates/jobs/ci.tests.yml -# eng/pipelines/templates/jobs/ci.yml -# eng/pipelines/templates/jobs/live.test.yml - -# Please use `$(TargetingString)` to refer to the python packages glob string. This was previously `${{ parameters.BuildTargetingString }}`. +# Please use `$(TargetingString)` to refer to the python packages glob string. This variable is set from resolve-package-targeting.yml. steps: - task: UsePythonVersion@0 displayName: 'Use Python 3.8' diff --git a/eng/pipelines/templates/steps/run_sphinx.yml b/eng/pipelines/templates/steps/run_sphinx.yml new file mode 100644 index 000000000000..babdfbcaa696 --- /dev/null +++ b/eng/pipelines/templates/steps/run_sphinx.yml @@ -0,0 +1,24 @@ +parameters: + ServiceDirectory: '' + WheelDirectory: '' + +# `$(TargetingString)` is set by a prior resolution step. +steps: + - pwsh: | + python ./scripts/devops_tasks/replace_requirements.py --service "${{ parameters.ServiceDirectory }}" -w "${{ parameters.WheelDirectory }}" $(TargetingString) + displayName: Pre-Build Dev Requirements + timeoutInMinutes: 15 + + - pwsh: | + Get-ChildItem $RepoRoot -Filter "$Toxenv-*.log" | ForEach-Object { Write-Host "Output for $($_.Name)"; Get-Content $_.FullName } + displayName: Dump Sphinx Logs + condition: failed() + + - pwsh: | + ./eng/scripts/invoke-tox-parallel.ps1 ` + -TargetingString "$(TargetingString)" ` + -RepoRoot "$(Build.Sourcesdirectory)" ` + -ServiceDirectory "${{ parameters.ServiceDirectory }}" ` + -WheelDirectory "${{ parameters.WheelDirectory }}" + Write-Host $LASTEXITCODE + displayName: 'Run Sphinx Environments' diff --git a/eng/pipelines/templates/steps/set-dev-build.yml b/eng/pipelines/templates/steps/set-dev-build.yml index 07de80a0bf78..26233110f3c8 100644 --- a/eng/pipelines/templates/steps/set-dev-build.yml +++ b/eng/pipelines/templates/steps/set-dev-build.yml @@ -2,20 +2,14 @@ parameters: ServiceDirectory: '' Condition: 'succeededOrFailed()' -# The variable TargetingString is set by template `eng/pipelines/templates/steps/targeting-string-resolve.yml`. This template is invoked from yml files: -# eng/pipelines/templates/jobs/ci.tests.yml -# eng/pipelines/templates/jobs/ci.yml -# eng/pipelines/templates/jobs/live.test.yml - -# Please use `$(TargetingString)` to refer to the python packages glob string. This was previously `${{ parameters.BuildTargetingString }}`. steps: + # this will never call save Save-Package-Properties directly as ServiceDirectory is not present + # it will however set an indicator for whether or not we're in a nightly build though. + # we will handle the Save-Package-Properties call ourselves - template: /eng/common/pipelines/templates/steps/daily-dev-build-variable.yml - parameters: - ServiceDirectory: ${{ parameters.ServiceDirectory }} - Condition: ${{ parameters.Condition }} - pwsh: | python -m pip install "tools/azure-sdk-tools[build]" - sdk_set_dev_version "$(TargetingString)" --service="${{parameters.ServiceDirectory}}" --build-id="$(Build.BuildNumber)" + sdk_set_dev_version "*" --build-id="$(Build.BuildNumber)" displayName: "Update package versions for dev build" condition: and(succeededOrFailed(), eq(variables['SetDevVersion'],'true'), ${{ parameters.Condition }}) diff --git a/eng/pipelines/templates/steps/targeting-string-resolve.yml b/eng/pipelines/templates/steps/targeting-string-resolve.yml deleted file mode 100644 index 268d6d92c3e7..000000000000 --- a/eng/pipelines/templates/steps/targeting-string-resolve.yml +++ /dev/null @@ -1,22 +0,0 @@ -parameters: - - name: BuildTargetingString - type: string - default: 'azure-*' - -steps: - - pwsh: | - $setting = "${{ parameters.BuildTargetingString }}" - - # if the variable is not set, it'll just come back as the variable name. otherwise it's set. - if ('$(BuildTargetingString)' -ne ('$' + '(BuildTargetingString)')) { - Write-Host "The variable named BuildTargetingString is set to $(BuildTargetingString)" - $setting = "$(BuildTargetingString)" - } - else { - Write-Host "We are falling back to the parameter value ${{ parameters.BuildTargetingString }}" - } - - Write-Host "Setting TargetingString to $setting" - Write-Host "##vso[task.setvariable variable=TargetingString]$setting" - displayName: Check override of targeting string - condition: and(succeededOrFailed(), eq(variables['TargetingString'],'')) diff --git a/eng/pipelines/templates/variables/globals.yml b/eng/pipelines/templates/variables/globals.yml index 065d73d83102..63309a3390f0 100644 --- a/eng/pipelines/templates/variables/globals.yml +++ b/eng/pipelines/templates/variables/globals.yml @@ -10,4 +10,4 @@ variables: BuildId: $(Build.BuildId) REPOROOT: $(Build.SourcesDirectory) WINDOWS_OUTPUTROOT: $(REPOROOT)\out - WindowsContainerImage: 'onebranch.azurecr.io/windows/ltsc2019/vse2022:latest' \ No newline at end of file + WindowsContainerImage: 'onebranch.azurecr.io/windows/ltsc2019/vse2022:latest' diff --git a/eng/pipelines/trigger-ml-sample-pipeline.yml b/eng/pipelines/trigger-ml-sample-pipeline.yml index b5546a69a4fb..6a0a57ee315a 100644 --- a/eng/pipelines/trigger-ml-sample-pipeline.yml +++ b/eng/pipelines/trigger-ml-sample-pipeline.yml @@ -26,7 +26,7 @@ jobs: vmImage: MMSUbuntu20.04 steps: - - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml + - template: /eng/pipelines/templates/steps/resolve-package-targeting.yml parameters: BuildTargetingString: "azure-*" diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 index 7d938ff1cea5..52b6dbeb9e6b 100644 --- a/eng/scripts/Language-Settings.ps1 +++ b/eng/scripts/Language-Settings.ps1 @@ -35,6 +35,7 @@ function Get-AllPackageInfoFromRepo ($serviceDirectory) Pop-Location } + foreach ($line in $allPkgPropLines) { $pkgInfo = ($line -Split " ") @@ -42,6 +43,9 @@ function Get-AllPackageInfoFromRepo ($serviceDirectory) $packageVersion = $pkgInfo[1] $isNewSdk = ($pkgInfo[2] -eq "True") $pkgDirectoryPath = $pkgInfo[3] + + $additionalValidationPackages = $pkgInfo[4] -Split "," + $serviceDirectoryName = Split-Path (Split-Path -Path $pkgDirectoryPath -Parent) -Leaf if ($packageName -match "mgmt") { @@ -55,6 +59,7 @@ function Get-AllPackageInfoFromRepo ($serviceDirectory) $pkgProp.IsNewSdk = $isNewSdk $pkgProp.SdkType = $sdkType $pkgProp.ArtifactName = $packageName + $pkgProp.AdditionalValidationPackages = $additionalValidationPackages $allPackageProps += $pkgProp } return $allPackageProps diff --git a/eng/scripts/Save-Package-Namespaces-Property.ps1 b/eng/scripts/Save-Package-Namespaces-Property.ps1 index 3c305d23c5f7..22d576e61992 100644 --- a/eng/scripts/Save-Package-Namespaces-Property.ps1 +++ b/eng/scripts/Save-Package-Namespaces-Property.ps1 @@ -23,9 +23,7 @@ producing docs. [CmdletBinding()] Param ( [Parameter(Mandatory = $True)] - [string] $ArtifactStagingDirectory, - [Parameter(Mandatory=$true)] - [array] $ArtifactsList + [string] $ArtifactStagingDirectory ) $ArtifactsList = $ArtifactsList | Where-Object -Not "skipPublishDocMs" @@ -43,19 +41,15 @@ if (-not (Test-Path -Path $ArtifactStagingDirectory)) { exit 1 } -Write-Host "" -Write-Host "ArtifactsList:" -$ArtifactsList | Format-Table -Property Name | Out-String | Write-Host - $packageInfoDirectory = Join-Path $ArtifactStagingDirectory "PackageInfo" - $foundError = $false -# At this point the packageInfo files should have been already been created. -# The only thing being done here is adding or updating namespaces for libraries -# that will be producing docs. -foreach($artifact in $ArtifactsList) { +$artifacts = Get-ChildItem -Path $packageInfoDirectory -File -Filter "*.json" +$artifacts | Format-Table -Property Name | Out-String | Write-Host + +# by this point, the PackageInfo folder will have a json file for each artifact +# we simply need to read each file, get the appropriate metadata, and add the namespaces if necessary +foreach($packageInfoFile in $artifacts) { # Get the version from the packageInfo file - $packageInfoFile = Join-Path $packageInfoDirectory "$($artifact.Name).json" Write-Host "processing $($packageInfoFile.FullName)" $packageInfo = ConvertFrom-Json (Get-Content $packageInfoFile -Raw) $version = $packageInfo.Version diff --git a/eng/scripts/distribute-packages-to-matrix.ps1 b/eng/scripts/distribute-packages-to-matrix.ps1 new file mode 100644 index 000000000000..6cc0c4024535 --- /dev/null +++ b/eng/scripts/distribute-packages-to-matrix.ps1 @@ -0,0 +1,197 @@ +<# +.SYNOPSIS +Distributes a set of packages to a platform matrix file by adding TargetingString property dynamically. +If an unexpected situation is encountered, the script will make no changes to the input file. + +.DESCRIPTION +Because of the way the platform matrix file is structured, we need to distribute the packages in a way that +honors the Include packages. We have these included this way because want to ensure that these python versions +run on that platform. This is aware of these additional configurations and will set TargetingString for them as well. +#> +param( + [parameter(Mandatory=$true)] + [string]$PackageInfoFolder, + [parameter(Mandatory=$true)] + [string]$PlatformMatrix +) + +Set-StrictMode -Version 4 + +if (!(Test-Path $PackageInfoFolder)) { + Write-Error "PackageInfo folder file not found: $PackageInfoFolder" + exit 1 +} + +if (!(Test-Path $PlatformMatrix)) { + Write-Error "Platform matrix file not found: $PlatformMatrix" + exit 1 +} + + +function Split-ArrayIntoBatches { + param ( + [Parameter(Mandatory=$true)] + [object[]]$InputArray, + + [int]$BatchSize = 5 + ) + + # Initialize an empty array to hold the batches + $batches = @() + + # Loop through the input array in increments of the batch size + for ($i = 0; $i -lt $InputArray.Count; $i += $BatchSize) { + # Create a batch containing up to $BatchSize elements + $batch = $InputArray[$i..[math]::Min($i + $BatchSize - 1, $InputArray.Count - 1)] + + # Add the batch to the list of batches + $batches += ,$batch + } + + return ,$batches +} + +$packageProperties = Get-ChildItem -Recurse "$PackageInfoFolder" *.json | % { Get-Content -Path $_.FullName | ConvertFrom-Json } +$matrix = Get-Content -Path $PlatformMatrix | ConvertFrom-Json + +$versionCount = $matrix.matrix.PythonVersion.Count +$includeCount = 0 +$includeObject = $null +if ($matrix.PSObject.Properties.Name -contains "include") { + $includeCount = $matrix.include.Count + $originalInclude = $matrix.include + # todo: reenable this + $matrix.include = @() + Write-Host "Original include type " + $originalInclude.GetType() +} +$batchCount = $versionCount + $includeCount +if ($batchCount -eq 0) { + Write-Error "No batches detected, skipping without updating platform matrix file $PlatformMatrix" + exit 1 +} + +$batchSize = 5 + +# batches is a hashtable because you cannot instantiate an array of empty arrays +# the add method will merely invoke and silently NOT add a new item +# using a hashtable bypasses this limitation +$batches = @{} +for ($i = 0; $i -lt $batchCount; $i++) { + $batches[$i] = @() +} +$batchCounter = 0 +$batchValues = @() + +# if there is an IMPORT, we should import that first and use THAT. we have a clear edge case where a nested include will NOT work. +# I am accepting that in interests of making this work with all our platform matrix json files throughout the repo +# TODO: implement grabbing the matrix from the import if it exists +$directIncludedPackages = $packageProperties | Where-Object { $_.IncludedForValidation -eq $false } +$indirectIncludedPackages = $packageProperties | Where-Object { $_.IncludedForValidation -eq $true } + +Write-Host "Direct Included Length = $($directIncludedPackages.Length)" +Write-Host "IndirectIncludedPackages Included Length = $($indirectIncludedPackages.Length)" + +# for python, for fast tests, I want the sets of packages to be broken up into sets of five. + +# I will assign all the direct included packages first. our goal is to get complete coverage of the direct included packages +# then, for the indirect packages, we will ADD them as extra TargetingString bundles to the matrix. +# this means that these "extra" packages will never run on any platform that's not present in INCLUDE, but frankly +# that's nbd IMO + +$directBatches = Split-ArrayIntoBatches -InputArray $directIncludedPackages -BatchSize $batchSize +Write-Host $directBatches.Length +Write-Host $directBatches[0].Length +# all directly included packages should have their tests invoked in full, so, lets use a basic storage service discovery. We'll have direct package batches of + +# ["azure-storage-blob", "azure-storage-blob-changefeed", "azure-storage-extensions", "azure-storage-file-datalake", "azure-storage-file-share"] +# ["azure-storage-queue"] + +# any other packages will be added to the matrix as additional targetingstring batches. it'll be sparse in that there won't be full matrix coverage + +# a basic platform matrix has the following: +# "matrix" -> "PythonVersion" = ["3.6", "3.7", "3.8", "3.9", "3.10"] +# "include" -> "ConfigName" -> "ActualJobName" -> PythonVersion=3.9 +# so what we need to do is for each batch + # 1. assign the batch * matrix size to the matrix as "TargetingString" + # 2. assign the batch to each include with the batch being the value for TargetingString for the include + # the easiest way to do this is to take a copy of the `Include` object at the beginning of this script, + # for each batch, assign the batch to the include object, and then add the include object to the matrix object + # this will have the result of multiplexing our includes, so we will need to update the name there + +if ($directBatches) { + # we need to ensure the presence of TargetingString in the matrix object + if (-not $matrix.matrix.PSObject.Properties["TargetingString"]) { + $matrix.matrix | Add-Member -Force -MemberType NoteProperty -Name TargetingString -Value @() + } + else { + $matrix.matrix.TargetingString = @() + } +} + +foreach($batch in $directBatches) { + $targetingString = ($batch | Select-Object -ExpandProperty Name) -join "," + + # we need to equal the number of python versions in the matrix (to ensure we get complete coverage) + # so we need to multiply the targeting string by the number of python versions, that'll ensure that + # across all python versions we see the same packages + $targetingStringArray = @($targetingString) * $versionCount + Write-Host "Returning batch:" + foreach($item in $targetingStringArray) { + Write-Host "`t$item" + } + + $matrix.matrix.TargetingString += $targetingStringArray + + # if there were any include objects, we need to duplicate them exactly and add the targeting string to each + # this means that the number of includes at the end of this operation will be incoming # of includes * the number of batches + if ($includeCount -gt 0) { + Write-Host "Walking include objects for $targetingString" + $includeCopy = $originalInclude | ConvertTo-Json -Depth 100 | ConvertFrom-Json + foreach ($configElement in $includeCopy) { + if ($configElement.PSObject.Properties) { + # an include looks like this + # { + # + # "CoverageConfig": { + # + # "ubuntu2004_39_coverage": { + # "OSVmImage": "env:LINUXVMIMAGE", + # "Pool": "env:LINUXPOOL", + # "PythonVersion": "3.9", + # "CoverageArg": "", + # "TestSamples": "false" + # } + # } + # } + $topLevelPropertyName = $configElement.PSObject.Properties.Name + $topLevelPropertyValue = $configElement.PSObject.Properties[$topLevelPropertyName].Value + + if ($topLevelPropertyValue.PSObject.Properties) { + $secondLevelPropertyName = $topLevelPropertyValue.PSObject.Properties.Name + $secondLevelPropertyValue = $topLevelPropertyValue.PSObject.Properties[$secondLevelPropertyName].Value + + $newSecondLevelName = "$secondLevelPropertyName" + "$targetingString" + + $topLevelPropertyValue.PSObject.Properties.Remove($secondLevelPropertyName) + $topLevelPropertyValue | Add-Member -MemberType NoteProperty -Name $newSecondLevelName -Value $secondLevelPropertyValue + + # add the targeting string property if it doesn't already exist + if (-not $topLevelPropertyValue.$newSecondLevelName.PSObject.Properties["TargetingString"]) { + $topLevelPropertyValue.$newSecondLevelName | Add-Member -Force -MemberType NoteProperty -Name TargetingString -Value "" + } + + # set the targeting string + $topLevelPropertyValue.$newSecondLevelName.TargetingString = $targetingString + } + } + + $matrix.include += $configElement + } + } +} + +foreach($batch in $indirectBatches) { + +} + +$matrix | ConvertTo-Json -Depth 100 | Set-Content -Path $PlatformMatrix diff --git a/eng/scripts/get_package_properties.py b/eng/scripts/get_package_properties.py index c0bea2a83592..99ed62fc878d 100644 --- a/eng/scripts/get_package_properties.py +++ b/eng/scripts/get_package_properties.py @@ -1,11 +1,76 @@ import argparse -import sys -import glob import os import re +from typing import Dict, List + from ci_tools.parsing import ParsedSetup +additional_pr_triggers: Dict[str, List[str]] = { + "azure-core":[ + os.path.join("/sdk", "servicebus", "azure-servicebus"), + os.path.join("/sdk", "eventhub", "azure-eventhub"), + os.path.join("/sdk", "tables", "azure-data-table"), + os.path.join("/sdk", "appconfig", "azure-appconfig"), + os.path.join("/sdk", "keyvault", "azure-keyvault-keys"), + os.path.join("/sdk", "identity", "azure-identity"), + os.path.join("/sdk", "core", "azure-mgmt-core"), + os.path.join("/sdk", "core", "azure-core-experimental"), + os.path.join("/sdk", "core", "azure-core-tracing-opentelemetry"), + os.path.join("/sdk", "core", "azure-core-tracing-opencensus"), + # todo: this currently won't be included, as azure-cosmos needs some special construction + # related to windows only + emulator + # os.path.join("/sdk", "cosmos", "azure-cosmos"), + os.path.join("/sdk", "ml", "azure-ai-ml"), + os.path.join("/sdk", "ai", "azure-ai-documentintelligence"), + os.path.join("/sdk", "ai", "azure-ai-inference"), + os.path.join("/sdk", "ai", "azure-ai-textanalytics"), + os.path.join("/sdk", "ai", "azure-ai-doctranslation"), + os.path.join("/sdk", "compute", "azure-mgmt-compute"), + os.path.join("/sdk", "communication", "azure-communication-chat"), + os.path.join("/sdk", "communication", "azure-communication-identity"), + os.path.join("/sdk", "storage", "azure-storage-blob") + ], + "azure-mgmt-core": [ + os.path.join("/sdk", "compute", "azure-mgmt-compute"), + os.path.join("/sdk", "network", "azure-mgmt-network"), + os.path.join("/sdk", "resource", "azure-mgmt-resource"), + os.path.join("/sdk", "keyvault", "azure-mgmt-keyvault") + ] +} + +# todo triggers based on paths and not files +# tools/ +# azure-storage-blob +# azure-servicebus +# azure-eventhub +# azure-data-table +# azure-appconfig +# azure-keyvault-keys +# azure-identity +# azure-mgmt-core +# azure-core-experimental +# azure-core-tracing-opentelemetry +# azure-core-tracing-opencensus +# azure-cosmos +# azure-ai-documentintelligence +# azure-ai-ml +# azure-ai-inference +# azure-ai-textanalytics +# azure-ai-doctranslation +# azure-mgmt-compute +# azure-communication-chat +# azure-communication-identity + +# eng/ +# azure-template +# azure-core + +# scripts/, doc/, common/, conda/ +# azure-template +# azure-core + + if __name__ == "__main__": parser = argparse.ArgumentParser(description="Get package version details from the repo") parser.add_argument("-s", "--search_path", required=True, help="The scope of the search") @@ -18,11 +83,19 @@ try: parsed = ParsedSetup.from_path(root) + dependent_packages = "" + if parsed.name in additional_pr_triggers: + dependent_packages = ",".join(additional_pr_triggers[parsed.name]) + print( - "{0} {1} {2} {3}".format( - parsed.name, parsed.version, parsed.is_new_sdk, os.path.dirname(parsed.setup_filename) + "{0} {1} {2} {3} {4}".format( + parsed.name, + parsed.version, + parsed.is_new_sdk, + os.path.dirname(parsed.setup_filename), + dependent_packages ) ) except: # Skip setup.py if the package cannot be parsed - pass + pass \ No newline at end of file diff --git a/eng/scripts/invoke-tox-parallel.ps1 b/eng/scripts/invoke-tox-parallel.ps1 new file mode 100644 index 000000000000..5724b4f0dbb3 --- /dev/null +++ b/eng/scripts/invoke-tox-parallel.ps1 @@ -0,0 +1,51 @@ + + +<# +.DESCRIPTION +This script is used to dispatch tox in parallel to multiple target packages. + +Because the sphinx invocation is IO-bound and do not influence each other, we are doing just simply to speed up +the total runtime. +#> + +param( + $TargetingString, + $RepoRoot, + $Check = "sphinx", + $ServiceDirectory = "", + $WheelDirectory = "" +) + +$pythonScript = Join-Path "$RepoRoot" "scripts" "devops_tasks" "dispatch_tox.py" +$jobs = @() +$packages = $TargetingString -split "," + +foreach ($package in $packages) { + $jobs += Start-Job -ScriptBlock { + param($Pkg, $ScriptPath, $RepoRoot, $Toxenv, $ServiceParam, $WheelParam) + $log = Join-Path $RepoRoot "$Toxenv-$Pkg.log" + Write-Host "& python $ScriptPath $Pkg --toxenv=$Toxenv --service `"$ServiceParam`" -w `"$WheelParam`" --disablecov 2>&1 >> $log" + & python $ScriptPath $Pkg --toxenv=$Toxenv --service "$ServiceParam" -w "$WheelParam" --disablecov 2>&1 >> $log + return $LASTEXITCODE + } -ArgumentList $package, $pythonScript, $RepoRoot, $Check, $ServiceDirectory, $WheelDirectory +} + +$jobs | ForEach-Object { $_ | Wait-Job } + +$exitCodes = $jobs | ForEach-Object { $_ | Receive-Job } +$nonZeroExit = $exitCodes | Where-Object { $_ -ne 0 } +$jobs | ForEach-Object { $_ | Remove-Job } + +Get-ChildItem $RepoRoot -Filter "$Toxenv-*.log" ` + | ForEach-Object { + Write-Host "Output for $($_.Name)" + Get-Content $_.FullName + } + +if ($nonZeroExit) { + Write-Host "One or more scripts failed with a non-zero exit code." + exit 1 +} else { + Write-Host "All scripts completed successfully." + exit 0 +} diff --git a/scripts/devops_tasks/calculate_targeted_packages.py b/scripts/devops_tasks/calculate_targeted_packages.py new file mode 100644 index 000000000000..f75ce345c39d --- /dev/null +++ b/scripts/devops_tasks/calculate_targeted_packages.py @@ -0,0 +1,49 @@ +import argparse +import json +import os + +from ci_tools.functions import discover_targeted_packages +from typing import Dict + +root_dir = os.path.abspath(os.path.join(os.path.abspath(__file__), "..", "..", "..")) + +def get_package_folder_from_path(file: str) -> str: + parts = file.split("/") + + if len(parts) < 3: + return "" + return os.sep.join(parts[:3]) + +def register(idx: Dict[str, str], key: str, value: str) -> None: + if key in idx: + idx[key].append(value) + else: + idx[key] = [value] + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Calculate the targeted packages') + parser.add_argument('input_json', type=str, help='Location of the diff.json file') + + args = parser.parse_args() + + + with open(args.input_json, 'r') as f: + diff = json.load(f) + + targeted_files = diff['ChangedFiles'] + + all_packages = {} # {package: [file]} + + for file in targeted_files: + # register(all_packages, , file) + package = get_package_folder_from_path(file) + if package: + package_directory = os.path.join(root_dir, package) + package_name = os.path.basename(package_directory) + + if os.path.exists(os.path.join(package_directory, "pyproject.toml")) or os.path.exists(os.path.join(package_directory, "setup.py")): + register(all_packages, package_name, file) + + packages = ",".join(all_packages.keys()) + print(f"This script sees the following packages [{packages}].") + print(f"##vso[task.setvariable variable=TargetingString;]{packages}") \ No newline at end of file diff --git a/scripts/devops_tasks/dispatch_tox.py b/scripts/devops_tasks/dispatch_tox.py index 03c2dc76e341..0317c18736ff 100644 --- a/scripts/devops_tasks/dispatch_tox.py +++ b/scripts/devops_tasks/dispatch_tox.py @@ -26,7 +26,7 @@ if __name__ == "__main__": parser = argparse.ArgumentParser( description=""" -This script is the single point for all checks invoked by CI within this repo. It works in two phases. +This script is the single point for all checks invoked by CI within this repo. It works in two phases. 1. Identify which packages in the repo are in scope for this script invocation, based on a glob string and a service directory. 2. Invoke one or multiple `tox` environments for each package identified as in scope. @@ -116,12 +116,14 @@ # We need to support both CI builds of everything and individual service # folders. This logic allows us to do both. - if args.service: + if args.service and args.service != "auto": service_dir = os.path.join("sdk", args.service) target_dir = os.path.join(root_dir, service_dir) else: target_dir = root_dir + logging.info(f"Beginning discovery for {args.service} and root dir {root_dir}. Resolving to {target_dir}.") + if args.filter_type == "None": args.filter_type = "Build" compatibility_filter = False @@ -133,7 +135,7 @@ ) if len(targeted_packages) == 0: - logging.info("No packages collected. Exit 0.") + logging.info(f"No packages collected for targeting string {args.glob_string} and root dir {root_dir}. Exit 0.") exit(0) logging.info(f"Executing prep_and_run_tox with the executable {sys.executable}.") diff --git a/scripts/devops_tasks/replace_requirements.py b/scripts/devops_tasks/replace_requirements.py new file mode 100644 index 000000000000..952a5db0e43c --- /dev/null +++ b/scripts/devops_tasks/replace_requirements.py @@ -0,0 +1,64 @@ +import os +import argparse +import logging + +root_dir = os.path.abspath(os.path.join(os.path.abspath(__file__), "..", "..", "..")) +test_tools_path = os.path.join(root_dir, "eng", "test_tools.txt") +dependency_tools_path = os.path.join(root_dir, "eng", "dependency_tools.txt") + +from ci_tools.functions import discover_targeted_packages +from ci_tools.scenario.generation import replace_dev_reqs + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description=""" +This script is a single point of access to replace relative requirements within the repo with prebuilt versions. +""" + ) + + parser.add_argument( + "glob_string", + nargs="?", + help=( + "A comma separated list of glob strings that will target the top level directories that contain packages." + 'Examples: All = "azure-*", Single = "azure-keyvault", Targeted Multiple = "azure-keyvault,azure-mgmt-resource"' + ), + ) + + parser.add_argument( + "--service", + help=("Name of service directory (under sdk/) to test. Example: --service applicationinsights"), + ) + + parser.add_argument( + "-w", + "--wheel_dir", + dest="wheel_dir", + help="Location for prebuilt artifacts (if any)", + ) + + args = parser.parse_args() + + # We need to support both CI builds of everything and individual service + # folders. This logic allows us to do both. + if args.service and args.service != "auto": + service_dir = os.path.join("sdk", args.service) + target_dir = os.path.join(root_dir, service_dir) + else: + target_dir = root_dir + + logging.info(f"Replacing local relative requirements in {test_tools_path} and {dependency_tools_path}.") + + logging.info(f"Beginning discovery for {args.service} and root dir {root_dir}. Resolving to {target_dir}.") + + targeted_packages = discover_targeted_packages( + args.glob_string, target_dir, "", "Build", False + ) + + for package in targeted_packages: + dev_requirements_path = os.path.join(package, "dev_requirements.txt") + + replace_dev_reqs(test_tools_path, package, args.wheel_dir) + replace_dev_reqs(dependency_tools_path, package, args.wheel_dir) + replace_dev_reqs(dev_requirements_path, package, args.wheel_dir) diff --git a/scripts/devops_tasks/set_tox_environment.py b/scripts/devops_tasks/set_tox_environment.py index 1dbe8140897f..fa689640ee55 100644 --- a/scripts/devops_tasks/set_tox_environment.py +++ b/scripts/devops_tasks/set_tox_environment.py @@ -46,7 +46,7 @@ def process_ci_skips(glob_string: str, service: str ) -> None: checks_with_global_skip = ["pylint", "verifywhl", "verifysdist" "bandit", "mypy", "pyright", "verifytypes"] root_dir = os.path.abspath(os.path.join(os.path.abspath(__file__), "..", "..", "..")) - if service: + if service and service != "auto": target_dir = os.path.join(root_dir, "sdk", service) else: target_dir = root_dir diff --git a/sdk/core/azure-core/perf.yml b/sdk/core/azure-core/perf.yml index a7774ec06b02..97e45420ba93 100644 --- a/sdk/core/azure-core/perf.yml +++ b/sdk/core/azure-core/perf.yml @@ -34,4 +34,4 @@ extends: Tests: ${{ parameters.Tests }} Arguments: ${{ parameters.Arguments }} Iterations: ${{ parameters.Iterations }} - AdditionalArguments: ${{ parameters.AdditionalArguments }} \ No newline at end of file + AdditionalArguments: ${{ parameters.AdditionalArguments }} diff --git a/sdk/core/azure-core/samples/test_example_async.py b/sdk/core/azure-core/samples/test_example_async.py index a94b1331f6bf..499775b0cfe2 100644 --- a/sdk/core/azure-core/samples/test_example_async.py +++ b/sdk/core/azure-core/samples/test_example_async.py @@ -23,6 +23,7 @@ # THE SOFTWARE. # # -------------------------------------------------------------------------- +import os from typing import Iterable, MutableSequence, Union import pytest from azure.core.pipeline import AsyncPipeline @@ -70,6 +71,8 @@ async def test_example_asyncio(): @pytest.mark.asyncio async def test_example_aiohttp(): + os.environ.pop("SSL_CERT_DIR", None) + os.environ.pop("REQUESTS_CA_BUNDLE", None) request = HttpRequest("GET", "https://bing.com") policies: Iterable[Union[AsyncHTTPPolicy, SansIOHTTPPolicy]] = [ UserAgentPolicy("myuseragent"), @@ -86,6 +89,8 @@ async def test_example_aiohttp(): @pytest.mark.asyncio async def test_example_async_pipeline(): + os.environ.pop("SSL_CERT_DIR", None) + os.environ.pop("REQUESTS_CA_BUNDLE", None) # [START build_async_pipeline] from azure.core.pipeline import AsyncPipeline from azure.core.pipeline.policies import AsyncRedirectPolicy, UserAgentPolicy @@ -108,6 +113,8 @@ async def test_example_async_pipeline(): @pytest.mark.asyncio async def test_example_async_pipeline_client(): + os.environ.pop("SSL_CERT_DIR", None) + os.environ.pop("REQUESTS_CA_BUNDLE", None) url = "https://bing.com" # [START build_async_pipeline_client] @@ -131,6 +138,8 @@ async def test_example_async_pipeline_client(): @pytest.mark.asyncio async def test_example_async_redirect_policy(): + os.environ.pop("SSL_CERT_DIR", None) + os.environ.pop("REQUESTS_CA_BUNDLE", None) url = "https://bing.com" request = HttpRequest("GET", url) @@ -159,6 +168,8 @@ async def test_example_async_redirect_policy(): @pytest.mark.asyncio async def test_example_async_retry_policy(): + os.environ.pop("SSL_CERT_DIR", None) + os.environ.pop("REQUESTS_CA_BUNDLE", None) url = "https://bing.com" request = HttpRequest("GET", "https://bing.com") policies: MutableSequence[Union[AsyncHTTPPolicy, SansIOHTTPPolicy]] = [ diff --git a/sdk/core/ci.yml b/sdk/core/ci.yml index 2f486f6901bb..d358db016420 100644 --- a/sdk/core/ci.yml +++ b/sdk/core/ci.yml @@ -38,6 +38,7 @@ extends: BuildTargetingString: "*" ValidateFormatting: true TestProxy: false + TestTimeoutInMinutes: 120 Artifacts: - name: azure-core safeName: azurecore diff --git a/sdk/core/corehttp/tests/async_tests/test_pipeline_async.py b/sdk/core/corehttp/tests/async_tests/test_pipeline_async.py index 007ef094d667..3e481311c146 100644 --- a/sdk/core/corehttp/tests/async_tests/test_pipeline_async.py +++ b/sdk/core/corehttp/tests/async_tests/test_pipeline_async.py @@ -22,6 +22,7 @@ import aiohttp import httpx import pytest +import os from utils import ASYNC_TRANSPORTS @@ -85,6 +86,8 @@ def on_response(self, request, response): @pytest.mark.asyncio @pytest.mark.parametrize("transport", ASYNC_TRANSPORTS) async def test_transport_socket_timeout(transport): + os.environ.pop("SSL_CERT_DIR", None) + os.environ.pop("REQUESTS_CA_BUNDLE", None) request = HttpRequest("GET", "https://bing.com") policies = [UserAgentPolicy("myusergant")] # Sometimes this will raise a read timeout, sometimes a socket timeout depending on timing. diff --git a/sdk/ml/azure-ai-ml/pyproject.toml b/sdk/ml/azure-ai-ml/pyproject.toml index ac1b99becca5..a46aa6bdd51f 100644 --- a/sdk/ml/azure-ai-ml/pyproject.toml +++ b/sdk/ml/azure-ai-ml/pyproject.toml @@ -3,7 +3,8 @@ mypy = true pyright = false type_check_samples = false pylint = true - +mindependency = false +latestdependency = false [tool.isort] profile = "black" @@ -18,7 +19,7 @@ extend_skip_glob = [ "*/.tox/*", ] -[tool.mypy] +[tool.mypy] python_version = "3.10" exclude = [ "azure/ai/ml/_restclient", @@ -32,8 +33,8 @@ exclude = [ "azure/ai/ml/_utils", "azure/ai/ml/exceptions.py", "azure/ai/ml/_exception_helper.py", -] -warn_unused_configs = true +] +warn_unused_configs = true follow_imports = "skip" ignore_missing_imports = true follow_imports_for_stubs = false diff --git a/sdk/pr.yml b/sdk/pr.yml new file mode 100644 index 000000000000..f7ae0940527c --- /dev/null +++ b/sdk/pr.yml @@ -0,0 +1,23 @@ +trigger: + branches: + exclude: + - pipelinev3-v2 + +pr: + branches: + include: + - pipelinev3* + +parameters: + - name: Service + type: string + default: auto + +extends: + template: ../eng/pipelines/templates/stages/archetype-sdk-client.yml + parameters: + ServiceDirectory: ${{ parameters.Service }} + BuildTargetingString: "*" + ValidateFormatting: true + TestProxy: true + TestTimeOutInMinutes: 180 \ No newline at end of file diff --git a/tools/azure-sdk-tools/ci_tools/build.py b/tools/azure-sdk-tools/ci_tools/build.py index dabdb11cb021..be83cfe1eb6a 100644 --- a/tools/azure-sdk-tools/ci_tools/build.py +++ b/tools/azure-sdk-tools/ci_tools/build.py @@ -9,9 +9,6 @@ from ci_tools.variables import discover_repo_root, get_artifact_directory from ci_tools.versioning.version_shared import set_version_py, set_dev_classifier from ci_tools.versioning.version_set_dev import get_dev_version, format_build_id -from ci_tools.logging import initialize_logger, run_logged - -logger = initialize_logger("build.py") def build() -> None: parser = argparse.ArgumentParser( @@ -100,13 +97,13 @@ def build() -> None: repo_root = discover_repo_root(args.repo) - if args.service: + if args.service and args.service != "auto": service_dir = os.path.join("sdk", args.service) target_dir = os.path.join(repo_root, service_dir) else: target_dir = repo_root - logger.debug(f"Searching for packages starting from {target_dir} with glob string {args.glob_string} and package filter {args.package_filter_string}") + logging.debug(f"Searching for packages starting from {target_dir} with glob string {args.glob_string} and package filter {args.package_filter_string}") targeted_packages = discover_targeted_packages( args.glob_string, @@ -150,7 +147,7 @@ def build_packages( build_apiview_artifact: bool = False, build_id: str = "", ): - logger.log(level=logging.INFO, msg=f"Generating {targeted_packages} using python{sys.version}") + logging.log(level=logging.INFO, msg=f"Generating {targeted_packages} using python{sys.version}") for package_root in targeted_packages: setup_parsed = ParsedSetup.from_path(package_root) @@ -162,7 +159,7 @@ def build_packages( new_version = get_dev_version(setup_parsed.version, build_id) - logger.log(level=logging.DEBUG, msg=f"{setup_parsed.name}: {setup_parsed.version} -> {new_version}") + logging.log(level=logging.DEBUG, msg=f"{setup_parsed.name}: {setup_parsed.version} -> {new_version}") set_version_py(setup_parsed.setup_filename, new_version) set_dev_classifier(setup_parsed.setup_filename, new_version) diff --git a/tools/azure-sdk-tools/ci_tools/functions.py b/tools/azure-sdk-tools/ci_tools/functions.py index c94f6f3a11d8..db242d6658ec 100644 --- a/tools/azure-sdk-tools/ci_tools/functions.py +++ b/tools/azure-sdk-tools/ci_tools/functions.py @@ -42,6 +42,9 @@ ] TEST_COMPATIBILITY_MAP = {} +TEST_PYTHON_DISTRO_INCOMPATIBILITY_MAP = { + "azure-storage-blob": "pypy" +} omit_regression = ( lambda x: "nspkg" not in x @@ -104,7 +107,12 @@ def apply_compatibility_filter(package_set: List[str]) -> List[str]: if pkg_specs_override: spec_set = SpecifierSet(pkg_specs_override) - if running_major_version in spec_set: + distro_compat = True + distro_incompat = TEST_PYTHON_DISTRO_INCOMPATIBILITY_MAP.get(os.path.basename(pkg), None) + if distro_incompat and distro_incompat in platform.python_implementation().lower(): + distro_compat = False + + if running_major_version in spec_set and distro_compat: collected_packages.append(pkg) logging.debug("Target packages after applying compatibility filter: {}".format(collected_packages)) @@ -201,19 +209,23 @@ def discover_targeted_packages( # glob the starting package set collected_packages = glob_packages(glob_string, target_root_dir) + logging.info(f"Results for glob_string \"{glob_string}\" and root directory \"{target_root_dir}\" are: {collected_packages}") # apply the additional contains filter collected_packages = [pkg for pkg in collected_packages if additional_contains_filter in pkg] + logging.info(f"Results after additional contains filter: \"{additional_contains_filter}\" {collected_packages}") # filter for compatibility, this means excluding a package that doesn't support py36 when we are running a py36 executable if compatibility_filter: collected_packages = apply_compatibility_filter(collected_packages) + logging.info(f"Results after compatibility filter: {collected_packages}") if not include_inactive: collected_packages = apply_inactive_filter(collected_packages) # Apply filter based on filter type. for e.g. Docs, Regression, Management collected_packages = apply_business_filter(collected_packages, filter_type) + logging.info(f"Results after business filter: {collected_packages}") return sorted(collected_packages)