From 6814ae83bd1e6368174b8f5265ec052d6c7e5a62 Mon Sep 17 00:00:00 2001 From: "Scott Beddall (from Dev Box)" Date: Wed, 10 Jul 2024 14:05:55 -0700 Subject: [PATCH 1/9] original draft of the adjustments. definitely going to be some discussion on the serviceInput parameter. probably want to go to a separate script entirely? add missing line --- eng/common/scripts/Generate-PR-Diff.ps1 | 2 +- eng/common/scripts/Package-Properties.ps1 | 14 +++++++++++++ .../scripts/Save-Package-Properties.ps1 | 21 ++++++++++++++----- eng/common/scripts/common.ps1 | 1 + 4 files changed, 32 insertions(+), 6 deletions(-) diff --git a/eng/common/scripts/Generate-PR-Diff.ps1 b/eng/common/scripts/Generate-PR-Diff.ps1 index 8239b219c37..d84f9e15ca7 100644 --- a/eng/common/scripts/Generate-PR-Diff.ps1 +++ b/eng/common/scripts/Generate-PR-Diff.ps1 @@ -45,7 +45,7 @@ $changedServices = Get-ChangedServices -ChangedFiles $changedFiles $result = [PSCustomObject]@{ "ChangedFiles" = $changedFiles "ChangedServices" = $changedServices - "PRNumber" = $env:SYSTEM_PULLREQUEST_PULLREQUESTNUMBER + "PRNumber" = if ($env:SYSTEM_PULLREQUEST_PULLREQUESTNUMBER) { $env:SYSTEM_PULLREQUEST_PULLREQUESTNUMBER } else { "-1" } } $result | ConvertTo-Json | Out-File $ArtifactName diff --git a/eng/common/scripts/Package-Properties.ps1 b/eng/common/scripts/Package-Properties.ps1 index 606998c8ea3..01318850407 100644 --- a/eng/common/scripts/Package-Properties.ps1 +++ b/eng/common/scripts/Package-Properties.ps1 @@ -105,6 +105,20 @@ function Get-PkgProperties return $null } +function Get-PrPkgProperties([string]$InputDiffJson) { + $pkgPropsResult = @() + + Write-Host "In Get-PRPkgProperties function" + + if ($GetPRPackageInfoFromRepoFn -and (Test-Path "Function:$GetPRPackageInfoFromRepoFn")) + { + Write-Host "Attempting to invoke the language one" + $pkgPropsResult = &$GetPRPackageInfoFromRepoFn -InputDiffJson $InputDiffJson + } + + return $pkgPropsResult +} + # Takes ServiceName and Repo Root Directory # Returns important properties for each package in the specified service, or entire repo if the serviceName is not specified # Returns a Table of service key to array values of PS Object with properties @ { pkgName, pkgVersion, pkgDirectoryPath, pkgReadMePath, pkgChangeLogPath } diff --git a/eng/common/scripts/Save-Package-Properties.ps1 b/eng/common/scripts/Save-Package-Properties.ps1 index 0e8ce26aa65..faf6640f64f 100644 --- a/eng/common/scripts/Save-Package-Properties.ps1 +++ b/eng/common/scripts/Save-Package-Properties.ps1 @@ -14,8 +14,8 @@ In cases of collisions where track 2 packages (IsNewSdk = true) have the same filename as track 1 packages (e.g. same artifact name or package name), the track 2 package properties will be written. -.PARAMETER serviceDirectory -Service directory in which to search for packages +.PARAMETER serviceInput +Service directory in which to search for packages, or file path ending in diff.json. .PARAMETER outDirectory Output location (generally a package artifact directory in DevOps) for JSON @@ -33,7 +33,7 @@ Verison property in that file. [CmdletBinding()] Param ( [Parameter(Mandatory=$True)] - [string] $serviceDirectory, + [string] $serviceInput, [Parameter(Mandatory=$True)] [string] $outDirectory, [switch] $addDevVersion @@ -92,7 +92,18 @@ function GetRelativePath($path) { } $exportedPaths = @{} -$allPackageProperties = Get-AllPkgProperties $serviceDirectory + +$allPackageProperties = @() + +if ($serviceInput.endswith("diff.json")) { + $allPackageProperties = Get-PrPkgProperties $serviceInput +} +else { + $allPackageProperties = Get-AllPkgProperties $serviceInput +} + +Write-Host $allPackageProperties + if ($allPackageProperties) { if (-not (Test-Path -Path $outDirectory)) @@ -137,6 +148,6 @@ if ($allPackageProperties) } else { - Write-Error "Package properties are not available for service directory $($serviceDirectory)" + Write-Error "Package properties are not available for service directory $($serviceInput)" exit 1 } diff --git a/eng/common/scripts/common.ps1 b/eng/common/scripts/common.ps1 index 0025d1c6fe6..d53c3b1655c 100644 --- a/eng/common/scripts/common.ps1 +++ b/eng/common/scripts/common.ps1 @@ -42,6 +42,7 @@ if (!(Get-Variable -Name "LanguageDisplayName" -ValueOnly -ErrorAction "Ignore") # Transformed Functions # Expected to be set in eng/scripts/Language-Settings.ps1 $GetPackageInfoFromRepoFn = "Get-${Language}-PackageInfoFromRepo" +$GetPRPackageInfoFromRepoFn = "Get-${Language}-PRPackageInfoFromRepo" $GetPackageInfoFromPackageFileFn = "Get-${Language}-PackageInfoFromPackageFile" $PublishGithubIODocsFn = "Publish-${Language}-GithubIODocs" $UpdateDocsMsPackagesFn = "Update-${Language}-DocsMsPackages" From 785bc68637b28e44c56b5769d1dabff907f7cea9 Mon Sep 17 00:00:00 2001 From: "Scott Beddall (from Dev Box)" Date: Mon, 15 Jul 2024 14:57:54 -0700 Subject: [PATCH 2/9] admendments as suggested by Wes --- eng/common/scripts/Package-Properties.ps1 | 30 ++++++++++++++++++----- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/eng/common/scripts/Package-Properties.ps1 b/eng/common/scripts/Package-Properties.ps1 index 01318850407..a9915101c56 100644 --- a/eng/common/scripts/Package-Properties.ps1 +++ b/eng/common/scripts/Package-Properties.ps1 @@ -105,18 +105,36 @@ function Get-PkgProperties return $null } +function Get-PackageFolderFromPath($file) { + $parts = $file -split "/" + + if ($parts.Length -lt 3) { + return "" + } + return [System.IO.Path]::Combine($parts[0], $parts[1], $parts[2]) +} + +# Takes an input diff json file generated from eng/common/scripts/Generate-PR-Diff.ps1 +# On PR builds, this function should be utilized instead of Get-AllPkgProperties. function Get-PrPkgProperties([string]$InputDiffJson) { - $pkgPropsResult = @() + $packagesWithChanges = @() - Write-Host "In Get-PRPkgProperties function" + $allPackageProperties = Get-AllPkgProperties + $diff = Get-Content $InputDiffJson | ConvertFrom-Json + $targetedFiles = $diff.ChangedFiles - if ($GetPRPackageInfoFromRepoFn -and (Test-Path "Function:$GetPRPackageInfoFromRepoFn")) + foreach($pkg in $allPackageProperties) { - Write-Host "Attempting to invoke the language one" - $pkgPropsResult = &$GetPRPackageInfoFromRepoFn -InputDiffJson $InputDiffJson + foreach($file in $targetedFiles) + { + $pkgFolder = Get-PackageFolderFromPath($file) + if ($pkgFolder.EndsWith($pkg.ArtifactName)) { + $packagesWithChanges += $pkg + } + } } - return $pkgPropsResult + return $packagesWithChanges } # Takes ServiceName and Repo Root Directory From 43e2a878dff39c85d8c82fd5e1263f6cdc47c54f Mon Sep 17 00:00:00 2001 From: "Scott Beddall (from Dev Box)" Date: Mon, 15 Jul 2024 15:01:13 -0700 Subject: [PATCH 3/9] remove new function ref in common.ps1 --- eng/common/scripts/common.ps1 | 1 - 1 file changed, 1 deletion(-) diff --git a/eng/common/scripts/common.ps1 b/eng/common/scripts/common.ps1 index d53c3b1655c..0025d1c6fe6 100644 --- a/eng/common/scripts/common.ps1 +++ b/eng/common/scripts/common.ps1 @@ -42,7 +42,6 @@ if (!(Get-Variable -Name "LanguageDisplayName" -ValueOnly -ErrorAction "Ignore") # Transformed Functions # Expected to be set in eng/scripts/Language-Settings.ps1 $GetPackageInfoFromRepoFn = "Get-${Language}-PackageInfoFromRepo" -$GetPRPackageInfoFromRepoFn = "Get-${Language}-PRPackageInfoFromRepo" $GetPackageInfoFromPackageFileFn = "Get-${Language}-PackageInfoFromPackageFile" $PublishGithubIODocsFn = "Publish-${Language}-GithubIODocs" $UpdateDocsMsPackagesFn = "Update-${Language}-DocsMsPackages" From 55c50379022f408c40fb937bc6eb8f522fdc0b4e Mon Sep 17 00:00:00 2001 From: "Scott Beddall (from Dev Box)" Date: Tue, 16 Jul 2024 10:52:06 -0700 Subject: [PATCH 4/9] remove debug statements --- eng/common/scripts/Save-Package-Properties.ps1 | 2 -- 1 file changed, 2 deletions(-) diff --git a/eng/common/scripts/Save-Package-Properties.ps1 b/eng/common/scripts/Save-Package-Properties.ps1 index faf6640f64f..8fea77067e2 100644 --- a/eng/common/scripts/Save-Package-Properties.ps1 +++ b/eng/common/scripts/Save-Package-Properties.ps1 @@ -102,8 +102,6 @@ else { $allPackageProperties = Get-AllPkgProperties $serviceInput } -Write-Host $allPackageProperties - if ($allPackageProperties) { if (-not (Test-Path -Path $outDirectory)) From 857869145282ff3c7971c4639287f4f8cfb6b471 Mon Sep 17 00:00:00 2001 From: "Scott Beddall (from Dev Box)" Date: Tue, 16 Jul 2024 11:34:02 -0700 Subject: [PATCH 5/9] adjustments to call Save-Package-Properties from other eng/common calls --- .../pipelines/templates/steps/daily-dev-build-variable.yml | 2 +- eng/common/scripts/Verify-RestApiSpecLocation.ps1 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/common/pipelines/templates/steps/daily-dev-build-variable.yml b/eng/common/pipelines/templates/steps/daily-dev-build-variable.yml index 37efd0bd031..ed34d8e64f2 100644 --- a/eng/common/pipelines/templates/steps/daily-dev-build-variable.yml +++ b/eng/common/pipelines/templates/steps/daily-dev-build-variable.yml @@ -9,7 +9,7 @@ steps: inputs: filePath: $(Build.SourcesDirectory)/eng/common/scripts/Save-Package-Properties.ps1 arguments: > - -ServiceDirectory ${{parameters.ServiceDirectory}} + -ServiceInput ${{parameters.ServiceDirectory}} -OutDirectory $(Build.ArtifactStagingDirectory)/PackageInfo pwsh: true workingDirectory: $(Pipeline.Workspace) diff --git a/eng/common/scripts/Verify-RestApiSpecLocation.ps1 b/eng/common/scripts/Verify-RestApiSpecLocation.ps1 index 8a420bd769f..f5bb1cef587 100644 --- a/eng/common/scripts/Verify-RestApiSpecLocation.ps1 +++ b/eng/common/scripts/Verify-RestApiSpecLocation.ps1 @@ -169,7 +169,7 @@ function Verify-PackageVersion() { if (-not (Test-Path -Path $PackageInfoDirectory)) { # Call Save-Package-Properties.ps1 script to generate package info json files $savePropertiesScriptPath = Join-Path -Path $PSScriptRoot "Save-Package-Properties.ps1" - & $savePropertiesScriptPath -serviceDirectory $ServiceDirectory -outDirectory $PackageInfoDirectory + & $savePropertiesScriptPath -serviceInput $ServiceDirectory -outDirectory $PackageInfoDirectory } } $pkgPropPath = Join-Path -Path $PackageInfoDirectory "$PackageName.json" From 1f8c1d48331ae71c121591a872d34a8deaac0d2c Mon Sep 17 00:00:00 2001 From: "Scott Beddall (from Dev Box)" Date: Wed, 17 Jul 2024 12:11:21 -0700 Subject: [PATCH 6/9] just check for file under pkg folder --- .../steps/daily-dev-build-variable.yml | 2 +- eng/common/scripts/Package-Properties.ps1 | 18 ++++-------------- eng/common/scripts/Save-Package-Properties.ps1 | 17 ++++++++++------- .../scripts/Verify-RestApiSpecLocation.ps1 | 2 +- 4 files changed, 16 insertions(+), 23 deletions(-) diff --git a/eng/common/pipelines/templates/steps/daily-dev-build-variable.yml b/eng/common/pipelines/templates/steps/daily-dev-build-variable.yml index ed34d8e64f2..37efd0bd031 100644 --- a/eng/common/pipelines/templates/steps/daily-dev-build-variable.yml +++ b/eng/common/pipelines/templates/steps/daily-dev-build-variable.yml @@ -9,7 +9,7 @@ steps: inputs: filePath: $(Build.SourcesDirectory)/eng/common/scripts/Save-Package-Properties.ps1 arguments: > - -ServiceInput ${{parameters.ServiceDirectory}} + -ServiceDirectory ${{parameters.ServiceDirectory}} -OutDirectory $(Build.ArtifactStagingDirectory)/PackageInfo pwsh: true workingDirectory: $(Pipeline.Workspace) diff --git a/eng/common/scripts/Package-Properties.ps1 b/eng/common/scripts/Package-Properties.ps1 index a9915101c56..cec8898d539 100644 --- a/eng/common/scripts/Package-Properties.ps1 +++ b/eng/common/scripts/Package-Properties.ps1 @@ -105,21 +105,10 @@ function Get-PkgProperties return $null } -function Get-PackageFolderFromPath($file) { - $parts = $file -split "/" - - if ($parts.Length -lt 3) { - return "" - } - return [System.IO.Path]::Combine($parts[0], $parts[1], $parts[2]) -} - -# Takes an input diff json file generated from eng/common/scripts/Generate-PR-Diff.ps1 -# On PR builds, this function should be utilized instead of Get-AllPkgProperties. function Get-PrPkgProperties([string]$InputDiffJson) { $packagesWithChanges = @() - $allPackageProperties = Get-AllPkgProperties + $allPackageProperties = Get-AllPkgProperties -ServiceDirectory media $diff = Get-Content $InputDiffJson | ConvertFrom-Json $targetedFiles = $diff.ChangedFiles @@ -127,8 +116,9 @@ function Get-PrPkgProperties([string]$InputDiffJson) { { foreach($file in $targetedFiles) { - $pkgFolder = Get-PackageFolderFromPath($file) - if ($pkgFolder.EndsWith($pkg.ArtifactName)) { + $filePath = Resolve-Path (Join-Path $RepoRoot $file) + $shouldInclude = $filePath -like "$($pkg.DirectoryPath)*" + if ($shouldInclude) { $packagesWithChanges += $pkg } } diff --git a/eng/common/scripts/Save-Package-Properties.ps1 b/eng/common/scripts/Save-Package-Properties.ps1 index 8fea77067e2..b27567f03c1 100644 --- a/eng/common/scripts/Save-Package-Properties.ps1 +++ b/eng/common/scripts/Save-Package-Properties.ps1 @@ -14,8 +14,11 @@ In cases of collisions where track 2 packages (IsNewSdk = true) have the same filename as track 1 packages (e.g. same artifact name or package name), the track 2 package properties will be written. -.PARAMETER serviceInput -Service directory in which to search for packages, or file path ending in diff.json. +.PARAMETER serviceDirectory +Service directory in which to search for packages. + +.PARAMETER prDiff +A file path leading to a file generated from Generate-PR-Diff.json. This parameter takes precedence over serviceDirectory, do not provide both. .PARAMETER outDirectory Output location (generally a package artifact directory in DevOps) for JSON @@ -32,10 +35,10 @@ Verison property in that file. [CmdletBinding()] Param ( - [Parameter(Mandatory=$True)] - [string] $serviceInput, + [string] $serviceDirectory, [Parameter(Mandatory=$True)] [string] $outDirectory, + [string] $prDiff, [switch] $addDevVersion ) @@ -95,11 +98,11 @@ $exportedPaths = @{} $allPackageProperties = @() -if ($serviceInput.endswith("diff.json")) { - $allPackageProperties = Get-PrPkgProperties $serviceInput +if ($prDiff) { + $allPackageProperties = Get-PrPkgProperties $prDiff } else { - $allPackageProperties = Get-AllPkgProperties $serviceInput + $allPackageProperties = Get-AllPkgProperties $serviceDirectory } if ($allPackageProperties) diff --git a/eng/common/scripts/Verify-RestApiSpecLocation.ps1 b/eng/common/scripts/Verify-RestApiSpecLocation.ps1 index f5bb1cef587..8a420bd769f 100644 --- a/eng/common/scripts/Verify-RestApiSpecLocation.ps1 +++ b/eng/common/scripts/Verify-RestApiSpecLocation.ps1 @@ -169,7 +169,7 @@ function Verify-PackageVersion() { if (-not (Test-Path -Path $PackageInfoDirectory)) { # Call Save-Package-Properties.ps1 script to generate package info json files $savePropertiesScriptPath = Join-Path -Path $PSScriptRoot "Save-Package-Properties.ps1" - & $savePropertiesScriptPath -serviceInput $ServiceDirectory -outDirectory $PackageInfoDirectory + & $savePropertiesScriptPath -serviceDirectory $ServiceDirectory -outDirectory $PackageInfoDirectory } } $pkgPropPath = Join-Path -Path $PackageInfoDirectory "$PackageName.json" From f715f26e7f335249aefa685e219d03f99fa83842 Mon Sep 17 00:00:00 2001 From: "Scott Beddall (from Dev Box)" Date: Wed, 17 Jul 2024 14:10:28 -0700 Subject: [PATCH 7/9] a debug value was left behind. remove it --- eng/common/scripts/Package-Properties.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/scripts/Package-Properties.ps1 b/eng/common/scripts/Package-Properties.ps1 index cec8898d539..040d012edb6 100644 --- a/eng/common/scripts/Package-Properties.ps1 +++ b/eng/common/scripts/Package-Properties.ps1 @@ -108,7 +108,7 @@ function Get-PkgProperties function Get-PrPkgProperties([string]$InputDiffJson) { $packagesWithChanges = @() - $allPackageProperties = Get-AllPkgProperties -ServiceDirectory media + $allPackageProperties = Get-AllPkgProperties $diff = Get-Content $InputDiffJson | ConvertFrom-Json $targetedFiles = $diff.ChangedFiles From ab5d4f359fc428ca8e88232da810f13d7489a57b Mon Sep 17 00:00:00 2001 From: Scott Beddall <45376673+scbedd@users.noreply.github.com> Date: Wed, 17 Jul 2024 14:14:06 -0700 Subject: [PATCH 8/9] Update eng/common/scripts/Save-Package-Properties.ps1 Co-authored-by: Wes Haggard --- eng/common/scripts/Save-Package-Properties.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/scripts/Save-Package-Properties.ps1 b/eng/common/scripts/Save-Package-Properties.ps1 index b27567f03c1..c234d8b28ed 100644 --- a/eng/common/scripts/Save-Package-Properties.ps1 +++ b/eng/common/scripts/Save-Package-Properties.ps1 @@ -149,6 +149,6 @@ if ($allPackageProperties) } else { - Write-Error "Package properties are not available for service directory $($serviceInput)" + Write-Error "Package properties are not available for service directory $serviceDirectory or $prdiff" exit 1 } From 4841183c9a54857a09c281aefb3c483b1480a246 Mon Sep 17 00:00:00 2001 From: "Scott Beddall (from Dev Box)" Date: Wed, 17 Jul 2024 14:19:56 -0700 Subject: [PATCH 9/9] resolve the path always --- eng/common/scripts/Package-Properties.ps1 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/eng/common/scripts/Package-Properties.ps1 b/eng/common/scripts/Package-Properties.ps1 index 040d012edb6..0e703a9d437 100644 --- a/eng/common/scripts/Package-Properties.ps1 +++ b/eng/common/scripts/Package-Properties.ps1 @@ -114,10 +114,12 @@ function Get-PrPkgProperties([string]$InputDiffJson) { foreach($pkg in $allPackageProperties) { + $pkgDirectory = Resolve-Path "$($pkg.DirectoryPath)" + foreach($file in $targetedFiles) { $filePath = Resolve-Path (Join-Path $RepoRoot $file) - $shouldInclude = $filePath -like "$($pkg.DirectoryPath)*" + $shouldInclude = $filePath -like "$pkgDirectory*" if ($shouldInclude) { $packagesWithChanges += $pkg }