Use a new branch for quing the TestPipeline, this should prevent chec…

eng/common/scripts/git-branch-push.ps1

@@ -27,25 +27,33 @@ param(
     [Parameter(Mandatory = $false)]
     [string] $PushArgs = "",
 
+    [string] $RemoteName = "azure-sdk-fork",
+
+    [Parameter(Mandatory = $false)]
+    [boolean] $SkipCommit = $false,
+
     [Parameter(Mandatory = $false)]
-    [boolean] $SkipCommit = $false
+    [boolean] $AmendCommit = $false
 )
 
 # This is necessay because of the janky git command output writing to stderr.
 # Without explicitly setting the ErrorActionPreference to continue the script
 # would fail the first time git wrote command output.
 $ErrorActionPreference = "Continue"
 
-Write-Host "git remote add azure-sdk-fork $GitUrl"
-git remote add azure-sdk-fork $GitUrl
-if ($LASTEXITCODE -ne 0)
+if (!(git remote | ? {$_ -eq $RemoteName}))
 {
-    Write-Error "Unable to add remote LASTEXITCODE=$($LASTEXITCODE), see command output above."
-    exit $LASTEXITCODE
+    Write-Host "git remote add $RemoteName $GitUrl"
+    git remote add $RemoteName $GitUrl
+    if ($LASTEXITCODE -ne 0)
+    {
+        Write-Error "Unable to add remote LASTEXITCODE=$($LASTEXITCODE), see command output above."
+        exit $LASTEXITCODE
+    }
 }
 
-Write-Host "git fetch azure-sdk-fork"
-git fetch azure-sdk-fork
+Write-Host "git fetch $RemoteName"
+git fetch $RemoteName
 if ($LASTEXITCODE -ne 0)
 {
     Write-Error "Unable to fetch remote LASTEXITCODE=$($LASTEXITCODE), see command output above."
@@ -61,8 +69,14 @@ if ($LASTEXITCODE -ne 0)
 }
 
 if (!$SkipCommit) {
-    Write-Host "git -c user.name=`"azure-sdk`" -c user.email=`"[email protected]`" commit -am `"$($CommitMsg)`""
-    git -c user.name="azure-sdk" -c user.email="[email protected]" commit -am "$($CommitMsg)"
+    if ($AmendCommit) {
+        Write-Host "git -c user.name=`"azure-sdk`" -c user.email=`"[email protected]`" commit -am `"$($CommitMsg)`""
+        git -c user.name="azure-sdk" -c user.email="[email protected]" commit --amend -am "$($CommitMsg)"
+    }
+    else {
+        Write-Host "git -c user.name=`"azure-sdk`" -c user.email=`"[email protected]`" commit --amend -am `"$($CommitMsg)`""
+        git -c user.name="azure-sdk" -c user.email="[email protected]" commit -am "$($CommitMsg)"
+    }
     if ($LASTEXITCODE -ne 0)
     {
         Write-Error "Unable to add files and create commit LASTEXITCODE=$($LASTEXITCODE), see command output above."
@@ -84,15 +98,15 @@ $tryNumber = 0
 do
 {
     $needsRetry = $false
-    Write-Host "git push azure-sdk-fork $PRBranchName $PushArgs"
-    git push azure-sdk-fork $PRBranchName $PushArgs
+    Write-Host "git push $RemoteName $PRBranchName $PushArgs"
+    git push $RemoteName $PRBranchName $PushArgs
     $tryNumber++
     if ($LASTEXITCODE -ne 0)
     {
         $needsRetry = $true
         Write-Host "Git push failed with LASTEXITCODE=$($LASTEXITCODE) Need to fetch and rebase: attempt number=$($tryNumber)"
-        Write-Host "git fetch azure-sdk-fork"
-        git fetch azure-sdk-fork
+        Write-Host "git fetch $RemoteName"
+        git fetch $RemoteName
         if ($LASTEXITCODE -ne 0)
         {
             Write-Error "Unable to fetch remote LASTEXITCODE=$($LASTEXITCODE), see command output above."
@@ -110,8 +124,8 @@ do
                 continue
             }
 
-            Write-Host "git reset --hard azure-sdk-fork/${PRBranchName}"
-            git reset --hard azure-sdk-fork/${PRBranchName}
+            Write-Host "git reset --hard $RemoteName/${PRBranchName}"
+            git reset --hard $RemoteName/${PRBranchName}
             if ($LASTEXITCODE -ne 0)
             {
                 Write-Error "Unable to hard reset branch LASTEXITCODE=$($LASTEXITCODE), see command output above."
@@ -127,6 +141,7 @@ do
                 exit $LASTEXITCODE
             }
 
+
             Write-Host "git add -A"
             git add -A
             if ($LASTEXITCODE -ne 0)

eng/common/scripts/modules/ChangeLog-Operations.psm1

@@ -1,6 +1,5 @@
 $RELEASE_TITLE_REGEX = "(?<releaseNoteTitle>^\#+.*(?<version>\b\d+\.\d+\.\d+([^0-9\s][^\s:]+)?)(\s(?<releaseStatus>\(Unreleased\)|\(\d{4}-\d{2}-\d{2}\)))?)"
 
-# Returns a Collection of changeLogEntry object containing changelog info for all version present in the gived CHANGELOG
 function Get-ChangeLogEntries {
   param (
     [Parameter(Mandatory = $true)]

eng/pipelines/eng-common-sync.yml

@@ -184,6 +184,20 @@ stages:
                         -AuthToken "$(azuresdk-github-pat)"
                       pwsh: true
 
+                  - ${{ each repo in parameters.Repos }}:
+                    - task: PowerShell@2
+                      displayName: Clean Up Sync Common Branches
+                      condition: succeeded()
+                      inputs:
+                        pwsh: true
+                        workingDirectory: $(System.DefaultWorkingDirectory)
+                        filePath: $(System.DefaultWorkingDirectory)/eng/common/scripts/Delete-RemoteBranches.ps1
+                        arguments: >
+                          -RepoOwner "Azure"
+                          -RepoName ${{ repo }}
+                          -BranchPrefix "sync-${{ parameters.DirectoryToSync }}-$(System.PullRequest.SourceBranch)-$(System.PullRequest.PullRequestNumber)"
+                          -AuthToken $(azuresdk-github-pat)
+
   - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}:
     - stage: CreateSync
       jobs:

eng/pipelines/templates/steps/sync-directory.yml

@@ -41,7 +41,7 @@ steps:
       workingDirectory: $(System.DefaultWorkingDirectory)
 
     - task: PowerShell@2
-      displayName: Push changes
+      displayName: Push changes for creating Sync PRs
       condition: and(succeeded(), eq(variables['HasChanges'], 'true'))
       inputs:
         pwsh: true
@@ -54,6 +54,20 @@ steps:
           -PushArgs "${{ parameters.PushArgs }}"
           -SkipCommit $${{parameters.SkipCheckingForChanges}}
 
+    - task: PowerShell@2
+      displayName: Push changes for Queueing Test Pipeline
+      condition: and(succeeded(), ne(variables['${{repo}}-template-definition-id'], ''))
+      inputs:
+        pwsh: true
+        workingDirectory: $(System.DefaultWorkingDirectory)/${{ repo }}
+        filePath: ${{ parameters.ScriptDirectory }}/git-branch-push.ps1
+        arguments: >
+          -PRBranchName "${{ parameters.UpstreamBranchName }}-ForTestPipeline"
+          -CommitMsg "${{ parameters.CommitMessage }}"
+          -GitUrl "https://$(azuresdk-github-pat)@github.com/${{ parameters.PROwner }}/${{ repo }}.git"
+          -PushArgs "${{ parameters.PushArgs }}"
+          -AmendCommit $True
+
     - task: PowerShell@2
       displayName: Queue test pipeline
       condition: and(succeeded(), ne(variables['${{repo}}-template-definition-id'], ''))
@@ -64,7 +78,7 @@ steps:
         arguments: >
           -Organization "azure-sdk"
           -Project "internal"
-          -SourceBranch "${{ parameters.UpstreamBranchName }}"
+          -SourceBranch "${{ parameters.UpstreamBranchName }}-ForTestPipeline"
           -DefinitionId "$(${{repo}}-template-definition-id)"
           -VsoQueuedPipelines "QueuedPipelines"
           -AuthToken "$(azuresdk-azure-sdk-devops-build-queuing-pat)"

eng/scripts/Verify-And-Merge-PRs.ps1

@@ -9,6 +9,7 @@ param(
 $ReadyForMerge = $true
 $mergablePRs = @()
 $headers = @{ }
+$RetryCount = 5
 
 if ($null -eq $ShouldMerge) {
   $ShouldMerge = $null -ne $AuthToken;
@@ -22,43 +23,60 @@ if ($AuthToken) {
 
 $PRData = Get-Content $PRDataArtifactPath
 # Confirm Mergability
-foreach ($prDataLine in $PRData)
-{
-  $repoOwner, $repoName, $prNumber = $prDataLine.Split(";")
 
-  $prApiUrl = "https://api.github.com/repos/${repoOwner}/${repoName}/pulls/${prNumber}"
-  $prUrl = "https://github.com/${repoOwner}/${repoName}/pull/${prNumber}"
-
-  try
+do 
+{
+  $unMergablePRs = @()
+  foreach ($prDataLine in $PRData)
   {
-    $response = Invoke-RestMethod -Headers $headers $prApiUrl
-    if ($response.merged) {
-      Write-Host "${prUrl} is merged."
-    }
-    elseif ($response.state -eq "closed") {
-      LogWarning "${prUrl} is closed. Please investigate why was not merged."
-      $ReadyForMerge = $false
-    }
-    elseif ($response.mergeable -and $response.mergeable_state -eq "clean") {
-      Write-Host "${prUrl} is ready to merge."
+    $repoOwner, $repoName, $prNumber = $prDataLine.Split(";")
 
-      $mergablePRs += @{ Url = $prApiUrl; HeadSHA = $response.head.sha }
-    }
-    elseif ($response.mergeable_state -ne "clean") {
-      LogWarning "${prUrl} is blocked ($($response.mergeable_state)). Please ensure all checks are green and reviewers have approved."
-      $ReadyForMerge = $false
+    $prApiUrl = "https://api.github.com/repos/${repoOwner}/${repoName}/pulls/${prNumber}"
+    $prUrl = "https://github.com/${repoOwner}/${repoName}/pull/${prNumber}"
+
+    try
+    {
+      $response = Invoke-RestMethod -Headers $headers $prApiUrl
+      if ($response.merged) {
+        Write-Host "${prUrl} is merged."
+      }
+      elseif ($response.state -eq "closed") {
+        LogWarning "${prUrl} is closed. Please investigate why was not merged."
+        $ReadyForMerge = $false
+      }
+      elseif ($response.mergeable -and $response.mergeable_state -eq "clean") {
+        Write-Host "${prUrl} is ready to merge."
+
+        $mergablePRs += @{ Url = $prApiUrl; HeadSHA = $response.head.sha }
+      }
+      elseif ($response.mergeable_state -ne "clean") {
+        LogWarning "${prUrl} is blocked ($($response.mergeable_state)). Please ensure all checks are green and reviewers have approved."
+        $ReadyForMerge = $false
+        $unMergablePRs += $prDataLine
+      }
+      else {
+        LogWarning "${prUrl} is in an unknown state please contact engineering system team to understand the state."
+        LogWarning $response
+        $ReadyForMerge = $false
+        $unMergablePRs += $prDataLine
+      }
     }
-    else {
-      LogWarning "${prUrl} is in an unknown state please contact engineering system team to understand the state."
-      LogWarning $response
-      $ReadyForMerge = $false
+    catch {
+      LogError "Invoke-RestMethod ${prApiUrl} failed with exception:`n$_"
+      exit 1
     }
   }
-  catch {
-    LogError "Invoke-RestMethod ${prApiUrl} failed with exception:`n$_"
-    exit 1
+  if ($unMergablePRs.Count -gt 0)
+  {
+    $PRData = $unMergablePRs
+    Start-Sleep -Seconds 30
+    $RetryCount--
+    LogDebug "Retrying merging for unmergable Prs"
+    $ReadyForMerge = $true
   }
 }
+while (($unMergablePRs.Count -gt 0) -and ($RetryCount -gt 0) -and $ShouldMerge)
+
 
 if (!$ReadyForMerge) {
   LogError "At least one sync PR is not able to be merged please investigate and then retry running this job to auto-merge them again"