diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/AzurePipelines/AzurePipelinesProcessor.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/AzurePipelines/AzurePipelinesProcessor.cs index 3aa446edf2b8..02ebe6020f52 100644 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/AzurePipelines/AzurePipelinesProcessor.cs +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/AzurePipelines/AzurePipelinesProcessor.cs @@ -6,6 +6,7 @@ using System.Linq; using System.Net; using System.Text; +using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; @@ -183,6 +184,30 @@ public async Task UploadBuildBlobsAsync(string account, Guid projectId, int buil await UploadBuildBlobAsync(account, build); } + public async Task GetBuildBlobNamesAsync(string projectName, DateTimeOffset minTime, DateTimeOffset maxTime, CancellationToken cancellationToken) + { + DateTimeOffset minDay = minTime.ToUniversalTime().Date; + DateTimeOffset maxDay = maxTime.ToUniversalTime().Date; + + DateTimeOffset[] days = Enumerable.Range(0, (int)(maxDay - minDay).TotalDays + 1) + .Select(offset => minDay.AddDays(offset)) + .ToArray(); + + List blobNames = []; + + foreach (DateTimeOffset day in days) + { + string blobPrefix = $"{projectName}/{day:yyyy/MM/dd}/"; + + await foreach (BlobItem blob in this.buildsContainerClient.GetBlobsAsync(prefix: blobPrefix, cancellationToken: cancellationToken)) + { + blobNames.Add(blob.Name); + } + } + + return [.. blobNames]; + } + public string GetBuildBlobName(Build build) { long changeTime = ((DateTimeOffset)build.LastChangedDate).ToUnixTimeSeconds(); diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/AzurePipelines/BuildCompleteQueueWorker.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/AzurePipelines/BuildCompleteQueueWorker.cs index dfb7b6a84f11..3608a16c8a9a 100644 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/AzurePipelines/BuildCompleteQueueWorker.cs +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/AzurePipelines/BuildCompleteQueueWorker.cs @@ -11,6 +11,7 @@ using Microsoft.ApplicationInsights; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; +using Newtonsoft.Json.Linq; namespace Azure.Sdk.Tools.PipelineWitness.AzurePipelines { diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/AzurePipelines/MissingAzurePipelineRunsWorker.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/AzurePipelines/MissingAzurePipelineRunsWorker.cs new file mode 100644 index 000000000000..4441c1f329b4 --- /dev/null +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/AzurePipelines/MissingAzurePipelineRunsWorker.cs @@ -0,0 +1,105 @@ +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Azure.Sdk.Tools.PipelineWitness.Configuration; +using Azure.Sdk.Tools.PipelineWitness.Services; +using Azure.Sdk.Tools.PipelineWitness.Services.WorkTokens; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.TeamFoundation.Build.WebApi; +using Microsoft.VisualStudio.Services.WebApi; + +namespace Azure.Sdk.Tools.PipelineWitness.AzurePipelines +{ + public class MissingAzurePipelineRunsWorker : PeriodicLockingBackgroundService + { + private readonly ILogger logger; + private readonly AzurePipelinesProcessor runProcessor; + private readonly BuildCompleteQueue buildCompleteQueue; + private readonly IOptions options; + private readonly EnhancedBuildHttpClient buildClient; + + public MissingAzurePipelineRunsWorker( + ILogger logger, + AzurePipelinesProcessor runProcessor, + IAsyncLockProvider asyncLockProvider, + VssConnection vssConnection, + BuildCompleteQueue buildCompleteQueue, + IOptions options) + : base( + logger, + asyncLockProvider, + options.Value.MissingPipelineRunsWorker) + { + this.logger = logger; + this.runProcessor = runProcessor; + this.buildCompleteQueue = buildCompleteQueue; + this.options = options; + + ArgumentNullException.ThrowIfNull(vssConnection); + + this.buildClient = vssConnection.GetClient(); + } + + protected override async Task ProcessAsync(CancellationToken cancellationToken) + { + var settings = this.options.Value; + + // search for builds that completed within this window + var buildMinTime = DateTimeOffset.UtcNow.Subtract(settings.MissingPipelineRunsWorker.LookbackPeriod); + var buildMaxTime = DateTimeOffset.UtcNow.Subtract(TimeSpan.FromHours(1)); + + foreach (string project in settings.Projects) + { + var knownBlobs = await this.runProcessor.GetBuildBlobNamesAsync(project, buildMinTime, buildMaxTime, cancellationToken); + + string continuationToken = null; + do + { + var completedBuilds = await this.buildClient.GetBuildsAsync2( + project, + minFinishTime: buildMinTime.DateTime, + maxFinishTime: buildMaxTime.DateTime, + statusFilter: BuildStatus.Completed, + continuationToken: continuationToken, + cancellationToken: cancellationToken); + + var skipCount = 0; + var enqueueCount = 0; + foreach (var build in completedBuilds) + { + var blobName = this.runProcessor.GetBuildBlobName(build); + + if (knownBlobs.Contains(blobName, StringComparer.InvariantCultureIgnoreCase)) + { + skipCount++; + continue; + } + + var queueMessage = new BuildCompleteQueueMessage + { + Account = settings.Account, + ProjectId = build.Project.Id, + BuildId = build.Id + }; + + this.logger.LogInformation("Enqueuing missing build {Project} {BuildId} for processing", build.Project.Name, build.Id); + await this.buildCompleteQueue.EnqueueMessageAsync(queueMessage); + enqueueCount++; + } + + this.logger.LogInformation("Enqueued {EnqueueCount} missing builds, skipped {SkipCount} existing builds in project {Project}", enqueueCount, skipCount, project); + + continuationToken = completedBuilds.ContinuationToken; + } while(!string.IsNullOrEmpty(continuationToken)); + } + } + + protected override Task ProcessExceptionAsync(Exception ex) + { + this.logger.LogError(ex, "Error processing missing builds"); + return Task.CompletedTask; + } + } +} diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Configuration/PeriodicProcessSettings.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Configuration/PeriodicProcessSettings.cs index 73164d3c2951..adce800d232f 100644 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Configuration/PeriodicProcessSettings.cs +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Configuration/PeriodicProcessSettings.cs @@ -24,6 +24,11 @@ public class PeriodicProcessSettings /// public TimeSpan CooldownPeriod { get; set; } + /// + /// Gets or sets the amount of history to process in each iteration + /// + public TimeSpan LookbackPeriod { get; set; } + /// /// Gets or sets the name of the distributed lock /// diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Configuration/PipelineWitnessSettings.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Configuration/PipelineWitnessSettings.cs index c67d59cc7340..f03340ba93e6 100644 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Configuration/PipelineWitnessSettings.cs +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Configuration/PipelineWitnessSettings.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; namespace Azure.Sdk.Tools.PipelineWitness.Configuration { @@ -85,6 +86,16 @@ public class PipelineWitnessSettings /// public PeriodicProcessSettings BuildDefinitionWorker { get; set; } + /// + /// Gets or sets the loops settins for the Missing Azure Pipline Runs worker + /// + public PeriodicProcessSettings MissingPipelineRunsWorker { get; set; } + + /// + /// Gets or sets the loops settins for the Missing GitHub Actions worker + /// + public PeriodicProcessSettings MissingGitHubActionsWorker { get; set; } + /// /// Gets or sets the artifact name used by the pipeline owners extraction build /// @@ -109,5 +120,15 @@ public class PipelineWitnessSettings /// Gets or sets the container to use for async locks /// public string CosmosAsyncLockContainer { get; set; } + + /// + /// Gets or sets the list of monitored GitHub repositories (Overrides GitHubRepositoriesSource) + /// + public string[] GitHubRepositories { get; set; } + + /// + /// Gets or sets the url for a list of monitored GitHub repositories + /// + public string GitHubRepositoriesSource { get; set; } } } diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Configuration/PostConfigureSettings.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Configuration/PostConfigureSettings.cs new file mode 100644 index 000000000000..16093eac8ec9 --- /dev/null +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Configuration/PostConfigureSettings.cs @@ -0,0 +1,50 @@ +using System; +using System.Linq; +using System.Net.Http; +using System.Net.Http.Json; +using System.Text.RegularExpressions; +using System.Threading; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace Azure.Sdk.Tools.PipelineWitness.Configuration; + +public class PostConfigureSettings : IPostConfigureOptions +{ + private readonly ILogger logger; + + public PostConfigureSettings(ILogger logger) + { + this.logger = logger; + } + + public void PostConfigure(string name, PipelineWitnessSettings options) + { + if (options.GitHubRepositories == null || options.GitHubRepositories.Length == 0) + { + options.GitHubRepositories = []; + + if (string.IsNullOrEmpty(options.GitHubRepositoriesSource)) + { + this.logger.LogWarning("No GitHubRepositories or GitHubRepositoriesSource configured"); + return; + } + + try + { + this.logger.LogInformation("Replacing settings property GitHubRepositories with values from {Source}", options.GitHubRepositoriesSource); + using var client = new HttpClient(); + + options.GitHubRepositories = client.GetFromJsonAsync(options.GitHubRepositoriesSource) + .ConfigureAwait(true) + .GetAwaiter() + .GetResult(); + } + catch (Exception ex) + { + this.logger.LogError(ex, "Error loading repository list from source"); + return; + } + } + } +} diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Controllers/GitHubEventsController.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Controllers/GitHubEventsController.cs index 191aeeb80b24..384a6f336993 100644 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Controllers/GitHubEventsController.cs +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Controllers/GitHubEventsController.cs @@ -83,16 +83,27 @@ private async Task ProcessWorkflowRunEventAsync() if (action == "completed") { - var queueMessage = new RunCompleteQueueMessage - { - Owner = eventMessage.GetProperty("repository").GetProperty("owner").GetProperty("login").GetString(), - Repository = eventMessage.GetProperty("repository").GetProperty("name").GetString(), - RunId = eventMessage.GetProperty("workflow_run").GetProperty("id").GetInt64(), - }; - - this.logger.LogInformation("Enqueuing GitHubRunCompleteMessage for {Owner}/{Repository} run {RunId}", queueMessage.Owner, queueMessage.Repository, queueMessage.RunId); + string owner = eventMessage.GetProperty("repository").GetProperty("owner").GetProperty("login").GetString(); + string repository = eventMessage.GetProperty("repository").GetProperty("name").GetString(); + long runId = eventMessage.GetProperty("workflow_run").GetProperty("id").GetInt64(); - await this.queueClient.SendMessageAsync(JsonSerializer.Serialize(queueMessage)); + if (this.settings.GitHubRepositories.Contains($"{owner}/{repository}", StringComparer.InvariantCultureIgnoreCase)) + { + this.logger.LogInformation("Enqueuing GitHubRunCompleteMessage for {Owner}/{Repository} run {RunId}", owner, repository, runId); + + var queueMessage = new RunCompleteQueueMessage + { + Owner = owner, + Repository = repository, + RunId = runId, + }; + + await this.queueClient.SendMessageAsync(JsonSerializer.Serialize(queueMessage)); + } + else + { + this.logger.LogInformation("Skipping message for unknown repostory {Owner}/{Repository}", owner, repository); + } } return Ok(); diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/GitHubActions/GitHubActionProcessor.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/GitHubActions/GitHubActionProcessor.cs index ecd69a8a50bf..6d1740cfda5f 100644 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/GitHubActions/GitHubActionProcessor.cs +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/GitHubActions/GitHubActionProcessor.cs @@ -66,6 +66,32 @@ public async Task ProcessAsync(string owner, string repository, long runId) } } + public async Task GetRunBlobNamesAsync(string repository, DateTimeOffset minTime, DateTimeOffset maxTime, CancellationToken cancellationToken) + { + DateTimeOffset minDay = minTime.ToUniversalTime().Date; + DateTimeOffset maxDay = maxTime.ToUniversalTime().Date; + + DateTimeOffset[] days = Enumerable.Range(0, (int)(maxDay - minDay).TotalDays + 1) + .Select(offset => minDay.AddDays(offset)) + .ToArray(); + + List blobNames = []; + + foreach (DateTimeOffset day in days) + { + string blobPrefix = $"{repository}/{day:yyyy/MM/dd}/".ToLower(); + + AsyncPageable blobs = this.runsContainerClient.GetBlobsAsync(prefix: blobPrefix, cancellationToken: cancellationToken); + + await foreach (BlobItem blob in blobs) + { + blobNames.Add(blob.Name); + } + } + + return blobNames.ToArray(); + } + public string GetRunBlobName(WorkflowRun run) { string repository = run.Repository.FullName; diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/GitHubActions/MissingGitHubActionsWorker.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/GitHubActions/MissingGitHubActionsWorker.cs new file mode 100644 index 000000000000..289cc1a05b2d --- /dev/null +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/GitHubActions/MissingGitHubActionsWorker.cs @@ -0,0 +1,107 @@ +using System; +using System.Linq; +using System.Net.Http; +using System.Net.Http.Json; +using System.Threading; +using System.Threading.Tasks; +using Azure.Sdk.Tools.PipelineWitness.Configuration; +using Azure.Sdk.Tools.PipelineWitness.Services; +using Azure.Sdk.Tools.PipelineWitness.Services.WorkTokens; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Octokit; + +namespace Azure.Sdk.Tools.PipelineWitness.GitHubActions +{ + public class MissingGitHubActionsWorker : PeriodicLockingBackgroundService + { + private readonly ILogger logger; + private readonly GitHubActionProcessor processor; + private readonly RunCompleteQueue queue; + private readonly IOptions options; + private readonly GitHubClient client; + + public MissingGitHubActionsWorker( + ILogger logger, + GitHubActionProcessor processor, + IAsyncLockProvider asyncLockProvider, + ICredentialStore credentials, + RunCompleteQueue queue, + IOptions options) + : base( + logger, + asyncLockProvider, + options.Value.MissingGitHubActionsWorker) + { + this.logger = logger; + this.processor = processor; + this.queue = queue; + this.options = options; + + if (credentials == null) + { + throw new ArgumentNullException(nameof(credentials)); + } + + this.client = new GitHubClient(new ProductHeaderValue("PipelineWitness", "1.0"), credentials); + } + + protected override async Task ProcessAsync(CancellationToken cancellationToken) + { + PipelineWitnessSettings settings = this.options.Value; + + var repositories = settings.GitHubRepositories; + + // search for builds that completed within this window + DateTimeOffset runMinTime = DateTimeOffset.UtcNow.Subtract(settings.MissingGitHubActionsWorker.LookbackPeriod); + DateTimeOffset runMaxTime = DateTimeOffset.UtcNow.Subtract(TimeSpan.FromHours(1)); + + foreach (string ownerAndRepository in repositories) + { + string owner = ownerAndRepository.Split('/')[0]; + string repository = ownerAndRepository.Split('/')[1]; + + string[] knownBlobs = await this.processor.GetRunBlobNamesAsync(ownerAndRepository, runMinTime, runMaxTime, cancellationToken); + + WorkflowRunsResponse listRunsResponse = await this.client.Actions.Workflows.Runs.List(owner, repository, new WorkflowRunsRequest + { + Created = $"{runMinTime:o}..{runMaxTime:o}", + Status = CheckRunStatusFilter.Completed, + }); + + var skipCount = 0; + var enqueueCount = 0; + + foreach (WorkflowRun run in listRunsResponse.WorkflowRuns) + { + var blobName = this.processor.GetRunBlobName(run); + + if (knownBlobs.Contains(blobName, StringComparer.InvariantCultureIgnoreCase)) + { + skipCount++; + continue; + } + + var queueMessage = new RunCompleteQueueMessage + { + Owner = owner, + Repository = repository, + RunId = run.Id + }; + + this.logger.LogInformation("Enqueuing missing run {Repository} {RunId} for processing", ownerAndRepository, run.Id); + await this.queue.EnqueueMessageAsync(queueMessage); + enqueueCount++; + } + + this.logger.LogInformation("Enqueued {EnqueueCount} missing runs, skipped {SkipCount} existing runs in repository {Repository}", enqueueCount, skipCount, ownerAndRepository); + } + } + + protected override Task ProcessExceptionAsync(Exception ex) + { + this.logger.LogError(ex, "Error processing missing builds"); + return Task.CompletedTask; + } + } +} diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Startup.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Startup.cs index a1dc3d65676d..06a6a7799285 100644 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Startup.cs +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Startup.cs @@ -37,6 +37,7 @@ public static void Configure(WebApplicationBuilder builder) builder.Services.Configure(settingsSection); builder.Services.AddSingleton(); builder.Services.AddSingleton, PostConfigureKeyVaultSettings>(); + builder.Services.AddSingleton, PostConfigureSettings>(); builder.Services.AddApplicationInsightsTelemetry(builder.Configuration); builder.Services.AddApplicationInsightsTelemetryProcessor(); @@ -66,6 +67,8 @@ public static void Configure(WebApplicationBuilder builder) builder.Services.AddHostedService(settings.GitHubActionRunsWorkerCount); builder.Services.AddHostedService(); + builder.Services.AddHostedService(); + builder.Services.AddHostedService(); } private static void AddHostedService(this IServiceCollection services, int instanceCount) where T : class, IHostedService diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Utilities/StringUtilities.cs b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Utilities/StringUtilities.cs index a41255d54f7d..bc666b63939b 100644 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Utilities/StringUtilities.cs +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/Utilities/StringUtilities.cs @@ -1,4 +1,5 @@ using System; +using System.Globalization; using System.Text.RegularExpressions; namespace Azure.Sdk.Tools.PipelineWitness.Utilities; diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/appsettings.development.json b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/appsettings.development.json index b31726f0d9ed..c4890017c2ac 100644 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/appsettings.development.json +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/appsettings.development.json @@ -19,11 +19,30 @@ "CosmosAccountUri": "https://pipelinewitnesstest.documents.azure.com", "GitHubWebhookSecret": "https://pipelinewitnesstest.vault.azure.net/secrets/github-webhook-validation-secret", "GitHubAccessToken": null, + + "GitHubRepositoriesSource": "https://raw.githubusercontent.com/Azure/azure-sdk-tools/users/pahallis/missing-build/tools/pipeline-witness/monitored-repos.json", + + "BuildCompleteWorkerCount": 1, + "GitHubActionRunsWorkerCount": 1, + "BuildDefinitionWorker": { + "Enabled": false, "LoopPeriod": "00:01:00", - "Enabled": true + "CooldownPeriod": "7.00:00:00" }, - "BuildCompleteWorkerCount": 1, - "GitHubActionRunsWorkerCount": 1 + + "MissingPipelineRunsWorker": { + "Enabled": true, + "LoopPeriod": "00:01:00", + "CooldownPeriod": "00:10:00", + "LookbackPeriod": "12:00:00" + }, + + "MissingGitHubActionsWorker": { + "Enabled": true, + "LoopPeriod": "00:01:00", + "CooldownPeriod": "00:10:00", + "LookbackPeriod": "12:00:00" + } } } diff --git a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/appsettings.json b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/appsettings.json index ff8cc53a7f6a..a3a970d5a9ec 100644 --- a/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/appsettings.json +++ b/tools/pipeline-witness/Azure.Sdk.Tools.PipelineWitness/appsettings.json @@ -29,11 +29,29 @@ "LockName": "BuildDefinitionWorker" }, + "MissingPipelineRunsWorker": { + "Enabled": true, + "LoopPeriod": "01:00:00", + "CooldownPeriod": "7.00:00:00", + "LookbackPeriod": "14.00:00:00", + "LockName": "MissingPipelineRunsWorker" + }, + + "MissingGitHubActionsWorker": { + "Enabled": true, + "LoopPeriod": "01:00:00", + "CooldownPeriod": "7.00:00:00", + "LookbackPeriod": "14.00:00:00", + "LockName": "MissingGitHubActionsWorker" + }, + "BuildCompleteQueueName": "azurepipelines-build-completed", "BuildCompleteWorkerCount": 10, "GitHubActionRunsQueueName": "github-actionrun-completed", "GitHubActionRunsWorkerCount": 10, + + "GitHubRepositoriesSource": "https://raw.githubusercontent.com/Azure/azure-sdk-tools/main/tools/pipeline-witness/monitored-repos.json", "GitHubWebhookSecret": "https://pipelinewitnessprod.vault.azure.net/secrets/github-webhook-validation-secret", "GitHubAccessToken": "https://pipelinewitnessprod.vault.azure.net/secrets/azuresdk-github-pat", "MessageLeasePeriod": "00:03:00",