refactor: wire up secondary service, with separate schedule

Ensure they share the same overlap identifier, so they are staggered.
iPromKnight · Nov 16, 2024 · dceb6cb · dceb6cb
1 parent 6402511
commit dceb6cb
Show file tree

Hide file tree

Showing 10 changed files with 69 additions and 22 deletions.
diff --git a/Zilean.sln.DotSettings.user b/Zilean.sln.DotSettings.user
@@ -1,13 +1,16 @@
 <wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ABulkConfig_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003FUsers_003FProm3theu5_003FAppData_003FRoaming_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fd6b09cbbbc4e7bf228e82618e4e0859c91be3b93407de754227ae1a771ea2a6_003FBulkConfig_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ACallSiteFactory_002Ecs_002Fl_003AC_0021_003FUsers_003FProm3theu5_003FAppData_003FRoaming_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FSourcesCache_003Ffc2027f7e776fc105cddb56b1a25eeb3895b3ae6f3aac854d786e63bd01f75e2_003FCallSiteFactory_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ACronExpression_002Ecs_002Fl_003AC_0021_003FUsers_003FProm3theu5_003FAppData_003FRoaming_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FSourcesCache_003F2c78f2a38e28ff68373887f2c71ed6595fcac3a938221853fc708d5ef51f739_003FCronExpression_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ADbContextBulkExtensions_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003FUsers_003FProm3theu5_003FAppData_003FRoaming_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fd1f8f68864fcaf754b3aefb6cb6777e56a17cfd9217db17992967674aa5cd1d_003FDbContextBulkExtensions_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ADbContextBulkTransaction_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003FUsers_003FProm3theu5_003FAppData_003FRoaming_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F1dfe90a67dda13ce28d4a1975cde7e7445f47ecbb8b4d73dd1bcc0c01acd88_003FDbContextBulkTransaction_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AExceptionDispatchInfo_002Ecs_002Fl_003AC_0021_003FUsers_003FProm3theu5_003FAppData_003FRoaming_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003Fc73b3c6c598640c592fd3c6fa226c286e90908_003F37_003F6c0b0159_003FExceptionDispatchInfo_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AExceptionDispatchInfo_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003FUsers_003FProm3theu5_003FAppData_003FRoaming_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fbd1d5c50194fea68ff3559c160230b0ab50f5acf4ce3061bffd6d62958e2182_003FExceptionDispatchInfo_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AHttpProtocol_002Ecs_002Fl_003AC_0021_003FUsers_003FProm3theu5_003FAppData_003FRoaming_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FSourcesCache_003F234a93d637d5e17f1de0ec244dc09d402281d71bab8d9d5bb507e14f1c26c9_003FHttpProtocol_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AIScheduledEventConfiguration_002Ecs_002Fl_003AC_0021_003FUsers_003FProm3theu5_003FAppData_003FRoaming_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FSourcesCache_003F1f1f4f12a09ac651d3b94226de6f7b5a6cda93c54e23d9707c1cf4a3c1a417_003FIScheduledEventConfiguration_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AJsonSerializer_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FUsers_003FProm3theu5_003FAppData_003FRoaming_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F5148e388a7db4994ad2ab3750386454116e910_003F72_003Fc63c4ed4_003FJsonSerializer_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AOpenApiServiceCollectionExtensions_002Ecs_002Fl_003AC_0021_003FUsers_003FProm3theu5_003FAppData_003FRoaming_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FSourcesCache_003Fba408bbefe5c813576b4417369c21d24248b94d040ec72a9753bc546f9308190_003FOpenApiServiceCollectionExtensions_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AScheduledEvent_002Ecs_002Fl_003AC_0021_003FUsers_003FProm3theu5_003FAppData_003FRoaming_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FSourcesCache_003Fb691f6facce7874b361c15a72a4e2181eaa3456f6484b161d7313b6321136d3_003FScheduledEvent_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AServiceCollection_002Ecs_002Fl_003AC_0021_003FUsers_003FProm3theu5_003FAppData_003FRoaming_003FJetBrains_003FRider2024_002E3_003Fresharper_002Dhost_003FSourcesCache_003F595e4f2361fd31cb66c7e9303f46cf8f2fe77d96eea9d0e42c17cae783aa74_003FServiceCollection_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
 
 

diff --git a/src/Zilean.ApiService/Features/Bootstrapping/ServiceCollectionExtensions.cs b/src/Zilean.ApiService/Features/Bootstrapping/ServiceCollectionExtensions.cs
@@ -9,14 +9,15 @@ public static IServiceCollection AddSwaggerSupport(this IServiceCollection servi
     public static IServiceCollection AddSchedulingSupport(this IServiceCollection services) =>
         services.AddScheduler();
 
-    public static IServiceCollection AddDataBootStrapping(this IServiceCollection services) =>
-        services.AddHostedService<BootstrapIndexesService>();
+    public static IServiceCollection AddStartupHostedService(this IServiceCollection services) =>
+        services.AddHostedService<StartupService>();
 
     public static IServiceCollection ConditionallyRegisterDmmJob(this IServiceCollection services, ZileanConfiguration configuration)
     {
         if (configuration.Dmm.EnableScraping)
         {
-            services.AddTransient<SyncJob>();
+            services.AddTransient<DmmSyncJob>();
+            services.AddTransient<GenericSyncJob>();
             services.AddSingleton<SyncOnDemandState>();
         }
 
@@ -29,9 +30,16 @@ public static IServiceProvider SetupScheduling(this IServiceProvider provider, Z
             {
                 if (configuration.Dmm.EnableScraping)
                 {
-                    scheduler.Schedule<SyncJob>()
+                    scheduler.Schedule<DmmSyncJob>()
                         .Cron(configuration.Dmm.ScrapeSchedule)
-                        .PreventOverlapping(nameof(SyncJob));
+                        .PreventOverlapping("SyncJobs");
+                }
+
+                if (configuration.Ingestion.EnableScraping)
+                {
+                    scheduler.Schedule<GenericSyncJob>()
+                        .Cron(configuration.Ingestion.ScrapeSchedule)
+                        .PreventOverlapping("SyncJobs");
                 }
             })
             .LogScheduledTaskProgress();

diff --git a/.../Bootstrapping/BootstrapIndexesService.cs → .../Features/Bootstrapping/StartupService.cs b/.../Bootstrapping/BootstrapIndexesService.cs → .../Features/Bootstrapping/StartupService.cs
@@ -1,6 +1,6 @@
 namespace Zilean.ApiService.Features.Bootstrapping;
 
-public class BootstrapIndexesService(
+public class StartupService(
     ZileanConfiguration configuration,
     IShellExecutionService executionService,
     IServiceProvider serviceProvider,
@@ -12,7 +12,7 @@ public class BootstrapIndexesService(
 
     public async Task StartingAsync(CancellationToken cancellationToken)
     {
-        var logger = loggerFactory.CreateLogger<BootstrapIndexesService>();
+        var logger = loggerFactory.CreateLogger<StartupService>();
         logger.LogInformation("Applying Migrations...");
         await using var asyncScope = serviceProvider.CreateAsyncScope();
         var dbContext = asyncScope.ServiceProvider.GetRequiredService<ZileanDbContext>();
@@ -30,7 +30,7 @@ public async Task StartedAsync(CancellationToken cancellationToken)
         {
             await using var asyncScope = serviceProvider.CreateAsyncScope();
             var dbContext = asyncScope.ServiceProvider.GetRequiredService<ZileanDbContext>();
-            var dmmJob = new SyncJob(executionService, loggerFactory.CreateLogger<SyncJob>(), dbContext);
+            var dmmJob = new DmmSyncJob(executionService, loggerFactory.CreateLogger<DmmSyncJob>(), dbContext);
             var shouldRun = await dmmJob.ShouldRunOnStartup();
             if (shouldRun)
             {

diff --git a/src/Zilean.ApiService/Features/Search/SearchEndpoints.cs b/src/Zilean.ApiService/Features/Search/SearchEndpoints.cs
@@ -34,29 +34,29 @@ private static RouteGroupBuilder Dmm(this RouteGroupBuilder group)
         return group;
     }
 
-    private static async Task PerformOnDemandScrape(HttpContext context, ILogger<GeneralInstance> logger, IShellExecutionService executionService, ILogger<SyncJob> syncLogger, IMutex mutex, SyncOnDemandState state, ZileanDbContext dbContext)
+    private static async Task PerformOnDemandScrape(HttpContext context, ILogger<GeneralInstance> logger, IShellExecutionService executionService, ILogger<DmmSyncJob> syncLogger, IMutex mutex, SyncOnDemandState state, ZileanDbContext dbContext)
     {
         if (state.IsRunning)
         {
             logger.LogWarning("On-demand scrape already running.");
             return;
         }
 
-        logger.LogInformation("Trying to schedule on-demand scrape with a 5 minute timeout on lock acquisition.");
+        logger.LogInformation("Trying to schedule on-demand scrape with a 1 minute timeout on lock acquisition.");
 
-        bool available = mutex.TryGetLock(nameof(SyncJob), 1);
+        bool available = mutex.TryGetLock(nameof(DmmSyncJob), 1);
 
         if(available)
         {
             try
             {
                 logger.LogInformation("On-demand scrape mutex lock acquired.");
                 state.IsRunning = true;
-                await new SyncJob(executionService, syncLogger, dbContext).Invoke();
+                await new DmmSyncJob(executionService, syncLogger, dbContext).Invoke();
             }
             finally
             {
-                mutex.Release(nameof(SyncJob));
+                mutex.Release(nameof(DmmSyncJob));
                 state.IsRunning = false;
             }
 

diff --git a/...ilean.ApiService/Features/Sync/SyncJob.cs → ...an.ApiService/Features/Sync/DmmSyncJob.cs b/...ilean.ApiService/Features/Sync/SyncJob.cs → ...an.ApiService/Features/Sync/DmmSyncJob.cs
@@ -1,21 +1,26 @@
 namespace Zilean.ApiService.Features.Sync;
 
-public class SyncJob(IShellExecutionService shellExecutionService, ILogger<SyncJob> logger, ZileanDbContext dbContext) : IInvocable, ICancellableInvocable
+public class DmmSyncJob(IShellExecutionService shellExecutionService, ILogger<DmmSyncJob> logger, ZileanDbContext dbContext) : IInvocable, ICancellableInvocable
 {
     public CancellationToken CancellationToken { get; set; }
+    private const string DmmSyncArg = "dmm-sync";
 
     public async Task Invoke()
     {
-        logger.LogInformation("SyncJob started");
+        logger.LogInformation("Dmm SyncJob started");
+
+        var argumentBuilder = ArgumentsBuilder.Create();
+        argumentBuilder.AppendArgument(DmmSyncArg, string.Empty, false, false);
 
         await shellExecutionService.ExecuteCommand(new ShellCommandOptions
         {
             Command = Path.Combine(AppContext.BaseDirectory, "scraper"),
+            ArgumentsBuilder = argumentBuilder,
             ShowOutput = true,
             CancellationToken = CancellationToken
         });
 
-        logger.LogInformation("SyncJob completed");
+        logger.LogInformation("Dmm SyncJob completed");
     }
 
     // ReSharper disable once MethodSupportsCancellation

diff --git a/src/Zilean.ApiService/Features/Sync/GenericSyncJob.cs b/src/Zilean.ApiService/Features/Sync/GenericSyncJob.cs
@@ -0,0 +1,28 @@
+namespace Zilean.ApiService.Features.Sync;
+
+public class GenericSyncJob(IShellExecutionService shellExecutionService, ILogger<GenericSyncJob> logger, ZileanDbContext dbContext) : IInvocable, ICancellableInvocable
+{
+    public CancellationToken CancellationToken { get; set; }
+    private const string GenericSyncArg = "generic-sync";
+
+    public async Task Invoke()
+    {
+        logger.LogInformation("Generic SyncJob started");
+
+        var argumentBuilder = ArgumentsBuilder.Create();
+        argumentBuilder.AppendArgument(GenericSyncArg, string.Empty, false, false);
+
+        await shellExecutionService.ExecuteCommand(new ShellCommandOptions
+        {
+            Command = Path.Combine(AppContext.BaseDirectory, "scraper"),
+            ArgumentsBuilder = argumentBuilder,
+            ShowOutput = true,
+            CancellationToken = CancellationToken
+        });
+
+        logger.LogInformation("Generic SyncJob completed");
+    }
+
+    // ReSharper disable once MethodSupportsCancellation
+    public Task<bool> ShouldRunOnStartup() => dbContext.ParsedPages.AnyAsync();
+}
diff --git a/src/Zilean.ApiService/Program.cs b/src/Zilean.ApiService/Program.cs
@@ -15,7 +15,7 @@
     .AddShellExecutionService()
     .ConditionallyRegisterDmmJob(zileanConfiguration)
     .AddZileanDataServices(zileanConfiguration)
-    .AddDataBootStrapping();
+    .AddStartupHostedService();
 
 var app = builder.Build();
 

diff --git a/src/Zilean.ApiService/Properties/launchSettings.json b/src/Zilean.ApiService/Properties/launchSettings.json
@@ -6,7 +6,9 @@
       "environmentVariables": {
         "ASPNETCORE_ENVIRONMENT": "Development",
         "ASPNETCORE_URLS": "http://+:8181",
-        "Zilean__Torrents__EnableEndpoint": "true"
+        "Zilean__Torrents__EnableEndpoint": "true",
+        "Zilean__Ingestion__EnableScraping": "true",
+        "Zilean__Dmm__EnableScraping": "true"
       }
     }
   }

diff --git a/src/Zilean.Scraper/Features/Ingestion/GenericIngestionScraping.cs b/src/Zilean.Scraper/Features/Ingestion/GenericIngestionScraping.cs
@@ -20,13 +20,13 @@ public async Task<int> Execute(CancellationToken cancellationToken)
             urlsToProcess.AddRange(urls);
         }
 
-        if (configuration.Ingestion.EnableZurgIngestion)
+        if (configuration.Ingestion.ZurgInstances.Count > 0)
         {
             logger.LogInformation("Adding Zurg instances to the list of URLs to process");
             urlsToProcess.AddRange(configuration.Ingestion.ZurgInstances);
         }
 
-        if (configuration.Ingestion.EnableZileanIngestion)
+        if (configuration.Ingestion.ZileanInstances.Count > 0)
         {
             logger.LogInformation("Adding Zilean instances to the list of URLs to process");
             urlsToProcess.AddRange(configuration.Ingestion.ZileanInstances);

diff --git a/src/Zilean.Shared/Features/Configuration/IngestionConfiguration.cs b/src/Zilean.Shared/Features/Configuration/IngestionConfiguration.cs
@@ -4,9 +4,10 @@ public class IngestionConfiguration
 {
     public List<string> ZurgInstances { get; set; } = [];
     public List<string> ZileanInstances { get; set; } = [];
-    public bool EnableZurgIngestion { get; set; } = false;
-    public bool EnableZileanIngestion { get; set; } = false;
+    public bool EnableScraping { get; set; } = false;
     public KubernetesConfiguration Kubernetes { get; set; } = new();
     public int BatchSize { get; set; } = 1000;
     public int MaxChannelSize { get; set; } = 5000;
+
+    public string ScrapeSchedule { get; set; } = "0 * * * *";
 }