From 562bd1433aa741c13e417a1608a628ee7093a0e6 Mon Sep 17 00:00:00 2001 From: PromKnight Date: Sun, 17 Nov 2024 03:56:13 +0000 Subject: [PATCH] feat: add blacklist feature and API key authentication - Introduced a blacklist feature to prevent processing of unwanted torrents. - Added API key authentication for securing API endpoints. - Updated database schema to include BlacklistedItems table. - Enhanced DmmScraping and GenericIngestionProcessor to filter out blacklisted torrents. - Implemented new endpoints for adding and removing items from the blacklist. - Updated configuration to support API key generation and management. - Improved Kubernetes service discovery with authentication type options. - Enhanced Swagger documentation with API key security scheme. --- README.md | 64 +++- .../Authentication/ApiKeyAuthentication.cs | 7 + .../ApiKeyAuthenticationHandler.cs | 32 ++ .../ApiKeyDocumentTransformer.cs | 45 +++ .../Authentication/OpenApiSecurityMetadata.cs | 6 + .../Features/Blacklist/BlacklistEndpoints.cs | 116 ++++++ .../Blacklist/BlacklistItemRequest.cs | 8 + .../ConfigurationUpdaterService.cs | 47 +++ .../ServiceCollectionExtensions.cs | 34 +- .../Features/Bootstrapping/StartupService.cs | 4 +- .../Bootstrapping/WebApplicationExtensions.cs | 1 + src/Zilean.ApiService/GlobalUsings.cs | 11 + src/Zilean.ApiService/Program.cs | 11 +- .../Properties/launchSettings.json | 8 +- src/Zilean.Database/GlobalUsings.cs | 1 + ...0241117004344_BlacklistedItems.Designer.cs | 337 ++++++++++++++++++ .../20241117004344_BlacklistedItems.cs | 40 +++ .../ZileanDbContextModelSnapshot.cs | 33 +- .../BlacklistedItemConfiguration.cs | 29 ++ .../Services/ITorrentInfoService.cs | 1 + .../Services/TorrentInfoService.cs | 12 + src/Zilean.Database/ZileanDbContext.cs | 4 +- .../Features/Ingestion/DmmScraping.cs | 20 +- .../Ingestion/GenericIngestionProcessor.cs | 10 +- .../Ingestion/KubernetesServiceDiscovery.cs | 10 +- .../Ingestion/TorrentInfoExtensions.cs | 7 +- .../Extensions/StringExtensions.cs | 2 +- .../Features/Blacklist/BlacklistedItem.cs | 13 + .../KubernetesAuthenticationType.cs | 7 + .../Configuration/KubernetesConfiguration.cs | 2 + .../Configuration/ProwlarrConfiguration.cs | 6 - .../Configuration/ZileanConfiguration.cs | 3 +- .../Features/Utilities/ApiKey.cs | 6 + .../Zilean.Tests/Tests/ConfigurationTests.cs | 7 - 34 files changed, 889 insertions(+), 55 deletions(-) create mode 100644 src/Zilean.ApiService/Features/Authentication/ApiKeyAuthentication.cs create mode 100644 src/Zilean.ApiService/Features/Authentication/ApiKeyAuthenticationHandler.cs create mode 100644 src/Zilean.ApiService/Features/Authentication/ApiKeyDocumentTransformer.cs create mode 100644 src/Zilean.ApiService/Features/Authentication/OpenApiSecurityMetadata.cs create mode 100644 src/Zilean.ApiService/Features/Blacklist/BlacklistEndpoints.cs create mode 100644 src/Zilean.ApiService/Features/Blacklist/BlacklistItemRequest.cs create mode 100644 src/Zilean.ApiService/Features/Bootstrapping/ConfigurationUpdaterService.cs create mode 100644 src/Zilean.Database/Migrations/20241117004344_BlacklistedItems.Designer.cs create mode 100644 src/Zilean.Database/Migrations/20241117004344_BlacklistedItems.cs create mode 100644 src/Zilean.Database/ModelConfiguration/BlacklistedItemConfiguration.cs create mode 100644 src/Zilean.Shared/Features/Blacklist/BlacklistedItem.cs create mode 100644 src/Zilean.Shared/Features/Configuration/KubernetesAuthenticationType.cs delete mode 100644 src/Zilean.Shared/Features/Configuration/ProwlarrConfiguration.cs create mode 100644 src/Zilean.Shared/Features/Utilities/ApiKey.cs diff --git a/README.md b/README.md index 406aee9..72afc1a 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,8 @@ The DMM import reruns on missing pages every hour. ```json { "Zilean": { + "ApiKey": "5c43b70d3be04308b72ada4f61515fb4e278b08c48ec4c8a87e954ec658f8e4e", + "FirstRun": false, "Dmm": { "EnableScraping": true, "EnableEndpoint": true, @@ -29,11 +31,8 @@ The DMM import reruns on missing pages every hour. "Database": { "ConnectionString": "Host=localhost;Database=zilean;Username=postgres;Password=postgres;Include Error Detail=true;Timeout=300;CommandTimeout=300;" }, - "Prowlarr": { - "EnableEndpoint": true - }, "Torrents": { - "EnableEndpoint": false + "EnableEndpoint": true }, "Imdb": { "EnableImportMatching": true, @@ -41,19 +40,19 @@ The DMM import reruns on missing pages every hour. "MinimumScoreMatch": 0.85 }, "Ingestion": { - "ZurgInstances": [], + "ZurgInstances": [ + { + "Url": "http://zurg:9999", + "EndpointType": 1 + } + ], "ZileanInstances": [], - "EnableScraping": false, + "EnableScraping": true, "Kubernetes": { "EnableServiceDiscovery": false, - "KubernetesSelectors": [ - { - "UrlTemplate": "http://zurg.{0}:9999", - "LabelSelector": "app.elfhosted.com/name=zurg", - "EndpointType": 1 - } - ], - "KubeConfigFile": "/$HOME/.kube/config" + "KubernetesSelectors": [], + "KubeConfigFile": "/$HOME/.kube/config", + "AuthenticationType": 0 }, "BatchSize": 500, "MaxChannelSize": 5000, @@ -124,7 +123,8 @@ The `Ingestion` section in the JSON configuration defines the behavior and optio "EndpointType": 1 } ], - "KubeConfigFile": "/$HOME/.kube/config" + "KubeConfigFile": "/$HOME/.kube/config", + "AuthenticationType": 0 }, "BatchSize": 500, "MaxChannelSize": 5000, @@ -192,6 +192,7 @@ The `Ingestion` section in the JSON configuration defines the behavior and optio - **`LabelSelector`**: Label selector to filter Kubernetes services. - **`EndpointType`**: Indicates the type of endpoint (0 = Zilean, 1 = Zurg). - **`KubeConfigFile`**: Path to the Kubernetes configuration file. + - **`AuthenticationType`**: Authentication type for Kubernetes service discovery (0 = ConfigFile, 1 = RoleBased). ### `BatchSize` - **Type**: `int` @@ -273,9 +274,21 @@ If `EnableServiceDiscovery` is set to `true` in the Kubernetes section, the appl "EndpointType": 1 } ], - "KubeConfigFile": "/$HOME/.kube/config" + "KubeConfigFile": "/$HOME/.kube/config", + "AuthenticationType": 0 +} +``` +### `AuthenticationType` +Defines the Types of authentication to use when connecting to the kubernetes service host. + +```csharp +public enum KubernetesAuthenticationType +{ + ConfigFile = 0, + RoleBased = 1 } ``` +note: In order for RBAC to work, the service account must have the correct permissions to list services in the namespace, and the `KUBERNETES_SERVICE_HOST` and `KUBERNETES_SERVICE_PORT` environment variables must be set. ### Behavior 1. The application uses the Kubernetes client to list services matching the `LabelSelector`. @@ -296,4 +309,21 @@ Key events in the ingestion process are logged: - Discovered URLs. - Filtered torrents (existing in the database). - Processed torrents (new and valid). -- Errors during processing or service discovery. \ No newline at end of file +- Errors during processing or service discovery. + +--- + +## Blacklisting + +The ingestion pipeline supports blacklisting infohashes to prevent them from being processed. This feature is useful for filtering out unwanted torrents or duplicates. +See the `/blacklist` endpoints for more information in scalar. +These endpoints are protected by the ApiKey that will be generated on first run of the application and stored in the settings.json file as well as a one time print to application logs on startup. +Blacklisting an item also removes it from the database. + +--- + +## Api Key + +The ApiKey is generated on first run of the application and stored in the settings.json file as well as a one time print to application logs on startup. +The key can also be cycled to a new key if you set the environment variable `ZILEAN__NEW__API__KEY` to `true` and restart the application. +To authenticate with the API, you must include the `ApiKey` in the request headers. The header key is `X-Api-Key` and will automatically be configured in scalar. \ No newline at end of file diff --git a/src/Zilean.ApiService/Features/Authentication/ApiKeyAuthentication.cs b/src/Zilean.ApiService/Features/Authentication/ApiKeyAuthentication.cs new file mode 100644 index 0000000..f8d5551 --- /dev/null +++ b/src/Zilean.ApiService/Features/Authentication/ApiKeyAuthentication.cs @@ -0,0 +1,7 @@ +namespace Zilean.ApiService.Features.Authentication; + +public static class ApiKeyAuthentication +{ + public const string Scheme = "ApiKey"; + public const string Policy = "ApiKeyPolicy"; +} diff --git a/src/Zilean.ApiService/Features/Authentication/ApiKeyAuthenticationHandler.cs b/src/Zilean.ApiService/Features/Authentication/ApiKeyAuthenticationHandler.cs new file mode 100644 index 0000000..3756af8 --- /dev/null +++ b/src/Zilean.ApiService/Features/Authentication/ApiKeyAuthenticationHandler.cs @@ -0,0 +1,32 @@ +namespace Zilean.ApiService.Features.Authentication; + +public class ApiKeyAuthenticationHandler( + IOptionsMonitor options, + ILoggerFactory logger, + UrlEncoder encoder, + ISystemClock clock, + ZileanConfiguration configuration) + : AuthenticationHandler(options, logger, encoder, clock) +{ + protected override Task HandleAuthenticateAsync() + { + if (!Request.Headers.TryGetValue("X-API-KEY", out var extractedApiKey)) + { + return Task.FromResult(AuthenticateResult.Fail("API Key was not provided")); + } + + var configuredApiKey = configuration.ApiKey; + if (string.IsNullOrEmpty(configuredApiKey) || extractedApiKey != configuredApiKey) + { + return Task.FromResult(AuthenticateResult.Fail("Invalid API Key")); + } + + var claims = new[] { new Claim(ClaimTypes.Name, "ApiKeyUser") }; + var identity = new ClaimsIdentity(claims, Scheme.Name); + var principal = new ClaimsPrincipal(identity); + var ticket = new AuthenticationTicket(principal, Scheme.Name); + + return Task.FromResult(AuthenticateResult.Success(ticket)); + } +} + diff --git a/src/Zilean.ApiService/Features/Authentication/ApiKeyDocumentTransformer.cs b/src/Zilean.ApiService/Features/Authentication/ApiKeyDocumentTransformer.cs new file mode 100644 index 0000000..160073a --- /dev/null +++ b/src/Zilean.ApiService/Features/Authentication/ApiKeyDocumentTransformer.cs @@ -0,0 +1,45 @@ +public class ApiKeyDocumentTransformer : IOpenApiDocumentTransformer +{ + public Task TransformAsync(OpenApiDocument document, OpenApiDocumentTransformerContext context, CancellationToken cancellationToken) + { + // Define the API key security scheme + var apiKeyScheme = new OpenApiSecurityScheme + { + Type = SecuritySchemeType.ApiKey, + Name = "X-API-KEY", + In = ParameterLocation.Header, + Description = "API Key required for accessing protected endpoints." + }; + + document.Components ??= new OpenApiComponents(); + document.Components.SecuritySchemes[ApiKeyAuthentication.Scheme] = apiKeyScheme; + + foreach (var group in context.DescriptionGroups) + { + foreach (var apiDescription in group.Items) + { + var metadata = apiDescription.ActionDescriptor.EndpointMetadata? + .OfType() + .FirstOrDefault(); + + if (metadata is { SecurityScheme: ApiKeyAuthentication.Scheme }) + { + var route = apiDescription.RelativePath; + if (document.Paths.TryGetValue("/" + route, out var pathItem)) + { + foreach (var operation in pathItem.Operations.Values) + { + operation.Security ??= []; + operation.Security.Add(new OpenApiSecurityRequirement + { + [apiKeyScheme] = Array.Empty() + }); + } + } + } + } + } + + return Task.CompletedTask; + } +} diff --git a/src/Zilean.ApiService/Features/Authentication/OpenApiSecurityMetadata.cs b/src/Zilean.ApiService/Features/Authentication/OpenApiSecurityMetadata.cs new file mode 100644 index 0000000..a160cbe --- /dev/null +++ b/src/Zilean.ApiService/Features/Authentication/OpenApiSecurityMetadata.cs @@ -0,0 +1,6 @@ +namespace Zilean.ApiService.Features.Authentication; + +public class OpenApiSecurityMetadata(string securityScheme) +{ + public string SecurityScheme { get; } = securityScheme; +} diff --git a/src/Zilean.ApiService/Features/Blacklist/BlacklistEndpoints.cs b/src/Zilean.ApiService/Features/Blacklist/BlacklistEndpoints.cs new file mode 100644 index 0000000..5caa089 --- /dev/null +++ b/src/Zilean.ApiService/Features/Blacklist/BlacklistEndpoints.cs @@ -0,0 +1,116 @@ +namespace Zilean.ApiService.Features.Blacklist; + +public static class BlacklistEndpoints +{ + private const string GroupName = "blacklist"; + private const string Add = "/add"; + private const string Remove = "/remove"; + + public static WebApplication MapBlacklistEndpoints(this WebApplication app) + { + app.MapGroup(GroupName) + .WithTags(GroupName) + .Torrents() + .DisableAntiforgery() + .RequireAuthorization(ApiKeyAuthentication.Policy) + .WithMetadata(new OpenApiSecurityMetadata(ApiKeyAuthentication.Scheme)); + + return app; + } + + private static RouteGroupBuilder Torrents(this RouteGroupBuilder group) + { + group.MapPut(Add, AddBlacklistItem) + .Produces(StatusCodes.Status204NoContent) + .Produces(StatusCodes.Status400BadRequest) + .Produces(StatusCodes.Status409Conflict); + + group.MapDelete(Remove, RemoveBlacklistItem) + .Produces(StatusCodes.Status204NoContent) + .Produces(StatusCodes.Status404NotFound) + .Produces(StatusCodes.Status400BadRequest); + + return group; + } + + private static async Task RemoveBlacklistItem(HttpContext context, ZileanDbContext dbContext, ILogger logger, [FromQuery] string infoHash) + { + try + { + if (string.IsNullOrWhiteSpace(infoHash)) + { + logger.LogWarning("Attempted to remove blacklisted item with empty info hash"); + return Results.BadRequest("InfoHash is required"); + } + + var item = await dbContext.BlacklistedItems.FirstOrDefaultAsync(x => x.InfoHash == infoHash); + + if (item == null) + { + logger.LogWarning("Attempted to remove non-existent blacklisted item {InfoHash}", infoHash); + return Results.NotFound(); + } + + dbContext.BlacklistedItems.Remove(item); + await dbContext.SaveChangesAsync(); + + logger.LogInformation("Removed blacklisted item {InfoHash}", infoHash); + + return Results.NoContent(); + } + catch (Exception e) + { + logger.LogError(e, "An error occurred while removing a blacklisted item"); + return Results.BadRequest("An error occurred while removing a blacklisted item"); + } + } + + private static async Task AddBlacklistItem(HttpContext context, ZileanDbContext dbContext, [AsParameters] BlacklistItemRequest request, ILogger logger) + { + try + { + if (string.IsNullOrWhiteSpace(request.info_hash)) + { + return Results.BadRequest("info_hash is required"); + } + + if (string.IsNullOrWhiteSpace(request.reason)) + { + return Results.BadRequest("reason is required"); + } + + if (await dbContext.BlacklistedItems.AnyAsync(x => x.InfoHash == request.info_hash)) + { + return Results.Conflict("Item already blacklisted"); + } + + var blacklistedItem = new BlacklistedItem + { + InfoHash = request.info_hash, + Reason = request.reason, + BlacklistedAt = DateTime.UtcNow + }; + + dbContext.BlacklistedItems.Add(blacklistedItem); + + var torrentInfo = await dbContext.Torrents.FirstOrDefaultAsync(x => x.InfoHash == request.info_hash); + + if (torrentInfo != null) + { + dbContext.Torrents.Remove(torrentInfo); + logger.LogInformation("Removed torrent {InfoHash} from database", request.info_hash); + } + + await dbContext.SaveChangesAsync(); + + return Results.NoContent(); + } + catch (Exception e) + { + logger.LogError(e, "An error occurred while adding a blacklisted item"); + return Results.BadRequest("An error occurred while adding a blacklisted item"); + } + } + + private abstract class BlacklistLogger; +} diff --git a/src/Zilean.ApiService/Features/Blacklist/BlacklistItemRequest.cs b/src/Zilean.ApiService/Features/Blacklist/BlacklistItemRequest.cs new file mode 100644 index 0000000..f246e33 --- /dev/null +++ b/src/Zilean.ApiService/Features/Blacklist/BlacklistItemRequest.cs @@ -0,0 +1,8 @@ +// ReSharper disable InconsistentNaming +namespace Zilean.ApiService.Features.Blacklist; + +public class BlacklistItemRequest +{ + public required string info_hash { get; set; } + public required string reason { get; set; } +} diff --git a/src/Zilean.ApiService/Features/Bootstrapping/ConfigurationUpdaterService.cs b/src/Zilean.ApiService/Features/Bootstrapping/ConfigurationUpdaterService.cs new file mode 100644 index 0000000..6bf72d6 --- /dev/null +++ b/src/Zilean.ApiService/Features/Bootstrapping/ConfigurationUpdaterService.cs @@ -0,0 +1,47 @@ +namespace Zilean.ApiService.Features.Bootstrapping; + +public class ConfigurationUpdaterService(ZileanConfiguration configuration, ILogger logger) : IHostedService +{ + private const string ResetApiKeyEnvVar = "ZILEAN__NEW__API__KEY"; + + public async Task StartAsync(CancellationToken cancellationToken) + { + bool firstRun = configuration.FirstRun; + + if (firstRun) + { + configuration.FirstRun = false; + } + + if (Environment.GetEnvironmentVariable(ResetApiKeyEnvVar) is "1" or "true") + { + configuration.ApiKey = ApiKey.Generate(); + logger.LogInformation("API Key regenerated:'{ApiKey}'", configuration.ApiKey); + logger.LogInformation("Please keep this key safe and secure."); + } + + var configurationFolderPath = Path.Combine(AppContext.BaseDirectory, ConfigurationLiterals.ConfigurationFolder); + var configurationFilePath = Path.Combine(configurationFolderPath, ConfigurationLiterals.SettingsConfigFilename); + + var configWrapper = new Dictionary + { + [ConfigurationLiterals.MainSettingsSectionName] = configuration, + }; + + await File.WriteAllTextAsync(configurationFilePath, + JsonSerializer.Serialize(configWrapper, + new JsonSerializerOptions + { + WriteIndented = true, + PropertyNamingPolicy = null, + }), cancellationToken); + + if (firstRun) + { + logger.LogInformation("Zilean API Key: '{ApiKey}'", configuration.ApiKey); + logger.LogInformation("Please keep this key safe and secure."); + } + } + + public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask; +} diff --git a/src/Zilean.ApiService/Features/Bootstrapping/ServiceCollectionExtensions.cs b/src/Zilean.ApiService/Features/Bootstrapping/ServiceCollectionExtensions.cs index c764399..14cf077 100644 --- a/src/Zilean.ApiService/Features/Bootstrapping/ServiceCollectionExtensions.cs +++ b/src/Zilean.ApiService/Features/Bootstrapping/ServiceCollectionExtensions.cs @@ -4,15 +4,20 @@ namespace Zilean.ApiService.Features.Bootstrapping; public static class ServiceCollectionExtensions { public static IServiceCollection AddSwaggerSupport(this IServiceCollection services) => - services.AddOpenApi("v2"); + services.AddOpenApi("v2", options => + { + options.AddDocumentTransformer(); + }); public static IServiceCollection AddSchedulingSupport(this IServiceCollection services) => services.AddScheduler(); - public static IServiceCollection AddStartupHostedService(this IServiceCollection services) => - services.AddHostedService(); + public static IServiceCollection AddStartupHostedServices(this IServiceCollection services) => + services.AddHostedService() + .AddHostedService(); - public static IServiceCollection ConditionallyRegisterDmmJob(this IServiceCollection services, ZileanConfiguration configuration) + public static IServiceCollection ConditionallyRegisterDmmJob(this IServiceCollection services, + ZileanConfiguration configuration) { if (configuration.Dmm.EnableScraping) { @@ -46,4 +51,25 @@ public static IServiceProvider SetupScheduling(this IServiceProvider provider, Z return provider; } + + public static IServiceCollection AddApiKeyAuthentication(this IServiceCollection services) + { + services.AddAuthentication(options => + { + options.DefaultScheme = "None"; + options.DefaultAuthenticateScheme = "None"; + }) + .AddScheme(ApiKeyAuthentication.Scheme, _ => { }); + + services.AddAuthorization(options => + { + options.AddPolicy(ApiKeyAuthentication.Policy, policy => + { + policy.AuthenticationSchemes.Add(ApiKeyAuthentication.Scheme); + policy.RequireAuthenticatedUser(); + }); + }); + + return services; + } } diff --git a/src/Zilean.ApiService/Features/Bootstrapping/StartupService.cs b/src/Zilean.ApiService/Features/Bootstrapping/StartupService.cs index 8e54433..adcde39 100644 --- a/src/Zilean.ApiService/Features/Bootstrapping/StartupService.cs +++ b/src/Zilean.ApiService/Features/Bootstrapping/StartupService.cs @@ -31,8 +31,8 @@ public async Task StartedAsync(CancellationToken cancellationToken) await using var asyncScope = serviceProvider.CreateAsyncScope(); var dbContext = asyncScope.ServiceProvider.GetRequiredService(); var dmmJob = new DmmSyncJob(executionService, loggerFactory.CreateLogger(), dbContext); - var shouldRun = await dmmJob.ShouldRunOnStartup(); - if (shouldRun) + var pagesExist = await dmmJob.ShouldRunOnStartup(); + if (!pagesExist) { await dmmJob.Invoke(); } diff --git a/src/Zilean.ApiService/Features/Bootstrapping/WebApplicationExtensions.cs b/src/Zilean.ApiService/Features/Bootstrapping/WebApplicationExtensions.cs index 3c33ad7..06961b1 100644 --- a/src/Zilean.ApiService/Features/Bootstrapping/WebApplicationExtensions.cs +++ b/src/Zilean.ApiService/Features/Bootstrapping/WebApplicationExtensions.cs @@ -16,5 +16,6 @@ public static WebApplication MapZileanEndpoints(this WebApplication app, ZileanC .MapImdbEndpoints(configuration) .MapTorznabEndpoints(configuration) .MapTorrentsEndpoints(configuration) + .MapBlacklistEndpoints() .MapHealthCheckEndpoints(); } diff --git a/src/Zilean.ApiService/GlobalUsings.cs b/src/Zilean.ApiService/GlobalUsings.cs index 17d1e22..5ef9895 100644 --- a/src/Zilean.ApiService/GlobalUsings.cs +++ b/src/Zilean.ApiService/GlobalUsings.cs @@ -4,19 +4,28 @@ global using System.Diagnostics.CodeAnalysis; global using System.Globalization; global using System.Reflection; +global using System.Security.Claims; +global using System.Text.Encodings.Web; global using System.Text.Json; global using System.Xml.Serialization; global using Coravel; global using Coravel.Invocable; global using Coravel.Scheduling.Schedule.Interfaces; +global using Microsoft.AspNetCore.Authentication; +global using Microsoft.AspNetCore.Authorization; global using Microsoft.AspNetCore.Http.HttpResults; global using Microsoft.AspNetCore.Mvc; +global using Microsoft.AspNetCore.OpenApi; global using Microsoft.EntityFrameworkCore; global using Microsoft.Extensions.DependencyInjection; global using Microsoft.Extensions.Logging; +global using Microsoft.Extensions.Options; global using Microsoft.IO; +global using Microsoft.OpenApi.Models; global using Scalar.AspNetCore; global using SimCube.Aspire.Features.Otlp; +global using Zilean.ApiService.Features.Authentication; +global using Zilean.ApiService.Features.Blacklist; global using Zilean.ApiService.Features.Bootstrapping; global using Zilean.ApiService.Features.HealthChecks; global using Zilean.ApiService.Features.Imdb; @@ -25,7 +34,9 @@ global using Zilean.ApiService.Features.Torrents; global using Zilean.ApiService.Features.Torznab; global using Zilean.Database; +global using Zilean.Database.Bootstrapping; global using Zilean.Database.Services; +global using Zilean.Shared.Features.Blacklist; global using Zilean.Shared.Features.Configuration; global using Zilean.Shared.Features.Dmm; global using Zilean.Shared.Features.Scraping; diff --git a/src/Zilean.ApiService/Program.cs b/src/Zilean.ApiService/Program.cs index caea6ac..07ebe94 100644 --- a/src/Zilean.ApiService/Program.cs +++ b/src/Zilean.ApiService/Program.cs @@ -1,6 +1,4 @@ -using Zilean.Database.Bootstrapping; - -var builder = WebApplication.CreateBuilder(args); +var builder = WebApplication.CreateBuilder(args); builder.Configuration.AddConfigurationFiles(); @@ -15,10 +13,14 @@ .AddShellExecutionService() .ConditionallyRegisterDmmJob(zileanConfiguration) .AddZileanDataServices(zileanConfiguration) - .AddStartupHostedService(); + .AddApiKeyAuthentication() + .AddStartupHostedServices(); var app = builder.Build(); +app.UseAuthentication(); +app.UseAuthorization(); + var logger = app.Services.GetRequiredService>(); app.MapDefaultEndpoints(); @@ -28,4 +30,5 @@ app.Services.SetupScheduling(zileanConfiguration); logger.LogInformation("Zilean API Service started."); + app.Run(); diff --git a/src/Zilean.ApiService/Properties/launchSettings.json b/src/Zilean.ApiService/Properties/launchSettings.json index e13dac4..afaf734 100644 --- a/src/Zilean.ApiService/Properties/launchSettings.json +++ b/src/Zilean.ApiService/Properties/launchSettings.json @@ -3,12 +3,16 @@ "profiles": { "Zilean.ApiService": { "commandName": "Project", + "launchBrowser": true, + "launchUrl": "http://localhost:8181/scalar/v2", "environmentVariables": { "ASPNETCORE_ENVIRONMENT": "Development", - "ASPNETCORE_URLS": "http://+:8181", + "ASPNETCORE_URLS": "http://localhost:8181", "Zilean__Torrents__EnableEndpoint": "true", "Zilean__Ingestion__EnableScraping": "true", - "Zilean__Dmm__EnableScraping": "true" + "Zilean__Dmm__EnableScraping": "true", + "ZILEAN__NEW__API__KEY": "false", + "Zilean__Database__ConnectionString": "Host=localhost;Database=zilean-testing;Username=postgres;Password=postgres;Include Error Detail=true;Timeout=300;CommandTimeout=300;" } } } diff --git a/src/Zilean.Database/GlobalUsings.cs b/src/Zilean.Database/GlobalUsings.cs index b452989..e4055ef 100644 --- a/src/Zilean.Database/GlobalUsings.cs +++ b/src/Zilean.Database/GlobalUsings.cs @@ -12,6 +12,7 @@ global using Spectre.Console; global using Zilean.Database.Dtos; global using Zilean.Database.ModelConfiguration; +global using Zilean.Shared.Features.Blacklist; global using Zilean.Shared.Features.Configuration; global using Zilean.Shared.Features.Dmm; global using Zilean.Shared.Features.Imdb; diff --git a/src/Zilean.Database/Migrations/20241117004344_BlacklistedItems.Designer.cs b/src/Zilean.Database/Migrations/20241117004344_BlacklistedItems.Designer.cs new file mode 100644 index 0000000..c3218c8 --- /dev/null +++ b/src/Zilean.Database/Migrations/20241117004344_BlacklistedItems.Designer.cs @@ -0,0 +1,337 @@ +// +using System; +using System.Text.Json; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using Zilean.Database; + +#nullable disable + +namespace Zilean.Database.Migrations +{ + [DbContext(typeof(ZileanDbContext))] + [Migration("20241117004344_BlacklistedItems")] + partial class BlacklistedItems + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "9.0.0") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("Zilean.Shared.Features.Blacklist.BlacklistedItem", b => + { + b.Property("InfoHash") + .HasColumnType("text") + .HasAnnotation("Relational:JsonPropertyName", "info_hash"); + + b.Property("BlacklistedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasDefaultValueSql("now() at time zone 'utc'") + .HasAnnotation("Relational:JsonPropertyName", "blacklisted_at"); + + b.Property("Reason") + .IsRequired() + .HasColumnType("text") + .HasAnnotation("Relational:JsonPropertyName", "reason"); + + b.HasKey("InfoHash"); + + b.HasIndex("InfoHash") + .IsUnique(); + + b.ToTable("BlacklistedItems", (string)null); + }); + + modelBuilder.Entity("Zilean.Shared.Features.Dmm.ParsedPages", b => + { + b.Property("Page") + .HasColumnType("text"); + + b.Property("EntryCount") + .HasColumnType("integer"); + + b.HasKey("Page"); + + b.ToTable("ParsedPages", (string)null); + }); + + modelBuilder.Entity("Zilean.Shared.Features.Dmm.TorrentInfo", b => + { + b.Property("InfoHash") + .HasColumnType("text") + .HasAnnotation("Relational:JsonPropertyName", "info_hash"); + + b.PrimitiveCollection("Audio") + .IsRequired() + .HasColumnType("text[]") + .HasAnnotation("Relational:JsonPropertyName", "audio"); + + b.Property("BitDepth") + .HasColumnType("text") + .HasAnnotation("Relational:JsonPropertyName", "bit_depth"); + + b.Property("Bitrate") + .HasColumnType("text") + .HasAnnotation("Relational:JsonPropertyName", "bitrate"); + + b.Property("Category") + .IsRequired() + .HasColumnType("text") + .HasAnnotation("Relational:JsonPropertyName", "category"); + + b.PrimitiveCollection("Channels") + .IsRequired() + .HasColumnType("text[]") + .HasAnnotation("Relational:JsonPropertyName", "channels"); + + b.Property("Codec") + .HasColumnType("text") + .HasAnnotation("Relational:JsonPropertyName", "codec"); + + b.Property("Complete") + .HasColumnType("boolean") + .HasAnnotation("Relational:JsonPropertyName", "complete"); + + b.Property("Container") + .HasColumnType("text") + .HasAnnotation("Relational:JsonPropertyName", "container"); + + b.Property("Converted") + .HasColumnType("boolean") + .HasAnnotation("Relational:JsonPropertyName", "converted"); + + b.Property("Country") + .HasColumnType("text") + .HasAnnotation("Relational:JsonPropertyName", "country"); + + b.Property("Date") + .HasColumnType("text") + .HasAnnotation("Relational:JsonPropertyName", "date"); + + b.Property("Documentary") + .HasColumnType("boolean") + .HasAnnotation("Relational:JsonPropertyName", "documentary"); + + b.Property("Dubbed") + .HasColumnType("boolean") + .HasAnnotation("Relational:JsonPropertyName", "dubbed"); + + b.Property("Edition") + .HasColumnType("text") + .HasAnnotation("Relational:JsonPropertyName", "edition"); + + b.Property("EpisodeCode") + .HasColumnType("text") + .HasAnnotation("Relational:JsonPropertyName", "episode_code"); + + b.PrimitiveCollection("Episodes") + .IsRequired() + .HasColumnType("integer[]") + .HasAnnotation("Relational:JsonPropertyName", "episodes"); + + b.Property("Extended") + .HasColumnType("boolean") + .HasAnnotation("Relational:JsonPropertyName", "extended"); + + b.Property("Extension") + .HasColumnType("text") + .HasAnnotation("Relational:JsonPropertyName", "extension"); + + b.Property("Group") + .HasColumnType("text") + .HasAnnotation("Relational:JsonPropertyName", "group"); + + b.Property("Hardcoded") + .HasColumnType("boolean") + .HasAnnotation("Relational:JsonPropertyName", "hardcoded"); + + b.PrimitiveCollection("Hdr") + .IsRequired() + .HasColumnType("text[]") + .HasAnnotation("Relational:JsonPropertyName", "hdr"); + + b.Property("ImdbId") + .HasColumnType("text") + .HasAnnotation("Relational:JsonPropertyName", "imdb_id"); + + b.Property("IngestedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasDefaultValueSql("now() at time zone 'utc'") + .HasAnnotation("Relational:JsonPropertyName", "ingested_at"); + + b.Property("Is3d") + .HasColumnType("boolean") + .HasAnnotation("Relational:JsonPropertyName", "_3d"); + + b.PrimitiveCollection("Languages") + .IsRequired() + .HasColumnType("text[]") + .HasAnnotation("Relational:JsonPropertyName", "languages"); + + b.Property("Network") + .HasColumnType("text") + .HasAnnotation("Relational:JsonPropertyName", "network"); + + b.Property("NormalizedTitle") + .IsRequired() + .HasColumnType("text") + .HasAnnotation("Relational:JsonPropertyName", "normalized_title"); + + b.Property("ParsedTitle") + .IsRequired() + .HasColumnType("text") + .HasAnnotation("Relational:JsonPropertyName", "parsed_title"); + + b.Property("Ppv") + .HasColumnType("boolean") + .HasAnnotation("Relational:JsonPropertyName", "ppv"); + + b.Property("Proper") + .HasColumnType("boolean") + .HasAnnotation("Relational:JsonPropertyName", "proper"); + + b.Property("Quality") + .HasColumnType("text") + .HasAnnotation("Relational:JsonPropertyName", "quality"); + + b.Property("RawTitle") + .IsRequired() + .HasColumnType("text") + .HasAnnotation("Relational:JsonPropertyName", "raw_title"); + + b.Property("Region") + .HasColumnType("text") + .HasAnnotation("Relational:JsonPropertyName", "region"); + + b.Property("Remastered") + .HasColumnType("boolean") + .HasAnnotation("Relational:JsonPropertyName", "remastered"); + + b.Property("Repack") + .HasColumnType("boolean") + .HasAnnotation("Relational:JsonPropertyName", "repack"); + + b.Property("Resolution") + .IsRequired() + .HasColumnType("text") + .HasAnnotation("Relational:JsonPropertyName", "resolution"); + + b.Property("Retail") + .HasColumnType("boolean") + .HasAnnotation("Relational:JsonPropertyName", "retail"); + + b.PrimitiveCollection("Seasons") + .IsRequired() + .HasColumnType("integer[]") + .HasAnnotation("Relational:JsonPropertyName", "seasons"); + + b.Property("Site") + .HasColumnType("text") + .HasAnnotation("Relational:JsonPropertyName", "site"); + + b.Property("Size") + .HasColumnType("text") + .HasAnnotation("Relational:JsonPropertyName", "size"); + + b.Property("Subbed") + .HasColumnType("boolean") + .HasAnnotation("Relational:JsonPropertyName", "subbed"); + + b.Property("Torrent") + .HasColumnType("boolean") + .HasAnnotation("Relational:JsonPropertyName", "torrent"); + + b.Property("Trash") + .HasColumnType("boolean") + .HasAnnotation("Relational:JsonPropertyName", "trash"); + + b.Property("Unrated") + .HasColumnType("boolean") + .HasAnnotation("Relational:JsonPropertyName", "unrated"); + + b.Property("Upscaled") + .HasColumnType("boolean") + .HasAnnotation("Relational:JsonPropertyName", "upscaled"); + + b.PrimitiveCollection("Volumes") + .IsRequired() + .HasColumnType("integer[]") + .HasAnnotation("Relational:JsonPropertyName", "volumes"); + + b.Property("Year") + .HasColumnType("integer") + .HasAnnotation("Relational:JsonPropertyName", "year"); + + b.HasKey("InfoHash"); + + b.HasIndex("ImdbId"); + + b.HasIndex("InfoHash") + .IsUnique(); + + b.ToTable("Torrents", (string)null); + }); + + modelBuilder.Entity("Zilean.Shared.Features.Imdb.ImdbFile", b => + { + b.Property("ImdbId") + .HasColumnType("text"); + + b.Property("Adult") + .HasColumnType("boolean"); + + b.Property("Category") + .HasColumnType("text"); + + b.Property("Title") + .HasColumnType("text"); + + b.Property("Year") + .HasColumnType("integer"); + + b.HasKey("ImdbId"); + + b.HasIndex("ImdbId") + .IsUnique(); + + b.ToTable("ImdbFiles", (string)null); + + b.HasAnnotation("Relational:JsonPropertyName", "imdb"); + }); + + modelBuilder.Entity("Zilean.Shared.Features.Statistics.ImportMetadata", b => + { + b.Property("Key") + .HasColumnType("text"); + + b.Property("Value") + .IsRequired() + .HasColumnType("jsonb"); + + b.HasKey("Key"); + + b.ToTable("ImportMetadata", (string)null); + }); + + modelBuilder.Entity("Zilean.Shared.Features.Dmm.TorrentInfo", b => + { + b.HasOne("Zilean.Shared.Features.Imdb.ImdbFile", "Imdb") + .WithMany() + .HasForeignKey("ImdbId"); + + b.Navigation("Imdb"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/Zilean.Database/Migrations/20241117004344_BlacklistedItems.cs b/src/Zilean.Database/Migrations/20241117004344_BlacklistedItems.cs new file mode 100644 index 0000000..1875097 --- /dev/null +++ b/src/Zilean.Database/Migrations/20241117004344_BlacklistedItems.cs @@ -0,0 +1,40 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace Zilean.Database.Migrations; + +/// +public partial class BlacklistedItems : Migration +{ + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "BlacklistedItems", + columns: table => new + { + InfoHash = table.Column(type: "text", nullable: false), + Reason = table.Column(type: "text", nullable: false), + BlacklistedAt = table.Column(type: "timestamp with time zone", nullable: false, defaultValueSql: "now() at time zone 'utc'") + }, + constraints: table => + { + table.PrimaryKey("PK_BlacklistedItems", x => x.InfoHash); + }); + + migrationBuilder.CreateIndex( + name: "IX_BlacklistedItems_InfoHash", + table: "BlacklistedItems", + column: "InfoHash", + unique: true); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "BlacklistedItems"); + } +} diff --git a/src/Zilean.Database/Migrations/ZileanDbContextModelSnapshot.cs b/src/Zilean.Database/Migrations/ZileanDbContextModelSnapshot.cs index 3fb782c..66dba1c 100644 --- a/src/Zilean.Database/Migrations/ZileanDbContextModelSnapshot.cs +++ b/src/Zilean.Database/Migrations/ZileanDbContextModelSnapshot.cs @@ -23,7 +23,32 @@ protected override void BuildModel(ModelBuilder modelBuilder) NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); - modelBuilder.Entity("Zilean.Shared.Features.Search.ParsedPages", b => + modelBuilder.Entity("Zilean.Shared.Features.Blacklist.BlacklistedItem", b => + { + b.Property("InfoHash") + .HasColumnType("text") + .HasAnnotation("Relational:JsonPropertyName", "info_hash"); + + b.Property("BlacklistedAt") + .ValueGeneratedOnAdd() + .HasColumnType("timestamp with time zone") + .HasDefaultValueSql("now() at time zone 'utc'") + .HasAnnotation("Relational:JsonPropertyName", "blacklisted_at"); + + b.Property("Reason") + .IsRequired() + .HasColumnType("text") + .HasAnnotation("Relational:JsonPropertyName", "reason"); + + b.HasKey("InfoHash"); + + b.HasIndex("InfoHash") + .IsUnique(); + + b.ToTable("BlacklistedItems", (string)null); + }); + + modelBuilder.Entity("Zilean.Shared.Features.Dmm.ParsedPages", b => { b.Property("Page") .HasColumnType("text"); @@ -36,7 +61,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.ToTable("ParsedPages", (string)null); }); - modelBuilder.Entity("Zilean.Shared.Features.Search.TorrentInfo", b => + modelBuilder.Entity("Zilean.Shared.Features.Dmm.TorrentInfo", b => { b.Property("InfoHash") .HasColumnType("text") @@ -292,10 +317,10 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.HasKey("Key"); - b.ToTable("ImportMetadata"); + b.ToTable("ImportMetadata", (string)null); }); - modelBuilder.Entity("Zilean.Shared.Features.Search.TorrentInfo", b => + modelBuilder.Entity("Zilean.Shared.Features.Dmm.TorrentInfo", b => { b.HasOne("Zilean.Shared.Features.Imdb.ImdbFile", "Imdb") .WithMany() diff --git a/src/Zilean.Database/ModelConfiguration/BlacklistedItemConfiguration.cs b/src/Zilean.Database/ModelConfiguration/BlacklistedItemConfiguration.cs new file mode 100644 index 0000000..536cc8d --- /dev/null +++ b/src/Zilean.Database/ModelConfiguration/BlacklistedItemConfiguration.cs @@ -0,0 +1,29 @@ +namespace Zilean.Database.ModelConfiguration; + +public class BlacklistedItemConfiguration: IEntityTypeConfiguration +{ + public void Configure(EntityTypeBuilder builder) + { + builder.ToTable("BlacklistedItems"); + + builder.HasKey(i => i.InfoHash); + + builder.Property(i => i.InfoHash) + .HasColumnType("text") + .HasAnnotation("Relational:JsonPropertyName", "info_hash"); + + builder.Property(i => i.Reason) + .IsRequired() + .HasColumnType("text") + .HasAnnotation("Relational:JsonPropertyName", "reason"); + + builder.Property(t => t.BlacklistedAt) + .IsRequired() + .HasColumnType("timestamp with time zone") + .HasDefaultValueSql("now() at time zone 'utc'") + .HasAnnotation("Relational:JsonPropertyName", "blacklisted_at"); + + builder.HasIndex(i => i.InfoHash) + .IsUnique(); + } +} diff --git a/src/Zilean.Database/Services/ITorrentInfoService.cs b/src/Zilean.Database/Services/ITorrentInfoService.cs index fd4f897..771bd00 100644 --- a/src/Zilean.Database/Services/ITorrentInfoService.cs +++ b/src/Zilean.Database/Services/ITorrentInfoService.cs @@ -6,4 +6,5 @@ public interface ITorrentInfoService Task SearchForTorrentInfoByOnlyTitle(string query); Task SearchForTorrentInfoFiltered(TorrentInfoFilter filter, int? limit = null); Task> GetExistingInfoHashesAsync(List infoHashes); + Task> GetBlacklistedItems(); } diff --git a/src/Zilean.Database/Services/TorrentInfoService.cs b/src/Zilean.Database/Services/TorrentInfoService.cs index 5185d0d..d51fb80 100644 --- a/src/Zilean.Database/Services/TorrentInfoService.cs +++ b/src/Zilean.Database/Services/TorrentInfoService.cs @@ -202,5 +202,17 @@ public async Task> GetExistingInfoHashesAsync(List infoH return [..existingHashes]; } + public async Task> GetBlacklistedItems() + { + await using var serviceScope = serviceProvider.CreateAsyncScope(); + await using var dbContext = serviceScope.ServiceProvider.GetRequiredService(); + + var existingHashes = await dbContext.BlacklistedItems + .Select(t => t.InfoHash) + .ToListAsync(); + + return [..existingHashes]; + } + private void WriteProgress(decimal @decimal) => logger.LogInformation("Storing torrent info: {Percentage:P}", @decimal); } diff --git a/src/Zilean.Database/ZileanDbContext.cs b/src/Zilean.Database/ZileanDbContext.cs index d2662c6..2410f1f 100644 --- a/src/Zilean.Database/ZileanDbContext.cs +++ b/src/Zilean.Database/ZileanDbContext.cs @@ -26,11 +26,13 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) modelBuilder.ApplyConfiguration(new TorrentInfoConfiguration()); modelBuilder.ApplyConfiguration(new ImdbFileConfiguration()); modelBuilder.ApplyConfiguration(new ParsedPagesConfiguration()); + modelBuilder.ApplyConfiguration(new ImportMetadataConfiguration()); + modelBuilder.ApplyConfiguration(new BlacklistedItemConfiguration()); } public DbSet Torrents => Set(); public DbSet ImdbFiles => Set(); public DbSet ParsedPages => Set(); - public DbSet ImportMetadata => Set(); + public DbSet BlacklistedItems => Set(); } diff --git a/src/Zilean.Scraper/Features/Ingestion/DmmScraping.cs b/src/Zilean.Scraper/Features/Ingestion/DmmScraping.cs index fcb87d4..4a44a26 100644 --- a/src/Zilean.Scraper/Features/Ingestion/DmmScraping.cs +++ b/src/Zilean.Scraper/Features/Ingestion/DmmScraping.cs @@ -109,7 +109,16 @@ await AnsiConsole.Progress() var parsedTorrents = await parseTorrentNameService.ParseAndPopulateAsync(distinctTorrents); - var finalizedTorrents = parsedTorrents.Where(torrentInfo => torrentInfo.WipeSomeTissue()).ToList(); + var blacklistedHashes = await torrentInfoService.GetBlacklistedItems(); + + var finalizedTorrents = parsedTorrents + .Where(torrentInfo => torrentInfo.WipeSomeTissue()) + .Where(torrentsInfo => !torrentsInfo.IsBlacklisted(blacklistedHashes)) + .ToList(); + + logger.LogInformation("Removed {Count} hashes due to blacklisting or possible adult titile matches", parsedTorrents.Count - finalizedTorrents.Count); + + logger.LogInformation("Parsed {Count} torrents", finalizedTorrents.Count); await torrentInfoService.StoreTorrentInfo(finalizedTorrents); } @@ -154,7 +163,14 @@ await Parallel.ForEachAsync(files, parallelOptions, async (file, ct) => var parsedTorrents = await parseTorrentNameService.ParseAndPopulateAsync(distinctTorrents); - var finalizedTorrents = parsedTorrents.Where(torrentInfo => torrentInfo.WipeSomeTissue()).ToList(); + var blacklistedHashes = await torrentInfoService.GetBlacklistedItems(); + + var finalizedTorrents = parsedTorrents + .Where(torrentInfo => torrentInfo.WipeSomeTissue()) + .Where(torrentInfo => !torrentInfo.IsBlacklisted(blacklistedHashes)) + .ToList(); + + logger.LogInformation("Removed {Count} hashes due to blacklisting or possible adult titile matches", parsedTorrents.Count - finalizedTorrents.Count); logger.LogInformation("Parsed {Count} torrents", finalizedTorrents.Count); diff --git a/src/Zilean.Scraper/Features/Ingestion/GenericIngestionProcessor.cs b/src/Zilean.Scraper/Features/Ingestion/GenericIngestionProcessor.cs index 32c0843..1a2f0ae 100644 --- a/src/Zilean.Scraper/Features/Ingestion/GenericIngestionProcessor.cs +++ b/src/Zilean.Scraper/Features/Ingestion/GenericIngestionProcessor.cs @@ -123,7 +123,15 @@ private async Task ProcessBatch(List> batch, CancellationTok if (torrents.Count != 0) { var parsedTorrents = await parseTorrentNameService.ParseAndPopulateAsync(newTorrents); - var finalizedTorrents = parsedTorrents.Where(torrentInfo => torrentInfo.WipeSomeTissue()).ToList(); + + var blacklistedHashes = await torrentInfoService.GetBlacklistedItems(); + + var finalizedTorrents = parsedTorrents + .Where(torrentInfo => torrentInfo.WipeSomeTissue()) + .Where(torrentsInfo => !torrentsInfo.IsBlacklisted(blacklistedHashes)) + .ToList(); + + logger.LogInformation("Removed {Count} hashes due to blacklisting or possible adult titile matches", parsedTorrents.Count - finalizedTorrents.Count); logger.LogInformation("Parsed {Count} torrents", finalizedTorrents.Count); await torrentInfoService.StoreTorrentInfo(finalizedTorrents); } diff --git a/src/Zilean.Scraper/Features/Ingestion/KubernetesServiceDiscovery.cs b/src/Zilean.Scraper/Features/Ingestion/KubernetesServiceDiscovery.cs index 14b15d4..b006a1f 100644 --- a/src/Zilean.Scraper/Features/Ingestion/KubernetesServiceDiscovery.cs +++ b/src/Zilean.Scraper/Features/Ingestion/KubernetesServiceDiscovery.cs @@ -12,8 +12,14 @@ public async Task> DiscoverUrlsAsync(CancellationToken can try { - var clientConfig = - KubernetesClientConfiguration.BuildConfigFromConfigFile(configuration.Ingestion.Kubernetes.KubeConfigFile); + var clientConfig = configuration.Ingestion.Kubernetes.AuthenticationType switch + { + KubernetesAuthenticationType.ConfigFile => KubernetesClientConfiguration.BuildConfigFromConfigFile(configuration + .Ingestion.Kubernetes.KubeConfigFile), + KubernetesAuthenticationType.RoleBased => KubernetesClientConfiguration.InClusterConfig(), + _ => throw new InvalidOperationException("Unknown authentication type") + }; + var kubernetesClient = new Kubernetes(clientConfig); List discoveredServices = []; diff --git a/src/Zilean.Scraper/Features/Ingestion/TorrentInfoExtensions.cs b/src/Zilean.Scraper/Features/Ingestion/TorrentInfoExtensions.cs index 370448e..4a3820c 100644 --- a/src/Zilean.Scraper/Features/Ingestion/TorrentInfoExtensions.cs +++ b/src/Zilean.Scraper/Features/Ingestion/TorrentInfoExtensions.cs @@ -1,4 +1,6 @@ -namespace Zilean.Scraper.Features.Ingestion; +using Zilean.Shared.Features.Blacklist; + +namespace Zilean.Scraper.Features.Ingestion; public static class TorrentInfoExtensions { @@ -6,4 +8,7 @@ public static bool WipeSomeTissue(this TorrentInfo torrent) => !((torrent.RawTitle.Contains(" xxx ", StringComparison.OrdinalIgnoreCase) || torrent.RawTitle.Contains(" xx ", StringComparison.OrdinalIgnoreCase)) && !torrent.ParsedTitle.Contains("XXX", StringComparison.OrdinalIgnoreCase)); + + public static bool IsBlacklisted(this TorrentInfo torrent, HashSet blacklistedItems) => + blacklistedItems.Any(x => x.Equals(torrent.InfoHash, StringComparison.OrdinalIgnoreCase)); } diff --git a/src/Zilean.Shared/Extensions/StringExtensions.cs b/src/Zilean.Shared/Extensions/StringExtensions.cs index b48482d..a9028f3 100644 --- a/src/Zilean.Shared/Extensions/StringExtensions.cs +++ b/src/Zilean.Shared/Extensions/StringExtensions.cs @@ -7,7 +7,7 @@ public static bool ContainsIgnoreCase(this string? source, string toCheck) => public static bool ContainsIgnoreCase(this IEnumerable? source, string toCheck) => source.Any(s => s.Contains(toCheck, StringComparison.OrdinalIgnoreCase)); - + public static bool IsNullOrWhiteSpace(this string? source) => string.IsNullOrWhiteSpace(source); } diff --git a/src/Zilean.Shared/Features/Blacklist/BlacklistedItem.cs b/src/Zilean.Shared/Features/Blacklist/BlacklistedItem.cs new file mode 100644 index 0000000..5d2848b --- /dev/null +++ b/src/Zilean.Shared/Features/Blacklist/BlacklistedItem.cs @@ -0,0 +1,13 @@ +namespace Zilean.Shared.Features.Blacklist; + +public class BlacklistedItem +{ + [JsonPropertyName("info_hash")] + public string? InfoHash { get; set; } + + [JsonPropertyName("reason")] + public string? Reason { get; set; } + + [JsonPropertyName("blacklisted_at")] + public DateTime? BlacklistedAt { get; set; } +} diff --git a/src/Zilean.Shared/Features/Configuration/KubernetesAuthenticationType.cs b/src/Zilean.Shared/Features/Configuration/KubernetesAuthenticationType.cs new file mode 100644 index 0000000..e025f16 --- /dev/null +++ b/src/Zilean.Shared/Features/Configuration/KubernetesAuthenticationType.cs @@ -0,0 +1,7 @@ +namespace Zilean.Shared.Features.Configuration; + +public enum KubernetesAuthenticationType +{ + ConfigFile = 0, + RoleBased = 1 +} diff --git a/src/Zilean.Shared/Features/Configuration/KubernetesConfiguration.cs b/src/Zilean.Shared/Features/Configuration/KubernetesConfiguration.cs index 0c418db..6e1f595 100644 --- a/src/Zilean.Shared/Features/Configuration/KubernetesConfiguration.cs +++ b/src/Zilean.Shared/Features/Configuration/KubernetesConfiguration.cs @@ -5,4 +5,6 @@ public class KubernetesConfiguration public bool EnableServiceDiscovery { get; set; } = false; public List KubernetesSelectors { get; set; } = []; public string KubeConfigFile { get; set; } = "/$HOME/.kube/config"; + + public KubernetesAuthenticationType AuthenticationType { get; set; } = KubernetesAuthenticationType.ConfigFile; } diff --git a/src/Zilean.Shared/Features/Configuration/ProwlarrConfiguration.cs b/src/Zilean.Shared/Features/Configuration/ProwlarrConfiguration.cs deleted file mode 100644 index 2bc98e1..0000000 --- a/src/Zilean.Shared/Features/Configuration/ProwlarrConfiguration.cs +++ /dev/null @@ -1,6 +0,0 @@ -namespace Zilean.Shared.Features.Configuration; - -public class ProwlarrConfiguration -{ - public bool EnableEndpoint { get; set; } = true; -} diff --git a/src/Zilean.Shared/Features/Configuration/ZileanConfiguration.cs b/src/Zilean.Shared/Features/Configuration/ZileanConfiguration.cs index c0ba023..43c1ada 100644 --- a/src/Zilean.Shared/Features/Configuration/ZileanConfiguration.cs +++ b/src/Zilean.Shared/Features/Configuration/ZileanConfiguration.cs @@ -8,10 +8,11 @@ public class ZileanConfiguration PropertyNamingPolicy = null, }; + public string? ApiKey { get; set; } = Utilities.ApiKey.Generate(); + public bool FirstRun { get; set; } = true; public DmmConfiguration Dmm { get; set; } = new(); public TorznabConfiguration Torznab { get; set; } = new(); public DatabaseConfiguration Database { get; set; } = new(); - public ProwlarrConfiguration Prowlarr { get; set; } = new(); public TorrentsConfiguration Torrents { get; set; } = new(); public ImdbConfiguration Imdb { get; set; } = new(); public IngestionConfiguration Ingestion { get; set; } = new(); diff --git a/src/Zilean.Shared/Features/Utilities/ApiKey.cs b/src/Zilean.Shared/Features/Utilities/ApiKey.cs new file mode 100644 index 0000000..63119ab --- /dev/null +++ b/src/Zilean.Shared/Features/Utilities/ApiKey.cs @@ -0,0 +1,6 @@ +namespace Zilean.Shared.Features.Utilities; + +public static class ApiKey +{ + public static string Generate() => $"{Guid.NewGuid():N}{Guid.NewGuid():N}"; +} diff --git a/tests/Zilean.Tests/Tests/ConfigurationTests.cs b/tests/Zilean.Tests/Tests/ConfigurationTests.cs index 97068f4..acc8304 100644 --- a/tests/Zilean.Tests/Tests/ConfigurationTests.cs +++ b/tests/Zilean.Tests/Tests/ConfigurationTests.cs @@ -21,9 +21,6 @@ public class ConfigurationTests "Database": { "ConnectionString": "Host=localhost;Database=zilean;Username=postgres;Password=postgres;Include Error Detail=true;Timeout=300;CommandTimeout=300;" }, - "Prowlarr": { - "EnableEndpoint": true - }, "Torrents": { "EnableEndpoint": true }, @@ -100,10 +97,6 @@ public void adds_json_configuration_file_to_builder_with_fake_filesystem_gets_in .Be( "Host=localhost;Database=zilean;Username=postgres;Password=postgres;Include Error Detail=true;Timeout=300;CommandTimeout=300;"); - // Prowlarr - zileanConfig.Prowlarr.Should().NotBeNull(); - zileanConfig.Prowlarr.EnableEndpoint.Should().BeTrue(); - // Torrents zileanConfig.Torrents.Should().NotBeNull(); zileanConfig.Torrents.EnableEndpoint.Should().BeTrue();