From 5d80de4ad8e88f1a2ff85a30c73aae5a787fc46b Mon Sep 17 00:00:00 2001 From: Romeo Dumitrescu Date: Sun, 17 Oct 2021 01:07:17 +0300 Subject: [PATCH] Hotfix: Garbage collection and ConfigMap watcher - Fixed garbage collection profile (broken commit) - Fixed ConfigMap mirror being disabled (broken commit) --- src/ES.Kubernetes.Reflector.sln | 2 +- .../Core/Mirroring/Constants/Annotations.cs | 76 +------------ .../Core/Mirroring/ResourceMirror.cs | 107 ++++++++---------- .../Core/Watchers/WatcherBackgroundService.cs | 5 +- .../ES.Kubernetes.Reflector.csproj | 1 + src/ES.Kubernetes.Reflector/Program.cs | 2 +- 6 files changed, 59 insertions(+), 134 deletions(-) diff --git a/src/ES.Kubernetes.Reflector.sln b/src/ES.Kubernetes.Reflector.sln index 82040c2..f726f1a 100644 --- a/src/ES.Kubernetes.Reflector.sln +++ b/src/ES.Kubernetes.Reflector.sln @@ -3,7 +3,7 @@ Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Version 17 VisualStudioVersion = 17.0.31710.8 MinimumVisualStudioVersion = 10.0.40219.1 -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ES.Kubernetes.Reflector", "ES.Kubernetes.Reflector\ES.Kubernetes.Reflector.csproj", "{96CDE0CF-7782-490B-8AF6-4219DB0236B3}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ES.Kubernetes.Reflector", "ES.Kubernetes.Reflector\ES.Kubernetes.Reflector.csproj", "{96CDE0CF-7782-490B-8AF6-4219DB0236B3}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution diff --git a/src/ES.Kubernetes.Reflector/Core/Mirroring/Constants/Annotations.cs b/src/ES.Kubernetes.Reflector/Core/Mirroring/Constants/Annotations.cs index 3bb6237..fe2e302 100644 --- a/src/ES.Kubernetes.Reflector/Core/Mirroring/Constants/Annotations.cs +++ b/src/ES.Kubernetes.Reflector/Core/Mirroring/Constants/Annotations.cs @@ -3,7 +3,6 @@ public static class Annotations { public const string Prefix = "reflector.v1.k8s.emberstack.com"; - public const string MetaPrefix = $"{Prefix}"; public static class Reflection { @@ -14,79 +13,10 @@ public static class Reflection public static string Reflects => $"{Prefix}/reflects"; - public static string MetaAutoReflects => $"{MetaPrefix}/auto-reflects"; - public static string MetaReflectedVersion => $"{MetaPrefix}/reflected-version"; - public static string MetaReflectedAt => $"{MetaPrefix}/reflected-at"; + public static string MetaAutoReflects => $"{Prefix}/auto-reflects"; + public static string MetaReflectedVersion => $"{Prefix}/reflected-version"; + public static string MetaReflectedAt => $"{Prefix}/reflected-at"; - #region Forti - - public static string FortiEnabled => $"{Prefix}/reflection-forti-enabled"; - public static string FortiHosts => $"{Prefix}/reflection-forti-hosts"; - public static string FortiCertificate => $"{Prefix}/reflection-forti-certificate"; - - #endregion - - - #region Ubiquiti - - public static string UbiquitiEnabled => $"{Prefix}/reflection-ubiquiti-enabled"; - public static string UbiquitiHosts => $"{Prefix}/reflection-ubiquiti-hosts"; - public static string UbiquitiCertificate => $"{Prefix}/reflection-ubiquiti-certificate"; - - #endregion - - #region VMware - - public static string VMwareEnabled => $"{Prefix}/reflection-vmware-enabled"; - public static string VMwareHosts => $"{Prefix}/reflection-vmware-hosts"; - public static string VMwareCertificate => $"{Prefix}/reflection-vmware-certificate"; - - #endregion - - #region FreeNAS - - public static string FreeNasEnabled => $"{Prefix}/reflection-freenas-enabled"; - public static string FreeNasHosts => $"{Prefix}/reflection-freenas-hosts"; - public static string FreeNasCertificate => $"{Prefix}/reflection-freenas-certificate"; - - #endregion - } - - public static class CertManagerCertificate - { - public static string SecretReflectionAllowed => $"{Prefix}/secret-reflection-allowed"; - public static string SecretReflectionAllowedNamespaces => $"{Prefix}/secret-reflection-allowed-namespaces"; - - public static string SecretReflectionAutoEnabled => $"{Prefix}/secret-reflection-auto-enabled"; - public static string SecretReflectionAutoNamespaces => $"{Prefix}/secret-reflection-auto-namespaces"; - - public static string SecretFortiEnabled => $"{Prefix}/secret-reflection-forti-enabled"; - public static string SecretFortiHosts => $"{Prefix}/secret-reflection-forti-hosts"; - public static string SecretFortiCertificate => $"{Prefix}/secret-reflection-forti-certificate"; - - #region Ubiquiti - - public static string SecretUbiquitiEnabled => $"{Prefix}/secret-reflection-ubiquiti-enabled"; - public static string SecretUbiquitiHosts => $"{Prefix}/secret-reflection-ubiquiti-hosts"; - public static string SecretUbiquitiCertificate => $"{Prefix}/secret-reflection-ubiquiti-certificate"; - - #endregion - - #region VMware - - public static string SecretVMwareEnabled => $"{Prefix}/secret-reflection-vmware-enabled"; - public static string SecretVMwareHosts => $"{Prefix}/secret-reflection-vmware-hosts"; - public static string SecretVMwareCertificate => $"{Prefix}/secret-reflection-vmware-certificate"; - - #endregion - - #region FreeNAS - - public static string SecretFreeNasEnabled => $"{Prefix}/secret-reflection-freenas-enabled"; - public static string SecretFreeNasHosts => $"{Prefix}/secret-reflection-freenas-hosts"; - public static string SecretFreeNasCertificate => $"{Prefix}/secret-reflection-freenas-certificate"; - - #endregion } } \ No newline at end of file diff --git a/src/ES.Kubernetes.Reflector/Core/Mirroring/ResourceMirror.cs b/src/ES.Kubernetes.Reflector/Core/Mirroring/ResourceMirror.cs index 4925de9..31df6b4 100644 --- a/src/ES.Kubernetes.Reflector/Core/Mirroring/ResourceMirror.cs +++ b/src/ES.Kubernetes.Reflector/Core/Mirroring/ResourceMirror.cs @@ -58,7 +58,8 @@ public async Task Handle(WatcherEvent notification, CancellationToken cancellati case TResource resource: if (await OnResourceIgnoreCheck(resource)) return; - Logger.LogTrace("Handling {eventType} {resourceType} {resourceRef}", notification.Type, resource.Kind, resource.GetRef()); + Logger.LogTrace("Handling {eventType} {resourceType} {resourceRef}", notification.Type, resource.Kind, + resource.GetRef()); var itemRef = resource.GetRef(); @@ -68,37 +69,37 @@ public async Task Handle(WatcherEvent notification, CancellationToken cancellati { case WatchEventType.Added: case WatchEventType.Modified: - { - await HandleUpsert(resource, notification.Type, cancellationToken); - } + { + await HandleUpsert(resource, notification.Type, cancellationToken); + } break; case WatchEventType.Deleted: + { + _propertiesCache.Remove(itemRef, out _); + var properties = resource.GetReflectionProperties(); + + + if (!properties.IsReflection) { - _propertiesCache.Remove(itemRef, out _); - var properties = resource.GetReflectionProperties(); - - - if (!properties.IsReflection) - { - if (properties.Allowed && properties.AutoEnabled && - _autoReflectionCache.TryGetValue(itemRef, out var reflectionList)) - foreach (var reflectionId in reflectionList.ToArray()) - { - Logger.LogDebug("Deleting {id} - Source {sourceId} has been deleted", reflectionId, - itemRef); - await OnResourceDelete(reflectionId); - } - - _autoSources.Remove(itemRef, out _); - _directReflectionCache.Remove(itemRef, out _); - _autoReflectionCache.Remove(itemRef, out _); - } - else - { - foreach (var item in _directReflectionCache) item.Value.Remove(itemRef); - foreach (var item in _autoReflectionCache) item.Value.Remove(itemRef); - } + if (properties.Allowed && properties.AutoEnabled && + _autoReflectionCache.TryGetValue(itemRef, out var reflectionList)) + foreach (var reflectionId in reflectionList.ToArray()) + { + Logger.LogDebug("Deleting {id} - Source {sourceId} has been deleted", reflectionId, + itemRef); + await OnResourceDelete(reflectionId); + } + + _autoSources.Remove(itemRef, out _); + _directReflectionCache.Remove(itemRef, out _); + _autoReflectionCache.Remove(itemRef, out _); } + else + { + foreach (var item in _directReflectionCache) item.Value.Remove(itemRef); + foreach (var item in _autoReflectionCache) item.Value.Remove(itemRef); + } + } break; default: return; @@ -106,28 +107,29 @@ public async Task Handle(WatcherEvent notification, CancellationToken cancellati break; case V1Namespace ns: - { - if (notification.Type != WatchEventType.Added) return; - Logger.LogTrace("Handling {eventType} {resourceType} {resourceRef}", notification.Type, ns.Kind, ns.GetRef()); + { + if (notification.Type != WatchEventType.Added) return; + Logger.LogTrace("Handling {eventType} {resourceType} {resourceRef}", notification.Type, ns.Kind, + ns.GetRef()); - foreach (var autoSourceRef in _autoSources.Keys) + foreach (var autoSourceRef in _autoSources.Keys) + { + var properties = _propertiesCache[autoSourceRef]; + if (properties.CanBeAutoReflectedToNamespace(ns.Name())) { - var properties = _propertiesCache[autoSourceRef]; - if (properties.CanBeAutoReflectedToNamespace(ns.Name())) - { - var reflectionRef = new KubeRef(ns.Name(), autoSourceRef.Name); - var autoReflectionList = _autoReflectionCache.GetOrAdd(autoSourceRef, new List()); + var reflectionRef = new KubeRef(ns.Name(), autoSourceRef.Name); + var autoReflectionList = _autoReflectionCache.GetOrAdd(autoSourceRef, new List()); - if (autoReflectionList.Contains(reflectionRef)) return; + if (autoReflectionList.Contains(reflectionRef)) return; - await ResourceReflect(autoSourceRef, reflectionRef, null, null, true); + await ResourceReflect(autoSourceRef, reflectionRef, null, null, true); - if (!autoReflectionList.Contains(reflectionRef)) - autoReflectionList.Add(reflectionRef); - } + if (!autoReflectionList.Contains(reflectionRef)) + autoReflectionList.Add(reflectionRef); } } + } break; } } @@ -250,7 +252,6 @@ private async Task HandleUpsert(TResource resource, WatchEventType eventType, Ca } await ResourceReflect(sourceRef, resourceRef, null, resource, false); - } @@ -261,7 +262,8 @@ private async Task HandleUpsert(TResource resource, WatchEventType eventType, Ca } } - private async Task TriggerAutoReflectionForSource(KubeRef resourceRef, KubeRef reflectionRef, CancellationToken cancellationToken) + private async Task TriggerAutoReflectionForSource(KubeRef resourceRef, KubeRef reflectionRef, + CancellationToken cancellationToken) { if (_notFoundCache.ContainsKey(resourceRef)) { @@ -279,6 +281,7 @@ private async Task TriggerAutoReflectionForSource(KubeRef resourceRef, KubeRef r await OnResourceDelete(reflectionRef); return; } + properties = resourceCached.GetReflectionProperties(); } else @@ -326,17 +329,14 @@ private async Task AutoReflectionForSource(KubeRef resourceRef, TResource? resou foreach (var kubeRef in toDelete) await OnResourceDelete(kubeRef); resourceInstance ??= await TryResourceGet(resourceRef); - if (resourceInstance is null) - { - return; - } + if (resourceInstance is null) return; var resource = resourceInstance; var toCreate = namespaces .Where(s => s.Name() != resourceRef.Namespace) .Where(s => matches.All(m => m.Namespace() != s.Name()) && properties.CanBeAutoReflectedToNamespace(s.Name())) - .Select(s => new KubeRef(s.Name(),resource.Name())).ToList(); + .Select(s => new KubeRef(s.Name(), resource.Name())).ToList(); var toUpdate = matches .Where(s => s.Namespace() != resourceRef.Namespace) @@ -359,16 +359,11 @@ private async Task AutoReflectionForSource(KubeRef resourceRef, TResource? resou autoReflectionList.AddRange(toSkip); autoReflectionList.AddRange(toUpdate); - foreach (var reflectionRef in toCreate) - { - await ResourceReflect(resourceRef, reflectionRef, resource, null, true); - - } + foreach (var reflectionRef in toCreate) await ResourceReflect(resourceRef, reflectionRef, resource, null, true); foreach (var reflectionRef in toUpdate) { var reflection = matches.Single(s => s.GetRef().Equals(reflectionRef)); await ResourceReflect(resourceRef, reflectionRef, resource, reflection, true); - } } @@ -466,8 +461,6 @@ private async Task ResourceReflect(KubeRef sourceId, KubeRef targetId, TResource } - - protected abstract Task OnResourceApplyPatch(V1Patch source, KubeRef refId); protected abstract Task OnResourceConfigurePatch(TResource source, JsonPatchDocument patchDoc); protected abstract Task OnResourceCreate(TResource item, string ns); diff --git a/src/ES.Kubernetes.Reflector/Core/Watchers/WatcherBackgroundService.cs b/src/ES.Kubernetes.Reflector/Core/Watchers/WatcherBackgroundService.cs index c2a830a..f11b8cc 100644 --- a/src/ES.Kubernetes.Reflector/Core/Watchers/WatcherBackgroundService.cs +++ b/src/ES.Kubernetes.Reflector/Core/Watchers/WatcherBackgroundService.cs @@ -32,9 +32,10 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) try { Logger.LogInformation("Requesting {type} resources", typeof(TResource).Name); - var watcher = OnGetWatcher(stoppingToken); + using var watcher = OnGetWatcher(stoppingToken); + var watchList = watcher.WatchAsync(); - await foreach (var (type, item) in watcher.WatchAsync() + await foreach (var (type, item) in watchList .WithCancellation(stoppingToken)) await Mediator.Publish(new WatcherEvent { diff --git a/src/ES.Kubernetes.Reflector/ES.Kubernetes.Reflector.csproj b/src/ES.Kubernetes.Reflector/ES.Kubernetes.Reflector.csproj index 75ae1e1..9c46ff1 100644 --- a/src/ES.Kubernetes.Reflector/ES.Kubernetes.Reflector.csproj +++ b/src/ES.Kubernetes.Reflector/ES.Kubernetes.Reflector.csproj @@ -5,6 +5,7 @@ enable enable Linux + fakse diff --git a/src/ES.Kubernetes.Reflector/Program.cs b/src/ES.Kubernetes.Reflector/Program.cs index 57ac033..baf52f1 100644 --- a/src/ES.Kubernetes.Reflector/Program.cs +++ b/src/ES.Kubernetes.Reflector/Program.cs @@ -57,7 +57,7 @@ container.RegisterType().AsImplementedInterfaces().SingleInstance(); container.RegisterType().AsImplementedInterfaces().SingleInstance(); - //container.RegisterType().AsImplementedInterfaces().SingleInstance(); + container.RegisterType().AsImplementedInterfaces().SingleInstance(); }); builder.WebHost.UseUrls("http://*:25080");