feat(kds): compact subscriptions in insights

[jakubdyszkiewicz]

Checklist prior to review

Opt-in to compact stale subscriptions. This way we can reduce size of ZoneInsight.
GUI always takes Config from the last subscriptions. If stats for specific resources are missing it looks like this

Here are 2 compacted subscription as well as one non compacted

  {
   "id": "cdf01176-53c9-419f-a90d-e81e04abe0b4",
   "globalInstanceId": "81142868492e-b743",
   "connectTime": "2023-10-04T10:22:40.927040625Z",
   "disconnectTime": "2023-10-04T10:24:42.051669417Z",
   "status": {
    "lastUpdateTime": "2023-10-04T10:23:41.944032375Z",
    "total": {
     "responsesSent": "57",
     "responsesAcknowledged": "54",
     "responsesRejected": "3"
    }
   },
   "version": {
    "kumaCp": {
     "version": "0.0.0-preview.vlocal-build",
     "gitTag": "local-build",
     "gitCommit": "local-build",
     "buildDate": "local-build",
     "kumaCpGlobalCompatible": true
    }
   }
  },
  {
   "id": "ce2cb3ae-6d18-49bf-b5d5-30170f5ca9bd",
   "globalInstanceId": "81142868492e-b743",
   "connectTime": "2023-10-04T10:25:30.831541884Z",
   "disconnectTime": "2023-10-04T10:26:53.499099214Z",
   "status": {
    "lastUpdateTime": "2023-10-04T10:25:31.945885760Z",
    "total": {
     "responsesSent": "34",
     "responsesAcknowledged": "34"
    }
   },
   "version": {
    "kumaCp": {
     "version": "0.0.0-preview.vlocal-build",
     "gitTag": "local-build",
     "gitCommit": "local-build",
     "buildDate": "local-build",
     "kumaCpGlobalCompatible": true
    }
   }
  },
  {
   "id": "b6d554f7-10fd-440e-9122-c4f3286c9bb3",
   "globalInstanceId": "81142868492e-b743",
   "connectTime": "2023-10-04T10:27:40.430499097Z",
   "status": {
    "lastUpdateTime": "2023-10-04T10:27:41.541478791Z",
    "total": {
     "responsesSent": "34",
     "responsesAcknowledged": "34"
    },
    "stat": {
     "CircuitBreaker": {
      "responsesSent": "1",
      "responsesAcknowledged": "1"
     },
     "Config": {
      "responsesSent": "1",
      "responsesAcknowledged": "1"
     },
     "ExternalService": {
      "responsesSent": "1",
      "responsesAcknowledged": "1"
     },
     "FaultInjection": {
      "responsesSent": "1",
      "responsesAcknowledged": "1"
     },
     "GlobalSecret": {
      "responsesSent": "1",
      "responsesAcknowledged": "1"
     },
     "HealthCheck": {
      "responsesSent": "1",
      "responsesAcknowledged": "1"
     },
     "Mesh": {
      "responsesSent": "1",
      "responsesAcknowledged": "1"
     },
     "MeshAccessLog": {
      "responsesSent": "1",
      "responsesAcknowledged": "1"
     },
     "MeshCircuitBreaker": {
      "responsesSent": "1",
      "responsesAcknowledged": "1"
     },
     "MeshFaultInjection": {
      "responsesSent": "1",
      "responsesAcknowledged": "1"
     },
     "MeshGateway": {
      "responsesSent": "1",
      "responsesAcknowledged": "1"
     },
     "MeshGatewayRoute": {
      "responsesSent": "1",
      "responsesAcknowledged": "1"
     },
     "MeshHTTPRoute": {
      "responsesSent": "1",
      "responsesAcknowledged": "1"
     },
     "MeshHealthCheck": {
      "responsesSent": "1",
      "responsesAcknowledged": "1"
     },
     "MeshLoadBalancingStrategy": {
      "responsesSent": "1",
      "responsesAcknowledged": "1"
     },
     "MeshProxyPatch": {
      "responsesSent": "1",
      "responsesAcknowledged": "1"
     },
     "MeshRateLimit": {
      "responsesSent": "1",
      "responsesAcknowledged": "1"
     },
     "MeshRetry": {
      "responsesSent": "1",
      "responsesAcknowledged": "1"
     },
     "MeshTCPRoute": {
      "responsesSent": "1",
      "responsesAcknowledged": "1"
     },
     "MeshTimeout": {
      "responsesSent": "1",
      "responsesAcknowledged": "1"
     },
     "MeshTrace": {
      "responsesSent": "1",
      "responsesAcknowledged": "1"
     },
     "MeshTrafficPermission": {
      "responsesSent": "1",
      "responsesAcknowledged": "1"
     },
     "ProxyTemplate": {
      "responsesSent": "1",
      "responsesAcknowledged": "1"
     },
     "RateLimit": {
      "responsesSent": "1",
      "responsesAcknowledged": "1"
     },
     "Retry": {
      "responsesSent": "1",
      "responsesAcknowledged": "1"
     },
     "Secret": {
      "responsesSent": "1",
      "responsesAcknowledged": "1"
     },
     "ServiceInsight": {
      "responsesSent": "1",
      "responsesAcknowledged": "1"
     },
     "Timeout": {
      "responsesSent": "1",
      "responsesAcknowledged": "1"
     },
     "TrafficLog": {
      "responsesSent": "1",
      "responsesAcknowledged": "1"
     },
     "TrafficPermission": {
      "responsesSent": "1",
      "responsesAcknowledged": "1"
     },
     "TrafficRoute": {
      "responsesSent": "1",
      "responsesAcknowledged": "1"
     },
     "TrafficTrace": {
      "responsesSent": "1",
      "responsesAcknowledged": "1"
     },
     "VirtualOutbound": {
      "responsesSent": "1",
      "responsesAcknowledged": "1"
     },
     "ZoneIngress": {
      "responsesSent": "1",
      "responsesAcknowledged": "1"
     }
    }
   },
   "version": {
    "kumaCp": {
     "version": "0.0.0-preview.vlocal-build",
     "gitTag": "local-build",
     "gitCommit": "local-build",
     "buildDate": "local-build",
     "kumaCpGlobalCompatible": true
    }
   },
   "generation": 15,
   "config": "{\"access\":{\"static\":{\"adminResources\":{\"groups\":[\"mesh-system:admin\"],\"users\":[\"mesh-system:admin\"]},\"generateDpToken\":{\"groups\":[\"mesh-system:admin\"],\"users\":[\"mesh-system:admin\"]},\"generateUserToken\":{\"groups\":[\"mesh-system:admin\"],\"users\":[\"mesh-system:admin\"]},\"generateZoneToken\":{\"groups\":[\"mesh-system:admin\"],\"users\":[\"mesh-system:admin\"]},\"viewClusters\":{\"groups\":[\"mesh-system:unauthenticated\",\"mesh-system:authenticated\"],\"users\":[]},\"viewConfigDump\":{\"groups\":[\"mesh-system:unauthenticated\",\"mesh-system:authenticated\"],\"users\":[]},\"viewStats\":{\"groups\":[\"mesh-system:unauthenticated\",\"mesh-system:authenticated\"],\"users\":[]}},\"type\":\"static\"},\"apiServer\":{\"auth\":{\"clientCertsDir\":\"\"},\"authn\":{\"localhostIsAdmin\":false,\"tokens\":{\"bootstrapAdminToken\":true,\"enableIssuer\":true,\"validator\":{\"publicKeys\":[],\"useSecrets\":true}},\"type\":\"tokens\"},\"basePath\":\"/\",\"corsAllowedDomains\":[\".*\"],\"gui\":{\"basePath\":\"/gui\",\"enabled\":true,\"rootUrl\":\"\"},\"http\":{\"enabled\":true,\"interface\":\"0.0.0.0\",\"port\":5681},\"https\":{\"enabled\":true,\"interface\":\"0.0.0.0\",\"port\":5682,\"requireClientCert\":false,\"tlsCaFile\":\"\",\"tlsCertFile\":\"/var/run/secrets/kuma.io/tls-cert/tls.crt\",\"tlsCipherSuites\":[],\"tlsKeyFile\":\"/var/run/secrets/kuma.io/tls-cert/tls.key\",\"tlsMaxVersion\":\"\",\"tlsMinVersion\":\"TLSv1_2\"},\"readOnly\":true,\"rootUrl\":\"\"},\"bootstrapServer\":{\"params\":{\"adminAccessLogPath\":\"/dev/null\",\"adminAddress\":\"127.0.0.1\",\"adminPort\":9901,\"xdsConnectTimeout\":\"1s\",\"xdsHost\":\"\",\"xdsPort\":5678}},\"defaults\":{\"skipMeshCreation\":false,\"skipTenantResources\":false},\"diagnostics\":{\"debugEndpoints\":false,\"serverPort\":5680,\"tlsCertFile\":\"/var/run/secrets/kuma.io/tls-cert/tls.crt\",\"tlsCipherSuites\":[],\"tlsEnabled\":false,\"tlsKeyFile\":\"/var/run/secrets/kuma.io/tls-cert/tls.key\",\"tlsMaxVersion\":\"\",\"tlsMinVersion\":\"TLSv1_2\"},\"dnsServer\":{\"CIDR\":\"240.0.0.0/4\",\"domain\":\"mesh\",\"serviceVipEnabled\":true,\"serviceVipPort\":80},\"dpServer\":{\"auth\":{\"type\":\"serviceAccountToken\",\"useTokenPath\":false},\"authn\":{\"dpProxy\":{\"dpToken\":{\"enableIssuer\":true,\"validator\":{\"publicKeys\":[],\"useSecrets\":true}},\"type\":\"serviceAccountToken\"},\"enableReloadableTokens\":false,\"zoneProxy\":{\"type\":\"serviceAccountToken\",\"zoneToken\":{\"enableIssuer\":true,\"validator\":{\"publicKeys\":[],\"useSecrets\":true}}}},\"hds\":{\"checkDefaults\":{\"healthyThreshold\":1,\"interval\":\"1s\",\"noTrafficInterval\":\"1s\",\"timeout\":\"2s\",\"unhealthyThreshold\":1},\"enabled\":false,\"interval\":\"5s\",\"refreshInterval\":\"10s\"},\"port\":5678,\"readHeaderTimeout\":\"5s\",\"tlsCertFile\":\"/var/run/secrets/kuma.io/tls-cert/tls.crt\",\"tlsCipherSuites\":[],\"tlsKeyFile\":\"/var/run/secrets/kuma.io/tls-cert/tls.key\",\"tlsMaxVersion\":\"\",\"tlsMinVersion\":\"TLSv1_2\"},\"environment\":\"kubernetes\",\"eventBus\":{\"bufferSize\":100},\"experimental\":{\"gatewayAPI\":false,\"ingressTagFilters\":[],\"kdsDeltaEnabled\":false,\"kdsEventBasedWatchdog\":{\"delayFullResync\":false,\"enabled\":false,\"flushInterval\":\"5s\",\"fullResyncInterval\":\"1m0s\"},\"kubeOutboundsAsVIPs\":true,\"useTagFirstVirtualOutboundModel\":false},\"general\":{\"dnsCacheTTL\":\"10s\",\"tlsCertFile\":\"/var/run/secrets/kuma.io/tls-cert/tls.crt\",\"tlsCipherSuites\":[],\"tlsKeyFile\":\"/var/run/secrets/kuma.io/tls-cert/tls.key\",\"tlsMaxVersion\":\"\",\"tlsMinVersion\":\"TLSv1_2\",\"workDir\":\"/home/nonroot/.kuma\"},\"interCp\":{\"catalog\":{\"heartbeatInterval\":\"5s\",\"instanceAddress\":\"10.42.0.10\",\"writerInterval\":\"15s\"},\"server\":{\"port\":5683,\"tlsCipherSuites\":[],\"tlsMaxVersion\":\"\",\"tlsMinVersion\":\"TLSv1_2\"}},\"metrics\":{\"controlPlane\":{\"reportResourcesCount\":true},\"dataplane\":{\"idleTimeout\":\"5m0s\",\"subscriptionLimit\":2},\"mesh\":{\"bufferSize\":1000,\"eventProcessors\":1,\"fullResyncInterval\":\"20s\",\"maxResyncTimeout\":\"0s\",\"minResyncInterval\":\"1s\",\"minResyncTimeout\":\"0s\"},\"zone\":{\"compactFinished\":false,\"idleTimeout\":\"5m0s\",\"subscriptionLimit\":10}},\"mode\":\"zone\",\"monitoringAssignmentServer\":{\"apiVersions\":[\"v1\"],\"assignmentRefreshInterval\":\"1s\",\"defaultFetchTimeout\":\"30s\",\"port\":5676,\"tlsCertFile\":\"/var/run/secrets/kuma.io/tls-cert/tls.crt\",\"tlsCipherSuites\":[],\"tlsEnabled\":false,\"tlsKeyFile\":\"/var/run/secrets/kuma.io/tls-cert/tls.key\",\"tlsMaxVersion\":\"\",\"tlsMinVersion\":\"TLSv1_2\"},\"multizone\":{\"global\":{\"kds\":{\"grpcPort\":5685,\"maxMsgSize\":10485760,\"msgSendTimeout\":\"1m0s\",\"nackBackoff\":\"5s\",\"refreshInterval\":\"1s\",\"tlsCertFile\":\"/var/run/secrets/kuma.io/tls-cert/tls.crt\",\"tlsCipherSuites\":[],\"tlsEnabled\":true,\"tlsKeyFile\":\"/var/run/secrets/kuma.io/tls-cert/tls.key\",\"tlsMaxVersion\":\"\",\"tlsMinVersion\":\"TLSv1_2\",\"zoneInsightFlushInterval\":\"10s\"}},\"zone\":{\"globalAddress\":\"grpcs://172.18.0.8:5685\",\"kds\":{\"maxMsgSize\":10485760,\"msgSendTimeout\":\"1m0s\",\"nackBackoff\":\"5s\",\"refreshInterval\":\"1s\",\"rootCaFile\":\"\",\"tlsSkipVerify\":true},\"name\":\"kuma-1-zone\"}},\"proxy\":{\"gateway\":{\"globalDownstreamMaxConnections\":50000}},\"reports\":{\"enabled\":false},\"runtime\":{\"kubernetes\":{\"admissionServer\":{\"address\":\"\",\"certDir\":\"/var/run/secrets/kuma.io/tls-cert\",\"port\":5443},\"clientConfig\":{\"burstQps\":100,\"qps\":100},\"controlPlaneServiceName\":\"kuma-control-plane\",\"controllersConcurrency\":{\"podController\":10},\"injector\":{\"builtinDNS\":{\"enabled\":true,\"port\":15053},\"caCertFile\":\"/var/run/secrets/kuma.io/tls-cert/ca.crt\",\"cniEnabled\":false,\"containerPatches\":[],\"ebpf\":{\"bpffsPath\":\"/sys/fs/bpf\",\"cgroupPath\":\"/sys/fs/cgroup\",\"enabled\":false,\"instanceIPEnvVarName\":\"INSTANCE_IP\",\"programsSourcePath\":\"/kuma/ebpf\"},\"exceptions\":{\"labels\":{\"openshift.io/build.name\":\"*\",\"openshift.io/deployer-pod-for.name\":\"*\"}},\"initContainer\":{\"image\":\"kumahq/kuma-init:0.0.0-preview.vlocal-build\"},\"sidecarContainer\":{\"drainTime\":\"30s\",\"envVars\":{},\"gid\":5678,\"image\":\"kumahq/kuma-dp:0.0.0-preview.vlocal-build\",\"livenessProbe\":{\"failureThreshold\":12,\"initialDelaySeconds\":60,\"periodSeconds\":5,\"timeoutSeconds\":3},\"readinessProbe\":{\"failureThreshold\":12,\"initialDelaySeconds\":1,\"periodSeconds\":5,\"successThreshold\":1,\"timeoutSeconds\":3},\"redirectPortInbound\":15006,\"redirectPortInboundV6\":15010,\"redirectPortOutbound\":15001,\"resources\":{\"limits\":{\"cpu\":\"1000m\",\"memory\":\"512Mi\"},\"requests\":{\"cpu\":\"50m\",\"memory\":\"64Mi\"}},\"uid\":5678,\"waitForDataplaneReady\":false},\"sidecarTraffic\":{\"excludeInboundPorts\":[],\"excludeOutboundPorts\":[]},\"transparentProxyV1\":false,\"virtualProbesEnabled\":true,\"virtualProbesPort\":9000},\"leaderElection\":{\"leaseDuration\":\"15s\",\"renewDeadline\":\"10s\"},\"marshalingCacheExpirationTime\":\"5m0s\",\"nodeTaintController\":{\"cniApp\":\"\",\"cniNamespace\":\"kube-system\",\"enabled\":false},\"serviceAccountName\":\"system:serviceaccount:kuma-system:kuma-control-plane\"},\"universal\":{\"dataplaneCleanupAge\":\"72h0m0s\"}},\"store\":{\"cache\":{\"enabled\":true,\"expirationTime\":\"1s\"},\"kubernetes\":{\"systemNamespace\":\"kuma-system\"},\"postgres\":{\"connectionTimeout\":5,\"dbName\":\"kuma\",\"driverName\":\"pgx\",\"healthCheckInterval\":\"30s\",\"host\":\"127.0.0.1\",\"maxConnectionLifetime\":\"1h0m0s\",\"maxConnectionLifetimeJitter\":\"1m0s\",\"maxIdleConnections\":50,\"maxListQueryElements\":0,\"maxOpenConnections\":50,\"maxReconnectInterval\":\"1m0s\",\"minOpenConnections\":0,\"minReconnectInterval\":\"10s\",\"password\":\"*****\",\"port\":15432,\"readReplica\":{\"host\":\"\",\"port\":5432,\"ratio\":100},\"tls\":{\"caPath\":\"\",\"certPath\":\"\",\"disableSSLSNI\":false,\"keyPath\":\"\",\"mode\":\"disable\"},\"user\":\"kuma\"},\"type\":\"kubernetes\",\"unsafeDelete\":true,\"upsert\":{\"conflictRetryBaseBackoff\":\"200ms\",\"conflictRetryJitterPercent\":30,\"conflictRetryMaxTimes\":10}},\"tracing\":{\"openTelemetry\":{}},\"xdsServer\":{\"dataplaneConfigurationRefreshInterval\":\"1s\",\"dataplaneDeregistrationDelay\":\"10s\",\"dataplaneStatusFlushInterval\":\"10s\",\"nackBackoff\":\"5s\"}}"
  }

• Link to relevant issue as well as docs and UI issues --
• This will not break child repos: it doesn't hardcode values (.e.g "kumahq" as a image registry) and it will work on Windows, system specific functions like syscall.Mkfifo have equivalent implementation on the other OS --
• Tests (Unit test, E2E tests, manual test on universal and k8s) --
• Do you need to update UPGRADE.md? --
• Does it need to be backported according to the backporting policy? (this GH action will add "backport" label based on these file globs, if you want to prevent it from adding the "backport" label use no-backport-autolabel label) --
• Do you need to explicitly set a > Changelog: entry here or add a ci/ label to run fewer/more tests?