Merge branch 'main' into improvement/calico_typha_monitoring

Netcracker · Oct 17, 2023 · 409fd8b · 409fd8b
2 parents ad53592 + 3af8622
commit 409fd8b
Show file tree

Hide file tree

Showing 8 changed files with 226 additions and 15 deletions.
diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
@@ -9,11 +9,12 @@ env:
 jobs:
   license:
      runs-on: ubuntu-latest
+     if: github.ref_name == 'main' || endsWith(github.ref_name, '_branch')
      steps:
        - uses: actions/checkout@v3
+         with:
+           token: ${{ secrets.NCCLPLCI_PAT }}
        - run: docker run -v "${PWD}:/src" -i ghcr.io/google/addlicense -v -c "${{ env.COPYRIGHT_COMPANY }}" -y "${{ env.COPYRIGHT_YEAR }}" $(find . -type f -name "*.py" | xargs echo)
-       # This currently not works for protected branches,
-       # but all pushes to them should be done through PRs that already contain inserted license.
        - uses: stefanzweifel/git-auto-commit-action@v4
          with:
            commit_message: Auto-update license header

diff --git a/documentation/Installation.md b/documentation/Installation.md
@@ -2509,15 +2509,24 @@ services:
 
 *Can restart service*: Always yes, container kube-apiserver.
 
-*OS specific*: No.
+*Overwrite files*: Yes, `/etc/kubernetes/audit-policy.yaml` backup is created.
 
-*Logging level*:
+*OS specific*: No
+
+For more information about Kubernetes auditing, refer to the official documentation at [https://kubernetes.io/docs/tasks/debug/debug-cluster/audit/](https://kubernetes.io/docs/tasks/debug/debug-cluster/audit/).
+
+**Logging level**:
 `None` - do not log;
 `Metadata` — log request metadata: user, request time, target resource (pod, namespace, etc.), action type (verb), etc.;
 `Request` — log metadata and request body;
 `RequestResponse` - log metadata, request body and response body.
 
-*omitStages*: To skip any stages.
+**omitStages**: The list of stages for which no events are created.
+
+By default, the following policy is installed:
+
+<details>
+  <summary>Default Policy</summary>
 
 ```yaml
 services:
@@ -2532,13 +2541,26 @@ services:
         # Don't log read-only requests
         - level: None
           verbs: ["watch", "get", "list"]
-        # Don't log checking API access by Calico API server
+        # Don't log checking access by internal services
         - level: None
-          users: ["system:serviceaccount:calico-apiserver:calico-apiserver"]
+          userGroups:
+            - "system:serviceaccounts:calico-apiserver"
+            - "system:nodes"
           verbs: ["create"]
           resources:
             - group: "authorization.k8s.io"
               resources: ["subjectaccessreviews"]
+            - group: "authentication.k8s.io"
+              resources: ["tokenreviews"]
+        # Don't log update of ingress-controller-leader ConfigMap by ingress-nginx.
+        # This reproduces only for v1.2.0 and can be removed after its support stop.
+        - level: None
+          users: ["system:serviceaccount:ingress-nginx:ingress-nginx"]
+          verbs: ["update"]
+          resources:
+            - group: ""
+              resources: ["configmaps"]
+              resourceNames: ["ingress-controller-leader"]
         # Log all other resources in core and extensions at the request level.
         - level: Metadata
           verbs: ["create", "update", "patch", "delete", "deletecollection"]
@@ -2595,6 +2617,69 @@ services:
           - group: "authentication.k8s.io"
             resources: ["tokenreviews"]
           - group: "authorization.k8s.io"
+          - group: "projectcalico.org"
+            resources:
+              - bgpconfigurations
+              - bgpfilters
+              - bgppeers
+              - blockaffinities
+              - caliconodestatuses
+              - clusterinformations
+              - felixconfigurations
+              - globalnetworkpolicies
+              - globalnetworksets
+              - hostendpoints
+              - ipamconfigurations
+              - ippools
+              - ipreservations
+              - kubecontrollersconfigurations
+              - networkpolicies
+              - networksets
+              - profiles
+          - group: "crd.projectcalico.org"
+            resources:
+              - bgpconfigurations
+              - bgpfilters
+              - bgppeers
+              - blockaffinities
+              - caliconodestatuses
+              - clusterinformations
+              - felixconfigurations
+              - globalnetworkpolicies
+              - globalnetworksets
+              - hostendpoints
+              - ipamblocks
+              - ipamconfigs
+              - ipamhandles
+              - ippools
+              - ipreservations
+              - kubecontrollersconfigurations
+              - networkpolicies
+              - networksets
+```
+</details>
+
+It is possible not only to redefine the default policy, but also to extend it. For more information, refer to [List Merge Strategy](#list-merge-strategy).
+
+For example, consider you have an [operator](https://kubernetes.io/docs/concepts/extend-kubernetes/operator/) that constantly updates some Pods' labels and some ConfigMap to maintain the leadership.
+If you do not see any benefit from logging of such events, they can be disabled by specifying the following in `cluster.yaml`:
+
+```yaml
+services:
+  audit:
+    cluster_policy:
+      rules:
+      - level: None
+        userGroups: ["system:serviceaccounts:operator-namespace"]
+        verbs: ["patch", "update"]
+        namespaces: ["operator-namespace"]
+        resources:
+        - group: ""
+          resources: [pods]
+        - group: ""
+          resources: [configmaps]
+          resourceNames: [controller-leader]
+      - '<<': merge
 ```
 
 ##### Audit Daemon

diff --git a/kubemarine/core/resources.py b/kubemarine/core/resources.py
@@ -195,11 +195,14 @@ def get_nodes_context(self) -> dict:
                 # temporary cluster instance to detect initial nodes context.
                 light_cluster = self._new_cluster_instance(deepcopy(self.context))
                 light_cluster.enrich(custom_enrichment_fns=light_cluster.get_facts_enrichment_fns())
-                self._nodes_context = light_cluster.detect_nodes_context()
+                self._nodes_context = self._detect_nodes_context(light_cluster)
                 light_cluster.connection_pool.close()
 
         return self._nodes_context
 
+    def _detect_nodes_context(self, light_cluster: c.KubernetesCluster) -> dict:
+        return light_cluster.detect_nodes_context()
+
     def _new_cluster_instance(self, context: dict) -> c.KubernetesCluster:
         from kubemarine.core import flow
         cluster_class = c.KubernetesCluster

diff --git a/kubemarine/demo.py b/kubemarine/demo.py
@@ -21,7 +21,7 @@
 import time
 from abc import ABC
 from copy import deepcopy
-from typing import List, Dict, Union, Any, Optional, Mapping, Iterable, IO, Tuple
+from typing import List, Dict, Union, Any, Optional, Mapping, Iterable, IO, Tuple, cast
 
 import fabric  # type: ignore[import]
 import invoke
@@ -200,7 +200,8 @@ def __init__(self, context: dict, raw_inventory: dict, procedure_inventory: dict
         self.last_cluster: Optional[FakeKubernetesCluster] = None
         self.fake_shell = fake_shell if fake_shell else FakeShell()
         self.fake_fs = fake_fs if fake_fs else FakeFS()
-        self._nodes_context = nodes_context
+        # Let's do not assign self._nodes_context directly to make it more close to the real enrichment.
+        self.fake_nodes_context = nodes_context
         self._procedure_inventory = procedure_inventory
 
     def _load_inventory(self) -> None:
@@ -210,13 +211,22 @@ def _load_inventory(self) -> None:
     def _store_inventory(self) -> None:
         self.stored_inventory = deepcopy(self.formatted_inventory())
 
+    def _detect_nodes_context(self, light_cluster: KubernetesCluster) -> dict:
+        if self.fake_nodes_context is not None:
+            return self.fake_nodes_context
+
+        return super()._detect_nodes_context(light_cluster)
+
+    def _create_cluster(self, context: dict) -> KubernetesCluster:
+        self.last_cluster = cast(FakeKubernetesCluster, super()._create_cluster(context))
+        return self.last_cluster
+
     def _new_cluster_instance(self, context: dict) -> FakeKubernetesCluster:
-        self.last_cluster = FakeKubernetesCluster(
+        return FakeKubernetesCluster(
             self.raw_inventory(), context,
             procedure_inventory=self.procedure_inventory(), logger=self.logger(),
             fake_shell=self.fake_shell, fake_fs=self.fake_fs
         )
-        return self.last_cluster
 
 
 class FakeConnection(fabric.connection.Connection):  # type: ignore[misc]

diff --git a/kubemarine/patches/__init__.py b/kubemarine/patches/__init__.py
@@ -23,9 +23,11 @@
 
 from kubemarine.core.patch import Patch
 from kubemarine.patches.p2_calico_typha_metrics import EnableCalicoTyphaMetrics
+from kubemarine.patches.p1_enable_calico_audit import EnableCalicoAudit
 
 patches: List[Patch] = [
     EnableCalicoTyphaMetrics(),
+    EnableCalicoAudit(),
 ]
 """
 List of patches that is sorted according to the Patch.priority() before execution.

diff --git a/kubemarine/patches/p1_enable_calico_audit.py b/kubemarine/patches/p1_enable_calico_audit.py
@@ -0,0 +1,57 @@
+# Copyright 2021-2023 NetCracker Technology Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from textwrap import dedent
+
+from kubemarine.core import yaml_merger
+from kubemarine.core.action import Action
+from kubemarine.core.patch import RegularPatch
+from kubemarine.core.resources import DynamicResources
+from kubemarine.procedures import install
+
+
+class TheAction(Action):
+    def __init__(self) -> None:
+        super().__init__("Enable audit of Calico resources")
+
+    def run(self, res: DynamicResources) -> None:
+        logger = res.logger()
+        raw_cluster_policy = res.raw_inventory().get('services', {}).get('audit', {}).get('cluster_policy', {})
+
+        if 'rules' not in raw_cluster_policy or yaml_merger.is_list_extends(raw_cluster_policy['rules']):
+            install.run_tasks(res, ['deploy.kubernetes.audit'])
+        else:
+            return logger.info("Audit policy is redefined in the inventory file. Nothing to change.")
+
+
+class EnableCalicoAudit(RegularPatch):
+    def __init__(self) -> None:
+        super().__init__("enable_calico_audit")
+
+    @property
+    def action(self) -> Action:
+        return TheAction()
+
+    @property
+    def description(self) -> str:
+        return dedent(
+            f"""\
+            Enable logging of Kubernetes audit events for changes of the Calico resources.
+            
+            If the Calico plugin or its API server are disabled, the policy is still reconfigured,
+            but new rules do not affect anything and are only reserved for possible future extensions.
+            
+            The patch is equivalent to `kubemarine install --tasks deploy.kubernetes.audit`.
+            """.rstrip()
+        )
diff --git a/kubemarine/resources/configurations/defaults.yaml b/kubemarine/resources/configurations/defaults.yaml
@@ -198,13 +198,26 @@ services:
         # Don't log read-only requests
         - level: None
           verbs: ["watch", "get", "list"]
-        # Don't log checking API access by Calico API server
+        # Don't log checking access by internal services
         - level: None
-          users: ["system:serviceaccount:calico-apiserver:calico-apiserver"]
+          userGroups:
+            - "system:serviceaccounts:calico-apiserver"
+            - "system:nodes"
           verbs: ["create"]
           resources:
             - group: "authorization.k8s.io"
               resources: ["subjectaccessreviews"]
+            - group: "authentication.k8s.io"
+              resources: ["tokenreviews"]
+        # Don't log update of ingress-controller-leader ConfigMap by ingress-nginx.
+        # This reproduces only for v1.2.0 and can be removed after its support stop.
+        - level: None
+          users: ["system:serviceaccount:ingress-nginx:ingress-nginx"]
+          verbs: ["update"]
+          resources:
+            - group: ""
+              resources: ["configmaps"]
+              resourceNames: ["ingress-controller-leader"]
         # Log all other resources in core and extensions at the request level.
         - level: Metadata
           verbs: ["create", "update", "patch", "delete", "deletecollection"]
@@ -261,6 +274,45 @@ services:
           - group: "authentication.k8s.io"
             resources: ["tokenreviews"]
           - group: "authorization.k8s.io"
+          - group: "projectcalico.org"
+            resources:
+              - bgpconfigurations
+              - bgpfilters
+              - bgppeers
+              - blockaffinities
+              - caliconodestatuses
+              - clusterinformations
+              - felixconfigurations
+              - globalnetworkpolicies
+              - globalnetworksets
+              - hostendpoints
+              - ipamconfigurations
+              - ippools
+              - ipreservations
+              - kubecontrollersconfigurations
+              - networkpolicies
+              - networksets
+              - profiles
+          - group: "crd.projectcalico.org"
+            resources:
+              - bgpconfigurations
+              - bgpfilters
+              - bgppeers
+              - blockaffinities
+              - caliconodestatuses
+              - clusterinformations
+              - felixconfigurations
+              - globalnetworkpolicies
+              - globalnetworksets
+              - hostendpoints
+              - ipamblocks
+              - ipamconfigs
+              - ipamhandles
+              - ippools
+              - ipreservations
+              - kubecontrollersconfigurations
+              - networkpolicies
+              - networksets
 
     rules:
       - '{% if services["cri"]["containerRuntime"] == "docker" %}-w /var/lib/docker -k docker{% endif %}'

diff --git a/kubemarine/resources/schemas/definitions/services/audit.json b/kubemarine/resources/schemas/definitions/services/audit.json
@@ -51,7 +51,8 @@
             "type": "object",
             "properties": {
               "group": {"type": "string"},
-              "resources": {"$ref": "../common/utils.json#/definitions/ArrayOfStrings"}
+              "resources": {"$ref": "../common/utils.json#/definitions/ArrayOfStrings"},
+              "resourceNames": {"$ref": "../common/utils.json#/definitions/ArrayOfStrings"}
             }
           }
         }