From 64a598b6976fe4d4211d74d252c470af7f641d89 Mon Sep 17 00:00:00 2001 From: ilia1243 <8808144+ilia1243@users.noreply.github.com> Date: Fri, 13 Oct 2023 16:49:29 +0300 Subject: [PATCH 1/2] Insert license only to protected branches (#522) --- .github/workflows/tests.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 77cffad51..b7582afdd 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -9,11 +9,12 @@ env: jobs: license: runs-on: ubuntu-latest + if: github.ref_name == 'main' || endsWith(github.ref_name, '_branch') steps: - uses: actions/checkout@v3 + with: + token: ${{ secrets.NCCLPLCI_PAT }} - run: docker run -v "${PWD}:/src" -i ghcr.io/google/addlicense -v -c "${{ env.COPYRIGHT_COMPANY }}" -y "${{ env.COPYRIGHT_YEAR }}" $(find . -type f -name "*.py" | xargs echo) - # This currently not works for protected branches, - # but all pushes to them should be done through PRs that already contain inserted license. - uses: stefanzweifel/git-auto-commit-action@v4 with: commit_message: Auto-update license header From 3af8622fe44f586729ef162c7502710b40d5e206 Mon Sep 17 00:00:00 2001 From: ilia1243 <8808144+ilia1243@users.noreply.github.com> Date: Tue, 17 Oct 2023 13:04:21 +0300 Subject: [PATCH 2/2] [CPDEV-93148] [CPDEV-93149] Clarify kubernetes audit rules (#516) * Add kubernetes audit rules for Calico resources. * Do not log update of ingress-controller-leader ConfigMap by ingress-nginx * Do not log checking API access by kubelet * Add resourceNames JSON hint for audit policy rules * Write patch that reconfigures audit policy * Add documentation * Update Installation.md * Rename patch file --------- Co-authored-by: Shoaib Mohammed <94443646+shmo1218@users.noreply.github.com> --- documentation/Installation.md | 95 ++++++++++++++++++- kubemarine/core/resources.py | 5 +- kubemarine/demo.py | 18 +++- kubemarine/patches/__init__.py | 2 + kubemarine/patches/p1_enable_calico_audit.py | 57 +++++++++++ .../resources/configurations/defaults.yaml | 56 ++++++++++- .../schemas/definitions/services/audit.json | 3 +- 7 files changed, 223 insertions(+), 13 deletions(-) create mode 100644 kubemarine/patches/p1_enable_calico_audit.py diff --git a/documentation/Installation.md b/documentation/Installation.md index 68c4170f3..9c31b075f 100644 --- a/documentation/Installation.md +++ b/documentation/Installation.md @@ -2508,15 +2508,24 @@ services: *Can restart service*: Always yes, container kube-apiserver. -*OS specific*: No. +*Overwrite files*: Yes, `/etc/kubernetes/audit-policy.yaml` backup is created. -*Logging level*: +*OS specific*: No + +For more information about Kubernetes auditing, refer to the official documentation at [https://kubernetes.io/docs/tasks/debug/debug-cluster/audit/](https://kubernetes.io/docs/tasks/debug/debug-cluster/audit/). + +**Logging level**: `None` - do not log; `Metadata` — log request metadata: user, request time, target resource (pod, namespace, etc.), action type (verb), etc.; `Request` — log metadata and request body; `RequestResponse` - log metadata, request body and response body. -*omitStages*: To skip any stages. +**omitStages**: The list of stages for which no events are created. + +By default, the following policy is installed: + +
+ Default Policy ```yaml services: @@ -2531,13 +2540,26 @@ services: # Don't log read-only requests - level: None verbs: ["watch", "get", "list"] - # Don't log checking API access by Calico API server + # Don't log checking access by internal services - level: None - users: ["system:serviceaccount:calico-apiserver:calico-apiserver"] + userGroups: + - "system:serviceaccounts:calico-apiserver" + - "system:nodes" verbs: ["create"] resources: - group: "authorization.k8s.io" resources: ["subjectaccessreviews"] + - group: "authentication.k8s.io" + resources: ["tokenreviews"] + # Don't log update of ingress-controller-leader ConfigMap by ingress-nginx. + # This reproduces only for v1.2.0 and can be removed after its support stop. + - level: None + users: ["system:serviceaccount:ingress-nginx:ingress-nginx"] + verbs: ["update"] + resources: + - group: "" + resources: ["configmaps"] + resourceNames: ["ingress-controller-leader"] # Log all other resources in core and extensions at the request level. - level: Metadata verbs: ["create", "update", "patch", "delete", "deletecollection"] @@ -2594,6 +2616,69 @@ services: - group: "authentication.k8s.io" resources: ["tokenreviews"] - group: "authorization.k8s.io" + - group: "projectcalico.org" + resources: + - bgpconfigurations + - bgpfilters + - bgppeers + - blockaffinities + - caliconodestatuses + - clusterinformations + - felixconfigurations + - globalnetworkpolicies + - globalnetworksets + - hostendpoints + - ipamconfigurations + - ippools + - ipreservations + - kubecontrollersconfigurations + - networkpolicies + - networksets + - profiles + - group: "crd.projectcalico.org" + resources: + - bgpconfigurations + - bgpfilters + - bgppeers + - blockaffinities + - caliconodestatuses + - clusterinformations + - felixconfigurations + - globalnetworkpolicies + - globalnetworksets + - hostendpoints + - ipamblocks + - ipamconfigs + - ipamhandles + - ippools + - ipreservations + - kubecontrollersconfigurations + - networkpolicies + - networksets +``` +
+ +It is possible not only to redefine the default policy, but also to extend it. For more information, refer to [List Merge Strategy](#list-merge-strategy). + +For example, consider you have an [operator](https://kubernetes.io/docs/concepts/extend-kubernetes/operator/) that constantly updates some Pods' labels and some ConfigMap to maintain the leadership. +If you do not see any benefit from logging of such events, they can be disabled by specifying the following in `cluster.yaml`: + +```yaml +services: + audit: + cluster_policy: + rules: + - level: None + userGroups: ["system:serviceaccounts:operator-namespace"] + verbs: ["patch", "update"] + namespaces: ["operator-namespace"] + resources: + - group: "" + resources: [pods] + - group: "" + resources: [configmaps] + resourceNames: [controller-leader] + - '<<': merge ``` ##### Audit Daemon diff --git a/kubemarine/core/resources.py b/kubemarine/core/resources.py index f9922c828..1cf57c1f5 100644 --- a/kubemarine/core/resources.py +++ b/kubemarine/core/resources.py @@ -195,11 +195,14 @@ def get_nodes_context(self) -> dict: # temporary cluster instance to detect initial nodes context. light_cluster = self._new_cluster_instance(deepcopy(self.context)) light_cluster.enrich(custom_enrichment_fns=light_cluster.get_facts_enrichment_fns()) - self._nodes_context = light_cluster.detect_nodes_context() + self._nodes_context = self._detect_nodes_context(light_cluster) light_cluster.connection_pool.close() return self._nodes_context + def _detect_nodes_context(self, light_cluster: c.KubernetesCluster) -> dict: + return light_cluster.detect_nodes_context() + def _new_cluster_instance(self, context: dict) -> c.KubernetesCluster: from kubemarine.core import flow cluster_class = c.KubernetesCluster diff --git a/kubemarine/demo.py b/kubemarine/demo.py index 293a3fe01..1ff426062 100644 --- a/kubemarine/demo.py +++ b/kubemarine/demo.py @@ -21,7 +21,7 @@ import time from abc import ABC from copy import deepcopy -from typing import List, Dict, Union, Any, Optional, Mapping, Iterable, IO, Tuple +from typing import List, Dict, Union, Any, Optional, Mapping, Iterable, IO, Tuple, cast import fabric # type: ignore[import] import invoke @@ -200,7 +200,8 @@ def __init__(self, context: dict, raw_inventory: dict, procedure_inventory: dict self.last_cluster: Optional[FakeKubernetesCluster] = None self.fake_shell = fake_shell if fake_shell else FakeShell() self.fake_fs = fake_fs if fake_fs else FakeFS() - self._nodes_context = nodes_context + # Let's do not assign self._nodes_context directly to make it more close to the real enrichment. + self.fake_nodes_context = nodes_context self._procedure_inventory = procedure_inventory def _load_inventory(self) -> None: @@ -210,13 +211,22 @@ def _load_inventory(self) -> None: def _store_inventory(self) -> None: self.stored_inventory = deepcopy(self.formatted_inventory()) + def _detect_nodes_context(self, light_cluster: KubernetesCluster) -> dict: + if self.fake_nodes_context is not None: + return self.fake_nodes_context + + return super()._detect_nodes_context(light_cluster) + + def _create_cluster(self, context: dict) -> KubernetesCluster: + self.last_cluster = cast(FakeKubernetesCluster, super()._create_cluster(context)) + return self.last_cluster + def _new_cluster_instance(self, context: dict) -> FakeKubernetesCluster: - self.last_cluster = FakeKubernetesCluster( + return FakeKubernetesCluster( self.raw_inventory(), context, procedure_inventory=self.procedure_inventory(), logger=self.logger(), fake_shell=self.fake_shell, fake_fs=self.fake_fs ) - return self.last_cluster class FakeConnection(fabric.connection.Connection): # type: ignore[misc] diff --git a/kubemarine/patches/__init__.py b/kubemarine/patches/__init__.py index 7958d24dd..69b6989eb 100644 --- a/kubemarine/patches/__init__.py +++ b/kubemarine/patches/__init__.py @@ -22,8 +22,10 @@ from typing import List from kubemarine.core.patch import Patch +from kubemarine.patches.p1_enable_calico_audit import EnableCalicoAudit patches: List[Patch] = [ + EnableCalicoAudit(), ] """ List of patches that is sorted according to the Patch.priority() before execution. diff --git a/kubemarine/patches/p1_enable_calico_audit.py b/kubemarine/patches/p1_enable_calico_audit.py new file mode 100644 index 000000000..bcf2e3de2 --- /dev/null +++ b/kubemarine/patches/p1_enable_calico_audit.py @@ -0,0 +1,57 @@ +# Copyright 2021-2023 NetCracker Technology Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from textwrap import dedent + +from kubemarine.core import yaml_merger +from kubemarine.core.action import Action +from kubemarine.core.patch import RegularPatch +from kubemarine.core.resources import DynamicResources +from kubemarine.procedures import install + + +class TheAction(Action): + def __init__(self) -> None: + super().__init__("Enable audit of Calico resources") + + def run(self, res: DynamicResources) -> None: + logger = res.logger() + raw_cluster_policy = res.raw_inventory().get('services', {}).get('audit', {}).get('cluster_policy', {}) + + if 'rules' not in raw_cluster_policy or yaml_merger.is_list_extends(raw_cluster_policy['rules']): + install.run_tasks(res, ['deploy.kubernetes.audit']) + else: + return logger.info("Audit policy is redefined in the inventory file. Nothing to change.") + + +class EnableCalicoAudit(RegularPatch): + def __init__(self) -> None: + super().__init__("enable_calico_audit") + + @property + def action(self) -> Action: + return TheAction() + + @property + def description(self) -> str: + return dedent( + f"""\ + Enable logging of Kubernetes audit events for changes of the Calico resources. + + If the Calico plugin or its API server are disabled, the policy is still reconfigured, + but new rules do not affect anything and are only reserved for possible future extensions. + + The patch is equivalent to `kubemarine install --tasks deploy.kubernetes.audit`. + """.rstrip() + ) diff --git a/kubemarine/resources/configurations/defaults.yaml b/kubemarine/resources/configurations/defaults.yaml index 42e27fd48..1f9dc4682 100644 --- a/kubemarine/resources/configurations/defaults.yaml +++ b/kubemarine/resources/configurations/defaults.yaml @@ -198,13 +198,26 @@ services: # Don't log read-only requests - level: None verbs: ["watch", "get", "list"] - # Don't log checking API access by Calico API server + # Don't log checking access by internal services - level: None - users: ["system:serviceaccount:calico-apiserver:calico-apiserver"] + userGroups: + - "system:serviceaccounts:calico-apiserver" + - "system:nodes" verbs: ["create"] resources: - group: "authorization.k8s.io" resources: ["subjectaccessreviews"] + - group: "authentication.k8s.io" + resources: ["tokenreviews"] + # Don't log update of ingress-controller-leader ConfigMap by ingress-nginx. + # This reproduces only for v1.2.0 and can be removed after its support stop. + - level: None + users: ["system:serviceaccount:ingress-nginx:ingress-nginx"] + verbs: ["update"] + resources: + - group: "" + resources: ["configmaps"] + resourceNames: ["ingress-controller-leader"] # Log all other resources in core and extensions at the request level. - level: Metadata verbs: ["create", "update", "patch", "delete", "deletecollection"] @@ -261,6 +274,45 @@ services: - group: "authentication.k8s.io" resources: ["tokenreviews"] - group: "authorization.k8s.io" + - group: "projectcalico.org" + resources: + - bgpconfigurations + - bgpfilters + - bgppeers + - blockaffinities + - caliconodestatuses + - clusterinformations + - felixconfigurations + - globalnetworkpolicies + - globalnetworksets + - hostendpoints + - ipamconfigurations + - ippools + - ipreservations + - kubecontrollersconfigurations + - networkpolicies + - networksets + - profiles + - group: "crd.projectcalico.org" + resources: + - bgpconfigurations + - bgpfilters + - bgppeers + - blockaffinities + - caliconodestatuses + - clusterinformations + - felixconfigurations + - globalnetworkpolicies + - globalnetworksets + - hostendpoints + - ipamblocks + - ipamconfigs + - ipamhandles + - ippools + - ipreservations + - kubecontrollersconfigurations + - networkpolicies + - networksets rules: - '{% if services["cri"]["containerRuntime"] == "docker" %}-w /var/lib/docker -k docker{% endif %}' diff --git a/kubemarine/resources/schemas/definitions/services/audit.json b/kubemarine/resources/schemas/definitions/services/audit.json index a60444092..fe7e26b80 100644 --- a/kubemarine/resources/schemas/definitions/services/audit.json +++ b/kubemarine/resources/schemas/definitions/services/audit.json @@ -51,7 +51,8 @@ "type": "object", "properties": { "group": {"type": "string"}, - "resources": {"$ref": "../common/utils.json#/definitions/ArrayOfStrings"} + "resources": {"$ref": "../common/utils.json#/definitions/ArrayOfStrings"}, + "resourceNames": {"$ref": "../common/utils.json#/definitions/ArrayOfStrings"} } } }