From 414d991cf6c45cede9f03904ae28668e94ef1c3b Mon Sep 17 00:00:00 2001 From: Billy McFall <22157057+Billy99@users.noreply.github.com> Date: Wed, 11 Dec 2024 16:23:09 -0500 Subject: [PATCH 1/3] doc: add design document for Cluster vs Namespace Add a design document for Cluster vs Namespace Scoped CRDs in bpfman. Signed-off-by: Billy McFall <22157057+Billy99@users.noreply.github.com> --- docs/design/clusterVsNamespaceScoped.md | 492 ++++++++++++++++++++++++ mkdocs.yml | 1 + 2 files changed, 493 insertions(+) create mode 100644 docs/design/clusterVsNamespaceScoped.md diff --git a/docs/design/clusterVsNamespaceScoped.md b/docs/design/clusterVsNamespaceScoped.md new file mode 100644 index 000000000..d53f68836 --- /dev/null +++ b/docs/design/clusterVsNamespaceScoped.md @@ -0,0 +1,492 @@ +# bpfman CRDs - Cluster vs Namespace Scoped + +## Status + +This design was implemented with +[bpfman-operator pull request #344](https://github.com/bpfman/bpfman-operator/pull/344). +The feature was first releases in the bpfman-operator v0.5.5 release. + + +## Introduction + +For security reasons, cluster admins may want to limit certain applications to only loading eBPF programs +within a given namespace. +Currently, all bpfman Custom Resource Definitions (CRDs) are Cluster scoped. +To provide cluster admins with tighter controls on eBPF program loading, some of the bpfman CRDs also need +to be Namespace scoped. + +Not all eBPF programs make sense to be namespaced scoped. +Some eBPF programs like kprobe cannot be constrained to a namespace. +The following programs will have a namespaced scoped variant: + +* Uprobe +* TC +* TCX +* XDP + +There will also be a namespace scoped BpfApplication variant that is limited to namespaced scoped +eBPF programs listed above. + +## Current Implementation + +Currently, the reconciler code is broken into two layers (for both the bpfman-operator and the bpfman-agent). +There is the \*Program layer, where there is a reconcile for each program type (Fentry, Fexit, Kprobe, etc). +At this layer, the program specific code handles creating the program specific structure. +The \*Program layer then calls the Common layer to handle processing that is common across all programs. + +There are some structures, then an interface that defines the set of methods the structure needs to support. + +### struct + +There are a set of structures (one for the BPF Program CRD and then one for each \*Program CRD) that define the +contents of the CRDs (bpfman-operator/apis/v1alpha). +Each object (BPF Program CRD and \*Program CRD) also has a List object. + +```go +type BpfProgram struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec BpfProgramSpec `json:"spec"` + // +optional + Status BpfProgramStatus `json:"status,omitempty"` +} +type BpfProgramList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []BpfProgram `json:"items"` +} + +type FentryProgram struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec FentryProgramSpec `json:"spec"` + // +optional + Status FentryProgramStatus `json:"status,omitempty"` +} +type FentryProgramList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []FentryProgram `json:"items"` +} + +: +``` + +There is a reconciler for each \*Program. +For the implementation, there is a common set of data used by each \*Program reconciler which is +contained in the base struct `ReconcilerCommon`. +Then there is a \*Program struct, which includes each \*Program’s Program struct and the base +struct `ReconcilerCommon`. +Below are the bpfman-agent structures, but the bpfman-operator follows the same pattern. + +```go +type ReconcilerCommon struct { + client.Client + Scheme *runtime.Scheme + GrpcConn *grpc.ClientConn + BpfmanClient gobpfman.BpfmanClient + Logger logr.Logger + NodeName string + progId *uint32 + finalizer string + recType string + appOwner metav1.Object // Set if the owner is an application +} + +type FentryProgramReconciler struct { + ReconcilerCommon + currentFentryProgram *bpfmaniov1alpha1.FentryProgram + ourNode *v1.Node +} + +type FexitProgramReconciler struct { + ReconcilerCommon + currentFexitProgram *bpfmaniov1alpha1.FexitProgram + ourNode *v1.Node +} + +: +``` + +### interface + +The `bpfmanReconciler` interface defines the set of methods the \*Program structs must implement to use the +common reconciler code. +Below are the bpfman-agent structures, but the bpfman-operator uses a `ProgramReconciler`, which follows +the same pattern. + +```go +type bpfmanReconciler interface { + SetupWithManager(mgr ctrl.Manager) error + Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) + getFinalizer() string + getOwner() metav1.Object + getRecType() string + getProgType() internal.ProgramType + getName() string + getExpectedBpfPrograms(ctx context.Context) + (*bpfmaniov1alpha1.BpfProgramList, error) + getLoadRequest(bpfProgram *bpfmaniov1alpha1.BpfProgram, + mapOwnerId *uint32) (*gobpfman.LoadRequest, error) + getNode() *v1.Node + getBpfProgramCommon() *bpfmaniov1alpha1.BpfProgramCommon + setCurrentProgram(program client.Object) error + getNodeSelector() *metav1.LabelSelector + getBpfGlobalData() map[string][]byte + getAppProgramId() string +} +``` + +There are also some common reconciler functions that perform common code. + +```go +func (r *ReconcilerCommon) reconcileCommon(ctx context.Context, + rec bpfmanReconciler, + programs []client.Object) (bool, ctrl.Result, error) { +: +} + +func (r *ReconcilerCommon) reconcileBpfProgram(ctx context.Context, + rec bpfmanReconciler, + loadedBpfPrograms map[string]*gobpfman.ListResponse_ListResult, + bpfProgram *bpfmaniov1alpha1.BpfProgram, + isNodeSelected bool, + isBeingDeleted bool, + mapOwnerStatus *MapOwnerParamStatus) +(bpfmaniov1alpha1.BpfProgramConditionType, error) { +: +} + +func (r *ReconcilerCommon) reconcileBpfProgramSuccessCondition( + isLoaded bool, + shouldBeLoaded bool, + isNodeSelected bool, + isBeingDeleted bool, + noContainersOnNode bool, + mapOwnerStatus *MapOwnerParamStatus) bpfmaniov1alpha1.BpfProgramConditionType { +: +} +``` + +So looks something like this: + +``` + --- FentryProgramReconciler + | func (r *FentryProgramReconciler) getFinalizer() string {} + | +bpfmanReconciler ----- FexitProgramReconciler + ReconcilerCommon | func (r *FexitProgramReconciler) getFinalizer() string {} + | + --- … +``` + +## Adding Namespaced Scoped CRDs + +While the contents are mostly the same for the namespace and cluster-scoped CRD in most cases, Kubernetes +requires different CRD for each type. + +### struct + +The set of CRD structures will need to be duplicated for each Namespaced scoped CRD (bpfman-operator/apis/v1alpha). +Note, data is the similar, just a new object. +The primary change is the existing `ContainerSelector` struct will be replaced with a `ContainerNsSelector`. +For Namespaced scoped CRDs, the `namespace` in the `ContainerSelector` is removed. +The `Namespace` field for the object is embedded the `metav1.ObjectMeta` structure. +Not all Program Types will have a Namespaced version, only those that can be contained by a namespace: + +* TC +* TCX +* Uprobe +* XDP + +The Application Program will also have a namespaced version, but it will only allow the Program Types +that are namespaced. + +```go +type BpfProgram struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec BpfProgramSpec `json:"spec"` + // +optional + Status BpfProgramStatus `json:"status,omitempty"` +} +type BpfProgramList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []BpfProgram `json:"items"` +} + +type BpfNsProgram struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec BpfProgramSpec `json:"spec"` + // +optional + Status BpfProgramStatus `json:"status,omitempty"` +} +type BpfNsProgramList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []BpfProgram `json:"items"` +} + +type TcProgram struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec TcProgramSpec `json:"spec"` + // +optional + Status TcProgramStatus `json:"status,omitempty"` +} +type TcProgramList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []TcProgram `json:"items"` +} + +type TcNsProgram struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec TcNsProgramSpec `json:"spec"` + // +optional + Status TcProgramStatus `json:"status,omitempty"` +} +type TcNsProgramList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []TcNsProgram `json:"items"` +} + +: +``` + +### interface + +The problem is that the `bpfmanReconciler` interface and common functions use the types `bpfmanReconciler`, +`BpfProgram` and `BpfProgramList`, which will need to be cluster or namespaced objects. + +To allow the common code to act on both Cluster or Namespaced objects, two new interfaces will be introduced. +First is `BpfProg`. +Both `BpfProgram` and `BpfNsProgram` need to implement these functions. + +```go +type BpfProg interface { + GetName() string + GetUID() types.UID + GetAnnotations() map[string]string + GetLabels() map[string]string + GetStatus() *bpfmaniov1alpha1.BpfProgramStatus + GetClientObject() client.Object +} +``` + +The second interface is `BpfProgList`. +Both `BpfProgramList` and `BpfNsProgramList` will need to implement these functions. +Because the list objects have lists of the `BpfProgram`or `BpfNsProgram`, the base interface is a generic, +where type T can be a either `BpfProgram` or `BpfNsProgram`. + +```go +type BpfProgList[T any] interface { + GetItems() []T +} +``` + +The reconciler base struct `ReconcilerCommon` then becomes a generic as well, and all references to the types +`bpfmanReconciler`, `BpfProgram` and `BpfProgramList` become the types `bpfmanReconciler[T,TL]`, `T` and `TL`. +Below are the bpfman-agent structures, but the bpfman-operator follows the same pattern. + +```go +type ReconcilerCommon[T BpfProg, TL BpfProgList[T]] struct { + : // Data is the same +} + +func (r *ReconcilerCommon) reconcileCommon(ctx context.Context, +rec bpfmanReconciler[T, TL], + programs []client.Object) (bool, ctrl.Result, error) { +: +} + +func (r *ReconcilerCommon) reconcileBpfProgram(ctx context.Context, + rec bpfmanReconciler[T, TL], + loadedBpfPrograms map[string]*gobpfman.ListResponse_ListResult, + bpfProgram *T, + isNodeSelected bool, + isBeingDeleted bool, + mapOwnerStatus *MapOwnerParamStatus) +(bpfmaniov1alpha1.BpfProgramConditionType, error) { +: +} + +func (r *ReconcilerCommon) reconcileBpfProgramSuccessCondition( + isLoaded bool, + shouldBeLoaded bool, + isNodeSelected bool, + isBeingDeleted bool, + noContainersOnNode bool, + mapOwnerStatus *MapOwnerParamStatus) bpfmaniov1alpha1.BpfProgramConditionType { +: +} +``` + +Same for the `bpfmanReconciler` interface. + +```go +type bpfmanReconciler[T BpfProg, TL BpfProgList[T]] interface { + SetupWithManager(mgr ctrl.Manager) error + Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) + getFinalizer() string + getOwner() metav1.Object + getRecType() string + getProgType() internal.ProgramType + getName() string + getExpectedBpfPrograms(ctx context.Context)(*TL, error) + getLoadRequest(bpfProgram *T, + mapOwnerId *uint32) (*gobpfman.LoadRequest, error) + getNode() *v1.Node + getBpfProgramCommon() *bpfmaniov1alpha1.BpfProgramCommon + setCurrentProgram(program client.Object) error + getNodeSelector() *metav1.LabelSelector + getBpfGlobalData() map[string][]byte + getAppProgramId() string +} +``` + +Issues arose when `ReconcilerCommon` functions needed to modify the `BpfProgram` or `BpfNsProgram` data. +For the modifications to be applied, the types need to be pointers `bpfmanReconciler[*T, *TL]`, `*T` and `*TL`. +However, the compiler would not allow this: + + cannot use type BpfProgList[*T] outside a type constraint: interface contains type constraints + +To work around this, a new layer was added. +A struct for cluster scoped code and a one for namespaced code. +So looks something like this: + +``` + +--- ClusterProgramReconciler + | | + | +--- FentryProgramReconciler + | | func (r *FentryProgramReconciler) getFinalizer() string {} + | | : + | | + | +--- FexitProgramReconciler + | | func (r *FexitProgramReconciler) getFinalizer() string {} + | | : + | : +bpfmanReconciler --+ + ReconcilerCommon | + +--- NamespaceProgramReconciler + | + +--- FentryNsProgramReconciler + | func (r *FentryProgramReconciler) getFinalizer() string {} + | : + | + +--- FexitNsProgramReconciler + | func (r *FexitProgramReconciler) getFinalizer() string {} + | : + : +``` + +```go +type ClusterProgramReconciler struct { + ReconcilerCommon[BpfProgram, BpfProgramList] +} + +type NamespaceProgramReconciler struct { + ReconcilerCommon[BpfNsProgram, BpfNsProgramList] +} +``` + +Several functions were added to the bpfmanReconciler interface that are implemented by these structures. + +```go +type bpfmanReconciler[T BpfProg, TL BpfProgList[T]] interface { + // BPF Cluster of Namespaced Reconciler + getBpfList(ctx context.Context, opts []client.ListOption) (*TL, error) + updateBpfStatus(ctx context.Context, bpfProgram *T, condition metav1.Condition) error + createBpfProgram( + attachPoint string, + rec bpfmanReconciler[T, TL], + annotations map[string]string, + ) (*T, error) + + // *Program Reconciler + SetupWithManager(mgr ctrl.Manager) error + : +} +``` + +And the \*Programs use the `ClusterProgramReconciler` or `NamespaceProgramReconciler` structs instead +of the `ReconcilerCommon` struct. + +```go +type TcProgramReconciler struct { + ClusterProgramReconciler + currentTcProgram *bpfmaniov1alpha1.TcProgram + interfaces []string + ourNode *v1.Node +} + +type TcNsProgramReconciler struct { + NamespaceProgramReconciler + currentTcNsProgram *bpfmaniov1alpha1.TcNsProgram + interfaces []string + ourNode *v1.Node +} + +: +``` + +## Naming + +In the existing codebase, all the CRDs are cluster scoped: + +* BpfApplicationProgram +* BpfProgram +* FentryProgram +* FexitProgram +* KprobeProgram +* ... + +Common practice is for cluster scoped objects to include "Cluster" in the name and +for namespaced objects to not have an identifier. +So the current CRDs SHOULD have been named: + +* ClusterBpfApplicationProgram +* ClusterBpfProgram +* ClusterFentryProgram +* ClusterFexitProgram +* ClusterKprobeProgram +* ... + +Around the same time this feature is being developed, another feature is being developed +which will break the loading and attaching of eBPF programs in bpfman into two steps. +As part of this feature, all the CRDs will be completely reworked. +With this in mind, the plan for adding namespace scoped CRDs is to make the namespaced CRDs +carry the identifier. +After the load/attach split work is complete, the CRDs will be renamed to follow the common +convention in which the cluster-scoped CRD names are prefixed with "Cluster". + +The current plan is for the namespaced scoped CRDs to use "NsProgram" identifier and cluster +scoped CRDs to use "Program" identifier. +With the new namespace scope feature, below are the list of CRDs supported by bpfman-operator: + +* BpfNsApplicationProgram +* BpfApplicationProgram +* BpfNsProgram +* BpfProgram +* FentryProgram +* FexitProgram +* KprobeProgram +* TcNsProgram +* TcProgram +* TcxNsProgram +* TcxProgram +* TracepointProgram +* UprobeNsProgram +* UprobeProgram +* XdpNsProgram +* XdpProgram diff --git a/mkdocs.yml b/mkdocs.yml index 77f5f5550..77d3f9b9d 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -82,6 +82,7 @@ nav: - Running the Examples as Non-Root on SELinux Distributions: developer-guide/k8s-selinux-distros.md - Design: - Daemonless: design/daemonless.md + - bpfman CRDs - Cluster vs Namespace Scoped: design/clusterVsNamespaceScoped.md - Blog: - blog/index.md - Community: From 5c4c58db2d83dbb4cb4014af1bb1ffb034b1f5e2 Mon Sep 17 00:00:00 2001 From: Billy McFall <22157057+Billy99@users.noreply.github.com> Date: Wed, 18 Dec 2024 14:21:22 -0500 Subject: [PATCH 2/3] doc: random doc updates before release Modifications to the docs before the release. Signed-off-by: Billy McFall <22157057+Billy99@users.noreply.github.com> --- README.md | 2 +- docs/blog/posts/ebpf-container-challenges.md | 2 +- docs/developer-guide/develop-operator.md | 2 +- docs/getting-started/building-bpfman.md | 16 ++++++++++------ docs/getting-started/example-bpf-k8s.md | 3 ++- .../operator-quick-start.md | 13 ++++++++++++- docs/getting-started/overview.md | 3 --- docs/getting-started/running-release.md | 4 ++-- docs/index.md | 2 +- mkdocs.yml | 2 +- 10 files changed, 31 insertions(+), 18 deletions(-) rename docs/{developer-guide => getting-started}/operator-quick-start.md (90%) diff --git a/README.md b/README.md index efd0eb846..093068fc7 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ Here are some links to help in your bpfman journey (all links are from the bpfma on setting up your development environment and building bpfman. - [Example eBPF Programs](https://bpfman.io/main/getting-started/example-bpf/) for some examples of eBPF programs written in Go, interacting with `bpfman`. -- [Deploying the bpfman-operator](https://bpfman.io/main/developer-guide/develop-operator/) for details on launching +- [Deploying the bpfman-operator](https://bpfman.io/main/getting-started/develop-operator/) for details on launching bpfman in a Kubernetes cluster. - [Meet the Community](https://bpfman.io/main/governance/meetings/) for details on community meeting details. diff --git a/docs/blog/posts/ebpf-container-challenges.md b/docs/blog/posts/ebpf-container-challenges.md index 71ac2d13a..aa89a4546 100644 --- a/docs/blog/posts/ebpf-container-challenges.md +++ b/docs/blog/posts/ebpf-container-challenges.md @@ -358,7 +358,7 @@ We will use a [kind](https://kind.sigs.k8s.io/) deployment with bpfman for this demo. See [Deploy Locally via KIND] for instructions on how to get this running. [bpfman.io_v1alpha1_uprobe_uprobeprogram_containers.yaml]:https://github.com/bpfman/bpfman/blob/main/bpfman-operator/config/samples/bpfman.io_v1alpha1_uprobe_uprobeprogram_containers.yaml -[Deploy Locally via KIND]:../../developer-guide/operator-quick-start.md#deploy-locally-via-kind +[Deploy Locally via KIND]:../../getting-started/operator-quick-start.md#deploy-locally-via-kind The container selector in the above yaml file is the following. diff --git a/docs/developer-guide/develop-operator.md b/docs/developer-guide/develop-operator.md index e85bcd621..5c0887371 100644 --- a/docs/developer-guide/develop-operator.md +++ b/docs/developer-guide/develop-operator.md @@ -237,7 +237,7 @@ quay.io/bpfman/bpfman-operator latest 4fe444b7abf1 2 minutes ago 141MB ## Running Locally in KIND -[Deploying the bpfman-operator](./operator-quick-start.md) goes into more detail on ways to +[Deploying the bpfman-operator](../getting-started/operator-quick-start.md) goes into more detail on ways to launch bpfman in a Kubernetes cluster. To run locally in a Kind cluster with an up to date build simply run: diff --git a/docs/getting-started/building-bpfman.md b/docs/getting-started/building-bpfman.md index e7bb2d939..99f15cbf1 100644 --- a/docs/getting-started/building-bpfman.md +++ b/docs/getting-started/building-bpfman.md @@ -36,10 +36,8 @@ Major kernel features leveraged by bpfman: * **Relaxed CAP_BPF Requirement:** Prior to Kernel 5.19, all eBPF system calls required CAP_BPF. This required userspace programs that wanted to access eBPF maps to have the CAP_BPF Linux capability. With the kernel 5.19 change, CAP_BPF is only required for load and unload requests. -* **TCX:** TCX support was added in Kernel 6.6 and is expected to be added to - bpfman in an upcoming release. TCX has performance improvements over TC and - adds support in the kernel for multiple TCX programs to run on a given TC hook - point. +* **TCX:** TCX has performance improvements over TC and adds support in the kernel for multiple TCX + programs to run on a given TC hook point. TCX support was added in Kernel 6.6. bpfman tested on older kernel versions: @@ -167,10 +165,10 @@ sudo dnf install perl sudo apt install perl ``` -### Install docker +### Install docker or podman To build the `bpfman-agent` and `bpfman-operator` using the provided Makefile and the -`make build-images` command, `docker` needs to be installed. +`make build-images` command, `docker` or `podman` needs to be installed. There are several existing guides: * Fedora: [https://developer.fedoraproject.org/tools/docker/docker-installation.html](https://developer.fedoraproject.org/tools/docker/docker-installation.html) @@ -380,3 +378,9 @@ man bpfman list manpage subcommand generation. So use `man bpfman load-file`, `man bpfman load-image`, `man bpfman load-image-xdp`, etc. to display the subcommand manpage files. + +## Building bpfman-operator + +Building and deploying bpfman-operator is covered in it's own section. +See [Deploying Example eBPF Programs On Kubernetes](./example-bpf-k8s.md) and +[Developing the bpfman-operator](../developer-guide/develop-operator.md). diff --git a/docs/getting-started/example-bpf-k8s.md b/docs/getting-started/example-bpf-k8s.md index b41a3b3c7..b68af8489 100644 --- a/docs/getting-started/example-bpf-k8s.md +++ b/docs/getting-started/example-bpf-k8s.md @@ -4,7 +4,7 @@ This section will describe launching eBPF enabled applications on a Kubernetes c The approach is slightly different when running on a Kubernetes cluster. This section assumes there is already a Kubernetes cluster running and `bpfman` is running in the cluster. -See [Deploying the bpfman-operator](../developer-guide/operator-quick-start.md) for details on +See [Deploying the bpfman-operator](./operator-quick-start.md) for details on deploying bpfman on a Kubernetes cluster, but the quickest solution is to run a Kubernetes KIND Cluster: ```console @@ -22,6 +22,7 @@ There is a CRD object for each eBPF program type bpfman supports. * FexitProgram CRD: [Fexit Sample yaml](https://github.com/bpfman/bpfman/blob/main/bpfman-operator/config/samples/bpfman.io_v1alpha1_fexit_fexitprogram.yaml) * KprobeProgram CRD: [Kprobe Examples yaml](https://github.com/bpfman/bpfman/tree/main/examples/config/base/go-kprobe-counter/bytecode.yaml) * TcProgram CRD: [TcProgram Examples yaml](https://github.com/bpfman/bpfman/tree/main/examples/config/base/go-tc-counter/bytecode.yaml) +* TcxProgram CRD: [TcxProgram Examples yaml](https://github.com/bpfman/bpfman/tree/main/examples/config/base/go-tcx-counter/bytecode.yaml) * TracepointProgram CRD: [Tracepoint Examples yaml](https://github.com/bpfman/bpfman/tree/main/examples/config/base/go-tracepoint-counter/bytecode.yaml) * UprobeProgram CRD: [Uprobe Examples yaml](https://github.com/bpfman/bpfman/tree/main/examples/config/base/go-uprobe-counter/bytecode.yaml) * XdpProgram CRD: [XdpProgram Examples yaml](https://github.com/bpfman/bpfman/tree/main/examples/config/base/go-xdp-counter/bytecode.yaml) diff --git a/docs/developer-guide/operator-quick-start.md b/docs/getting-started/operator-quick-start.md similarity index 90% rename from docs/developer-guide/operator-quick-start.md rename to docs/getting-started/operator-quick-start.md index 8bc3d1c34..2102b18fc 100644 --- a/docs/developer-guide/operator-quick-start.md +++ b/docs/getting-started/operator-quick-start.md @@ -13,6 +13,13 @@ It runs on the control plane and is composed of the containers `bpfman-operator` The operator is responsible for launching the bpfman Daemonset, which runs on every node. The bpfman Daemonset is composed of the containers `bpfman`, `bpfman-agent`, and `node-driver-registrar`. +Described below are two ways to deploy bpfman in a Kubernetes cluster: + +* [Deploy Locally via KIND](#deploy-locally-via-kind): Easiest way to deploy bpfman in a Kubernetes cluster + and great for testing. +* [Deploy To Openshift Cluster](#deploy-to-openshift-cluster): Special steps are needed to deploy on an + Openshift cluster because SELinux is enable. + ### Deploy Locally via KIND After reviewing the possible make targets it's quick and easy to get bpfman deployed locally on your system @@ -158,9 +165,13 @@ NAME BPFFUNCTIONNAME NODESELECTOR PRIORITY INTERFACESELECT xdp-pass-all-nodes pass {} 0 {"primarynodeinterface":true} ["pass","dispatcher_return"] ``` +There are sample yamls for each of the support program type in the +[config/samples](https://github.com/bpfman/bpfman-operator/tree/main/config/samples) +directory. + ## API Types Overview -See [api-spec.md](./api-spec.md) for a more detailed description of all the bpfman Kubernetes API types. +See [api-spec.md](../developer-guide/api-spec.md) for a more detailed description of all the bpfman Kubernetes API types. ### Multiple Program CRDs diff --git a/docs/getting-started/overview.md b/docs/getting-started/overview.md index 29fbb1cb5..e8735417e 100644 --- a/docs/getting-started/overview.md +++ b/docs/getting-started/overview.md @@ -12,9 +12,6 @@ For applications written in other languages, bpfman provides `bpfman-rpc`, a Rus based bpfman RPC server binary. Non-Rust applications can send a RPC message to the server, which translate the RPC request into a bpfman library call. -The long term solution is to leverage the Rust Foreign Function Interface (FFI) -feature, which enables a different (foreign) programming language to call Rust -functions, but that is not supported at the moment. ![bpfman library](../img/bpfman_library.png) diff --git a/docs/getting-started/running-release.md b/docs/getting-started/running-release.md index 315a8aba2..f95870600 100644 --- a/docs/getting-started/running-release.md +++ b/docs/getting-started/running-release.md @@ -20,7 +20,7 @@ for deploying a released version of `bpfman` from an rpm as a systemd service an [Deploying Example eBPF Programs On Local Host](./example-bpf-local.md) for further information on how to test and interact with `bpfman`. -[Deploying the bpfman-operator](../developer-guide/operator-quick-start.md) contains +[Deploying the bpfman-operator](./operator-quick-start.md) contains more details on deploying `bpfman` in a Kubernetes deployment and [Deploying Example eBPF Programs On Kubernetes](./example-bpf-k8s.md) contains more details on interacting with `bpfman` running in a Kubernetes deployment. @@ -98,7 +98,7 @@ kubectl apply -f https://github.com/bpfman/bpfman/releases/download/v${BPFMAN_RE There are other example programs in the [Releases](https://github.com/bpfman/bpfman/releases) page. -Continue in [Deploying the bpfman-operator](../developer-guide/operator-quick-start.md) or +Continue in [Deploying the bpfman-operator](./operator-quick-start.md) or [Deploying Example eBPF Programs On Kubernetes](./example-bpf-k8s.md) if desired. Keep in mind that prior to v0.4.0, `bpfman` was released as `bpfd`. So follow the release specific documentation. diff --git a/docs/index.md b/docs/index.md index 97c432e60..c0e32d249 100644 --- a/docs/index.md +++ b/docs/index.md @@ -147,7 +147,7 @@ For more details, please see the following: on setting up your development environment and building bpfman. - [Example eBPF Programs](./getting-started/example-bpf.md) for some examples of eBPF programs written in Go, interacting with `bpfman`. -- [Deploying the bpfman-operator](./developer-guide/operator-quick-start.md) for +- [Deploying the bpfman-operator](./getting-started/operator-quick-start.md) for details on launching bpfman in a Kubernetes cluster. - [Meet the Community](./governance/MEETINGS.md) for details on community meeting details. diff --git a/mkdocs.yml b/mkdocs.yml index 77d3f9b9d..342302593 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -56,6 +56,7 @@ nav: - Setup and Building: getting-started/building-bpfman.md - Launching bpfman: getting-started/launching-bpfman.md - Deploying Example eBPF Programs On Local Host: getting-started/example-bpf-local.md + - Deploying the bpfman-operator: getting-started/operator-quick-start.md - Deploying Example eBPF Programs On Kubernetes: getting-started/example-bpf-k8s.md - Run bpfman From Release Image: getting-started/running-release.md - Run bpfman From RPM: getting-started/running-rpm.md @@ -65,7 +66,6 @@ nav: - Developer Documentation: - Contributing: governance/CONTRIBUTING.md - Reviewing Guide: governance/REVIEWING.md - - Deploying the bpfman-operator: developer-guide/operator-quick-start.md - Developing the bpfman-operator: developer-guide/develop-operator.md - Kubernetes CRD API-Reference: developer-guide/api-spec.md - eBPF Bytecode Image Specifications: developer-guide/shipping-bytecode.md From 0c8e358a7cad2f3c2259d6cfa1e407828d79fc41 Mon Sep 17 00:00:00 2001 From: Billy McFall <22157057+Billy99@users.noreply.github.com> Date: Wed, 18 Dec 2024 15:44:29 -0500 Subject: [PATCH 3/3] doc: add namespaced CRDs to docs Update the documentation for namespace scoped CRDs. Signed-off-by: Billy McFall <22157057+Billy99@users.noreply.github.com> --- docs/getting-started/operator-quick-start.md | 285 +++++++++++++++++-- 1 file changed, 259 insertions(+), 26 deletions(-) diff --git a/docs/getting-started/operator-quick-start.md b/docs/getting-started/operator-quick-start.md index 2102b18fc..1f37acfb7 100644 --- a/docs/getting-started/operator-quick-start.md +++ b/docs/getting-started/operator-quick-start.md @@ -96,14 +96,69 @@ Independent of the method used to deploy, if the bpfman-operator came up success you will see the bpfman-daemon and bpfman-operator pods running without errors: ```bash -kubectl get pods -n bpfman +$ kubectl get pods -n bpfman NAME READY STATUS RESTARTS AGE bpfman-daemon-w24pr 3/3 Running 0 130m bpfman-operator-78cf9c44c6-rv7f2 2/2 Running 0 132m ``` +## API Types Overview + +Refer to [api-spec.md](../developer-guide/api-spec.md) for a more detailed description of all the bpfman Kubernetes API types. + +### Cluster Scoped Versus Namespaced Scoped CRDs + +For security reasons, cluster admins may want to limit certain applications to only loading eBPF programs +within a given namespace. +To provide these tighter controls on eBPF program loading, some of the bpfman Custom Resource Definitions (CRDs) +are Namespace scoped. +Not all eBPF programs make sense to be namespaced scoped. +The namespaced scoped CRDs use the "NsProgram" identifier and cluster scoped CRDs to use "Program" +identifier. + +### Multiple Program CRDs + +The multiple `*Program` CRDs are the bpfman Kubernetes API objects most relevant to users and can be used to +understand clusterwide state for an eBPF program. +It's designed to express how, and where eBPF programs are to be deployed within a Kubernetes cluster. +Currently bpfman supports: + +* `fentryProgram` +* `fexitProgram` +* `kprobeProgram` +* `tcProgram` and `tcNsProgram` +* `tcxProgram` and `tcxNsProgram` +* `tracepointProgram` +* `uprobeProgram` and `uprobeNsProgam` +* `xdpProgram` and `xdpNsProgram` + +There is also the `bpfApplication` and `bpfNsApplication` CRDs, which are +designed for managing eBPF programs at an application level within a Kubernetes cluster. +These CRD allows Kubernetes users to define which eBPF programs are essential for an application's operations +and specify how these programs should be deployed across the cluster. +With cluster scoped variant (`bpfApplication`), any variation of the cluster scoped +eBPF programs can be loaded. +With namespace scoped variant (`bpfNsApplication`), any variation of the namespace scoped +eBPF programs can be loaded. + +### BpfProgram and BpfNsProgram CRD + +The `BpfProgram` and `BpfNsProgram` CRDs are used internally by the bpfman-deployment to keep track of per +node bpfman state such as map pin points, and to report node specific errors back to the user. +Kubernetes users/controllers are only allowed to view these objects, NOT create or edit them. + +Applications wishing to use bpfman to deploy/manage their eBPF programs in Kubernetes will make use of this +object to find references to the bpfMap pin points (`spec.maps`) in order to configure their eBPF programs. + ## Deploy an eBPF Program to the cluster +There are sample yamls for each of the support program type in the +[bpfman-operator/config/samples](https://github.com/bpfman/bpfman-operator/tree/main/config/samples) +directory. + +### Deploy Cluster Scoped Sample + +Any of the cluster scoped samples can be applied as is. To test the deployment simply deploy one of the sample `xdpPrograms`: ```bash @@ -115,7 +170,7 @@ If loading of the XDP Program to the selected nodes was successful it will be re back to the user via the `xdpProgram`'s status field: ```bash -kubectl get xdpprogram xdp-pass-all-nodes -o yaml +$ kubectl get xdpprogram xdp-pass-all-nodes -o yaml apiVersion: bpfman.io/v1alpha1 kind: XdpProgram metadata: @@ -160,39 +215,217 @@ status: To see information in listing form simply run: ```bash -kubectl get xdpprogram -o wide +$ kubectl get xdpprogram -o wide NAME BPFFUNCTIONNAME NODESELECTOR PRIORITY INTERFACESELECTOR PROCEEDON xdp-pass-all-nodes pass {} 0 {"primarynodeinterface":true} ["pass","dispatcher_return"] ``` -There are sample yamls for each of the support program type in the -[config/samples](https://github.com/bpfman/bpfman-operator/tree/main/config/samples) -directory. +To view each attachment point on each node, use the `bpfProgram` object: -## API Types Overview +```bash +$ kubectl get bpfprograms +NAME TYPE STATUS AGE +xdp-pass-all-nodes-f3def00d xdp bpfmanLoaded 56s -See [api-spec.md](../developer-guide/api-spec.md) for a more detailed description of all the bpfman Kubernetes API types. -### Multiple Program CRDs +$ kubectl get bpfprograms xdp-pass-all-nodes-f3def00d -o yaml +apiVersion: bpfman.io/v1alpha1 +kind: BpfProgram +metadata: + annotations: + bpfman.io.xdpprogramcontroller/interface: eth0 + bpfman.io/ProgramId: "26577" + bpfman.io/bpfProgramAttachPoint: eth0 + creationTimestamp: "2024-12-18T22:26:55Z" + finalizers: + - bpfman.io.xdpprogramcontroller/finalizer + generation: 1 + labels: + bpfman.io/appProgramId: "" + bpfman.io/ownedByProgram: xdp-pass-all-nodes + kubernetes.io/hostname: bpfman-deployment-control-plane + name: xdp-pass-all-nodes-f3def00d + ownerReferences: + - apiVersion: bpfman.io/v1alpha1 + blockOwnerDeletion: true + controller: true + kind: XdpProgram + name: xdp-pass-all-nodes + uid: 7685a5b6-a626-4483-8c20-06b29643a2a8 + resourceVersion: "8430" + uid: 83c5a80d-2dca-46ce-806b-6fdf7bde901f +spec: + type: xdp +status: + conditions: + - lastTransitionTime: "2024-12-18T22:27:11Z" + message: Successfully loaded bpfProgram + reason: bpfmanLoaded + status: "True" + type: Loaded +``` -The multiple `*Program` CRDs are the bpfman Kubernetes API objects most relevant to users and can be used to -understand clusterwide state for an eBPF program. -It's designed to express how, and where eBPF programs are to be deployed within a Kubernetes cluster. -Currently bpfman supports: +### Deploy Namespace Scoped Sample -* `fentryProgram` -* `fexitProgram` -* `kprobeProgram` -* `tcProgram` -* `tracepointProgram` -* `uprobeProgram` -* `xdpProgram` +The namespace scoped samples need a namespace and pods to attach to. +A yaml has been created that will create a `Namespace` called "acme". +([bpfman-operator/hack/namespace_scoped.yaml](https://github.com/bpfman/bpfman-operator/blob/main/hack/namespace_scoped.yaml)). +The reason for namespace scoped CRDs is to limit an application or user to a namespace. +To this end, this yaml also creates a `ServiceAccount`, `Role`, `RoleBinding` and `Secret`. -## BpfProgram CRD +```bash +cd bpfman-operator +kubectl apply -f hack/namespace_scoped.yaml + namespace/acme created + serviceaccount/test-account created + role.rbac.authorization.k8s.io/test-account created + rolebinding.rbac.authorization.k8s.io/test-account created + secret/test-account-token created +``` -The `BpfProgram` CRD is used internally by the bpfman-deployment to keep track of per node bpfman state -such as map pin points, and to report node specific errors back to the user. -Kubernetes users/controllers are only allowed to view these objects, NOT create or edit them. +To create a `kubeconfig` file that limits access to the created namespace, use the script +[bpfman-operator/hack/namespace_scoped.sh](https://github.com/bpfman/bpfman-operator/blob/main/hack/nginx-deployment.sh). +The script needs to know the name of the `Cluster`, `Namespace`, `Service Account` and `Secret`. +The script defaults these fields to what is currently in +[bpfman-operator/hack/namespace_scoped.yaml](https://github.com/bpfman/bpfman-operator/blob/main/hack/namespace_scoped.yaml). +However, if a file is passed to the script, it will look for the `Secret` object and attempt to extract the values. +This can be used if the names are changed or a different yaml file is used. +The output of the script is the contents of a `kubeconfig`. +This can be printed to the console or redirected to a file. -Applications wishing to use bpfman to deploy/manage their eBPF programs in Kubernetes will make use of this -object to find references to the bpfMap pin points (`spec.maps`) in order to configure their eBPF programs. +```bash +./hack/namespace_scoped.sh hack/namespace_scoped.yaml > /tmp/kubeconfig +``` + +To use the `kubeconfig` file, select the session to limit access in and run: + +```bash +export KUBECONFIG=/tmp/kubeconfig +``` + +From within this limited access session, a sample `nginx` deployment can be created in the same namespace using +[bpfman-operator/hack/namespace_scoped.yaml](https://github.com/bpfman/bpfman-operator/blob/main/hack/nginx-deployment.yaml). + +```bash +kubectl apply -f hack/nginx-deployment.yaml + deployment.apps/nginx-deployment created +``` + +Finally, load any of the namespaced samples from +[bpfman-operator/config/samples](https://github.com/bpfman/bpfman-operator/tree/main/config/samples). +They are of the format: `bpfman.io_v1alpha1_*nsprogram.yaml` + +```bash +kubectl apply -f config/samples/bpfman.io_v1alpha1_tc_pass_tcnsprogram.yaml + tcnsprogram.bpfman.io/tc-containers created +``` + +The status for each namespaced program is reported via the \*NsProgram status field and further +information can be seen in the resulting BpfNsProgram CRDs. +As an example, the following commands display the information of the TC program loaded in the acme +namespace with the command above. + +```bash +$ kubectl get tcnsprograms +NAME BPFFUNCTIONNAME NODESELECTOR STATUS +tc-containers pass {} ReconcileSuccess + + +$ kubectl get tcnsprograms tc-containers -o yaml +apiVersion: bpfman.io/v1alpha1 +kind: TcNsProgram +metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"bpfman.io/v1alpha1","kind":"TcNsProgram","metadata":{"annotations":{},"labels":{"app.kubernetes.io/name":"tcnsprogram"},"name":"tc-containers","namespace":"acme"},"spec":{"bpffunctionname":"pass","bytecode":{"image":{"url":"quay.io/bpfman-bytecode/tc_pass:latest"}},"containers":{"containernames":["nginx"],"pods":{"matchLabels":{"app":"nginx"}}},"direction":"ingress","globaldata":{"GLOBAL_u32":[13,12,11,10],"GLOBAL_u8":[1]},"interfaceselector":{"interfaces":["eth0"]},"nodeselector":{},"priority":0}} + creationTimestamp: "2024-12-18T22:22:52Z" + finalizers: + - bpfman.io.operator/finalizer + generation: 2 + labels: + app.kubernetes.io/name: tcnsprogram + name: tc-containers + namespace: acme + resourceVersion: "7993" + uid: 49291f28-49dc-4486-9119-af7c31569de3 +spec: + bpffunctionname: pass + bytecode: + image: + imagepullpolicy: IfNotPresent + url: quay.io/bpfman-bytecode/tc_pass:latest + containers: + containernames: + - nginx + pods: + matchLabels: + app: nginx + direction: ingress + globaldata: + GLOBAL_u8: AQ== + GLOBAL_u32: DQwLCg== + interfaceselector: + interfaces: + - eth0 + mapownerselector: {} + nodeselector: {} + priority: 0 + proceedon: + - pipe + - dispatcher_return +status: + conditions: + - lastTransitionTime: "2024-12-18T22:23:11Z" + message: bpfProgramReconciliation Succeeded on all nodes + reason: ReconcileSuccess + status: "True" + type: ReconcileSuccess +``` + +To view each attachment point on each node, use the `bpfNsProgram` object: + +```bash +$ kubectl get bpfnsprograms +NAME TYPE STATUS AGE +tc-containers-6494dbed tc bpfmanLoaded 12m +tc-containers-7dcde5ab tc bpfmanLoaded 12m + + +$ kubectl get bpfnsprograms tc-containers-6494dbed -o yaml +apiVersion: bpfman.io/v1alpha1 +kind: BpfNsProgram +metadata: + annotations: + bpfman.io.tcnsprogramcontroller/containerpid: "3256" + bpfman.io.tcnsprogramcontroller/interface: eth0 + bpfman.io/ProgramId: "26575" + bpfman.io/bpfProgramAttachPoint: eth0-ingress-nginx-deployment-57d84f57dc-lgc6f-nginx + creationTimestamp: "2024-12-18T22:23:08Z" + finalizers: + - bpfman.io.tcnsprogramcontroller/finalizer + generation: 1 + labels: + bpfman.io/appProgramId: "" + bpfman.io/ownedByProgram: tc-containers + kubernetes.io/hostname: bpfman-deployment-control-plane + name: tc-containers-6494dbed + namespace: acme + ownerReferences: + - apiVersion: bpfman.io/v1alpha1 + blockOwnerDeletion: true + controller: true + kind: TcNsProgram + name: tc-containers + uid: 49291f28-49dc-4486-9119-af7c31569de3 + resourceVersion: "7992" + uid: c913eea4-71e0-4d5d-b664-078abac36c40 +spec: + type: tc +status: + conditions: + - lastTransitionTime: "2024-12-18T22:23:11Z" + message: Successfully loaded bpfProgram + reason: bpfmanLoaded + status: "True" + type: Loaded +```