Initial bpfman-operator support for Load/Attach Split

Makefile

@@ -208,8 +208,8 @@ COMMON_FLAGS ?= ${VERIFY_FLAG} --go-header-file $(shell pwd)/hack/boilerplate.go
 .PHONY: manifests
 manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
 	$(CONTROLLER_GEN) crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases
-	$(CONTROLLER_GEN) rbac:roleName=agent-role paths="./controllers/bpfman-agent/..." output:rbac:artifacts:config=config/rbac/bpfman-agent
-	$(CONTROLLER_GEN) rbac:roleName=operator-role paths="./controllers/bpfman-operator" output:rbac:artifacts:config=config/rbac/bpfman-operator
+	$(CONTROLLER_GEN) rbac:roleName=agent-role paths="./controllers/bpfman-agent/...;./controllers/app-agent/..." output:rbac:artifacts:config=config/rbac/bpfman-agent
+	$(CONTROLLER_GEN) rbac:roleName=operator-role paths="./controllers/bpfman-operator;./controllers/app-operator" output:rbac:artifacts:config=config/rbac/bpfman-operator
 
 .PHONY: generate
 generate: manifests generate-register generate-deepcopy generate-typed-clients generate-typed-listers generate-typed-informers ## Generate ALL auto-generated code.

PROJECT

@@ -129,4 +129,11 @@ resources:
   kind: BpfNsApplication
   path: github.com/bpfman/bpfman-operator/apis/v1alpha1
   version: v1alpha1
+- api:
+    crdVersion: v1
+  controller: true
+  domain: bpfman.io
+  kind: BpfApplicationState
+  path: github.com/bpfman/bpfman-operator/apis/v1alpha1
+  version: v1alpha1
 version: "3"

TODO.md

@@ -0,0 +1,155 @@
+# Intro
+
+The code has support for XDP, TCX, and Fentry programs in a BpfApplication.
+
+It's written so that Dave's load/attach split code should drop in pretty easily,
+but it's not using it yet.  I'm simulating the attachments by reloading the code
+for each attachment (like we do today).
+
+# Observation/Question
+Fentry and Fexit programs break the mold.
+
+Fentry and Fexit programs need to be loaded separately for each attach point, so
+the user must specify the BPF function name and attach point together for each
+attachment.  The user can then attach or detach that program later, but if the
+user wants to attach the same Fentry/Fexit program to a different attach point,
+the program must be loaded again with the new attach point.
+
+For other program types, the user can load a program, and then attach or detach
+the program to/from multiple attach points at any time after it has been loaded.
+
+Some differences that result from these requirements:
+- Each Fentry/Fexit attach point results in a unique bpf program ID (even if
+  they all use the same bpf function)
+- For other types, a given bpf program can have one bpf program ID (assigned
+  when it's loaded), plus multiple attach IDs (assigened when it is attached)
+- We don't need an attach ID for Fentry/Fexit programs.
+
+We need to do one of the following:
+- Not support the user modifying Fentry/Fexit attach points after the initial
+  BpfApplication load.
+- Load the program if they add an attach point (which would result in an
+  independent set of global data), and unload the program if they remove an
+  attachment.
+
+Yaml options:
+
+**Option 1:**  The current design uses a map indexed by the bpffunction name, so
+we can only list a given bpffunction name once followed by a list of attach
+points as shown below.  This represents Fentry/Fexit programs the same way as
+others, but they would need to behave differently as outlined above.
+
+```yaml
+programs:
+  tcx_stats:
+    type: TCX
+    tcx:
+      attach_points:
+        - interfaceselector:
+            primarynodeinterface: true
+          priority: 500
+          direction: ingress
+        - interfaceselector:
+            interfaces:
+              - eth1
+          priority: 100
+          direction: egress
+  test_fentry:
+    type: Fentry
+    fentry:
+      attach_points:
+        - function_name: do_unlinkat
+          attach: true
+        - function_name: tcp_connect
+          attach: false
+```
+
+**Options 2:**  Use a slice, and allow the same Fentry/Fexit functions to be
+included multiple times. The is more like the bpfman api, but potentially more
+cumbersome for Fentry/Fexit programs.
+
+```yaml
+  programs:
+    - type: TCX
+      tcx:
+        bpffunctionname: tcx_stats
+        attach_points:
+          - interfaceselector:
+              primarynodeinterface: true
+            priority: 500
+            direction: ingress
+          - interfaceselector:
+              interfaces:
+                - eth0
+            priority: 100
+            direction: egress
+            containers:
+              namespace: bpfman
+              pods:
+                matchLabels:
+                  name: bpfman-daemon
+              containernames:
+                - bpfman
+                - bpfman-agent
+    - type: Fentry
+      fentry:
+        bpffunctionname: tcx_stats
+        function_name: do_unlinkat
+        attach: true
+    - type: Fentry
+      fentry:
+        bpffunctionname: tcx_stats
+        function_name: tcp_connect
+        attach: true
+```
+
+# New Code
+
+The new code is mainly in these directories:
+
+## Updated & working APIs:
+
+- apis/v1alpha1/fentryProgram_types.go
+- apis/v1alpha1/xdpProgram_types.go
+- apis/v1alpha1/tcxProgram_types.go
+- apis/v1alpha1/bpfApplication_types.go
+- apis/v1alpha1/bpfApplicationState_types.go
+
+Note: the rest are partially updated.
+
+## New Agent:
+
+- controllers/app-agent
+
+## New Operator:
+
+- controllers/app-operator
+
+Note: I left the old bpfman-agent and bpfman-operator code unchanged (except as
+needed due to CRD changed).  It should work, but it's not being initialized when
+we run the operator.
+
+# Testing:
+
+- Unit tests for the agent and the operator
+- The following working samples:
+    - config/samples/bpfman.io_v1alpha1_bpfapplication.yaml (XDP & TCX)
+    - config/samples/bpfman.io_v1alpha1_fentry_bpfapplication.yaml (Fentry)
+
+# TODO:
+
+(In no particular order.)
+
+- Implement Fentry/Fexit solution.
+- Integrate with the new bpfman code with load/attach split (of course)
+- Create a bpf.o file with all the application types for both cluster and
+  namespace scoped BpfApplicaitons.
+- Redo the status/condition values.  I’m currently using the existing framework
+  and some values for status/conditions, but I intend to create a new set of
+  conditions/status values that make more sense for the new design.
+- Review all comments and logs.
+- Maybe make more code common.
+- Support the rest of the program types (including namespace-scoped CRDs).
+- Delete old directories.
+- Lots more testing and code cleanup.
+- Lots of other stuff (I'm sure).

apis/v1alpha1/bpfApplicationState_types.go

@@ -0,0 +1,167 @@
+/*
+Copyright 2023 The bpfman Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	metav1types "k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+// BpfApplicationProgramState defines the desired state of BpfApplication
+// +union
+// +kubebuilder:validation:XValidation:rule="has(self.type) && self.type == 'XDP' ?  has(self.xdp) : !has(self.xdp)",message="xdp configuration is required when type is XDP, and forbidden otherwise"
+// // +kubebuilder:validation:XValidation:rule="has(self.type) && self.type == 'TC' ?  has(self.tc) : !has(self.tc)",message="tc configuration is required when type is TC, and forbidden otherwise"
+// +kubebuilder:validation:XValidation:rule="has(self.type) && self.type == 'TCX' ?  has(self.tcx) : !has(self.tcx)",message="tcx configuration is required when type is TCX, and forbidden otherwise"
+// +kubebuilder:validation:XValidation:rule="has(self.type) && self.type == 'Fentry' ?  has(self.fentry) : !has(self.fentry)",message="fentry configuration is required when type is Fentry, and forbidden otherwise"
+// // +kubebuilder:validation:XValidation:rule="has(self.type) && self.type == 'Fexit' ?  has(self.fexit) : !has(self.fexit)",message="fexit configuration is required when type is Fexit, and forbidden otherwise"
+// // +kubebuilder:validation:XValidation:rule="has(self.type) && self.type == 'Kprobe' ?  has(self.kprobe) : !has(self.kprobe)",message="kprobe configuration is required when type is Kprobe, and forbidden otherwise"
+// // +kubebuilder:validation:XValidation:rule="has(self.type) && self.type == 'Kretprobe' ?  has(self.kretprobe) : !has(self.kretprobe)",message="kretprobe configuration is required when type is Kretprobe, and forbidden otherwise"
+// // +kubebuilder:validation:XValidation:rule="has(self.type) && self.type == 'Uprobe' ?  has(self.uprobe) : !has(self.uprobe)",message="uprobe configuration is required when type is Uprobe, and forbidden otherwise"
+// // +kubebuilder:validation:XValidation:rule="has(self.type) && self.type == 'Uretprobe' ?  has(self.uretprobe) : !has(self.uretprobe)",message="uretprobe configuration is required when type is Uretprobe, and forbidden otherwise"
+// // +kubebuilder:validation:XValidation:rule="has(self.type) && self.type == 'Tracepoint' ?  has(self.tracepoint) : !has(self.tracepoint)",message="tracepoint configuration is required when type is Tracepoint, and forbidden otherwise"
+type BpfApplicationProgramState struct {
+	BpfProgramStateCommon `json:",inline"`
+	// Type specifies the bpf program type
+	// +unionDiscriminator
+	// +kubebuilder:validation:Required
+	// +kubebuilder:validation:Enum:="XDP";"TC";"TCX";"Fentry";"Fexit";"Kprobe";"Kretprobe";"Uprobe";"Uretprobe";"Tracepoint"
+	Type EBPFProgType `json:"type,omitempty"`
+
+	// xdp defines the desired state of the application's XdpPrograms.
+	// +unionMember
+	// +optional
+	XDP *XdpProgramInfoState `json:"xdp,omitempty"`
+
+	// tc defines the desired state of the application's TcPrograms.
+	// +unionMember
+	// +optional
+	TC *TcProgramInfoState `json:"tc,omitempty"`
+
+	// tcx defines the desired state of the application's TcxPrograms.
+	// +unionMember
+	// +optional
+	TCX *TcxProgramInfoState `json:"tcx,omitempty"`
+
+	// fentry defines the desired state of the application's FentryPrograms.
+	// +unionMember
+	// +optional
+	Fentry *FentryProgramInfoState `json:"fentry,omitempty"`
+
+	// // fexit defines the desired state of the application's FexitPrograms.
+	// // +unionMember
+	// // +optional
+	// Fexit *FexitProgramInfoState `json:"fexit,omitempty"`
+
+	// // kprobe defines the desired state of the application's KprobePrograms.
+	// // +unionMember
+	// // +optional
+	// Kprobe *KprobeProgramInfoState `json:"kprobe,omitempty"`
+
+	// // kretprobe defines the desired state of the application's KretprobePrograms.
+	// // +unionMember
+	// // +optional
+	// Kretprobe *KprobeProgramInfoState `json:"kretprobe,omitempty"`
+
+	// // uprobe defines the desired state of the application's UprobePrograms.
+	// // +unionMember
+	// // +optional
+	// Uprobe *UprobeProgramInfoState `json:"uprobe,omitempty"`
+
+	// // uretprobe defines the desired state of the application's UretprobePrograms.
+	// // +unionMember
+	// // +optional
+	// Uretprobe *UprobeProgramInfoState `json:"uretprobe,omitempty"`
+
+	// // tracepoint defines the desired state of the application's TracepointPrograms.
+	// // +unionMember
+	// // +optional
+	// Tracepoint *TracepointProgramInfoState `json:"tracepoint,omitempty"`
+}
+
+// BpfApplicationSpec defines the desired state of BpfApplication
+type BpfApplicationStateSpec struct {
+	// Node is the name of the node for this BpfApplicationStateSpec.
+	Node string `json:"node"`
+	// The number of times the BpfApplicationState has been updated.  Set to 1
+	// when the object is created, then it is incremented prior to each update.
+	// This allows us to verify that the API server has the updated object prior
+	// to starting a new Reconcile operation.
+	UpdateCount int64 `json:"updatecount"`
+	// AppLoadStatus reflects the status of loading the bpf application on the
+	// given node.
+	AppLoadStatus BpfProgramConditionType `json:"apploadstatus"`
+	// Programs is a list of bpf programs contained in the parent application.
+	// It is a map from the bpf program name to BpfApplicationProgramState
+	// elements.
+	Programs []BpfApplicationProgramState `json:"programs,omitempty"`
+}
+
+// +genclient
+// +genclient:nonNamespaced
+// +kubebuilder:object:root=true
+// +kubebuilder:subresource:status
+// +kubebuilder:resource:scope=Cluster
+
+// BpfApplicationState contains the per-node state of a BpfApplication.
+// ANF-TODO: I can't get the Node to display in the kubectl output.
+// +kubebuilder:printcolumn:name="Node",type=string,JSONPath=".spec.node"
+// +kubebuilder:printcolumn:name="Status",type=string,JSONPath=`.status.conditions[0].reason`
+// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp"
+type BpfApplicationState struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   BpfApplicationStateSpec `json:"spec,omitempty"`
+	Status BpfAppStatus            `json:"status,omitempty"`
+}
+
+// +kubebuilder:object:root=true
+// BpfApplicationStateList contains a list of BpfApplicationState objects
+type BpfApplicationStateList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []BpfApplicationState `json:"items"`
+}
+
+func (an BpfApplicationState) GetName() string {
+	return an.Name
+}
+
+func (an BpfApplicationState) GetUID() metav1types.UID {
+	return an.UID
+}
+
+func (an BpfApplicationState) GetAnnotations() map[string]string {
+	return an.Annotations
+}
+
+func (an BpfApplicationState) GetLabels() map[string]string {
+	return an.Labels
+}
+
+func (an BpfApplicationState) GetStatus() *BpfAppStatus {
+	return &an.Status
+}
+
+func (an BpfApplicationState) GetClientObject() client.Object {
+	return &an
+}
+
+func (anl BpfApplicationStateList) GetItems() []BpfApplicationState {
+	return anl.Items
+}

apis/v1alpha1/bpfApplication_types.go

@@ -68,6 +68,7 @@ const (
 // +kubebuilder:validation:XValidation:rule="has(self.type) && self.type == 'Uretprobe' ?  has(self.uretprobe) : !has(self.uretprobe)",message="uretprobe configuration is required when type is Uretprobe, and forbidden otherwise"
 // +kubebuilder:validation:XValidation:rule="has(self.type) && self.type == 'Tracepoint' ?  has(self.tracepoint) : !has(self.tracepoint)",message="tracepoint configuration is required when type is Tracepoint, and forbidden otherwise"
 type BpfApplicationProgram struct {
+	BpfProgramCommon `json:",inline"`
 	// Type specifies the bpf program type
 	// +unionDiscriminator
 	// +kubebuilder:validation:Required
@@ -128,19 +129,12 @@ type BpfApplicationProgram struct {
 // BpfApplicationSpec defines the desired state of BpfApplication
 type BpfApplicationSpec struct {
 	BpfAppCommon `json:",inline"`
-
-	// Programs is a list of bpf programs supported for a specific application.
-	// It's possible that the application can selectively choose which program(s)
-	// to run from this list.
-	// +kubebuilder:validation:MinItems:=1
+	// Programs is the list of bpf programs in the BpfApplication that should be
+	// loaded. The application can selectively choose which program(s) to run
+	// from this list based on the optional attach points provided.
 	Programs []BpfApplicationProgram `json:"programs,omitempty"`
 }
 
-// BpfApplicationStatus defines the observed state of BpfApplication
-type BpfApplicationStatus struct {
-	BpfProgramStatusCommon `json:",inline"`
-}
-
 // +genclient
 // +genclient:nonNamespaced
 //+kubebuilder:object:root=true
@@ -155,8 +149,8 @@ type BpfApplication struct {
 	metav1.TypeMeta   `json:",inline"`
 	metav1.ObjectMeta `json:"metadata,omitempty"`
 
-	Spec   BpfApplicationSpec   `json:"spec,omitempty"`
-	Status BpfApplicationStatus `json:"status,omitempty"`
+	Spec   BpfApplicationSpec `json:"spec,omitempty"`
+	Status BpfAppStatus       `json:"status,omitempty"`
 }
 
 // +kubebuilder:object:root=true

apis/v1alpha1/bpfNsApplication_types.go

@@ -85,7 +85,7 @@ type BpfNsApplication struct {
 	metav1.ObjectMeta `json:"metadata,omitempty"`
 
 	Spec   BpfNsApplicationSpec `json:"spec,omitempty"`
-	Status BpfApplicationStatus `json:"status,omitempty"`
+	Status BpfAppStatus         `json:"status,omitempty"`
 }
 
 // +kubebuilder:object:root=true