[Multitenancy]: Add PodNetwork CRD (#2133)

* Add PN CRD

* fix push error

* minor updates

* Add PN CRD

* update crdgen

* update network to VnetGuid

* minor update

* add doc.go

* update readme

* update crd fields

* update crdgen

* Update crd/external/podnetwork/api/v1alpha1/podnetwork.go

Co-authored-by: Evan Baker <[email protected]>
Signed-off-by: aggarwal0009 <[email protected]>

* address pr comments

---------

Signed-off-by: aggarwal0009 <[email protected]>
Co-authored-by: Evan Baker <[email protected]>

.github/workflows/crdgen.yaml

@@ -26,6 +26,8 @@ jobs:
       run: make -C crd/nodenetworkconfig
     - name: Regenerate MultitenantNetworkContainer CRD
       run: make -C crd/multitenantnetworkcontainer
+    - name: Regenerate PodNetwork CRD
+      run: make -C crd/external/podnetwork
     - name: Regenerate NodeInfo CRD
       run: make -C crd/nodeinfo
     - name: Regenerate MultitenantPodNetworkConfig CRD

crd/external/podnetwork/Makefile

@@ -0,0 +1,19 @@
+.DEFAULT_GOAL: all
+
+REPO_ROOT = $(shell git rev-parse --show-toplevel)
+TOOLS_DIR = $(REPO_ROOT)/build/tools
+TOOLS_BIN_DIR = $(REPO_ROOT)/build/tools/bin
+CONTROLLER_GEN = $(TOOLS_BIN_DIR)/controller-gen
+
+all: generate manifests 
+
+generate: $(CONTROLLER_GEN)
+	$(CONTROLLER_GEN) object paths="./..."
+
+.PHONY: manifests
+manifests: $(CONTROLLER_GEN)
+	mkdir -p manifests
+	$(CONTROLLER_GEN) crd paths="./..." output:crd:artifacts:config=manifests/
+
+$(CONTROLLER_GEN):
+	@make -C $(REPO_ROOT) $(CONTROLLER_GEN)

crd/external/podnetwork/README.md

@@ -0,0 +1,6 @@
+# PodNetwork CRDs
+
+This CRD is added to enable VNET multitenancy – which will be watched and managed by the control plane.
+
+PodNetwork objects need to be created by Orchestrator in the subnet delegation flow.
+These represent a Cx subnet already delegated by the customer to the Orchestrator and locked with a Service Association Link (SAL) on network RP.

crd/external/podnetwork/api/v1alpha1/groupversion_info.go

@@ -0,0 +1,23 @@
+//go:build !ignore_uncovered
+// +build !ignore_uncovered
+
+// Package v1alpha1 contains API Schema definitions for the networking v1alpha1 API group
+// +kubebuilder:object:generate=true
+// +groupName=acn.azure.com
+package v1alpha1
+
+import (
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"sigs.k8s.io/controller-runtime/pkg/scheme"
+)
+
+var (
+	// GroupVersion is group version used to register these objects
+	GroupVersion = schema.GroupVersion{Group: "acn.azure.com", Version: "v1alpha1"}
+
+	// SchemeBuilder is used to add go types to the GroupVersionKind scheme
+	SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion}
+
+	// AddToScheme adds the types in this group-version to the given scheme.
+	AddToScheme = SchemeBuilder.AddToScheme
+)

crd/external/podnetwork/api/v1alpha1/podnetwork.go

@@ -0,0 +1,67 @@
+//go:build !ignore_uncovered
+// +build !ignore_uncovered
+
+package v1alpha1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// Important: Run "make" to regenerate code after modifying this file
+
+// +kubebuilder:object:root=true
+
+// PodNetwork is the Schema for the PodNetworks API
+// +kubebuilder:resource:scope=Namespaced
+// +kubebuilder:resource:shortName=pn
+// +kubebuilder:subresource:status
+// +kubebuilder:printcolumn:name="Status",type=string,priority=1,JSONPath=`.status.status`
+// +kubebuilder:printcolumn:name="Address Prefixes",type=string,priority=1,JSONPath=`.status.addressPrefixes`
+// +kubebuilder:printcolumn:name="Network",type=string,priority=1,JSONPath=`.spec.vnetGUID`
+// +kubebuilder:printcolumn:name="Subnet",type=string,priority=1,JSONPath=`.spec.subnetResourceID`
+type PodNetwork struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   PodNetworkSpec   `json:"spec,omitempty"`
+	Status PodNetworkStatus `json:"status,omitempty"`
+}
+
+// +kubebuilder:object:root=true
+
+// PodNetworkList contains a list of PodNetwork
+type PodNetworkList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []PodNetwork `json:"items"`
+}
+
+// PodNetworkSpec defines the desired state of PodNetwork
+type PodNetworkSpec struct {
+	// +kubebuilder:validation:Optional
+	// customer vnet guid
+	VnetGUID string `json:"vnetGUID,omitempty"`
+	// customer subnet id
+	SubnetResourceID string `json:"subnetResourceID,omitempty"`
+}
+
+// Status indicates the status of PN
+// +kubebuilder:validation:Enum=Ready;InUse;SubnetNotDelegated
+type Status string
+
+const (
+	Ready              Status = "Ready"
+	InUse              Status = "InUse"
+	SubnetNotDelegated Status = "SubnetNotDelegated"
+)
+
+// PodNetworkStatus defines the observed state of PodNetwork
+type PodNetworkStatus struct {
+	// +kubebuilder:validation:Optional
+	Status          Status   `json:"status,omitempty"`
+	AddressPrefixes []string `json:"addressPrefixes,omitempty"`
+}
+
+func init() {
+	SchemeBuilder.Register(&PodNetwork{}, &PodNetworkList{})
+}

crd/external/podnetwork/client.go

@@ -0,0 +1,84 @@
+package podnetwork
+
+import (
+	"context"
+	"reflect"
+
+	"github.com/Azure/azure-container-networking/crd"
+	"github.com/Azure/azure-container-networking/crd/external/podnetwork/api/v1alpha1"
+	"github.com/pkg/errors"
+	v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	typedv1 "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/rest"
+)
+
+// Scheme is a runtime scheme containing the client-go scheme and the PodNetwork scheme.
+var Scheme = runtime.NewScheme()
+
+func init() {
+	_ = scheme.AddToScheme(Scheme)
+	_ = v1alpha1.AddToScheme(Scheme)
+}
+
+// Installer provides methods to manage the lifecycle of the PodNetwork resource definition.
+type Installer struct {
+	cli typedv1.CustomResourceDefinitionInterface
+}
+
+func NewInstaller(c *rest.Config) (*Installer, error) {
+	cli, err := crd.NewCRDClientFromConfig(c)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to init crd client")
+	}
+	return &Installer{
+		cli: cli,
+	}, nil
+}
+
+func (i *Installer) create(ctx context.Context, res *v1.CustomResourceDefinition) (*v1.CustomResourceDefinition, error) {
+	res, err := i.cli.Create(ctx, res, metav1.CreateOptions{})
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to create podnetwork crd")
+	}
+	return res, nil
+}
+
+// Install installs the embedded PodNetwork CRD definition in the cluster.
+func (i *Installer) Install(ctx context.Context) (*v1.CustomResourceDefinition, error) {
+	podnetwork, err := GetPodNetworks()
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get embedded podnetwork crd")
+	}
+	return i.create(ctx, podnetwork)
+}
+
+// InstallOrUpdate installs the embedded PodNetwork CRD definition in the cluster or updates it if present.
+func (i *Installer) InstallOrUpdate(ctx context.Context) (*v1.CustomResourceDefinition, error) {
+	podNetwork, err := GetPodNetworks()
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get embedded podnetwork crd")
+	}
+	current, err := i.create(ctx, podNetwork)
+	if !apierrors.IsAlreadyExists(err) {
+		return current, err
+	}
+	if current == nil {
+		current, err = i.cli.Get(ctx, podNetwork.Name, metav1.GetOptions{})
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to get existing podnetwork crd")
+		}
+	}
+	if !reflect.DeepEqual(podNetwork.Spec.Versions, current.Spec.Versions) {
+		podNetwork.SetResourceVersion(current.GetResourceVersion())
+		previous := *current
+		current, err = i.cli.Update(ctx, podNetwork, metav1.UpdateOptions{})
+		if err != nil {
+			return &previous, errors.Wrap(err, "failed to update existing podnetwork crd")
+		}
+	}
+	return current, nil
+}

crd/external/podnetwork/embed.go

@@ -0,0 +1,24 @@
+package podnetwork
+
+import (
+	_ "embed"
+
+	"github.com/pkg/errors"
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	"sigs.k8s.io/yaml"
+)
+
+// PodNetworkYAML embeds the CRD YAML for downstream consumers.
+//
+//go:embed manifests/acn.azure.com_podnetworks.yaml
+var PodNetworkYAML []byte
+
+// GetPodNetworks parses the raw []byte PodNetwork in
+// to a CustomResourceDefinition and returns it or an unmarshalling error.
+func GetPodNetworks() (*apiextensionsv1.CustomResourceDefinition, error) {
+	podNetworks := &apiextensionsv1.CustomResourceDefinition{}
+	if err := yaml.Unmarshal(PodNetworkYAML, &podNetworks); err != nil {
+		return nil, errors.Wrap(err, "error unmarshalling embedded PodNetwork")
+	}
+	return podNetworks, nil
+}

crd/external/podnetwork/embed_test.go

@@ -0,0 +1,21 @@
+package podnetwork
+
+import (
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+const filename = "manifests/acn.azure.com_podnetworks.yaml"
+
+func TestEmbed(t *testing.T) {
+	b, err := os.ReadFile(filename)
+	assert.NoError(t, err)
+	assert.Equal(t, b, PodNetworkYAML)
+}
+
+func TestGetPodNetworks(t *testing.T) {
+	_, err := GetPodNetworks()
+	assert.NoError(t, err)
+}