Specify labels/annotations not reflected in VirtualNode API Spec

Author: fra98

apis/virtualkubelet/v1alpha1/virtualnode_types.go

@@ -30,6 +30,10 @@ type Affinity struct {
 
 // OffloadingPatch contains the information to patch the virtual node.
 type OffloadingPatch struct {
+	// AnnotationsNotReflected is the list of annotations (key) that must not be reflected
+	AnnotationsNotReflected []string `json:"annotationsNotReflected,omitempty"`
+	// LabelsNotReflected is the list of labels (key) that must not be reflected
+	LabelsNotReflected []string `json:"labelsNotReflected,omitempty"`
 	// NodeSelector contains the node selector to target the remote cluster.
 	NodeSelector map[string]string `json:"nodeSelector,omitempty"`
 	// Tolerations contains the tolerations to target the remote cluster.

apis/virtualkubelet/v1alpha1/zz_generated.deepcopy.go

@@ -206,23 +206,21 @@ func main() {
 
 	// Options for the virtual kubelet.
 	virtualKubeletOpts := &forge.VirtualKubeletOpts{
-		ContainerImage:          *kubeletImage,
-		ExtraAnnotations:        kubeletExtraAnnotations.StringMap,
-		ExtraLabels:             kubeletExtraLabels.StringMap,
-		ExtraArgs:               kubeletExtraArgs.StringList,
-		NodeExtraAnnotations:    nodeExtraAnnotations,
-		NodeExtraLabels:         nodeExtraLabels,
-		RequestsCPU:             kubeletCPURequests.Quantity,
-		RequestsRAM:             kubeletRAMRequests.Quantity,
-		LimitsCPU:               kubeletCPULimits.Quantity,
-		LimitsRAM:               kubeletRAMLimits.Quantity,
-		IpamEndpoint:            *kubeletIpamServer,
-		MetricsAddress:          kubeletMetricsAddress,
-		MetricsEnabled:          kubeletMetricsEnabled,
-		ReflectorsWorkers:       reflectorsWorkers,
-		ReflectorsType:          reflectorsType,
-		LabelsNotReflected:      labelsNotReflected.StringList,
-		AnnotationsNotReflected: annotationsNotReflected.StringList,
+		ContainerImage:       *kubeletImage,
+		ExtraAnnotations:     kubeletExtraAnnotations.StringMap,
+		ExtraLabels:          kubeletExtraLabels.StringMap,
+		ExtraArgs:            kubeletExtraArgs.StringList,
+		NodeExtraAnnotations: nodeExtraAnnotations,
+		NodeExtraLabels:      nodeExtraLabels,
+		RequestsCPU:          kubeletCPURequests.Quantity,
+		RequestsRAM:          kubeletRAMRequests.Quantity,
+		LimitsCPU:            kubeletCPULimits.Quantity,
+		LimitsRAM:            kubeletRAMLimits.Quantity,
+		IpamEndpoint:         *kubeletIpamServer,
+		MetricsAddress:       kubeletMetricsAddress,
+		MetricsEnabled:       kubeletMetricsEnabled,
+		ReflectorsWorkers:    reflectorsWorkers,
+		ReflectorsType:       reflectorsType,
 	}
 
 	clusterIdentity := clusterIdentityFlags.ReadOrDie()
@@ -435,7 +433,8 @@ func main() {
 	}
 
 	resourceOfferReconciler := resourceoffercontroller.NewResourceOfferController(
-		mgr, idManager, *resyncPeriod, *offerDisableAutoAccept)
+		mgr, idManager, *resyncPeriod, *offerDisableAutoAccept,
+		labelsNotReflected.StringList, annotationsNotReflected.StringList)
 	if err = resourceOfferReconciler.SetupWithManager(mgr); err != nil {
 		klog.Fatal(err)
 	}

cmd/liqo-controller-manager/main.go

@@ -63,9 +63,6 @@ func InstallFlags(flags *pflag.FlagSet, o *Opts) {
 	flags.Var(&o.NodeExtraAnnotations, "node-extra-annotations", "Extra annotations to add to the Virtual Node")
 	flags.Var(&o.NodeExtraLabels, "node-extra-labels", "Extra labels to add to the Virtual Node")
 
-	flags.Var(&o.LabelsNotReflected, "labels-not-reflected", "List of labels (key) that must not be reflected")
-	flags.Var(&o.AnnotationsNotReflected, "annotations-not-reflected", "List of annotations (key) that must not be reflected")
-
 	flags.BoolVar(&o.EnableAPIServerSupport, "enable-apiserver-support", false,
 		"Enable offloaded pods to interact back with the local Kubernetes API server")
 	flags.BoolVar(&o.EnableStorage, "enable-storage", false, "Enable the Liqo storage reflection")

cmd/virtual-kubelet/root/flag.go

@@ -101,9 +101,6 @@ type Opts struct {
 	// Type of reflection to use for each reflected resource
 	ReflectorsType map[string]*string
 
-	LabelsNotReflected      argsutils.StringList
-	AnnotationsNotReflected argsutils.StringList
-
 	NodeLeaseDuration time.Duration
 	NodePingInterval  time.Duration
 	NodePingTimeout   time.Duration
@@ -152,9 +149,6 @@ func NewOpts() *Opts {
 		ReflectorsWorkers: initReflectionWorkers(),
 		ReflectorsType:    initReflectionType(),
 
-		LabelsNotReflected:      argsutils.StringList{},
-		AnnotationsNotReflected: argsutils.StringList{},
-
 		NodeLeaseDuration: node.DefaultLeaseDuration * time.Second,
 		NodePingInterval:  node.DefaultPingInterval,
 		NodePingTimeout:   DefaultNodePingTimeout,

cmd/virtual-kubelet/root/opts.go

@@ -162,9 +162,6 @@ func runRootCommand(ctx context.Context, c *Opts) error {
 		HomeAPIServerHost: c.HomeAPIServerHost,
 		HomeAPIServerPort: c.HomeAPIServerPort,
 
-		LabelsNotReflected:      c.LabelsNotReflected.StringList,
-		AnnotationsNotReflected: c.AnnotationsNotReflected.StringList,
-
 		OffloadingPatch: vn.Spec.OffloadingPatch,
 	}
 

cmd/virtual-kubelet/root/root.go

@@ -377,6 +377,18 @@ spec:
                             x-kubernetes-map-type: atomic
                         type: object
                     type: object
+                  annotationsNotReflected:
+                    description: AnnotationsNotReflected is the list of annotations
+                      (key) that must not be reflected
+                    items:
+                      type: string
+                    type: array
+                  labelsNotReflected:
+                    description: LabelsNotReflected is the list of labels (key) that
+                      must not be reflected
+                    items:
+                      type: string
+                    type: array
                   nodeSelector:
                     additionalProperties:
                       type: string

deployments/liqo/crds/virtualkubelet.liqo.io_virtualnodes.yaml

@@ -26,14 +26,17 @@ import (
 func NewResourceOfferController(
 	mgr manager.Manager,
 	identityReader identitymanager.IdentityReader,
-	resyncPeriod time.Duration, disableAutoAccept bool) *ResourceOfferReconciler {
+	resyncPeriod time.Duration, disableAutoAccept bool,
+	labelsNotReflected, annotationsNotReflected []string) *ResourceOfferReconciler {
 	return &ResourceOfferReconciler{
 		Client: mgr.GetClient(),
 		Scheme: mgr.GetScheme(),
 
-		identityReader:    identityReader,
-		eventsRecorder:    mgr.GetEventRecorderFor("ResourceOffer"),
-		disableAutoAccept: disableAutoAccept,
-		resyncPeriod:      resyncPeriod,
+		identityReader:          identityReader,
+		eventsRecorder:          mgr.GetEventRecorderFor("ResourceOffer"),
+		disableAutoAccept:       disableAutoAccept,
+		resyncPeriod:            resyncPeriod,
+		labelsNotReflected:      labelsNotReflected,
+		annotationsNotReflected: annotationsNotReflected,
 	}
 }

pkg/liqo-controller-manager/resourceoffer-controller/controller_config.go

@@ -41,10 +41,12 @@ type ResourceOfferReconciler struct {
 	client.Client
 	Scheme *runtime.Scheme
 
-	identityReader    identitymanager.IdentityReader
-	eventsRecorder    record.EventRecorder
-	disableAutoAccept bool
-	resyncPeriod      time.Duration
+	identityReader          identitymanager.IdentityReader
+	eventsRecorder          record.EventRecorder
+	disableAutoAccept       bool
+	resyncPeriod            time.Duration
+	labelsNotReflected      []string
+	annotationsNotReflected []string
 }
 
 //+kubebuilder:rbac:groups=sharing.liqo.io,resources=resourceoffers,verbs=get;list;watch;create;update;patch;delete

pkg/liqo-controller-manager/resourceoffer-controller/resourceoffer_controller.go

@@ -32,6 +32,7 @@ import (
 	"github.com/liqotech/liqo/pkg/discovery"
 	virtualnodectrl "github.com/liqotech/liqo/pkg/liqo-controller-manager/virtualnode-controller"
 	foreigncluster "github.com/liqotech/liqo/pkg/utils/foreignCluster"
+	"github.com/liqotech/liqo/pkg/utils/slice"
 )
 
 // setControllerReference sets owner reference to the related ForeignCluster.
@@ -126,6 +127,15 @@ func (r *ResourceOfferReconciler) getVirtualNodeMutator(fc *discoveryv1alpha1.Fo
 		virtualNode.Spec.StorageClasses = resourceOffer.Spec.StorageClasses
 		virtualNode.Spec.IngressClasses = resourceOffer.Spec.IngressClasses
 		virtualNode.Spec.LoadBalancerClasses = resourceOffer.Spec.LoadBalancerClasses
+
+		if virtualNode.Spec.OffloadingPatch == nil {
+			virtualNode.Spec.OffloadingPatch = &virtualkubeletv1alpha1.OffloadingPatch{}
+		}
+		virtualNode.Spec.OffloadingPatch.AnnotationsNotReflected = slice.Merge(
+			r.annotationsNotReflected, virtualNode.Spec.OffloadingPatch.AnnotationsNotReflected)
+		virtualNode.Spec.OffloadingPatch.LabelsNotReflected = slice.Merge(
+			r.labelsNotReflected, virtualNode.Spec.OffloadingPatch.LabelsNotReflected)
+
 		return controllerutil.SetControllerReference(resourceOffer, virtualNode, r.Scheme)
 	}
 }

pkg/liqo-controller-manager/resourceoffer-controller/resourceoffer_controller_methods.go

@@ -116,7 +116,7 @@ var _ = Describe("ResourceOffer Controller", func() {
 		identityReader := fake.NewIdentityReader().Add(remoteClusterIdentity.ClusterID,
 			testNamespace, kubeconfigSecretName, cluster.GetCfg())
 
-		controller = NewResourceOfferController(mgr, identityReader, 10*time.Second, true)
+		controller = NewResourceOfferController(mgr, identityReader, 10*time.Second, true, []string{}, []string{})
 		if err := controller.SetupWithManager(mgr); err != nil {
 			By(err.Error())
 			os.Exit(1)

pkg/liqo-controller-manager/resourceoffer-controller/resourceoffer_controller_test.go

@@ -18,6 +18,7 @@ import (
 	"os"
 	"time"
 
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	discoveryv1alpha1 "github.com/liqotech/liqo/apis/discovery/v1alpha1"
@@ -73,14 +74,27 @@ func ApplyOptions() metav1.ApplyOptions {
 type ForgingOpts struct {
 	LabelsNotReflected      []string
 	AnnotationsNotReflected []string
-	OffloadingPatch         *virtualkubeletv1alpha1.OffloadingPatch
+	NodeSelector            map[string]string
+	Tolerations             []corev1.Toleration
+	Affinity                *virtualkubeletv1alpha1.Affinity
 }
 
 // NewForgingOpts returns a new ForgingOpts instance.
-func NewForgingOpts(labelsNotReflected, annotationsNotReflected []string, offloadingPatch *virtualkubeletv1alpha1.OffloadingPatch) ForgingOpts {
+func NewForgingOpts(offloadingPatch *virtualkubeletv1alpha1.OffloadingPatch) ForgingOpts {
+	if offloadingPatch == nil {
+		return NewEmptyForgingOpts()
+	}
+
 	return ForgingOpts{
-		LabelsNotReflected:      labelsNotReflected,
-		AnnotationsNotReflected: annotationsNotReflected,
-		OffloadingPatch:         offloadingPatch,
+		LabelsNotReflected:      offloadingPatch.LabelsNotReflected,
+		AnnotationsNotReflected: offloadingPatch.AnnotationsNotReflected,
+		NodeSelector:            offloadingPatch.NodeSelector,
+		Tolerations:             offloadingPatch.Tolerations,
+		Affinity:                offloadingPatch.Affinity,
 	}
 }
+
+// NewEmptyForgingOpts returns a new ForgingOpts instance with empty fields.
+func NewEmptyForgingOpts() ForgingOpts {
+	return ForgingOpts{}
+}

pkg/virtualKubelet/forge/forge.go

@@ -79,9 +79,6 @@ type InitConfig struct {
 	HomeAPIServerHost string
 	HomeAPIServerPort string
 
-	LabelsNotReflected      []string
-	AnnotationsNotReflected []string
-
 	OffloadingPatch *vkalpha1.OffloadingPatch
 }
 
@@ -135,8 +132,10 @@ func NewLiqoProvider(ctx context.Context, cfg *InitConfig, eb record.EventBroadc
 	}
 
 	podreflector := workload.NewPodReflector(cfg.RemoteConfig, remoteMetricsClient, ipamClient, &podReflectorConfig, cfg.ReflectorsConfigs[generic.Pod])
-	reflectionManager := manager.New(localClient, remoteClient, localLiqoClient, remoteLiqoClient,
-		cfg.InformerResyncPeriod, eb, cfg.LabelsNotReflected, cfg.AnnotationsNotReflected, cfg.OffloadingPatch).
+
+	forgingOpts := forge.NewForgingOpts(cfg.OffloadingPatch)
+
+	reflectionManager := manager.New(localClient, remoteClient, localLiqoClient, remoteLiqoClient, cfg.InformerResyncPeriod, eb, &forgingOpts).
 		With(podreflector).
 		With(exposition.NewServiceReflector(cfg.ReflectorsConfigs[generic.Service], cfg.EnableLoadBalancer, cfg.RemoteRealLoadBalancerClassName)).
 		With(exposition.NewIngressReflector(cfg.ReflectorsConfigs[generic.Ingress], cfg.EnableIngress, cfg.RemoteRealIngressClassName)).

pkg/virtualKubelet/provider/provider.go

@@ -26,9 +26,9 @@ import (
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/record"
 	"k8s.io/klog/v2"
+	"k8s.io/utils/ptr"
 	"k8s.io/utils/trace"
 
-	virtualkubeletv1alpha1 "github.com/liqotech/liqo/apis/virtualkubelet/v1alpha1"
 	liqoclient "github.com/liqotech/liqo/pkg/client/clientset/versioned"
 	liqoinformers "github.com/liqotech/liqo/pkg/client/informers/externalversions"
 	traceutils "github.com/liqotech/liqo/pkg/utils/trace"
@@ -62,8 +62,7 @@ type manager struct {
 
 // New returns a new manager to start the reflection towards a remote cluster.
 func New(local, remote kubernetes.Interface, localLiqo, remoteLiqo liqoclient.Interface, resync time.Duration,
-	eb record.EventBroadcaster, labelsNotReflected, annotationsNotReflected []string,
-	offloadingPatch *virtualkubeletv1alpha1.OffloadingPatch) Manager {
+	eb record.EventBroadcaster, forgingOpts *forge.ForgingOpts) Manager {
 	// Configure the field selector to retrieve only the pods scheduled on the current virtual node.
 	localPodTweakListOptions := func(opts *metav1.ListOptions) {
 		opts.FieldSelector = fields.OneTermEqualSelector("spec.nodeName", forge.LiqoNodeName).String()
@@ -84,7 +83,7 @@ func New(local, remote kubernetes.Interface, localLiqo, remoteLiqo liqoclient.In
 		started: false,
 		stop:    make(map[string]context.CancelFunc),
 
-		forgingOpts: forge.NewForgingOpts(labelsNotReflected, annotationsNotReflected, offloadingPatch),
+		forgingOpts: ptr.Deref(forgingOpts, forge.NewEmptyForgingOpts()),
 	}
 }
 

pkg/virtualKubelet/reflection/manager/manager.go

@@ -27,6 +27,7 @@ import (
 	virtualkubeletv1alpha1 "github.com/liqotech/liqo/apis/virtualkubelet/v1alpha1"
 	liqoclient "github.com/liqotech/liqo/pkg/client/clientset/versioned"
 	liqoclientfake "github.com/liqotech/liqo/pkg/client/clientset/versioned/fake"
+	"github.com/liqotech/liqo/pkg/virtualKubelet/forge"
 	reflectionfake "github.com/liqotech/liqo/pkg/virtualKubelet/reflection/generic/fake"
 )
 
@@ -37,15 +38,14 @@ var _ = Describe("Manager tests", func() {
 	)
 
 	var (
-		mgr                     Manager
-		localClient             kubernetes.Interface
-		remoteClient            kubernetes.Interface
-		localLiqoClient         liqoclient.Interface
-		remoteLiqoClient        liqoclient.Interface
-		broadcaster             record.EventBroadcaster
-		labelsNotReflected      []string
-		annotationsNotReflected []string
-		offloadingPatch         *virtualkubeletv1alpha1.OffloadingPatch
+		mgr              Manager
+		localClient      kubernetes.Interface
+		remoteClient     kubernetes.Interface
+		localLiqoClient  liqoclient.Interface
+		remoteLiqoClient liqoclient.Interface
+		broadcaster      record.EventBroadcaster
+		offloadingPatch  virtualkubeletv1alpha1.OffloadingPatch
+		forgingOpts      forge.ForgingOpts
 
 		ctx    context.Context
 		cancel context.CancelFunc
@@ -58,12 +58,12 @@ var _ = Describe("Manager tests", func() {
 		localLiqoClient = liqoclientfake.NewSimpleClientset()
 		remoteLiqoClient = liqoclientfake.NewSimpleClientset()
 		broadcaster = record.NewBroadcaster()
+		forgingOpts = forge.NewForgingOpts(&offloadingPatch)
 	})
 	AfterEach(func() { cancel() })
 
 	JustBeforeEach(func() {
-		mgr = New(localClient, remoteClient, localLiqoClient, remoteLiqoClient, 1*time.Hour, broadcaster,
-			labelsNotReflected, annotationsNotReflected, offloadingPatch)
+		mgr = New(localClient, remoteClient, localLiqoClient, remoteLiqoClient, 1*time.Hour, broadcaster, forgingOpts)
 	})
 
 	Context("a new manager is created", func() {

pkg/virtualKubelet/reflection/manager/manager_test.go

@@ -331,11 +331,11 @@ func (npr *NamespacedPodReflector) ForgeShadowPod(ctx context.Context, local *co
 			saSecretRetriever, ipGetter, npr.config.HomeAPIServerHost, npr.config.HomeAPIServerPort),
 		forge.ServiceAccountMutator(npr.config.APIServerSupport, local.Annotations))
 
-	if forgingOpts.OffloadingPatch != nil {
+	if forgingOpts != nil {
 		mutators = append(mutators,
-			forge.NodeSelectorMutator(forgingOpts.OffloadingPatch.NodeSelector),
-			forge.TolerationsMutator(forgingOpts.OffloadingPatch.Tolerations),
-			forge.AffinityMutator(forgingOpts.OffloadingPatch.Affinity))
+			forge.NodeSelectorMutator(forgingOpts.NodeSelector),
+			forge.TolerationsMutator(forgingOpts.Tolerations),
+			forge.AffinityMutator(forgingOpts.Affinity))
 	}
 
 	// Forge the target shadowpod object.

pkg/virtualKubelet/reflection/workload/podns.go

@@ -97,14 +97,6 @@ func forgeVKContainers(
 	args = appendArgsReflectorsWorkers(args, opts.ReflectorsWorkers)
 	args = appendArgsReflectorsType(args, opts.ReflectorsType)
 
-	if len(opts.LabelsNotReflected) > 0 {
-		args = append(args, stringifyArgument(string(LabelsNotReflected), strings.Join(opts.LabelsNotReflected, ",")))
-	}
-
-	if len(opts.AnnotationsNotReflected) > 0 {
-		args = append(args, stringifyArgument(string(AnnotationsNotReflected), strings.Join(opts.AnnotationsNotReflected, ",")))
-	}
-
 	if extraAnnotations := opts.NodeExtraAnnotations.StringMap; len(extraAnnotations) != 0 {
 		args = append(args, stringifyArgument(string(NodeExtraAnnotations), opts.NodeExtraAnnotations.String()))
 	}