Virtual Node: specify labels/annotations not reflected in API Spec

apis/virtualkubelet/v1alpha1/virtualnode_types.go

@@ -30,6 +30,10 @@ type Affinity struct {
 
 // OffloadingPatch contains the information to patch the virtual node.
 type OffloadingPatch struct {
+	// AnnotationsNotReflected is the list of annotations (key) that must not be reflected
+	AnnotationsNotReflected []string `json:"annotationsNotReflected,omitempty"`
+	// LabelsNotReflected is the list of labels (key) that must not be reflected
+	LabelsNotReflected []string `json:"labelsNotReflected,omitempty"`
 	// NodeSelector contains the node selector to target the remote cluster.
 	NodeSelector map[string]string `json:"nodeSelector,omitempty"`
 	// Tolerations contains the tolerations to target the remote cluster.

cmd/liqo-controller-manager/main.go

@@ -206,23 +206,21 @@ func main() {
 
 	// Options for the virtual kubelet.
 	virtualKubeletOpts := &forge.VirtualKubeletOpts{
-		ContainerImage:          *kubeletImage,
-		ExtraAnnotations:        kubeletExtraAnnotations.StringMap,
-		ExtraLabels:             kubeletExtraLabels.StringMap,
-		ExtraArgs:               kubeletExtraArgs.StringList,
-		NodeExtraAnnotations:    nodeExtraAnnotations,
-		NodeExtraLabels:         nodeExtraLabels,
-		RequestsCPU:             kubeletCPURequests.Quantity,
-		RequestsRAM:             kubeletRAMRequests.Quantity,
-		LimitsCPU:               kubeletCPULimits.Quantity,
-		LimitsRAM:               kubeletRAMLimits.Quantity,
-		IpamEndpoint:            *kubeletIpamServer,
-		MetricsAddress:          kubeletMetricsAddress,
-		MetricsEnabled:          kubeletMetricsEnabled,
-		ReflectorsWorkers:       reflectorsWorkers,
-		ReflectorsType:          reflectorsType,
-		LabelsNotReflected:      labelsNotReflected.StringList,
-		AnnotationsNotReflected: annotationsNotReflected.StringList,
+		ContainerImage:       *kubeletImage,
+		ExtraAnnotations:     kubeletExtraAnnotations.StringMap,
+		ExtraLabels:          kubeletExtraLabels.StringMap,
+		ExtraArgs:            kubeletExtraArgs.StringList,
+		NodeExtraAnnotations: nodeExtraAnnotations,
+		NodeExtraLabels:      nodeExtraLabels,
+		RequestsCPU:          kubeletCPURequests.Quantity,
+		RequestsRAM:          kubeletRAMRequests.Quantity,
+		LimitsCPU:            kubeletCPULimits.Quantity,
+		LimitsRAM:            kubeletRAMLimits.Quantity,
+		IpamEndpoint:         *kubeletIpamServer,
+		MetricsAddress:       kubeletMetricsAddress,
+		MetricsEnabled:       kubeletMetricsEnabled,
+		ReflectorsWorkers:    reflectorsWorkers,
+		ReflectorsType:       reflectorsType,
 	}
 
 	clusterIdentity := clusterIdentityFlags.ReadOrDie()
@@ -435,7 +433,8 @@ func main() {
 	}
 
 	resourceOfferReconciler := resourceoffercontroller.NewResourceOfferController(
-		mgr, idManager, *resyncPeriod, *offerDisableAutoAccept)
+		mgr, idManager, *resyncPeriod, *offerDisableAutoAccept,
+		labelsNotReflected.StringList, annotationsNotReflected.StringList)
 	if err = resourceOfferReconciler.SetupWithManager(mgr); err != nil {
 		klog.Fatal(err)
 	}

cmd/virtual-kubelet/root/flag.go

@@ -63,9 +63,6 @@ func InstallFlags(flags *pflag.FlagSet, o *Opts) {
 	flags.Var(&o.NodeExtraAnnotations, "node-extra-annotations", "Extra annotations to add to the Virtual Node")
 	flags.Var(&o.NodeExtraLabels, "node-extra-labels", "Extra labels to add to the Virtual Node")
 
-	flags.Var(&o.LabelsNotReflected, "labels-not-reflected", "List of labels (key) that must not be reflected")
-	flags.Var(&o.AnnotationsNotReflected, "annotations-not-reflected", "List of annotations (key) that must not be reflected")
-
 	flags.BoolVar(&o.EnableAPIServerSupport, "enable-apiserver-support", false,
 		"Enable offloaded pods to interact back with the local Kubernetes API server")
 	flags.BoolVar(&o.EnableStorage, "enable-storage", false, "Enable the Liqo storage reflection")

cmd/virtual-kubelet/root/opts.go

@@ -101,9 +101,6 @@ type Opts struct {
 	// Type of reflection to use for each reflected resource
 	ReflectorsType map[string]*string
 
-	LabelsNotReflected      argsutils.StringList
-	AnnotationsNotReflected argsutils.StringList
-
 	NodeLeaseDuration time.Duration
 	NodePingInterval  time.Duration
 	NodePingTimeout   time.Duration
@@ -152,9 +149,6 @@ func NewOpts() *Opts {
 		ReflectorsWorkers: initReflectionWorkers(),
 		ReflectorsType:    initReflectionType(),
 
-		LabelsNotReflected:      argsutils.StringList{},
-		AnnotationsNotReflected: argsutils.StringList{},
-
 		NodeLeaseDuration: node.DefaultLeaseDuration * time.Second,
 		NodePingInterval:  node.DefaultPingInterval,
 		NodePingTimeout:   DefaultNodePingTimeout,

cmd/virtual-kubelet/root/root.go

@@ -162,9 +162,6 @@ func runRootCommand(ctx context.Context, c *Opts) error {
 		HomeAPIServerHost: c.HomeAPIServerHost,
 		HomeAPIServerPort: c.HomeAPIServerPort,
 
-		LabelsNotReflected:      c.LabelsNotReflected.StringList,
-		AnnotationsNotReflected: c.AnnotationsNotReflected.StringList,
-
 		OffloadingPatch: vn.Spec.OffloadingPatch,
 	}
 

deployments/liqo/crds/virtualkubelet.liqo.io_virtualnodes.yaml

@@ -377,6 +377,18 @@ spec:
                             x-kubernetes-map-type: atomic
                         type: object
                     type: object
+                  annotationsNotReflected:
+                    description: AnnotationsNotReflected is the list of annotations
+                      (key) that must not be reflected
+                    items:
+                      type: string
+                    type: array
+                  labelsNotReflected:
+                    description: LabelsNotReflected is the list of labels (key) that
+                      must not be reflected
+                    items:
+                      type: string
+                    type: array
                   nodeSelector:
                     additionalProperties:
                       type: string

docs/usage/reflection.md

@@ -10,6 +10,8 @@ Briefly, the set of supported resources includes (by category):
 * [**Configuration**](UsageReflectionConfiguration): *ConfigMaps*, *Secrets*, *ServiceAccounts*
 * [**Event**](UsageReflectionEvent): *Events*
 
+(UsageReflectionPolicies)=
+
 ## Reflection policies
 
 Liqo implements two different reflection policies:
@@ -40,6 +42,19 @@ liqoctl install ... --set "reflection.secret.workers=0"
 ```
 ````
 
+(UsageReflectionLabelsAnnots)=
+
+## Disabling the reflection of specific labels and annotations
+
+In some cases, it could be useful to **not propagate** to the remote clusters some labels/annotations present on reflected resources.
+This can be useful to avoid reflecting labels/annotations that lead to conflicts between the local and remote resources (e.g., the ones added by cloud providers and that are tied to the configuration of the hosting cluster), thus **preventing infinite reconciliations** of the reflected resource.
+
+You can disable the reflection of custom labels and annotations by configuring at install-time respectively the Helm values `reflection.skip.labels` and `reflection.skip.annotations` with the list of **keys** that must not be reflected.
+To modify the list of not-reflected labels/annotations if Liqo is already installed you can either:
+
+* Patch the individual virtual nodes fields `spec.offloadingPatch.labelsNotReflected` and `spec.offloadingPatch.annotationsNotReflected`.
+* Patch the *liqo-controller-manager* args or upgrade Liqo with the new Helm values (but keep in mind that existing virtual nodes will not be updated dynamically, so you should unpeer first and peer again to apply the new changes to existing peers).
+
 (UsageReflectionPods)=
 
 ## Pods offloading

pkg/liqo-controller-manager/resourceoffer-controller/controller_config.go

@@ -26,14 +26,17 @@ import (
 func NewResourceOfferController(
 	mgr manager.Manager,
 	identityReader identitymanager.IdentityReader,
-	resyncPeriod time.Duration, disableAutoAccept bool) *ResourceOfferReconciler {
+	resyncPeriod time.Duration, disableAutoAccept bool,
+	labelsNotReflected, annotationsNotReflected []string) *ResourceOfferReconciler {
 	return &ResourceOfferReconciler{
 		Client: mgr.GetClient(),
 		Scheme: mgr.GetScheme(),
 
-		identityReader:    identityReader,
-		eventsRecorder:    mgr.GetEventRecorderFor("ResourceOffer"),
-		disableAutoAccept: disableAutoAccept,
-		resyncPeriod:      resyncPeriod,
+		identityReader:          identityReader,
+		eventsRecorder:          mgr.GetEventRecorderFor("ResourceOffer"),
+		disableAutoAccept:       disableAutoAccept,
+		resyncPeriod:            resyncPeriod,
+		labelsNotReflected:      labelsNotReflected,
+		annotationsNotReflected: annotationsNotReflected,
 	}
 }

pkg/liqo-controller-manager/resourceoffer-controller/resourceoffer_controller.go

@@ -41,10 +41,12 @@ type ResourceOfferReconciler struct {
 	client.Client
 	Scheme *runtime.Scheme
 
-	identityReader    identitymanager.IdentityReader
-	eventsRecorder    record.EventRecorder
-	disableAutoAccept bool
-	resyncPeriod      time.Duration
+	identityReader          identitymanager.IdentityReader
+	eventsRecorder          record.EventRecorder
+	disableAutoAccept       bool
+	resyncPeriod            time.Duration
+	labelsNotReflected      []string
+	annotationsNotReflected []string
 }
 
 //+kubebuilder:rbac:groups=sharing.liqo.io,resources=resourceoffers,verbs=get;list;watch;create;update;patch;delete

pkg/liqo-controller-manager/resourceoffer-controller/resourceoffer_controller_methods.go

@@ -32,6 +32,7 @@ import (
 	"github.com/liqotech/liqo/pkg/discovery"
 	virtualnodectrl "github.com/liqotech/liqo/pkg/liqo-controller-manager/virtualnode-controller"
 	foreigncluster "github.com/liqotech/liqo/pkg/utils/foreignCluster"
+	"github.com/liqotech/liqo/pkg/utils/slice"
 )
 
 // setControllerReference sets owner reference to the related ForeignCluster.
@@ -126,6 +127,15 @@ func (r *ResourceOfferReconciler) getVirtualNodeMutator(fc *discoveryv1alpha1.Fo
 		virtualNode.Spec.StorageClasses = resourceOffer.Spec.StorageClasses
 		virtualNode.Spec.IngressClasses = resourceOffer.Spec.IngressClasses
 		virtualNode.Spec.LoadBalancerClasses = resourceOffer.Spec.LoadBalancerClasses
+
+		if virtualNode.Spec.OffloadingPatch == nil {
+			virtualNode.Spec.OffloadingPatch = &virtualkubeletv1alpha1.OffloadingPatch{}
+		}
+		virtualNode.Spec.OffloadingPatch.AnnotationsNotReflected = slice.Merge(
+			r.annotationsNotReflected, virtualNode.Spec.OffloadingPatch.AnnotationsNotReflected)
+		virtualNode.Spec.OffloadingPatch.LabelsNotReflected = slice.Merge(
+			r.labelsNotReflected, virtualNode.Spec.OffloadingPatch.LabelsNotReflected)
+
 		return controllerutil.SetControllerReference(resourceOffer, virtualNode, r.Scheme)
 	}
 }

pkg/liqo-controller-manager/resourceoffer-controller/resourceoffer_controller_test.go

@@ -116,7 +116,7 @@ var _ = Describe("ResourceOffer Controller", func() {
 		identityReader := fake.NewIdentityReader().Add(remoteClusterIdentity.ClusterID,
 			testNamespace, kubeconfigSecretName, cluster.GetCfg())
 
-		controller = NewResourceOfferController(mgr, identityReader, 10*time.Second, true)
+		controller = NewResourceOfferController(mgr, identityReader, 10*time.Second, true, []string{}, []string{})
 		if err := controller.SetupWithManager(mgr); err != nil {
 			By(err.Error())
 			os.Exit(1)

pkg/virtualKubelet/forge/forge.go

@@ -18,6 +18,7 @@ import (
 	"os"
 	"time"
 
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	discoveryv1alpha1 "github.com/liqotech/liqo/apis/discovery/v1alpha1"
@@ -73,14 +74,27 @@ func ApplyOptions() metav1.ApplyOptions {
 type ForgingOpts struct {
 	LabelsNotReflected      []string
 	AnnotationsNotReflected []string
-	OffloadingPatch         *virtualkubeletv1alpha1.OffloadingPatch
+	NodeSelector            map[string]string
+	Tolerations             []corev1.Toleration
+	Affinity                *virtualkubeletv1alpha1.Affinity
 }
 
 // NewForgingOpts returns a new ForgingOpts instance.
-func NewForgingOpts(labelsNotReflected, annotationsNotReflected []string, offloadingPatch *virtualkubeletv1alpha1.OffloadingPatch) ForgingOpts {
+func NewForgingOpts(offloadingPatch *virtualkubeletv1alpha1.OffloadingPatch) ForgingOpts {
+	if offloadingPatch == nil {
+		return NewEmptyForgingOpts()
+	}
+
 	return ForgingOpts{
-		LabelsNotReflected:      labelsNotReflected,
-		AnnotationsNotReflected: annotationsNotReflected,
-		OffloadingPatch:         offloadingPatch,
+		LabelsNotReflected:      offloadingPatch.LabelsNotReflected,
+		AnnotationsNotReflected: offloadingPatch.AnnotationsNotReflected,
+		NodeSelector:            offloadingPatch.NodeSelector,
+		Tolerations:             offloadingPatch.Tolerations,
+		Affinity:                offloadingPatch.Affinity,
 	}
 }
+
+// NewEmptyForgingOpts returns a new ForgingOpts instance with empty fields.
+func NewEmptyForgingOpts() ForgingOpts {
+	return ForgingOpts{}
+}

pkg/virtualKubelet/provider/provider.go

@@ -79,9 +79,6 @@ type InitConfig struct {
 	HomeAPIServerHost string
 	HomeAPIServerPort string
 
-	LabelsNotReflected      []string
-	AnnotationsNotReflected []string
-
 	OffloadingPatch *vkalpha1.OffloadingPatch
 }
 
@@ -135,8 +132,10 @@ func NewLiqoProvider(ctx context.Context, cfg *InitConfig, eb record.EventBroadc
 	}
 
 	podreflector := workload.NewPodReflector(cfg.RemoteConfig, remoteMetricsClient, ipamClient, &podReflectorConfig, cfg.ReflectorsConfigs[generic.Pod])
-	reflectionManager := manager.New(localClient, remoteClient, localLiqoClient, remoteLiqoClient,
-		cfg.InformerResyncPeriod, eb, cfg.LabelsNotReflected, cfg.AnnotationsNotReflected, cfg.OffloadingPatch).
+
+	forgingOpts := forge.NewForgingOpts(cfg.OffloadingPatch)
+
+	reflectionManager := manager.New(localClient, remoteClient, localLiqoClient, remoteLiqoClient, cfg.InformerResyncPeriod, eb, &forgingOpts).
 		With(podreflector).
 		With(exposition.NewServiceReflector(cfg.ReflectorsConfigs[generic.Service], cfg.EnableLoadBalancer, cfg.RemoteRealLoadBalancerClassName)).
 		With(exposition.NewIngressReflector(cfg.ReflectorsConfigs[generic.Ingress], cfg.EnableIngress, cfg.RemoteRealIngressClassName)).

pkg/virtualKubelet/reflection/manager/manager.go

@@ -26,9 +26,9 @@ import (
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/record"
 	"k8s.io/klog/v2"
+	"k8s.io/utils/ptr"
 	"k8s.io/utils/trace"
 
-	virtualkubeletv1alpha1 "github.com/liqotech/liqo/apis/virtualkubelet/v1alpha1"
 	liqoclient "github.com/liqotech/liqo/pkg/client/clientset/versioned"
 	liqoinformers "github.com/liqotech/liqo/pkg/client/informers/externalversions"
 	traceutils "github.com/liqotech/liqo/pkg/utils/trace"
@@ -62,8 +62,7 @@ type manager struct {
 
 // New returns a new manager to start the reflection towards a remote cluster.
 func New(local, remote kubernetes.Interface, localLiqo, remoteLiqo liqoclient.Interface, resync time.Duration,
-	eb record.EventBroadcaster, labelsNotReflected, annotationsNotReflected []string,
-	offloadingPatch *virtualkubeletv1alpha1.OffloadingPatch) Manager {
+	eb record.EventBroadcaster, forgingOpts *forge.ForgingOpts) Manager {
 	// Configure the field selector to retrieve only the pods scheduled on the current virtual node.
 	localPodTweakListOptions := func(opts *metav1.ListOptions) {
 		opts.FieldSelector = fields.OneTermEqualSelector("spec.nodeName", forge.LiqoNodeName).String()
@@ -84,7 +83,7 @@ func New(local, remote kubernetes.Interface, localLiqo, remoteLiqo liqoclient.In
 		started: false,
 		stop:    make(map[string]context.CancelFunc),
 
-		forgingOpts: forge.NewForgingOpts(labelsNotReflected, annotationsNotReflected, offloadingPatch),
+		forgingOpts: ptr.Deref(forgingOpts, forge.NewEmptyForgingOpts()),
 	}
 }
 

pkg/virtualKubelet/reflection/manager/manager_test.go

@@ -27,6 +27,7 @@ import (
 	virtualkubeletv1alpha1 "github.com/liqotech/liqo/apis/virtualkubelet/v1alpha1"
 	liqoclient "github.com/liqotech/liqo/pkg/client/clientset/versioned"
 	liqoclientfake "github.com/liqotech/liqo/pkg/client/clientset/versioned/fake"
+	"github.com/liqotech/liqo/pkg/virtualKubelet/forge"
 	reflectionfake "github.com/liqotech/liqo/pkg/virtualKubelet/reflection/generic/fake"
 )
 
@@ -37,15 +38,14 @@ var _ = Describe("Manager tests", func() {
 	)
 
 	var (
-		mgr                     Manager
-		localClient             kubernetes.Interface
-		remoteClient            kubernetes.Interface
-		localLiqoClient         liqoclient.Interface
-		remoteLiqoClient        liqoclient.Interface
-		broadcaster             record.EventBroadcaster
-		labelsNotReflected      []string
-		annotationsNotReflected []string
-		offloadingPatch         *virtualkubeletv1alpha1.OffloadingPatch
+		mgr              Manager
+		localClient      kubernetes.Interface
+		remoteClient     kubernetes.Interface
+		localLiqoClient  liqoclient.Interface
+		remoteLiqoClient liqoclient.Interface
+		broadcaster      record.EventBroadcaster
+		offloadingPatch  virtualkubeletv1alpha1.OffloadingPatch
+		forgingOpts      forge.ForgingOpts
 
 		ctx    context.Context
 		cancel context.CancelFunc
@@ -58,12 +58,12 @@ var _ = Describe("Manager tests", func() {
 		localLiqoClient = liqoclientfake.NewSimpleClientset()
 		remoteLiqoClient = liqoclientfake.NewSimpleClientset()
 		broadcaster = record.NewBroadcaster()
+		forgingOpts = forge.NewForgingOpts(&offloadingPatch)
 	})
 	AfterEach(func() { cancel() })
 
 	JustBeforeEach(func() {
-		mgr = New(localClient, remoteClient, localLiqoClient, remoteLiqoClient, 1*time.Hour, broadcaster,
-			labelsNotReflected, annotationsNotReflected, offloadingPatch)
+		mgr = New(localClient, remoteClient, localLiqoClient, remoteLiqoClient, 1*time.Hour, broadcaster, &forgingOpts)
 	})
 
 	Context("a new manager is created", func() {

pkg/virtualKubelet/reflection/workload/podns.go

@@ -331,11 +331,11 @@ func (npr *NamespacedPodReflector) ForgeShadowPod(ctx context.Context, local *co
 			saSecretRetriever, ipGetter, npr.config.HomeAPIServerHost, npr.config.HomeAPIServerPort),
 		forge.ServiceAccountMutator(npr.config.APIServerSupport, local.Annotations))
 
-	if forgingOpts.OffloadingPatch != nil {
+	if forgingOpts != nil {
 		mutators = append(mutators,
-			forge.NodeSelectorMutator(forgingOpts.OffloadingPatch.NodeSelector),
-			forge.TolerationsMutator(forgingOpts.OffloadingPatch.Tolerations),
-			forge.AffinityMutator(forgingOpts.OffloadingPatch.Affinity))
+			forge.NodeSelectorMutator(forgingOpts.NodeSelector),
+			forge.TolerationsMutator(forgingOpts.Tolerations),
+			forge.AffinityMutator(forgingOpts.Affinity))
 	}
 
 	// Forge the target shadowpod object.