Skip to content

Commit

Permalink
Get default registration namespace from template manifest
Browse files Browse the repository at this point in the history 
Signed-off-by: Jian Qiu <[email protected]>
  • Loading branch information
@qiujian16
qiujian16 committed Apr 16, 2024
1 parent 40b0497 commit 4bbd743
Show file tree
Hide file tree
Showing 41 changed files with 1,672 additions and 1,515 deletions.
4 changes: 2 additions & 2 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,9 @@ require (
k8s.io/klog/v2 v2.120.1
k8s.io/kube-aggregator v0.29.3
k8s.io/utils v0.0.0-20240310230437-4693a0247e57
open-cluster-management.io/addon-framework v0.9.1
open-cluster-management.io/addon-framework v0.9.1-0.20240416063208-ecb7f349df05
open-cluster-management.io/api v0.13.1-0.20240411131856-8f6aa25f111c
open-cluster-management.io/sdk-go v0.13.1-0.20240415030117-612344aae744
open-cluster-management.io/sdk-go v0.13.1-0.20240416030555-aa744f426379
sigs.k8s.io/controller-runtime v0.17.3
sigs.k8s.io/kube-storage-version-migrator v0.0.6-0.20230721195810-5c8923c5ff96
)
Expand Down
8 changes: 4 additions & 4 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -423,12 +423,12 @@ k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/A
k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA=
k8s.io/utils v0.0.0-20240310230437-4693a0247e57 h1:gbqbevonBh57eILzModw6mrkbwM0gQBEuevE/AaBsHY=
k8s.io/utils v0.0.0-20240310230437-4693a0247e57/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
open-cluster-management.io/addon-framework v0.9.1 h1:m6n/W29G/4KzMx+8mgC9P/ybuiyNHVy+O6AHEaWbfQg=
open-cluster-management.io/addon-framework v0.9.1/go.mod h1:OEIFCEXhZKO/Grv08CB0T+TGzS0bLshw4G9u7Vw8dw0=
open-cluster-management.io/addon-framework v0.9.1-0.20240416063208-ecb7f349df05 h1:oj4IEyxmQFMxdpnOPE/Y7Xkyfkz4hGVkcYZ4dn8WtKU=
open-cluster-management.io/addon-framework v0.9.1-0.20240416063208-ecb7f349df05/go.mod h1:K+/TwCUz2PZ7L+svaodw6029S/iPJ7eVX21F7NMw+Kw=
open-cluster-management.io/api v0.13.1-0.20240411131856-8f6aa25f111c h1:/iUoY6/PqBmcBq3v0+UBFvIcI39k/QPRGqpOv9XtDIc=
open-cluster-management.io/api v0.13.1-0.20240411131856-8f6aa25f111c/go.mod h1:CuCPEzXDvOyxBB0H1d1eSeajbHqaeGEKq9c63vQc63w=
open-cluster-management.io/sdk-go v0.13.1-0.20240415030117-612344aae744 h1:dBO6eK3gHSoRpl8OckW1zyOp35BOI48rYgoCznrPn40=
open-cluster-management.io/sdk-go v0.13.1-0.20240415030117-612344aae744/go.mod h1:w2OaxtCyegxeyFLU42UQ3oxUz01QdsBQkcHI17T/l48=
open-cluster-management.io/sdk-go v0.13.1-0.20240416030555-aa744f426379 h1:8jXVHfgy+wgXq1mrWC1mTieoP77WsAAHNpzILMIzWB0=
open-cluster-management.io/sdk-go v0.13.1-0.20240416030555-aa744f426379/go.mod h1:w2OaxtCyegxeyFLU42UQ3oxUz01QdsBQkcHI17T/l48=
sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.28.0 h1:TgtAeesdhpm2SGwkQasmbeqDo8th5wOBA5h/AjTKA4I=
sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.28.0/go.mod h1:VHVDI/KrK4fjnV61bE2g3sA7tiETLn8sooImelsCx3Y=
sigs.k8s.io/controller-runtime v0.17.3 h1:65QmN7r3FWgTxDMz9fvGnO1kbf2nu+acg9p2R9oYYYk=
Expand Down
46 changes: 40 additions & 6 deletions pkg/addon/templateagent/template_agent.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,12 +118,11 @@ func (a *CRDTemplateAgentAddon) GetAgentAddonOptions() agent.AgentAddonOptions {
},
SupportedConfigGVRs: supportedConfigGVRs,
Registration: &agent.RegistrationOption{
CSRConfigurations: a.TemplateCSRConfigurationsFunc(),
PermissionConfig: a.TemplatePermissionConfigFunc(),
CSRApproveCheck: a.TemplateCSRApproveCheckFunc(),
CSRSign: a.TemplateCSRSignFunc(),
AgentInstallNamespace: utils.AgentInstallNamespaceFromDeploymentConfigFunc(
utils.NewAddOnDeploymentConfigGetter(a.addonClient)),
CSRConfigurations: a.TemplateCSRConfigurationsFunc(),
PermissionConfig: a.TemplatePermissionConfigFunc(),
CSRApproveCheck: a.TemplateCSRApproveCheckFunc(),
CSRSign: a.TemplateCSRSignFunc(),
AgentInstallNamespace: a.TemplateAgentRegistrationNamespaceFunc,
},
AgentDeployTriggerClusterFilter: func(old, new *clusterv1.ManagedCluster) bool {
return utils.ClusterImageRegistriesAnnotationChanged(old, new) ||
Expand Down Expand Up @@ -233,3 +232,38 @@ func (a *CRDTemplateAgentAddon) getDesiredAddOnTemplateInner(

return template.DeepCopy(), nil
}

// TemplateAgentRegistrationNamespaceFunc reads deployment resource in the manifests and use that namespace as the default
// registration namespace. If addonDeploymentConfig is set, uses the namespace in it.
func (a *CRDTemplateAgentAddon) TemplateAgentRegistrationNamespaceFunc(addon *addonapiv1alpha1.ManagedClusterAddOn) (string, error) {
template, err := a.getDesiredAddOnTemplateInner(addon.Name, addon.Status.ConfigReferences)
if err != nil {
return "", err
}

// pick the namespace of the first deployment
var desiredNS = "open-cluster-management-agent-addon"
for _, manifest := range template.Spec.AgentSpec.Workload.Manifests {
object := &unstructured.Unstructured{}
if err := object.UnmarshalJSON(manifest.Raw); err != nil {
a.logger.Error(err, "failed to extract the object")
continue
}
if object.GetKind() != "Deployment" {
continue
}

desiredNS = object.GetNamespace()
break
}

overrideNs, err := utils.AgentInstallNamespaceFromDeploymentConfigFunc(
utils.NewAddOnDeploymentConfigGetter(a.addonClient))(addon)
if err != nil {
return "", err
}
if len(overrideNs) > 0 {
desiredNS = overrideNs
}
return desiredNS, nil
}
187 changes: 136 additions & 51 deletions pkg/addon/templateagent/template_agent_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@ import (
clusterv1 "open-cluster-management.io/api/cluster/v1"
clusterv1apha1 "open-cluster-management.io/api/cluster/v1alpha1"
workapiv1 "open-cluster-management.io/api/work/v1"

testingcommon "open-cluster-management.io/ocm/pkg/common/testing"
)

func TestAddonTemplateAgentManifests(t *testing.T) {
Expand Down Expand Up @@ -258,7 +260,7 @@ func TestAddonTemplateAgentManifests(t *testing.T) {
addonInformerFactory,
kubeInformers.Rbac().V1().RoleBindings().Lister(),
addonfactory.GetAddOnDeploymentConfigValues(
addonfactory.NewAddOnDeploymentConfigGetter(addonClient),
utils.NewAddOnDeploymentConfigGetter(addonClient),
addonfactory.ToAddOnCustomizedVariableValues,
ToAddOnNodePlacementPrivateValues,
ToAddOnRegistriesPrivateValues,
Expand All @@ -279,10 +281,8 @@ func TestAgentInstallNamespace(t *testing.T) {
addonName := "hello"
clusterName := "cluster1"

s := runtime.NewScheme()
_ = scheme.AddToScheme(s)
_ = clusterv1apha1.Install(s)
_ = addonapiv1alpha1.Install(s)
deployment := testingcommon.NewUnstructured("apps/v1", "Deployment", "test-ns", "test")
deploymentRaw, _ := deployment.MarshalJSON()

cases := []struct {
name string
Expand All @@ -294,6 +294,18 @@ func TestAgentInstallNamespace(t *testing.T) {
}{
{
name: "install namespace is set",
addonTemplate: &addonapiv1alpha1.AddOnTemplate{
ObjectMeta: metav1.ObjectMeta{
Name: "hello-template",
},
Spec: addonapiv1alpha1.AddOnTemplateSpec{
AgentSpec: workapiv1.ManifestWorkSpec{
Workload: workapiv1.ManifestsTemplate{
Manifests: []workapiv1.Manifest{},
},
},
},
},
addonDeploymentConfig: &addonapiv1alpha1.AddOnDeploymentConfig{
ObjectMeta: metav1.ObjectMeta{
Name: "hello-config",
Expand All @@ -319,6 +331,74 @@ func TestAgentInstallNamespace(t *testing.T) {
ObjectMeta: metav1.ObjectMeta{
Name: "hello-template",
},
Spec: addonapiv1alpha1.AddOnTemplateSpec{
AgentSpec: workapiv1.ManifestWorkSpec{
Workload: workapiv1.ManifestsTemplate{
Manifests: []workapiv1.Manifest{},
},
},
},
},
managedClusterAddonBuilder: newManagedClusterAddonBuilder(
&addonapiv1alpha1.ManagedClusterAddOn{
ObjectMeta: metav1.ObjectMeta{
Name: addonName,
Namespace: clusterName,
},
},
),
expected: "open-cluster-management-agent-addon",
},
{
name: "has deployment in the template",
addonTemplate: &addonapiv1alpha1.AddOnTemplate{
ObjectMeta: metav1.ObjectMeta{
Name: "hello-template",
},
Spec: addonapiv1alpha1.AddOnTemplateSpec{
AgentSpec: workapiv1.ManifestWorkSpec{
Workload: workapiv1.ManifestsTemplate{
Manifests: []workapiv1.Manifest{
{RawExtension: runtime.RawExtension{Raw: deploymentRaw}},
},
},
},
},
},
managedClusterAddonBuilder: newManagedClusterAddonBuilder(
&addonapiv1alpha1.ManagedClusterAddOn{
ObjectMeta: metav1.ObjectMeta{
Name: addonName,
Namespace: clusterName,
},
},
),
expected: "test-ns",
},
{
name: "override deployment in the template",
addonTemplate: &addonapiv1alpha1.AddOnTemplate{
ObjectMeta: metav1.ObjectMeta{
Name: "hello-template",
},
Spec: addonapiv1alpha1.AddOnTemplateSpec{
AgentSpec: workapiv1.ManifestWorkSpec{
Workload: workapiv1.ManifestsTemplate{
Manifests: []workapiv1.Manifest{
{RawExtension: runtime.RawExtension{Raw: deploymentRaw}},
},
},
},
},
},
addonDeploymentConfig: &addonapiv1alpha1.AddOnDeploymentConfig{
ObjectMeta: metav1.ObjectMeta{
Name: "hello-config",
Namespace: "default",
},
Spec: addonapiv1alpha1.AddOnDeploymentConfigSpec{
AgentInstallNamespace: "test-install-namespace",
},
},
managedClusterAddonBuilder: newManagedClusterAddonBuilder(
&addonapiv1alpha1.ManagedClusterAddOn{
Expand All @@ -328,60 +408,65 @@ func TestAgentInstallNamespace(t *testing.T) {
},
},
),
expected: "",
expected: "test-install-namespace",
},
}

for _, tc := range cases {
var managedClusterAddon *addonapiv1alpha1.ManagedClusterAddOn
var objs []runtime.Object
if tc.managedClusterAddonBuilder != nil {
if tc.addonTemplate != nil {
tc.managedClusterAddonBuilder.withAddonTemplate(tc.addonTemplate)
objs = append(objs, tc.addonTemplate)
t.Run(tc.name, func(t *testing.T) {
var managedClusterAddon *addonapiv1alpha1.ManagedClusterAddOn
var objs []runtime.Object
if tc.managedClusterAddonBuilder != nil {
if tc.addonTemplate != nil {
tc.managedClusterAddonBuilder.withAddonTemplate(tc.addonTemplate)
objs = append(objs, tc.addonTemplate)
}
if tc.addonDeploymentConfig != nil {
tc.managedClusterAddonBuilder.withAddonDeploymentConfig(tc.addonDeploymentConfig)
objs = append(objs, tc.addonDeploymentConfig)
}
managedClusterAddon = tc.managedClusterAddonBuilder.build()
objs = append(objs, managedClusterAddon)
}
if tc.addonDeploymentConfig != nil {
tc.managedClusterAddonBuilder.withAddonDeploymentConfig(tc.addonDeploymentConfig)
objs = append(objs, tc.addonDeploymentConfig)
hubKubeClient := fakekube.NewSimpleClientset()
addonClient := fakeaddon.NewSimpleClientset(objs...)

addonInformerFactory := addoninformers.NewSharedInformerFactory(addonClient, 30*time.Minute)
if managedClusterAddon != nil {
mcaStore := addonInformerFactory.Addon().V1alpha1().ManagedClusterAddOns().Informer().GetStore()
if err := mcaStore.Add(managedClusterAddon); err != nil {
t.Fatal(err)
}
}
managedClusterAddon = tc.managedClusterAddonBuilder.build()
objs = append(objs, managedClusterAddon)
}
hubKubeClient := fakekube.NewSimpleClientset()
addonClient := fakeaddon.NewSimpleClientset(objs...)

addonInformerFactory := addoninformers.NewSharedInformerFactory(addonClient, 30*time.Minute)
if managedClusterAddon != nil {
mcaStore := addonInformerFactory.Addon().V1alpha1().ManagedClusterAddOns().Informer().GetStore()
if err := mcaStore.Add(managedClusterAddon); err != nil {
t.Fatal(err)
if tc.addonTemplate != nil {
atStore := addonInformerFactory.Addon().V1alpha1().AddOnTemplates().Informer().GetStore()
if err := atStore.Add(tc.addonTemplate); err != nil {
t.Fatal(err)
}
}
}
if tc.addonTemplate != nil {
atStore := addonInformerFactory.Addon().V1alpha1().AddOnTemplates().Informer().GetStore()
if err := atStore.Add(tc.addonTemplate); err != nil {
kubeInformers := kubeinformers.NewSharedInformerFactoryWithOptions(hubKubeClient, 10*time.Minute)

agentAddon := NewCRDTemplateAgentAddon(
ctx,
addonName,
hubKubeClient,
addonClient,
addonInformerFactory,
kubeInformers.Rbac().V1().RoleBindings().Lister(),
addonfactory.GetAddOnDeploymentConfigValues(
utils.NewAddOnDeploymentConfigGetter(addonClient),
addonfactory.ToAddOnCustomizedVariableValues,
ToAddOnNodePlacementPrivateValues,
ToAddOnRegistriesPrivateValues,
),
)

ns, err := agentAddon.GetAgentAddonOptions().Registration.AgentInstallNamespace(managedClusterAddon)
if err != nil {
t.Fatal(err)
}
}
kubeInformers := kubeinformers.NewSharedInformerFactoryWithOptions(hubKubeClient, 10*time.Minute)

agentAddon := NewCRDTemplateAgentAddon(
ctx,
addonName,
hubKubeClient,
addonClient,
addonInformerFactory,
kubeInformers.Rbac().V1().RoleBindings().Lister(),
addonfactory.GetAddOnDeploymentConfigValues(
addonfactory.NewAddOnDeploymentConfigGetter(addonClient),
addonfactory.ToAddOnCustomizedVariableValues,
ToAddOnNodePlacementPrivateValues,
ToAddOnRegistriesPrivateValues,
),
)

ns := agentAddon.GetAgentAddonOptions().Registration.AgentInstallNamespace(managedClusterAddon)
assert.Equal(t, tc.expected, ns, tc.name)
assert.Equal(t, tc.expected, ns, tc.name)
})
}
}

Expand Down Expand Up @@ -536,7 +621,7 @@ func TestAgentManifestConfigs(t *testing.T) {
addonInformerFactory,
kubeInformers.Rbac().V1().RoleBindings().Lister(),
addonfactory.GetAddOnDeploymentConfigValues(
addonfactory.NewAddOnDeploymentConfigGetter(addonClient),
utils.NewAddOnDeploymentConfigGetter(addonClient),
addonfactory.ToAddOnCustomizedVariableValues,
ToAddOnNodePlacementPrivateValues,
ToAddOnRegistriesPrivateValues,
Expand Down
Loading

0 comments on commit 4bbd743

Please sign in to comment.