From 0ac1f9e631325eb0e8bbf2d95087328bb8417802 Mon Sep 17 00:00:00 2001 From: Stefan Prodan Date: Tue, 16 Feb 2021 02:06:13 +0200 Subject: [PATCH 1/2] Implement var substitution from ConfigMaps and Secrets Signed-off-by: Stefan Prodan --- api/v1beta1/kustomization_types.go | 23 +++++++ api/v1beta1/zz_generated.deepcopy.go | 20 ++++++ ...mize.toolkit.fluxcd.io_kustomizations.yaml | 28 +++++++++ controllers/kustomization_controller.go | 38 ++++++------ controllers/kustomization_controller_test.go | 38 ++++++++++++ controllers/kustomization_generator.go | 53 +++++++++++++--- docs/api/kustomize.md | 62 +++++++++++++++++++ docs/spec/v1beta1/kustomization.md | 30 +++++++-- 8 files changed, 260 insertions(+), 32 deletions(-) diff --git a/api/v1beta1/kustomization_types.go b/api/v1beta1/kustomization_types.go index ab443f7c..9cb40efb 100644 --- a/api/v1beta1/kustomization_types.go +++ b/api/v1beta1/kustomization_types.go @@ -166,6 +166,29 @@ type PostBuild struct { // e.g. ${var:=default}, ${var:position} and ${var/substring/replacement}. // +optional Substitute map[string]string `json:"substitute,omitempty"` + + // SubstituteFrom holds references to ConfigMaps and Secrets containing + // the variables and their values to be substituted in the YAML manifests. + // The ConfigMap and the Secret data keys represent the var names and they + // must match the vars declared in the manifests for the substitution to happen. + // +optional + SubstituteFrom []SubstituteReference `json:"substituteFrom,omitempty"` +} + +// SubstituteReference contains a reference to a resource containing +// the variables name and value. +type SubstituteReference struct { + // Kind of the values referent, valid values are ('Secret', 'ConfigMap'). + // +kubebuilder:validation:Enum=Secret;ConfigMap + // +required + Kind string `json:"kind"` + + // Name of the values referent. Should reside in the same namespace as the + // referring resource. + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=253 + // +required + Name string `json:"name"` } // KustomizationStatus defines the observed state of a kustomization. diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 068c548d..74448f61 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -248,6 +248,11 @@ func (in *PostBuild) DeepCopyInto(out *PostBuild) { (*out)[key] = val } } + if in.SubstituteFrom != nil { + in, out := &in.SubstituteFrom, &out.SubstituteFrom + *out = make([]SubstituteReference, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PostBuild. @@ -303,3 +308,18 @@ func (in *SnapshotEntry) DeepCopy() *SnapshotEntry { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SubstituteReference) DeepCopyInto(out *SubstituteReference) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SubstituteReference. +func (in *SubstituteReference) DeepCopy() *SubstituteReference { + if in == nil { + return nil + } + out := new(SubstituteReference) + in.DeepCopyInto(out) + return out +} diff --git a/config/crd/bases/kustomize.toolkit.fluxcd.io_kustomizations.yaml b/config/crd/bases/kustomize.toolkit.fluxcd.io_kustomizations.yaml index 1918ac12..2873efed 100644 --- a/config/crd/bases/kustomize.toolkit.fluxcd.io_kustomizations.yaml +++ b/config/crd/bases/kustomize.toolkit.fluxcd.io_kustomizations.yaml @@ -263,6 +263,34 @@ spec: support for bash string replacement functions e.g. ${var:=default}, ${var:position} and ${var/substring/replacement}. type: object + substituteFrom: + description: SubstituteFrom holds references to ConfigMaps and + Secrets containing the variables and their values to be substituted + in the YAML manifests. The ConfigMap and the Secret data keys + represent the var names and they must match the vars declared + in the manifests for the substitution to happen. + items: + description: SubstituteReference contains a reference to a resource + containing the variables name and value. + properties: + kind: + description: Kind of the values referent, valid values are + ('Secret', 'ConfigMap'). + enum: + - Secret + - ConfigMap + type: string + name: + description: Name of the values referent. Should reside + in the same namespace as the referring resource. + maxLength: 253 + minLength: 1 + type: string + required: + - kind + - name + type: object + type: array type: object prune: description: Prune enables garbage collection. diff --git a/controllers/kustomization_controller.go b/controllers/kustomization_controller.go index 5076c282..ea92fdf9 100644 --- a/controllers/kustomization_controller.go +++ b/controllers/kustomization_controller.go @@ -298,19 +298,20 @@ func (r *KustomizationReconciler) reconcile( ), err } - // generate kustomization.yaml and calculate the manifests checksum - checksum, err := r.generate(kustomization, dirPath) + // create any necessary kube-clients for impersonation + impersonation := NewKustomizeImpersonation(kustomization, r.Client, r.StatusPoller, dirPath) + kubeClient, statusPoller, err := impersonation.GetClient(ctx) if err != nil { return kustomizev1.KustomizationNotReady( kustomization, source.GetArtifact().Revision, - kustomizev1.BuildFailedReason, + meta.ReconciliationFailedReason, err.Error(), - ), err + ), fmt.Errorf("failed to build kube client: %w", err) } - // build the kustomization and generate the GC snapshot - snapshot, err := r.build(kustomization, checksum, dirPath) + // generate kustomization.yaml and calculate the manifests checksum + checksum, err := r.generate(ctx, kubeClient, kustomization, dirPath) if err != nil { return kustomizev1.KustomizationNotReady( kustomization, @@ -320,16 +321,15 @@ func (r *KustomizationReconciler) reconcile( ), err } - // create any necessary kube-clients for impersonation - impersonation := NewKustomizeImpersonation(kustomization, r.Client, r.StatusPoller, dirPath) - client, statusPoller, err := impersonation.GetClient(ctx) + // build the kustomization and generate the GC snapshot + snapshot, err := r.build(ctx, kustomization, checksum, dirPath) if err != nil { return kustomizev1.KustomizationNotReady( kustomization, source.GetArtifact().Revision, - meta.ReconciliationFailedReason, + kustomizev1.BuildFailedReason, err.Error(), - ), fmt.Errorf("failed to build kube client: %w", err) + ), err } // dry-run apply @@ -355,7 +355,7 @@ func (r *KustomizationReconciler) reconcile( } // prune - err = r.prune(ctx, client, kustomization, checksum) + err = r.prune(ctx, kubeClient, kustomization, checksum) if err != nil { return kustomizev1.KustomizationNotReady( kustomization, @@ -490,12 +490,12 @@ func (r *KustomizationReconciler) getSource(ctx context.Context, kustomization k return source, nil } -func (r *KustomizationReconciler) generate(kustomization kustomizev1.Kustomization, dirPath string) (string, error) { - gen := NewGenerator(kustomization) - return gen.WriteFile(dirPath) +func (r *KustomizationReconciler) generate(ctx context.Context, kubeClient client.Client, kustomization kustomizev1.Kustomization, dirPath string) (string, error) { + gen := NewGenerator(kustomization, kubeClient) + return gen.WriteFile(ctx, dirPath) } -func (r *KustomizationReconciler) build(kustomization kustomizev1.Kustomization, checksum, dirPath string) (*kustomizev1.Snapshot, error) { +func (r *KustomizationReconciler) build(ctx context.Context, kustomization kustomizev1.Kustomization, checksum, dirPath string) (*kustomizev1.Snapshot, error) { timeout := kustomization.GetTimeout() ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() @@ -540,7 +540,7 @@ func (r *KustomizationReconciler) build(kustomization kustomizev1.Kustomization, } // run post-build actions - resources, err = runPostBuildActions(kustomization, resources) + resources, err = runPostBuildActions(ctx, r.Client, kustomization, resources) if err != nil { return nil, fmt.Errorf("post-build actions failed: %w", err) } @@ -678,7 +678,7 @@ func (r *KustomizationReconciler) applyWithRetry(ctx context.Context, kustomizat return changeSet, nil } -func (r *KustomizationReconciler) prune(ctx context.Context, client client.Client, kustomization kustomizev1.Kustomization, newChecksum string) error { +func (r *KustomizationReconciler) prune(ctx context.Context, kubeClient client.Client, kustomization kustomizev1.Kustomization, newChecksum string) error { if !kustomization.Spec.Prune || kustomization.Status.Snapshot == nil { return nil } @@ -686,7 +686,7 @@ func (r *KustomizationReconciler) prune(ctx context.Context, client client.Clien return nil } - gc := NewGarbageCollector(client, *kustomization.Status.Snapshot, newChecksum, logr.FromContext(ctx)) + gc := NewGarbageCollector(kubeClient, *kustomization.Status.Snapshot, newChecksum, logr.FromContext(ctx)) if output, ok := gc.Prune(kustomization.GetTimeout(), kustomization.GetName(), diff --git a/controllers/kustomization_controller_test.go b/controllers/kustomization_controller_test.go index 561a698c..2349abb0 100644 --- a/controllers/kustomization_controller_test.go +++ b/controllers/kustomization_controller_test.go @@ -150,6 +150,32 @@ var _ = Describe("KustomizationReconciler", func() { Expect(k8sClient.Status().Update(context.Background(), repository)).Should(Succeed()) defer k8sClient.Delete(context.Background(), repository) + configName := types.NamespacedName{ + Name: fmt.Sprintf("%s", randStringRunes(5)), + Namespace: namespace.Name, + } + config := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: configName.Name, + Namespace: configName.Namespace, + }, + Data: map[string]string{"zone": "\naz-1a\n"}, + } + Expect(k8sClient.Create(context.Background(), config)).Should(Succeed()) + + secretName := types.NamespacedName{ + Name: fmt.Sprintf("%s", randStringRunes(5)), + Namespace: namespace.Name, + } + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretName.Name, + Namespace: secretName.Namespace, + }, + StringData: map[string]string{"zone": "\naz-1b\n"}, + } + Expect(k8sClient.Create(context.Background(), secret)).Should(Succeed()) + kName := types.NamespacedName{ Name: fmt.Sprintf("%s", randStringRunes(5)), Namespace: namespace.Name, @@ -173,6 +199,16 @@ var _ = Describe("KustomizationReconciler", func() { Validation: "client", PostBuild: &kustomizev1.PostBuild{ Substitute: map[string]string{"region": "eu-central-1"}, + SubstituteFrom: []kustomizev1.SubstituteReference{ + { + Kind: "ConfigMap", + Name: configName.Name, + }, + { + Kind: "Secret", + Name: secretName.Name, + }, + }, }, HealthChecks: []meta.NamespacedObjectKindReference{ { @@ -213,6 +249,7 @@ var _ = Describe("KustomizationReconciler", func() { Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: "test", Namespace: "test"}, sa)).Should(Succeed()) Expect(sa.Labels["environment"]).To(Equal("dev")) Expect(sa.Labels["region"]).To(Equal("eu-central-1")) + Expect(sa.Labels["zone"]).To(Equal("az-1b")) }, Entry("namespace-sa", refTestCase{ artifacts: []testserver.File{ @@ -236,6 +273,7 @@ metadata: labels: environment: ${env:=dev} region: "${region}" + zone: "${zone}" `, }, }, diff --git a/controllers/kustomization_generator.go b/controllers/kustomization_generator.go index a26fbdaf..82c98293 100644 --- a/controllers/kustomization_generator.go +++ b/controllers/kustomization_generator.go @@ -17,12 +17,16 @@ limitations under the License. package controllers import ( + "context" "crypto/sha1" "encoding/json" "fmt" "io/ioutil" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" "os" "path/filepath" + "sigs.k8s.io/controller-runtime/pkg/client" "strings" "github.com/drone/envsubst" @@ -45,18 +49,20 @@ const ( type KustomizeGenerator struct { kustomization kustomizev1.Kustomization + client.Client } -func NewGenerator(kustomization kustomizev1.Kustomization) *KustomizeGenerator { +func NewGenerator(kustomization kustomizev1.Kustomization, kubeClient client.Client) *KustomizeGenerator { return &KustomizeGenerator{ kustomization: kustomization, + Client: kubeClient, } } -func (kg *KustomizeGenerator) WriteFile(dirPath string) (string, error) { +func (kg *KustomizeGenerator) WriteFile(ctx context.Context, dirPath string) (string, error) { kfile := filepath.Join(dirPath, konfig.DefaultKustomizationFileName()) - checksum, err := kg.checksum(dirPath) + checksum, err := kg.checksum(ctx, dirPath) if err != nil { return "", err } @@ -236,7 +242,7 @@ func (kg *KustomizeGenerator) generateKustomization(dirPath string) error { return ioutil.WriteFile(kfile, kd, os.ModePerm) } -func (kg *KustomizeGenerator) checksum(dirPath string) (string, error) { +func (kg *KustomizeGenerator) checksum(ctx context.Context, dirPath string) (string, error) { if err := kg.generateKustomization(dirPath); err != nil { return "", fmt.Errorf("kustomize create failed: %w", err) } @@ -253,7 +259,7 @@ func (kg *KustomizeGenerator) checksum(dirPath string) (string, error) { } // run post-build actions - resources, err = runPostBuildActions(kg.kustomization, resources) + resources, err = runPostBuildActions(ctx, kg.Client, kg.kustomization, resources) if err != nil { return "", fmt.Errorf("post-build actions failed: %w", err) } @@ -342,14 +348,45 @@ func buildKustomization(fs filesys.FileSystem, dirPath string) (resmap.ResMap, e } // runPostBuildActions runs actions on the multi-doc YAML manifest generated by kustomize build -func runPostBuildActions(kustomization kustomizev1.Kustomization, manifests []byte) ([]byte, error) { +func runPostBuildActions(ctx context.Context, kubeClient client.Client, kustomization kustomizev1.Kustomization, manifests []byte) ([]byte, error) { if kustomization.Spec.PostBuild == nil { return manifests, nil } + vars := make(map[string]string) + + // load vars from ConfigMaps and Secrets data keys + for _, reference := range kustomization.Spec.PostBuild.SubstituteFrom { + namespacedName := types.NamespacedName{Namespace: kustomization.Namespace, Name: reference.Name} + switch reference.Kind { + case "ConfigMap": + resource := &corev1.ConfigMap{} + if err := kubeClient.Get(ctx, namespacedName, resource); err != nil { + return nil, fmt.Errorf("substitute from 'ConfigMap/%s' error: %w", reference.Name, err) + } + for k, v := range resource.Data { + vars[k] = strings.Replace(v, "\n", "", -1) + } + case "Secret": + resource := &corev1.Secret{} + if err := kubeClient.Get(ctx, namespacedName, resource); err != nil { + return nil, fmt.Errorf("substitute from 'Secret/%s' error: %w", reference.Name, err) + } + for k, v := range resource.Data { + vars[k] = strings.Replace(string(v), "\n", "", -1) + } + } + } + + // load in-line vars (overrides the ones from resources) + if kustomization.Spec.PostBuild.Substitute != nil { + for k, v := range kustomization.Spec.PostBuild.Substitute { + vars[k] = strings.Replace(v, "\n", "", -1) + } + } + // run bash variable substitutions - vars := kustomization.Spec.PostBuild.Substitute - if vars != nil && len(vars) > 0 { + if len(vars) > 0 { output, err := envsubst.Eval(string(manifests), func(s string) string { return vars[s] }) diff --git a/docs/api/kustomize.md b/docs/api/kustomize.md index 83eb1617..d257674c 100644 --- a/docs/api/kustomize.md +++ b/docs/api/kustomize.md @@ -902,6 +902,23 @@ Includes support for bash string replacement functions e.g. ${var:=default}, ${var:position} and ${var/substring/replacement}.

+ + +substituteFrom
+ + +[]SubstituteReference + + + + +(Optional) +

SubstituteFrom holds references to ConfigMaps and Secrets containing +the variables and their values to be substituted in the YAML manifests. +The ConfigMap and the Secret data keys represent the var names and they +must match the vars declared in the manifests for the substitution to happen.

+ + @@ -997,6 +1014,51 @@ map[string]string +

SubstituteReference +

+

+(Appears on: +PostBuild) +

+

SubstituteReference contains a reference to a resource containing +the variables name and value.

+
+
+ + + + + + + + + + + + + + + + + +
FieldDescription
+kind
+ +string + +		 +

Kind of the values referent, valid values are (‘Secret’, ‘ConfigMap’).

+
+name
+ +string + +		 +

Name of the values referent. Should reside in the same namespace as the +referring resource.

+
+
+

This page was automatically generated with gen-crd-api-reference-docs

diff --git a/docs/spec/v1beta1/kustomization.md b/docs/spec/v1beta1/kustomization.md index aae8a496..0c461a43 100644 --- a/docs/spec/v1beta1/kustomization.md +++ b/docs/spec/v1beta1/kustomization.md @@ -164,6 +164,13 @@ type PostBuild struct { // e.g. ${var:=default}, ${var:position} and ${var/substring/replacement}. // +optional Substitute map[string]string `json:"substitute,omitempty"` + + // SubstituteFrom holds references to ConfigMaps and Secrets containing + // the variables and their values to be substituted in the YAML manifests. + // The ConfigMap and the Secret data keys represent the var names and they + // must match the vars declared in the manifests for the substitution to happen. + // +optional + SubstituteFrom []SubstituteReference `json:"substituteFrom,omitempty"` } ``` @@ -688,6 +695,10 @@ spec: With `spec.postBuild.substitute` you can provide a map of key/value pairs holding the variables to be substituted in the final YAML manifest, after kustomize build. +With `spec.postBuild.substituteFrom` you can provide a list of ConfigMaps and Secrets +from which the variables are loaded. +The ConfigMap and Secret data keys are used as the var names. + This offers basic templating for your manifests including support for [bash string replacement functions](https://github.com/drone/envsubst) e.g.: @@ -709,9 +720,9 @@ metadata: ``` You can specify the variables and their values in the Kustomization definition under -the `substitute` post build section: +`substitute` and/or `substituteFrom` post build section: -````yaml +```yaml apiVersion: kustomize.toolkit.fluxcd.io/v1beta1 kind: Kustomization metadata: @@ -723,10 +734,19 @@ spec: substitute: cluster_env: "prod" cluster_region: "eu-central-1" -```` + substituteFrom: + - kind: ConfigMap + name: cluster-config + - kind: Secret + name: cluster-secret-config +``` + +The var values which are specified in-line with `substitute` +take precedence over the ones in `substituteFrom`. -Note that you should prefix the variables that get replaced by kustomize-controller -to avoid conflicts with any existing scripts embedded in ConfigMaps or container commands. +Note that if you want to avoid var substitutions in scripts embedded in ConfigMaps or container commands, +you must use the format `$var` instead of `${var}`. All the undefined variables in the format `${var}` +will be substituted with string empty, unless a default is provided e.g. `${var:=default}`. You can replicate the controller post-build substitutions locally using [kustomize](https://github.com/kubernetes-sigs/kustomize) From 401fec6c8d4e05f97d9a99ad6aef44f2ef6a95e8 Mon Sep 17 00:00:00 2001 From: Stefan Prodan Date: Tue, 16 Feb 2021 10:17:10 +0200 Subject: [PATCH 2/2] Allow disabling var substitution for certain resources Signed-off-by: Stefan Prodan --- api/v1beta1/kustomization_types.go | 1 + controllers/kustomization_controller.go | 27 +++++--- controllers/kustomization_gc.go | 3 +- controllers/kustomization_generator.go | 80 +++++----------------- controllers/kustomization_varsub.go | 90 +++++++++++++++++++++++++ docs/spec/v1beta1/kustomization.md | 11 ++- 6 files changed, 137 insertions(+), 75 deletions(-) create mode 100644 controllers/kustomization_varsub.go diff --git a/api/v1beta1/kustomization_types.go b/api/v1beta1/kustomization_types.go index 9cb40efb..b01b3ae6 100644 --- a/api/v1beta1/kustomization_types.go +++ b/api/v1beta1/kustomization_types.go @@ -33,6 +33,7 @@ const ( KustomizationKind = "Kustomization" KustomizationFinalizer = "finalizers.fluxcd.io" MaxConditionMessageLength = 20000 + DisabledValue = "disabled" ) // KustomizationSpec defines the desired state of a kustomization. diff --git a/controllers/kustomization_controller.go b/controllers/kustomization_controller.go index ea92fdf9..eb15a599 100644 --- a/controllers/kustomization_controller.go +++ b/controllers/kustomization_controller.go @@ -517,9 +517,9 @@ func (r *KustomizationReconciler) build(ctx context.Context, kustomization kusto return nil, fmt.Errorf("kustomize build failed: %w", err) } - // check if resources are encrypted and decrypt them before generating the final YAML - if kustomization.Spec.Decryption != nil { - for _, res := range m.Resources() { + for _, res := range m.Resources() { + // check if resources are encrypted and decrypt them before generating the final YAML + if kustomization.Spec.Decryption != nil { outRes, err := dec.Decrypt(res) if err != nil { return nil, fmt.Errorf("decryption failed for '%s': %w", res.GetName(), err) @@ -532,6 +532,21 @@ func (r *KustomizationReconciler) build(ctx context.Context, kustomization kusto } } } + + // run variable substitutions + if kustomization.Spec.PostBuild != nil { + outRes, err := substituteVariables(ctx, r.Client, kustomization, res) + if err != nil { + return nil, fmt.Errorf("var substitution failed for '%s': %w", res.GetName(), err) + } + + if outRes != nil { + _, err = m.Replace(res) + if err != nil { + return nil, err + } + } + } } resources, err := m.AsYaml() @@ -539,12 +554,6 @@ func (r *KustomizationReconciler) build(ctx context.Context, kustomization kusto return nil, fmt.Errorf("kustomize build failed: %w", err) } - // run post-build actions - resources, err = runPostBuildActions(ctx, r.Client, kustomization, resources) - if err != nil { - return nil, fmt.Errorf("post-build actions failed: %w", err) - } - manifestsFile := filepath.Join(dirPath, fmt.Sprintf("%s.yaml", kustomization.GetUID())) if err := fs.WriteFile(manifestsFile, resources); err != nil { return nil, err diff --git a/controllers/kustomization_gc.go b/controllers/kustomization_gc.go index cdb8f54c..b17d93e3 100644 --- a/controllers/kustomization_gc.go +++ b/controllers/kustomization_gc.go @@ -144,9 +144,8 @@ func (kgc *KustomizeGarbageCollector) isStale(obj unstructured.Unstructured) boo func (kgc *KustomizeGarbageCollector) shouldSkip(obj unstructured.Unstructured) bool { key := fmt.Sprintf("%s/prune", kustomizev1.GroupVersion.Group) - val := "disabled" - return obj.GetLabels()[key] == val || obj.GetAnnotations()[key] == val + return obj.GetLabels()[key] == kustomizev1.DisabledValue || obj.GetAnnotations()[key] == kustomizev1.DisabledValue } func (kgc *KustomizeGarbageCollector) matchingLabels(name, namespace string) client.MatchingLabels { diff --git a/controllers/kustomization_generator.go b/controllers/kustomization_generator.go index 82c98293..8ec0e076 100644 --- a/controllers/kustomization_generator.go +++ b/controllers/kustomization_generator.go @@ -22,14 +22,11 @@ import ( "encoding/json" "fmt" "io/ioutil" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/types" "os" "path/filepath" - "sigs.k8s.io/controller-runtime/pkg/client" "strings" - "github.com/drone/envsubst" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/kustomize/api/filesys" "sigs.k8s.io/kustomize/api/k8sdeps/kunstruct" "sigs.k8s.io/kustomize/api/konfig" @@ -253,15 +250,26 @@ func (kg *KustomizeGenerator) checksum(ctx context.Context, dirPath string) (str return "", fmt.Errorf("kustomize build failed: %w", err) } - resources, err := m.AsYaml() - if err != nil { - return "", fmt.Errorf("kustomize build failed: %w", err) + // run variable substitutions + if kg.kustomization.Spec.PostBuild != nil { + for _, res := range m.Resources() { + outRes, err := substituteVariables(ctx, kg.Client, kg.kustomization, res) + if err != nil { + return "", fmt.Errorf("var substitution failed for '%s': %w", res.GetName(), err) + } + + if outRes != nil { + _, err = m.Replace(res) + if err != nil { + return "", err + } + } + } } - // run post-build actions - resources, err = runPostBuildActions(ctx, kg.Client, kg.kustomization, resources) + resources, err := m.AsYaml() if err != nil { - return "", fmt.Errorf("post-build actions failed: %w", err) + return "", fmt.Errorf("kustomize build failed: %w", err) } return fmt.Sprintf("%x", sha1.Sum(resources)), nil @@ -346,55 +354,3 @@ func buildKustomization(fs filesys.FileSystem, dirPath string) (resmap.ResMap, e k := krusty.MakeKustomizer(fs, buildOptions) return k.Run(dirPath) } - -// runPostBuildActions runs actions on the multi-doc YAML manifest generated by kustomize build -func runPostBuildActions(ctx context.Context, kubeClient client.Client, kustomization kustomizev1.Kustomization, manifests []byte) ([]byte, error) { - if kustomization.Spec.PostBuild == nil { - return manifests, nil - } - - vars := make(map[string]string) - - // load vars from ConfigMaps and Secrets data keys - for _, reference := range kustomization.Spec.PostBuild.SubstituteFrom { - namespacedName := types.NamespacedName{Namespace: kustomization.Namespace, Name: reference.Name} - switch reference.Kind { - case "ConfigMap": - resource := &corev1.ConfigMap{} - if err := kubeClient.Get(ctx, namespacedName, resource); err != nil { - return nil, fmt.Errorf("substitute from 'ConfigMap/%s' error: %w", reference.Name, err) - } - for k, v := range resource.Data { - vars[k] = strings.Replace(v, "\n", "", -1) - } - case "Secret": - resource := &corev1.Secret{} - if err := kubeClient.Get(ctx, namespacedName, resource); err != nil { - return nil, fmt.Errorf("substitute from 'Secret/%s' error: %w", reference.Name, err) - } - for k, v := range resource.Data { - vars[k] = strings.Replace(string(v), "\n", "", -1) - } - } - } - - // load in-line vars (overrides the ones from resources) - if kustomization.Spec.PostBuild.Substitute != nil { - for k, v := range kustomization.Spec.PostBuild.Substitute { - vars[k] = strings.Replace(v, "\n", "", -1) - } - } - - // run bash variable substitutions - if len(vars) > 0 { - output, err := envsubst.Eval(string(manifests), func(s string) string { - return vars[s] - }) - if err != nil { - return nil, fmt.Errorf("variable substitution failed: %w", err) - } - manifests = []byte(output) - } - - return manifests, nil -} diff --git a/controllers/kustomization_varsub.go b/controllers/kustomization_varsub.go new file mode 100644 index 00000000..2052eb90 --- /dev/null +++ b/controllers/kustomization_varsub.go @@ -0,0 +1,90 @@ +package controllers + +import ( + "context" + "fmt" + "strings" + + "github.com/drone/envsubst" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/kustomize/api/resource" + "sigs.k8s.io/yaml" + + kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1beta1" +) + +// substituteVariables replaces the vars with their values in the specified resource. +// If a resource is labeled or annotated with +// 'kustomize.toolkit.fluxcd.io/substitute: disabled' the substitution is skipped. +func substituteVariables( + ctx context.Context, + kubeClient client.Client, + kustomization kustomizev1.Kustomization, + res *resource.Resource) (*resource.Resource, error) { + resData, err := res.AsYAML() + if err != nil { + return nil, err + } + + key := fmt.Sprintf("%s/substitute", kustomizev1.GroupVersion.Group) + + if res.GetLabels()[key] == kustomizev1.DisabledValue || res.GetAnnotations()[key] == kustomizev1.DisabledValue { + return nil, nil + } + + vars := make(map[string]string) + + // load vars from ConfigMaps and Secrets data keys + for _, reference := range kustomization.Spec.PostBuild.SubstituteFrom { + namespacedName := types.NamespacedName{Namespace: kustomization.Namespace, Name: reference.Name} + switch reference.Kind { + case "ConfigMap": + resource := &corev1.ConfigMap{} + if err := kubeClient.Get(ctx, namespacedName, resource); err != nil { + return nil, fmt.Errorf("substitute from 'ConfigMap/%s' error: %w", reference.Name, err) + } + for k, v := range resource.Data { + vars[k] = strings.Replace(v, "\n", "", -1) + } + case "Secret": + resource := &corev1.Secret{} + if err := kubeClient.Get(ctx, namespacedName, resource); err != nil { + return nil, fmt.Errorf("substitute from 'Secret/%s' error: %w", reference.Name, err) + } + for k, v := range resource.Data { + vars[k] = strings.Replace(string(v), "\n", "", -1) + } + } + } + + // load in-line vars (overrides the ones from resources) + if kustomization.Spec.PostBuild.Substitute != nil { + for k, v := range kustomization.Spec.PostBuild.Substitute { + vars[k] = strings.Replace(v, "\n", "", -1) + } + } + + // run bash variable substitutions + if len(vars) > 0 { + output, err := envsubst.Eval(string(resData), func(s string) string { + return vars[s] + }) + if err != nil { + return nil, fmt.Errorf("variable substitution failed: %w", err) + } + + jsonData, err := yaml.YAMLToJSON([]byte(output)) + if err != nil { + return nil, fmt.Errorf("YAMLToJSON: %w", err) + } + + err = res.UnmarshalJSON(jsonData) + if err != nil { + return nil, fmt.Errorf("UnmarshalJSON: %w", err) + } + } + + return res, nil +} diff --git a/docs/spec/v1beta1/kustomization.md b/docs/spec/v1beta1/kustomization.md index 0c461a43..96943e27 100644 --- a/docs/spec/v1beta1/kustomization.md +++ b/docs/spec/v1beta1/kustomization.md @@ -736,9 +736,9 @@ spec: cluster_region: "eu-central-1" substituteFrom: - kind: ConfigMap - name: cluster-config + name: cluster-vars - kind: Secret - name: cluster-secret-config + name: cluster-secret-vars ``` The var values which are specified in-line with `substitute` @@ -748,6 +748,13 @@ Note that if you want to avoid var substitutions in scripts embedded in ConfigMa you must use the format `$var` instead of `${var}`. All the undefined variables in the format `${var}` will be substituted with string empty, unless a default is provided e.g. `${var:=default}`. +You can disable the variable substitution for certain resources by either +labeling or annotating them with: + +```yaml +kustomize.toolkit.fluxcd.io/substitute: disabled +``` + You can replicate the controller post-build substitutions locally using [kustomize](https://github.com/kubernetes-sigs/kustomize) and Drone's [envsubst](https://github.com/drone/envsubst):