Merge pull request #18 from cloudogu/feature/configurable_build_images

Feature/configurable build images
cloudogu · May 18, 2021 · 811d51e · 811d51e
2 parents f525222 + 98ab14b
commit 811d51e
Show file tree

Hide file tree

Showing 25 changed files with 292 additions and 129 deletions.
diff --git a/README.md b/README.md
@@ -319,6 +319,24 @@ First of all there are some mandatory properties e.g. the information about your
 
 ---
 
+## Build Images
+The GitOps-build-lib uses some docker images internally (To run Helm or Kubectl commands and specific Validators inside a docker container).  
+All of these have set default images, but you can change them if you wish to.
+
+```groovy
+def gitopsConfig = [
+        buildImages: [
+            // These are used to run helm and kubectl commands in the core logic
+            helm: 'ghcr.io/cloudogu/helm:3.5.4-1',
+            kubectl: 'lachlanevenson/k8s-kubectl:v1.19.3',
+            // These are used for each specific validator via an imageRef property inside the validators config. See [Validators] for examples.
+            kubeval: 'ghcr.io/cloudogu/helm:3.5.4-1',
+            helmKubeval: 'ghcr.io/cloudogu/helm:3.5.4-1',
+            yamllint: 'cytopia/yamllint:1.25-0.7'
+        ]
+]
+```
+
 ## Stages
 The GitOps-build-lib supports builds on multiple stages. A stage is defined by a name and contains a namespace (used to
 generate the resources) and a deployment-flag. If no stages is passed into the gitops-config by the user, the default
@@ -691,6 +709,40 @@ node {
 }
 ```
 
+Each Validator has a property called `imageRef` and `image` inside the `config` property.  
+`imageRef`'s value defaults to the key in the `buildImages` property and will replace the key-name with the actual image corresponding to the value.
+
+```groovy
+def gitopsConfig = [
+        validators: [
+            kubeval: [
+                enabled: true,
+                config: [
+                    imageRef: 'kubeval' // this corresponds to the key/value pair in buildImages which internally will become imageRef: 'ghcr.io/cloudogu/helm:3.5.4-1'
+                ]
+            ]
+        ]
+    ]
+```
+
+If you wish to change the image, you can do so by changing the image in the corresponding key/value pair in `buildImages` or by setting the image directly via the `image` property.
+First, the library will check if the `image` property is set and if so, will use its value as the image.
+
+```groovy
+def gitopsConfig = [
+        validators: [
+            kubeval: [
+                enabled: true,
+                config: [
+                    image: 'ghcr.io/cloudogu/helm:3.5.4-1'
+                ]
+            ]
+        ]
+    ]
+```
+
+If the `image` value is not set, it resorts to the `imageRef` property.
+
 ### Custom validators
 
 The library offers a convenient base class [`com.cloudogu.gitops.gitopsbuildlib.Validator`](src/com/cloudogu/gitopsbuildlib/Validator.groovy).

diff --git a/src/com/cloudogu/gitopsbuildlib/deployment/Deployment.groovy b/src/com/cloudogu/gitopsbuildlib/deployment/Deployment.groovy
@@ -69,10 +69,6 @@ abstract class Deployment {
         dockerWrapper.withDockerImage(image, body)
     }
 
-    void withHelm(Closure body) {
-        dockerWrapper.withHelm(body)
-    }
-
     // Dummy kubeConfig, so we can use `kubectl --dry-run=client`
     String writeKubeConfig() {
         String kubeConfigPath = "${script.pwd()}/.kube/config"

diff --git a/src/com/cloudogu/gitopsbuildlib/deployment/helm/Helm.groovy b/src/com/cloudogu/gitopsbuildlib/deployment/helm/Helm.groovy
@@ -1,14 +1,13 @@
 package com.cloudogu.gitopsbuildlib.deployment.helm
 
 import com.cloudogu.gitopsbuildlib.deployment.Deployment
-import com.cloudogu.gitopsbuildlib.deployment.GitopsTool
+import com.cloudogu.gitopsbuildlib.deployment.SourceType
 import com.cloudogu.gitopsbuildlib.deployment.helm.helmrelease.ArgoCDRelease
 import com.cloudogu.gitopsbuildlib.deployment.helm.helmrelease.FluxV1Release
 import com.cloudogu.gitopsbuildlib.deployment.helm.helmrelease.HelmRelease
 import com.cloudogu.gitopsbuildlib.deployment.helm.repotype.GitRepo
 import com.cloudogu.gitopsbuildlib.deployment.helm.repotype.HelmRepo
 import com.cloudogu.gitopsbuildlib.deployment.helm.repotype.RepoType
-import com.cloudogu.gitopsbuildlib.deployment.SourceType
 
 class Helm extends Deployment {
 
@@ -35,20 +34,18 @@ class Helm extends Deployment {
 
     @Override
     def preValidation(String stage) {
-        def helmConfig = gitopsConfig.deployments.helm
-        def application = gitopsConfig.application
         def sourcePath = gitopsConfig.deployments.sourcePath
 
-        chartRepo.prepareRepo(helmConfig, helmChartTempDir, chartRootDir)
+        chartRepo.prepareRepo(gitopsConfig, helmChartTempDir, chartRootDir)
 
         // writing the merged-values.yaml via writeYaml into a file has the advantage, that it gets formatted as valid yaml
         // This makes it easier to read in and indent for the inline use in the helmRelease.
         // It enables us to reuse the `fileToInlineYaml` function, without writing a complex formatting logic.
-        script.writeFile file: "${script.env.WORKSPACE}/${helmChartTempDir}/mergedValues.yaml", text: mergeValuesFiles(helmConfig, ["${script.env.WORKSPACE}/${sourcePath}/values-${stage}.yaml", "${script.env.WORKSPACE}/${sourcePath}/values-shared.yaml"] as String[])
+        script.writeFile file: "${script.env.WORKSPACE}/${helmChartTempDir}/mergedValues.yaml", text: mergeValuesFiles(gitopsConfig, ["${script.env.WORKSPACE}/${sourcePath}/values-${stage}.yaml", "${script.env.WORKSPACE}/${sourcePath}/values-shared.yaml"] as String[])
 
-        updateYamlValue("${script.env.WORKSPACE}/${helmChartTempDir}/mergedValues.yaml", helmConfig)
+        updateYamlValue("${script.env.WORKSPACE}/${helmChartTempDir}/mergedValues.yaml", gitopsConfig)
 
-        script.writeFile file: "${stage}/${application}/applicationRelease.yaml", text: helmRelease.create(helmConfig, application, getNamespace(stage), "${script.env.WORKSPACE}/${helmChartTempDir}/mergedValues.yaml")
+        script.writeFile file: "${stage}/${gitopsConfig.application}/applicationRelease.yaml", text: helmRelease.create(gitopsConfig, getNamespace(stage), "${script.env.WORKSPACE}/${helmChartTempDir}/mergedValues.yaml")
     }
 
     @Override
@@ -65,7 +62,9 @@ class Helm extends Deployment {
         }
     }
 
-    private void updateYamlValue(String yamlFilePath, Map helmConfig) {
+    private void updateYamlValue(String yamlFilePath, Map gitopsConfig) {
+        def helmConfig = gitopsConfig.deployments.helm
+
         def data = script.readYaml file: yamlFilePath
         helmConfig.updateValues.each {
             String[] paths = it["fieldPath"].split("\\.")
@@ -81,7 +80,9 @@ class Helm extends Deployment {
         script.writeYaml file: yamlFilePath, data: data, overwrite: true
     }
 
-    private String mergeValuesFiles(Map helmConfig, String[] valuesFiles) {
+    private String mergeValuesFiles(Map gitopsConfig, String[] valuesFiles) {
+        def helmConfig = gitopsConfig.deployments.helm
+
         String mergedValuesFile = ""
 
         def chartDir = ''
@@ -91,7 +92,7 @@ class Helm extends Deployment {
             chartDir = helmConfig.chartName
         }
 
-        withHelm {
+        withDockerImage(gitopsConfig.buildImages.helm) {
             String helmScript = "helm values ${script.env.WORKSPACE}/${helmChartTempDir}/${chartRootDir}/${chartDir} ${valuesFilesWithParameter(valuesFiles)}"
             mergedValuesFile = script.sh returnStdout: true, script: helmScript
         }

diff --git a/src/com/cloudogu/gitopsbuildlib/deployment/helm/helmrelease/ArgoCDRelease.groovy b/src/com/cloudogu/gitopsbuildlib/deployment/helm/helmrelease/ArgoCDRelease.groovy
@@ -12,32 +12,37 @@ class ArgoCDRelease extends HelmRelease {
     }
 
     @Override
-    String create(Map helmConfig, String application, String namespace, String mergedValuesFile) {
+    String create(Map gitopsConfig, String namespace, String mergedValuesFile) {
+        Map helmConfig = gitopsConfig.deployments.helm
+        String application = gitopsConfig.application
+
         String helmRelease = ""
         if (helmConfig.repoType == 'GIT') {
-            helmRelease = createResourcesFromGitRepo(helmConfig, application, mergedValuesFile)
+            helmRelease = createResourcesFromGitRepo(gitopsConfig, application, mergedValuesFile)
         } else if (helmConfig.repoType == 'HELM') {
-            helmRelease = createResourcesFromHelmRepo(helmConfig, application, mergedValuesFile)
+            helmRelease = createResourcesFromHelmRepo(gitopsConfig, application, mergedValuesFile)
         }
         return helmRelease
     }
 
-    private String createResourcesFromGitRepo(Map helmConfig, String application, String mergedValuesFile) {
+    private String createResourcesFromGitRepo(Map gitopsConfig, String application, String mergedValuesFile) {
+        Map helmConfig = gitopsConfig.deployments.helm
+
         def chartPath = ''
         if (helmConfig.containsKey('chartPath')) {
             chartPath = helmConfig.chartPath
         }
 
-        return createHelmRelease(chartPath as String, application, mergedValuesFile)
+        return createHelmRelease(chartPath as String, application, gitopsConfig.buildImages.helm, mergedValuesFile)
     }
 
-    private String createResourcesFromHelmRepo(Map helmConfig, String application, String mergedValuesFile) {
-        return createHelmRelease(helmConfig.chartName as String, application, mergedValuesFile)
+    private String createResourcesFromHelmRepo(Map gitopsConfig, String application, String mergedValuesFile) {
+        return createHelmRelease(gitopsConfig.deployments.helm.chartName, application, gitopsConfig.buildImages.helm, mergedValuesFile)
     }
 
-    private String createHelmRelease(String chartPath, String application, String mergedValuesFile) {
+    private String createHelmRelease(def chartPath, String application, def helmImage, String mergedValuesFile) {
         String helmRelease = ""
-        dockerWrapper.withHelm {
+        dockerWrapper.withDockerImage(helmImage) {
             String templateScript = "helm template ${application} ${script.env.WORKSPACE}/.helmChartTempDir/chart/${chartPath} -f ${mergedValuesFile}"
             helmRelease = script.sh returnStdout: true, script: templateScript
         }

diff --git a/src/com/cloudogu/gitopsbuildlib/deployment/helm/helmrelease/FluxV1Release.groovy b/src/com/cloudogu/gitopsbuildlib/deployment/helm/helmrelease/FluxV1Release.groovy
@@ -7,7 +7,10 @@ class FluxV1Release extends HelmRelease {
     }
 
     @Override
-    String create(Map helmConfig, String application, String namespace, String mergedValuesFile) {
+    String create(Map gitopsConfig, String namespace, String mergedValuesFile) {
+        Map helmConfig = gitopsConfig.deployments.helm
+        String application = gitopsConfig.application
+
         def values = fileToInlineYaml(mergedValuesFile)
         def chart = getChart(helmConfig)
         return """apiVersion: helm.fluxcd.io/v1

diff --git a/src/com/cloudogu/gitopsbuildlib/deployment/helm/helmrelease/HelmRelease.groovy b/src/com/cloudogu/gitopsbuildlib/deployment/helm/helmrelease/HelmRelease.groovy
@@ -8,7 +8,7 @@ abstract class HelmRelease {
         this.script = script
     }
 
-    abstract String create(Map helmConfig, String application, String namespace, String mergedValuesFile)
+    abstract String create(Map gitopsConfig, String namespace, String mergedValuesFile)
 
     String fileToInlineYaml(String fileContents) {
         String values = ""

diff --git a/src/com/cloudogu/gitopsbuildlib/deployment/helm/repotype/GitRepo.groovy b/src/com/cloudogu/gitopsbuildlib/deployment/helm/repotype/GitRepo.groovy
@@ -7,7 +7,8 @@ class GitRepo extends RepoType {
     }
 
     @Override
-    void prepareRepo(Map helmConfig, String helmChartTempDir, String chartRootDir) {
+    void prepareRepo(Map gitopsConfig, String helmChartTempDir, String chartRootDir) {
+        def helmConfig = gitopsConfig.deployments.helm
 
         getHelmChartFromGitRepo(helmConfig, helmChartTempDir, chartRootDir)
 
@@ -16,7 +17,7 @@ class GitRepo extends RepoType {
             chartPath = helmConfig.chartPath
         }
 
-        withHelm {
+        withDockerImage(gitopsConfig.buildImages.helm) {
             script.sh "helm dep update ${script.env.WORKSPACE}/.helmChartTempDir/${chartRootDir}/${chartPath}"
         }
     }

diff --git a/src/com/cloudogu/gitopsbuildlib/deployment/helm/repotype/HelmRepo.groovy b/src/com/cloudogu/gitopsbuildlib/deployment/helm/repotype/HelmRepo.groovy
@@ -7,7 +7,8 @@ class HelmRepo extends RepoType {
     }
 
     @Override
-    void prepareRepo(Map helmConfig, String helmChartTempDir, String chartRootDir) {
+    void prepareRepo(Map gitopsConfig, String helmChartTempDir, String chartRootDir) {
+        def helmConfig = gitopsConfig.deployments.helm
 
         if (helmConfig.containsKey('credentialsId') && helmConfig.credentialsId) {
             script.withCredentials([
@@ -17,15 +18,16 @@ class HelmRepo extends RepoType {
                     passwordVariable: 'PASSWORD')
             ]) {
                 String credentialArgs = " --username ${script.USERNAME} --password ${script.PASSWORD}"
-                addAndPullRepo(helmConfig, helmChartTempDir, chartRootDir, credentialArgs)
+                addAndPullRepo(gitopsConfig, helmChartTempDir, chartRootDir, credentialArgs)
             }
         } else {
-            addAndPullRepo(helmConfig, helmChartTempDir, chartRootDir)
+            addAndPullRepo(gitopsConfig, helmChartTempDir, chartRootDir)
         }
     }
 
-    private void addAndPullRepo(Map helmConfig, String helmChartTempDir, String chartRootDir, String credentialArgs = "") {
-        withHelm {
+    private void addAndPullRepo(Map gitopsConfig, String helmChartTempDir, String chartRootDir, String credentialArgs = "") {
+        def helmConfig = gitopsConfig.deployments.helm
+        withDockerImage(gitopsConfig.buildImages.helm) {
             script.sh "helm repo add chartRepo ${helmConfig.repoUrl}${credentialArgs}"
             script.sh "helm repo update"
             // helm pull also executes helm dependency so we don't need to do it in this step

diff --git a/src/com/cloudogu/gitopsbuildlib/deployment/helm/repotype/RepoType.groovy b/src/com/cloudogu/gitopsbuildlib/deployment/helm/repotype/RepoType.groovy
@@ -12,9 +12,9 @@ abstract class RepoType {
         dockerWrapper = new DockerWrapper(script)
     }
 
-    abstract void prepareRepo(Map helmConfig, String helmChartTempDir, String chartRootDir)
+    abstract void prepareRepo(Map gitopsConfig, String helmChartTempDir, String chartRootDir)
 
-    void withHelm(Closure body) {
-        dockerWrapper.withHelm(body)
+    void withDockerImage(String image, Closure body) {
+        dockerWrapper.withDockerImage(image, body)
     }
 }
diff --git a/src/com/cloudogu/gitopsbuildlib/docker/DockerWrapper.groovy b/src/com/cloudogu/gitopsbuildlib/docker/DockerWrapper.groovy
@@ -1,9 +1,6 @@
 package com.cloudogu.gitopsbuildlib.docker
 
 class DockerWrapper {
-
-    protected static String getHelmImage() { 'ghcr.io/cloudogu/helm:3.5.4-1' }
-
     protected def script
 
     DockerWrapper(def script) {
@@ -20,12 +17,4 @@ class DockerWrapper {
             body()
         }
     }
-
-    void withHelm(Closure body) {
-        script.cesBuildLib.Docker.new(script).image(helmImage).inside(
-            "${script.pwd().equals(script.env.WORKSPACE) ? '' : "-v ${script.env.WORKSPACE}:${script.env.WORKSPACE}"}"
-        ) {
-            body()
-        }
-    }
 }
diff --git a/src/com/cloudogu/gitopsbuildlib/validation/HelmKubeval.groovy b/src/com/cloudogu/gitopsbuildlib/validation/HelmKubeval.groovy
@@ -18,10 +18,7 @@ class HelmKubeval extends Validator {
     void validate(String targetDirectory, Map validatorConfig, Map gitopsConfig) {
         Map deployments = gitopsConfig.deployments as Map
         String args = argsParser.parse(validatorConfig)
-
-        withDockerImage(validatorConfig.image) {
-            script.sh "helm kubeval ${targetDirectory}/chart/${getChartDir(deployments)} -f ${targetDirectory}/mergedValues.yaml -v ${validatorConfig.k8sSchemaVersion}${args}"
-        }
+        script.sh "helm kubeval ${targetDirectory}/chart/${getChartDir(deployments)} -f ${targetDirectory}/mergedValues.yaml -v ${validatorConfig.k8sSchemaVersion}${args}"
     }
 
     private String getChartDir(Map deployments) {

diff --git a/src/com/cloudogu/gitopsbuildlib/validation/Kubeval.groovy b/src/com/cloudogu/gitopsbuildlib/validation/Kubeval.groovy
@@ -17,13 +17,9 @@ class Kubeval extends Validator {
     }
 
     @Override
-    void validate(String targetDirectory, Map config, Map gitopsConfig) {
-
-        String args = argsParser.parse(config)
-
-        withDockerImage(config.image) {
-            script.sh "kubeval -d ${targetDirectory} -v ${config.k8sSchemaVersion}${args}"
-        }
+    void validate(String targetDirectory, Map validatorConfig, Map gitopsConfig) {
+        String args = argsParser.parse(validatorConfig)
+        script.sh "kubeval -d ${targetDirectory} -v ${validatorConfig.k8sSchemaVersion}${args}"
     }
 
     @Override

diff --git a/src/com/cloudogu/gitopsbuildlib/validation/Validator.groovy b/src/com/cloudogu/gitopsbuildlib/validation/Validator.groovy
@@ -3,6 +3,7 @@ package com.cloudogu.gitopsbuildlib.validation
 import com.cloudogu.gitopsbuildlib.deployment.GitopsTool
 import com.cloudogu.gitopsbuildlib.deployment.SourceType
 import com.cloudogu.gitopsbuildlib.docker.DockerWrapper
+import org.codehaus.groovy.runtime.NullObject
 
 abstract class Validator {
 
@@ -17,7 +18,9 @@ abstract class Validator {
     void validate(boolean enabled, GitopsTool gitopsTool, SourceType sourceType, String targetDirectory, Map validatorConfig, Map gitopsConfig) {
         if (enabled && getSupportedGitopsTools().contains(gitopsTool) && getSupportedSourceTypes().contains(sourceType)) {
             script.echo "Starting validator ${this.getClass().getSimpleName()} for ${gitopsTool.name()} in ${sourceType.name()} resources"
-            validate(targetDirectory, validatorConfig, gitopsConfig)
+            withDockerImage(getImage(gitopsConfig, validatorConfig)) {
+                validate(targetDirectory, validatorConfig, gitopsConfig)
+            }
         } else {
             script.echo "Skipping validator ${this.getClass().getSimpleName()} because it is configured as enabled=false or doesn't support the given gitopsTool=${gitopsTool.name()} or sourceType=${sourceType.name()}"
         }
@@ -30,4 +33,14 @@ abstract class Validator {
     protected void withDockerImage(String image, Closure body) {
         dockerWrapper.withDockerImage(image, body)
     }
+
+    protected String getImage(Map gitopsConfig, Map validatorConfig) {
+        if (validatorConfig.containsKey('image')) {
+            return validatorConfig.image
+        } else if (validatorConfig.containsKey('imageRef') && gitopsConfig.buildImages.containsKey(validatorConfig.imageRef)) {
+            return gitopsConfig.buildImages[validatorConfig.imageRef]
+        } else {
+            return null
+        }
+    }
 }
diff --git a/src/com/cloudogu/gitopsbuildlib/validation/Yamllint.groovy b/src/com/cloudogu/gitopsbuildlib/validation/Yamllint.groovy
@@ -18,12 +18,10 @@ class Yamllint extends Validator {
 
     @Override
     void validate(String targetDirectory, Map validatorConfig, Map gitopsConfig) {
-        withDockerImage(validatorConfig.image) {
-            script.sh "yamllint " +
-                "${validatorConfig.profile ? "-d ${validatorConfig.profile} " : ''}" +
-                '-f standard ' + // non-colored for CI-server  
-                "${targetDirectory}"
-        }
+        script.sh "yamllint " +
+            "${validatorConfig.profile ? "-d ${validatorConfig.profile} " : ''}" +
+            '-f standard ' + // non-colored for CI-server
+            "${targetDirectory}"
     }
 
     @Override