control-service: Expose supported python versions in helm

As part of the initiative to support multiple python versions in data job deployments, new configurations were introduced in the form of `datajobs.deployment.supportedPythonVersions` and `datajobs.deployment.defaultPythonVersion` configuration properties. To facilitate proper production setup of the Control Service with the new functionality, the configuration needs to be exposed in the helm chart. This change updates the values.yaml and deployment.yaml files of the Control Service helm chart to expose the supportedPythonVersions and defaultPythonVersion configurations and set them as environment variables. Testing Done: None Signed-off-by: Andon Andonov <[email protected]>
vmware · Apr 25, 2023 · f560e22 · f560e22
1 parent 5b7a8e4
commit f560e22
Show file tree

Hide file tree

Showing 3 changed files with 44 additions and 75 deletions.
diff --git a/...cts/control-service/projects/helm_charts/pipelines-control-service/templates/_helpers.tpl b/...cts/control-service/projects/helm_charts/pipelines-control-service/templates/_helpers.tpl
@@ -133,55 +133,6 @@ Also, we can't use a single if because lazy evaluation is not an option
 {{- end -}}
 {{- end -}}
 
-{{/*
-Return the proper Control Service data jobs base image where data job run in.
-*/}}
-{{- define "pipelines-control-service.deploymentDataJobBaseImage" -}}
-{{- $globalRegistryOverrideName := .Values.deploymentDataJobBaseImage.globalRegistryOverride -}}
-{{- $registryName := .Values.deploymentDataJobBaseImage.registry -}}
-{{- $repositoryName := .Values.deploymentDataJobBaseImage.repository -}}
-{{- $tag := .Values.deploymentDataJobBaseImage.tag | toString -}}
-{{/*
-Helm 2.11 supports the assignment of a value to a variable defined in a different scope,
-but Helm 2.9 and 2.10 doesn't support it, so we need to implement this if-else logic.
-Also, we can't use a single if because lazy evaluation is not an option
-*/}}
-{{- if $globalRegistryOverrideName }}
-    {{- printf "%s/%s:%s" $globalRegistryOverrideName $repositoryName $tag -}}
-{{- else if .Values.global.imageRegistry }}
-    {{- printf "%s/%s:%s" .Values.global.imageRegistry $repositoryName $tag -}}
-{{- else -}}
-    {{- printf "%s/%s:%s" $registryName $repositoryName $tag -}}
-{{- end -}}
-{{- end -}}
-
-{{/*
-Return the proper Control Service vdk image where the vdk (sdk) is installed and used during data job runs.
-*/}}
-{{- define "pipelines-control-service.deploymentVdkDistributionImageRepository" -}}
-    {{- $globalRegistryOverrideName := .Values.deploymentVdkDistributionImage.globalRegistryOverride -}}
-    {{- $registryName := .Values.deploymentVdkDistributionImage.registry -}}
-    {{- $repositoryName := .Values.deploymentVdkDistributionImage.repository -}}
-
-    {{/*
-    Helm 2.11 supports the assignment of a value to a variable defined in a different scope,
-    but Helm 2.9 and 2.10 doesn't support it, so we need to implement this if-else logic.
-    Also, we can't use a single if because lazy evaluation is not an option
-    */}}
-    {{- if $globalRegistryOverrideName }}
-        {{- printf "%s/%s" $globalRegistryOverrideName $repositoryName -}}
-    {{- else if .Values.global.imageRegistry }}
-        {{- printf "%s/%s" .Values.global.imageRegistry $repositoryName -}}
-    {{- else -}}
-        {{- printf "%s/%s" $registryName $repositoryName -}}
-    {{- end -}}
-{{- end -}}
-
-{{- define "pipelines-control-service.deploymentVdkDistributionImage" -}}
-    {{- $tag := .Values.deploymentVdkDistributionImage.tag | toString -}}
-    {{- printf "%s:%s" (include "pipelines-control-service.deploymentVdkDistributionImageRepository" .) $tag -}}
-{{- end -}}
-
 {{/*
 Create the name of the deployment Kubernetes namespace used by Data Jobs
 */}}
@@ -234,7 +185,7 @@ VDK distribution docker repository secret name
 {{- end -}}
 
 {{- define "shouldCreateVdkSdkDockerRepoSecret" }}
-  {{- if and (.Values.deploymentVdkDistributionImage.registryUsernameReadOnly) (.Values.deploymentVdkDistributionImage.registryPasswordReadOnly) }}
+  {{- if and (.Values.deploymentVdkDistributionImageRegistry.registryUsernameReadOnly) (.Values.deploymentVdkDistributionImageRegistry.registryPasswordReadOnly) }}
     true
   {{- end }}
 {{- end }}
@@ -260,7 +211,7 @@ Image Pull Secret in json format
 {{- end }}
 
 {{- define "vdkSdkImagePullSecretJson" }}
-    {{ include "buildImagePullSecretJson" (list .Values.deploymentVdkDistributionImage.registry .Values.deploymentVdkDistributionImage.registryUsernameReadOnly .Values.deploymentVdkDistributionImage.registryPasswordReadOnly) }}
+    {{ include "buildImagePullSecretJson" (list .Values.deploymentVdkDistributionImageRegistry.registry .Values.deploymentVdkDistributionImageRegistry.registryUsernameReadOnly .Values.deploymentVdkDistributionImageRegistry.registryPasswordReadOnly) }}
 {{- end }}
 
 {{- define "pipelinesControlServicePullSecretJson" }}
@@ -276,5 +227,5 @@ Image Pull Secret in json format
 {{- end }}
 
 {{- define "basePullSecretJson" }}
-    {{ include "buildImagePullSecretJson" (list .Values.deploymentDataJobBaseImage.registry .Values.deploymentDataJobBaseImage.username .Values.deploymentDataJobBaseImage.password) }}
+    {{ include "buildImagePullSecretJson" (list .Values.deploymentDataJobBaseImageRegistry.registry .Values.deploymentDataJobBaseImageRegistry.username .Values.deploymentDataJobBaseImageRegistry.password) }}
 {{- end }}
diff --git a/.../control-service/projects/helm_charts/pipelines-control-service/templates/deployment.yaml b/.../control-service/projects/helm_charts/pipelines-control-service/templates/deployment.yaml
@@ -111,12 +111,8 @@ spec:
             {{- end }}
             - name: DOCKER_REPOSITORY_URL
               value: "{{ .Values.deploymentDockerRepository }}"
-            - name: DOCKER_VDK_BASE_IMAGE
-              value: {{ template "pipelines-control-service.deploymentVdkDistributionImage" . }}
             - name: DATAJOBS_BUILDER_IMAGE
               value: {{ template "pipelines-control-service.deploymentBuilderImage" . }}
-            - name: DATAJOBS_DEPLOYMENT_DATAJOBBASEIMAGE
-              value: {{ template "pipelines-control-service.deploymentDataJobBaseImage" . }}
             - name: SERVER_MAX_HTTP_HEADER_SIZE
               value: "{{ .Values.server.maxHttpHeaderSize }}"
             - name: DB_JDBC_URL
@@ -275,6 +271,14 @@ spec:
               value: "{{ .Values.dataJob.deployment.initContainer.resources.limits.memory }}"
             - name: DATAJOBS_DEPLOYMENT_INITCONTAINER_RESOURCES_LIMITS_CPU
               value: "{{ .Values.dataJob.deployment.initContainer.resources.limits.cpu }}"
+            {{- if .Values.deploymentSupportedPythonVersions }}
+            - name: DATAJOBS_DEPLOYMENT_SUPPORTED_PYTHON_VERSIONS
+              value: "{{ toJson .Values.deploymentSupportedPythonVersions }}"
+            {{- end }}
+            {{- if .Values.deploymentDefaultPythonVersion }}
+            - name: DATAJOBS_DEPLOYMENT_DEFAULT_PYTHON_VERSION
+              value: "{{ .Values.deploymentDefaultPythonVersion }}"
+            {{- end }}
 
 
 {{- if .Values.extraVars }}

diff --git a/projects/control-service/projects/helm_charts/pipelines-control-service/values.yaml b/projects/control-service/projects/helm_charts/pipelines-control-service/values.yaml
@@ -247,14 +247,9 @@ deploymentDockerRegistryPassword: ""
 deploymentDockerRegistryUsernameReadOnly: ""
 deploymentDockerRegistryPasswordReadOnly: ""
 
-## Image name of VDK which will be used to run the data jobs.
-## It is recommended to use image with same tag (e.g release or latest)
-## As it will be pulled before any execution thus making sure all jobs use the same and latest version of VDK.
-## The image should contain installation of VDK (vdk) which will be used to run the data jobs (vdk run command)
-## Only the installed python modules (vdk and its dependencies) will be used from the image.
-## Everything else is effectively discarded since another image is used as base during execution.
-## Override if you build VDK SDK yourself instead of using one provided by default.
-deploymentVdkDistributionImage:
+
+## Registry and associated credentials to be set as part of vdk sdk repo docker secret
+deploymentVdkDistributionImageRegistry:
   registry: registry.hub.docker.com/versatiledatakit
 
   ## Image registry types. Your account need to have pull permission over the registry. The supported values are:
@@ -268,22 +263,41 @@ deploymentVdkDistributionImage:
   registryUsernameReadOnly: ""
   registryPasswordReadOnly: ""
 
-  repository: quickstart-vdk
-  tag: "release"
-
 
-# The base image which will be used to create the image where data job would be run.
-# On top of it the job source and its dependencies are installed for each job.
-# It can be customized further by installing other native dependencies necessary to run different kinds of libraries
-# For example the below default image adds support for Oracle client on top of standard python image.
-# The SDK (vdk) is installed separate image (to enable zero time upgrades) see deploymentVdkDistributionImage
-deploymentDataJobBaseImage:
+## Registry and credentials to be set as part of the image pull secret in kubernetes.
+deploymentDataJobBaseImageRegistry:
   registry: registry.hub.docker.com/versatiledatakit
-  repository: data-job-base-python-3.7
-  tag: "latest"
   username:
   password:
 
+
+## The map of python versions for data job deployments supported by the Control Service.
+## All values should be strings, and follow the format
+## {"pythonVersion": {"baseImage": "<repository>:<tag>", "vdkImage": "<registry>/<repository>:<tag>"}}
+##
+## The value of baseImage will be used to create the image, where data job would be run.
+## On top of it the job source and its dependencies are installed for each job.
+## It can be customized further by installing other native dependencies necessary to run different kinds of libraries
+##
+## The value of vdkImage is the image name of VDK, which will be used to run the data jobs.
+## It is recommended to use image with same tag (e.g., release or latest),
+## as it will be pulled before any execution, thus making sure all jobs use the same and latest version of VDK.
+## The image should contain installation of VDK (vdk), which will be used to run the data jobs (vdk run command)
+## Only the installed python modules (vdk and its dependencies) will be used from the image.
+## Everything else is effectively discarded since another image is used as base during execution.
+##
+## Example: {"3.9": {"baseImage": "python:3.9-latest", "vdkImage": "example-registry.com/some-user/vdk:3.9-release"}}
+deploymentSupportedPythonVersions:
+  "3.7":
+    baseImage: "registry.hub.docker.com/versatiledatakit/data-job-base-python-3.7:latest"
+    vdkImage: "registry.hub.docker.com/versatiledatakit/quickstart-vdk:release"
+
+
+## Default python version to be used for data job deployments. The value should be a string and would be
+## used when a user has not provided a specific python_version for their data job's deployment.
+## Example: "3.9"
+deploymentDefaultPythonVersion: "3.7"
+
 ## ini formatted options that provides default and per-job VDK options.
 ## Allows to provide VDK configuration via environment variables.
 ## They will be set as environment variables during execution of Data Job