From 213f204a2e6182373621eaef61d9d75ea0422af7 Mon Sep 17 00:00:00 2001 From: Andon Andonov Date: Thu, 20 Apr 2023 18:02:15 +0300 Subject: [PATCH] control-service: Expose supported python versions in helm As part of the initiative to support multiple python versions in data job deployments, new configurations were introduced in the form of `datajobs.deployment.supportedPythonVersions` and `datajobs.deployment.defaultPythonVersion` configuration properties. To facilitate proper production setup of the Control Service with the new functionality, the configuration needs to be exposed in the helm chart. This change updates the values.yaml and deployment.yaml files of the Control Service helm chart to expose the supportedPythonVersions and defaultPythonVersion configurations and set them as environment variables. Testing Done: CI/CD and local run of `helm lint` on the chart. Signed-off-by: Andon Andonov --- .../templates/deployment.yaml | 8 ++++++ .../pipelines-control-service/values.yaml | 28 +++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/projects/control-service/projects/helm_charts/pipelines-control-service/templates/deployment.yaml b/projects/control-service/projects/helm_charts/pipelines-control-service/templates/deployment.yaml index 75c05c5af5..4c32b5f5fc 100644 --- a/projects/control-service/projects/helm_charts/pipelines-control-service/templates/deployment.yaml +++ b/projects/control-service/projects/helm_charts/pipelines-control-service/templates/deployment.yaml @@ -275,6 +275,14 @@ spec: value: "{{ .Values.dataJob.deployment.initContainer.resources.limits.memory }}" - name: DATAJOBS_DEPLOYMENT_INITCONTAINER_RESOURCES_LIMITS_CPU value: "{{ .Values.dataJob.deployment.initContainer.resources.limits.cpu }}" + {{- if .Values.deploymentDefaultPythonVersion }} + - name: DATAJOBS_DEPLOYMENT_DEFAULT_PYTHON_VERSION + value: "{{ .Values.deploymentDefaultPythonVersion }}" + {{- end }} + {{- if .Values.deploymentSupportedPythonVersions }} + - name: DATAJOBS_DEPLOYMENT_SUPPORTED_PYTHON_VERSIONS + value: {{ toJson .Values.deploymentSupportedPythonVersions }} + {{- end }} {{- if .Values.extraVars }} diff --git a/projects/control-service/projects/helm_charts/pipelines-control-service/values.yaml b/projects/control-service/projects/helm_charts/pipelines-control-service/values.yaml index 7019425a63..5295825646 100644 --- a/projects/control-service/projects/helm_charts/pipelines-control-service/values.yaml +++ b/projects/control-service/projects/helm_charts/pipelines-control-service/values.yaml @@ -284,6 +284,34 @@ deploymentDataJobBaseImage: username: password: + +## The map of python versions for data job deployments supported by the Control Service. +## All values should be strings, and follow the format +## {"pythonVersion": {"baseImage": ":", "vdkImage": "/:"}} +## +## The value of baseImage will be used to create the image, where data job would be run. +## On top of it the job source and its dependencies are installed for each job. +## It can be customized further by installing other native dependencies necessary to run different kinds of libraries +## +## The value of vdkImage is the image name of VDK, which will be used to run the data jobs. +## It is recommended to use image with same tag (e.g., release or latest), +## as it will be pulled before any execution, thus making sure all jobs use the same and latest version of VDK. +## The image should contain installation of VDK (vdk), which will be used to run the data jobs (vdk run command) +## Only the installed python modules (vdk and its dependencies) will be used from the image. +## Everything else is effectively discarded since another image is used as base during execution. +## +## Example: {"3.9": {"baseImage": "python:3.9-latest", "vdkImage": "example-registry.com/some-user/vdk:3.9-release"}} +deploymentSupportedPythonVersions: + 3.7: + baseImage: "registry.hub.docker.com/versatiledatakit/data-job-base-python-3.7:latest" + vdkImage: "registry.hub.docker.com/versatiledatakit/quickstart-vdk:release" + + +## Default python version to be used for data job deployments. The value should be a string and would be +## used when a user has not provided a specific python_version for their data job's deployment. +## Example: "3.9" +deploymentDefaultPythonVersion: "3.7" + ## ini formatted options that provides default and per-job VDK options. ## Allows to provide VDK configuration via environment variables. ## They will be set as environment variables during execution of Data Job