From ab196a96c6c9c793d9d9d54bc7680f59696cf1ea Mon Sep 17 00:00:00 2001 From: Tristan <74349933+trispera@users.noreply.github.com> Date: Fri, 27 Sep 2024 11:24:55 +0300 Subject: [PATCH] Update MLflow Helm Chart (following bitnami): (#37) * Update MLflow Helm Chart (following bitnami): - MLflow version 2.16 - `postgresql.enabled=false` by default (recommended to use CSC PUKKI) - `minio.enabled=false` by default (recommended to use CSC ALLAS) - Edit README - EDIT NOTES: `oc` instead of `kubectl` - `compatibility.openshift.adaptSecurityContext=auto`. It won't apply the different `SecurityContext` * Update charts/mlflow/README.md Co-authored-by: Alvaro Gonzalez * Update README and NOTES.txt following suggestions --------- Co-authored-by: Alvaro Gonzalez --- .gitignore | 3 +- charts/mlflow/Chart.yaml | 20 +- charts/mlflow/README.md | 56 ++- charts/mlflow/templates/NOTES.txt | 26 +- charts/mlflow/templates/_helpers.tpl | 52 +- charts/mlflow/templates/extra-list.yaml | 2 +- charts/mlflow/templates/run/dep-job.yaml | 32 +- .../mlflow/templates/run/networkpolicy.yaml | 61 +++ charts/mlflow/templates/run/pvc.yaml | 2 +- .../mlflow/templates/run/service-account.yaml | 2 +- .../templates/run/source-configmap.yaml | 2 +- .../templates/tracking/auth-secret.yaml | 2 +- .../tracking/configmap-overrides.yaml | 2 +- .../mlflow/templates/tracking/deployment.yaml | 56 ++- .../templates/tracking/externaldb-secret.yaml | 2 +- .../templates/tracking/externals3-secret.yaml | 9 +- charts/mlflow/templates/tracking/hpa.yaml | 2 +- .../tracking/ingress-tls-secrets.yaml | 2 +- charts/mlflow/templates/tracking/ingress.yaml | 2 +- .../templates/tracking/networkpolicy.yaml | 34 +- charts/mlflow/templates/tracking/pdb.yaml | 9 +- charts/mlflow/templates/tracking/pvc.yaml | 2 +- .../templates/tracking/service-account.yaml | 2 +- charts/mlflow/templates/tracking/service.yaml | 2 +- .../templates/tracking/servicemonitor.yaml | 2 +- .../mlflow/templates/tracking/tls-secret.yaml | 2 +- charts/mlflow/templates/tracking/vpa.yaml | 2 +- charts/mlflow/values.schema.json | 275 ++++++----- charts/mlflow/values.yaml | 455 +++++++++++------- 29 files changed, 738 insertions(+), 382 deletions(-) create mode 100644 charts/mlflow/templates/run/networkpolicy.yaml diff --git a/.gitignore b/.gitignore index 8e53e9c..8688e56 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ .DS_Store -Chart.lock \ No newline at end of file +Chart.lock +*.tgz \ No newline at end of file diff --git a/charts/mlflow/Chart.yaml b/charts/mlflow/Chart.yaml index 904f48c..2b2327b 100644 --- a/charts/mlflow/Chart.yaml +++ b/charts/mlflow/Chart.yaml @@ -1,4 +1,4 @@ -# Copyright VMware, Inc. +# Copyright Broadcom, Inc. All Rights Reserved. # SPDX-License-Identifier: APACHE-2.0 annotations: @@ -6,29 +6,28 @@ annotations: licenses: Apache-2.0 images: | - name: git - image: docker.io/bitnami/git:2.43.0-debian-11-r1 + image: docker.io/bitnami/git:2.46.1-debian-12-r1 - name: mlflow - image: docker.io/bitnami/mlflow:2.9.2-debian-11-r0 + image: docker.io/bitnami/mlflow:2.16.2-debian-12-r3 - name: os-shell - image: docker.io/bitnami/os-shell:11-debian-11-r92 + image: docker.io/bitnami/os-shell:12-debian-12-r30 apiVersion: v2 -appVersion: 2.9.2 +appVersion: 2.16.2 dependencies: - condition: minio.enabled name: minio repository: oci://registry-1.docker.io/bitnamicharts - version: 12.x.x + version: 14.x.x - condition: postgresql.enabled name: postgresql repository: oci://registry-1.docker.io/bitnamicharts - version: 13.2.28 + version: 15.x.x - name: common repository: oci://registry-1.docker.io/bitnamicharts tags: - bitnami-common version: 2.x.x description: MLflow is an open-source platform designed to manage the end-to-end machine learning lifecycle. It allows you to track experiments, package code into reproducible runs, and share and deploy models. - Link to the repo https://github.com/CSCfi/helm-charts home: https://bitnami.com icon: https://bitnami.com/assets/stacks/mlflow/img/mlflow-stack-220x234.png keywords: @@ -38,10 +37,11 @@ keywords: - machine - learning maintainers: -- name: VMware, Inc. +- name: Broadcom, Inc. All Rights Reserved. url: https://github.com/bitnami/charts name: mlflow sources: +- https://github.com/bitnami/charts/tree/main/bitnami/mlflow - https://github.com/bitnami/containers/tree/main/bitnami/mlflow - https://github.com/mlflow/mlflow -version: 0.4.0 +version: 1.5.7 diff --git a/charts/mlflow/README.md b/charts/mlflow/README.md index ddab14c..4071ef3 100644 --- a/charts/mlflow/README.md +++ b/charts/mlflow/README.md @@ -5,10 +5,15 @@ ## Introduction This Helm chart deploys MLflow on Rahti2. +It is highly recommended to use the Helm CLI instead of the WebUI of Rahti2. If so, you can clone the GitHub repository from [here](https://github.com/CSCfi/helm-charts). +Helm CLI allows you: +- to download the necessary dependencies in order to run the chart, if you decide to run PostgreSQL and MinIO in Rahti2. +- to set the necessary values (see command below), if you decide to run a PostgreSQL instance externally and to use an external S3 service. + ## Test and Deploy Different steps are necessary to deploy this Helm Chart to Rahti2: -1. If you want to use CSC external S3 service (Allas), be sure to create Allas credentials. +1. By default, this Helm Chart will use the CSC S3 service Allas. Be sure to create Allas credentials. You can achieve this by [sourcing](https://docs.csc.fi/cloud/pouta/install-client/#configure-your-terminal-environment-for-openstack) your cPouta project and then type this command: ```sh @@ -19,36 +24,73 @@ Different steps are necessary to deploy this Helm Chart to Rahti2: You can also use another external S3 service instead of Allas. -2. Deploy MLflow: +2. By default, it also uses our CSC database service named [Pukki](https://pukki.dbaas.csc.fi). Be sure to have a database created on this service. +During the process of creation of database, it will ask you the `Allowed CIDRs`. Rahti2 has a common egress IP which is `86.50.229.150`. If you want a dedicated egress IP, you can send a ticket to [servicedesk@csc.fi](mailto:servicedesk@csc.fi). More information [here](https://docs.csc.fi/cloud/rahti2/networking/#egress-ips). + + A database named `mlflow_auth` must be created when launching your instance. This database is needed for the auth module (only if `tracking.auth.enabled=true` which is the case by default). + +3. Deploy MLflow: ```sh - helm install mlflow . --set externalS3.accessKeyID={ACCESS_KEY} --set externalS3.accessKeySecret={SECRET_KEY} --set externalS3.bucket=mlflow + helm install mlflow . --set externalS3.accessKeyID={ACCESS_KEY} \ + --set externalS3.accessKeySecret={SECRET_KEY} \ + --set externalS3.bucket={BUCKET_NAME} \ + --set externalDatabase.host={DB_PUBLIC_IP} \ + --set externalDatabase.user={DB_USER} \ + --set externalDatabase.password={DB_PASSWORD} \ + --set externalDatabase.database={DB_NAME} ``` _Replace {ACCESS_KEY} by the access key previously created_ _Replace {SECRET_KEY} by the secret key previously created_ _Replace {BUCKET_NAME} by the name of the bucket previously created_ + _Replace {DB_PUBLIC_IP} by the public IP of your databse created on Pukki_ + _Replace {DB_USER} by the user created on Pukki_ + _Replace {DB_NAME} by the database created on Pukki_ Alternatively, you can edit the `values.yaml`: ```yaml + [...] + externalDatabase: + host: '' + user: '' + database: '' + password: '' + [...] externalS3: accessKeyID: '' accessKeySecret: '' - bucket: 'mlflow' + bucket: '' ``` -To access MLflow tracking webpage, run this command to retrieve `user` password: +After the deployment, the Web URL will be displayed in the NOTES. To access MLflow tracking webpage, run this command to retrieve `user` password: ```sh -echo Password: $(oc get secret --namespace {YOUR_NAMESPACE} mlflow-tracking -o jsonpath="{.data.admin-password }" | base64 -d) +echo Password: $(oc get secret --namespace {YOUR_NAMESPACE} mlflow-tracking -o jsonpath="{ .data.admin-password }" | base64 -d) ``` _Replace {YOUR_NAMESPACE} by the name of your project in Rahti_ -You can edit the `config.yaml`. Instead of deleting your deployment and recreating a new one, Helm lets you `upgrade` your release. Use this command: +You can edit the `values.yaml`. Instead of deleting your deployment and recreating a new one, Helm lets you `upgrade` your release. Use this command: ```sh helm upgrade mlflow . --set externalS3.accessKeyID={ACCESS_KEY} --set externalS3.accessKeySecret={SECRET_KEY} --set externalS3.bucket={BUCKET_NAME} ``` +## NOTES +You can use this template by deploying PostgreSQL and MINIO in Rahti2. You can enable these parameters by editing the `values.yaml`: +```yaml +[...] +postgresql: + enabled: true +[...] +minio: + enabled: true +``` + +**It is highly recommended to use our other services (Pukki and Allas) in a production environment.** + +If, for some reasons, the Rahti2 node crashes while you have PostgreSQL and MinIO running, it can cause disruptions and corruption in your database. +Pukki also has automatic backups for your databases. + ## Project status ## Links diff --git a/charts/mlflow/templates/NOTES.txt b/charts/mlflow/templates/NOTES.txt index 8675e00..af6e62a 100644 --- a/charts/mlflow/templates/NOTES.txt +++ b/charts/mlflow/templates/NOTES.txt @@ -12,11 +12,11 @@ The chart has been deployed in diagnostic mode. All probes have been disabled an Get the list of pods by executing: - kubectl get pods --namespace {{ include "common.names.namespace" . | quote }} -l app.kubernetes.io/instance={{ .Release.Name }} + oc get pods --namespace {{ include "common.names.namespace" . | quote }} -l app.kubernetes.io/instance={{ .Release.Name }} Access the pod you want to debug by executing - kubectl exec --namespace {{ include "common.names.namespace" . | quote }} -ti -- bash + oc exec --namespace {{ include "common.names.namespace" . | quote }} -ti -- bash {{- else }} @@ -27,19 +27,19 @@ The following command will be executed: {{- include "common.tplvalues.render" (dict "value" .Values.run.source.launchCommand "context" $) | nindent 2 }} You can see the logs of each running node with: - kubectl logs [POD_NAME] + oc logs [POD_NAME] and the list of pods: - kubectl get pods --namespace {{ include "common.names.namespace" . }} -l "app.kubernetes.io/name={{ include "common.names.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" + oc get pods --namespace {{ include "common.names.namespace" . }} -l "app.kubernetes.io/name={{ include "common.names.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" {{- else }} You didn't specify any entrypoint to your code. To run it, you can either deploy again using the `source.launchCommand` option to specify your entrypoint, or execute it manually by jumping into the pods: 1. Get the running pods - kubectl get pods --namespace {{ include "common.names.namespace" . }} -l "app.kubernetes.io/name={{ include "common.names.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" + oc get pods --namespace {{ include "common.names.namespace" . }} -l "app.kubernetes.io/name={{ include "common.names.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" 2. Get into a pod - kubectl exec -ti [POD_NAME] bash + oc exec -ti [POD_NAME] bash 3. Execute your script as you would normally do. {{- end }} @@ -68,21 +68,21 @@ To access your MLflow site from outside the cluster follow the steps below: {{- if contains "NodePort" .Values.tracking.service.type }} - export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "mlflow.v0.tracking.fullname" . }}) - export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + export NODE_PORT=$(oc get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "mlflow.v0.tracking.fullname" . }}) + export NODE_IP=$(oc get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") echo "MLflow URL: {{ include "mlflow.v0.tracking.protocol" . }}://$NODE_IP:$NODE_PORT/" {{- else if contains "LoadBalancer" .Values.tracking.service.type }} NOTE: It may take a few minutes for the LoadBalancer IP to be available. - Watch the status with: 'kubectl get svc --namespace {{ .Release.Namespace }} -w {{ include "mlflow.v0.tracking.fullname" . }}' + Watch the status with: 'oc get svc --namespace {{ .Release.Namespace }} -w {{ include "mlflow.v0.tracking.fullname" . }}' - export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "mlflow.v0.tracking.fullname" . }} --template "{{ "{{ range (index .status.loadBalancer.ingress 0) }}{{ . }}{{ end }}" }}") + export SERVICE_IP=$(oc get svc --namespace {{ .Release.Namespace }} {{ include "mlflow.v0.tracking.fullname" . }} --template "{{ "{{ range (index .status.loadBalancer.ingress 0) }}{{ . }}{{ end }}" }}") echo "MLflow URL: {{ include "mlflow.v0.tracking.protocol" . }}://$SERVICE_IP{{- if ne $port "80" }}:{{ include "mlflow.v0.tracking.port" . }}{{ end }}/" {{- else if contains "ClusterIP" .Values.tracking.service.type }} - kubectl port-forward --namespace {{ .Release.Namespace }} svc/{{ include "mlflow.v0.tracking.fullname" . }} {{ include "mlflow.v0.tracking.port" . }}:{{ include "mlflow.v0.tracking.port" . }} & + oc port-forward --namespace {{ .Release.Namespace }} svc/{{ include "mlflow.v0.tracking.fullname" . }} {{ include "mlflow.v0.tracking.port" . }}:{{ include "mlflow.v0.tracking.port" . }} & echo "MLflow URL: {{ include "mlflow.v0.tracking.protocol" . }}://127.0.0.1{{- if ne $port "80" }}:{{ include "mlflow.v0.tracking.port" . }}{{ end }}//" @@ -100,8 +100,8 @@ To access your MLflow site from outside the cluster follow the steps below: {{- if .Values.tracking.enabled }} 3. Login with the following credentials below to see your blog: - echo Username: $(kubectl get secret --namespace {{ .Release.Namespace }} {{ include "mlflow.v0.tracking.fullname" . }} -o jsonpath="{ .data.{{ include "mlflow.v0.tracking.userKey" . }} }" | base64 -d) - echo Password: $(kubectl get secret --namespace {{ .Release.Namespace }} {{ include "mlflow.v0.tracking.fullname" . }} -o jsonpath="{.data.{{ include "mlflow.v0.tracking.passwordKey" . }} }" | base64 -d) + echo Username: $(oc get secret --namespace {{ .Release.Namespace }} {{ include "mlflow.v0.tracking.fullname" . }} -o jsonpath="{ .data.{{ include "mlflow.v0.tracking.userKey" . }} }" | base64 -d) + echo Password: $(oc get secret --namespace {{ .Release.Namespace }} {{ include "mlflow.v0.tracking.fullname" . }} -o jsonpath="{ .data.{{ include "mlflow.v0.tracking.passwordKey" . }} }" | base64 -d) {{- end }} {{- end }} {{- end }} diff --git a/charts/mlflow/templates/_helpers.tpl b/charts/mlflow/templates/_helpers.tpl index 9c95bab..bf9a5e1 100644 --- a/charts/mlflow/templates/_helpers.tpl +++ b/charts/mlflow/templates/_helpers.tpl @@ -1,5 +1,5 @@ {{/* -Copyright VMware, Inc. +Copyright Broadcom, Inc. All Rights Reserved. SPDX-License-Identifier: APACHE-2.0 */}} @@ -28,7 +28,7 @@ Return the proper image name (for the init container wait-permissions image) Return the proper Docker Image Registry Secret Names */}} {{- define "mlflow.v0.imagePullSecrets" -}} -{{- include "common.images.pullSecrets" (dict "images" (list .Values.image .Values.waitContainer.image .Values.volumePermissions.image) "global" .Values.global) -}} +{{- include "common.images.renderPullSecrets" (dict "images" (list .Values.image .Values.waitContainer.image .Values.volumePermissions.image) "context" $) -}} {{- end -}} {{/* @@ -121,7 +121,7 @@ Init container definition for copying the certificates image: {{ include "mlflow.v0.image" . }} imagePullPolicy: {{ .Values.image.pullPolicy }} {{- if .Values.tracking.containerSecurityContext.enabled }} - securityContext: {{- omit .Values.tracking.containerSecurityContext "enabled" | toYaml | nindent 4 }} + securityContext: {{- include "common.compatibility.renderSecurityContext" (dict "secContext" .Values.tracking.containerSecurityContext "context" $) | nindent 4 }} {{- end }} command: - bash @@ -147,7 +147,7 @@ Init container definition for waiting for the database to be ready image: {{ include "mlflow.v0.image" . }} imagePullPolicy: {{ .Values.image.pullPolicy }} {{- if .Values.tracking.containerSecurityContext.enabled }} - securityContext: {{- omit .Values.tracking.containerSecurityContext "enabled" | toYaml | nindent 4 }} + securityContext: {{- include "common.compatibility.renderSecurityContext" (dict "secContext" .Values.tracking.containerSecurityContext "context" $) | nindent 4 }} {{- end }} command: - bash @@ -164,7 +164,7 @@ Init container definition for waiting for the database to be ready image: {{ include "mlflow.v0.waitContainer.image" . }} imagePullPolicy: {{ .Values.waitContainer.image.pullPolicy }} {{- if .Values.tracking.containerSecurityContext.enabled }} - securityContext: {{- omit .Values.tracking.containerSecurityContext "enabled" | toYaml | nindent 4 }} + securityContext: {{- include "common.compatibility.renderSecurityContext" (dict "secContext" .Values.tracking.containerSecurityContext "context" $) | nindent 4 }} {{- end }} command: - bash @@ -234,7 +234,7 @@ Init container definition for upgrading the database image: {{ include "mlflow.v0.image" . }} imagePullPolicy: {{ .Values.image.pullPolicy }} {{- if .Values.tracking.containerSecurityContext.enabled }} - securityContext: {{- omit .Values.tracking.containerSecurityContext "enabled" | toYaml | nindent 4 }} + securityContext: {{- include "common.compatibility.renderSecurityContext" (dict "secContext" .Values.tracking.containerSecurityContext "context" $) | nindent 4 }} {{- end }} command: - mlflow @@ -261,7 +261,7 @@ Init container definition for upgrading the database image: {{ include "mlflow.v0.image" . }} imagePullPolicy: {{ .Values.image.pullPolicy }} {{- if .Values.tracking.containerSecurityContext.enabled }} - securityContext: {{- omit .Values.tracking.containerSecurityContext "enabled" | toYaml | nindent 4 }} + securityContext: {{- include "common.compatibility.renderSecurityContext" (dict "secContext" .Values.tracking.containerSecurityContext "context" $) | nindent 4 }} {{- end }} command: - python @@ -392,6 +392,17 @@ Return the PostgreSQL Hostname {{- end -}} {{- end -}} +{{/* +Return the Database Dialect(+Driver) +*/}} +{{- define "mlflow.v0.database.dialectDriver" -}} +{{- if .Values.postgresql.enabled -}} + {{- print "postgresql" -}} +{{- else -}} + {{- print .Values.externalDatabase.dialectDriver -}} +{{- end -}} +{{- end -}} + {{/* Return the PostgreSQL Hostname */}} @@ -492,14 +503,14 @@ Retrieve key of the PostgreSQL secret Retrieve the URI of the database */}} {{- define "mlflow.v0.database.uri" -}} -{{- printf "postgresql://%s:$(MLFLOW_DATABASE_PASSWORD)@%s:%v/%s" (include "mlflow.v0.database.user" .) (include "mlflow.v0.database.host" .) (include "mlflow.v0.database.port" .) (include "mlflow.v0.database.name" .) -}} +{{- printf "%s://%s:$(MLFLOW_DATABASE_PASSWORD)@%s:%v/%s" (include "mlflow.v0.database.dialectDriver" .) (include "mlflow.v0.database.user" .) (include "mlflow.v0.database.host" .) (include "mlflow.v0.database.port" .) (include "mlflow.v0.database.name" .) -}} {{- end -}} {{/* Retrieve the URI of the auth database */}} {{- define "mlflow.v0.database-auth.uri" -}} -{{- printf "postgresql://%s:$(MLFLOW_DATABASE_PASSWORD)@%s:%v/%s" (include "mlflow.v0.database.user" .) (include "mlflow.v0.database.host" .) (include "mlflow.v0.database.port" .) (include "mlflow.v0.database-auth.name" .) -}} +{{- printf "%s://%s:$(MLFLOW_DATABASE_PASSWORD)@%s:%v/%s" (include "mlflow.v0.database.dialectDriver" .) (include "mlflow.v0.database.user" .) (include "mlflow.v0.database.host" .) (include "mlflow.v0.database.port" .) (include "mlflow.v0.database-auth.name" .) -}} {{- end -}} {{/* @@ -519,10 +530,12 @@ Return the volume-permissions init container chown {{ .Values.volumePermissions.containerSecurityContext.runAsUser }}:{{ .Values.podSecurityContext.fsGroup }} {{ .Values.persistence.mountPath }} find {{ .Values.persistence.mountPath }} -mindepth 1 -maxdepth 1 -not -name ".snapshot" -not -name "lost+found" | xargs chown -R {{ .Values.volumePermissions.containerSecurityContext.runAsUser }}:{{ .Values.podSecurityContext.fsGroup }} {{- if .Values.volumePermissions.containerSecurityContext.enabled }} - securityContext: {{- omit .Values.volumePermissions.containerSecurityContext "enabled" | toYaml | nindent 4 }} + securityContext: {{- include "common.compatibility.renderSecurityContext" (dict "secContext" .Values.volumePermissions.containerSecurityContext "context" $) | nindent 4 }} {{- end }} {{- if .Values.volumePermissions.resources }} resources: {{- toYaml .Values.volumePermissions.resources | nindent 12 }} + {{- else if ne .Values.volumePermissions.resourcesPreset "none" }} + resources: {{- include "common.resources.preset" (dict "type" .Values.volumePermissions.resourcesPreset) | nindent 12 }} {{- end }} volumeMounts: - name: data @@ -531,6 +544,11 @@ Return the volume-permissions init container mountPath: /tmp {{- end -}} + +{{/* +Deal with external artifact storage +*/}} + {{/* Return MinIO(TM) fullname */}} @@ -538,6 +556,7 @@ Return MinIO(TM) fullname {{- include "common.names.dependency.fullname" (dict "chartName" "minio" "chartValues" .Values.minio "context" $) -}} {{- end -}} + {{/* Return whether S3 is enabled */}} @@ -635,6 +654,15 @@ Return the S3 secret access key inside the secret {{- end -}} {{- end -}} +{{/* +Return whether GCS is enabled +*/}} +{{- define "mlflow.v0.gcs.enabled" -}} + {{- if and (not .Values.minio.enabled) (not .Values.externalS3.host) .Values.externalGCS.bucket -}} + {{- true }} + {{- end -}} +{{- end -}} + {{/* Return the proper git image name @@ -660,7 +688,7 @@ Return the definition of the git clone init container [[ -f "/opt/bitnami/scripts/git/entrypoint.sh" ]] && source "/opt/bitnami/scripts/git/entrypoint.sh" git clone {{ .Values.run.source.git.repository }} {{ if .Values.run.source.git.revision }}--branch {{ .Values.run.source.git.revision }}{{ end }} /app {{- if .Values.run.containerSecurityContext.enabled }} - securityContext: {{- omit .Values.run.containerSecurityContext "enabled" | toYaml | nindent 4 }} + securityContext: {{- include "common.compatibility.renderSecurityContext" (dict "secContext" .Values.run.containerSecurityContext "context" $) | nindent 4 }} {{- end }} volumeMounts: - name: source @@ -683,7 +711,7 @@ Init container definition for waiting for the database to be ready image: {{ include "mlflow.v0.waitContainer.image" .context }} imagePullPolicy: {{ .context.Values.waitContainer.image.pullPolicy }} {{- if .context.Values.waitContainer.containerSecurityContext.enabled }} - securityContext: {{- omit .context.Values.waitContainer.containerSecurityContext "enabled" | toYaml | nindent 4 }} + securityContext: {{- include "common.compatibility.renderSecurityContext" (dict "secContext" .context.Values.waitContainer.containerSecurityContext "context" .context) | nindent 4 }} {{- end }} command: - bash diff --git a/charts/mlflow/templates/extra-list.yaml b/charts/mlflow/templates/extra-list.yaml index 2d35a58..329f5c6 100644 --- a/charts/mlflow/templates/extra-list.yaml +++ b/charts/mlflow/templates/extra-list.yaml @@ -1,5 +1,5 @@ {{- /* -Copyright VMware, Inc. +Copyright Broadcom, Inc. All Rights Reserved. SPDX-License-Identifier: APACHE-2.0 */}} diff --git a/charts/mlflow/templates/run/dep-job.yaml b/charts/mlflow/templates/run/dep-job.yaml index 4e93381..4c1ccba 100644 --- a/charts/mlflow/templates/run/dep-job.yaml +++ b/charts/mlflow/templates/run/dep-job.yaml @@ -1,5 +1,5 @@ {{- /* -Copyright VMware, Inc. +Copyright Broadcom, Inc. All Rights Reserved. SPDX-License-Identifier: APACHE-2.0 */}} @@ -44,8 +44,9 @@ spec: app.kubernetes.io/part-of: mlflow app.kubernetes.io/component: run spec: - {{- include "mlflow.v0.imagePullSecrets" . | indent 6 }} + {{- include "mlflow.v0.imagePullSecrets" . | nindent 6 }} serviceAccountName: {{ template "mlflow.v0.run.serviceAccountName" . }} + automountServiceAccountToken: {{ .Values.run.automountServiceAccountToken }} {{- if .Values.run.hostAliases }} hostAliases: {{- include "common.tplvalues.render" (dict "value" .Values.run.hostAliases "context" $) | nindent 8 }} {{- end }} @@ -67,7 +68,7 @@ spec: runtimeClassName: {{ .Values.run.runtimeClassName | quote }} {{- end }} {{- if .Values.run.podSecurityContext.enabled }} - securityContext: {{- omit .Values.run.podSecurityContext "enabled" | toYaml | nindent 8 }} + securityContext: {{- include "common.compatibility.renderSecurityContext" (dict "secContext" .Values.run.podSecurityContext "context" $) | nindent 8 }} {{- end }} {{- if .Values.run.tolerations }} tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.run.tolerations "context" .) | nindent 8 }} @@ -107,7 +108,7 @@ spec: image: {{ include "mlflow.v0.image" . }} imagePullPolicy: {{ .Values.image.pullPolicy | quote }} {{- if .Values.run.containerSecurityContext.enabled }} - securityContext: {{- omit .Values.run.containerSecurityContext "enabled" | toYaml | nindent 12 }} + securityContext: {{- include "common.compatibility.renderSecurityContext" (dict "secContext" .Values.run.containerSecurityContext "context" $) | nindent 12 }} {{- end }} {{- if .Values.diagnosticMode.enabled }} command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 12 }} @@ -170,37 +171,18 @@ spec: {{- end }} {{- if .Values.run.resources }} resources: {{- toYaml .Values.run.resources | nindent 12 }} + {{- else if ne .Values.run.resourcesPreset "none" }} + resources: {{- include "common.resources.preset" (dict "type" .Values.run.resourcesPreset) | nindent 12 }} {{- end }} {{- if not .Values.diagnosticMode.enabled }} {{- if .Values.run.customLivenessProbe }} livenessProbe: {{- include "common.tplvalues.render" (dict "value" .Values.run.customLivenessProbe "context" $) | nindent 12 }} - {{- else if .Values.run.livenessProbe.enabled }} - livenessProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.run.livenessProbe "enabled") "context" $) | nindent 12 }} - exec: - command: - - python - - -c - - import mlflow; mlflow.__version__ {{- end }} {{- if .Values.run.customReadinessProbe }} readinessProbe: {{- include "common.tplvalues.render" (dict "value" .Values.run.customReadinessProbe "context" $) | nindent 12 }} - {{- else if .Values.run.readinessProbe.enabled }} - readinessProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.run.readinessProbe "enabled") "context" $) | nindent 12 }} - exec: - command: - - python - - -c - - import mlflow; mlflow.__version__ {{- end }} {{- if .Values.run.customStartupProbe }} startupProbe: {{- include "common.tplvalues.render" (dict "value" .Values.run.customStartupProbe "context" $) | nindent 12 }} - {{- else if .Values.run.startupProbe.enabled }} - startupProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.run.startupProbe "enabled") "context" $) | nindent 12 }} - exec: - command: - - python - - -c - - import mlflow; mlflow.__version__ {{- end }} {{- end }} {{- if .Values.run.lifecycleHooks }} diff --git a/charts/mlflow/templates/run/networkpolicy.yaml b/charts/mlflow/templates/run/networkpolicy.yaml new file mode 100644 index 0000000..5fffd04 --- /dev/null +++ b/charts/mlflow/templates/run/networkpolicy.yaml @@ -0,0 +1,61 @@ +{{- /* +Copyright Broadcom, Inc. All Rights Reserved. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{- if and .Values.run.enabled .Values.run.networkPolicy.enabled }} +kind: NetworkPolicy +apiVersion: {{ include "common.capabilities.networkPolicy.apiVersion" . }} +metadata: + name: {{ template "mlflow.v0.run.fullname" . }} + namespace: {{ include "common.names.namespace" . | quote }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + app.kubernetes.io/part-of: mlflow + app.kubernetes.io/component: run + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" (dict "value" .Values.commonAnnotations "context" $) | nindent 4 }} + {{- end }} +spec: + {{- $podLabels := include "common.tplvalues.merge" ( dict "values" ( list .Values.tracking.podLabels .Values.commonLabels ) "context" . ) }} + podSelector: + matchLabels: {{- include "common.labels.matchLabels" ( dict "customLabels" $podLabels "context" $ ) | nindent 6 }} + app.kubernetes.io/part-of: mlflow + app.kubernetes.io/component: run + policyTypes: + - Ingress + - Egress + {{- if .Values.run.networkPolicy.allowExternalEgress }} + egress: + - {} + {{- else }} + egress: + # Allow dns resolution + - ports: + - port: 53 + protocol: UDP + - port: 53 + protocol: TCP + # Allow connection to these sources for downloading models + - port: 80 + - port: 443 + - port: 22 + {{- if .Values.tracking.enabled }} + # Allow outbound connections to other pods + - ports: + - port: {{ .Values.tracking.containerPorts.http }} + - port: {{ include "mlflow.v0.tracking.port" . }} + to: + - podSelector: + matchLabels: {{- include "common.labels.matchLabels" ( dict "customLabels" $podLabels "context" $ ) | nindent 14 }} + app.kubernetes.io/part-of: mlflow + app.kubernetes.io/component: tracking + {{- end }} + {{- if .Values.run.networkPolicy.extraEgress }} + {{- include "common.tplvalues.render" (dict "value" .Values.tracking.networkPolicy.extraEgress "context" $) | nindent 4 }} + {{- end }} + {{- end }} + ingress: + {{- if .Values.run.networkPolicy.extraIngress }} + {{- include "common.tplvalues.render" (dict "value" .Values.tracking.networkPolicy.extraIngress "context" $) | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/mlflow/templates/run/pvc.yaml b/charts/mlflow/templates/run/pvc.yaml index d808494..daea8fd 100644 --- a/charts/mlflow/templates/run/pvc.yaml +++ b/charts/mlflow/templates/run/pvc.yaml @@ -1,5 +1,5 @@ {{- /* -Copyright VMware, Inc. +Copyright Broadcom, Inc. All Rights Reserved. SPDX-License-Identifier: APACHE-2.0 */}} diff --git a/charts/mlflow/templates/run/service-account.yaml b/charts/mlflow/templates/run/service-account.yaml index 4716cf7..b4e3e70 100644 --- a/charts/mlflow/templates/run/service-account.yaml +++ b/charts/mlflow/templates/run/service-account.yaml @@ -1,5 +1,5 @@ {{- /* -Copyright VMware, Inc. +Copyright Broadcom, Inc. All Rights Reserved. SPDX-License-Identifier: APACHE-2.0 */}} diff --git a/charts/mlflow/templates/run/source-configmap.yaml b/charts/mlflow/templates/run/source-configmap.yaml index e19e4fc..3edfdaf 100644 --- a/charts/mlflow/templates/run/source-configmap.yaml +++ b/charts/mlflow/templates/run/source-configmap.yaml @@ -1,5 +1,5 @@ {{- /* -Copyright VMware, Inc. +Copyright Broadcom, Inc. All Rights Reserved. SPDX-License-Identifier: APACHE-2.0 */}} diff --git a/charts/mlflow/templates/tracking/auth-secret.yaml b/charts/mlflow/templates/tracking/auth-secret.yaml index d08ec22..bdf159d 100644 --- a/charts/mlflow/templates/tracking/auth-secret.yaml +++ b/charts/mlflow/templates/tracking/auth-secret.yaml @@ -1,5 +1,5 @@ {{- /* -Copyright VMware, Inc. +Copyright Broadcom, Inc. All Rights Reserved. SPDX-License-Identifier: APACHE-2.0 */}} diff --git a/charts/mlflow/templates/tracking/configmap-overrides.yaml b/charts/mlflow/templates/tracking/configmap-overrides.yaml index 0dd8df3..f76f129 100644 --- a/charts/mlflow/templates/tracking/configmap-overrides.yaml +++ b/charts/mlflow/templates/tracking/configmap-overrides.yaml @@ -1,5 +1,5 @@ {{- /* -Copyright VMware, Inc. +Copyright Broadcom, Inc. All Rights Reserved. SPDX-License-Identifier: APACHE-2.0 */}} diff --git a/charts/mlflow/templates/tracking/deployment.yaml b/charts/mlflow/templates/tracking/deployment.yaml index dc6ef26..8a4b2d7 100644 --- a/charts/mlflow/templates/tracking/deployment.yaml +++ b/charts/mlflow/templates/tracking/deployment.yaml @@ -1,5 +1,5 @@ {{- /* -Copyright VMware, Inc. +Copyright Broadcom, Inc. All Rights Reserved. SPDX-License-Identifier: APACHE-2.0 */}} @@ -43,7 +43,8 @@ spec: app.kubernetes.io/component: tracking spec: serviceAccountName: {{ include "mlflow.v0.tracking.serviceAccountName" . }} - {{- include "mlflow.v0.imagePullSecrets" . | indent 6 }} + {{- include "mlflow.v0.imagePullSecrets" . | nindent 6 }} + automountServiceAccountToken: {{ .Values.tracking.automountServiceAccountToken }} {{- if .Values.tracking.hostAliases }} hostAliases: {{- include "common.tplvalues.render" (dict "value" .Values.tracking.hostAliases "context" $) | nindent 8 }} {{- end }} @@ -71,7 +72,7 @@ spec: topologySpreadConstraints: {{- include "common.tplvalues.render" (dict "value" .Values.tracking.topologySpreadConstraints "context" .) | nindent 8 }} {{- end }} {{- if .Values.tracking.podSecurityContext.enabled }} - securityContext: {{- omit .Values.tracking.podSecurityContext "enabled" | toYaml | nindent 8 }} + securityContext: {{- include "common.compatibility.renderSecurityContext" (dict "secContext" .Values.tracking.podSecurityContext "context" $) | nindent 8 }} {{- end }} {{- if .Values.tracking.terminationGracePeriodSeconds }} terminationGracePeriodSeconds: {{ .Values.tracking.terminationGracePeriodSeconds }} @@ -112,7 +113,7 @@ spec: image: {{ include "mlflow.v0.image" . }} imagePullPolicy: {{ .Values.image.pullPolicy }} {{- if .Values.tracking.containerSecurityContext.enabled }} - securityContext: {{- omit .Values.tracking.containerSecurityContext "enabled" | toYaml | nindent 12 }} + securityContext: {{- include "common.compatibility.renderSecurityContext" (dict "secContext" .Values.tracking.containerSecurityContext "context" $) | nindent 12 }} {{- end }} {{- if .Values.diagnosticMode.enabled }} command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 12 }} @@ -140,12 +141,17 @@ spec: {{- end }} {{- if (include "mlflow.v0.s3.enabled" .) }} - --artifacts-destination=s3://{{ include "mlflow.v0.s3.bucket" . }} + {{- else if (include "mlflow.v0.gcs.enabled" .) }} + - --artifacts-destination=gs://{{ .Values.externalGCS.bucket }} {{- else }} - --artifacts-destination={{ .Values.tracking.persistence.mountPath }}/mlartifacts {{- end }} {{- if and (not (include "mlflow.v0.s3.serveArtifacts" .)) (include "mlflow.v0.s3.enabled" .) }} - --default-artifact-root=s3://{{ include "mlflow.v0.s3.bucket" . }} - --no-serve-artifacts + {{- else if and (not .Values.externalGCS.serveArtifacts) (include "mlflow.v0.gcs.enabled" .) }} + - --default-artifact-root=gs://{{ .Values.externalGCS.bucket }} + - --no-serve-artifacts {{ else }} - --serve-artifacts {{- end }} @@ -172,6 +178,7 @@ spec: key: {{ include "mlflow.v0.database.passwordKey" . | quote }} {{- end }} {{- if (include "mlflow.v0.s3.enabled" .) }} + {{- if .Values.externalS3.useCredentialsInSecret }} - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: @@ -182,9 +189,22 @@ spec: secretKeyRef: name: {{ include "mlflow.v0.s3.secretName" . }} key: {{ include "mlflow.v0.s3.secretAccessKeyKey" . | quote }} + {{- end }} + {{- end }} + {{- if (include "mlflow.v0.s3.enabled" .) }} - name: MLFLOW_S3_ENDPOINT_URL value: {{ printf "%s://%s:%v" (include "mlflow.v0.s3.protocol" .) (include "mlflow.v0.s3.host" .) (include "mlflow.v0.s3.port" .) | quote }} {{- end }} + {{- if (include "mlflow.v0.gcs.enabled" .) }} + {{- if .Values.externalGCS.useCredentialsInSecret }} + - name: GOOGLE_APPLICATION_CREDENTIALS + value: /bitnami/gcs/key.json + {{- end }} + {{- if .Values.externalGCS.googleCloudProject }} + - name: GOOGLE_CLOUD_PROJECT + value: {{ .Values.externalGCS.googleCloudProject }} + {{- end }} + {{- end }} {{- if .Values.tracking.extraEnvVars }} {{- include "common.tplvalues.render" (dict "value" .Values.tracking.extraEnvVars "context" $) | nindent 12 }} {{- end }} @@ -199,6 +219,8 @@ spec: {{- end }} {{- if .Values.tracking.resources }} resources: {{- toYaml .Values.tracking.resources | nindent 12 }} + {{- else if ne .Values.tracking.resourcesPreset "none" }} + resources: {{- include "common.resources.preset" (dict "type" .Values.tracking.resourcesPreset) | nindent 12 }} {{- end }} ports: - name: {{ include "mlflow.v0.tracking.portName" . }} @@ -208,14 +230,11 @@ spec: livenessProbe: {{- include "common.tplvalues.render" (dict "value" .Values.tracking.customLivenessProbe "context" $) | nindent 12 }} {{- else if .Values.tracking.livenessProbe.enabled }} livenessProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.tracking.livenessProbe "enabled") "context" $) | nindent 12 }} - {{- if .Values.tracking.auth.enabled }} - tcpSocket: - port: {{ include "mlflow.v0.tracking.portName" . }} - {{- else }} - httpGet: - path: / - port: {{ include "mlflow.v0.tracking.portName" . }} - {{- end }} + exec: + command: + - pgrep + - -f + - mlflow.server {{- end }} {{- if .Values.tracking.customReadinessProbe }} readinessProbe: {{- include "common.tplvalues.render" (dict "value" .Values.tracking.customReadinessProbe "context" $) | nindent 12 }} @@ -269,6 +288,11 @@ spec: {{- if .Values.tracking.persistence.subPath }} subPath: {{ .Values.tracking.persistence.subPath }} {{- end }} + {{- if and (include "mlflow.v0.gcs.enabled" .) .Values.externalGCS.useCredentialsInSecret }} + - name: gcs-key + mountPath: /bitnami/gcs/key.json + subPath: key.json + {{- end }} {{- if .Values.tracking.extraVolumeMounts }} {{- include "common.tplvalues.render" (dict "value" .Values.tracking.extraVolumeMounts "context" $) | nindent 12 }} {{- end }} @@ -306,4 +330,12 @@ spec: {{- if .Values.tracking.extraVolumes }} {{- include "common.tplvalues.render" (dict "value" .Values.tracking.extraVolumes "context" $) | nindent 8 }} {{- end }} + {{- if and (include "mlflow.v0.gcs.enabled" .) .Values.externalGCS.useCredentialsInSecret }} + - name: gcs-key + secret: + secretName: {{ .Values.externalGCS.existingSecret }} + items: + - key: {{ .Values.externalGCS.existingSecretKey }} + path: key.json + {{- end }} {{- end }} diff --git a/charts/mlflow/templates/tracking/externaldb-secret.yaml b/charts/mlflow/templates/tracking/externaldb-secret.yaml index 9d36a82..aa6e976 100644 --- a/charts/mlflow/templates/tracking/externaldb-secret.yaml +++ b/charts/mlflow/templates/tracking/externaldb-secret.yaml @@ -1,5 +1,5 @@ {{- /* -Copyright VMware, Inc. +Copyright Broadcom, Inc. All Rights Reserved. SPDX-License-Identifier: APACHE-2.0 */}} diff --git a/charts/mlflow/templates/tracking/externals3-secret.yaml b/charts/mlflow/templates/tracking/externals3-secret.yaml index e14fa97..5e88000 100644 --- a/charts/mlflow/templates/tracking/externals3-secret.yaml +++ b/charts/mlflow/templates/tracking/externals3-secret.yaml @@ -1,9 +1,14 @@ {{- /* -Copyright VMware, Inc. +Copyright Broadcom, Inc. All Rights Reserved. SPDX-License-Identifier: APACHE-2.0 */}} -{{- if and .Values.tracking.enabled (not .Values.minio.enabled) (not .Values.externalS3.existingSecret) }} +{{- if and + .Values.tracking.enabled + .Values.externalS3.useCredentialsInSecret + (not .Values.minio.enabled) + (not .Values.externalS3.existingSecret) +}} apiVersion: v1 kind: Secret metadata: diff --git a/charts/mlflow/templates/tracking/hpa.yaml b/charts/mlflow/templates/tracking/hpa.yaml index 221382f..49e26c7 100644 --- a/charts/mlflow/templates/tracking/hpa.yaml +++ b/charts/mlflow/templates/tracking/hpa.yaml @@ -1,5 +1,5 @@ {{- /* -Copyright VMware, Inc. +Copyright Broadcom, Inc. All Rights Reserved. SPDX-License-Identifier: APACHE-2.0 */}} diff --git a/charts/mlflow/templates/tracking/ingress-tls-secrets.yaml b/charts/mlflow/templates/tracking/ingress-tls-secrets.yaml index cb767ba..3b645d7 100644 --- a/charts/mlflow/templates/tracking/ingress-tls-secrets.yaml +++ b/charts/mlflow/templates/tracking/ingress-tls-secrets.yaml @@ -1,5 +1,5 @@ {{- /* -Copyright VMware, Inc. +Copyright Broadcom, Inc. All Rights Reserved. SPDX-License-Identifier: APACHE-2.0 */}} diff --git a/charts/mlflow/templates/tracking/ingress.yaml b/charts/mlflow/templates/tracking/ingress.yaml index 7992e80..ba27618 100644 --- a/charts/mlflow/templates/tracking/ingress.yaml +++ b/charts/mlflow/templates/tracking/ingress.yaml @@ -1,5 +1,5 @@ {{- /* -Copyright VMware, Inc. +Copyright Broadcom, Inc. All Rights Reserved. SPDX-License-Identifier: APACHE-2.0 */}} diff --git a/charts/mlflow/templates/tracking/networkpolicy.yaml b/charts/mlflow/templates/tracking/networkpolicy.yaml index d7cbdd0..5cd70a3 100644 --- a/charts/mlflow/templates/tracking/networkpolicy.yaml +++ b/charts/mlflow/templates/tracking/networkpolicy.yaml @@ -1,5 +1,5 @@ {{- /* -Copyright VMware, Inc. +Copyright Broadcom, Inc. All Rights Reserved. SPDX-License-Identifier: APACHE-2.0 */}} @@ -24,6 +24,10 @@ spec: policyTypes: - Ingress - Egress + {{- if .Values.tracking.networkPolicy.allowExternalEgress }} + egress: + - {} + {{- else }} egress: # Allow dns resolution - ports: @@ -31,20 +35,44 @@ spec: protocol: UDP - port: 53 protocol: TCP + # Allow connection to these sources for downloading models + - port: 80 + - port: 443 + - port: 22 # Allow outbound connections to other pods - ports: - port: {{ .Values.tracking.containerPorts.http }} - port: {{ include "mlflow.v0.tracking.port" . }} - - port: {{ include "mlflow.v0.database.port" . }} - - port: {{ include "mlflow.v0.s3.port" . }} to: - podSelector: matchLabels: {{- include "common.labels.matchLabels" ( dict "customLabels" $podLabels "context" $ ) | nindent 14 }} app.kubernetes.io/part-of: mlflow app.kubernetes.io/component: tracking + # Allow outbound connections to the database + - ports: + - port: {{ include "mlflow.v0.database.port" . }} + {{- if .Values.postgresql.enabled }} + to: + - podSelector: + matchLabels: + app.kubernetes.io/name: postgresql + app.kubernetes.io/instance: {{ .Release.Name }} + {{- end }} + # Allow outbound connections to the S3 backend + - ports: + - port: {{ include "mlflow.v0.s3.port" . }} + {{- if .Values.minio.enabled }} + - port: {{ .Values.minio.containerPorts.api }} + to: + - podSelector: + matchLabels: + app.kubernetes.io/name: minio + app.kubernetes.io/instance: {{ .Release.Name }} + {{- end }} {{- if .Values.tracking.networkPolicy.extraEgress }} {{- include "common.tplvalues.render" (dict "value" .Values.tracking.networkPolicy.extraEgress "context" $) | nindent 4 }} {{- end }} + {{- end }} ingress: - ports: - port: {{ .Values.tracking.containerPorts.http }} diff --git a/charts/mlflow/templates/tracking/pdb.yaml b/charts/mlflow/templates/tracking/pdb.yaml index 6cdaad7..466fd1f 100644 --- a/charts/mlflow/templates/tracking/pdb.yaml +++ b/charts/mlflow/templates/tracking/pdb.yaml @@ -1,10 +1,9 @@ {{- /* -Copyright VMware, Inc. +Copyright Broadcom, Inc. All Rights Reserved. SPDX-License-Identifier: APACHE-2.0 */}} -{{- $replicaCount := int .Values.tracking.replicaCount }} -{{- if and .Values.tracking.enabled .Values.tracking.pdb.create (or (gt $replicaCount 1) .Values.tracking.autoscaling.hpa.enabled) }} +{{- if and .Values.tracking.enabled .Values.tracking.pdb.create }} apiVersion: {{ include "common.capabilities.policy.apiVersion" . }} kind: PodDisruptionBudget metadata: @@ -20,8 +19,8 @@ spec: {{- if .Values.tracking.pdb.minAvailable }} minAvailable: {{ .Values.tracking.pdb.minAvailable }} {{- end }} - {{- if .Values.tracking.pdb.maxUnavailable }} - maxUnavailable: {{ .Values.tracking.pdb.maxUnavailable }} + {{- if or .Values.tracking.pdb.maxUnavailable ( not .Values.tracking.pdb.minAvailable ) }} + maxUnavailable: {{ .Values.tracking.pdb.maxUnavailable | default 1 }} {{- end }} {{- $podLabels := include "common.tplvalues.merge" (dict "values" (list .Values.tracking.podLabels .Values.commonLabels) "context" .) | fromYaml }} selector: diff --git a/charts/mlflow/templates/tracking/pvc.yaml b/charts/mlflow/templates/tracking/pvc.yaml index aa2a3ca..3dad31e 100644 --- a/charts/mlflow/templates/tracking/pvc.yaml +++ b/charts/mlflow/templates/tracking/pvc.yaml @@ -1,5 +1,5 @@ {{- /* -Copyright VMware, Inc. +Copyright Broadcom, Inc. All Rights Reserved. SPDX-License-Identifier: APACHE-2.0 */}} diff --git a/charts/mlflow/templates/tracking/service-account.yaml b/charts/mlflow/templates/tracking/service-account.yaml index 377bb42..413ac55 100644 --- a/charts/mlflow/templates/tracking/service-account.yaml +++ b/charts/mlflow/templates/tracking/service-account.yaml @@ -1,5 +1,5 @@ {{- /* -Copyright VMware, Inc. +Copyright Broadcom, Inc. All Rights Reserved. SPDX-License-Identifier: APACHE-2.0 */}} diff --git a/charts/mlflow/templates/tracking/service.yaml b/charts/mlflow/templates/tracking/service.yaml index 183d0f0..6729fe9 100644 --- a/charts/mlflow/templates/tracking/service.yaml +++ b/charts/mlflow/templates/tracking/service.yaml @@ -1,5 +1,5 @@ {{- /* -Copyright VMware, Inc. +Copyright Broadcom, Inc. All Rights Reserved. SPDX-License-Identifier: APACHE-2.0 */}} diff --git a/charts/mlflow/templates/tracking/servicemonitor.yaml b/charts/mlflow/templates/tracking/servicemonitor.yaml index 84b3520..e0c7898 100644 --- a/charts/mlflow/templates/tracking/servicemonitor.yaml +++ b/charts/mlflow/templates/tracking/servicemonitor.yaml @@ -1,5 +1,5 @@ {{- /* -Copyright VMware, Inc. +Copyright Broadcom, Inc. All Rights Reserved. SPDX-License-Identifier: APACHE-2.0 */}} diff --git a/charts/mlflow/templates/tracking/tls-secret.yaml b/charts/mlflow/templates/tracking/tls-secret.yaml index e2e5ef6..b67d33b 100644 --- a/charts/mlflow/templates/tracking/tls-secret.yaml +++ b/charts/mlflow/templates/tracking/tls-secret.yaml @@ -1,5 +1,5 @@ {{- /* -Copyright VMware, Inc. +Copyright Broadcom, Inc. All Rights Reserved. SPDX-License-Identifier: APACHE-2.0 */}} diff --git a/charts/mlflow/templates/tracking/vpa.yaml b/charts/mlflow/templates/tracking/vpa.yaml index 2aec418..fc99b4c 100644 --- a/charts/mlflow/templates/tracking/vpa.yaml +++ b/charts/mlflow/templates/tracking/vpa.yaml @@ -1,5 +1,5 @@ {{- /* -Copyright VMware, Inc. +Copyright Broadcom, Inc. All Rights Reserved. SPDX-License-Identifier: APACHE-2.0 */}} diff --git a/charts/mlflow/values.schema.json b/charts/mlflow/values.schema.json index 33556ba..87ac82d 100644 --- a/charts/mlflow/values.schema.json +++ b/charts/mlflow/values.schema.json @@ -100,17 +100,12 @@ "registry": { "type": "string", "description": "mlflow image registry", - "default": "docker.io" + "default": "REGISTRY_NAME" }, "repository": { "type": "string", "description": "mlflow image repository", - "default": "bitnami/mlflow" - }, - "tag": { - "type": "string", - "description": "mlflow image tag (immutable tags are recommended)", - "default": "2.7.1-debian-11-r0" + "default": "REPOSITORY_NAME/mlflow" }, "digest": { "type": "string", @@ -141,17 +136,12 @@ "registry": { "type": "string", "description": "Git image registry", - "default": "docker.io" + "default": "REGISTRY_NAME" }, "repository": { "type": "string", "description": "Git image repository", - "default": "bitnami/git" - }, - "tag": { - "type": "string", - "description": "Git image tag (immutable tags are recommended)", - "default": "2.42.0-debian-11-r20" + "default": "REPOSITORY_NAME/git" }, "digest": { "type": "string", @@ -337,20 +327,27 @@ "description": "Enabled mlflow pods' Security Context", "default": true }, + "fsGroupChangePolicy": { + "type": "string", + "description": "Set filesystem group change policy", + "default": "Always" + }, + "sysctls": { + "type": "array", + "description": "Set kernel settings using the sysctl interface", + "default": [], + "items": {} + }, + "supplementalGroups": { + "type": "array", + "description": "Set filesystem extra groups", + "default": [], + "items": {} + }, "fsGroup": { "type": "number", "description": "Set mlflow pod's Security Context fsGroup", "default": 1001 - }, - "seccompProfile": { - "type": "object", - "properties": { - "type": { - "type": "string", - "description": "Set container's Security Context seccomp profile", - "default": "RuntimeDefault" - } - } } } }, @@ -372,6 +369,11 @@ "description": "Set containers' Security Context runAsGroup", "default": 1001 }, + "privileged": { + "type": "boolean", + "description": "Set containers' Security Context privileged", + "default": false + }, "runAsNonRoot": { "type": "boolean", "description": "Set containers' Security Context runAsNonRoot", @@ -401,6 +403,16 @@ } } } + }, + "seccompProfile": { + "type": "object", + "properties": { + "type": { + "type": "string", + "description": "Set container's Security Context seccomp profile", + "default": "RuntimeDefault" + } + } } } }, @@ -507,6 +519,11 @@ "description": "Add an init container to run mlflow db upgrade", "default": false }, + "automountServiceAccountToken": { + "type": "boolean", + "description": "Mount Service Account token in pod", + "default": false + }, "hostAliases": { "type": "array", "description": "mlflow pods host aliases", @@ -542,9 +559,9 @@ "default": false }, "minAvailable": { - "type": "number", + "type": "string", "description": "Minimum number/percentage of pods that should remain scheduled", - "default": 1 + "default": "1" }, "maxUnavailable": { "type": "string", @@ -847,11 +864,6 @@ "description": "Ingress path type", "default": "ImplementationSpecific" }, - "apiVersion": { - "type": "string", - "description": "Force Ingress API version (automatically detected if not set)", - "default": "" - }, "hostname": { "type": "string", "description": "Default host for the ingress record", @@ -920,23 +932,28 @@ "enabled": { "type": "boolean", "description": "Enable creation of NetworkPolicy resources", - "default": false + "default": true }, "allowExternal": { "type": "boolean", "description": "The Policy model to apply", "default": true }, + "allowExternalEgress": { + "type": "boolean", + "description": "Allow the pod to access any range of port and all destinations.", + "default": true + }, "extraIngress": { "type": "array", "description": "Add extra ingress rules to the NetworkPolicy", - "default": [], + "default": "[]", "items": {} }, "extraEgress": { "type": "array", "description": "Add extra ingress rules to the NetworkPolicy", - "default": [], + "default": "[]", "items": {} } } @@ -1039,26 +1056,6 @@ "description": "Enable the export of Prometheus metrics", "default": false }, - "annotations": { - "type": "object", - "properties": { - "prometheus": { - "type": "object", - "properties": { - "io/scrape": { - "type": "string", - "description": "", - "default": "true" - }, - "io/port": { - "type": "string", - "description": "", - "default": "{{ .Values.tracking.service.ports.http }}" - } - } - } - } - }, "serviceMonitor": { "type": "object", "properties": { @@ -1329,20 +1326,27 @@ "description": "Enabled Run pods' Security Context", "default": true }, + "fsGroupChangePolicy": { + "type": "string", + "description": "Set filesystem group change policy", + "default": "Always" + }, + "sysctls": { + "type": "array", + "description": "Set kernel settings using the sysctl interface", + "default": [], + "items": {} + }, + "supplementalGroups": { + "type": "array", + "description": "Set filesystem extra groups", + "default": [], + "items": {} + }, "fsGroup": { "type": "number", "description": "Set Run pod's Security Context fsGroup", "default": 1001 - }, - "seccompProfile": { - "type": "object", - "properties": { - "type": { - "type": "string", - "description": "Set Run container's Security Context seccomp profile", - "default": "RuntimeDefault" - } - } } } }, @@ -1369,6 +1373,11 @@ "description": "Set Run containers' Security Context runAsNonRoot", "default": true }, + "privileged": { + "type": "boolean", + "description": "Set Run containers' Security Context privileged", + "default": false + }, "readOnlyRootFilesystem": { "type": "boolean", "description": "Set Run containers' Security Context runAsNonRoot", @@ -1393,6 +1402,16 @@ } } } + }, + "seccompProfile": { + "type": "object", + "properties": { + "type": { + "type": "string", + "description": "Set Run container's Security Context seccomp profile", + "default": "RuntimeDefault" + } + } } } }, @@ -1406,6 +1425,11 @@ "description": "Name of the runtime class to be used by pod(s)", "default": "" }, + "automountServiceAccountToken": { + "type": "boolean", + "description": "Mount Service Account token in pod", + "default": false + }, "hostAliases": { "type": "array", "description": "run pods host aliases", @@ -1534,6 +1558,38 @@ "default": [], "items": {} }, + "networkPolicy": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "description": "Enable creation of NetworkPolicy resources", + "default": true + }, + "allowExternal": { + "type": "boolean", + "description": "The Policy model to apply", + "default": true + }, + "allowExternalEgress": { + "type": "boolean", + "description": "Allow the pod to access any range of port and all destinations.", + "default": true + }, + "extraIngress": { + "type": "array", + "description": "Add extra ingress rules to the NetworkPolicy", + "default": "[]", + "items": {} + }, + "extraEgress": { + "type": "array", + "description": "Add extra ingress rules to the NetworkPolicy", + "default": "[]", + "items": {} + } + } + }, "source": { "type": "object", "properties": { @@ -1586,7 +1642,7 @@ "create": { "type": "boolean", "description": "Enable creation of ServiceAccount for Run pods", - "default": false + "default": true }, "name": { "type": "string", @@ -1686,17 +1742,12 @@ "registry": { "type": "string", "description": "OS Shell + Utility image registry", - "default": "docker.io" + "default": "REGISTRY_NAME" }, "repository": { "type": "string", "description": "OS Shell + Utility image repository", - "default": "bitnami/os-shell" - }, - "tag": { - "type": "string", - "description": "OS Shell + Utility image tag (immutable tags are recommended)", - "default": "11-debian-11-r81" + "default": "REPOSITORY_NAME/os-shell" }, "pullPolicy": { "type": "string", @@ -1752,17 +1803,12 @@ "registry": { "type": "string", "description": "Init container wait-container image registry", - "default": "docker.io" + "default": "REGISTRY_NAME" }, "repository": { "type": "string", "description": "Init container wait-container image name", - "default": "bitnami/os-shell" - }, - "tag": { - "type": "string", - "description": "Init container wait-container image tag", - "default": "11-debian-11-r83" + "default": "REPOSITORY_NAME/os-shell" }, "digest": { "type": "string", @@ -1777,8 +1823,10 @@ "pullSecrets": { "type": "array", "description": "Specify docker-registry secret names as an array", - "default": [], - "items": {} + "default": "[]", + "items": { + "type": "string" + } } } }, @@ -1787,27 +1835,32 @@ "properties": { "enabled": { "type": "boolean", - "description": "Enabled APISIX containers' Security Context", + "description": "Enabled containers' Security Context", "default": true }, "runAsUser": { "type": "number", - "description": "Set APISIX containers' Security Context runAsUser", + "description": "Set containers' Security Context runAsUser", "default": 1001 }, "runAsNonRoot": { "type": "boolean", - "description": "Set APISIX containers' Security Context runAsNonRoot", + "description": "Set containers' Security Context runAsNonRoot", "default": true }, + "privileged": { + "type": "boolean", + "description": "Set containers' Security Context privileged", + "default": false + }, "readOnlyRootFilesystem": { "type": "boolean", - "description": "Set APISIX containers' Security Context runAsNonRoot", + "description": "Set containers' Security Context runAsNonRoot", "default": true }, "allowPrivilegeEscalation": { "type": "boolean", - "description": "Set APISIX container's privilege escalation", + "description": "Set container's privilege escalation", "default": false }, "capabilities": { @@ -1815,7 +1868,7 @@ "properties": { "drop": { "type": "array", - "description": "Set APISIX container's Security Context runAsNonRoot", + "description": "Set container's Security Context runAsNonRoot", "default": [ "ALL" ], @@ -1824,6 +1877,16 @@ } } } + }, + "seccompProfile": { + "type": "object", + "properties": { + "type": { + "type": "string", + "description": "Set container's Security Context seccomp profile", + "default": "RuntimeDefault" + } + } } } } @@ -1884,26 +1947,6 @@ } } } - }, - "initdb": { - "type": "object", - "properties": { - "scripts": { - "type": "object", - "properties": { - "create_auth_db": { - "type": "object", - "properties": { - "sh": { - "type": "string", - "description": "", - "default": "#!/bin/bash\nPGPASSWORD=$POSTGRES_POSTGRES_PASSWORD psql -U postgres <<< \"CREATE DATABASE {{ include \"mlflow.v0.database-auth.name\" . }}\"\nPGPASSWORD=$POSTGRES_POSTGRES_PASSWORD psql -U postgres <<< \"GRANT ALL PRIVILEGES ON DATABASE {{ include \"mlflow.v0.database-auth.name\" . }} to {{ .Values.auth.username }}\"\nPGPASSWORD=$POSTGRES_POSTGRES_PASSWORD psql -U postgres <<< \"ALTER DATABASE {{ include \"mlflow.v0.database-auth.name\" . }} OWNER TO {{ .Values.auth.username }}\"\n" - } - } - } - } - } - } } } } @@ -1912,6 +1955,12 @@ "externalDatabase": { "type": "object", "properties": { + "dialectDriver": { + "type": "string", + "description": "Database dialect(+driver)", + "default": "postgresql", + "pattern": "^(mysql|mssql|sqlite|postgresql)(\\+\\w+)?$" + }, "host": { "type": "string", "description": "Database host", @@ -2057,6 +2106,11 @@ "description": "External S3 port number", "default": 443 }, + "useCredentialsInSecret": { + "type": "boolean", + "description": "Whether to use a secret to store the S3 credentials", + "default": true + }, "accessKeyID": { "type": "string", "description": "External S3 access key ID", @@ -2091,8 +2145,13 @@ "type": "string", "description": "External S3 bucket", "default": "mlflow" + }, + "serveArtifacts": { + "type": "boolean", + "description": "Whether artifact serving is enabled", + "default": true } } } } -} \ No newline at end of file +} diff --git a/charts/mlflow/values.yaml b/charts/mlflow/values.yaml index 13281ff..0085e37 100644 --- a/charts/mlflow/values.yaml +++ b/charts/mlflow/values.yaml @@ -1,4 +1,4 @@ -# Copyright VMware, Inc. +# Copyright Broadcom, Inc. All Rights Reserved. # SPDX-License-Identifier: APACHE-2.0 ## @section Global parameters @@ -9,7 +9,8 @@ ## @param global.imageRegistry Global Docker image registry ## @param global.imagePullSecrets Global Docker registry secret names as an array -## @param global.storageClass Global StorageClass for Persistent Volume(s) +## @param global.defaultStorageClass Global default StorageClass for Persistent Volume(s) +## @param global.storageClass DEPRECATED: use global.defaultStorageClass instead ## global: imageRegistry: "" @@ -18,8 +19,17 @@ global: ## - myRegistryKeySecretName ## imagePullSecrets: [] + defaultStorageClass: "" storageClass: "" - + ## Compatibility adaptations for Kubernetes platforms + ## + compatibility: + ## Compatibility adaptations for Openshift + ## + openshift: + ## @param global.compatibility.openshift.adaptSecurityContext Adapt the securityContext sections of the deployment to make them compatible with Openshift restricted-v2 SCC: remove runAsUser, runAsGroup and fsGroup and let the platform use their allowed default IDs. Possible values: auto (apply if the detected running cluster is Openshift), force (perform the adaptation always), disabled (do not perform adaptation) + ## + adaptSecurityContext: auto ## @section Common parameters ## @@ -43,11 +53,10 @@ commonLabels: {} commonAnnotations: {} ## @param clusterDomain Kubernetes cluster domain name ## -clusterDomain: "2.rahtiapp.fi" +clusterDomain: 2.rahtiapp.fi ## @param extraDeploy Array of extra objects to deploy with the release ## extraDeploy: [] - ## Enable diagnostic mode in the deployment ## diagnosticMode: @@ -62,7 +71,6 @@ diagnosticMode: ## args: - infinity - ## @section MLflow common Parameters ## @@ -79,11 +87,11 @@ diagnosticMode: image: registry: docker.io repository: bitnami/mlflow - tag: 2.9.2-debian-11-r0 + tag: 2.16.2-debian-12-r3 digest: "" ## Specify a imagePullPolicy ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' - ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## ref: https://kubernetes.io/docs/concepts/containers/images/#pre-pulled-images ## pullPolicy: IfNotPresent ## Optionally specify an array of imagePullSecrets. @@ -97,7 +105,6 @@ image: ## Enable debug mode ## debug: false - ## Bitnami git image version ## ref: https://hub.docker.com/r/bitnami/git/tags/ ## @param gitImage.registry [default: REGISTRY_NAME] Git image registry @@ -110,7 +117,7 @@ image: gitImage: registry: docker.io repository: bitnami/git - tag: 2.43.0-debian-11-r1 + tag: 2.46.1-debian-12-r1 digest: "" pullPolicy: IfNotPresent ## Optionally specify an array of imagePullSecrets. @@ -121,7 +128,6 @@ gitImage: ## - myRegistryKeySecretName ## pullSecrets: [] - ## @section MLflow Tracking parameters ## tracking: @@ -189,24 +195,40 @@ tracking: ## customStartupProbe: {} ## mlflow resource requests and limits - ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ - ## @param tracking.resources.limits The resources limits for the mlflow containers - ## @param tracking.resources.requests The requested resources for the mlflow containers + ## ref: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + ## @param tracking.resourcesPreset Set container resources according to one common preset (allowed values: none, nano, micro, small, medium, large, xlarge, 2xlarge). This is ignored if tracking.resources is set (tracking.resources is recommended for production). + ## More information: https://github.com/bitnami/charts/blob/main/bitnami/common/templates/_resources.tpl#L15 ## - resources: - limits: {} - requests: {} + resourcesPreset: "medium" + ## @param tracking.resources Set container requests and limits for different resources like CPU or memory (essential for production workloads) + ## Example: + ## resources: + ## requests: + ## cpu: 2 + ## memory: 512Mi + ## limits: + ## cpu: 3 + ## memory: 1024Mi + ## + resources: {} ## Configure Pods Security Context ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod ## @param tracking.podSecurityContext.enabled Enabled mlflow pods' Security Context + ## @param tracking.podSecurityContext.fsGroupChangePolicy Set filesystem group change policy + ## @param tracking.podSecurityContext.sysctls Set kernel settings using the sysctl interface + ## @param tracking.podSecurityContext.supplementalGroups Set filesystem extra groups ## @param tracking.podSecurityContext.fsGroup Set mlflow pod's Security Context fsGroup ## podSecurityContext: - enabled: false + enabled: true + fsGroupChangePolicy: Always + sysctls: [] + supplementalGroups: [] fsGroup: 1001 ## Configure Container Security Context ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container ## @param tracking.containerSecurityContext.enabled Enabled containers' Security Context + ## @param tracking.containerSecurityContext.seLinuxOptions [object,nullable] Set SELinux options in container ## @param tracking.containerSecurityContext.runAsUser Set containers' Security Context runAsUser ## @param tracking.containerSecurityContext.runAsGroup Set containers' Security Context runAsGroup ## @param tracking.containerSecurityContext.privileged Set containers' Security Context privileged @@ -217,7 +239,8 @@ tracking: ## @param tracking.containerSecurityContext.seccompProfile.type Set container's Security Context seccomp profile ## containerSecurityContext: - enabled: false + enabled: true + seLinuxOptions: {} runAsUser: 1001 runAsGroup: 1001 runAsNonRoot: true @@ -228,7 +251,6 @@ tracking: drop: ["ALL"] seccompProfile: type: "RuntimeDefault" - ## Basic authentication ## @param tracking.auth.enabled Enable basic authentication ## @param tracking.auth.username Admin username @@ -282,6 +304,9 @@ tracking: ## @param tracking.runUpgradeDB Add an init container to run mlflow db upgrade ## runUpgradeDB: false + ## @param tracking.automountServiceAccountToken Mount Service Account token in pod + ## + automountServiceAccountToken: false ## @param tracking.hostAliases mlflow pods host aliases ## https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/ ## @@ -306,12 +331,11 @@ tracking: ## ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb ## @param tracking.pdb.create Enable/disable a Pod Disruption Budget creation ## @param tracking.pdb.minAvailable Minimum number/percentage of pods that should remain scheduled - ## @param tracking.pdb.maxUnavailable Maximum number/percentage of pods that may be made unavailable ## pdb: - create: false - minAvailable: 1 + create: true + minAvailable: "" maxUnavailable: "" ## Autoscaling configuration ## ref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/ @@ -379,7 +403,7 @@ tracking: ## affinity: {} ## @param tracking.nodeSelector Node labels for mlflow pods assignment - ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/assign-pods-nodes/ ## nodeSelector: {} ## @param tracking.tolerations Tolerations for mlflow pods assignment @@ -394,7 +418,6 @@ tracking: ## Can be set to RollingUpdate or OnDelete ## type: RollingUpdate - ## @param tracking.priorityClassName mlflow pods' priorityClassName ## priorityClassName: "" @@ -456,7 +479,6 @@ tracking: ## command: ['sh', '-c', 'echo "hello world"'] ## initContainers: [] - ## @section MLflow Tracking Traffic Exposure Parameters ## @@ -511,7 +533,7 @@ tracking: extraPorts: [] ## @param tracking.service.sessionAffinity Control where client requests go, to the same pod or round-robin ## Values: ClientIP or None - ## ref: https://kubernetes.io/docs/user-guide/services/ + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/ ## sessionAffinity: None ## @param tracking.service.sessionAffinityConfig Additional settings for the sessionAffinity @@ -521,7 +543,7 @@ tracking: ## sessionAffinityConfig: {} ## mlflow ingress parameters - ## ref: http://kubernetes.io/docs/user-guide/ingress/ + ## ref: https://kubernetes.io/docs/concepts/services-networking/ingress/ ## ingress: ## @param tracking.ingress.enabled Enable ingress record generation for mlflow @@ -621,19 +643,21 @@ tracking: ## name: http ## extraRules: [] - ## Network Policy configuration ## ref: https://kubernetes.io/docs/concepts/services-networking/network-policies/ ## networkPolicy: ## @param tracking.networkPolicy.enabled Enable creation of NetworkPolicy resources ## - enabled: false + enabled: true ## @param tracking.networkPolicy.allowExternal The Policy model to apply - ## When set to false, only pods with the correct client label will have network access to the ports Keycloak is - ## listening on. When true, Keycloak will accept connections from any source (with the correct destination port). + ## When set to false, only pods with the correct client label will have network access to the ports MLFlow is + ## listening on. When true, MLFlow will accept connections from any source (with the correct destination port). ## allowExternal: true + ## @param tracking.networkPolicy.allowExternalEgress Allow the pod to access any range of port and all destinations. + ## + allowExternalEgress: true ## @param tracking.networkPolicy.extraIngress [array] Add extra ingress rules to the NetworkPolicy ## e.g: ## extraIngress: @@ -673,12 +697,11 @@ tracking: ## ingressNSMatchLabels: {} ingressNSPodMatchLabels: {} - ## @section MLflow Tracking Persistence Parameters ## ## Enable persistence using Persistent Volume Claims - ## ref: https://kubernetes.io/docs/user-guide/persistent-volumes/ + ## ref: https://kubernetes.io/docs/concepts/storage/persistent-volumes/ ## persistence: ## @param tracking.persistence.enabled Enable persistence using Persistent Volume Claims @@ -714,7 +737,7 @@ tracking: ## @param tracking.persistence.existingClaim The name of an existing PVC to use for persistence ## existingClaim: "" - ## @param tracking.persistence.selector Selector to match an existing Persistent Volume for WordPress data PVC + ## @param tracking.persistence.selector Selector to match an existing Persistent Volume for MLflow data PVC ## If set, the PVC can't have a PV dynamically provisioned for it ## E.g. ## selector: @@ -725,7 +748,6 @@ tracking: ## @param tracking.persistence.dataSource Custom PVC data source ## dataSource: {} - ## @section MLflow Tracking Other Parameters ## serviceAccount: @@ -742,7 +764,6 @@ tracking: ## @param tracking.serviceAccount.automountServiceAccountToken Automount service account token for the server service account ## automountServiceAccountToken: false - ## @section MLflow Tracking Metrics Parameters ## metrics: @@ -798,7 +819,6 @@ tracking: ## prometheus: my-prometheus ## selector: {} - ## @section MLflow Run Parameters ## run: @@ -839,50 +859,6 @@ run: ## @param run.terminationGracePeriodSeconds Run termination grace period (in seconds) ## terminationGracePeriodSeconds: "" - ## Configure extra options for Run containers' liveness, readiness and startup probes - ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes - ## @param run.livenessProbe.enabled Enable livenessProbe on Run nodes - ## @param run.livenessProbe.initialDelaySeconds Initial delay seconds for livenessProbe - ## @param run.livenessProbe.periodSeconds Period seconds for livenessProbe - ## @param run.livenessProbe.timeoutSeconds Timeout seconds for livenessProbe - ## @param run.livenessProbe.failureThreshold Failure threshold for livenessProbe - ## @param run.livenessProbe.successThreshold Success threshold for livenessProbe - ## - livenessProbe: - enabled: true - initialDelaySeconds: 5 - periodSeconds: 30 - timeoutSeconds: 20 - failureThreshold: 5 - successThreshold: 1 - ## @param run.readinessProbe.enabled Enable readinessProbe on Run nodes - ## @param run.readinessProbe.initialDelaySeconds Initial delay seconds for readinessProbe - ## @param run.readinessProbe.periodSeconds Period seconds for readinessProbe - ## @param run.readinessProbe.timeoutSeconds Timeout seconds for readinessProbe - ## @param run.readinessProbe.failureThreshold Failure threshold for readinessProbe - ## @param run.readinessProbe.successThreshold Success threshold for readinessProbe - ## - readinessProbe: - enabled: true - initialDelaySeconds: 5 - periodSeconds: 30 - timeoutSeconds: 20 - failureThreshold: 5 - successThreshold: 1 - ## @param run.startupProbe.enabled Enable startupProbe on Run containers - ## @param run.startupProbe.initialDelaySeconds Initial delay seconds for startupProbe - ## @param run.startupProbe.periodSeconds Period seconds for startupProbe - ## @param run.startupProbe.timeoutSeconds Timeout seconds for startupProbe - ## @param run.startupProbe.failureThreshold Failure threshold for startupProbe - ## @param run.startupProbe.successThreshold Success threshold for startupProbe - ## - startupProbe: - enabled: false - initialDelaySeconds: 5 - periodSeconds: 30 - timeoutSeconds: 5 - failureThreshold: 5 - successThreshold: 1 ## @param run.customLivenessProbe Custom livenessProbe that overrides the default one ## customLivenessProbe: {} @@ -893,24 +869,40 @@ run: ## customStartupProbe: {} ## run resource requests and limits - ## ref: https://kubernetes.io/docs/user-guide/compute-resources/ - ## @param run.resources.limits The resources limits for the run containers - ## @param run.resources.requests The requested resources for the run containers + ## ref: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + ## @param run.resourcesPreset Set container resources according to one common preset (allowed values: none, nano, micro, small, medium, large, xlarge, 2xlarge). This is ignored if run.resources is set (run.resources is recommended for production). + ## More information: https://github.com/bitnami/charts/blob/main/bitnami/common/templates/_resources.tpl#L15 ## - resources: - limits: {} - requests: {} + resourcesPreset: "small" + ## @param run.resources Set container requests and limits for different resources like CPU or memory (essential for production workloads) + ## Example: + ## resources: + ## requests: + ## cpu: 2 + ## memory: 512Mi + ## limits: + ## cpu: 3 + ## memory: 1024Mi + ## + resources: {} ## Configure Pods Security Context ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod ## @param run.podSecurityContext.enabled Enabled Run pods' Security Context + ## @param run.podSecurityContext.fsGroupChangePolicy Set filesystem group change policy + ## @param run.podSecurityContext.sysctls Set kernel settings using the sysctl interface + ## @param run.podSecurityContext.supplementalGroups Set filesystem extra groups ## @param run.podSecurityContext.fsGroup Set Run pod's Security Context fsGroup ## podSecurityContext: - enabled: false + enabled: true + fsGroupChangePolicy: Always + sysctls: [] + supplementalGroups: [] fsGroup: 1001 ## Configure Container Security Context ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod ## @param run.containerSecurityContext.enabled Enabled Run containers' Security Context + ## @param run.containerSecurityContext.seLinuxOptions [object,nullable] Set SELinux options in container ## @param run.containerSecurityContext.runAsUser Set Run containers' Security Context runAsUser ## @param run.containerSecurityContext.runAsGroup Set Run containers' Security Context runAsGroup ## @param run.containerSecurityContext.runAsNonRoot Set Run containers' Security Context runAsNonRoot @@ -921,7 +913,8 @@ run: ## @param run.containerSecurityContext.seccompProfile.type Set Run container's Security Context seccomp profile ## containerSecurityContext: - enabled: false + enabled: true + seLinuxOptions: {} runAsUser: 1001 runAsGroup: 1001 runAsNonRoot: true @@ -939,6 +932,9 @@ run: ## ref: https://kubernetes.io/docs/concepts/containers/runtime-class/ ##https://github.com/microsoft/MlflowExamples runtimeClassName: "" + ## @param run.automountServiceAccountToken Mount Service Account token in pod + ## + automountServiceAccountToken: false ## @param run.hostAliases run pods host aliases ## https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/ ## @@ -986,7 +982,7 @@ run: ## affinity: {} ## @param run.nodeSelector Node labels for Run pods assignment - ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/assign-pods-nodes/ ## nodeSelector: {} ## @param run.tolerations Tolerations for Run pods assignment @@ -1041,6 +1037,60 @@ run: ## command: ['sh', '-c', 'echo "hello world"'] ## initContainers: [] + ## Network Policy configuration + ## ref: https://kubernetes.io/docs/concepts/services-networking/network-policies/ + ## + networkPolicy: + ## @param run.networkPolicy.enabled Enable creation of NetworkPolicy resources + ## + enabled: true + ## @param run.networkPolicy.allowExternal The Policy model to apply + ## When set to false, only pods with the correct client label will have network access to the ports MLFlow is + ## listening on. When true, MLFlow will accept connections from any source (with the correct destination port). + ## + allowExternal: true + ## @param run.networkPolicy.allowExternalEgress Allow the pod to access any range of port and all destinations. + ## + allowExternalEgress: true + ## @param run.networkPolicy.extraIngress [array] Add extra ingress rules to the NetworkPolicy + ## e.g: + ## extraIngress: + ## - ports: + ## - port: 1234 + ## from: + ## - podSelector: + ## - matchLabels: + ## - role: frontend + ## - podSelector: + ## - matchExpressions: + ## - key: role + ## operator: In + ## values: + ## - frontend + ## + extraIngress: [] + ## @param run.networkPolicy.extraEgress [array] Add extra ingress rules to the NetworkPolicy + ## e.g: + ## extraEgress: + ## - ports: + ## - port: 1234 + ## to: + ## - podSelector: + ## - matchLabels: + ## - role: frontend + ## - podSelector: + ## - matchExpressions: + ## - key: role + ## operator: In + ## values: + ## - frontend + ## + extraEgress: [] + ## @param run.networkPolicy.ingressNSMatchLabels [object] Labels to match to allow traffic from other namespaces + ## @param run.networkPolicy.ingressNSPodMatchLabels [object] Pod labels to match to allow traffic from other namespaces + ## + ingressNSMatchLabels: {} + ingressNSPodMatchLabels: {} ## Source code parameters ## source: @@ -1071,14 +1121,13 @@ run: ## mountPath: /.ssh/ ## extraVolumeMounts: [] - ## Service account for Run to use ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ ## serviceAccount: ## @param run.serviceAccount.create Enable creation of ServiceAccount for Run pods ## - create: false + create: true ## @param run.serviceAccount.name The name of the ServiceAccount to use ## If not set and create is true, a name is generated using the common.names.fullname template ## @@ -1092,7 +1141,7 @@ run: annotations: {} ## @section Mlflow Run persistence paramaters ## Enable persistence using Persistent Volume Claims - ## ref: https://kubernetes.io/docs/user-guide/persistent-volumes/ + ## ref: https://kubernetes.io/docs/concepts/storage/persistent-volumes/ ## persistence: ## @param run.persistence.enabled Use a PVC to persist data @@ -1139,7 +1188,6 @@ run: ## @param run.persistence.annotations Persistent Volume annotations ## annotations: {} - ## @section Init Container Parameters ## @@ -1162,7 +1210,7 @@ volumePermissions: image: registry: docker.io repository: bitnami/os-shell - tag: 11-debian-11-r92 + tag: 12-debian-12-r30 pullPolicy: IfNotPresent ## Optionally specify an array of imagePullSecrets. ## Secrets must be manually created in the namespace. @@ -1173,25 +1221,35 @@ volumePermissions: ## pullSecrets: [] ## Init container's resource requests and limits - ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ - ## @param volumePermissions.resources.limits The resources limits for the init container - ## @param volumePermissions.resources.requests The requested resources for the init container + ## ref: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + ## @param volumePermissions.resourcesPreset Set container resources according to one common preset (allowed values: none, nano, micro, small, medium, large, xlarge, 2xlarge). This is ignored if volumePermissions.resources is set (volumePermissions.resources is recommended for production). + ## More information: https://github.com/bitnami/charts/blob/main/bitnami/common/templates/_resources.tpl#L15 ## - resources: - limits: {} - requests: {} + resourcesPreset: "nano" + ## @param volumePermissions.resources Set container requests and limits for different resources like CPU or memory (essential for production workloads) + ## Example: + ## resources: + ## requests: + ## cpu: 2 + ## memory: 512Mi + ## limits: + ## cpu: 3 + ## memory: 1024Mi + ## + resources: {} ## Init container Container Security Context ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container ## @param volumePermissions.containerSecurityContext.enabled Set container security context settings + ## @param volumePermissions.containerSecurityContext.seLinuxOptions [object,nullable] Set SELinux options in container ## @param volumePermissions.containerSecurityContext.runAsUser Set init container's Security Context runAsUser ## NOTE: when runAsUser is set to special value "auto", init container will try to chown the ## data folder to auto-determined user&group, using commands: `id -u`:`id -G | cut -d" " -f2` ## "auto" is especially useful for OpenShift which has scc with dynamic user ids (and 0 is not allowed) ## containerSecurityContext: - enabled: false + enabled: true + seLinuxOptions: {} runAsUser: 0 - waitContainer: ## @param waitContainer.image.registry [default: REGISTRY_NAME] Init container wait-container image registry ## @param waitContainer.image.repository [default: REPOSITORY_NAME/os-shell] Init container wait-container image name @@ -1201,7 +1259,7 @@ waitContainer: image: registry: docker.io repository: bitnami/os-shell - tag: 11-debian-11-r92 + tag: 12-debian-12-r30 digest: "" ## @param waitContainer.image.pullPolicy Init container wait-container image pull policy ## @@ -1215,11 +1273,12 @@ waitContainer: ## - myRegistryKeySecretName ## pullSecrets: [] - ## Configure Container Security Context ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container ## @param waitContainer.containerSecurityContext.enabled Enabled containers' Security Context + ## @param waitContainer.containerSecurityContext.seLinuxOptions [object,nullable] Set SELinux options in container ## @param waitContainer.containerSecurityContext.runAsUser Set containers' Security Context runAsUser + ## @param waitContainer.containerSecurityContext.runAsGroup Set containers' Security Context runAsGroup ## @param waitContainer.containerSecurityContext.runAsNonRoot Set containers' Security Context runAsNonRoot ## @param waitContainer.containerSecurityContext.privileged Set containers' Security Context privileged ## @param waitContainer.containerSecurityContext.readOnlyRootFilesystem Set containers' Security Context runAsNonRoot @@ -1228,8 +1287,10 @@ waitContainer: ## @param waitContainer.containerSecurityContext.seccompProfile.type Set container's Security Context seccomp profile ## containerSecurityContext: - enabled: false + enabled: true + seLinuxOptions: {} runAsUser: 1001 + runAsGroup: 1001 runAsNonRoot: true privileged: false readOnlyRootFilesystem: true @@ -1238,7 +1299,6 @@ waitContainer: drop: ["ALL"] seccompProfile: type: "RuntimeDefault" - ## @section PostgreSQL chart configuration ## ref: https://github.com/bitnami/charts/blob/main/bitnami/postgresql/values.yaml ## @param postgresql.enabled Switch to enable or disable the PostgreSQL helm chart @@ -1250,7 +1310,16 @@ waitContainer: ## @param postgresql.primary.service.ports.postgresql PostgreSQL service port ## postgresql: - enabled: true + enabled: false + ## Compatibility adaptations for Kubernetes platforms + ## + compatibility: + ## Compatibility adaptations for Openshift + ## + openshift: + ## @param global.compatibility.openshift.adaptSecurityContext Adapt the securityContext sections of the deployment to make them compatible with Openshift restricted-v2 SCC: remove runAsUser, runAsGroup and fsGroup and let the platform use their allowed default IDs. Possible values: auto (apply if the detected running cluster is Openshift), force (perform the adaptation always), disabled (do not perform adaptation) + ## + adaptSecurityContext: auto auth: username: mlflow password: "" @@ -1258,17 +1327,6 @@ postgresql: existingSecret: "" architecture: standalone primary: - podSecurityContext: - enabled: false - containerSecurityContext: - enabled: false - runAsNonRoot: true - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - seccompProfile: - type: RuntimeDefault service: ports: postgresql: 5432 @@ -1281,47 +1339,32 @@ postgresql: PGPASSWORD=$POSTGRES_POSTGRES_PASSWORD psql -U postgres <<< "CREATE DATABASE {{ include "mlflow.v0.database-auth.name" . }}" PGPASSWORD=$POSTGRES_POSTGRES_PASSWORD psql -U postgres <<< "GRANT ALL PRIVILEGES ON DATABASE {{ include "mlflow.v0.database-auth.name" . }} to {{ .Values.auth.username }}" PGPASSWORD=$POSTGRES_POSTGRES_PASSWORD psql -U postgres <<< "ALTER DATABASE {{ include "mlflow.v0.database-auth.name" . }} OWNER TO {{ .Values.auth.username }}" + ## PostgreSQL Primary resource requests and limits + ## ref: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ + ## @param primary.resourcesPreset Set container resources according to one common preset (allowed values: none, nano, micro, small, medium, large, xlarge, 2xlarge). This is ignored if primary.resources is set (primary.resources is recommended for production). + ## More information: https://github.com/bitnami/charts/blob/main/bitnami/common/templates/_resources.tpl#L15 + ## + resourcesPreset: none + ## @param resources Set container requests and limits for different resources like CPU or memory (essential for production workloads) + ## Example: + ## resources: + ## requests: + ## cpu: 2 + ## memory: 512Mi + ## limits: + ## cpu: 3 + ## memory: 1024Mi + ## + resources: {} + ## Enable persistence using Persistent Volume Claims + ## ref: https://kubernetes.io/docs/concepts/storage/persistent-volumes/ + ## + persistence: + size: 8Gi - readReplicas: - podSecurityContext: - enabled: false - containerSecurityContext: - enabled: false - runAsNonRoot: true - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - seccompProfile: - type: RuntimeDefault - - backup: - cronjob: - podSecurityContext: - enabled: false - containerSecurityContext: - enabled: false - runAsNonRoot: true - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - seccompProfile: - type: RuntimeDefault - - metrics: - containerSecurityContext: - enabled: false - runAsNonRoot: true - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - seccompProfile: - type: RuntimeDefault - ## @section External PostgreSQL configuration ## All of these values are only used when postgresql.enabled is set to false +## @param externalDatabase.dialectDriver Database Dialect(+Driver) ## @param externalDatabase.host Database host ## @param externalDatabase.port Database port number ## @param externalDatabase.user Non-root username @@ -1332,15 +1375,16 @@ postgresql: ## @param externalDatabase.existingSecretPasswordKey Name of an existing secret key containing the database credentials ## externalDatabase: - host: postgresql-svc + dialectDriver: "postgresql" + host: "" port: 5432 user: mlflow - database: mlflow_db + database: mlflow authDatabase: mlflow_auth password: "" - existingSecret: mlflow-externaldb + existingSecret: "" existingSecretPasswordKey: "db-password" - + ## @section MinIO® chart parameters ## @extra minio For full list of MinIO® values configurations please refere [here](https://github.com/bitnami/charts/tree/main/bitnami/minio) ## @@ -1349,6 +1393,15 @@ minio: ## to be used as an objstore for Mlflow ## enabled: false + ## Compatibility adaptations for Kubernetes platforms + ## + compatibility: + ## Compatibility adaptations for Openshift + ## + openshift: + ## @param global.compatibility.openshift.adaptSecurityContext Adapt the securityContext sections of the deployment to make them compatible with Openshift restricted-v2 SCC: remove runAsUser, runAsGroup and fsGroup and let the platform use their allowed default IDs. Possible values: auto (apply if the detected running cluster is Openshift), force (perform the adaptation always), disabled (do not perform adaptation) + ## + adaptSecurityContext: auto ## MinIO® authentication parameters ## auth: @@ -1364,7 +1417,6 @@ minio: ## @param minio.defaultBuckets Comma, semi-colon or space separated list of MinIO® buckets to create ## defaultBuckets: "mlflow" - ## @param minio.provisioning.enabled Enable/disable MinIO® provisioning job ## @param minio.provisioning.extraCommands Extra commands to run on MinIO® provisioning job ## @@ -1372,7 +1424,27 @@ minio: enabled: true # We need to allow downloads in order for the UI to work extraCommands: ["mc anonymous set download provisioning/mlflow"] - + ## MinIO® containers' resource requests and limits + ## ref: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ + ## We usually recommend not to specify default resources and to leave this as a conscious + ## choice for the user. This also increases chances charts run on environments with little + ## resources, such as Minikube. If you do want to specify resources, uncomment the following + ## lines, adjust them as necessary, and remove the curly braces after 'resources:'. + ## @param resourcesPreset Set container resources according to one common preset (allowed values: none, nano, micro, small, medium, large, xlarge, 2xlarge). This is ignored if resources is set (resources is recommended for production). + ## More information: https://github.com/bitnami/charts/blob/main/bitnami/common/templates/_resources.tpl#L15 + ## + resourcesPreset: "none" + ## @param resources Set container requests and limits for different resources like CPU or memory (essential for production workloads) + ## Example: + ## resources: + ## requests: + ## cpu: 2 + ## memory: 512Mi + ## limits: + ## cpu: 3 + ## memory: 1024Mi + ## + resources: {} ## @param minio.tls.enabled Enable/disable MinIO® TLS support ## tls: @@ -1386,11 +1458,40 @@ minio: loadBalancerIP: "" ports: api: 80 - + ## MinIO® containers' resource requests and limits + ## ref: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ + ## We usually recommend not to specify default resources and to leave this as a conscious + ## choice for the user. This also increases chances charts run on environments with little + ## resources, such as Minikube. If you do want to specify resources, uncomment the following + ## lines, adjust them as necessary, and remove the curly braces after 'resources:'. + ## @param resourcesPreset Set container resources according to one common preset (allowed values: none, nano, micro, small, medium, large, xlarge, 2xlarge). This is ignored if resources is set (resources is recommended for production). + ## More information: https://github.com/bitnami/charts/blob/main/bitnami/common/templates/_resources.tpl#L15 + ## + resourcesPreset: "none" + ## @param resources Set container requests and limits for different resources like CPU or memory (essential for production workloads) + ## Example: + ## resources: + ## requests: + ## cpu: 2 + ## memory: 512Mi + ## limits: + ## cpu: 3 + ## memory: 1024Mi + ## + resources: {} + ## Enable persistence using Persistent Volume Claims + ## ref: https://kubernetes.io/docs/concepts/storage/persistent-volumes/ + ## + persistence: + ## @param persistence.size PVC Storage Request for MinIO® data volume + ## + size: 8Gi + ## @section External S3 parameters ## All of these values are only used when minio.enabled is set to false -## @param externalS3.host External S3 host +## @param externalS3.host External S3 host. When using AWS S3, include appropriate [regional code](https://docs.aws.amazon.com/general/latest/gr/s3.html#s3_region), e.g. "eu-central-1.amazonaws.com ## @param externalS3.port External S3 port number +## @param externalS3.useCredentialsInSecret Whether to use a secret to store the S3 credentials ## @param externalS3.accessKeyID External S3 access key ID ## @param externalS3.accessKeySecret External S3 access key secret ## @param externalS3.existingSecret Name of an existing secret resource containing the S3 credentials @@ -1403,13 +1504,31 @@ minio: externalS3: host: "a3s.fi" port: 443 + useCredentialsInSecret: true accessKeyID: "" accessKeySecret: "" existingSecret: "" existingSecretAccessKeyIDKey: "root-user" existingSecretKeySecretKey: "root-password" - protocol: "s3" - bucket: "mlflow" + protocol: "https" + bucket: "" + serveArtifacts: true + +## @section External Google Cloud Storage parameters +## All of these values are only used when minio.enabled is set to false and externalS3 is not configured (host is empty) +## @param externalGCS.bucket GCS bucket name. Activate gcs artifact storage if set +## @param externalGCS.googleCloudProject Google Cloud Project to use (optional, needed when using "default application credentials") +## @param externalGCS.useCredentialsInSecret Whether to read the GCS application credentials from a secret +## @param externalGCS.existingSecret Name of an existing secret key containing the application credentials file (required when useCredentialsInSecret is true) +## @param externalGCS.existingSecretKey Key in the existing secret containing the application credentials (required when useCredentialsInSecret is true) +## @param externalGCS.serveArtifacts Whether artifact serving is enabled +## +externalGCS: + bucket: "" + googleCloudProject: "" + useCredentialsInSecret: false + existingSecret: "" + existingSecretKey: "" serveArtifacts: true route: