From 8fe0758c2679085fa96bf77c328ab6e88cdab661 Mon Sep 17 00:00:00 2001 From: ivakoleva Date: Mon, 31 Oct 2022 16:17:02 +0200 Subject: [PATCH 1/4] versatile-data-kit: postgresql embedded For embedded DB for control-service metadata storage, we need to use the Bitnami-available chart of PostgreSQL. Added postgresql `values.yaml` configurations. In case `postgresql.enabled` is `true`, PostgreSQL embedded is being prioritized for usages. It is also the default embedded database. Testing Done: ci/cd Signed-off-by: ivakoleva --- .../pipelines-control-service/Chart.yaml | 4 +++ .../pipelines-control-service/README.md | 5 ++- .../templates/_helpers.tpl | 11 +++++-- .../templates/secret_jdbc.yaml | 2 +- .../pipelines-control-service/values.yaml | 32 +++++++++++++++++-- 5 files changed, 47 insertions(+), 7 deletions(-) diff --git a/projects/control-service/projects/helm_charts/pipelines-control-service/Chart.yaml b/projects/control-service/projects/helm_charts/pipelines-control-service/Chart.yaml index 14f874bdbf..b30845d52e 100644 --- a/projects/control-service/projects/helm_charts/pipelines-control-service/Chart.yaml +++ b/projects/control-service/projects/helm_charts/pipelines-control-service/Chart.yaml @@ -19,6 +19,10 @@ dependencies: tags: - bitnami-common version: 1.x.x +- name: postgresql + version: "12.0.0" + repository: https://charts.bitnami.com/bitnami + condition: postgresql.enabled - name: cockroachdb version: "3.0.7" repository: "https://charts.helm.sh/stable" diff --git a/projects/control-service/projects/helm_charts/pipelines-control-service/README.md b/projects/control-service/projects/helm_charts/pipelines-control-service/README.md index fbbe44645a..b9a120db55 100644 --- a/projects/control-service/projects/helm_charts/pipelines-control-service/README.md +++ b/projects/control-service/projects/helm_charts/pipelines-control-service/README.md @@ -23,7 +23,9 @@ Note: integration with bitnami is being planned which will replace above repo. ## Introduction This chart bootstraps a Versatile Data Kit deployment on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. -It also packages the [Cockroach chart](https://github.com/helm/charts/tree/master/stable/cockroachdb) which is required for bootstrapping a Cockroach deployment for the database requirements of the Versatile Data Kit +It also packages a requirement for bootstrapping a database deployment for Versatile Data Kit's metadata: +- [Bitnami PostgreSQL chart](https://github.com/bitnami/charts/tree/main/bitnami/postgresql) +- [Cockroach chart](https://github.com/helm/charts/tree/master/stable/cockroachdb) ## Installing the Chart To install the chart with the release name `my-release`: @@ -51,6 +53,7 @@ The command removes all the Kubernetes components associated with the chart and For a full list of configuration parameters of the Versatile Data Kit chart, and their documentation see [values.yaml](values.yaml) file. Look through it and make sure to have set all [Required] Parameters as per instruction in the property doc. +For PostgreSQL database parameters see https://github.com/bitnami/charts/tree/main/bitnami/postgresql For Cockroach database parameters see https://github.com/helm/charts/tree/master/stable/cockroachdb Specify each parameter using the `--set key=value` or `--set-file key=filePath` argument to `helm install`. For example, diff --git a/projects/control-service/projects/helm_charts/pipelines-control-service/templates/_helpers.tpl b/projects/control-service/projects/helm_charts/pipelines-control-service/templates/_helpers.tpl index b1c1380454..25e79de8d5 100644 --- a/projects/control-service/projects/helm_charts/pipelines-control-service/templates/_helpers.tpl +++ b/projects/control-service/projects/helm_charts/pipelines-control-service/templates/_helpers.tpl @@ -182,12 +182,17 @@ Create the name of the deployment Kubernetes namespace used by System and builde {{- end -}} {{/* -Generate default JDBC credentials for local CockroachDB instance. +Generate default JDBC credentials for local embedded database instance (CockroachDB or PostgreSQL). */}} {{- define "pipelines-control-service.jdbcSecret" -}} -USERNAME: {{ default "root" .Values.database.username | b64enc | quote }} +{{- if and (not .Values.postgresql.enabled) .Values.cockroachdb.enabled -}} + USERNAME: {{ default "root" .Values.database.username | b64enc | quote }} + JDBC: {{ default (printf "jdbc:postgresql://%s-cockroachdb-public:26257/defaultdb?sslmode=disable" .Release.Name) .Values.database.jdbcUrl | b64enc |quote }} +{{- else -}} + USERNAME: {{ default "postgres" .Values.database.username | b64enc | quote }} + JDBC: {{ default (printf "jdbc:postgresql://%s-postgresql-public:5432/postgres?sslmode=disable" .Release.Name) .Values.database.jdbcUrl | b64enc |quote }} +{{- end -}} PASSWORD: {{ default "" .Values.database.password | b64enc | quote }} -JDBC: {{ default (printf "jdbc:postgresql://%s-cockroachdb-public:26257/defaultdb?sslmode=disable" .Release.Name) .Values.database.jdbcUrl | b64enc |quote }} {{- end -}} {{/* diff --git a/projects/control-service/projects/helm_charts/pipelines-control-service/templates/secret_jdbc.yaml b/projects/control-service/projects/helm_charts/pipelines-control-service/templates/secret_jdbc.yaml index 3854d3e7f1..b12c65ece3 100644 --- a/projects/control-service/projects/helm_charts/pipelines-control-service/templates/secret_jdbc.yaml +++ b/projects/control-service/projects/helm_charts/pipelines-control-service/templates/secret_jdbc.yaml @@ -1,4 +1,4 @@ -## Create a secret with JDBC credentials for CockroachDB only if externalSecretName is not supplied. +## Create a secret with JDBC credentials for embedded database (PostgreSQL or CockroachDB) only if externalSecretName is not supplied. ## If any of the fields are empty, we fall-back to defaults (f.e. local dev environment) {{- if not .Values.database.externalSecretName }} apiVersion: v1 diff --git a/projects/control-service/projects/helm_charts/pipelines-control-service/values.yaml b/projects/control-service/projects/helm_charts/pipelines-control-service/values.yaml index b77873430c..a717c61b5f 100644 --- a/projects/control-service/projects/helm_charts/pipelines-control-service/values.yaml +++ b/projects/control-service/projects/helm_charts/pipelines-control-service/values.yaml @@ -428,10 +428,38 @@ resources: ## and some deploy specific configuration like execution schedule ## Database schema is internal implementation detail and should not be queried directly. -## Configuration for the embedded database, set cockroachdb.enabled to false if you want to use external database +## Configuration for a PostgreSQL embedded database, set postgresql.enabled to false if you want to use external database +## See https://github.com/bitnami/charts/tree/main/bitnami/postgresql +postgresql: + enabled: true + storage: + persistentVolume: + size: 1Gi + statefulset: + replicas: 3 + resources: + limits: + cpu: 500m + memory: 500Mi + requests: + cpu: 500m + memory: 500Mi + init: + resources: + limits: + cpu: 500m + memory: 500Mi + requests: + cpu: 500m + memory: 500Mi + service: + public: + type: ClusterIP + +## Configuration for a CockroachDB embedded database, set cockroachdb.enabled to false if you want to use external database ## See https://github.com/cockroachdb/helm-charts/tree/master/cockroachdb cockroachdb: - enabled: true + enabled: false storage: persistentVolume: size: 1Gi From 29cc133b9473c7671708162f1fd5c4c1629b9c73 Mon Sep 17 00:00:00 2001 From: Antoni Ivanov Date: Mon, 9 Aug 2021 13:56:24 +0300 Subject: [PATCH 2/4] versatile-data-kit: postgresql embedded For embedded DB for control-service metadata storage, we need to use the Bitnami-available chart of PostgreSQL. Added postgresql `values.yaml` configurations. In case `postgresql.enabled` is `true`, PostgreSQL embedded is being prioritized for usages. It is also the default embedded database. Testing Done: ci/cd Signed-off-by: ivakoleva --- .../pipelines-control-service/templates/_helpers.tpl | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/projects/control-service/projects/helm_charts/pipelines-control-service/templates/_helpers.tpl b/projects/control-service/projects/helm_charts/pipelines-control-service/templates/_helpers.tpl index 25e79de8d5..c7c6d2965f 100644 --- a/projects/control-service/projects/helm_charts/pipelines-control-service/templates/_helpers.tpl +++ b/projects/control-service/projects/helm_charts/pipelines-control-service/templates/_helpers.tpl @@ -186,13 +186,14 @@ Generate default JDBC credentials for local embedded database instance (Cockroac */}} {{- define "pipelines-control-service.jdbcSecret" -}} {{- if and (not .Values.postgresql.enabled) .Values.cockroachdb.enabled -}} - USERNAME: {{ default "root" .Values.database.username | b64enc | quote }} - JDBC: {{ default (printf "jdbc:postgresql://%s-cockroachdb-public:26257/defaultdb?sslmode=disable" .Release.Name) .Values.database.jdbcUrl | b64enc |quote }} +USERNAME: {{ default "root" .Values.database.username | b64enc | quote }} +PASSWORD: {{ default "" .Values.database.password | b64enc | quote }} +JDBC: {{ default (printf "jdbc:postgresql://%s-cockroachdb-public:26257/defaultdb?sslmode=disable" .Release.Name) .Values.database.jdbcUrl | b64enc |quote }} {{- else -}} - USERNAME: {{ default "postgres" .Values.database.username | b64enc | quote }} - JDBC: {{ default (printf "jdbc:postgresql://%s-postgresql-public:5432/postgres?sslmode=disable" .Release.Name) .Values.database.jdbcUrl | b64enc |quote }} -{{- end -}} +USERNAME: {{ default "postgres" .Values.database.username | b64enc | quote }} PASSWORD: {{ default "" .Values.database.password | b64enc | quote }} +JDBC: {{ default (printf "jdbc:postgresql://%s-postgresql-public:5432/postgres?sslmode=disable" .Release.Name) .Values.database.jdbcUrl | b64enc |quote }} +{{- end -}} {{- end -}} {{/* From 4dea7ed7d6f9a006ca8da8192b3d2a43f2b26966 Mon Sep 17 00:00:00 2001 From: ivakoleva Date: Mon, 31 Oct 2022 19:18:45 +0200 Subject: [PATCH 3/4] versatile-data-kit: postgresql embedded For embedded DB for control-service metadata storage, we need to use the Bitnami-available chart of PostgreSQL. Added postgresql `values.yaml` configurations. In case `postgresql.enabled` is `true`, PostgreSQL embedded is being prioritized for usages. It is also the default embedded database. Testing Done: verified with helm template --debug, also postgresql artifacts with helm install Signed-off-by: ivakoleva --- .../pipelines-control-service/values.yaml | 27 +++++++++---------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/projects/control-service/projects/helm_charts/pipelines-control-service/values.yaml b/projects/control-service/projects/helm_charts/pipelines-control-service/values.yaml index a717c61b5f..fa1d68df9f 100644 --- a/projects/control-service/projects/helm_charts/pipelines-control-service/values.yaml +++ b/projects/control-service/projects/helm_charts/pipelines-control-service/values.yaml @@ -432,11 +432,19 @@ resources: ## See https://github.com/bitnami/charts/tree/main/bitnami/postgresql postgresql: enabled: true - storage: - persistentVolume: + architecture: standalone # replication +# readReplicas: +# replicaCount: 2 +# resources: +# limits: +# cpu: 500m +# memory: 500Mi +# requests: +# cpu: 500m +# memory: 500Mi + primary: + persistence: size: 1Gi - statefulset: - replicas: 3 resources: limits: cpu: 500m @@ -444,16 +452,7 @@ postgresql: requests: cpu: 500m memory: 500Mi - init: - resources: - limits: - cpu: 500m - memory: 500Mi - requests: - cpu: 500m - memory: 500Mi - service: - public: + service: type: ClusterIP ## Configuration for a CockroachDB embedded database, set cockroachdb.enabled to false if you want to use external database From 578c8c36430c52d7718067965d587c27bc8c08de Mon Sep 17 00:00:00 2001 From: ivakoleva Date: Tue, 8 Nov 2022 18:29:30 +0200 Subject: [PATCH 4/4] versatile-data-kit: postgresql embedded For embedded DB for control-service metadata storage, we need to support the Bitnami-available chart of PostgreSQL. Added postgresql `values.yaml` configurations. In case `cockroachdb.enabled` is `true`, CockroachDB embedded is still being prioritized for usages. Also, CockroachDB remains the default embedded database. Testing Done: verified `postgresql.enabled` with helm template --debug, also postgresql artifacts with helm install. Signed-off-by: ivakoleva --- .../pipelines-control-service/templates/_helpers.tpl | 10 +++++----- .../helm_charts/pipelines-control-service/values.yaml | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/projects/control-service/projects/helm_charts/pipelines-control-service/templates/_helpers.tpl b/projects/control-service/projects/helm_charts/pipelines-control-service/templates/_helpers.tpl index bde9c50793..b329670b85 100644 --- a/projects/control-service/projects/helm_charts/pipelines-control-service/templates/_helpers.tpl +++ b/projects/control-service/projects/helm_charts/pipelines-control-service/templates/_helpers.tpl @@ -185,14 +185,14 @@ Create the name of the deployment Kubernetes namespace used by System and builde Generate default JDBC credentials for local embedded database instance (CockroachDB or PostgreSQL). */}} {{- define "pipelines-control-service.jdbcSecret" -}} -{{- if and (not .Values.postgresql.enabled) .Values.cockroachdb.enabled -}} -USERNAME: {{ default "root" .Values.database.username | b64enc | quote }} -PASSWORD: {{ default "" .Values.database.password | b64enc | quote }} -JDBC: {{ default (printf "jdbc:postgresql://%s-cockroachdb-public:26257/defaultdb?sslmode=disable" .Release.Name) .Values.database.jdbcUrl | b64enc |quote }} -{{- else -}} +{{- if and (not .Values.cockroachdb.enabled) .Values.postgresql.enabled -}} USERNAME: {{ default "postgres" .Values.database.username | b64enc | quote }} PASSWORD: {{ default "" .Values.database.password | b64enc | quote }} JDBC: {{ default (printf "jdbc:postgresql://%s-postgresql-public:5432/postgres?sslmode=disable" .Release.Name) .Values.database.jdbcUrl | b64enc |quote }} +{{- else -}} +USERNAME: {{ default "root" .Values.database.username | b64enc | quote }} +PASSWORD: {{ default "" .Values.database.password | b64enc | quote }} +JDBC: {{ default (printf "jdbc:postgresql://%s-cockroachdb-public:26257/defaultdb?sslmode=disable" .Release.Name) .Values.database.jdbcUrl | b64enc |quote }} {{- end -}} {{- end -}} diff --git a/projects/control-service/projects/helm_charts/pipelines-control-service/values.yaml b/projects/control-service/projects/helm_charts/pipelines-control-service/values.yaml index e97c259ba7..4e8da43016 100644 --- a/projects/control-service/projects/helm_charts/pipelines-control-service/values.yaml +++ b/projects/control-service/projects/helm_charts/pipelines-control-service/values.yaml @@ -439,7 +439,7 @@ resources: ## Configuration for a PostgreSQL embedded database, set postgresql.enabled to false if you want to use external database ## See https://github.com/bitnami/charts/tree/main/bitnami/postgresql postgresql: - enabled: true + enabled: false architecture: standalone # replication # readReplicas: # replicaCount: 2 @@ -466,7 +466,7 @@ postgresql: ## Configuration for a CockroachDB embedded database, set cockroachdb.enabled to false if you want to use external database ## See https://github.com/cockroachdb/helm-charts/tree/master/cockroachdb cockroachdb: - enabled: false + enabled: true storage: persistentVolume: size: 1Gi