diff --git a/.github/ISSUE_TEMPLATE/bug_report.yaml b/.github/ISSUE_TEMPLATE/bug_report.yaml index f996cbd..9306474 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yaml +++ b/.github/ISSUE_TEMPLATE/bug_report.yaml @@ -8,15 +8,6 @@ body: value: | Thanks for taking the time to fill out this bug report! - - type: input - id: title - attributes: - label: Title - description: A clear and concise title of the issue - placeholder: "Type the issue title here" - validations: - required: true - - type: checkboxes id: labels attributes: diff --git a/.github/ISSUE_TEMPLATE/feature_request.yaml b/.github/ISSUE_TEMPLATE/feature_request.yaml index 47215db..5d3c055 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yaml +++ b/.github/ISSUE_TEMPLATE/feature_request.yaml @@ -8,15 +8,6 @@ body: value: | Thanks for taking the time to suggest a feature! - - type: input - id: title - attributes: - label: Title - description: A clear and concise title of the feature request - placeholder: "Type the feature title here" - validations: - required: true - - type: textarea id: description attributes: diff --git a/.github/workflows/helm-tests.yml b/.github/workflows/helm-tests.yml index de66e9d..64b3e67 100644 --- a/.github/workflows/helm-tests.yml +++ b/.github/workflows/helm-tests.yml @@ -1,44 +1,47 @@ name: CI - -# On every pull request, but only on push to main on: push: - branches: - - main - paths: [] - # only run jobs if some code have changed - #- 'helm-chart/eoapi/**' + branches: [ "main" ] pull_request: - branches: - - main + branches: [ "main" ] + types: [ opened, reopened, synchronize, labeled ] jobs: helm-tests: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + - uses: d3adb5/helm-unittest-action@v2 with: - helm-version: v3.8.2 + helm-version: v3.15.2 github-token: ${{ secrets.GITHUB_TOKEN }} + - run: | cd helm-chart helm unittest eoapi -f 'tests/*.yaml' -v eoapi/test-helm-values.yaml - integration-tests: - if: github.event.pull_request.head.repo.full_name == github.repository + integration-tests-gcp: + # run on: + # - all pushes to specified branch(es) + # - a PR was just labeled 'test-integration' + # - a PR with 'test-integration' label was opened, reopened, or synchronized + if: | + github.event_name == 'push' || + github.event.label.name == 'test-integration-gcp' || + contains( github.event.pull_request.labels.*.name, 'test-integration-gcp') permissions: contents: 'read' id-token: 'write' needs: helm-tests runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - - uses: azure/setup-helm@v3 + - uses: azure/setup-helm@v4 with: - version: v3.8.2 - token: ${{ secrets.GITHUB_TOKEN }} + version: v3.15.2 + #token: ${{ secrets.GITHUB_TOKEN }} - name: last commit sha if PR if: ${{ github.event_name == 'pull_request' }} @@ -60,13 +63,13 @@ jobs: echo "RELEASE_NAME=eoapi$COMMITSHA$SALT" >> $GITHUB_ENV - id: 'auth' - uses: 'google-github-actions/auth@v1' + uses: 'google-github-actions/auth@v2' with: service_account: 'k8seed-deploy@devseed-labs.iam.gserviceaccount.com' credentials_json: ${{ secrets.GH_ACTIONS_SA_JSON }} - name: setup gcloud sdk - uses: google-github-actions/setup-gcloud@v1 + uses: google-github-actions/setup-gcloud@v2 with: version: '>= 363.0.0' project_id: 'devseed-labs' @@ -78,24 +81,13 @@ jobs: - name: helm render/install eoapi templates run: | - export PGUSER=username - export POSTGRES_USER=username - export PGPASSWORD=password - export POSTGRES_PASSWORD=password export GITSHA='${{github.sha}}' cd helm-chart helm install $RELEASE_NAME \ - --namespace eoapitest \ + --namespace $RELEASE_NAME \ --create-namespace \ - --set db.settings.secrets.POSTGRES_HOST=pgstac-$RELEASE_NAME \ - --set db.settings.secrets.POSTGRES_HOST_READER=pgstac-$RELEASE_NAME \ - --set db.settings.secrets.POSTGRES_HOST_WRITER=pgstac-$RELEASE_NAME \ - --set db.settings.secrets.PGUSER=$PGUSER \ - --set db.settings.secrets.POSTGRES_USER=$POSTGRES_USER \ - --set db.settings.secrets.PGPASSWORD=$PGPASSWORD \ - --set db.settings.secrets.POSTGRES_PASSWORD=$POSTGRES_PASSWORD \ -f ./eoapi/values.yaml \ -f ./eoapi/test-unittest-values.yaml \ ./eoapi @@ -109,7 +101,7 @@ jobs: timeout-minutes: 10 continue-on-error: true run: | - kubectl config set-context --current --namespace=eoapitest + kubectl config set-context --current --namespace=$RELEASE_NAME while [[ -z "$(kubectl get pod | grep "^raster-$RELEASE_NAME-.*$" | cut -d' ' -f1 | xargs -I{} kubectl logs pod/{} | grep "GET /.*/healthz" | head -n 1)" ]]; do echo "still waiting for raster service to start..." sleep 1 @@ -131,6 +123,9 @@ jobs: run: | echo "The previous step failed or timed out. Running cleanup logic..." helm uninstall $RELEASE_NAME + + kubectl delete ns/$RELEASE_NAME + # force GH action to show failed result exit 128 @@ -145,7 +140,7 @@ jobs: id: testrunner continue-on-error: true run: | - kubectl config set-context --current --namespace=eoapitest + kubectl config set-context --current --namespace=$RELEASE_NAME PUBLICIP='http://'$(kubectl -n ingress-nginx get svc/ingress-nginx-controller -o jsonpath='{.spec.loadBalancerIP}') echo '#################################' echo vector=$PUBLICIP/vector$RELEASE_NAME @@ -163,18 +158,24 @@ jobs: head -n 5 .github/workflows/tests/test_stac.py pytest .github/workflows/tests/test_stac.py - sed -i "s|raster_endpoint\=.*$|raster_endpoint\='$PUBLICIP/raster$RELEASE_NAME'|g" .github/workflows/tests/test_raster.py - head -n 5 .github/workflows/tests/test_raster.py - pytest .github/workflows/tests/test_raster.py + # TODO: fix raster tests + #sed -i "s|raster_endpoint\=.*$|raster_endpoint\='$PUBLICIP/raster$RELEASE_NAME'|g" .github/workflows/tests/test_raster.py + #head -n 5 .github/workflows/tests/test_raster.py + #pytest .github/workflows/tests/test_raster.py - name: cleanup if tests faile if: steps.testrunner.outcome == 'failure' run: | echo "The previous step failed or timed out. Running cleanup logic..." helm uninstall $RELEASE_NAME + + kubectl delete ns/$RELEASE_NAME + # force GH action to show failed result exit 128 - name: helm uinstall eoapi templates run: | helm uninstall $RELEASE_NAME + + kubectl delete ns/$RELEASE_NAME diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 205f015..fb4fab6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -10,7 +10,7 @@ jobs: if: github.actor == 'ranchodeluxe' || github.actor == 'gcorradini' || github.actor == 'sunu' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 @@ -19,11 +19,10 @@ jobs: git config user.name "$GITHUB_ACTOR" git config user.email "$GITHUB_ACTOR@users.noreply.github.com" - - uses: azure/setup-helm@v3 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - uses: azure/setup-helm@v4 with: - helm-version: v3.8.2 + version: v3.15.2 + #token: ${{ secrets.GITHUB_TOKEN }} - name: update gh-pages with content from main env: diff --git a/.github/workflows/tests/test_vector.py b/.github/workflows/tests/test_vector.py index 70a08ef..21a89ff 100644 --- a/.github/workflows/tests/test_vector.py +++ b/.github/workflows/tests/test_vector.py @@ -23,6 +23,10 @@ def test_vector_api(): assert resp.headers["content-type"] == "application/json" assert resp.json()["conformsTo"] + # refresh to get newest catalog + resp = client.get(f"{vector_endpoint}/refresh") + assert resp.status_code == 200 + # collections resp = client.get(f"{vector_endpoint}/collections") assert resp.status_code == 200 diff --git a/README.md b/README.md index 2871c7f..8b58d00 100644 --- a/README.md +++ b/README.md @@ -37,47 +37,56 @@ If you don't have a k8s cluster set up on AWS or GCP then follow an IaC guide be -## Helm Installation +## Helm Installation -Once you have a k8s cluster set up you can `helm install` eoAPI as follows: +Once you have a k8s cluster set up you can `helm install` eoAPI with the following steps: -1. `helm install` from https://devseed.com/eoapi-k8s/: +0. `eoapi-k8s` depends on the [Crunchydata Postgresql Operator](https://access.crunchydata.com/documentation/postgres-operator/latest/installation/helm). Install that first: + + ```python + $ helm install --set disable_check_for_upgrades=true pgo oci://registry.developers.crunchydata.com/crunchydata/pgo + ``` + + +1. Add the eoapi repo from https://devseed.com/eoapi-k8s/: ```python - # add the eoapi helm repo locally $ helm repo add eoapi https://devseed.com/eoapi-k8s/ - - # list out the eoapi chart versions - $ helm search repo eoapi --versions - NAME CHART VERSION APP VERSION DESCRIPTION - eoapi/eoapi 0.1.1 0.1.0 Create a full Earth Observation API with Metada... - eoapi/eoapi 0.1.2 0.1.0 Create a full Earth Observation API with Metada... - - # add the required secret overrides to an arbitrarily named `.yaml` file (`config.yaml` below) - $ cat config.yaml - db: - settings: - secrets: - PGUSER: "username" - POSTGRES_USER: "username" - PGPASSWORD: "password" - POSTGRES_PASSWORD: "password" - - # then run `helm install` with those overrides - $ helm install -n eoapi --create-namespace eoapi eoapi/eoapi --version 0.1.2 -f config.yaml ``` -2. or `helm install` from this repo's `helm-chart/` folder: +2. List out the eoapi chart versions + + ```python + $ helm search repo eoapi --versions + NAME CHART VERSION APP VERSION DESCRIPTION + eoapi/eoapi 0.2.14 0.3.1 Create a full Earth Observation API with Metada... + eoapi/eoapi 0.1.13 0.2.11 Create a full Earth Observation API with Metada... + ``` +3. Optionally override keys/values in the default `values.yaml` with a custom `config.yaml` like below: + + ```python + $ cat config.yaml + vector: + enable: false + pgstacBootstrap: + settings: + envVars: + LOAD_FIXTURES: "0" + RUN_FOREVER: "1" + ``` +4. Then `helm install` with those `config.yaml` values: + + ```python + $ helm install -n eoapi --create-namespace eoapi eoapi/eoapi --version 0.1.2 -f config.yaml + ``` + +5. or check out this repo and `helm install` from this repo's `helm-chart/` folder: ```python ###################################################### # create os environment variables for required secrets ###################################################### $ export GITSHA=$(git rev-parse HEAD | cut -c1-10) - $ export PGUSER=s00pers3cr3t - $ export POSTGRES_USER=s00pers3cr3t - $ export POSTGRES_PASSWORD=superuserfoobar - $ export PGPASSWORD=foobar $ cd ./helm-chart @@ -85,10 +94,6 @@ Once you have a k8s cluster set up you can `helm install` eoAPI as follows: --namespace eoapi \ --create-namespace \ --set gitSha=$GITSHA \ - --set db.settings.secrets.PGUSER=$PGUSER \ - --set db.settings.secrets.POSTGRES_USER=$POSTGRES_USER \ - --set db.settings.secrets.PGPASSWORD=$PGPASSWORD \ - --set db.settings.secrets.POSTGRES_PASSWORD=$POSTGRES_PASSWORD \ eoapi \ ./eoapi ``` diff --git a/docs/aws-eks.md b/docs/aws-eks.md index 783cdcb..87e047a 100644 --- a/docs/aws-eks.md +++ b/docs/aws-eks.md @@ -2,7 +2,7 @@ This is a verbose walkthrough. It uses `eksctl` and assumes you already have an AWS account, have the [eksctl prerequisites installed](https://docs.aws.amazon.com/eks/latest/userguide/getting-started-eksctl.html) including `eksctl` and `helm`. -If you are familiar with Terraform would like an IaC choice that is more terse consider setting up your cluster with that: https://github.com/developmentseed/eoapi-k8s-terraform +If you're familiar with Terraform and would like an IaC choice that is more terse consider setting up your cluster with that: https://github.com/developmentseed/eoapi-k8s-terraform ## Table of Contents: diff --git a/docs/configuration.md b/docs/configuration.md index e25e94d..c82abea 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -10,19 +10,16 @@ $ head -n 9 /values.schema.json "$schema": "http://json-schema.org/schema#", "type": "object", "required": [ - "db", "service", "gitSha" ], ``` -Most of the required fields have common-sense defaults except traditional username and password secrets under `db`. +Most of the required fields have common-sense defaults. The table below and the `values.yaml` comments should explain what the options and defaults are: | **Values Key** | **Description** | **Default** | **Choices** | |:-------------------------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------|:------------|:------------------------| -| `db.settings.secrets.PGUSER`
`db.settings.secrets.PGPASSWORD` | username and password used by application for connections
https://www.postgresql.org/docs/current/libpq-envars.html | | | -| `db.settings.secrets.POSTGRES_USER`
`db.settings.secrets.POSTGRES_PASSWORD` | username and password used by
base postgresl image for admin purposes
see https://www.postgresql.org/docs/current/libpq-envars.html | | | | `service.port` | the port that all vector/raster/stac services run on
used in `kind: Service` and `kind: Ingress` | 8080 | your favorite port | | `gitSha` | sha attached to a `kind: Deployment` key `metadata.labels` | gitshaABC123 | your favorite sha | @@ -31,23 +28,11 @@ The table below and the `values.yaml` comments should explain what the options a ## Default Configuration -Running `helm install` from https://devseed.com/eoapi-k8s/ with this simple `config.yml` overrides below -should spin up similar infrastructure in EKS or GKE: - -```python -$ cat config.yaml -db: - settings: - secrets: - PGUSER: "username" - POSTGRES_USER: "username" - PGPASSWORD: "password" - POSTGRES_PASSWORD: "password" -``` +Running `helm install` from https://devseed.com/eoapi-k8s/ should spin up similar infrastructure in EKS or GKE: In EKS or GKE you'll by default get: -* a pgstac PostgreSQL database deployment and service +* a HA PostgreSQL database deployment and service via [Crunchdata's Postgresl Operator](https://access.crunchydata.com/documentation/postgres-operator) * the same vector and raster data fixtures used for testing loaded into the DB * a load balancer and nginx-compatible ingress with the following path rewrites: * a `/stac` service for `stac_fastapi.pgstac` @@ -69,7 +54,7 @@ Here's a simplified high-level diagram to grok: | **Values Key** | **Description** | **Default** | **Choices** | |:-----------------|:-----------------------------------------------------------------------------------------------------------------------------------------------|:-----------|:--------------| -| `autoscaling.type` | a simple example of a default metric (`cpu`) and custom metric (`requestRate`) to scale by. NOTE: `requestRate` is based on nginx metrics and currently isn't supported for `ingress.className: alb/gce` options yet. It will throw an error during install if you attemp this. If selecting `both` the metric that results in the "highest amount of change" wins. See https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#scaling-on-multiple-metrics for more info | requestRate | requestRate
cpu
both
| +| `autoscaling.type` | a simple example of a default metric (`cpu`) and custom metric (`requestRate`) to scale by. If selecting `both` the metric that results in the "highest amount of change" wins. See [k8s documentation](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#scaling-on-multiple-metrics) for more info | requestRate | requestRate
cpu
both
| #### `autoscaling.behaviour.[scaleDown||scaleUp]` diff --git a/docs/gcp-gke.md b/docs/gcp-gke.md index bf25e5a..1933d80 100644 --- a/docs/gcp-gke.md +++ b/docs/gcp-gke.md @@ -2,7 +2,7 @@ This is a verbose walkthrough. It uses `gcloud` and assumes you already have an GCP account and project where you want to run eoapi. We also assume that you have some prerequisites installed including `gcloud`, `kubectl` and `helm`. -If you are familiar with Terraform would like an IaC choice that is more terse consider setting up your cluster with that: https://github.com/developmentseed/eoapi-k8s-terraform +If you're familiar with Terraform and would like an IaC choice that is more terse consider setting up your cluster with that: https://github.com/developmentseed/eoapi-k8s-terraform # Table of Contents - [Pre-requisites](#pre-requisites) diff --git a/helm-chart/eoapi/Chart.lock b/helm-chart/eoapi/Chart.lock new file mode 100644 index 0000000..a0899da --- /dev/null +++ b/helm-chart/eoapi/Chart.lock @@ -0,0 +1,9 @@ +dependencies: +- name: pgo + repository: "" + version: 5.5.2 +- name: postgrescluster + repository: "" + version: 5.5.2 +digest: sha256:88b79b5e364db5d6d1b636c5533628b2d874584f119b1630ec0b16ee6d0f9d88 +generated: "2024-06-06T11:50:50.141657-07:00" diff --git a/helm-chart/eoapi/Chart.yaml b/helm-chart/eoapi/Chart.yaml index d8f3ea9..5214acb 100644 --- a/helm-chart/eoapi/Chart.yaml +++ b/helm-chart/eoapi/Chart.yaml @@ -17,10 +17,15 @@ kubeVersion: ">=1.23.0-0" # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: "0.2.14" +version: "0.3.0" # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "0.3.1" +appVersion: "0.3.2" + +dependencies: + - name: postgrescluster + version: 5.5.2 + respository: "file://charts/postgrescluster" diff --git a/helm-chart/eoapi/charts/postgrescluster/Chart.yaml b/helm-chart/eoapi/charts/postgrescluster/Chart.yaml new file mode 100644 index 0000000..8fd2e3e --- /dev/null +++ b/helm-chart/eoapi/charts/postgrescluster/Chart.yaml @@ -0,0 +1,7 @@ +apiVersion: v2 +name: postgrescluster +description: A Helm chart for Kubernetes +type: application +# The version below should match the version on the PostgresCluster CRD +version: 5.5.2 +appVersion: 5.5.2 diff --git a/helm-chart/eoapi/charts/postgrescluster/templates/NOTES.txt b/helm-chart/eoapi/charts/postgrescluster/templates/NOTES.txt new file mode 100644 index 0000000..13d1374 --- /dev/null +++ b/helm-chart/eoapi/charts/postgrescluster/templates/NOTES.txt @@ -0,0 +1,31 @@ +Thank you for deploying a Crunchy PostgreSQL cluster! + + (((((((((((((((((((((( + (((((((((((((%%%%%%%((((((((((((((( + (((((((((((%%% %%%%(((((((((((( + (((((((((((%%( (((( ( %%%((((((((((( + (((((((((((((%% (( ,(( %%%((((((((((( + (((((((((((((((%% *%%/ %%%%%%%(((((((((( + (((((((((((((((((((%%(( %%%%%%%%%%#(((((%%%%%%%%%%#(((((((((((( + ((((((((((((((((((%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%(((((((((((((( + *((((((((((((((((((((%%%%%% /%%%%%%%%%%%%%%%%%%%(((((((((((((((( + (((((((((((((((((((((((%%%/ .%, %%%((((((((((((((((((, + ((((((((((((((((((((((% %#((((((((((((((((( +(((((((((((((((%%%%%% #%((((((((((((((((( +((((((((((((((%% %%(((((((((((((((, +((((((((((((%%%#% % %%((((((((((((((( +((((((((((((%. % % #(((((((((((((( +(((((((((((%% % %%* %((((((((((((( +#(###(###(#%% %%% %% %%% #%%#(###(###(# +###########%%%%% /%%%%%%%%%%%%% %% %%%%% ,%%####### +###############%% %%%%%% %%% %%%%%%%% %%##### + ################%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% %%## + ################%% %%%%%%%%%%%%%%%%% %%%% % + ##############%# %% (%%%%%%% %%%%%% + #############% %%%%% %%%%%%%%%%% + ###########% %%%%%%%%%%% %%%%%%%%% + #########%% %% %%%%%%%%%%%%%%%# + ########%% %% %%%%%%%%% + ######%% %% %%%%%% + ####%%% %%%%% % + %% %%%% diff --git a/helm-chart/eoapi/charts/postgrescluster/templates/_azure.tpl b/helm-chart/eoapi/charts/postgrescluster/templates/_azure.tpl new file mode 100644 index 0000000..e3cfc4c --- /dev/null +++ b/helm-chart/eoapi/charts/postgrescluster/templates/_azure.tpl @@ -0,0 +1,12 @@ +{{/* Allow for Azure secret information to be stored in a Secret */}} +{{- define "postgres.azure" }} +[global] +{{- if .azure }} + {{- if .azure.account }} +repo{{ add .index 1 }}-azure-account={{ .azure.account }} + {{- end }} + {{- if .azure.key }} +repo{{ add .index 1 }}-azure-key={{ .azure.key }} + {{- end }} +{{- end }} +{{ end }} diff --git a/helm-chart/eoapi/charts/postgrescluster/templates/_gcs.tpl b/helm-chart/eoapi/charts/postgrescluster/templates/_gcs.tpl new file mode 100644 index 0000000..43e5c0b --- /dev/null +++ b/helm-chart/eoapi/charts/postgrescluster/templates/_gcs.tpl @@ -0,0 +1,7 @@ +{{/* Allow for GCS secret information to be stored in a Secret */}} +{{- define "postgres.gcs" }} +[global] +{{- if .gcs }} +repo{{ add .index 1 }}-gcs-key=/etc/pgbackrest/conf.d/gcs-key.json +{{- end }} +{{ end }} diff --git a/helm-chart/eoapi/charts/postgrescluster/templates/_s3.tpl b/helm-chart/eoapi/charts/postgrescluster/templates/_s3.tpl new file mode 100644 index 0000000..7607723 --- /dev/null +++ b/helm-chart/eoapi/charts/postgrescluster/templates/_s3.tpl @@ -0,0 +1,18 @@ +{{/* Allow for S3 secret information to be stored in a Secret */}} +{{- define "postgres.s3" }} +[global] +{{- if .s3 }} + {{- if .s3.key }} +repo{{ add .index 1 }}-s3-key={{ .s3.key }} + {{- end }} + {{- if .s3.keySecret }} +repo{{ add .index 1 }}-s3-key-secret={{ .s3.keySecret }} + {{- end }} + {{- if .s3.keyType }} +repo{{ add .index 1 }}-s3-key-type={{ .s3.keyType }} + {{- end }} + {{- if .s3.encryptionPassphrase }} +repo{{ add .index 1 }}-cipher-pass={{ .s3.encryptionPassphrase }} + {{- end }} +{{- end }} +{{ end }} diff --git a/helm-chart/eoapi/charts/postgrescluster/templates/pgbackrest-secret.yaml b/helm-chart/eoapi/charts/postgrescluster/templates/pgbackrest-secret.yaml new file mode 100644 index 0000000..e22bd37 --- /dev/null +++ b/helm-chart/eoapi/charts/postgrescluster/templates/pgbackrest-secret.yaml @@ -0,0 +1,41 @@ +{{- if or .Values.multiBackupRepos .Values.s3 .Values.gcs .Values.azure }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ default .Release.Name .Values.name }}-pgbackrest-secret +type: Opaque +data: +{{- if .Values.multiBackupRepos }} + {{- range $index, $repo := .Values.multiBackupRepos }} + {{- if $repo.s3 }} + {{- $args := dict "s3" $repo.s3 "index" $index }} + s3.conf: |- + {{ include "postgres.s3" $args | b64enc }} + {{- else if $repo.gcs }} + {{- $args := dict "gcs" $repo.gcs "index" $index }} + gcs.conf: |- + {{ include "postgres.gcs" $args | b64enc }} + gcs-key.json: |- + {{ $repo.gcs.key | b64enc }} + {{- else if $repo.azure }} + {{- $args := dict "azure" $repo.azure "index" $index }} + azure.conf: |- + {{ include "postgres.azure" $args | b64enc }} + {{- end }} +{{- end }} +{{- else if .Values.s3 }} + {{- $args := dict "s3" .Values.s3 "index" 0 }} + s3.conf: |- + {{ include "postgres.s3" $args | b64enc }} +{{- else if .Values.gcs }} + {{- $args := dict "gcs" .Values.gcs "index" 0 }} + gcs.conf: |- + {{ include "postgres.gcs" $args | b64enc }} + gcs-key.json: |- + {{ .Values.gcs.key | b64enc }} +{{- else if .Values.azure }} + {{- $args := dict "azure" .Values.azure "index" 0 }} + azure.conf: |- + {{ include "postgres.azure" $args | b64enc }} +{{- end }} +{{- end }} diff --git a/helm-chart/eoapi/charts/postgrescluster/templates/postgres.yaml b/helm-chart/eoapi/charts/postgrescluster/templates/postgres.yaml new file mode 100644 index 0000000..5326136 --- /dev/null +++ b/helm-chart/eoapi/charts/postgrescluster/templates/postgres.yaml @@ -0,0 +1,214 @@ +apiVersion: postgres-operator.crunchydata.com/v1beta1 +kind: PostgresCluster +metadata: + name: {{ default .Release.Name .Values.name }} +spec: + postgresVersion: {{ required "You must set the version of Postgres to deploy." .Values.postgresVersion }} + {{- if .Values.postGISVersion }} + postGISVersion: {{ quote .Values.postGISVersion }} + {{- end }} + {{- if .Values.imagePostgres }} + image: {{ .Values.imagePostgres | quote }} + {{- end }} + {{- if .Values.port }} + port: {{ .Values.port }} + {{- end }} + {{- if .Values.instances }} + instances: +{{ toYaml .Values.instances | indent 4 }} + {{- else }} + instances: + - name: {{ default "instance1" .Values.instanceName | quote }} + replicas: {{ default 1 .Values.instanceReplicas }} + dataVolumeClaimSpec: + {{- if .Values.instanceStorageClassName }} + storageClassName: {{ .Values.instanceStorageClassName | quote }} + {{- end }} + accessModes: + - "ReadWriteOnce" + resources: + requests: + storage: {{ default "1Gi" .Values.instanceSize | quote }} + {{- if or .Values.instanceMemory .Values.instanceCPU }} + resources: + limits: + cpu: {{ default "" .Values.instanceCPU | quote }} + memory: {{ default "" .Values.instanceMemory | quote }} + {{- end }} + {{- end }} + backups: + pgbackrest: + {{- if .Values.imagePgBackRest }} + image: {{ .Values.imagePgBackRest | quote }} + {{- end }} + {{- if .Values.pgBackRestConfig }} +{{ toYaml .Values.pgBackRestConfig | indent 6 }} + {{- else if .Values.multiBackupRepos }} + configuration: + - secret: + name: {{ default .Release.Name .Values.name }}-pgbackrest-secret + global: + {{- range $index, $repo := .Values.multiBackupRepos }} + {{- if or $repo.s3 $repo.gcs $repo.azure }} + repo{{ add $index 1 }}-path: /pgbackrest/{{ $.Release.Namespace }}/{{ default $.Release.Name $.Values.name }}/repo{{ add $index 1 }} + {{- end }} + {{- end }} + repos: + {{- range $index, $repo := .Values.multiBackupRepos }} + - name: repo{{ add $index 1 }} + {{- if $repo.volume }} + volume: + volumeClaimSpec: + {{- if $repo.volume.backupsStorageClassName }} + storageClassName: {{ .Values.backupsStorageClassName | quote }} + {{- end }} + accessModes: + - "ReadWriteOnce" + resources: + requests: + storage: {{ default "1Gi" $repo.volume.backupsSize | quote }} + {{- else if $repo.s3 }} + s3: + bucket: {{ $repo.s3.bucket | quote }} + endpoint: {{ $repo.s3.endpoint | quote }} + region: {{ $repo.s3.region | quote }} + {{- else if $repo.gcs }} + gcs: + bucket: {{ $repo.gcs.bucket | quote }} + {{- else if $repo.azure }} + azure: + container: {{ $repo.azure.container | quote }} + {{- end }} + {{- end }} + {{- else if .Values.s3 }} + configuration: + - secret: + name: {{ default .Release.Name .Values.name }}-pgbackrest-secret + global: + repo1-path: /pgbackrest/{{ .Release.Namespace }}/{{ default .Release.Name .Values.name }}/repo1 + {{- if .Values.s3.encryptionPassphrase }} + repo1-cipher-type: aes-256-cbc + {{- end }} + repos: + - name: repo1 + s3: + bucket: {{ .Values.s3.bucket | quote }} + endpoint: {{ .Values.s3.endpoint | quote }} + region: {{ .Values.s3.region | quote }} + {{- else if .Values.gcs }} + configuration: + - secret: + name: {{ default .Release.Name .Values.name }}-pgbackrest-secret + global: + repo1-path: /pgbackrest/{{ .Release.Namespace }}/{{ default .Release.Name .Values.name }}/repo1 + repos: + - name: repo1 + gcs: + bucket: {{ .Values.gcs.bucket | quote }} + {{- else if .Values.azure }} + configuration: + - secret: + name: {{ default .Release.Name .Values.name }}-pgbackrest-secret + global: + repo1-path: /pgbackrest/{{ .Release.Namespace }}/{{ default .Release.Name .Values.name }}/repo1 + repos: + - name: repo1 + azure: + container: {{ .Values.azure.container | quote }} + {{- else }} + repos: + - name: repo1 + volume: + volumeClaimSpec: + {{- if .Values.backupsStorageClassName }} + storageClassName: {{ .Values.backupsStorageClassName | quote }} + {{- end }} + accessModes: + - "ReadWriteOnce" + resources: + requests: + storage: {{ default "1Gi" .Values.backupsSize | quote }} + {{- end }} + {{- if or .Values.pgBouncerReplicas .Values.pgBouncerConfig }} + proxy: + pgBouncer: + {{- if .Values.imagePgBouncer }} + image: {{ .Values.imagePgBouncer | quote }} + {{- end }} + {{- if .Values.pgBouncerConfig }} +{{ toYaml .Values.pgBouncerConfig | indent 6 }} + {{- else }} + replicas: {{ .Values.pgBouncerReplicas }} + {{- end }} + {{- end }} + {{- if .Values.patroni }} + patroni: +{{ toYaml .Values.patroni | indent 4 }} + {{- end }} + {{- if .Values.users }} + users: +{{ toYaml .Values.users | indent 4 }} + {{- end }} + {{- if .Values.service }} + service: +{{ toYaml .Values.service | indent 4 }} + {{- end }} + {{- if .Values.dataSource }} + dataSource: +{{ toYaml .Values.dataSource | indent 4 }} + {{- end }} + {{- if .Values.databaseInitSQL }} + databaseInitSQL: + name: {{ required "A ConfigMap name is required for running bootstrap SQL." .Values.databaseInitSQL.name | quote }} + key: {{ required "A key in a ConfigMap containing any bootstrap SQL is required." .Values.databaseInitSQL.key | quote }} + {{- end }} + {{- if .Values.imagePullPolicy }} + imagePullPolicy: {{ .Values.imagePullPolicy | quote }} + {{- end }} + {{- if .Values.imagePullSecrets }} + imagePullSecrets: +{{ toYaml .Values.imagePullSecrets | indent 4 }} + {{- end }} + {{- if .Values.disableDefaultPodScheduling }} + disableDefaultPodScheduling: true + {{- end }} + {{- if .Values.metadata }} + metadata: +{{ toYaml .Values.metadata | indent 4 }} + {{- end }} + {{- if .Values.monitoring }} + monitoring: + pgmonitor: + exporter: + image: {{ default "" .Values.imageExporter | quote }} + {{- if .Values.monitoringConfig }} +{{ toYaml .Values.monitoringConfig | indent 8 }} + {{- end }} + {{- end }} + {{- if .Values.shutdown }} + shutdown: true + {{- end }} + {{- if .Values.standby }} + standby: + enabled: {{ .Values.standby.enabled }} + repoName: {{ .Values.standby.repoName }} + host: {{ .Values.standby.host }} + port: {{ .Values.standby.port }} + {{- end }} + {{- if .Values.supplementalGroups }} + supplementalGroups: +{{ toYaml .Values.supplementalGroups | indent 4 }} + {{- end }} + {{- if .Values.openshift }} + openshift: true + {{- else if eq .Values.openshift false }} + openshift: false + {{- end }} + {{- if .Values.customTLSSecret }} + customTLSSecret: +{{ toYaml .Values.customTLSSecret | indent 4 }} + {{- end }} + {{- if .Values.customReplicationTLSSecret }} + customReplicationTLSSecret: +{{ toYaml .Values.customReplicationTLSSecret | indent 4 }} + {{- end }} diff --git a/helm-chart/eoapi/charts/postgrescluster/values.yaml b/helm-chart/eoapi/charts/postgrescluster/values.yaml new file mode 100644 index 0000000..7321f3a --- /dev/null +++ b/helm-chart/eoapi/charts/postgrescluster/values.yaml @@ -0,0 +1,316 @@ +--- +# For a full explanation of how to set up the custom resource, please refer to +# the documentation: +# https://access.crunchydata.com/documentation/postgres-operator/v5/ + +########### +# General # +########### + +# name is the name of the cluster. This defaults to the name of the Helm +# release. +# name: hippo + +# postgresVersion sets the version to deploy. This version number needs to be +# available as one of the "RELATED_IMAGE_POSTGRES_..." images as part of the PGO +# installation if you want to deploy the image without setting the "postgres" +# image variable. This value is required. +postgresVersion: 16 + +# postGISVersion if set and coupled with a PostGIS enabled container, enables +# PostGIS. This version number needs to be available as one of the +# "RELATED_IMAGE_POSTGRES_..." images as part of the PGO installation if you +# want to deploy the image without setting the "postgres" image variable. +# postGISVersion: 3.1 + +# NOTE: pgBackRest is enabled by default. It must be set in +# "RELATED_IMAGE_PGBACKREST" on the PGO deployment, otherwise you will need to +# override the "pgBackRest" image. + +# pgBouncerReplicas sets the number of pgBouncer instances to deploy. The +# default is 0. You need to set this to at least 1 to deploy pgBouncer or set +# "pgBouncerConfig". Setting "pgBouncerConfig" will override the value of +# pgBouncerReplicas. The "RELATED_IMAGE_PGBOUNCER" in the PGO deployment must be +# set if you want to enable this without explicitly setting "pgBouncer". +# pgBouncerReplicas: 1 + +# monitoring enables the ability to monitor the Postgres cluster through a +# metrics exporter that can be scraped by Prometheus. This defaults to the value +# below. +# monitoring: false + +################### +# Image Overrides # +################### + +# imagePostgres can be a Postgres or GIS-enabled Postgres image. This defaults to the +# below value. "postgresVersion" needs to match the version of Postgres that is +# used here. If using the GIS-enabled Postgres image, you need to ensure +# "postGISVersion" matches the version of PostGIS used. +# imagePostgres: registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi8-16.3-0 + +# imagePgBackRest is the pgBackRest backup utility image. This defaults to the +# below value. +# imagePgBackRest: registry.developers.crunchydata.com/crunchydata/crunchy-pgbackrest:ubi8-2.51-0 + +# imagePgBouncer is the image for the PgBouncer connection pooler. This defaults +# to the below value. +# imagePgBouncer: registry.developers.crunchydata.com/crunchydata/crunchy-pgbouncer:ubi8-1.22-0 + +# imageExporter is the image name for the exporter used as a part of monitoring. +# This defaults to the value below. +# imageExporter: registry.developers.crunchydata.com/crunchydata/crunchy-postgres-exporter:ubi8-0.15.0-6 + +########################### +# Basic Postgres Settings # +########################### + +# instanceName lets you set the name of your instances. This defaults to +# the value below. Setting "instances" overrides this value. +# instanceName: instance1 + +# instanceSize sets the size of the volume that contains the data. This defaults +# to the value below. Settings "instances" overrides this value. +# instanceSize: 1Gi + +# instanceStorageClassName sets the storage class for the volume that contains the data. +# This defaults to the "default" storage class defined in the cluster. +# See: 'kubectl get storageclasses.storage.k8s.io | grep default' +# Settings "instances" overrides this value. +# instanceStorageClassName: "hostpath" + +# instanceMemory sets the memory limit for the Postgres instances. This defaults +# to no limit being set, but an example value is set below. Settings "instances" +# overrides this value. +# instanceMemory: 2Gi + +# instanceCPU sets the CPU limit for the Postgres instances. This defaults to +# no limit being set, but an example value is set below. Setting "instances" +# overrides this value. +# instanceCPU: 1000m + +# instanceReplicas lets you set the total number of Postgres replicas. This +# defaults to the value below. More than on replica enables high availability +# (HA). Settings "instances" overrides this value. +# instanceReplicas: 1 + +############################## +# Advanced Postgres Settings # +############################## + +# instances allows you to define one or more Postgres instance sets. By default, +# PGO will only deploy a single instance. Each instance set has similar +# characteristics to the other instances in the set, e.g. storage size, resource +# etc. You can have multiple replicas within an instance set. +# +# This allows you to fully customize the topology of your Postgres instances. +# +# For example, to set up an instance set with HA (due to the default pod +# topology spread constraints) +# +# instances: +# - name: pgha1 +# replicas: 2 +# dataVolumeClaimSpec: +# accessModes: +# - "ReadWriteOnce" +# resources: +# requests: +# storage: 1Gi +# instances: {} + +# port sets the port that Postgres listens on. Defaults to 5432. +# port: 5432 + +# patroni lets you set the Patroni configuration for the Postgres cluster. +# for example, to set up synchronous replication: +# patroni: +# dynamicConfiguration: +# synchronous_mode: true +# postgresql: +# parameters: +# synchronous_commit: "on" +# patroni: {} + +# users sets any custom Postgres users and databases that they have access to +# as well as any permissions associated with the user account. +# users: {} + +# dataSource specifies a data source for bootstrapping a Postgres cluster. +# dataSource: {} + +# customTLSSecret references a Secret that contains the relevant information for +# bringing external TLS artifacts to a PostgreSQL cluster. This provides the +# TLS for the cluster itself. +# customTLSSecret: {} + +# customReplicationTLSSecret references a Secret that contains the relevant +# information for bringing external TLS artifacts to a PostgreSQL cluster. This +# provides the information for the replication user. +# customReplicationTLSSecret: {} + +# databaseInitSQL references a ConfigMap that contains a SQL file that should be +# run a cluster bootstrap. +# databaseInitSQL: +# name: bootstrap-sql +# key: bootstrap.sql + +# standby sets whether to run this as a standby cluster. Setting "enabled" to +# "true" enables the standby cluster while "repoName" points to a pgBackRest +# archive to replay WAL files from, and "host" and "port" point to a primary +# cluster from which to stream data. +# standby: +# enabled: false +# repoName: repo1 +# host: "192.0.2.2" +# port: 5432 + +# shutdown when set scales the entire workload to zero. By default, this is not +# set. +# shutdown: true + +################################# +# Backups / pgBackRest Settings # +################################# + +# backupsSize sets the storage size of the backups to a volume in Kubernetes. +# can be overridden by "pgBackRestConfig", if set. Defaults to the value below. +# backupsSize: 1Gi + +# backupsStorageClassName sets the storage class to a class existing in Kubernetes. +# Defaults to the "default" storage class defined in the cluster. +# Can be overridden by "pgBackRestConfig", if set. +# backupsStorageClassName: "hostpath" + +# s3 allows for AWS S3 or an S3 compatible storage system to be used for +# backups. This allows for a quick setup with S3; if you need more advanced +# setup, use pgBackRestConfig. +# s3: +# # bucket specifies the S3 bucket to use, +# bucket: "" +# # endpoint specifies the S3 endpoint to use. +# endpoint: "" +# # region specifies the S3 region to use. If your S3 storage system does not +# # use "region", fill this in with a random value. +# region: "" +# # key is the S3 key. This is stored in a Secret. +# key: "" +# # keySecret is the S3 key secret. This is stored in a Secret. +# keySecret: "" +# # keyType can be configured to enable IAM integration via AssumeRole +# # For more info, see the documentation at https://access.crunchydata.com/documentation/postgres-operator/v5/tutorial/backups/#using-an-aws-integrated-identity-provider-and-role +# keyType: "" +# # encryptionPassphrase is an optional parameter to enable encrypted backups +# # with pgBackRest. This is encrypted by pgBackRest and does not use S3's +# # built-in encryption system. +# encryptionPassphrase: "" + +# gcs allows for Google Cloud Storage (GCS) to be used for backups. This allows +# for a quick setup with GCS; if you need a more advanced setup, use +# "pgBackRestConfig". +# gcs: +# # bucket is the name of the GCS bucket that the backups will be stored in. +# bucket: "" +# # key is a multi-line string that contains the GCS key, which is a JSON +# # structure. +# key: | +# {} + +# azure allows for Azure Blob Storage to be used for backups. This allows +# for a quick setup with Azure Blob Storage; if you need a more advanced setup, +# use "pgBackRestConfig". +# azure: +# # account is the name of the Azure account to be used. +# account: "" +# # key is the Secret key used associated with the Azure account. +# key: "" +# # container is the Azure container that the backups will be stored in. +# container: "" + +# multiBackupRepos allows for backing up to multiple repositories. This is +# effectively uses the "quickstarts" for each of the backup types (volume, s3, +# gcs, azure). You can have any permutation of these types. You can set up to 4. +# can be overwritten by "pgBackRestConfig". +# +# You can't set "multiBackupRepos" and any of the individual quickstarts at the +# same time. "multiBackupRepos" will take precedence. +# +# Below is an example that enables one of each backup type. +# All available quickstart options are presented below; please see the backup types +# if you want to see how each option works. +# multiBackupRepos: +# - volume: +# backupsSize: 1Gi +# - s3: +# bucket: "" +# endpoint: "" +# region: "" +# key: "" +# keySecret: "" +# keyType: "" +# - gcs: +# bucket: "" +# key: | +# {} +# - azure: +# account: "" +# key: "" +# container: "" + +# pgBackRestConfig allows for the configuration of every pgBackRest option +# except for "image", which is set by "pgBackRest". +# pgBackRestConfig: {} + +################################ +# Pooling / pgBouncer Settings # +################################ + +# pgBouncerConfig sets all of the pgBouncer portions of the spec except for +# image. To set image, you need to set the "pgBouncer" setting. +# pgBouncerConfig: {} + +####################### +# Monitoring Settings # +####################### + +# monitoringConfig sets all of the monitoring portions of the spec except for the +# image. To set the image, which also enables monitoring, you need to set the +# "monitoring" setting. +# monitoringConfig: {} + +####################### +# Kubernetes Settings # +####################### + +# metadata contains any metadata that should be applied to all PGO managed +# objects in this Postgres cluster. This includes "annotations" and "labels" as +# sub-keys. +# metadata: {} + +# service customizes the Service that exposes the Postgres primary. +# service: {} + +# imagePullPolicy sets the pull policy for all the images. This defaults to +# the Kubernetes heuristic: +# https://kubernetes.io/docs/concepts/containers/images/#imagepullpolicy-defaulting +# imagePullPolicy: IfNotPresent + +# imagePullSecrets references Secrets that credentials for pulling image from +# private repositories +# imagePullSecrets: [] + +# supplementalGroups sets any group IDs that should be assigned to +# Pods, particularly around file system constraints within a system +# supplementalGroups: [] + +# disableDefaultPodScheduling if set to true, will disable any of the default +# scheduling constraints for Pods, such as the default Pod Topology Spread +# Constraints. If set to false or unset, the default scheduling constraints will +# be used in addition to any customizations that are added in. +# disableDefaultPodScheduling: false + +# openshift can be set explicitly if this is an OpenShift cluster or a cluster +# that uses a SecurityContextConstraint. This usually does not need to be set, +# but you may want to explicitly set it to "false" when using an SCC like +# "anyuid" +# openshift: false diff --git a/helm-chart/eoapi/initdb-data/pgstac-setup.py b/helm-chart/eoapi/initdb-data/pgstac-setup.py new file mode 100644 index 0000000..6b6bcd9 --- /dev/null +++ b/helm-chart/eoapi/initdb-data/pgstac-setup.py @@ -0,0 +1,53 @@ +import os +import psycopg +from psycopg import sql +from psycopg.conninfo import make_conninfo +from pypgstac.db import PgstacDB +from pypgstac.migrate import Migrate + +admin_db_conninfo = make_conninfo(os.environ['PGADMIN_URI']) +print("[ REGISTER ]: postgis") +with psycopg.connect(admin_db_conninfo, autocommit=True) as conn: + with conn.cursor() as cur: + cur.execute(sql.SQL("CREATE EXTENSION IF NOT EXISTS postgis;")) + +pgdb = PgstacDB(dsn=os.environ['PGADMIN_URI'], debug=True) +print(f"[ VERSION ]: {pgdb.version=}") +Migrate(pgdb).run_migration(pgdb.version) + +with psycopg.connect(admin_db_conninfo, autocommit=True) as conn: + with conn.cursor() as cur: + # NOTE: most of these should've been set up by postgresql operator + # see `helm-chart/eoapi/values.yaml:postgrescluster` but in case + # they haven't been + cur.execute( + sql.SQL( + "GRANT CONNECT ON DATABASE {db_name} TO {username};" + "GRANT CREATE ON DATABASE {db_name} TO {username};" # Allow schema creation + "GRANT USAGE ON SCHEMA public TO {username};" + "ALTER DEFAULT PRIVILEGES IN SCHEMA public " + "GRANT ALL PRIVILEGES ON TABLES TO {username};" + "ALTER DEFAULT PRIVILEGES IN SCHEMA public " + "GRANT ALL PRIVILEGES ON SEQUENCES TO {username};" + "GRANT pgstac_read TO {username};" + "GRANT pgstac_ingest TO {username};" + "GRANT pgstac_admin TO {username};" + ).format( + db_name=sql.Identifier(os.environ["POSTGRES_DBNAME"]), + username=sql.Identifier(os.environ["POSTGRES_USER"]), + ) + ) + +with psycopg.connect(admin_db_conninfo, autocommit=True) as conn: + with conn.cursor() as cur: + cur.execute( + sql.SQL( + "INSERT INTO pgstac.pgstac_settings (name, value) " + " VALUES " + " ('context', 'auto')," + " ('context_estimated_count', '100000')," + " ('context_estimated_cost', '100000')," + " ('context_stats_ttl', '1 day')" + " ON CONFLICT ON CONSTRAINT pgstac_settings_pkey DO UPDATE SET value = excluded.value;" + ) + ) diff --git a/helm-chart/eoapi/templates/_helpers.tpl b/helm-chart/eoapi/templates/_helpers.tpl index e22b55b..52b9aa9 100644 --- a/helm-chart/eoapi/templates/_helpers.tpl +++ b/helm-chart/eoapi/templates/_helpers.tpl @@ -66,13 +66,80 @@ Create pgstac host string depending if .Values.testing */}} {{- define "eoapi.pgstacHostName" -}} {{- if .Values.testing }} -{{- printf "%s-%s" "pgstac" .Release.Name }} +{{- printf "%s-%s" "pgstacbootstrap" .Release.Name }} {{- else }} {{/* need to match what is default in values.yamls */}} -{{- printf "%s" "pgstac" }} +{{- printf "%s" "pgstacbootstrap" }} {{- end }} {{- end }} +{{/* +Secrets for postgres/postgis access have to be +derived from what the crunchydata operator creates + +Also note that we want to use the pgbouncer- +but currently it doesn't support `search_path` parameters +(https://github.com/pgbouncer/pgbouncer/pull/73) which +are required for much of *pgstac +*/}} +{{- define "eoapi.pgstacSecrets" -}} +{{- range $userName, $v := .Values.postgrescluster.users -}} +{{/* do not render anything for the "postgres" user */}} +{{- if not (eq (index $v "name") "postgres") }} +- name: POSTGRES_USER + valueFrom: + secretKeyRef: + name: {{ $.Release.Name }}-pguser-{{ index $v "name" }} + key: user +- name: POSTGRES_PORT + valueFrom: + secretKeyRef: + name: {{ $.Release.Name }}-pguser-{{ index $v "name" }} + key: port +- name: POSTGRES_HOST + valueFrom: + secretKeyRef: + name: {{ $.Release.Name }}-pguser-{{ index $v "name" }} + key: host +- name: POSTGRES_HOST_READER + valueFrom: + secretKeyRef: + name: {{ $.Release.Name }}-pguser-{{ index $v "name" }} + key: host +- name: POSTGRES_HOST_WRITER + valueFrom: + secretKeyRef: + name: {{ $.Release.Name }}-pguser-{{ index $v "name" }} + key: host +- name: POSTGRES_PASS + valueFrom: + secretKeyRef: + name: {{ $.Release.Name }}-pguser-{{ index $v "name" }} + key: password +- name: POSTGRES_DBNAME + valueFrom: + secretKeyRef: + name: {{ $.Release.Name }}-pguser-{{ index $v "name" }} + key: dbname +- name: PGBOUNCER_URI + valueFrom: + secretKeyRef: + name: {{ $.Release.Name }}-pguser-{{ index $v "name" }} + key: pgbouncer-uri +- name: DATABASE_URL + valueFrom: + secretKeyRef: + name: {{ $.Release.Name }}-pguser-{{ index $v "name" }} + key: uri +{{- end }} +{{- end }} +- name: PGADMIN_URI + valueFrom: + secretKeyRef: + name: {{ .Release.Name }}-pguser-postgres + key: uri +{{- end }} + {{/* values.schema.json doesn't play nice combined value checks so we use this helper function to check autoscaling rules @@ -89,3 +156,29 @@ so we use this helper function to check autoscaling rules {{- end }} {{- end }} {{- end -}} + +{{/* +validate: +1. the .Values.postgrescluster.users array does not have more than two elements. +2. at least one of the users is named "postgres". +*/}} +{{- define "eoapi.validatePostgresCluster" -}} +{{- $users := .Values.postgrescluster.users | default (list) -}} + +{{- if gt (len $users) 2 -}} + {{- fail "The users array in postgrescluster should not have more than two users declared b/c the last user declared will override all secrets generated in eoapi.pgstacSecrets" -}} +{{- end -}} + +{{- $hasPostgres := false -}} +{{- range $index, $user := $users -}} + {{- if eq $user.name "postgres" -}} + {{- $hasPostgres = true -}} + {{- end -}} +{{- end -}} + +{{- if not $hasPostgres -}} + {{- fail "The users array in postgrescluster must contain at least one user named 'postgres'." -}} +{{- end -}} + +{{- end -}} + diff --git a/helm-chart/eoapi/templates/db/configmap.yaml b/helm-chart/eoapi/templates/db/configmap.yaml deleted file mode 100644 index 0cdb8e2..0000000 --- a/helm-chart/eoapi/templates/db/configmap.yaml +++ /dev/null @@ -1,37 +0,0 @@ ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: initdb-sql-config-{{ $.Release.Name }} -data: - initdb.sql: | - {{- range $path, $bytes := $.Files.Glob "initdb-data/*.sql" -}} - {{ $.Files.Get $path | nindent 4 }} - {{- end }} ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: initdb-json-config-{{ $.Release.Name }} -data: - {{- range $path, $bytes := $.Files.Glob "initdb-data/*.json" -}} - {{- base $path | nindent 2 -}}: | {{- $.Files.Get $path | nindent 4 -}} - {{- end }} ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: initdb-sh-config-{{ $.Release.Name }} -data: - load.sh: | - #!/bin/bash - apt update -y && apt install python3 python3-pip -y - pip install pypgstac[psycopg] - DSN="postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST/$POSTGRES_DB" - pypgstac pgready --dsn $DSN - pypgstac load collections /opt/initdb/json-data/noaa-emergency-response.json --dsn $DSN --method insert_ignore - pypgstac load items /opt/initdb/json-data/noaa-eri-nashville2020.json --dsn $DSN --method insert_ignore - psql $DSN -f /opt/initdb/sql-data/initdb.sql - echo "DONE LOADING!!!!!!" - # run it forever like a docker process should - tail -f /dev/null diff --git a/helm-chart/eoapi/templates/db/deployment.yaml b/helm-chart/eoapi/templates/db/deployment.yaml deleted file mode 100644 index 2ce2010..0000000 --- a/helm-chart/eoapi/templates/db/deployment.yaml +++ /dev/null @@ -1,74 +0,0 @@ ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "eoapi.pgstacHostName" . | nindent 8 }} - labels: - app: {{ include "eoapi.pgstacHostName" . | nindent 10 }} -spec: - selector: - matchLabels: - app: {{ include "eoapi.pgstacHostName" . | nindent 12 }} - strategy: - type: Recreate - template: - metadata: - labels: - app: {{ include "eoapi.pgstacHostName" . | nindent 14 }} - spec: - restartPolicy: Always - containers: - - name: pgstac - image: {{ .Values.db.image.name }}:{{ .Values.db.image.tag }} - args: - {{- toYaml .Values.db.command | nindent 12 }} - envFrom: - - secretRef: - name: pgstac-secrets-{{ $.Release.Name }} - ports: - - containerPort: 5432 - resources: - limits: - cpu: {{ .Values.db.settings.resources.limits.cpu }} - memory: {{ .Values.db.settings.resources.limits.memory }} - requests: - cpu: {{ .Values.db.settings.resources.requests.cpu }} - memory: {{ .Values.db.settings.resources.requests.memory }} - volumeMounts: - - mountPath: /var/lib/postgresql/data - name: pgstac-claim-{{ $.Release.Name }} - {{- if .Values.db.enable_data_fixtures }} - - name: loader - image: {{ .Values.db.image.name }}:{{ .Values.db.image.tag }} - command: - - "sh" - args: - - "/opt/initdb/load.sh" - envFrom: - - secretRef: - name: pgstac-secrets-{{ $.Release.Name }} - ports: - - containerPort: 6543 - volumeMounts: - - mountPath: /var/lib/postgresql/data - name: pgstac-claim-{{ $.Release.Name }} - - mountPath: /opt/initdb/sql-data - name: initdb-sql-volume-{{ $.Release.Name }} - - mountPath: /opt/initdb/json-data - name: initdb-json-volume-{{ $.Release.Name }} - - mountPath: /opt/initdb/ - name: initdb-sh-volume-{{ $.Release.Name }} - {{- end }} - volumes: - - name: pgstac-claim-{{ $.Release.Name }} - persistentVolumeClaim: - claimName: pgstac-claim-{{ $.Release.Name }} - - name: initdb-sql-volume-{{ $.Release.Name }} - configMap: - name: initdb-sql-config-{{ $.Release.Name }} - - name: initdb-json-volume-{{ $.Release.Name }} - configMap: - name: initdb-json-config-{{ $.Release.Name }} - - name: initdb-sh-volume-{{ $.Release.Name }} - configMap: - name: initdb-sh-config-{{ $.Release.Name }} diff --git a/helm-chart/eoapi/templates/db/pvc.yaml b/helm-chart/eoapi/templates/db/pvc.yaml deleted file mode 100644 index a28af8f..0000000 --- a/helm-chart/eoapi/templates/db/pvc.yaml +++ /dev/null @@ -1,11 +0,0 @@ ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: pgstac-claim-{{ $.Release.Name }} -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: {{ .Values.db.settings.resources.requests.storage }} \ No newline at end of file diff --git a/helm-chart/eoapi/templates/db/secrets.yaml b/helm-chart/eoapi/templates/db/secrets.yaml deleted file mode 100644 index 246baf4..0000000 --- a/helm-chart/eoapi/templates/db/secrets.yaml +++ /dev/null @@ -1,18 +0,0 @@ ---- -apiVersion: v1 -kind: Secret -metadata: - name: pgstac-secrets-{{ $.Release.Name }} -type: "Opaque" -stringData: - {{- range $envKey, $envValue := .Values.db.settings.secrets }} - {{ upper $envKey }}: "{{ $envValue }}" - {{- /* stac-utils seems to require different environment variable for postgres so handle here via if/else to - avoid having to pass more arg secrets */ -}} - {{- if eq $envKey "PGPASSWORD" }} - POSTGRES_PASS: "{{ $envValue }}" - {{- end }} - {{- if eq $envKey "PGDATABASE" }} - POSTGRES_DBNAME: "{{ $envValue }}" - {{- end }} - {{- end }} \ No newline at end of file diff --git a/helm-chart/eoapi/templates/db/service.yaml b/helm-chart/eoapi/templates/db/service.yaml deleted file mode 100644 index 2854eba..0000000 --- a/helm-chart/eoapi/templates/db/service.yaml +++ /dev/null @@ -1,14 +0,0 @@ ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app: {{ include "eoapi.pgstacHostName" . | nindent 10 }} - name: {{ include "eoapi.pgstacHostName" . | nindent 8 }} -spec: - ports: - - name: "5432" - port: 5432 - targetPort: 5432 - selector: - app: {{ include "eoapi.pgstacHostName" . | nindent 10 }} diff --git a/helm-chart/eoapi/templates/pgstacboostrap/configmap.yaml b/helm-chart/eoapi/templates/pgstacboostrap/configmap.yaml new file mode 100644 index 0000000..0225d70 --- /dev/null +++ b/helm-chart/eoapi/templates/pgstacboostrap/configmap.yaml @@ -0,0 +1,64 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: initdb-sql-config-{{ $.Release.Name }} +data: + initdb.sql: | + {{- range $path, $bytes := $.Files.Glob "initdb-data/*.sql" -}} + {{ $.Files.Get $path | nindent 4 }} + {{- end }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: initdb-json-config-{{ $.Release.Name }} +data: + {{- range $path, $bytes := $.Files.Glob "initdb-data/*.json" -}} + {{- base $path | nindent 2 -}}: | {{- $.Files.Get $path | nindent 4 -}} + {{- end }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: pgstac-setup-config-{{ $.Release.Name }} +data: + pgstac-migrate.py: | + {{- range $path, $bytes := $.Files.Glob "initdb-data/*.py" -}} + {{- $.Files.Get $path | nindent 4 -}} + {{- end }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: initdb-sh-config-{{ $.Release.Name }} +data: + run-forever.sh: | + #!/bin/bash + pypgstac pgready --dsn $PGADMIN_URI + while true; do sleep 86400; done + apt-and-pip-install.sh: | + #!/bin/bash + apt update -y && apt install python3 python3-pip -y + pip install pypgstac[psycopg] + run-migrate-and-load.sh: | + #!/bin/bash + bash /opt/initdb/apt-and-pip-install.sh + # make sure crunchydata postgresql operator has seeded our secrets and we're ready to go + pypgstac pgready --dsn $PGADMIN_URI + # run migrations + python3 /opt/initdb/python-scripts/pgstac-migrate.py + + if [ "$LOAD_FIXTURES" = "true" ]; then + pypgstac load collections /opt/initdb/json-data/noaa-emergency-response.json --dsn $PGADMIN_URI --method insert_ignore + pypgstac load items /opt/initdb/json-data/noaa-eri-nashville2020.json --dsn $PGADMIN_URI --method insert_ignore + psql $PGADMIN_URI -f /opt/initdb/sql-data/initdb.sql + echo "DONE LOADING!!!!!!" + fi + + if [ "$KEEP_ALIVE" = "true" ]; then + bash /opt/initdb/run-forever.sh + fi + + # let the k8's pod know we've completed successfully + exit 0 diff --git a/helm-chart/eoapi/templates/pgstacboostrap/job.yaml b/helm-chart/eoapi/templates/pgstacboostrap/job.yaml new file mode 100644 index 0000000..e2110b7 --- /dev/null +++ b/helm-chart/eoapi/templates/pgstacboostrap/job.yaml @@ -0,0 +1,55 @@ +{{- if .Values.pgstacBootstrap.enabled }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "eoapi.pgstacHostName" . | nindent 8 }} + labels: + app: {{ include "eoapi.pgstacHostName" . | nindent 10 }} +spec: + template: + metadata: + labels: + app: {{ include "eoapi.pgstacHostName" . | nindent 14 }} + spec: + restartPolicy: Never + containers: + - name: pgstacbootstrap + image: {{ .Values.pgstacBootstrap.image.name }}:{{ .Values.pgstacBootstrap.image.tag }} + command: {{- toYaml .Values.pgstacBootstrap.command | nindent 12 }} + args: + {{- toYaml .Values.pgstacBootstrap.args | nindent 12 }} + ports: + - containerPort: 5432 + resources: + {{- toYaml .Values.pgstacBootstrap.settings.resources | nindent 12 }} + volumeMounts: + - mountPath: /opt/initdb/sql-data + name: initdb-sql-volume-{{ $.Release.Name }} + - mountPath: /opt/initdb/json-data + name: initdb-json-volume-{{ $.Release.Name }} + - mountPath: /opt/initdb/ + name: initdb-sh-volume-{{ $.Release.Name }} + - mountPath: /opt/initdb/python-scripts + name: pgstac-setup-volume-{{ $.Release.Name }} + env: + - name: LOAD_FIXTURES + value: {{ .Values.pgstacBootstrap.settings.envVars.LOAD_FIXTURES | quote }} + - name: KEEP_ALIVE + value: {{ .Values.pgstacBootstrap.settings.envVars.KEEP_ALIVE | quote }} + {{ include "eoapi.pgstacSecrets" . | nindent 12 }} + volumes: + - name: initdb-sql-volume-{{ $.Release.Name }} + configMap: + name: initdb-sql-config-{{ $.Release.Name }} + - name: initdb-json-volume-{{ $.Release.Name }} + configMap: + name: initdb-json-config-{{ $.Release.Name }} + - name: initdb-sh-volume-{{ $.Release.Name }} + configMap: + name: initdb-sh-config-{{ $.Release.Name }} + - name: pgstac-setup-volume-{{ $.Release.Name }} + configMap: + name: pgstac-setup-config-{{ $.Release.Name }} + backoffLimit: 1 +{{- end }} diff --git a/helm-chart/eoapi/templates/services/deployment.yaml b/helm-chart/eoapi/templates/services/deployment.yaml index 529ce9e..678bb40 100644 --- a/helm-chart/eoapi/templates/services/deployment.yaml +++ b/helm-chart/eoapi/templates/services/deployment.yaml @@ -49,18 +49,13 @@ spec: ports: - containerPort: {{ $.Values.service.port }} resources: - limits: - cpu: {{ index $v "settings" "resources" "limits" "cpu" }} - memory: {{ index $v "settings" "resources" "limits" "memory" }} - requests: - cpu: {{ index $v "settings" "resources" "requests" "cpu" }} - memory: {{ index $v "settings" "resources" "requests" "memory" }} + {{- toYaml (index $v "settings" "resources") | nindent 10 }} + env: + {{- include "eoapi.pgstacSecrets" $ | nindent 12 }} envFrom: # NOTE: there's no reason we need to use a `ConfigMap` or `Secret` here to get os env vars into the pod. # we could just template them out here immediately with `value: $_` but this allows us # to store them in k8s intermediately and change them and then bounce deploys if needed - - secretRef: - name: pgstac-secrets-{{ $.Release.Name }} - configMapRef: name: {{ $serviceName }}-envvar-configmap-{{ $.Release.Name }} --- diff --git a/helm-chart/eoapi/templates/services/nginx-doc-server.yaml b/helm-chart/eoapi/templates/services/nginx-doc-server.yaml index 30723e6..ed626d3 100644 --- a/helm-chart/eoapi/templates/services/nginx-doc-server.yaml +++ b/helm-chart/eoapi/templates/services/nginx-doc-server.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: ConfigMap metadata: - name: nginx-root-html + name: nginx-root-html-{{ .Release.Name }} data: index.html: | @@ -23,37 +23,37 @@ data: apiVersion: apps/v1 kind: Deployment metadata: - name: doc-server + name: doc-server-{{ .Release.Name }} spec: replicas: 1 selector: matchLabels: - app: doc-server + app: doc-server-{{ .Release.Name }} template: metadata: labels: - app: doc-server + app: doc-server-{{ .Release.Name }} spec: containers: - name: doc-server image: nginx:alpine volumeMounts: - - name: doc-html + - name: doc-html-{{ .Release.Name }} mountPath: /usr/share/nginx/html ports: - containerPort: 80 volumes: - - name: doc-html + - name: doc-html-{{ .Release.Name }} configMap: - name: nginx-root-html + name: nginx-root-html-{{ .Release.Name }} --- apiVersion: v1 kind: Service metadata: - name: doc-server + name: doc-server-{{ .Release.Name }} spec: selector: - app: doc-server + app: doc-server-{{ .Release.Name }} ports: - protocol: TCP port: 80 diff --git a/helm-chart/eoapi/test-helm-values.yaml b/helm-chart/eoapi/test-helm-values.yaml index 0f617fe..5b0612f 100644 --- a/helm-chart/eoapi/test-helm-values.yaml +++ b/helm-chart/eoapi/test-helm-values.yaml @@ -4,12 +4,4 @@ ingress: className: "nginx" enabled: true -db: - settings: - secrets: - PGUSER: 'foobar' - POSTGRES_USER: 'superfoobar' - PGPASSWORD: 's3cr3t' - POSTGRES_PASSWORD: 's00p3rs3cr3t' - gitSha: 'ABC123' \ No newline at end of file diff --git a/helm-chart/eoapi/test-unittest-values.yaml b/helm-chart/eoapi/test-unittest-values.yaml index 4bc9ee1..05b1c6e 100644 --- a/helm-chart/eoapi/test-unittest-values.yaml +++ b/helm-chart/eoapi/test-unittest-values.yaml @@ -3,12 +3,11 @@ testing: true ingress: enabled: true className: "nginx" -db: +pgstacBootstrap: enabled: true settings: resources: requests: - storage: "100Mi" cpu: "256m" memory: "1024Mi" limits: @@ -44,3 +43,6 @@ vector: requests: cpu: "256m" memory: "1024Mi" + envVars: + # needs to on so we can call /refresh for integration tests + TIPG_DEBUG: "True" diff --git a/helm-chart/eoapi/tests/config_tests.yaml b/helm-chart/eoapi/tests/config_tests.yaml index 6398b2f..aba0e02 100644 --- a/helm-chart/eoapi/tests/config_tests.yaml +++ b/helm-chart/eoapi/tests/config_tests.yaml @@ -15,7 +15,7 @@ tests: pattern: ^vector-envvar-configmap-RELEASE-NAME$ - equal: path: data.TIPG_CATALOG_TTL - value: "0" + value: "300" - it: "raster configmap defaults" set: raster.enabled: true @@ -43,4 +43,4 @@ tests: pattern: ^stac-envvar-configmap-RELEASE-NAME$ - equal: path: data.WEB_CONCURRENCY - value: "10" + value: "5" diff --git a/helm-chart/eoapi/tests/db_deploy_tests.yaml b/helm-chart/eoapi/tests/db_deploy_tests.yaml deleted file mode 100644 index 7ea8b7e..0000000 --- a/helm-chart/eoapi/tests/db_deploy_tests.yaml +++ /dev/null @@ -1,61 +0,0 @@ -suite: db defaults deployment -templates: - - templates/db/deployment.yaml -tests: - - it: "db deploy defaults" - set: - db.enabled: true - db.enable_data_fixtures: false - asserts: - - isKind: - of: Deployment - - matchRegex: - path: metadata.name - pattern: ^pgstac$ - - equal: - path: spec.strategy.type - value: "Recreate" - - equal: - path: spec.template.spec.containers[0].name - value: "pgstac" - - isNull: - path: spec.template.spec.containers[1].name - - it: "db deploy testing=true" - set: - testing: true - db.enabled: true - db.enable_data_fixtures: true - asserts: - - isKind: - of: Deployment - - matchRegex: - path: metadata.name - pattern: ^pgstac-RELEASE-NAME$ - - equal: - path: spec.strategy.type - value: "Recreate" - - equal: - path: spec.template.spec.containers[0].name - value: "pgstac" - - equal: - path: spec.template.spec.containers[1].name - value: "loader" - - it: "db deploy with fixtures" - set: - db.enabled: true - db.enable_data_fixtures: true - asserts: - - isKind: - of: Deployment - - matchRegex: - path: metadata.name - pattern: ^pgstac$ - - equal: - path: spec.strategy.type - value: "Recreate" - - equal: - path: spec.template.spec.containers[0].name - value: "pgstac" - - equal: - path: spec.template.spec.containers[1].name - value: "loader" \ No newline at end of file diff --git a/helm-chart/eoapi/tests/db_tests.yaml b/helm-chart/eoapi/tests/db_tests.yaml deleted file mode 100644 index 24d20a4..0000000 --- a/helm-chart/eoapi/tests/db_tests.yaml +++ /dev/null @@ -1,27 +0,0 @@ -suite: db defaults secrets -templates: - - templates/db/secrets.yaml -tests: - - it: "db secrets defaults" - set: - db.settings.secrets.PGPASSWORD: "foobar" - db.settings.secrets.PGDATABASE: "dbar" - asserts: - - isKind: - of: Secret - - matchRegex: - path: metadata.name - pattern: ^pgstac-secrets-RELEASE-NAME$ - - equal: - path: stringData.PGPASSWORD - value: "foobar" - - equal: - path: stringData.PGDATABASE - value: "dbar" - # make sure the if/else switch adds our stac-specific ones - - equal: - path: stringData.POSTGRES_PASS - value: "foobar" - - equal: - path: stringData.POSTGRES_DBNAME - value: "dbar" diff --git a/helm-chart/eoapi/values.schema.json b/helm-chart/eoapi/values.schema.json index 221596b..502d09c 100644 --- a/helm-chart/eoapi/values.schema.json +++ b/helm-chart/eoapi/values.schema.json @@ -2,59 +2,10 @@ "$schema": "http://json-schema.org/schema#", "type": "object", "required": [ - "db", "service", "gitSha" ], "properties": { - "db": { - "type": "object", - "required": [ - "settings", - "environment" - ], - "properties": { - "settings": { - "type": "object", - "required": [ - "secrets" - ], - "properties": { - "secrets": { - "type": "object", - "required": [ - "PGPASSWORD", - "PGUSER", - "POSTGRES_PASSWORD", - "POSTGRES_USER" - ], - "properties": { - "PGPASSWORD": { - "type": "string", - "pattern": "^.+$" - }, - "PGUSER": { - "type": "string", - "pattern": "^.+$" - }, - "POSTGRES_PASSWORD": { - "type": "string", - "pattern": "^.+$" - }, - "POSTGRES_USER": { - "type": "string", - "pattern": "^.+$" - } - } - } - } - }, - "environment": { - "type": "string", - "pattern": "^(k8s|rds|cloudsql)$" - } - } - }, "service": { "type": "object", "required": [ diff --git a/helm-chart/eoapi/values.yaml b/helm-chart/eoapi/values.yaml index fe400f2..0805289 100644 --- a/helm-chart/eoapi/values.yaml +++ b/helm-chart/eoapi/values.yaml @@ -1,84 +1,36 @@ -# default values for eoapi - -comment: > - some postgres variables are secrets that should be passed in via os environment variables. - for example, to pass username or password information during `helm template` or `helm install` do: +comment_install: > + `service` and `gitSha` are required and defaulted value keys. + a manual installation looks like this: $ export GITSHA=$(git rev-parse HEAD | cut -c1-10) - $ export PGUSER=s00pers3cr3t - $ export POSTGRES_USER=s00pers3cr3t - $ export POSTGRES_PASSWORD=superuserfoobar - $ export PGPASSWORD=foobar - $ helm install \ --namespace eoapi \ --create-namespace \ --set gitSha=$GITSHA \ - --set db.settings.secrets.PGUSER=$PGUSER \ - --set db.settings.secrets.POSTGRES_USER=$POSTGRES_USER \ - --set db.settings.secrets.PGPASSWORD=$PGPASSWORD \ - --set db.settings.secrets.POSTGRES_PASSWORD=$POSTGRES_PASSWORD \ eoapi \ ./eoapi - $ helm template . \ - -s template/db/.yaml - -f values.yaml \ - --set gitSha=$GITSHA \ - --set db.settings.secrets.PGUSER=$PGUSER \ - --set db.settings.secrets.POSTGRES_USER=$POSTGRES_USER \ - --set db.settings.secrets.PGPASSWORD=$PGPASSWORD \ - --set db.settings.secrets.POSTGRES_PASSWORD=$POSTGRES_PASSWORD \ +# the chart on the gh-pages branch will provide +# the correct updated value otherwise it's defaulted +gitSha: "gitshaABC123" -db: - environment: "k8s" - enabled: true - image: - name: ghcr.io/stac-utils/pgstac - tag: v0.8.2 - command: - - "postgres" - - "-N" - - "500" - # toggle to true||false if you want the db test fixtures loaded - enable_data_fixtures: true - settings: - resources: - requests: - storage: "100Mi" - cpu: "512m" - memory: "1024Mi" - limits: - cpu: "512m" - memory: "1024Mi" - secrets: - POSTGRES_DB: "postgis" - POSTGRES_USER: "" - POSTGRES_PASSWORD: "" - POSTGRES_PORT: "5432" - POSTGRES_HOST: "pgstac" - POSTGRES_HOST_READER: "pgstac" - POSTGRES_HOST_WRITER: "pgstac" - DB_MIN_CONN_SIZE: "1" - DB_MAX_CONN_SIZE: "15" - # default connect: https://www.postgresql.org/docs/current/libpq-envars.html - PGDATA: "/var/lib/postgresql/data/pgdata" - PGUSER: "" - PGPASSWORD: "" - PGDATABASE: "postgis" - +###################### +# TESTING +###################### # only used in CI for running parallel helm installs testing: false -# the chart on the gh-pages branch will provide the correct updated value otherwise it's defaulted -gitSha: "gitshaABC123" +###################### +# SERVICE & INGRESS +###################### service: port: 8080 ingress: - # `"nginx"` will create a `kind:Service` with a `spec.port:ClusterIP` and a single Load Balancer and path rewrites for /vector, /stac, /raster + # `"nginx"` will create a `kind:Service` with a `spec.port:ClusterIP` + # and a single Load Balancer and path rewrites for /vector, /stac, /raster enabled: true className: "nginx" host: "" @@ -87,8 +39,95 @@ ingress: secretName: eoapi-tls certManager: false certManagerIssuer: letsencrypt-prod - cerrtManagerEmail: "" + certManagerEmail: "" + + +###################### +# DATABASE +###################### +comment_db: > + We use the crunchydata postgres operator/cluster charts as the k8s internal HA solution for eoapi. + Those charts are therefore listed as a dependency of this chart in `Chart.yaml`. + 0. make sure to install the operator first `helm install --set disable_check_for_upgrades=true pgo oci://registry.developers.crunchydata.com/crunchydata/pgo` + 1. it will create a postgres cluster: see `postgrescluster.instances` spec below + 2. will will also create user credentials and mount their secrets: see `postgrescluster.users` spec below + + The `postgrescluster` specs below are pass-through values to configure those separate + charts. For more information read https://access.crunchydata.com/documentation/postgres-operator/latest + +# this is declared as a dependency of eoapi in helm-chart/eoapi/Chart.yaml +postgrescluster: + postgresVersion: 16 + postGISVersion: 3.4 + pgBouncerReplicas: 1 + monitoring: false + backupsSize: 20Gi + instances: + - name: eoapi + replicas: 1 + dataVolumeClaimSpec: + accessModes: + - "ReadWriteOnce" + resources: + requests: + storage: "10Gi" + cpu: "1024m" + memory: "3048Mi" + # https://access.crunchydata.com/documentation/postgres-operator/latest/architecture/user-management + users: + # `postgres` always is a `SUPERUSER` and cannot be adjusted with `options` flag + # we use this block here to create secrets for the `postgres` user that we can mount and lookup + - name: postgres + databases: + - eoapi + - postgres + # if you try to grant `options: 'SUPERUSER'` below to your eoapi service user + # then they will not be able to connect through the pgbouncer connection pooler, so be aware + # https://access.crunchydata.com/documentation/crunchy-postgres-containers/2.4.2/container-specifications/crunchy-pgbouncer/ + # this user is currently unprivileged and the `pgstacBootstrap` container below will grant privileges + # when it boots via the `postgres` superuser above. + # see `/helm-chart/eoapi/templates/pgstacbootstrap/configmap.yaml:initdb-sh-config` + - name: eoapi + databases: + - eoapi + - postgres + # default `password.type` is ASCII which follows the character set US-ASCII + # but which can contain characters that `asyncpg` dislikes + # see https://github.com/MagicStack/asyncpg/issues/1151 + password: + type: AlphaNumeric + +# `pgstacBootstrap` is a pod that by default runs pgstac schema migrations +# and optionally loads some fixtures for testing and examples +# using the LOAD_FIXTURES env var below +pgstacBootstrap: + enabled: true + image: + name: ghcr.io/stac-utils/pgstac + tag: v0.8.2 + command: + - "sh" + args: + - "/opt/initdb/run-migrate-and-load.sh" + settings: + resources: + requests: + cpu: "512m" + memory: "1024Mi" + limits: + cpu: "512m" + memory: "1024Mi" + envVars: + # toggle to "false" if you don't want fixtures default loaded + LOAD_FIXTURES: "true" + # toggle to `true` if you want to keep the job running as db jumpbox + KEEP_ALIVE: "false" + + +###################### +# API SERVICES +###################### raster: enabled: true autoscaling: @@ -115,7 +154,6 @@ raster: - "--host=$(HOST)" - "--port=$(PORT)" settings: - timeout: 10 # https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ resources: limits: @@ -125,13 +163,6 @@ raster: cpu: "256m" memory: "3072Mi" envVars: - ############## - # uvicorn - ############## - HOST: "0.0.0.0" - PORT: "8080" - # https://www.uvicorn.org/settings/#production - WEB_CONCURRENCY: "10" ############## # titiler ############## @@ -145,6 +176,13 @@ raster: PYTHONWARNINGS: "ignore" VSI_CACHE: "TRUE" VSI_CACHE_SIZE: "5000000" # 5 MB (per file-handle) + ############## + # uvicorn + ############## + HOST: "0.0.0.0" + PORT: "8080" + # https://www.uvicorn.org/settings/#production + WEB_CONCURRENCY: "5" stac: enabled: true @@ -172,7 +210,6 @@ stac: - "--host=$(HOST)" - "--port=$(PORT)" settings: - timeout: 10 # https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ resources: limits: @@ -188,7 +225,7 @@ stac: HOST: "0.0.0.0" PORT: "8080" # https://www.uvicorn.org/settings/#production - WEB_CONCURRENCY: "10" + WEB_CONCURRENCY: "5" vector: enabled: true @@ -216,7 +253,6 @@ vector: - "--host=$(HOST)" - "--port=$(PORT)" settings: - timeout: 10 # https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ resources: limits: @@ -229,14 +265,16 @@ vector: ############## # tipg ############## - TIPG_CATALOG_TTL: "0" + TIPG_CATALOG_TTL: "300" + TIPG_DEBUG: "True" ############## # uvicorn ############## HOST: "0.0.0.0" PORT: "8080" # https://www.uvicorn.org/settings/#production - WEB_CONCURRENCY: "10" + WEB_CONCURRENCY: "5" + docServer: enabled: true