From 6d560a1b9224eeddb28f2d5c86dd22f6cac189a5 Mon Sep 17 00:00:00 2001 From: Tanner Doshier Date: Tue, 14 Jan 2025 15:25:09 -0500 Subject: [PATCH] app: Update `template-application-flask` to version 0.4.0 (#8) Manual migration for the template switch to the Platform CLI[1]. [1] https://github.com/navapbc/template-application-flask/commit/6a23ef57e615a3c3af47ea7ffeaebc206b0df85a --- .dockleconfig | 6 +----- .../{ci-openapi.yml => ci-app-openapi.yml} | 4 ++-- .github/workflows/ci-app.yml | 8 ++++---- .gitignore | 20 ------------------- .hadolint.yaml | 7 +------ .template-application-flask/app.yml | 5 +++++ .template-flask-version | 1 - .template-version | 1 - app/.dockleconfig | 8 ++++++++ app/.hadolint.yaml | 11 ++++++++++ app/Dockerfile | 3 +-- app/Makefile | 6 +++--- app/bin/wait-for-local-db.sh | 10 +++++++--- app/docker-compose.debug.yml | 5 ++--- app/docker-compose.override.yml.example | 3 +-- app/docker-compose.yml | 17 ++++++---------- app/local.env | 6 +++--- app/pyproject.toml | 3 +-- docs/app/README.md | 2 +- docs/app/getting-started.md | 2 +- ...enerated-openapi-spec-to-source-control.md | 2 +- 21 files changed, 59 insertions(+), 71 deletions(-) rename .github/workflows/{ci-openapi.yml => ci-app-openapi.yml} (92%) delete mode 100644 .gitignore create mode 100644 .template-application-flask/app.yml delete mode 100644 .template-flask-version delete mode 100644 .template-version create mode 100644 app/.dockleconfig create mode 100644 app/.hadolint.yaml diff --git a/.dockleconfig b/.dockleconfig index 2c7e44a..f7bc976 100644 --- a/.dockleconfig +++ b/.dockleconfig @@ -1,8 +1,4 @@ # This file is allows you to specify a list of files that is acceptable to Dockle # To allow multiple files, use a list of names, example below. Make sure to remove the leading # # DOCKLE_ACCEPT_FILES="file1,path/to/file2,file3/path,etc" -# https://github.com/goodwithtech/dockle#accept-suspicious-environment-variables--files--file-extensions -# DOCKLE_ACCEPT_FILES="file1,path/to/file2,file3/path,etc" - -# The apiflask/settings file is a stub file that apiflask creates, and has no sensitive data in. We are ignoring it since it is unused -DOCKLE_ACCEPT_FILES=app/.venv/lib/python3.12/site-packages/apiflask/settings.py +# https://github.com/goodwithtech/dockle#accept-suspicious-environment-variables--files--file-extensions \ No newline at end of file diff --git a/.github/workflows/ci-openapi.yml b/.github/workflows/ci-app-openapi.yml similarity index 92% rename from .github/workflows/ci-openapi.yml rename to .github/workflows/ci-app-openapi.yml index d101586..52a5ce7 100644 --- a/.github/workflows/ci-openapi.yml +++ b/.github/workflows/ci-app-openapi.yml @@ -6,7 +6,7 @@ on: paths: - app/** - Makefile - - .github/workflows/ci-openapi.yml + - .github/workflows/ci-app-openapi.yml defaults: run: @@ -24,7 +24,7 @@ jobs: update-openapi-docs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: # Checkout the feature branch associated with the pull request ref: ${{ github.head_ref }} diff --git a/.github/workflows/ci-app.yml b/.github/workflows/ci-app.yml index 1487460..5408b98 100644 --- a/.github/workflows/ci-app.yml +++ b/.github/workflows/ci-app.yml @@ -1,4 +1,4 @@ -name: CI - App +name: CI - app on: push: @@ -24,7 +24,7 @@ jobs: name: Lint runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Run format check run: make format-check @@ -35,7 +35,7 @@ jobs: name: Security scan runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Run security linting run: make lint-security @@ -43,7 +43,7 @@ jobs: name: Test runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Start tests run: | diff --git a/.gitignore b/.gitignore deleted file mode 100644 index 8fc4637..0000000 --- a/.gitignore +++ /dev/null @@ -1,20 +0,0 @@ -# Ignore Terraform lock files -# As of Feb 2023, Terraform lock files, while well intentioned, have a tendency -# to get into a confusing state that requires recreating the lock file, which -# defeats the purpose. Moreover, lock files are per environment, which can make -# it difficult for people to upgrade dependencies (e.g. upgrade an AWS provider) -# across environments if certain environments are locked down (e.g. production). -.terraform.lock.hcl - -# MacOS files -.DS_Store - -# Ignore develop-specific VS code settings files -.vscode - -# Ignore local environment variables which can contain environment secrets -.env -.envrc - -# Python testing stuff -*__pycache__* diff --git a/.hadolint.yaml b/.hadolint.yaml index 92af65c..d552e35 100644 --- a/.hadolint.yaml +++ b/.hadolint.yaml @@ -1,11 +1,6 @@ # List of settings and ignore or safelist findings for the hadolint scanner + # For more information on any settings you can specify, see the actions' documentation here # https://github.com/hadolint/hadolint#configure failure-threshold: warning ignored: [] -override: - info: - # Casts the apt-get install = finding as info - # We have this set since there is no way to specify version for - # build-essentials in the Dockerfile - - DL3008 diff --git a/.template-application-flask/app.yml b/.template-application-flask/app.yml new file mode 100644 index 0000000..9fa465a --- /dev/null +++ b/.template-application-flask/app.yml @@ -0,0 +1,5 @@ +# Changes here will be overwritten by Copier +_commit: v0.4.0 +_src_path: https://github.com/navapbc/template-application-flask +app_local_port: 8080 +app_name: app diff --git a/.template-flask-version b/.template-flask-version deleted file mode 100644 index d56a5df..0000000 --- a/.template-flask-version +++ /dev/null @@ -1 +0,0 @@ -f715d8d0be3ea1bfc3456e113ade445b51a0b80c diff --git a/.template-version b/.template-version deleted file mode 100644 index 06a4a93..0000000 --- a/.template-version +++ /dev/null @@ -1 +0,0 @@ -f7661c4b4dc2bdf1333af3c0f0310a49d0ca1449 diff --git a/app/.dockleconfig b/app/.dockleconfig new file mode 100644 index 0000000..48ec38a --- /dev/null +++ b/app/.dockleconfig @@ -0,0 +1,8 @@ +# This file is allows you to specify a list of files that is acceptable to Dockle +# To allow multiple files, use a list of names, example below. Make sure to remove the leading # +# DOCKLE_ACCEPT_FILES="file1,path/to/file2,file3/path,etc" +# https://github.com/goodwithtech/dockle#accept-suspicious-environment-variables--files--file-extensions +# DOCKLE_ACCEPT_FILES="file1,path/to/file2,file3/path,etc" + +# The apiflask/settings file is a stub file that apiflask creates, and has no sensitive data in. We are ignoring it since it is unused +DOCKLE_ACCEPT_FILES=app/.venv/lib/python3.13/site-packages/apiflask/settings.py diff --git a/app/.hadolint.yaml b/app/.hadolint.yaml new file mode 100644 index 0000000..92af65c --- /dev/null +++ b/app/.hadolint.yaml @@ -0,0 +1,11 @@ +# List of settings and ignore or safelist findings for the hadolint scanner +# For more information on any settings you can specify, see the actions' documentation here +# https://github.com/hadolint/hadolint#configure +failure-threshold: warning +ignored: [] +override: + info: + # Casts the apt-get install = finding as info + # We have this set since there is no way to specify version for + # build-essentials in the Dockerfile + - DL3008 diff --git a/app/Dockerfile b/app/Dockerfile index bb2a8d1..38ccb62 100644 --- a/app/Dockerfile +++ b/app/Dockerfile @@ -5,8 +5,7 @@ # needs to be called `release` in order to integrate with the repo's # top-level Makefile FROM python:3.13-slim AS base -# See /docs/app/README.md#Upgrading Python -# for details on upgrading your Python version +# See docs for details on upgrading your Python version # Install poetry, the package manager. # https://python-poetry.org diff --git a/app/Makefile b/app/Makefile index 2abddea..1d215fc 100644 --- a/app/Makefile +++ b/app/Makefile @@ -2,7 +2,7 @@ # Constants ################################################## -APP_NAME := main-app +APP_NAME := app # Adding this to the end of a script that outputs JSON will convert # it to a readable format with timestamps and color-coding. @@ -118,8 +118,8 @@ check: format-check lint test init-db: start-db db-migrate start-db: - docker compose up --detach main-db - ./bin/wait-for-local-db.sh + docker compose up --detach $(APP_NAME)-db + DB_NAME=$(APP_NAME)-db ./bin/wait-for-local-db.sh ## Destroy current DB, setup new one db-recreate: clean-volumes init-db diff --git a/app/bin/wait-for-local-db.sh b/app/bin/wait-for-local-db.sh index e54ee6b..4a2a142 100755 --- a/app/bin/wait-for-local-db.sh +++ b/app/bin/wait-for-local-db.sh @@ -10,10 +10,14 @@ NO_COLOR='\033[0m' MAX_WAIT_TIME=30 # seconds WAIT_TIME=0 +DB_NAME="${DB_NAME:=${PGDATABASE}}" + +DOCKER_DB_SERVICE_NAME="${DOCKER_DB_SERVICE_NAME:=database}" + # Use pg_isready to wait for the DB to be ready to accept connections # We check every 3 seconds and consider it failed if it gets to 30+ # https://www.postgresql.org/docs/current/app-pg-isready.html -until pg_isready -h localhost -d main-db -q; +until pg_isready -h localhost -d "${DB_NAME}" -q; do echo "waiting on Postgres DB to initialize..." sleep 3 @@ -21,8 +25,8 @@ do WAIT_TIME=$(($WAIT_TIME+3)) if [ $WAIT_TIME -gt $MAX_WAIT_TIME ] then - echo -e "${RED}ERROR: Database appears to not be starting up, running \"docker logs main-db\" to troubleshoot.${NO_COLOR}" - docker logs main-db + echo -e "${RED}ERROR: Database appears to not be starting up, running \"docker logs ${DOCKER_DB_SERVICE_NAME}\" to troubleshoot.${NO_COLOR}" + docker logs "${DOCKER_DB_SERVICE_NAME}" exit 1 fi done diff --git a/app/docker-compose.debug.yml b/app/docker-compose.debug.yml index 2b840fa..62d68a2 100644 --- a/app/docker-compose.debug.yml +++ b/app/docker-compose.debug.yml @@ -1,14 +1,13 @@ # run with `docker compose -f` -# combines ports and env vars with the main docker-compose.yml main-app service +# combines ports and env vars with the main docker-compose.yml app service services: - main-app: + app: build: context: ./ target: dev args: - RUN_UID=${RUN_UID:-4000} - RUN_USER=${RUN_USER:-app} - container_name: main-app env_file: local.env command: [ "poetry", "run", "python", "-m", "debugpy", diff --git a/app/docker-compose.override.yml.example b/app/docker-compose.override.yml.example index 94443e8..f43dd7b 100644 --- a/app/docker-compose.override.yml.example +++ b/app/docker-compose.override.yml.example @@ -1,6 +1,5 @@ services: - - main-app: + app: env_file: - local.env - .env diff --git a/app/docker-compose.yml b/app/docker-compose.yml index f340dff..0243688 100644 --- a/app/docker-compose.yml +++ b/app/docker-compose.yml @@ -1,21 +1,19 @@ services: - - main-db: + app-db: image: postgres:16-alpine - container_name: main-db command: postgres -c "log_lock_waits=on" -N 1000 -c "fsync=off" environment: POSTGRES_PASSWORD: secret123 # Set user to app to match user from template-infra (https://github.com/navapbc/template-infra) # This is also needed for the initial migration that alters defaut privileges to grant - # table privileges to the app user (see /app/src/db/migrations/versions/2023_08_10_default_table_privileges.py) + # table privileges to the app user (see src/db/migrations/versions/2023_08_10_default_table_privileges.py) POSTGRES_USER: app ports: - "5432:5432" volumes: - - dbdata:/var/lib/postgresql/data + - app-dbdata:/var/lib/postgresql/data - main-app: + app: build: context: ./ target: dev @@ -23,7 +21,6 @@ services: - RUN_UID=${RUN_UID:-4000} - RUN_USER=${RUN_USER:-app} command: ["poetry", "run", "flask", "--app", "src.app", "run", "--host", "0.0.0.0", "--port", "8080", "--reload"] - container_name: main-app env_file: - path: ./local.env required: true @@ -34,9 +31,7 @@ services: volumes: - ./:/app depends_on: - - main-db + - app-db volumes: - dbdata: - - + app-dbdata: diff --git a/app/local.env b/app/local.env index 4aed5e9..6f1947e 100644 --- a/app/local.env +++ b/app/local.env @@ -1,6 +1,6 @@ # Local environment variables # Used by docker-compose and it can be loaded -# by calling load_local_env_vars() from app/src/util/local.py +# by calling load_local_env_vars() from src/util/local.py # # To override values for your own personal local development # create an override.env file in the same directory. @@ -51,7 +51,7 @@ API_AUTH_TOKEN=LOCAL_AUTH_12345678 ############################ # Set DB_HOST to localhost if accessing a non-dockerized database -DB_HOST=main-db +DB_HOST=app-db DB_NAME=app DB_USER=app DB_SCHEMA=public @@ -71,7 +71,7 @@ HIDE_SQL_PARAMETER_LOGS=TRUE # committing them. Set these in your shell # by doing `export AWS_ACCESS_KEY_ID=whatever` # if you are running the app directly, or -# in your `app/.env` if you are running the +# in your `override.env` if you are running the # app in a Docker container AWS_ACCESS_KEY_ID=DO_NOT_SET_HERE AWS_SECRET_ACCESS_KEY=DO_NOT_SET_HERE diff --git a/app/pyproject.toml b/app/pyproject.toml index 1f67965..c3f374e 100644 --- a/app/pyproject.toml +++ b/app/pyproject.toml @@ -6,8 +6,7 @@ packages = [{ include = "src" }] authors = ["Nava Engineering "] [tool.poetry.dependencies] -# See /docs/app/README.md#Upgrading Python -# for details on upgrading your Python version +# See docs for details on upgrading your Python version python = "~3.13" SQLAlchemy = {version = "^2.0.21", extras = ["mypy"]} alembic = "^1.12.0" diff --git a/docs/app/README.md b/docs/app/README.md index 288c6ae..64920a7 100644 --- a/docs/app/README.md +++ b/docs/app/README.md @@ -117,7 +117,7 @@ The API can be run in debug mode that allows for remote attach debugging (curren - First create a file `./vscode/launch.json` - as shown below. (Default name of `Python: Remote Attach`) - Start the server in debug mode via `make start-debug` or `make start-debug run-logs`. - - This will start the `main-app` service with port 5678 exposed. + - This will start the `app` service with port 5678 exposed. - The server will start in waiting mode, waiting for you to attach the debugger (see `/src/app.py`) before continuing to run. diff --git a/docs/app/getting-started.md b/docs/app/getting-started.md index fc8d55c..1b08b29 100644 --- a/docs/app/getting-started.md +++ b/docs/app/getting-started.md @@ -32,7 +32,7 @@ A very simple [docker-compose.yml](/app/docker-compose.yml) has been included to ## (Optional) Configure local secrets -If you need to pass secrets to the application via environment variables, copy the provided [/app/docker-compose.override.yml.example](/docker-compose.override.yml.example) to `/app/docker-compose.override.yml`. Then create an `/app/.env` file with your secrets. The override will pass this file to the Docker container with your application. +If you need to pass secrets to the application via environment variables, copy the provided [docker-compose.override.yml.example](/app/docker-compose.override.yml.example) to `/app/docker-compose.override.yml`. Then create an `/app/.env` file with your secrets. The override will pass this file to the Docker container with your application. ```bash cp docker-compose.override.yml.example docker-compose.override.yml diff --git a/docs/decisions/0003-keep-generated-openapi-spec-to-source-control.md b/docs/decisions/0003-keep-generated-openapi-spec-to-source-control.md index 290d0e9..477a629 100644 --- a/docs/decisions/0003-keep-generated-openapi-spec-to-source-control.md +++ b/docs/decisions/0003-keep-generated-openapi-spec-to-source-control.md @@ -18,6 +18,6 @@ With the switch from Connexion to APIFlask (see [Connection replacement ADR](./0 We chose to keep the openapi.yml file in source control because we want changes to the API to be called out explicitly so that developers do not accidentally make backwards-incompatible changes to the API as part of a code change. This is particularly important since the API spec is now implicit as the OpenAPI specification is automatically generated from the code. -We chose to keep the openapi.yml file in sync with the API application automatically using a [CI workflow that generates the OpenAPI and pushes and changes to the PR branch](../../.github/workflows/ci-openapi.yml). This reduces the amount of manual work required by the engineer compared to a CI check that only checks for diffs but does not make the change. That said, we don't feel strongly about this decision so are open to changes in the future. +We chose to keep the openapi.yml file in sync with the API application automatically using a CI workflow that generates the OpenAPI and pushes and changes to the PR branch (`.github/workflows/ci--openapi.yml`). This reduces the amount of manual work required by the engineer compared to a CI check that only checks for diffs but does not make the change. That said, we don't feel strongly about this decision so are open to changes in the future. To minimize developer confusion, we chose to rename the `openapi.yml` file to `openapi.generated.yml` to clearly indicate that it is a generated file and not something that the developer should manually adjust.