From fde487c9e8f5701067a779f1e65ab571ab759a7b Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Thu, 21 Apr 2022 04:18:11 +1000 Subject: [PATCH] Reassert idp-sql and django tests (#7779) ## Description #7342 introduced a change where nox would check for `test_*.py` and `*_test.py` files. Django and IDP-SQL had their tests in `test/`, so this glob didn't find these files, and these tests haven't been run for a bit. Outstanding issues with these tests: *dynamic PR status update follows* --- * Django: Postgres 9.6 EOL ``` Step #6 - "Migrate database": django.db.utils.ProgrammingError: column c.relispartition does not exist Step #6 - "Migrate database": LINE 3: CASE WHEN c.relispartition THEN 'p' WHEN c.relki... Step #6 - "Migrate database": ``` Debugging seems to indicate this happens on old versions of postgres. [Django 4.0 no longer supports Postgres 9.6](https://docs.djangoproject.com/en/4.0/releases/4.0/#dropped-support-for-postgresql-9-6). `test-instance-pg` is Postgres 9.6, which is EOL as of November 2021. Suggest creating a newer versioned instance of a postgres testing database (happy to help here where I can). Update: test-instance-pg has been updated to Postgres 14 --- * IDP: Linting issues ``` ./middleware.py:31:29: ANN401 Dynamically typed expressions (typing.Any) are disallowed def decorated_function(*args: Any, **kwargs: Any) -> Any: ^ ./middleware.py:31:42: ANN401 Dynamically typed expressions (typing.Any) are disallowed def decorated_function(*args: Any, **kwargs: Any) -> Any: ^ ./middleware.py:31:61: ANN401 Dynamically typed expressions (typing.Any) are disallowed def decorated_function(*args: Any, **kwargs: Any) -> Any: ^ ``` This is being addressed as of v2.9.0 of flake8-annotations. https://github.com/sco1/flake8-annotations/pull/136 It will require adding `--allow-star-arg-any` to the flake8 call. Update: while this flag could be used, I opted to use the `TypeVar` method in #7789 --- ## Checklist - [ ] I have followed [Sample Guidelines from AUTHORING_GUIDE.MD](https://github.com/GoogleCloudPlatform/python-docs-samples/blob/main/AUTHORING_GUIDE.md) - [ ] README is updated to include [all relevant information](https://github.com/GoogleCloudPlatform/python-docs-samples/blob/main/AUTHORING_GUIDE.md#readme-file) - [ ] **Tests** pass: `nox -s py-3.6` (see [Test Environment Setup](https://github.com/GoogleCloudPlatform/python-docs-samples/blob/main/AUTHORING_GUIDE.md#test-environment-setup)) - [ ] **Lint** pass: `nox -s lint` (see [Test Environment Setup](https://github.com/GoogleCloudPlatform/python-docs-samples/blob/main/AUTHORING_GUIDE.md#test-environment-setup)) - [ ] These samples need a new **API enabled** in testing projects to pass (let us know which ones) - [ ] These samples need a new/updated **env vars** in testing projects set to pass (let us know which ones) - [ ] Please **merge** this PR for me once it is approved. - [ ] This sample adds a new sample directory, and I updated the [CODEOWNERS file](https://github.com/GoogleCloudPlatform/python-docs-samples/blob/main/.github/CODEOWNERS) with the codeowners for this sample --- run/django/{test => }/e2e_test.py | 4 ++-- run/django/{test => }/e2e_test_cleanup.yaml | 14 +++++++------- run/django/{test => }/e2e_test_setup.yaml | 16 ++++++++-------- run/django/{test => }/retry.sh | 0 run/idp-sql/README.md | 4 ++-- run/idp-sql/{test => }/e2e_test.py | 4 ++-- run/idp-sql/{test => }/e2e_test_cleanup.yaml | 8 ++++---- run/idp-sql/{test => }/e2e_test_setup.yaml | 10 +++++----- run/idp-sql/middleware.py | 6 ++++-- run/idp-sql/{test => }/retry.sh | 0 10 files changed, 34 insertions(+), 32 deletions(-) rename run/django/{test => }/e2e_test.py (98%) rename run/django/{test => }/e2e_test_cleanup.yaml (71%) rename run/django/{test => }/e2e_test_setup.yaml (89%) rename run/django/{test => }/retry.sh (100%) rename run/idp-sql/{test => }/e2e_test.py (98%) rename run/idp-sql/{test => }/e2e_test_cleanup.yaml (75%) rename run/idp-sql/{test => }/e2e_test_setup.yaml (87%) rename run/idp-sql/{test => }/retry.sh (100%) diff --git a/run/django/test/e2e_test.py b/run/django/e2e_test.py similarity index 98% rename from run/django/test/e2e_test.py rename to run/django/e2e_test.py index 8acf44c3dc57..50432e728395 100644 --- a/run/django/test/e2e_test.py +++ b/run/django/e2e_test.py @@ -92,7 +92,7 @@ def deployed_service() -> str: "--project", GOOGLE_CLOUD_PROJECT, "--config", - "./test/e2e_test_setup.yaml", + "./e2e_test_setup.yaml", "--substitutions", ] + substitutions @@ -124,7 +124,7 @@ def deployed_service() -> str: "--project", GOOGLE_CLOUD_PROJECT, "--config", - "./test/e2e_test_cleanup.yaml", + "./e2e_test_cleanup.yaml", "--substitutions", ] + substitutions diff --git a/run/django/test/e2e_test_cleanup.yaml b/run/django/e2e_test_cleanup.yaml similarity index 71% rename from run/django/test/e2e_test_cleanup.yaml rename to run/django/e2e_test_cleanup.yaml index 6fc9c52027e4..06e227f21cd9 100644 --- a/run/django/test/e2e_test_cleanup.yaml +++ b/run/django/e2e_test_cleanup.yaml @@ -19,25 +19,25 @@ steps: args: - "-ec" - | - ./test/retry.sh "gcloud secrets describe ${_SECRET_SETTINGS_NAME}" \ + ./retry.sh "gcloud secrets describe ${_SECRET_SETTINGS_NAME}" \ "gcloud secrets delete ${_SECRET_SETTINGS_NAME} --quiet --project $PROJECT_ID" - ./test/retry.sh "gcloud secrets describe ${_SECRET_PASSWORD_NAME}" \ + ./retry.sh "gcloud secrets describe ${_SECRET_PASSWORD_NAME}" \ "gcloud secrets delete ${_SECRET_PASSWORD_NAME} --quiet --project $PROJECT_ID" - ./test/retry.sh "gsutil ls gs://${_STORAGE_BUCKET}" \ + ./retry.sh "gsutil ls gs://${_STORAGE_BUCKET}" \ "gsutil -m rm -r gs://${_STORAGE_BUCKET}" - ./test/retry.sh "gcloud container images describe gcr.io/${PROJECT_ID}/${_SERVICE}:${_VERSION}" \ + ./retry.sh "gcloud container images describe gcr.io/${PROJECT_ID}/${_SERVICE}:${_VERSION}" \ "gcloud container images delete gcr.io/${PROJECT_ID}/${_SERVICE}:${_VERSION} --quiet" - ./test/retry.sh "gcloud run services describe ${_SERVICE} --region ${_REGION} --platform ${_PLATFORM}" \ + ./retry.sh "gcloud run services describe ${_SERVICE} --region ${_REGION} --platform ${_PLATFORM}" \ "gcloud run services delete ${_SERVICE} --region ${_REGION} --platform ${_PLATFORM} --async --quiet" - WAIT=30 ./test/retry.sh "gcloud sql databases describe ${_DB_NAME} --instance ${_DB_INSTANCE} --project $PROJECT_ID" \ + WAIT=30 ./retry.sh "gcloud sql databases describe ${_DB_NAME} --instance ${_DB_INSTANCE} --project $PROJECT_ID" \ "gcloud sql databases delete ${_DB_NAME} --instance ${_DB_INSTANCE} --quiet --project $PROJECT_ID" - ./test/retry.sh "gcloud sql users list --filter \"name=${_DB_USER}\" --instance ${_DB_INSTANCE}" \ + ./retry.sh "gcloud sql users list --filter \"name=${_DB_USER}\" --instance ${_DB_INSTANCE}" \ "gcloud sql users delete ${_DB_USER} --instance ${_DB_INSTANCE} --quiet --project $PROJECT_ID" substitutions: diff --git a/run/django/test/e2e_test_setup.yaml b/run/django/e2e_test_setup.yaml similarity index 89% rename from run/django/test/e2e_test_setup.yaml rename to run/django/e2e_test_setup.yaml index d20093b99f26..dcaea33c75a3 100644 --- a/run/django/test/e2e_test_setup.yaml +++ b/run/django/e2e_test_setup.yaml @@ -19,7 +19,7 @@ steps: args: - "-c" - | - WAIT=30 ./test/retry.sh "gcloud sql databases create ${_DB_NAME} \ + WAIT=30 ./retry.sh "gcloud sql databases create ${_DB_NAME} \ --instance ${_DB_INSTANCE} \ --project ${PROJECT_ID}" @@ -30,7 +30,7 @@ steps: - "-c" - | echo -n "${_DB_PASS}" > db_password - ./test/retry.sh "gcloud sql users create ${_DB_USER} \ + ./retry.sh "gcloud sql users create ${_DB_USER} \ --password $(cat db_password) \ --instance ${_DB_INSTANCE} \ --project ${PROJECT_ID}" @@ -42,7 +42,7 @@ steps: args: - "-c" - | - ./test/retry.sh "gsutil mb \ + ./retry.sh "gsutil mb \ -l ${_REGION} \ -p ${PROJECT_ID} \ gs://${_STORAGE_BUCKET}" @@ -59,7 +59,7 @@ steps: SECRET_KEY=$(cat /dev/urandom | LC_ALL=C tr -dc '[:alpha:]' | fold -w 30 | head -n1) PASSWORD_NAME=${_SECRET_PASSWORD_NAME}" > ${_SECRET_SETTINGS_NAME} - ./test/retry.sh "gcloud secrets create ${_SECRET_SETTINGS_NAME} \ + ./retry.sh "gcloud secrets create ${_SECRET_SETTINGS_NAME} \ --project $PROJECT_ID \ --data-file=${_SECRET_SETTINGS_NAME}" @@ -72,7 +72,7 @@ steps: echo -n "${_SECRET_PASSWORD_VALUE}" > ${_SECRET_PASSWORD_NAME} - ./test/retry.sh "gcloud secrets create ${_SECRET_PASSWORD_NAME} \ + ./retry.sh "gcloud secrets create ${_SECRET_PASSWORD_NAME} \ --project $PROJECT_ID \ --data-file=${_SECRET_PASSWORD_NAME}" @@ -89,7 +89,7 @@ steps: args: - "-c" - | - ./test/retry.sh "docker build -t gcr.io/${PROJECT_ID}/${_SERVICE}:${_VERSION} ." + ./retry.sh "docker build -t gcr.io/${PROJECT_ID}/${_SERVICE}:${_VERSION} ." - id: "Push Container Image" name: "gcr.io/cloud-builders/docker" @@ -97,7 +97,7 @@ steps: args: - "-c" - | - ./test/retry.sh "docker push gcr.io/${PROJECT_ID}/${_SERVICE}:${_VERSION}" + ./retry.sh "docker push gcr.io/${PROJECT_ID}/${_SERVICE}:${_VERSION}" - id: "Migrate database" name: "gcr.io/google-appengine/exec-wrapper" @@ -142,7 +142,7 @@ steps: args: - "-c" - | - ./test/retry.sh "gcloud run deploy ${_SERVICE} \ + ./retry.sh "gcloud run deploy ${_SERVICE} \ --project $PROJECT_ID \ --image gcr.io/${PROJECT_ID}/${_SERVICE}:${_VERSION} \ --no-allow-unauthenticated \ diff --git a/run/django/test/retry.sh b/run/django/retry.sh similarity index 100% rename from run/django/test/retry.sh rename to run/django/retry.sh diff --git a/run/idp-sql/README.md b/run/idp-sql/README.md index 0170eb9c37fe..4da4ac4e2e53 100644 --- a/run/idp-sql/README.md +++ b/run/idp-sql/README.md @@ -72,7 +72,7 @@ to be set. ### Unit tests ``` -pytest test/test_app.py +pytest test_app.py ``` ### System Tests @@ -82,5 +82,5 @@ export GOOGLE_CLOUD_PROJECT= export CLOUD_SQL_CONNECTION_NAME= export DB_PASSWORD= export IDP_KEY= # See tutorial for creation of this key ("API_KEY") -pytest test/e2e_test.py +pytest e2e_test.py ``` diff --git a/run/idp-sql/test/e2e_test.py b/run/idp-sql/e2e_test.py similarity index 98% rename from run/idp-sql/test/e2e_test.py rename to run/idp-sql/e2e_test.py index 2182acf5a59b..3d4c25790e60 100644 --- a/run/idp-sql/test/e2e_test.py +++ b/run/idp-sql/e2e_test.py @@ -114,7 +114,7 @@ def deployed_service() -> str: "--project", GOOGLE_CLOUD_PROJECT, "--config", - "./test/e2e_test_setup.yaml", + "./e2e_test_setup.yaml", "--substitutions", ] + substitutions @@ -166,7 +166,7 @@ def deployed_service() -> str: "--project", GOOGLE_CLOUD_PROJECT, "--config", - "./test/e2e_test_cleanup.yaml", + "./e2e_test_cleanup.yaml", "--substitutions", ] + substitutions diff --git a/run/idp-sql/test/e2e_test_cleanup.yaml b/run/idp-sql/e2e_test_cleanup.yaml similarity index 75% rename from run/idp-sql/test/e2e_test_cleanup.yaml rename to run/idp-sql/e2e_test_cleanup.yaml index 55324b199aaf..16a0714053b0 100644 --- a/run/idp-sql/test/e2e_test_cleanup.yaml +++ b/run/idp-sql/e2e_test_cleanup.yaml @@ -20,16 +20,16 @@ steps: args: - '-ec' - | - ./test/retry.sh "gcloud run services describe ${_SERVICE} --region ${_REGION} --platform ${_PLATFORM}" \ + ./retry.sh "gcloud run services describe ${_SERVICE} --region ${_REGION} --platform ${_PLATFORM}" \ "gcloud run services delete ${_SERVICE} --region ${_REGION} --platform ${_PLATFORM} --quiet" - ./test/retry.sh "gcloud secrets describe ${_SERVICE}-secrets" \ + ./retry.sh "gcloud secrets describe ${_SERVICE}-secrets" \ "gcloud secrets delete ${_SERVICE}-secrets --quiet --project $PROJECT_ID" - ./test/retry.sh "gcloud container images describe gcr.io/${PROJECT_ID}/${_SERVICE}:${_VERSION}" \ + ./retry.sh "gcloud container images describe gcr.io/${PROJECT_ID}/${_SERVICE}:${_VERSION}" \ "gcloud container images delete gcr.io/${PROJECT_ID}/${_SERVICE}:${_VERSION} --quiet" - WAIT=30 ./test/retry.sh "gcloud sql databases describe ${_DB_NAME} --instance ${_DB_INSTANCE} --project $PROJECT_ID" \ + WAIT=30 ./retry.sh "gcloud sql databases describe ${_DB_NAME} --instance ${_DB_INSTANCE} --project $PROJECT_ID" \ "gcloud sql databases delete ${_DB_NAME} --instance ${_DB_INSTANCE} --quiet --project $PROJECT_ID" substitutions: diff --git a/run/idp-sql/test/e2e_test_setup.yaml b/run/idp-sql/e2e_test_setup.yaml similarity index 87% rename from run/idp-sql/test/e2e_test_setup.yaml rename to run/idp-sql/e2e_test_setup.yaml index 6fbcb2884746..703abd8482f5 100644 --- a/run/idp-sql/test/e2e_test_setup.yaml +++ b/run/idp-sql/e2e_test_setup.yaml @@ -20,7 +20,7 @@ steps: args: - '-c' - | - WAIT=30 ./test/retry.sh "gcloud sql databases create ${_DB_NAME} \ + WAIT=30 ./retry.sh "gcloud sql databases create ${_DB_NAME} \ --instance ${_DB_INSTANCE} \ --project $PROJECT_ID" @@ -35,7 +35,7 @@ steps: sed -i "s/\"DB_NAME\": \"postgres\"/\"DB_NAME\": \"${_DB_NAME}\"/" postgres-secrets.json sed -i "s/\"DB_USER\": \"postgres\"/\"DB_USER\": \"${_DB_USER}\"/" postgres-secrets.json - ./test/retry.sh "gcloud secrets create ${_SERVICE}-secrets \ + ./retry.sh "gcloud secrets create ${_SERVICE}-secrets \ --project $PROJECT_ID \ --replication-policy="automatic" \ --data-file=postgres-secrets.json" \ @@ -47,7 +47,7 @@ steps: args: - '-c' - | - ./test/retry.sh "docker build -t gcr.io/${PROJECT_ID}/${_SERVICE}:${_VERSION} ." + ./retry.sh "docker build -t gcr.io/${PROJECT_ID}/${_SERVICE}:${_VERSION} ." - id: 'Push Container Image' name: 'gcr.io/cloud-builders/docker' @@ -55,7 +55,7 @@ steps: args: - '-c' - | - ./test/retry.sh "docker push gcr.io/${PROJECT_ID}/${_SERVICE}:${_VERSION}" + ./retry.sh "docker push gcr.io/${PROJECT_ID}/${_SERVICE}:${_VERSION}" - id: 'Deploy to Cloud Run' name: 'gcr.io/cloud-builders/gcloud:latest' @@ -63,7 +63,7 @@ steps: args: - '-c' - | - ./test/retry.sh "gcloud beta run deploy ${_SERVICE} \ + ./retry.sh "gcloud beta run deploy ${_SERVICE} \ --project $PROJECT_ID \ --image gcr.io/${PROJECT_ID}/${_SERVICE}:${_VERSION} \ --allow-unauthenticated \ diff --git a/run/idp-sql/middleware.py b/run/idp-sql/middleware.py index cebfe4d47f68..d359762aa68d 100644 --- a/run/idp-sql/middleware.py +++ b/run/idp-sql/middleware.py @@ -14,7 +14,7 @@ from functools import wraps -from typing import Any, Callable, Dict +from typing import Callable, Dict, TypeVar import firebase_admin from firebase_admin import auth # noqa: F401 @@ -22,13 +22,15 @@ import structlog +a = TypeVar("a") + default_app = firebase_admin.initialize_app() # [START cloudrun_user_auth_jwt] def jwt_authenticated(func: Callable[..., int]) -> Callable[..., int]: @wraps(func) - def decorated_function(*args: Any, **kwargs: Any) -> Any: + def decorated_function(*args: a, **kwargs: a) -> a: header = request.headers.get("Authorization", None) if header: token = header.split(" ")[1] diff --git a/run/idp-sql/test/retry.sh b/run/idp-sql/retry.sh similarity index 100% rename from run/idp-sql/test/retry.sh rename to run/idp-sql/retry.sh