Merge pull request #5 from lsst-sqre/tickets/DM-32386

[DM-32386] Fixes found during test deployment
lsst-sqre · Nov 11, 2021 · 4d1c36d · 4d1c36d
2 parents dc7546f + 2b6332b
commit 4d1c36d
Show file tree

Hide file tree

Showing 34 changed files with 516 additions and 184 deletions.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -101,7 +101,7 @@ jobs:
         with:
           context: .
           push: true
-          tags: lsstsqre/vocutouts:${{ steps.vars.outputs.tag }}
+          tags: lsstsqre/vo-cutouts:${{ steps.vars.outputs.tag }}
           cache-from: type=local,src=/tmp/.buildx-cache
           cache-to: type=local,dest=/tmp/.buildx-cache-new
 

diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -2,7 +2,11 @@
 Change log
 ##########
 
-0.1.0 (unreleased)
+0.1.0 (2021-11-11)
 ==================
 
-- Initial simple version.
+Initial version, which uses a colon-separated tuple as the ``ID`` parameter and has an initial proof-of-concept backend that runs ``pipetask`` via ``subprocess`` and does not do correct conversion of coordinates to pixels.
+
+This is only a proof of concept release.
+Some of the things it does are very slow and block the whole asyncio process.
+The backend will be changed significantly before the first production release.
diff --git a/Dockerfile b/Dockerfile
@@ -14,7 +14,7 @@
 #   - Runs a non-root user.
 #   - Sets up the entrypoint and port.
 
-FROM python:3.9.5-slim-buster as base-image
+FROM python:3.9.8-slim-bullseye as base-image
 
 # Update system packages
 COPY scripts/install-base-packages.sh .
@@ -55,6 +55,9 @@ RUN useradd --create-home appuser
 # Copy the virtualenv
 COPY --from=install-image /opt/venv /opt/venv
 
+# Copy the startup script
+COPY scripts/start-frontend.sh /start-frontend.sh
+
 # Make sure we use the virtualenv
 ENV PATH="/opt/venv/bin:$PATH"
 
@@ -64,5 +67,5 @@ USER appuser
 # Expose the port.
 EXPOSE 8080
 
-# Run the application via uvicorn.
-CMD ["uvicorn", "vocutouts.main:app", "--host", "0.0.0.0", "--port", "8080"]
+# Run the application.
+CMD ["/start-frontend.sh"]
diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -0,0 +1,23 @@
+# Run Redis and PostgreSQL for testing with a local development server.
+# This is used by tox -e run, which starts gafaelfawr in the foreground.
+
+version: "3"
+services:
+  redis:
+    image: "redis:latest"
+    hostname: "redis"
+    container_name: "redis"
+    command:
+      - "redis-server"
+    ports:
+      - "6379:6379"
+  postgresql:
+    image: "postgres:latest"
+    hostname: "postgresql"
+    container_name: "postgresql"
+    environment:
+      POSTGRES_PASSWORD: "INSECURE-PASSWORD"
+      POSTGRES_USER: "vo-cutouts"
+      POSTGRES_DB: "vo-cutouts"
+    ports:
+      - "5432:5432"
diff --git a/pyproject.toml b/pyproject.toml
@@ -53,3 +53,9 @@ include_trailing_comma = true
 multi_line_output = 3
 known_first_party = ["vocutouts", "tests"]
 skip = ["docs/conf.py"]
+
+[tool.pytest.ini_options]
+python_files = [
+    "tests/*.py",
+    "tests/*/*.py"
+]
diff --git a/requirements/dev.txt b/requirements/dev.txt
@@ -16,7 +16,7 @@ asgiref==3.4.1
     #   uvicorn
 attrs==21.2.0
     # via pytest
-backports.entry-points-selectable==1.1.0
+backports.entry-points-selectable==1.1.1
     # via virtualenv
 certifi==2021.10.8
     # via
@@ -32,13 +32,13 @@ click==8.0.3
     # via
     #   -c requirements/main.txt
     #   uvicorn
-coverage[toml]==6.0.2
+coverage[toml]==6.1.2
     # via
     #   -r requirements/dev.in
     #   pytest-cov
 distlib==0.3.3
     # via virtualenv
-filelock==3.3.1
+filelock==3.3.2
     # via virtualenv
 greenlet==1.1.2
     # via
@@ -61,7 +61,7 @@ httpx==0.20.0
     # via
     #   -c requirements/main.txt
     #   -r requirements/dev.in
-identify==2.3.1
+identify==2.3.5
     # via pre-commit
 idna==3.3
     # via
@@ -78,17 +78,17 @@ mypy-extensions==0.4.3
     # via mypy
 nodeenv==1.6.0
     # via pre-commit
-packaging==21.0
+packaging==21.2
     # via pytest
 platformdirs==2.4.0
     # via virtualenv
 pluggy==1.0.0
     # via pytest
 pre-commit==2.15.0
     # via -r requirements/dev.in
-py==1.10.0
+py==1.11.0
     # via pytest
-pyparsing==3.0.2
+pyparsing==2.4.7
     # via packaging
 pytest==6.2.5
     # via
@@ -102,7 +102,7 @@ pytest-cov==3.0.0
     # via -r requirements/dev.in
 pytest-timeout==2.0.1
     # via -r requirements/dev.in
-python-dotenv==0.19.1
+python-dotenv==0.19.2
     # via
     #   -c requirements/main.txt
     #   uvicorn
@@ -126,11 +126,11 @@ sniffio==1.2.0
     #   asgi-lifespan
     #   httpcore
     #   httpx
-sqlalchemy[asyncio,mypy]==1.4.26
+sqlalchemy[asyncio,mypy]==1.4.27
     # via
     #   -c requirements/main.txt
     #   -r requirements/dev.in
-sqlalchemy2-stubs==0.0.2a18
+sqlalchemy2-stubs==0.0.2a19
     # via sqlalchemy
 toml==0.10.2
     # via
@@ -152,7 +152,7 @@ uvloop==0.16.0
     # via
     #   -c requirements/main.txt
     #   uvicorn
-virtualenv==20.9.0
+virtualenv==20.10.0
     # via pre-commit
 watchgod==0.7
     # via

diff --git a/requirements/main.in b/requirements/main.in
@@ -15,14 +15,18 @@ uvicorn[standard]
 # Other dependencies.
 astropy
 asyncpg
+click
 dramatiq[redis]
+google-cloud-storage
 jinja2
 psycopg2
 safir
 sqlalchemy[asyncio]
 structlog
 
 # Copied from requirements.txt for lsst.daf.butler.
+boto3>=1.13
+botocore>=1.15
 click>7.0
 httpx
 pydantic
@@ -32,6 +36,6 @@ pyyaml>=5.1
 psutil>=5.7
 
 # Not yet available on PyPI.
-git+https://github.com/lsst/daf_butler@master#egg=daf-butler
+git+https://github.com/lsst/daf_butler@tickets/DM-32386#egg=daf-butler
 git+https://github.com/lsst/sphgeom@master#egg=lsst_sphgeom
 git+https://github.com/lsst/utils@master#egg=lsst_utils
diff --git a/requirements/main.txt b/requirements/main.txt
@@ -16,16 +16,29 @@ astropy==4.3.1
     #   daf-butler
 asyncpg==0.24.0
     # via -r requirements/main.in
+boto3==1.20.4
+    # via -r requirements/main.in
+botocore==1.23.4
+    # via
+    #   -r requirements/main.in
+    #   boto3
+    #   s3transfer
+cachetools==4.2.4
+    # via google-auth
 certifi==2021.10.8
-    # via httpx
+    # via
+    #   httpx
+    #   requests
 charset-normalizer==2.0.7
-    # via httpx
+    # via
+    #   httpx
+    #   requests
 click==8.0.3
     # via
     #   -r requirements/main.in
     #   daf-butler
     #   uvicorn
-daf-butler @ git+https://github.com/lsst/daf_butler@master
+daf-butler @ git+https://github.com/lsst/daf_butler@tickets/DM-32386
     # via -r requirements/main.in
 deprecated==1.2.13
     # via daf-butler
@@ -35,6 +48,25 @@ fastapi==0.70.0
     # via
     #   -r requirements/main.in
     #   safir
+google-api-core==2.2.2
+    # via
+    #   google-cloud-core
+    #   google-cloud-storage
+google-auth==2.3.3
+    # via
+    #   google-api-core
+    #   google-cloud-core
+    #   google-cloud-storage
+google-cloud-core==2.2.1
+    # via google-cloud-storage
+google-cloud-storage==1.42.3
+    # via -r requirements/main.in
+google-crc32c==1.3.0
+    # via google-resumable-media
+google-resumable-media==2.1.0
+    # via google-cloud-storage
+googleapis-common-protos==1.53.0
+    # via google-api-core
 greenlet==1.1.2
     # via sqlalchemy
 gunicorn==20.1.0
@@ -54,9 +86,14 @@ httpx==0.20.0
 idna==3.3
     # via
     #   anyio
+    #   requests
     #   rfc3986
-jinja2==3.0.2
+jinja2==3.0.3
     # via -r requirements/main.in
+jmespath==0.10.0
+    # via
+    #   boto3
+    #   botocore
 lsst-sphgeom @ git+https://github.com/lsst/sphgeom@master
     # via
     #   -r requirements/main.in
@@ -67,24 +104,37 @@ lsst-utils @ git+https://github.com/lsst/utils@master
     #   daf-butler
 markupsafe==2.0.1
     # via jinja2
-numpy==1.21.3
+numpy==1.21.4
     # via
     #   astropy
     #   pyerfa
-prometheus-client==0.11.0
+prometheus-client==0.12.0
     # via dramatiq
+protobuf==3.19.1
+    # via
+    #   google-api-core
+    #   google-cloud-storage
+    #   googleapis-common-protos
 psutil==5.8.0
     # via -r requirements/main.in
 psycopg2==2.9.1
     # via -r requirements/main.in
+pyasn1==0.4.8
+    # via
+    #   pyasn1-modules
+    #   rsa
+pyasn1-modules==0.2.8
+    # via google-auth
 pydantic==1.8.2
     # via
     #   -r requirements/main.in
     #   fastapi
     #   safir
-pyerfa==2.0.0
+pyerfa==2.0.0.1
     # via astropy
-python-dotenv==0.19.1
+python-dateutil==2.8.2
+    # via botocore
+python-dotenv==0.19.2
     # via uvicorn
 python-multipart==0.0.5
     # via -r requirements/main.in
@@ -95,18 +145,30 @@ pyyaml==6.0
     #   uvicorn
 redis==3.5.3
     # via dramatiq
+requests==2.26.0
+    # via
+    #   google-api-core
+    #   google-cloud-storage
 rfc3986[idna2008]==1.5.0
     # via httpx
-safir==2.1.0
+rsa==4.7.2
+    # via google-auth
+s3transfer==0.5.0
+    # via boto3
+safir==2.2.0
     # via -r requirements/main.in
 six==1.16.0
-    # via python-multipart
+    # via
+    #   google-auth
+    #   google-cloud-storage
+    #   python-dateutil
+    #   python-multipart
 sniffio==1.2.0
     # via
     #   anyio
     #   httpcore
     #   httpx
-sqlalchemy[asyncio]==1.4.26
+sqlalchemy[asyncio]==1.4.27
     # via
     #   -r requirements/main.in
     #   daf-butler
@@ -121,6 +183,10 @@ structlog==21.2.0
     #   safir
 typing-extensions==3.10.0.2
     # via pydantic
+urllib3==1.26.7
+    # via
+    #   botocore
+    #   requests
 uvicorn[standard]==0.15.0
     # via -r requirements/main.in
 uvloop==0.16.0
@@ -129,7 +195,7 @@ watchgod==0.7
     # via uvicorn
 websockets==10.0
     # via uvicorn
-wrapt==1.13.2
+wrapt==1.13.3
     # via deprecated
 
 # The following packages are considered to be unsafe in a requirements file:

diff --git a/scripts/start-frontend.sh b/scripts/start-frontend.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+#
+# Start the vo-cutouts frontend application.  Currently creates the database
+# and then starts the server.  Eventually, this will call Alembic to handle
+# database migrations.
+
+set -eu
+
+vo-cutouts init
+uvicorn vocutouts.main:app --host 0.0.0.0 --port 8080
diff --git a/setup.cfg b/setup.cfg
@@ -39,7 +39,7 @@ where = src
 
 [options.entry_points]
 console_scripts =
-    vocutouts = vocutouts.cli:main
+    vo-cutouts = vocutouts.cli:main
 
 [flake8]
 max-line-length = 79