Remap ports to the non-conflicting range

.github/workflows/ci_cd.yml

@@ -142,10 +142,6 @@ jobs:
       - name: Start API, ingest and index test data
         run: just init
 
-      - name: Smoke test ReDoc site
-        run: |
-          curl --fail 'http://localhost:8000/v1/?format=openapi'
-
       - name: Run API tests
         run: just api-test
 
@@ -162,6 +158,31 @@ jobs:
           name: api_logs
           path: api_logs
 
+  test-redoc:
+    name: Check for API consumer docs
+    runs-on: ubuntu-latest
+    needs:
+      - build-images
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Setup just
+        uses: extractions/setup-just@v1
+
+      - name: Download all images
+        uses: actions/download-artifact@v2
+        with:
+          path: /tmp
+
+      - name: Load all images
+        run: |
+          docker load --input /tmp/api/api.tar
+          docker load --input /tmp/ingestion_server/ingestion_server.tar
+
+      - name: Test ReDoc site
+        run: just api-doctest
+
   django-check:
     name: Run Django check
     runs-on: ubuntu-latest

api/README.md

@@ -9,7 +9,7 @@ The API has two sets of documentation.
       ```bash
       just sphinx-live
       ```
-    - visiting the `https://localhost:3000/` endpoint
+    - visiting the `https://localhost:50230/` endpoint
   - contain more details on how to contribute to the API project
 
 - [Consumer docs](https://api.openverse.engineering/)

api/catalog/settings.py

@@ -60,14 +60,6 @@
         "0.0.0.0",
     ]
 
-# Domains that shortened links may point to
-SHORT_URL_WHITELIST = {
-    "api-dev.openverse.engineering",
-    "api.openverse.engineering",
-    "localhost:8000",
-}
-SHORT_URL_PATH_WHITELIST = ["/v1/list", "/v1/images/"]
-
 USE_S3 = config("USE_S3", default=False, cast=bool)
 
 # Application definition

api/docs/guides/quickstart.md

@@ -30,7 +30,7 @@ Ensure that you have installed `mkcert` (and the corresponding NSS tools). You c
    just up
    ```
 
-5. Point your browser to `http://localhost:8000`. You should be able to see the API documentation.
+5. Point your browser to `http://localhost:50280`. You should be able to see the API documentation.
    ![API ReDoc](/_static/api_redoc.png)
 
 6. Load the sample data. This could take a couple of minutes.

api/docs/guides/test.md

@@ -12,12 +12,3 @@
    ```bash
    just api-test
    ```
-
-## On the host
-
-1. Start the Django dev server. See the {doc}`"Run" <./run>` document for instructions.
-
-2. Run the tests in a terminal on the host.
-   ```bash
-   just api-testlocal
-   ```

api/run.sh

@@ -2,10 +2,10 @@
 
 set -e
 
-while [[ "$(curl --insecure -s -o /dev/null -w '%{http_code}' http://es:9200/)" != "200" ]]
-do
-  echo "Waiting for Elasticsearch connection..."
-  sleep 2
+es_url="http://${ELASTICSEARCH_URL:-es}:${ELASTICSEARCH_PORT:-9200}/_cluster/health"
+while [ "$(curl -s -o /dev/null -w '%{http_code}' "$es_url")" != "200" ]; do
+  echo "Waiting for Elasticsearch connection..." && sleep 5;
 done
+echo "Elasticsearch connection established!"
 
 exec "$@"

docker-compose.yml

@@ -3,7 +3,7 @@ services:
   db:
     image: postgres:13.2-alpine
     ports:
-      - "5432:5432"
+      - "50254:5432"
     env_file:
       - postgres/env.docker
     healthcheck:
@@ -13,17 +13,17 @@ services:
 
   thumbnails:
     image: ghcr.io/wordpress/openverse-imaginary:latest
-    ports:
-      - "8222:8222"
+    expose:
+      - "8222"
     environment:
       PORT: 8222
       MALLOC_ARENA_MAX: 2
     command: ["-enable-url-source"]
 
   upstream_db:
     image: postgres:13.2-alpine
-    ports:
-      - "5433:5432"
+    expose:
+      - "5432"
     volumes:
       - catalog-postgres:/var/lib/postgresql/data
       - ./sample_data:/sample_data
@@ -35,12 +35,12 @@ services:
   cache:
     image: redis:4.0.10
     ports:
-      - "6379:6379"
+      - "50263:6379"
 
   es:
     image: docker.elastic.co/elasticsearch/elasticsearch:7.12.0
     ports:
-      - "9200:9200"
+      - "50292:9200"
     environment:
       # disable XPack
       # https://www.elastic.co/guide/en/elasticsearch/reference/5.3/docker.html#_security_note
@@ -72,8 +72,8 @@ services:
     volumes:
       - ./api:/api
     ports:
-      - "8000:8000"
-      - "3000:3000"
+      - "50280:8000" # Django
+      - "50230:3000" # Sphinx (unused by default; see `sphinx-live` recipe)
     depends_on:
       - db
       - es
@@ -88,9 +88,10 @@ services:
     image: openverse-ingestion_server
     command: gunicorn -c ./gunicorn.conf.py
     ports:
-      - "8001:8001"
+      - "50281:8001"
     depends_on:
       - db
+      - upstream_db
       - es
       - indexer_worker
     volumes:
@@ -104,10 +105,11 @@ services:
     build: ./ingestion_server/
     image: openverse-ingestion_server
     command: gunicorn -c ./gunicorn_worker.conf.py
-    ports:
-      - "8002:8002"
+    expose:
+      - "8002"
     depends_on:
       - db
+      - upstream_db
       - es
     volumes:
       - ./ingestion_server:/ingestion_server
@@ -119,12 +121,15 @@ services:
   proxy:
     image: nginx:alpine
     ports:
-      - "9080:9080"
-      - "9443:9443"
+      - "50200:9080"
+      - "50243:9443"
+    environment:
+      HTTPS_PORT: 50243 # See `ports` mapping above.
     depends_on:
       - web
     volumes:
-      - ./nginx:/etc/nginx/conf.d
+      - ./nginx/templates:/etc/nginx/templates
+      - ./nginx/certs:/etc/nginx/certs
 
 volumes:
   api-postgres:

ingestion_server/test/gen_integration_compose.py

@@ -46,11 +46,16 @@ def _map_ports(conf: dict):
     """
 
     for service_name, service in conf["services"].items():
-        ports = service["ports"]
-        ports = [
-            f"{service_ports[service_name]}:{port.split(':')[1]}" for port in ports
-        ]
-        service["ports"] = ports
+        if "ports" in service:
+            ports = service["ports"]
+            ports = [
+                f"{service_ports[service_name]}:{port.split(':')[1]}" for port in ports
+            ]
+            service["ports"] = ports
+        elif "expose" in service and service_name in service_ports:
+            exposes = service["expose"]
+            ports = [f"{service_ports[service_name]}:{expose}" for expose in exposes]
+            service["ports"] = ports
 
 
 def _fixup_env(conf: dict):

ingestion_server/test/test_constants.py

@@ -5,5 +5,4 @@
     "db": 65432,  # from 5432
     "es": 60200,  # from 9200
     "ingestion_server": 60001,  # from 8001
-    "indexer_worker": 60002,  # from 8002
 }

justfile

@@ -73,8 +73,14 @@ env:
     cp api/env.template api/.env
     cp ingestion_server/env.template ingestion_server/.env
 
+# Ensure all services are up and running
+@_api-up:
+    just up
+    just wait-for-ing
+    just wait-for-web
+
 # Load sample data into the Docker Compose services
-init: up wait-for-es wait-for-ing wait-for-web
+init: _api-up
     ./load_sample_data.sh
 
 
@@ -109,17 +115,16 @@ cert:
 
 # Check the health of Elasticsearch
 @es-health es_host:
-    -curl -s -o /dev/null -w '%{http_code}' 'http://{{ es_host }}/_cluster/health?pretty'
+    -curl -s -o /dev/null -w '%{http_code}' 'http://{{ es_host }}/_cluster/health'
 
 # Wait for Elasticsearch to be healthy
-@wait-for-es es_host="localhost:9200":
+@wait-for-es es_host="localhost:50292":
     just _loop \
     '"$(just es-health {{ es_host }})" != "200"' \
     "Waiting for Elasticsearch to be healthy..."
 
-# Check if the media is indexed in Elasticsearch
 @check-index index="image":
-    -curl -sb -H "Accept:application/json" "http://localhost:9200/_cat/indices/{{ index }}" | grep -o "{{ index }}" | wc -l | xargs
+    -curl -sb -H "Accept:application/json" "http://localhost:50292/_cat/indices/{{ index }}" | grep -o "{{ index }}" | wc -l | xargs
 
 # Wait for the media to be indexed in Elasticsearch
 @wait-for-index index="image":
@@ -137,7 +142,7 @@ _ing-install:
     cd ingestion_server && pipenv install --dev
 
 # Perform the given action on the given model by invoking the ingestion-server API
-_ing-api data port="8001":
+_ing-api data port="50281":
     curl \
       -X POST \
       -H 'Content-Type: application/json' \
@@ -149,7 +154,7 @@ _ing-api data port="8001":
     -curl -s -o /dev/null -w '%{http_code}' 'http://{{ ing_host }}/'
 
 # Wait for the ingestion-server to be healthy
-@wait-for-ing ing_host="localhost:8001":
+@wait-for-ing ing_host="localhost:50281":
     just _loop \
     '"$(just ing-health {{ ing_host }})" != "200"' \
     "Waiting for the ingestion-server to be healthy..."
@@ -181,25 +186,22 @@ _api-install:
 
 # Check the health of the API
 @web-health:
-    -curl -s -o /dev/null -w '%{http_code}' 'http://localhost:8000/healthcheck'
+    -curl -s -o /dev/null -w '%{http_code}' 'http://localhost:50280/healthcheck'
 
 # Wait for the API to be healthy
 @wait-for-web:
     just _loop \
     '"$(just web-health)" != "200"' \
     "Waiting for the API to be healthy..."
 
-@_api-up: up wait-for-es wait-for-ing wait-for-web
-    exit 0
+# Run smoke test for the API docs
+api-doctest: _api-up
+    curl --fail 'http://localhost:50280/v1/?format=openapi'
 
 # Run API tests inside Docker
-@api-test *args: _api-up
+api-test *args: _api-up
     just exec web ./test/run_test.sh {{ args }}
 
-# Run API tests locally
-api-testlocal *args:
-    cd api && pipenv run ./test/run_test.sh {{ args }}
-
 # Run Django administrative commands locally
 dj-local +args:
     cd api && pipenv run python manage.py {{ args }}
@@ -210,7 +212,7 @@ dj-local +args:
 
 # Make a test cURL request to the API
 stats media="images":
-    curl "http://localhost:8000/v1/{{ media }}/stats/"
+    curl "http://localhost:50280/v1/{{ media }}/stats/"
 
 # Get Django shell with IPython
 ipython:
@@ -222,13 +224,13 @@ ipython:
 ##########
 
 # Compile Sphinx documentation into HTML output
-sphinx-make: up wait-for-es wait-for-ing wait-for-web
+sphinx-make: _api-up
     just exec web sphinx-build -M html docs/ build/
 
 # Serve Sphinx documentation via a live-reload server
-sphinx-live port="3000": up wait-for-es wait-for-ing wait-for-web
-    just exec web sphinx-autobuild --host 0.0.0.0 --port {{ port }} docs/ build/html/
+sphinx-live: _api-up
+    just exec web sphinx-autobuild --host 0.0.0.0 --port 3000 docs/ build/html/
 
 # Serve the Sphinx documentation from the HTML output directory
-sphinx-serve dir="api" port="3001":
+sphinx-serve dir="api" port="50231":
     cd {{ dir }}/build/html && pipenv run python -m http.server {{ port }}

nginx/web.conf → nginx/templates/web.conf.template

@@ -3,7 +3,7 @@ server {
   server_name localhost;
 
   location / {
-    return 301 https://$host:9443$request_uri;
+    return 301 https://$host:${HTTPS_PORT}$request_uri;
   }
 }
 
@@ -19,6 +19,6 @@ server {
     proxy_set_header X-Forwarded-Proto $scheme;
   }
 
-  ssl_certificate     /etc/nginx/conf.d/certs/openverse.crt;
-  ssl_certificate_key /etc/nginx/conf.d/certs/openverse.key;
+  ssl_certificate     /etc/nginx/certs/openverse.crt;
+  ssl_certificate_key /etc/nginx/certs/openverse.key;
 }