Install python dev packages only for local development

api/Dockerfile

@@ -19,6 +19,8 @@ ENV PYTHONBUFFERED=1
 # Activate the virtualenv
 ENV PATH="/venv/bin:$PATH"
 
+ARG PIPENV_INSTALL_ARGS
+
 # - Install system packages needed for building Python dependencies
 # - Create a virtualenv inside `/venv`
 # - Install Pipenv to install Python dependencies
@@ -32,7 +34,7 @@ RUN apt-get update \
 COPY Pipfile Pipfile.lock ./
 
 # Install Python dependencies system-wide (uses the active virtualenv)
-RUN pipenv install --system --deploy --dev
+RUN pipenv install --system --deploy ${PIPENV_INSTALL_ARGS}
 
 #######
 # API #

api/justfile

@@ -31,6 +31,7 @@ wait-up: up
 init: wait-up
     cd .. && ./load_sample_data.sh
 
+# Recreate volumes and containers specific to the API profile from scratch
 recreate:
     just ../down -v
     just up "--force-recreate --build"

docker-compose.override.yml

@@ -0,0 +1,165 @@
+###############
+# Development #
+###############
+
+version: "2.4"
+services:
+  db:
+    profiles:
+      - ingestion_server
+      - api
+    image: postgres:13.2-alpine
+    ports:
+      - "50254:5432"
+    volumes:
+      - api-postgres:/var/lib/postgresql/data
+    env_file:
+      - docker/db/env.docker
+    healthcheck:
+      test: "pg_isready -U deploy -d openledger"
+
+  upstream_db:
+    profiles:
+      - ingestion_server
+      - api
+    build: ./docker/upstream_db/
+    image: openverse-upstream_db
+    expose:
+      - "5432"
+    volumes:
+      - catalog-postgres:/var/lib/postgresql/data
+      - ./sample_data:/sample_data
+    env_file:
+      - docker/upstream_db/env.docker
+    healthcheck:
+      test: "pg_isready -U deploy -d openledger"
+
+  plausible_db:
+    profiles:
+      - frontend
+    image: postgres:13.2-alpine
+    expose:
+      - "5432"
+    volumes:
+      - plausible-postgres:/var/lib/postgresql/data
+    env_file:
+      - ./docker/plausible_db/env.docker
+    healthcheck:
+      test: "pg_isready -U deploy -d plausible"
+
+  plausible_ch:
+    profiles:
+      - frontend
+    image: clickhouse/clickhouse-server:22.6-alpine
+    volumes:
+      - plausible-clickhouse:/var/lib/clickhouse
+      - ./docker/clickhouse/clickhouse-config.xml:/etc/clickhouse-server/config.d/logging.xml:ro
+      - ./docker/clickhouse/clickhouse-user-config.xml:/etc/clickhouse-server/users.d/logging.xml:ro
+    ulimits:
+      nofile:
+        soft: 262144
+        hard: 262144
+
+  plausible:
+    profiles:
+      - frontend
+    image: plausible/analytics:latest
+    ports:
+      - "50288:8000"
+    command: sh -c "sleep 10 && /entrypoint.sh db createdb && /entrypoint.sh db migrate && /entrypoint.sh run"
+    depends_on:
+      - plausible_db
+      - plausible_ch
+    env_file:
+      - docker/plausible/env.docker
+
+  cache:
+    profiles:
+      - api
+    image: redis:4.0.10
+    ports:
+      - "50263:6379"
+
+  es:
+    profiles:
+      - ingestion_server
+      - api
+    image: docker.elastic.co/elasticsearch/elasticsearch:7.12.0
+    ports:
+      - "50292:9200"
+    env_file:
+      - docker/es/env.docker
+    healthcheck:
+      test:
+        [
+          "CMD-SHELL",
+          "curl -si -XGET 'localhost:9200/_cluster/health?pretty' | grep -qE 'yellow|green'",
+        ]
+      interval: 10s
+      timeout: 60s
+      retries: 10
+    ulimits:
+      nofile:
+        soft: 65536
+        hard: 65536
+    # Memory limit for ES, as it tends to be a memory hoarder
+    # Set this value to an empty string to remove the limit
+    # https://docs.docker.com/compose/compose-file/compose-file-v2/#cpu-and-other-resources
+    mem_limit: ${ES_MEM_LIMIT:-4294967296} # 4 GiB in bytes
+    volumes:
+      - es-data:/usr/share/elasticsearch/data
+
+  web:
+    build:
+      context: ./api/
+      target: api
+      args:
+        SEMANTIC_VERSION: ${SEMANTIC_VERSION:-v1.0.0}
+        PIPENV_INSTALL_ARGS: "--dev"
+    depends_on:
+      - db
+      - es
+      - cache
+
+  ingestion_server:
+    build:
+      context: ./ingestion_server/
+      args:
+        PIPENV_INSTALL_ARGS: "--dev"
+    depends_on:
+      - db
+      - upstream_db
+      - es
+      - indexer_worker
+
+  indexer_worker:
+    build:
+      context: ./ingestion_server/
+      args:
+        PIPENV_INSTALL_ARGS: "--dev"
+    depends_on:
+      - db
+      - upstream_db
+      - es
+
+  proxy:
+    profiles:
+      - api
+    image: nginx:alpine
+    ports:
+      - "50200:9080"
+      - "50243:9443"
+    environment:
+      HTTPS_PORT: 50243 # See `ports` mapping above.
+    depends_on:
+      - web
+    volumes:
+      - ./docker/nginx/templates:/etc/nginx/templates
+      - ./docker/nginx/certs:/etc/nginx/certs
+
+volumes:
+  api-postgres:
+  catalog-postgres:
+  plausible-postgres:
+  plausible-clickhouse:
+  es-data:

docker-compose.yml

@@ -1,110 +1,5 @@
 version: "2.4"
 services:
-  db:
-    profiles:
-      - ingestion_server
-      - api
-    image: postgres:13.2-alpine
-    ports:
-      - "50254:5432"
-    volumes:
-      - api-postgres:/var/lib/postgresql/data
-    env_file:
-      - docker/db/env.docker
-    healthcheck:
-      test: "pg_isready -U deploy -d openledger"
-
-  upstream_db:
-    profiles:
-      - ingestion_server
-      - api
-    build: ./docker/upstream_db/
-    image: openverse-upstream_db
-    expose:
-      - "5432"
-    volumes:
-      - catalog-postgres:/var/lib/postgresql/data
-      - ./sample_data:/sample_data
-    env_file:
-      - docker/upstream_db/env.docker
-    healthcheck:
-      test: "pg_isready -U deploy -d openledger"
-
-  plausible_db:
-    profiles:
-      - frontend
-    image: postgres:13.2-alpine
-    expose:
-      - "5432"
-    volumes:
-      - plausible-postgres:/var/lib/postgresql/data
-    env_file:
-      - ./docker/plausible_db/env.docker
-    healthcheck:
-      test: "pg_isready -U deploy -d plausible"
-
-  plausible_ch:
-    profiles:
-      - frontend
-    image: clickhouse/clickhouse-server:22.6-alpine
-    volumes:
-      - plausible-clickhouse:/var/lib/clickhouse
-      - ./docker/clickhouse/clickhouse-config.xml:/etc/clickhouse-server/config.d/logging.xml:ro
-      - ./docker/clickhouse/clickhouse-user-config.xml:/etc/clickhouse-server/users.d/logging.xml:ro
-    ulimits:
-      nofile:
-        soft: 262144
-        hard: 262144
-
-  plausible:
-    profiles:
-      - frontend
-    image: plausible/analytics:latest
-    ports:
-      - "50288:8000"
-    command: sh -c "sleep 10 && /entrypoint.sh db createdb && /entrypoint.sh db migrate && /entrypoint.sh run"
-    depends_on:
-      - plausible_db
-      - plausible_ch
-    env_file:
-      - docker/plausible/env.docker
-
-  cache:
-    profiles:
-      - api
-    image: redis:4.0.10
-    ports:
-      - "50263:6379"
-
-  es:
-    profiles:
-      - ingestion_server
-      - api
-    image: docker.elastic.co/elasticsearch/elasticsearch:7.12.0
-    ports:
-      - "50292:9200"
-    env_file:
-      - docker/es/env.docker
-    healthcheck:
-      test:
-        [
-          "CMD-SHELL",
-          "curl -si -XGET 'localhost:9200/_cluster/health?pretty' | grep -qE 'yellow|green'",
-        ]
-      interval: 10s
-      timeout: 60s
-      retries: 10
-    ulimits:
-      nofile:
-        soft: 65536
-        hard: 65536
-    # Memory limit for ES, as it tends to be a memory hoarder
-    # Set this value to an empty string to remove the limit
-    # https://docs.docker.com/compose/compose-file/compose-file-v2/#cpu-and-other-resources
-    mem_limit: ${ES_MEM_LIMIT:-4294967296} # 4 GiB in bytes
-    volumes:
-      - es-data:/usr/share/elasticsearch/data
-
   web:
     profiles:
       - api
@@ -119,10 +14,6 @@ services:
     ports:
       - "50280:8000" # Django
       - "50230:3000" # Sphinx (unused by default; see `sphinx-live` recipe)
-    depends_on:
-      - db
-      - es
-      - cache
     env_file:
       - api/env.docker
       - api/.env
@@ -142,9 +33,6 @@ services:
     ports:
       - "50281:8001"
     depends_on:
-      - db
-      - upstream_db
-      - es
       - indexer_worker
     volumes:
       - ./ingestion_server:/ingestion_server
@@ -162,35 +50,9 @@ services:
     command: gunicorn -c ./gunicorn_worker.conf.py
     expose:
       - "8002"
-    depends_on:
-      - db
-      - upstream_db
-      - es
     volumes:
       - ./ingestion_server:/ingestion_server
     env_file:
       - ingestion_server/env.docker
     stdin_open: true
     tty: true
-
-  proxy:
-    profiles:
-      - api
-    image: nginx:alpine
-    ports:
-      - "50200:9080"
-      - "50243:9443"
-    environment:
-      HTTPS_PORT: 50243 # See `ports` mapping above.
-    depends_on:
-      - web
-    volumes:
-      - ./docker/nginx/templates:/etc/nginx/templates
-      - ./docker/nginx/certs:/etc/nginx/certs
-
-volumes:
-  api-postgres:
-  catalog-postgres:
-  plausible-postgres:
-  plausible-clickhouse:
-  es-data:

ingestion_server/Dockerfile

@@ -8,6 +8,8 @@ ENV PYTHONBUFFERED=1
 # Activate the virtualenv
 ENV PATH="/venv/bin:$PATH"
 
+ARG PIPENV_INSTALL_ARGS
+
 # - Install system packages needed for building Python dependencies
 # - Create a virtualenv inside `/venv`
 # - Install Pipenv to install Python dependencies
@@ -21,7 +23,7 @@ RUN apt-get update \
 COPY Pipfile Pipfile.lock ./
 
 # Install Python dependencies system-wide (uses the active virtualenv)
-RUN pipenv install --system --deploy --dev
+RUN pipenv install --system --deploy ${PIPENV_INSTALL_ARGS}
 
 ####################
 # Ingestion server #
@@ -53,12 +55,12 @@ RUN apt-get update \
       && mkdir /worker_state
 
 # Create a non-root user
-RUN useradd ingestionu
-RUN chown ingestionu /worker_state
-USER ingestionu
+RUN useradd opener
+RUN chown opener /worker_state
+USER opener
 
 # Copy code into the final image
-COPY --chown=ingestionu . /ingestion_server/
+COPY --chown=opener . /ingestion_server/
 
 # Exposes
 # - 8001: Gunicorn server for `ingestion_server` Falcon app