diff --git a/api/Dockerfile b/api/Dockerfile index c8d64b31bd1..d1a05452f2e 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -19,6 +19,8 @@ ENV PYTHONBUFFERED=1 # Activate the virtualenv ENV PATH="/venv/bin:$PATH" +ARG PIPENV_INSTALL_ARGS + # - Install system packages needed for building Python dependencies # - Create a virtualenv inside `/venv` # - Install Pipenv to install Python dependencies @@ -32,7 +34,7 @@ RUN apt-get update \ COPY Pipfile Pipfile.lock ./ # Install Python dependencies system-wide (uses the active virtualenv) -RUN pipenv install --system --deploy --dev +RUN pipenv install --system --deploy ${PIPENV_INSTALL_ARGS} ####### # API # diff --git a/api/justfile b/api/justfile index 3563bea6c20..3d1f913a2bb 100644 --- a/api/justfile +++ b/api/justfile @@ -31,6 +31,7 @@ wait-up: up init: wait-up cd .. && ./load_sample_data.sh +# Recreate volumes and containers specific to the API profile from scratch recreate: just ../down -v just up "--force-recreate --build" diff --git a/docker-compose.override.yml b/docker-compose.override.yml new file mode 100644 index 00000000000..f8f3d320917 --- /dev/null +++ b/docker-compose.override.yml @@ -0,0 +1,165 @@ +############### +# Development # +############### + +version: "2.4" +services: + db: + profiles: + - ingestion_server + - api + image: postgres:13.2-alpine + ports: + - "50254:5432" + volumes: + - api-postgres:/var/lib/postgresql/data + env_file: + - docker/db/env.docker + healthcheck: + test: "pg_isready -U deploy -d openledger" + + upstream_db: + profiles: + - ingestion_server + - api + build: ./docker/upstream_db/ + image: openverse-upstream_db + expose: + - "5432" + volumes: + - catalog-postgres:/var/lib/postgresql/data + - ./sample_data:/sample_data + env_file: + - docker/upstream_db/env.docker + healthcheck: + test: "pg_isready -U deploy -d openledger" + + plausible_db: + profiles: + - frontend + image: postgres:13.2-alpine + expose: + - "5432" + volumes: + - plausible-postgres:/var/lib/postgresql/data + env_file: + - ./docker/plausible_db/env.docker + healthcheck: + test: "pg_isready -U deploy -d plausible" + + plausible_ch: + profiles: + - frontend + image: clickhouse/clickhouse-server:22.6-alpine + volumes: + - plausible-clickhouse:/var/lib/clickhouse + - ./docker/clickhouse/clickhouse-config.xml:/etc/clickhouse-server/config.d/logging.xml:ro + - ./docker/clickhouse/clickhouse-user-config.xml:/etc/clickhouse-server/users.d/logging.xml:ro + ulimits: + nofile: + soft: 262144 + hard: 262144 + + plausible: + profiles: + - frontend + image: plausible/analytics:latest + ports: + - "50288:8000" + command: sh -c "sleep 10 && /entrypoint.sh db createdb && /entrypoint.sh db migrate && /entrypoint.sh run" + depends_on: + - plausible_db + - plausible_ch + env_file: + - docker/plausible/env.docker + + cache: + profiles: + - api + image: redis:4.0.10 + ports: + - "50263:6379" + + es: + profiles: + - ingestion_server + - api + image: docker.elastic.co/elasticsearch/elasticsearch:7.12.0 + ports: + - "50292:9200" + env_file: + - docker/es/env.docker + healthcheck: + test: + [ + "CMD-SHELL", + "curl -si -XGET 'localhost:9200/_cluster/health?pretty' | grep -qE 'yellow|green'", + ] + interval: 10s + timeout: 60s + retries: 10 + ulimits: + nofile: + soft: 65536 + hard: 65536 + # Memory limit for ES, as it tends to be a memory hoarder + # Set this value to an empty string to remove the limit + # https://docs.docker.com/compose/compose-file/compose-file-v2/#cpu-and-other-resources + mem_limit: ${ES_MEM_LIMIT:-4294967296} # 4 GiB in bytes + volumes: + - es-data:/usr/share/elasticsearch/data + + web: + build: + context: ./api/ + target: api + args: + SEMANTIC_VERSION: ${SEMANTIC_VERSION:-v1.0.0} + PIPENV_INSTALL_ARGS: "--dev" + depends_on: + - db + - es + - cache + + ingestion_server: + build: + context: ./ingestion_server/ + args: + PIPENV_INSTALL_ARGS: "--dev" + depends_on: + - db + - upstream_db + - es + - indexer_worker + + indexer_worker: + build: + context: ./ingestion_server/ + args: + PIPENV_INSTALL_ARGS: "--dev" + depends_on: + - db + - upstream_db + - es + + proxy: + profiles: + - api + image: nginx:alpine + ports: + - "50200:9080" + - "50243:9443" + environment: + HTTPS_PORT: 50243 # See `ports` mapping above. + depends_on: + - web + volumes: + - ./docker/nginx/templates:/etc/nginx/templates + - ./docker/nginx/certs:/etc/nginx/certs + +volumes: + api-postgres: + catalog-postgres: + plausible-postgres: + plausible-clickhouse: + es-data: diff --git a/docker-compose.yml b/docker-compose.yml index b1b4121c6fc..d02ce417bca 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,110 +1,5 @@ version: "2.4" services: - db: - profiles: - - ingestion_server - - api - image: postgres:13.2-alpine - ports: - - "50254:5432" - volumes: - - api-postgres:/var/lib/postgresql/data - env_file: - - docker/db/env.docker - healthcheck: - test: "pg_isready -U deploy -d openledger" - - upstream_db: - profiles: - - ingestion_server - - api - build: ./docker/upstream_db/ - image: openverse-upstream_db - expose: - - "5432" - volumes: - - catalog-postgres:/var/lib/postgresql/data - - ./sample_data:/sample_data - env_file: - - docker/upstream_db/env.docker - healthcheck: - test: "pg_isready -U deploy -d openledger" - - plausible_db: - profiles: - - frontend - image: postgres:13.2-alpine - expose: - - "5432" - volumes: - - plausible-postgres:/var/lib/postgresql/data - env_file: - - ./docker/plausible_db/env.docker - healthcheck: - test: "pg_isready -U deploy -d plausible" - - plausible_ch: - profiles: - - frontend - image: clickhouse/clickhouse-server:22.6-alpine - volumes: - - plausible-clickhouse:/var/lib/clickhouse - - ./docker/clickhouse/clickhouse-config.xml:/etc/clickhouse-server/config.d/logging.xml:ro - - ./docker/clickhouse/clickhouse-user-config.xml:/etc/clickhouse-server/users.d/logging.xml:ro - ulimits: - nofile: - soft: 262144 - hard: 262144 - - plausible: - profiles: - - frontend - image: plausible/analytics:latest - ports: - - "50288:8000" - command: sh -c "sleep 10 && /entrypoint.sh db createdb && /entrypoint.sh db migrate && /entrypoint.sh run" - depends_on: - - plausible_db - - plausible_ch - env_file: - - docker/plausible/env.docker - - cache: - profiles: - - api - image: redis:4.0.10 - ports: - - "50263:6379" - - es: - profiles: - - ingestion_server - - api - image: docker.elastic.co/elasticsearch/elasticsearch:7.12.0 - ports: - - "50292:9200" - env_file: - - docker/es/env.docker - healthcheck: - test: - [ - "CMD-SHELL", - "curl -si -XGET 'localhost:9200/_cluster/health?pretty' | grep -qE 'yellow|green'", - ] - interval: 10s - timeout: 60s - retries: 10 - ulimits: - nofile: - soft: 65536 - hard: 65536 - # Memory limit for ES, as it tends to be a memory hoarder - # Set this value to an empty string to remove the limit - # https://docs.docker.com/compose/compose-file/compose-file-v2/#cpu-and-other-resources - mem_limit: ${ES_MEM_LIMIT:-4294967296} # 4 GiB in bytes - volumes: - - es-data:/usr/share/elasticsearch/data - web: profiles: - api @@ -119,10 +14,6 @@ services: ports: - "50280:8000" # Django - "50230:3000" # Sphinx (unused by default; see `sphinx-live` recipe) - depends_on: - - db - - es - - cache env_file: - api/env.docker - api/.env @@ -142,9 +33,6 @@ services: ports: - "50281:8001" depends_on: - - db - - upstream_db - - es - indexer_worker volumes: - ./ingestion_server:/ingestion_server @@ -162,35 +50,9 @@ services: command: gunicorn -c ./gunicorn_worker.conf.py expose: - "8002" - depends_on: - - db - - upstream_db - - es volumes: - ./ingestion_server:/ingestion_server env_file: - ingestion_server/env.docker stdin_open: true tty: true - - proxy: - profiles: - - api - image: nginx:alpine - ports: - - "50200:9080" - - "50243:9443" - environment: - HTTPS_PORT: 50243 # See `ports` mapping above. - depends_on: - - web - volumes: - - ./docker/nginx/templates:/etc/nginx/templates - - ./docker/nginx/certs:/etc/nginx/certs - -volumes: - api-postgres: - catalog-postgres: - plausible-postgres: - plausible-clickhouse: - es-data: diff --git a/ingestion_server/Dockerfile b/ingestion_server/Dockerfile index 40893b7eac7..9821c535f96 100644 --- a/ingestion_server/Dockerfile +++ b/ingestion_server/Dockerfile @@ -8,6 +8,8 @@ ENV PYTHONBUFFERED=1 # Activate the virtualenv ENV PATH="/venv/bin:$PATH" +ARG PIPENV_INSTALL_ARGS + # - Install system packages needed for building Python dependencies # - Create a virtualenv inside `/venv` # - Install Pipenv to install Python dependencies @@ -21,7 +23,7 @@ RUN apt-get update \ COPY Pipfile Pipfile.lock ./ # Install Python dependencies system-wide (uses the active virtualenv) -RUN pipenv install --system --deploy --dev +RUN pipenv install --system --deploy ${PIPENV_INSTALL_ARGS} #################### # Ingestion server # @@ -53,12 +55,12 @@ RUN apt-get update \ && mkdir /worker_state # Create a non-root user -RUN useradd ingestionu -RUN chown ingestionu /worker_state -USER ingestionu +RUN useradd opener +RUN chown opener /worker_state +USER opener # Copy code into the final image -COPY --chown=ingestionu . /ingestion_server/ +COPY --chown=opener . /ingestion_server/ # Exposes # - 8001: Gunicorn server for `ingestion_server` Falcon app diff --git a/justfile b/justfile index 91c879c8e19..78db8960279 100644 --- a/justfile +++ b/justfile @@ -102,7 +102,7 @@ env: DOCKER_FILE := "-f " + ( if IS_PROD == "true" { "ingestion_server/docker-compose.yml" } - else { "docker-compose.yml" } + else { "docker-compose.yml -f docker-compose.override.yml" } ) # Run `docker-compose` configured with the correct files and environment