From 28d921144b85162f8ecee295c55e2acb7887474f Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Mon, 25 May 2020 14:03:25 +0200 Subject: [PATCH 1/2] Use celery concurrency 1 and use 2 containers This will give us similar parity with production, where there can't be 2 builds building in the same server at the same time. --- dockerfiles/docker-compose.yml | 30 +++++++++++++++++++++++++++--- dockerfiles/entrypoints/build.sh | 2 +- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/dockerfiles/docker-compose.yml b/dockerfiles/docker-compose.yml index 605b715..7511344 100644 --- a/dockerfiles/docker-compose.yml +++ b/dockerfiles/docker-compose.yml @@ -1,7 +1,8 @@ version: '3' volumes: - build-user-builds: + build-default-user-builds: + build-large-user-builds: storage_data: postgres_data: postgres_backups_data: @@ -87,7 +88,7 @@ services: readthedocs: command: ["../../docker/celery.sh"] - build: + build-default: &build volumes: - ${PWD}/common/dockerfiles/entrypoints/common.sh:/usr/src/app/docker/common.sh - ${PWD}/common/dockerfiles/entrypoints/build.sh:/usr/src/app/docker/build.sh @@ -98,7 +99,7 @@ services: # between the build container (git commands), and the container that # is created inside the build container (sphinx commands). # Because of this, we need to use a shared volume between them - - build-user-builds:/usr/src/app/checkouts/readthedocs.org/user_builds + - build-default-user-builds:/usr/src/app/checkouts/readthedocs.org/user_builds # Docker in Docker - /var/run/docker.sock:/var/run/docker.sock @@ -108,12 +109,35 @@ services: - cache environment: - DOCKER_NO_RELOAD + - BUILD_USER_BUILDS_VOLUME=build-default-user-builds + - CELERY_QUEUE=build:default stdin_open: true tty: true networks: readthedocs: command: ["../../docker/build.sh"] + build-large: + <<: *build + environment: + - DOCKER_NO_RELOAD + - BUILD_USER_BUILDS_VOLUME=build-large-user-builds + - CELERY_QUEUE=build:large + volumes: + - ${PWD}/common/dockerfiles/entrypoints/common.sh:/usr/src/app/docker/common.sh + - ${PWD}/common/dockerfiles/entrypoints/build.sh:/usr/src/app/docker/build.sh + - ${PWD}/../readthedocs-ext:/usr/src/app/checkouts/readthedocs-ext + + # The python code at readthedocs/doc_builder/environments.py + # mounts `self.project.doc_path`. We need to share this path + # between the build container (git commands), and the container that + # is created inside the build container (sphinx commands). + # Because of this, we need to use a shared volume between them + - build-large-user-builds:/usr/src/app/checkouts/readthedocs.org/user_builds + + # Docker in Docker + - /var/run/docker.sock:/var/run/docker.sock + cache: image: redis:3.2.7 networks: diff --git a/dockerfiles/entrypoints/build.sh b/dockerfiles/entrypoints/build.sh index f604244..f13a3e0 100755 --- a/dockerfiles/entrypoints/build.sh +++ b/dockerfiles/entrypoints/build.sh @@ -2,7 +2,7 @@ ../../docker/common.sh -CMD="python3 -m celery worker -A ${CELERY_APP_NAME}.worker -Ofair -c 2 -Q builder,celery,default,build01 -l DEBUG" +CMD="python3 -m celery worker -A ${CELERY_APP_NAME}.worker -Ofair -c 1 -Q builder,celery,default,build01,{CELERY_QUEUE} -l DEBUG" if [ -n "${DOCKER_NO_RELOAD}" ]; then echo "Running Docker with no reload" From a78d01b21195012c767521b168765f43d1da5ff2 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Mon, 8 Jun 2020 12:31:28 +0200 Subject: [PATCH 2/2] Use `docker-compose --scale build=N` to manually set N builders --- dockerfiles/docker-compose.yml | 30 +++--------------------------- dockerfiles/entrypoints/build.sh | 2 +- dockerfiles/tasks.py | 7 ++++--- 3 files changed, 8 insertions(+), 31 deletions(-) diff --git a/dockerfiles/docker-compose.yml b/dockerfiles/docker-compose.yml index 7511344..605b715 100644 --- a/dockerfiles/docker-compose.yml +++ b/dockerfiles/docker-compose.yml @@ -1,8 +1,7 @@ version: '3' volumes: - build-default-user-builds: - build-large-user-builds: + build-user-builds: storage_data: postgres_data: postgres_backups_data: @@ -88,7 +87,7 @@ services: readthedocs: command: ["../../docker/celery.sh"] - build-default: &build + build: volumes: - ${PWD}/common/dockerfiles/entrypoints/common.sh:/usr/src/app/docker/common.sh - ${PWD}/common/dockerfiles/entrypoints/build.sh:/usr/src/app/docker/build.sh @@ -99,7 +98,7 @@ services: # between the build container (git commands), and the container that # is created inside the build container (sphinx commands). # Because of this, we need to use a shared volume between them - - build-default-user-builds:/usr/src/app/checkouts/readthedocs.org/user_builds + - build-user-builds:/usr/src/app/checkouts/readthedocs.org/user_builds # Docker in Docker - /var/run/docker.sock:/var/run/docker.sock @@ -109,35 +108,12 @@ services: - cache environment: - DOCKER_NO_RELOAD - - BUILD_USER_BUILDS_VOLUME=build-default-user-builds - - CELERY_QUEUE=build:default stdin_open: true tty: true networks: readthedocs: command: ["../../docker/build.sh"] - build-large: - <<: *build - environment: - - DOCKER_NO_RELOAD - - BUILD_USER_BUILDS_VOLUME=build-large-user-builds - - CELERY_QUEUE=build:large - volumes: - - ${PWD}/common/dockerfiles/entrypoints/common.sh:/usr/src/app/docker/common.sh - - ${PWD}/common/dockerfiles/entrypoints/build.sh:/usr/src/app/docker/build.sh - - ${PWD}/../readthedocs-ext:/usr/src/app/checkouts/readthedocs-ext - - # The python code at readthedocs/doc_builder/environments.py - # mounts `self.project.doc_path`. We need to share this path - # between the build container (git commands), and the container that - # is created inside the build container (sphinx commands). - # Because of this, we need to use a shared volume between them - - build-large-user-builds:/usr/src/app/checkouts/readthedocs.org/user_builds - - # Docker in Docker - - /var/run/docker.sock:/var/run/docker.sock - cache: image: redis:3.2.7 networks: diff --git a/dockerfiles/entrypoints/build.sh b/dockerfiles/entrypoints/build.sh index f13a3e0..2d7735a 100755 --- a/dockerfiles/entrypoints/build.sh +++ b/dockerfiles/entrypoints/build.sh @@ -2,7 +2,7 @@ ../../docker/common.sh -CMD="python3 -m celery worker -A ${CELERY_APP_NAME}.worker -Ofair -c 1 -Q builder,celery,default,build01,{CELERY_QUEUE} -l DEBUG" +CMD="python3 -m celery worker -A ${CELERY_APP_NAME}.worker -Ofair -c 1 -Q builder,celery,default,build01,build:default,build:large -l DEBUG" if [ -n "${DOCKER_NO_RELOAD}" ]; then echo "Running Docker with no reload" diff --git a/dockerfiles/tasks.py b/dockerfiles/tasks.py index 4470879..6051bf9 100644 --- a/dockerfiles/tasks.py +++ b/dockerfiles/tasks.py @@ -27,19 +27,20 @@ def down(c, volumes=False): c.run(f'{DOCKER_COMPOSE_COMMAND} down', pty=True) @task -def up(c, no_search=False, init=False, no_reload=False): +def up(c, no_search=False, init=False, no_reload=False, scale_build=1): """Start all the docker containers for a Read the Docs instance""" INIT = 'INIT=' DOCKER_NO_RELOAD = 'DOCKER_NO_RELOAD=' + SCALE = f'--scale build={scale_build}' if init: INIT = 'INIT=t' if no_reload: DOCKER_NO_RELOAD = 'DOCKER_NO_RELOAD=t' if no_search: - c.run(f'{INIT} {DOCKER_NO_RELOAD} docker-compose -f {DOCKER_COMPOSE} -f {DOCKER_COMPOSE_OVERRIDE} up', pty=True) + c.run(f'{INIT} {DOCKER_NO_RELOAD} docker-compose -f {DOCKER_COMPOSE} -f {DOCKER_COMPOSE_OVERRIDE} up {SCALE}', pty=True) else: - c.run(f'{INIT} {DOCKER_NO_RELOAD} {DOCKER_COMPOSE_COMMAND} up', pty=True) + c.run(f'{INIT} {DOCKER_NO_RELOAD} {DOCKER_COMPOSE_COMMAND} up {SCALE}', pty=True) @task def shell(c, running=False, container='web'):