Use docker-compose --scale build=N to manually set N builders

readthedocs · Jun 8, 2020 · 97e6f5b · 97e6f5b
1 parent 28d9211
commit 97e6f5b
Show file tree

Hide file tree

Showing 3 changed files with 6 additions and 30 deletions.
diff --git a/dockerfiles/docker-compose.yml b/dockerfiles/docker-compose.yml
@@ -1,8 +1,7 @@
 version: '3'
 
 volumes:
-  build-default-user-builds:
-  build-large-user-builds:
+  build-user-builds:
   storage_data:
   postgres_data:
   postgres_backups_data:
@@ -88,7 +87,7 @@ services:
       readthedocs:
     command: ["../../docker/celery.sh"]
 
-  build-default: &build
+  build:
     volumes:
       - ${PWD}/common/dockerfiles/entrypoints/common.sh:/usr/src/app/docker/common.sh
       - ${PWD}/common/dockerfiles/entrypoints/build.sh:/usr/src/app/docker/build.sh
@@ -99,7 +98,7 @@ services:
       # between the build container (git commands), and the container that
       # is created inside the build container (sphinx commands).
       # Because of this, we need to use a shared volume between them
-      - build-default-user-builds:/usr/src/app/checkouts/readthedocs.org/user_builds
+      - build-user-builds:/usr/src/app/checkouts/readthedocs.org/user_builds
 
       # Docker in Docker
       - /var/run/docker.sock:/var/run/docker.sock
@@ -109,35 +108,12 @@ services:
       - cache
     environment:
       - DOCKER_NO_RELOAD
-      - BUILD_USER_BUILDS_VOLUME=build-default-user-builds
-      - CELERY_QUEUE=build:default
     stdin_open: true
     tty: true
     networks:
       readthedocs:
     command: ["../../docker/build.sh"]
 
-  build-large:
-    <<: *build
-    environment:
-      - DOCKER_NO_RELOAD
-      - BUILD_USER_BUILDS_VOLUME=build-large-user-builds
-      - CELERY_QUEUE=build:large
-    volumes:
-      - ${PWD}/common/dockerfiles/entrypoints/common.sh:/usr/src/app/docker/common.sh
-      - ${PWD}/common/dockerfiles/entrypoints/build.sh:/usr/src/app/docker/build.sh
-      - ${PWD}/../readthedocs-ext:/usr/src/app/checkouts/readthedocs-ext
-
-      # The python code at readthedocs/doc_builder/environments.py
-      # mounts `self.project.doc_path`. We need to share this path
-      # between the build container (git commands), and the container that
-      # is created inside the build container (sphinx commands).
-      # Because of this, we need to use a shared volume between them
-      - build-large-user-builds:/usr/src/app/checkouts/readthedocs.org/user_builds
-
-      # Docker in Docker
-      - /var/run/docker.sock:/var/run/docker.sock
-
   cache:
     image: redis:3.2.7
     networks:

diff --git a/dockerfiles/entrypoints/build.sh b/dockerfiles/entrypoints/build.sh
@@ -2,7 +2,7 @@
 
 ../../docker/common.sh
 
-CMD="python3 -m celery worker -A ${CELERY_APP_NAME}.worker -Ofair -c 1 -Q builder,celery,default,build01,{CELERY_QUEUE} -l DEBUG"
+CMD="python3 -m celery worker -A ${CELERY_APP_NAME}.worker -Ofair -c 1 -Q builder,celery,default,build01,build:default,build:large -l DEBUG"
 
 if [ -n "${DOCKER_NO_RELOAD}" ]; then
   echo "Running Docker with no reload"

diff --git a/dockerfiles/tasks.py b/dockerfiles/tasks.py
@@ -27,7 +27,7 @@ def down(c, volumes=False):
         c.run(f'{DOCKER_COMPOSE_COMMAND} down', pty=True)
 
 @task
-def up(c, no_search=False, init=False, no_reload=False):
+def up(c, no_search=False, init=False, no_reload=False, scale_build=1):
     """Start all the docker containers for a Read the Docs instance"""
     INIT = 'INIT='
     DOCKER_NO_RELOAD = 'DOCKER_NO_RELOAD='
@@ -37,7 +37,7 @@ def up(c, no_search=False, init=False, no_reload=False):
         DOCKER_NO_RELOAD = 'DOCKER_NO_RELOAD=t'
 
     if no_search:
-        c.run(f'{INIT} {DOCKER_NO_RELOAD} docker-compose -f {DOCKER_COMPOSE} -f {DOCKER_COMPOSE_OVERRIDE} up', pty=True)
+        c.run(f'{INIT} {DOCKER_NO_RELOAD} docker-compose -f {DOCKER_COMPOSE} -f {DOCKER_COMPOSE_OVERRIDE} up --scale build={scale_build}', pty=True)
     else:
         c.run(f'{INIT} {DOCKER_NO_RELOAD} {DOCKER_COMPOSE_COMMAND} up', pty=True)