Merge pull request #8453 from readthedocs/humitos/build-new-docker-image

readthedocs/doc_builder/environments.py

@@ -782,8 +782,8 @@ def __init__(self, *args, **kwargs):
         if self.project.has_feature(Feature.USE_TESTING_BUILD_IMAGE):
             self.container_image = 'readthedocs/build:testing'
         # the image set by user or,
-        if self.config and self.config.build.image:
-            self.container_image = self.config.build.image
+        if self.config and self.config.docker_image:
+            self.container_image = self.config.docker_image
         # the image overridden by the project (manually set by an admin).
         if self.project.container_image:
             self.container_image = self.project.container_image

readthedocs/doc_builder/python_environments.py

@@ -8,6 +8,7 @@
 import logging
 import os
 import shutil
+import tarfile
 
 import yaml
 from django.conf import settings
@@ -22,6 +23,7 @@
 from readthedocs.doc_builder.loader import get_builder_class
 from readthedocs.projects.constants import LOG_TEMPLATE
 from readthedocs.projects.models import Feature
+from readthedocs.storage import build_tools_storage
 
 log = logging.getLogger(__name__)
 
@@ -72,6 +74,112 @@ def delete_existing_venv_dir(self):
             )
             shutil.rmtree(venv_dir)
 
+    def install_build_tools(self):
+        if settings.RTD_DOCKER_COMPOSE:
+            # Create a symlink for ``root`` user to use the same ``.asdf``
+            # installation than ``docs`` user. Required for local building
+            # since everything is run as ``root`` when using Local Development
+            # instance
+            cmd = [
+                'ln',
+                '-s',
+                os.path.join(settings.RTD_DOCKER_WORKDIR, '.asdf'),
+                '/root/.asdf',
+            ]
+            self.build_env.run(
+                *cmd,
+            )
+
+        for tool, version in self.config.build.tools.items():
+            version = version.full_version  # e.g. 3.9 -> 3.9.7
+
+            # TODO: generate the correct path for the Python version
+            # tool_path = f'{self.config.build.os}/{tool}/2021-08-30/{version}.tar.gz'
+            tool_path = f'{self.config.build.os}-{tool}-{version}.tar.gz'
+            tool_version_cached = build_tools_storage.exists(tool_path)
+            if tool_version_cached:
+                remote_fd = build_tools_storage.open(tool_path, mode='rb')
+                with tarfile.open(fileobj=remote_fd) as tar:
+                    # Extract it on the shared path between host and Docker container
+                    extract_path = os.path.join(self.project.doc_path, 'tools')
+                    tar.extractall(extract_path)
+
+                    # Move the extracted content to the ``asdf`` installation
+                    cmd = [
+                        'mv',
+                        f'{extract_path}/{version}',
+                        os.path.join(
+                            settings.RTD_DOCKER_WORKDIR,
+                            f'.asdf/installs/{tool}/{version}',
+                        ),
+                    ]
+                    self.build_env.run(
+                        *cmd,
+                    )
+            else:
+                log.debug(
+                    'Cached version for tool not found. os=%s tool=%s version=% filename=%s',
+                    self.config.build.os,
+                    tool,
+                    version,
+                    tool_path,
+                )
+                # If the tool version selected is not available from the
+                # cache we compile it at build time
+                cmd = [
+                    # TODO: make this environment variable to work
+                    # 'PYTHON_CONFIGURE_OPTS="--enable-shared"',
+                    'asdf',
+                    'install',
+                    tool,
+                    version,
+                ]
+                self.build_env.run(
+                    *cmd,
+                )
+
+            # Make the tool version chosen by the user the default one
+            cmd = [
+                'asdf',
+                'global',
+                tool,
+                version,
+            ]
+            self.build_env.run(
+                *cmd,
+            )
+
+            # Recreate shims for this tool to make the new version
+            # installed available
+            cmd = [
+                'asdf',
+                'reshim',
+                tool,
+            ]
+            self.build_env.run(
+                *cmd,
+            )
+
+            if all([
+                    tool == 'python',
+                    # Do not install them if the tool version was cached
+                    not tool_version_cached,
+                    # Do not install them on conda/mamba
+                    self.config.python_interpreter == 'python',
+            ]):
+                # Install our own requirements if the version is compiled
+                cmd = [
+                    'python',
+                    '-mpip',
+                    'install',
+                    '-U',
+                    'virtualenv',
+                    'setuptools',
+                ]
+                self.build_env.run(
+                    *cmd,
+                )
+
     def install_requirements(self):
         """Install all requirements from the config object."""
         for install in self.config.python.install:
@@ -214,7 +322,7 @@ def is_obsolete(self):
         env_build_hash = env_build.get('hash', None)
 
         if isinstance(self.build_env, DockerBuildEnvironment):
-            build_image = self.config.build.image or DOCKER_IMAGE
+            build_image = self.config.docker_image
             image_hash = self.build_env.image_hash
         else:
             # e.g. LocalBuildEnvironment
@@ -269,7 +377,7 @@ def save_environment_json(self):
         }
 
         if isinstance(self.build_env, DockerBuildEnvironment):
-            build_image = self.config.build.image or DOCKER_IMAGE
+            build_image = self.config.docker_image
             data.update({
                 'build': {
                     'image': build_image,
@@ -317,6 +425,7 @@ def setup_base(self):
         cli_args.append(
             self.venv_path(),
         )
+
         self.build_env.run(
             self.config.python_interpreter,
             *cli_args,
@@ -496,6 +605,11 @@ def conda_bin_name(self):
 
         See https://github.com/QuantStack/mamba
         """
+        # Config file using ``build.tools.python``
+        if self.config.using_build_tools:
+            return self.config.python_interpreter
+
+        # Config file using ``conda``
         if self.project.has_feature(Feature.CONDA_USES_MAMBA):
             return 'mamba'
         return 'conda'
@@ -556,8 +670,12 @@ def setup_base(self):
             self._append_core_requirements()
             self._show_environment_yaml()
 
-        # TODO: remove it when ``mamba`` is installed in the Docker image
-        if self.project.has_feature(Feature.CONDA_USES_MAMBA):
+        if all([
+                # The project has CONDA_USES_MAMBA feature enabled and,
+                self.project.has_feature(Feature.CONDA_USES_MAMBA),
+                # the project is not using ``build.tools``
+                not self.config.using_build_tools,
+        ]):
             self._install_mamba()
 
         self.build_env.run(

readthedocs/projects/tasks.py

@@ -762,7 +762,10 @@ def run_build(self, record):
         # Environment used for building code, usually with Docker
         with self.build_env:
             python_env_cls = Virtualenv
-            if self.config.conda is not None:
+            if any([
+                    self.config.conda is not None,
+                    self.config.python_interpreter in ('conda', 'mamba'),
+            ]):
                 log.info(
                     LOG_TEMPLATE,
                     {
@@ -1164,6 +1167,10 @@ def setup_python_environment(self):
         else:
             self.python_env.delete_existing_build_dir()
 
+        # Install all ``build.tools`` specified by the user
+        if self.config.using_build_tools:
+            self.python_env.install_build_tools()
+
         self.python_env.setup_base()
         self.python_env.save_environment_json()
         self.python_env.install_core_requirements()

readthedocs/rtd_tests/tests/test_celery.py

@@ -6,6 +6,7 @@
 from unittest.mock import MagicMock, patch
 
 from allauth.socialaccount.models import SocialAccount
+from django.conf import settings
 from django.contrib.auth.models import User
 from django.test import TestCase
 from django_dynamic_fixture import get
@@ -541,3 +542,156 @@ def test_install_apt_packages(self, load_config, run):
                 user='root:root',
             )
         )
+
+    @patch('readthedocs.projects.tasks.UpdateDocsTaskStep.build_docs', new=MagicMock)
+    @patch('readthedocs.projects.tasks.UpdateDocsTaskStep.setup_vcs', new=MagicMock)
+    @patch.object(BuildEnvironment, 'run')
+    @patch('readthedocs.doc_builder.config.load_config')
+    def test_build_tools(self, load_config, build_run):
+        config = BuildConfigV2(
+            {},
+            {
+                'version': 2,
+                'build': {
+                    'os': 'ubuntu-20.04',
+                    'tools': {
+                        'python': '3.10',
+                        'nodejs': '16',
+                        'rust': '1.55',
+                        'golang': '1.17',
+                    },
+                },
+            },
+            source_file='readthedocs.yml',
+        )
+        config.validate()
+        load_config.return_value = config
+
+        version = self.project.versions.first()
+        build = get(
+            Build,
+            project=self.project,
+            version=version,
+        )
+        with mock_api(self.repo):
+            result = tasks.update_docs_task.delay(
+                version.pk,
+                build_pk=build.pk,
+                record=False,
+                intersphinx=False,
+            )
+        self.assertTrue(result.successful())
+        self.assertEqual(build_run.call_count, 14)
+
+        python_version = settings.RTD_DOCKER_BUILD_SETTINGS['tools']['python']['3.10']
+        nodejs_version = settings.RTD_DOCKER_BUILD_SETTINGS['tools']['nodejs']['16']
+        rust_version = settings.RTD_DOCKER_BUILD_SETTINGS['tools']['rust']['1.55']
+        golang_version = settings.RTD_DOCKER_BUILD_SETTINGS['tools']['golang']['1.17']
+        self.assertEqual(
+            build_run.call_args_list,
+            [
+                mock.call('asdf', 'install', 'python', python_version),
+                mock.call('asdf', 'global', 'python', python_version),
+                mock.call('asdf', 'reshim', 'python'),
+                mock.call('python', '-mpip', 'install', '-U', 'virtualenv', 'setuptools'),
+                mock.call('asdf', 'install', 'nodejs', nodejs_version),
+                mock.call('asdf', 'global', 'nodejs', nodejs_version),
+                mock.call('asdf', 'reshim', 'nodejs'),
+                mock.call('asdf', 'install', 'rust', rust_version),
+                mock.call('asdf', 'global', 'rust', rust_version),
+                mock.call('asdf', 'reshim', 'rust'),
+                mock.call('asdf', 'install', 'golang', golang_version),
+                mock.call('asdf', 'global', 'golang', golang_version),
+                mock.call('asdf', 'reshim', 'golang'),
+                mock.ANY,
+            ],
+        )
+
+    @patch('readthedocs.projects.tasks.UpdateDocsTaskStep.build_docs', new=MagicMock)
+    @patch('readthedocs.projects.tasks.UpdateDocsTaskStep.setup_vcs', new=MagicMock)
+    @patch('readthedocs.doc_builder.python_environments.tarfile')
+    @patch('readthedocs.doc_builder.python_environments.build_tools_storage')
+    @patch.object(BuildEnvironment, 'run')
+    @patch('readthedocs.doc_builder.config.load_config')
+    def test_build_tools_cached(self, load_config, build_run, build_tools_storage, tarfile):
+        config = BuildConfigV2(
+            {},
+            {
+                'version': 2,
+                'build': {
+                    'os': 'ubuntu-20.04',
+                    'tools': {
+                        'python': '3.10',
+                        'nodejs': '16',
+                        'rust': '1.55',
+                        'golang': '1.17',
+                    },
+                },
+            },
+            source_file='readthedocs.yml',
+        )
+        config.validate()
+        load_config.return_value = config
+
+        build_tools_storage.open.return_value = b''
+        build_tools_storage.exists.return_value = True
+        tarfile.open.return_value.__enter__.return_value.extract_all.return_value = None
+
+        version = self.project.versions.first()
+        build = get(
+            Build,
+            project=self.project,
+            version=version,
+        )
+        with mock_api(self.repo):
+            result = tasks.update_docs_task.delay(
+                version.pk,
+                build_pk=build.pk,
+                record=False,
+                intersphinx=False,
+            )
+        self.assertTrue(result.successful())
+        self.assertEqual(build_run.call_count, 13)
+
+        python_version = settings.RTD_DOCKER_BUILD_SETTINGS['tools']['python']['3.10']
+        nodejs_version = settings.RTD_DOCKER_BUILD_SETTINGS['tools']['nodejs']['16']
+        rust_version = settings.RTD_DOCKER_BUILD_SETTINGS['tools']['rust']['1.55']
+        golang_version = settings.RTD_DOCKER_BUILD_SETTINGS['tools']['golang']['1.17']
+        self.assertEqual(
+            # NOTE: casting the first argument as `list()` shows a better diff
+            # explaining where the problem is
+            list(build_run.call_args_list),
+            [
+                mock.call(
+                    'mv',
+                    # Use mock.ANY here because path differs when ran locally
+                    # and on CircleCI
+                    mock.ANY,
+                    f'/home/docs/.asdf/installs/python/{python_version}',
+                ),
+                mock.call('asdf', 'global', 'python', python_version),
+                mock.call('asdf', 'reshim', 'python'),
+                mock.call(
+                    'mv',
+                    mock.ANY,
+                    f'/home/docs/.asdf/installs/nodejs/{nodejs_version}',
+                ),
+                mock.call('asdf', 'global', 'nodejs', nodejs_version),
+                mock.call('asdf', 'reshim', 'nodejs'),
+                mock.call(
+                    'mv',
+                    mock.ANY,
+                    f'/home/docs/.asdf/installs/rust/{rust_version}',
+                ),
+                mock.call('asdf', 'global', 'rust', rust_version),
+                mock.call('asdf', 'reshim', 'rust'),
+                mock.call(
+                    'mv',
+                    mock.ANY,
+                    f'/home/docs/.asdf/installs/golang/{golang_version}',
+                ),
+                mock.call('asdf', 'global', 'golang', golang_version),
+                mock.call('asdf', 'reshim', 'golang'),
+                mock.ANY,
+            ],
+        )

readthedocs/settings/base.py

@@ -311,6 +311,7 @@ def USE_PROMOS(self):  # noqa
     # https://docs.readthedocs.io/page/development/settings.html#rtd-build-media-storage
     RTD_BUILD_MEDIA_STORAGE = 'readthedocs.builds.storage.BuildMediaFileSystemStorage'
     RTD_BUILD_ENVIRONMENT_STORAGE = 'readthedocs.builds.storage.BuildMediaFileSystemStorage'
+    RTD_BUILD_TOOLS_STORAGE = 'readthedocs.builds.storage.BuildMediaFileSystemStorage'
     RTD_BUILD_COMMANDS_STORAGE = 'readthedocs.builds.storage.BuildMediaFileSystemStorage'
 
     @property