Trac #30010: Split sage_setup.docbuild out to a separate distribution sage_docbuild

`sage_setup.docbuild`, created in #19127, has very different
dependencies compared to `sage_setup`:

It depends on `sagelib` and `sphinx`,
whereas the rest of `sage_setup` is for building `sagelib`.

We remove the nesting within `sage_setup`, creating a new top-level
package `sage_docbuild`.

We split it out as a separate pip-installable distribution package
`sage_docbuild`.

Using the `spkg-src` script, a pip-installable tarball can be generated,
which could be uploaded to PyPI -- for the use by external packages that
want to build documentation.

This is preparation for #29868 (pip-installable packages sagemath-doc-
src, sagemath-doc-inventory, sagemath-doc-html, sagemath-doc-pdf)

URL: https://trac.sagemath.org/30010
Reported by: mkoeppe
Ticket author(s): Matthias Koeppe
Reviewer(s): John Palmieri

Author: Release Manager

build/bin/sage-site

@@ -162,7 +162,7 @@ if [ "$1" = "-docbuild" -o "$1" = "--docbuild" ]; then
         export OMP_NUM_THREADS=1
     fi
 
-    exec sage-python -m sage_setup.docbuild "$@" </dev/null
+    exec sage-python -m sage_docbuild "$@" </dev/null
 fi
 
 if [ "$1" = '-git-branch' -o "$1" = '--git-branch' ]; then

build/make/Makefile.in

@@ -335,7 +335,7 @@ base: $(inst_patch) $(inst_pkgconf)
 # Building the documentation has many dependencies, because all
 # documented modules are imported and because we use matplotlib to
 # produce plots.
-DOC_DEPENDENCIES = sagelib $(inst_sphinx) \
+DOC_DEPENDENCIES = sagelib sage_docbuild $(inst_sphinx) \
 	| $(SAGERUNTIME) $(inst_maxima) $(inst_networkx) $(inst_scipy) $(inst_sympy) \
 	$(inst_matplotlib) $(inst_pillow) $(inst_mathjax) $(inst_mpmath) \
 	$(inst_ipykernel) $(inst_jupyter_client) $(inst_conway_polynomials) \
@@ -391,7 +391,10 @@ sagelib-clean:
 	 rm -rf sage/ext/interpreters) \
 	&& (cd "$(SAGE_ROOT)/build/pkgs/sagelib/src/" && rm -rf build)
 
-build-clean: clean doc-clean sagelib-clean
+sage_docbuild-clean:
+	(cd "$(SAGE_ROOT)/build/pkgs/sage_docbuild/src" && rm -rf build)
+
+build-clean: clean doc-clean sagelib-clean sage_docbuild-clean
 
 # Special target for cleaning up a broken GCC install detected by configure
 # This should check for the .clean-broken-gcc stamp, and if found clean

build/pkgs/sage_docbuild/dependencies

@@ -0,0 +1 @@
+$(PYTHON) sphinx ../pkgs/sage_docbuild/src/sage_docbuild/*.py ../pkgs/sage_docbuild/src/sage_docbuild/ext/*.py | $(PYTHON_TOOLCHAIN) sagelib

build/pkgs/sage_docbuild/package-version.txt

@@ -0,0 +1 @@
+../sagelib/package-version.txt

build/pkgs/sage_docbuild/spkg-install

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+# From sage-spkg.
+# For type=script packages, the build rule in build/make/Makefile sources
+# sage-env but not sage-dist-helpers.
+lib="$SAGE_ROOT/build/bin/sage-dist-helpers"
+source "$lib"
+if [ $? -ne 0 ]; then
+    echo >&2 "Error: failed to source $lib"
+    echo >&2 "Is $SAGE_ROOT the correct SAGE_ROOT?"
+    exit 1
+fi
+# We build the wheel directly with "setup.py bdist_wheel", not with "pip wheel",
+# because pip does not handle our symlinks correctly.
+cd src && sdh_setup_bdist_wheel && sdh_store_and_pip_install_wheel .

build/pkgs/sage_docbuild/spkg-src

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+#
+# Script to prepare an sdist tarball for sage_docbuild
+# This script is not used during build.
+#
+# HOW TO MAKE THE TARBALL:
+# ./sage --sh build/pkgs/sage_docbuild/spkg-src
+
+if [ -z "$SAGE_ROOT" ] ; then
+    echo >&2 "Error - SAGE_ROOT undefined ... exiting"
+    echo >&2 "Maybe run 'sage -sh'?"
+    exit 1
+fi
+
+# Exit on failure
+set -e
+
+cd build/pkgs/sage_docbuild
+
+cd src
+python3 -u setup.py --no-user-cfg sdist --dist-dir "$SAGE_DISTFILES"

build/pkgs/sage_docbuild/src/MANIFEST.in

@@ -0,0 +1 @@
+include VERSION.txt

build/pkgs/sage_docbuild/src/README.rst

@@ -0,0 +1,23 @@
+================================================================================
+ Sage: Open Source Mathematics Software: Build system of the Sage documentation
+================================================================================
+
+About SageMath
+--------------
+
+   "Creating a Viable Open Source Alternative to
+    Magma, Maple, Mathematica, and MATLAB"
+
+   Copyright (C) 2005-2020 The Sage Development Team
+
+   https://www.sagemath.org
+
+SageMath fully supports all major Linux distributions, recent versions of macOS, and Windows (using Cygwin or Windows Subsystem for Linux).
+
+The traditional and recommended way to install SageMath is from source via Sage-the-distribution (https://www.sagemath.org/download-source.html).  Sage-the-distribution first builds a large number of open source packages from source (unless it finds suitable versions installed in the system) and then installs the Sage Library (sagelib, implemented in Python and Cython).
+
+
+About this pip-installable source distribution
+----------------------------------------------
+
+This is the build system of the Sage documentation, based on Sphinx.

build/pkgs/sage_docbuild/src/VERSION.txt

@@ -0,0 +1 @@
+../package-version.txt

build/pkgs/sage_docbuild/src/requirements.txt

@@ -0,0 +1,2 @@
+#sage
+sphinx

build/pkgs/sage_docbuild/src/sage_docbuild

@@ -0,0 +1 @@
+../../../../src/sage_docbuild

build/pkgs/sage_docbuild/src/setup.cfg

@@ -0,0 +1,33 @@
+[metadata]
+name = sage_docbuild
+version = file: VERSION.txt
+description = Sage: Open Source Mathematics Software: Build system of the Sage documentation
+long_description = file: README.rst
+long_description_content_type = text/x-rst
+license = GNU General Public License (GPL) v2 or later
+author = The Sage Developers
+author_email = [email protected]
+url = https://www.sagemath.org
+
+classifiers =
+    Development Status :: 6 - Mature
+    Intended Audience :: Education
+    Intended Audience :: Science/Research
+    License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)
+    Operating System :: POSIX
+    Operating System :: MacOS :: MacOS X
+    Programming Language :: Python :: 3 :: Only
+    Programming Language :: Python :: 3.6
+    Programming Language :: Python :: 3.7
+    Programming Language :: Python :: 3.8
+    Programming Language :: Python :: 3.9
+    Programming Language :: Python :: Implementation :: CPython
+    Topic :: Scientific/Engineering :: Mathematics
+
+[options]
+packages =
+    sage_docbuild
+    sage_docbuild.ext
+
+install_requires =
+    sphinx

build/pkgs/sage_docbuild/src/setup.py

@@ -0,0 +1,5 @@
+#!/usr/bin/env python
+
+from setuptools import setup
+
+setup()

build/pkgs/sage_docbuild/src/tox.ini

@@ -0,0 +1,30 @@
+# First pip-install tox:
+#
+#   ./sage -pip install tox
+#
+# To build and test in the tox environment:
+#
+# ./sage -sh -c '(cd build/pkgs/sage_docbuild/src && tox)'
+#
+# To test interactively:
+#
+# build/pkgs/sage_docbuild/src/.tox/python/bin/python
+#
+[tox]
+
+[testenv]
+deps = -rrequirements.txt
+
+setenv =
+    # Sage scripts like to use $HOME/.sage
+    HOME={envdir}
+
+whitelist_externals =
+    bash
+
+commands =
+    # Beware of the treacherous non-src layout.
+    #python -c 'import sys; "" in sys.path and sys.path.remove(""); import sage_docbuild'
+
+# TODO: Add tests after adding the dependency on sagelib to
+# requirements.txt

build/pkgs/sage_docbuild/type

@@ -0,0 +1 @@
+standard

build/pkgs/sagelib/src/MANIFEST.in

@@ -6,3 +6,5 @@ prune .tox
 
 graft   sage/libs/gap/test
 prune sage/ext/interpreters   # In particular, __init__.py must not be present in the distribution; or sage_setup.autogen.interpreters.rebuild will not generate the code
+
+prune sage_docbuild     # Shipped by sage_docbuild

src/doc/en/developer/sage_manuals.rst

@@ -176,7 +176,7 @@ Building the Manuals
 
 All of the Sage manuals are built using the ``sage --docbuild``
 script.  The content of the ``sage --docbuild`` script is defined in
-``SAGE_ROOT/src/sage_setup/docbuild/__init__.py``.  It is a thin wrapper around
+``SAGE_ROOT/src/sage_docbuild/__init__.py``.  It is a thin wrapper around
 the ``sphinx-build`` script which does all of the real work.  It is
 designed to be a replacement for the default Makefiles generated by
 the ``sphinx-quickstart`` script.  The general form of the command

src/sage/docs/conf.py

@@ -12,17 +12,14 @@
 import sphinx.ext.intersphinx as intersphinx
 from IPython.lib.lexers import IPythonConsoleLexer, IPyLexer
 
-# If your extensions are in another directory, add it here.
-sys.path.append(os.path.join(SAGE_SRC, "sage_setup", "docbuild", "ext"))
-
 # General configuration
 # ---------------------
 
 # Add any Sphinx extension module names here, as strings. They can be extensions
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
-extensions = ['inventory_builder',
-              'multidocs',
-              'sage_autodoc',
+extensions = ['sage_docbuild.ext.inventory_builder',
+              'sage_docbuild.ext.multidocs',
+              'sage_docbuild.ext.sage_autodoc',
               'sphinx.ext.graphviz',
               'sphinx.ext.inheritance_diagram',
               'sphinx.ext.todo',

src/sage/doctest/control.py

@@ -700,6 +700,7 @@ def all_files():
             # don't make sense to run outside a build environment
             if have_git:
                 self.files.append(opj(SAGE_SRC, 'sage_setup'))
+                self.files.append(opj(SAGE_SRC, 'sage_docbuild'))
             self.files.append(SAGE_DOC_SRC)
 
         if self.options.all or (self.options.new and not have_git):

src/sage_setup/docbuild/__init__.py src/sage_docbuild/__init__.py

@@ -90,16 +90,16 @@ def builder_helper(type):
 
     Check that :trac:`25161` has been resolved::
 
-        sage: from sage_setup.docbuild import DocBuilder, setup_parser
+        sage: from sage_docbuild import DocBuilder, setup_parser
         sage: DocBuilder._options = setup_parser().parse_args([])[0] # builder_helper needs _options to be set
 
-        sage: import sage_setup.docbuild.sphinxbuild
+        sage: import sage_docbuild.sphinxbuild
         sage: def raiseBaseException():
         ....:     raise BaseException("abort pool operation")
-        sage: original_runsphinx, sage_setup.docbuild.sphinxbuild.runsphinx = sage_setup.docbuild.sphinxbuild.runsphinx, raiseBaseException
+        sage: original_runsphinx, sage_docbuild.sphinxbuild.runsphinx = sage_docbuild.sphinxbuild.runsphinx, raiseBaseException
 
-        sage: from sage_setup.docbuild import builder_helper, build_ref_doc
-        sage: from sage_setup.docbuild import _build_many as build_many
+        sage: from sage_docbuild import builder_helper, build_ref_doc
+        sage: from sage_docbuild import _build_many as build_many
         sage: helper = builder_helper("html")
         sage: try:
         ....:     build_many(build_ref_doc, [("docname", "en", "html", {})])
@@ -180,7 +180,7 @@ def _output_dir(self, type):
 
         EXAMPLES::
 
-            sage: from sage_setup.docbuild import DocBuilder
+            sage: from sage_docbuild import DocBuilder
             sage: b = DocBuilder('tutorial')
             sage: b._output_dir('html')
             '.../html/en/tutorial'
@@ -197,7 +197,7 @@ def _doctrees_dir(self):
 
         EXAMPLES::
 
-            sage: from sage_setup.docbuild import DocBuilder
+            sage: from sage_docbuild import DocBuilder
             sage: b = DocBuilder('tutorial')
             sage: b._doctrees_dir()
             '.../doctrees/en/tutorial'
@@ -212,7 +212,7 @@ def _output_formats(self):
 
         EXAMPLES::
 
-            sage: from sage_setup.docbuild import DocBuilder
+            sage: from sage_docbuild import DocBuilder
             sage: b = DocBuilder('tutorial')
             sage: b._output_formats()
             ['changes', 'html', 'htmlhelp', 'inventory', 'json', 'latex', 'linkcheck', 'pickle', 'web']
@@ -236,7 +236,7 @@ def pdf(self):
 
         EXAMPLES::
 
-            sage: from sage_setup.docbuild import DocBuilder
+            sage: from sage_docbuild import DocBuilder
             sage: b = DocBuilder('tutorial')
             sage: b.pdf() #not tested
         """
@@ -288,7 +288,7 @@ def clean(self, *args):
 
 def build_many(target, args, processes=None):
     """
-    Thin wrapper around `sage_setup.docbuild.utils.build_many` which uses the
+    Thin wrapper around `sage_docbuild.utils.build_many` which uses the
     docbuild settings ``NUM_THREADS`` and ``ABORT_ON_ERROR``.
     """
     if processes is None:
@@ -369,7 +369,7 @@ def get_all_documents(self):
 
         EXAMPLES::
 
-            sage: from sage_setup.docbuild import AllBuilder
+            sage: from sage_docbuild import AllBuilder
             sage: documents = AllBuilder().get_all_documents()
             sage: 'en/tutorial' in documents
             True
@@ -525,7 +525,7 @@ def _output_dir(self, type, lang='en'):
 
         EXAMPLES::
 
-            sage: from sage_setup.docbuild import ReferenceBuilder
+            sage: from sage_docbuild import ReferenceBuilder
             sage: b = ReferenceBuilder('reference')
             sage: b._output_dir('html')
             '.../html/en/reference'
@@ -688,7 +688,7 @@ def get_all_documents(self, refdir):
 
         EXAMPLES::
 
-            sage: from sage_setup.docbuild import ReferenceBuilder
+            sage: from sage_docbuild import ReferenceBuilder
             sage: b = ReferenceBuilder('reference')
             sage: refdir = os.path.join(os.environ['SAGE_DOC_SRC'], 'en', b.name)
             sage: sorted(b.get_all_documents(refdir))
@@ -1046,7 +1046,7 @@ def auto_rest_filename(self, module_name):
 
         EXAMPLES::
 
-            sage: from sage_setup.docbuild import ReferenceSubBuilder
+            sage: from sage_docbuild import ReferenceSubBuilder
             sage: ReferenceSubBuilder("reference").auto_rest_filename("sage.combinat.partition")
             '.../doc/en/reference/sage/combinat/partition.rst'
         """
@@ -1616,7 +1616,7 @@ def setup_logger(verbose=1, color=True):
 
     EXAMPLES::
 
-        sage: from sage_setup.docbuild import setup_logger, logger
+        sage: from sage_docbuild import setup_logger, logger
         sage: setup_logger()
         sage: type(logger)
         <class 'logging.Logger'>

src/sage_setup/docbuild/__main__.py src/sage_docbuild/__main__.py

@@ -172,7 +172,7 @@ def _check_errors(self, line):
         EXAMPLES::
 
             sage: from sys import stdout
-            sage: from sage_setup.docbuild.sphinxbuild import SageSphinxLogger
+            sage: from sage_docbuild.sphinxbuild import SageSphinxLogger
             sage: logger = SageSphinxLogger(stdout, "doctesting")
             sage: logger._log_line("Segmentation fault!\n") # indirect doctest
             [doctestin] Segmentation fault!
@@ -200,7 +200,7 @@ def _log_line(self, line):
         EXAMPLES::
 
             sage: from sys import stdout
-            sage: from sage_setup.docbuild.sphinxbuild import SageSphinxLogger
+            sage: from sage_docbuild.sphinxbuild import SageSphinxLogger
             sage: logger = SageSphinxLogger(stdout, "doctesting")
             sage: logger._log_line("building documentation…\n")
             [doctestin] building documentation…
@@ -243,7 +243,7 @@ def raise_errors(self):
         EXAMPLES::
 
             sage: from sys import stdout
-            sage: from sage_setup.docbuild.sphinxbuild import SageSphinxLogger
+            sage: from sage_docbuild.sphinxbuild import SageSphinxLogger
             sage: logger = SageSphinxLogger(stdout, "doctesting")
             sage: logger._log_line("This is a SEVERE error\n")
             [doctestin] This is a SEVERE error

src/sage_setup/docbuild/build_options.py src/sage_docbuild/build_options.py

@@ -42,7 +42,7 @@ def build_many(target, args, processes=None):
 
     EXAMPLES::
 
-        sage: from sage_setup.docbuild.utils import build_many
+        sage: from sage_docbuild.utils import build_many
         sage: def target(N):
         ....:     import time
         ....:     time.sleep(float(0.1))