From 92b9d80c30d75b75e535e4dec59a9c4c8fc96a64 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Thu, 2 May 2024 18:13:05 -0700 Subject: [PATCH 01/14] .ci/write-dockerfile.sh: Update for src/pyproject.toml after #36982 --- .ci/write-dockerfile.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/write-dockerfile.sh b/.ci/write-dockerfile.sh index 0aa53414553..29485ea1283 100755 --- a/.ci/write-dockerfile.sh +++ b/.ci/write-dockerfile.sh @@ -247,7 +247,7 @@ $ADD Makefile VERSION.txt COPYING.txt condarc.yml README.md bootstrap bootstrap- $ADD config/config.rpath /new/config/config.rpath $ADD src/doc/bootstrap /new/src/doc/bootstrap $ADD src/bin /new/src/bin -$ADD src/Pipfile.m4 src/pyproject.toml.m4 src/requirements.txt.m4 src/setup.cfg.m4 src/VERSION.txt /new/src/ +$ADD src/Pipfile.m4 src/pyproject.toml src/requirements.txt.m4 src/setup.cfg.m4 src/VERSION.txt /new/src/ $ADD m4 /new/m4 $ADD pkgs /new/pkgs $ADD build /new/build From e0fc352131500ac80075a708c8bfcb939e58d6ee Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Thu, 2 May 2024 19:40:18 -0700 Subject: [PATCH 02/14] Makefile (CONFIGURE_DEPENDENCIES): Add src/pyproject.toml --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 8c9a4504b49..00989597a11 100644 --- a/Makefile +++ b/Makefile @@ -351,6 +351,7 @@ ptestoptionallong-nodoc: # CONFIGURE_DEPENDENCIES is the list of files that influence the generation of 'configure'. CONFIGURE_DEPENDENCIES = \ configure.ac src/bin/sage-version.sh m4/*.m4 \ + src/pyproject.toml \ build/pkgs/*/spkg-configure.m4 \ build/pkgs/*/type build/pkgs/*/SPKG.rst \ build/pkgs/*/checksums.ini build/pkgs/*/requirements.txt \ From 404a2c1a2f21f65bfc4404bcd29a6239ecaf0749 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Thu, 2 May 2024 20:13:41 -0700 Subject: [PATCH 03/14] .ci/write-dockerfile.sh: Quoting fix for updated docker client --- .ci/write-dockerfile.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/write-dockerfile.sh b/.ci/write-dockerfile.sh index 29485ea1283..c5896f7d434 100755 --- a/.ci/write-dockerfile.sh +++ b/.ci/write-dockerfile.sh @@ -271,7 +271,7 @@ RUN if [ -d /sage ]; then \ fi WORKDIR /sage -ARG BOOTSTRAP=${BOOTSTRAP-./bootstrap} +ARG BOOTSTRAP="${BOOTSTRAP-./bootstrap}" $RUN sh -x -c "\${BOOTSTRAP}" $ENDRUN $THEN_SAVE_STATUS FROM bootstrapped as configured From 5393c9417961a24a84118e608deb279af16373cb Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Thu, 2 May 2024 22:10:16 -0700 Subject: [PATCH 04/14] .ci/write-dockerfile.sh (with-targets): Re-bootstrap after copying in src --- .ci/write-dockerfile.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.ci/write-dockerfile.sh b/.ci/write-dockerfile.sh index c5896f7d434..15b2bf764de 100755 --- a/.ci/write-dockerfile.sh +++ b/.ci/write-dockerfile.sh @@ -324,6 +324,7 @@ RUN if command -v git; then \ else \ rm -rf /sage/src; \ mv /new/src /sage/src; \ + ./bootstrap && ./config.status; \ fi ARG TARGETS="build" From e335e9318225e5909095afafc64af72d30e1b00f Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sun, 5 May 2024 12:10:40 -0700 Subject: [PATCH 05/14] tox.ini, .ci/write-dockerfile.sh: Fix handling of CONFIGURE_ARGS, add doc --- .ci/write-dockerfile.sh | 41 +++++++++++++++++++++++++++++++---------- tox.ini | 1 + 2 files changed, 32 insertions(+), 10 deletions(-) diff --git a/.ci/write-dockerfile.sh b/.ci/write-dockerfile.sh index 15b2bf764de..9eb2c322b5e 100755 --- a/.ci/write-dockerfile.sh +++ b/.ci/write-dockerfile.sh @@ -1,20 +1,41 @@ #! /usr/bin/env bash -## Write a Dockerfile to stdout that tests that the packages listed in the debian.txt/fedora.txt files of standard spkg exist -## and satisfy the requirements tested by spkg-configure.m4 -## This is called by $SAGE_ROOT/tox.ini set -e -SYSTEM="${1:-debian}" shopt -s extglob +## +## Write a Dockerfile for portability testing to stdout. +## +## This script needs to be run from SAGE_ROOT (root of the Sage repository). +## It is called by $SAGE_ROOT/tox.ini for all environments 'tox -e docker-...' +## +## Positional arguments: +## +SYSTEM="${1:-debian}" SAGE_PACKAGE_LIST_ARGS="${2:-:standard:}" WITH_SYSTEM_SPKG="${3:-yes}" IGNORE_MISSING_SYSTEM_PACKAGES="${4:-no}" EXTRA_SAGE_PACKAGES="${5:-_bootstrap}" -# +## +## Environment variables that take influence: +## +## - BOOTSTRAP +## - CONFIGURE_ARGS +## - DEVTOOLSET +## - DIST_UPGRADE +## - DOCKER_BUILDKIT +## - EXTRA_PATH +## - EXTRA_REPOSITORIES +## - EXTRA_SYSTEM_PACKAGES +## - FULL_BASE_IMAGE_AND_TAG +## - SKIP_SYSTEM_PKG_INSTALL +## - USE_CONDARC +## - __CHOWN +## - __SUDO +## STRIP_COMMENTS="sed s/#.*//;" SAGE_ROOT=. export PATH="$SAGE_ROOT"/build/bin:$PATH SYSTEM_PACKAGES=$EXTRA_SYSTEM_PACKAGES -CONFIGURE_ARGS="--enable-option-checking " +SYSTEM_CONFIGURE_ARGS="--enable-option-checking " for SPKG in $(sage-package list --has-file=spkg-configure.m4 $SAGE_PACKAGE_LIST_ARGS) $EXTRA_SAGE_PACKAGES; do SYSTEM_PACKAGE=$(sage-get-system-packages $SYSTEM $SPKG) if [ -n "${SYSTEM_PACKAGE}" ]; then @@ -24,7 +45,7 @@ for SPKG in $(sage-package list --has-file=spkg-configure.m4 $SAGE_PACKAGE_LIST_ # shell-quote package if necessary SYSTEM_PACKAGES+=$(printf " %q" "$a") done - CONFIGURE_ARGS+="--with-system-${SPKG}=${WITH_SYSTEM_SPKG} " + SYSTEM_CONFIGURE_ARGS+="--with-system-${SPKG}=${WITH_SYSTEM_SPKG} " fi done echo "# Automatically generated by SAGE_ROOT/.ci/write-dockerfile.sh" @@ -277,15 +298,15 @@ $RUN sh -x -c "\${BOOTSTRAP}" $ENDRUN $THEN_SAVE_STATUS FROM bootstrapped as configured #:configuring: RUN $CHECK_STATUS_THEN mkdir -p logs/pkgs; rm -f config.log; ln -s logs/pkgs/config.log config.log -ARG EXTRA_CONFIGURE_ARGS="${CONFIGURE_ARGS}" +ARG CONFIGURE_ARGS="${CONFIGURE_ARGS:---enable-build-as-root}" EOF if [ ${WITH_SYSTEM_SPKG} = "force" ]; then cat < Date: Tue, 7 May 2024 13:07:56 -0700 Subject: [PATCH 06/14] tox.ini (docker): Pass through HOME to fix use with current macOS Docker Desktop client --- tox.ini | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index a529254ce77..12858706384 100644 --- a/tox.ini +++ b/tox.ini @@ -148,6 +148,12 @@ passenv = # Use DOCKER_BUILDKIT=0 for legacy builder docker: DOCKER_BUILDKIT docker: BUILDKIT_INLINE_CACHE + docker: DOCKER_HOST + # On macOS, the docker CLI may need $HOME to find the socket + # $HOME/Library/Containers/com.docker.docker/Data/docker-cli.sock + # Otherwise may get the error "Cannot connect to the Docker daemon at + # unix:///var/run/docker.sock. Is the docker daemon running?" + docker: HOME # Set for example to "with-system-packages configured with-targets-pre with-targets" # to tag intermediate images. docker: DOCKER_TARGETS @@ -198,8 +204,8 @@ setenv = # Whether to add the system packages needed for bootstrapping EXTRA_SAGE_PACKAGES_0=_bootstrap nobootstrap: EXTRA_SAGE_PACKAGES_0= - # local envs need HOME set, also Docker 19.03 needs HOME - {local,docker}: HOME={envdir} + # local envs need HOME set + local: HOME={envdir} # for local envs we can guess the package system if it is not provided as a factor local: SYSTEM=auto # From 754bb9f6ae66a5bb1c8d2a8826e990e56996abb5 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Tue, 7 May 2024 15:48:33 -0700 Subject: [PATCH 07/14] .ci/write-dockerfile.sh: Fix handling of .gitignore while src/ has not been copied in fully --- .ci/write-dockerfile.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.ci/write-dockerfile.sh b/.ci/write-dockerfile.sh index 9eb2c322b5e..312f8adb656 100755 --- a/.ci/write-dockerfile.sh +++ b/.ci/write-dockerfile.sh @@ -277,8 +277,9 @@ $ADD .upstream.d /new/.upstream.d RUN if [ -d /sage ]; then \ echo "### Incremental build from \$(cat /sage/VERSION.txt)" && \ if command -v git; then \ + printf '/src\n!/src/doc/bootstrap\n!/src/bin\n!/src/*.m4\n!/src/*.toml\n!/src/VERSION.txt\n' >> /sage/.gitignore && \ (cd /new && \ - echo /src >> .gitignore && \ + printf '/src\n!/src/doc/bootstrap\n!/src/bin\n!/src/*.m4\n!/src/*.toml\n!/src/VERSION.txt\n' >> .gitignore && \ ./.ci/retrofit-worktree.sh worktree-image /sage); \ else \ for a in local logs; do \ From d3f9c60079a291118fe0ccedb913c51c9ebb5a19 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Tue, 7 May 2024 15:51:09 -0700 Subject: [PATCH 08/14] .gitignore: Ignore venvs within pkgs/*/ --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 7c6e7b55356..1d7eacdf9e4 100644 --- a/.gitignore +++ b/.gitignore @@ -204,6 +204,8 @@ build/pkgs/wheel/version_requirements.txt /pkgs/*/MANIFEST /pkgs/*/*.egg-info /pkgs/*/.tox +/pkgs/*/venv* +/pkgs/*/.venv* /pkgs/sage-conf_pypi/sage_root/config.log /pkgs/sage-conf_pypi/sage_root/config.status From 614d98a19caf367f5f0dfee4940f936eace91646 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Tue, 7 May 2024 21:22:17 -0700 Subject: [PATCH 09/14] src/sage/misc/package_dir.py: Mark two doctests '# known bug' --- src/sage/misc/package_dir.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sage/misc/package_dir.py b/src/sage/misc/package_dir.py index 71ec08b3ae0..50cb0fc5514 100644 --- a/src/sage/misc/package_dir.py +++ b/src/sage/misc/package_dir.py @@ -273,14 +273,14 @@ def is_package_or_sage_namespace_package_dir(path, *, distribution_filter=None): sage: directory = os.path.join(sage.libs.__path__[0], 'mpfr'); directory '.../sage/libs/mpfr' - sage: is_package_or_sage_namespace_package_dir(directory) + sage: is_package_or_sage_namespace_package_dir(directory) # known bug (seen in build.yml) True :mod:`sage` is designated to become an implicit namespace package:: sage: directory = sage.__path__[0]; directory '.../sage' - sage: is_package_or_sage_namespace_package_dir(directory) + sage: is_package_or_sage_namespace_package_dir(directory) # known bug (seen in build.yml) True Not a package:: From c2be0e08b2e475cc0d70b7c6dc96781adb43f472 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Thu, 9 May 2024 00:20:10 -0700 Subject: [PATCH 10/14] tox.ini (docker): Supply BUILD_TAG to write-dockerfile.sh, for use by the default from_docker_tag in docker.yml --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 12858706384..41c92f576ea 100644 --- a/tox.ini +++ b/tox.ini @@ -765,7 +765,7 @@ commands = local: bash -c 'if [ ! -d prefix -o -L prefix ]; then rm -f prefix; ln -sf {env:PREFIX:{envdir}/local} prefix; fi' ##commands = - docker: bash -c '.ci/write-dockerfile.sh {env:SYSTEM} "{env:SAGE_PACKAGE_LIST_ARGS:}" {env:WITH_SYSTEM_SPKG} {env:IGNORE_MISSING_SYSTEM_PACKAGES} "{env:ALL_EXTRA_SAGE_PACKAGES}" > {envdir}/Dockerfile' + docker: bash -c 'BUILD_TAG={env:DOCKER_TAG:$(git describe --dirty --always)} .ci/write-dockerfile.sh {env:SYSTEM} "{env:SAGE_PACKAGE_LIST_ARGS:}" {env:WITH_SYSTEM_SPKG} {env:IGNORE_MISSING_SYSTEM_PACKAGES} "{env:ALL_EXTRA_SAGE_PACKAGES}" > {envdir}/Dockerfile' # From https://hub.docker.com/r/multiarch/ubuntu-core/ # configure binfmt-support on the Docker host (works locally or remotely, i.e: using boot2docker) docker-{arm64,armhf}: docker run --rm --privileged multiarch/qemu-user-static:register --reset From 9ae5c1867939e8161e9254079b04ae3d378added Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Thu, 9 May 2024 21:33:26 -0700 Subject: [PATCH 11/14] .ci/write-dockerfile.sh: Check that 'git worktree' works before using retrofit-worktree.sh --- .ci/write-dockerfile.sh | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/.ci/write-dockerfile.sh b/.ci/write-dockerfile.sh index 312f8adb656..e0afeaec76f 100755 --- a/.ci/write-dockerfile.sh +++ b/.ci/write-dockerfile.sh @@ -276,12 +276,9 @@ $ADD .ci /new/.ci $ADD .upstream.d /new/.upstream.d RUN if [ -d /sage ]; then \ echo "### Incremental build from \$(cat /sage/VERSION.txt)" && \ - if command -v git; then \ - printf '/src\n!/src/doc/bootstrap\n!/src/bin\n!/src/*.m4\n!/src/*.toml\n!/src/VERSION.txt\n' >> /sage/.gitignore && \ - (cd /new && \ - printf '/src\n!/src/doc/bootstrap\n!/src/bin\n!/src/*.m4\n!/src/*.toml\n!/src/VERSION.txt\n' >> .gitignore && \ - ./.ci/retrofit-worktree.sh worktree-image /sage); \ - else \ + printf '/src\n!/src/doc/bootstrap\n!/src/bin\n!/src/*.m4\n!/src/*.toml\n!/src/VERSION.txt\n' >> /sage/.gitignore && \ + printf '/src\n!/src/doc/bootstrap\n!/src/bin\n!/src/*.m4\n!/src/*.toml\n!/src/VERSION.txt\n' >> /new/.gitignore && \ + if ! (cd /new && ./.ci/retrofit-worktree.sh worktree-image /sage); then \ for a in local logs; do \ if [ -d /sage/\$a ]; then mv /sage/\$a /new/; fi; \ done; \ @@ -340,13 +337,11 @@ ENV SAGE_CHECK=warn ENV SAGE_CHECK_PACKAGES="!cython,!r,!python3,!gap,!cysignals,!linbox,!git,!ppl,!cmake,!rpy2,!sage_sws2rst" $ADD .gitignore /new/.gitignore $ADD src /new/src -RUN if command -v git; then \ - cd /new && rm -rf .git && \ - ./.ci/retrofit-worktree.sh worktree-pre /sage; \ - else \ - rm -rf /sage/src; \ - mv /new/src /sage/src; \ - ./bootstrap && ./config.status; \ +RUN cd /new && rm -rf .git && \ + if ! ./.ci/retrofit-worktree.sh worktree-pre /sage; then \ + rm -rf /sage/src; \ + mv src /sage/src; \ + ./bootstrap && ./config.status; \ fi ARG TARGETS="build" From b16750c8f967b9e7633c53bcb145956e37aa4f84 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sun, 12 May 2024 14:14:12 -0700 Subject: [PATCH 12/14] .ci/write-dockerfile.sh (with-targets): Use .ci from /sage; in git-free fallback, re-bootstrap in /sage --- .ci/write-dockerfile.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/write-dockerfile.sh b/.ci/write-dockerfile.sh index e0afeaec76f..578680d76f2 100755 --- a/.ci/write-dockerfile.sh +++ b/.ci/write-dockerfile.sh @@ -338,10 +338,10 @@ ENV SAGE_CHECK_PACKAGES="!cython,!r,!python3,!gap,!cysignals,!linbox,!git,!ppl,! $ADD .gitignore /new/.gitignore $ADD src /new/src RUN cd /new && rm -rf .git && \ - if ! ./.ci/retrofit-worktree.sh worktree-pre /sage; then \ + if ! /sage/.ci/retrofit-worktree.sh worktree-pre /sage; then \ rm -rf /sage/src; \ mv src /sage/src; \ - ./bootstrap && ./config.status; \ + cd /sage && ./bootstrap && ./config.status; \ fi ARG TARGETS="build" From fe470b554495bf224280e51ee777f28f5d252180 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sun, 12 May 2024 15:31:39 -0700 Subject: [PATCH 13/14] .ci/retrofit-worktree.sh: Exit earlier if git is not present --- .ci/retrofit-worktree.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.ci/retrofit-worktree.sh b/.ci/retrofit-worktree.sh index b9c880a839f..b399c4b4043 100755 --- a/.ci/retrofit-worktree.sh +++ b/.ci/retrofit-worktree.sh @@ -12,11 +12,13 @@ export GIT_AUTHOR_EMAIL="ci-sage@example.com" export GIT_COMMITTER_NAME="$GIT_AUTHOR_NAME" export GIT_COMMITTER_EMAIL="$GIT_AUTHOR_EMAIL" +set -e + # Set globally for other parts of the workflow git config --global user.name "$GIT_AUTHOR_NAME" git config --global user.email "$GIT_AUTHOR_EMAIL" -set -ex +set -x # If actions/checkout downloaded our source tree using the GitHub REST API # instead of with git (because do not have git installed in our image), From 4c076bbaea77dd1b5bc7cf43bd99006bff9b756e Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sun, 12 May 2024 23:47:47 -0700 Subject: [PATCH 14/14] .ci/retrofit-worktree.sh: Prune old worktree if needed --- .ci/retrofit-worktree.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.ci/retrofit-worktree.sh b/.ci/retrofit-worktree.sh index b399c4b4043..5dcb6e2b88f 100755 --- a/.ci/retrofit-worktree.sh +++ b/.ci/retrofit-worktree.sh @@ -38,6 +38,10 @@ git tag -f new # The changed files now show up as uncommitted changes. # The final "git add -N" makes sure that files that were added in "new" do not show # as untracked files, which would be removed by "git clean -fx". +if [ -L $WORKTREE_NAME ]; then + rm -f $WORKTREE_NAME + git worktree prune --verbose +fi git worktree add --detach $WORKTREE_NAME rm -rf $WORKTREE_DIRECTORY/.git && mv $WORKTREE_NAME/.git $WORKTREE_DIRECTORY/ rm -rf $WORKTREE_NAME && ln -s $WORKTREE_DIRECTORY $WORKTREE_NAME