Restore CI Linux made useful

.ci/retrofit-worktree.sh

@@ -20,27 +20,30 @@ git config --global user.email "$GIT_AUTHOR_EMAIL"
 
 set -x
 
-# If actions/checkout downloaded our source tree using the GitHub REST API
-# instead of with git (because do not have git installed in our image),
-# we first make the source tree a repo.
+# If actions/checkout downloaded our source tree (in current directory) using the
+# GitHub REST API instead of with git, we first make the source tree a repo.
 if [ ! -d .git ]; then git init; fi
 
-# Commit and tag this state of the source tree "new". This is what we want to build and test.
+# Commit and tag this state of the source tree "new". The current directory
+# contains "new" source files with which we want to build and test.
 git add -A && git commit --quiet --allow-empty -m "new"
 git tag -f new
 
 # Our container image contains a source tree in $WORKTREE_DIRECTORY with a full build of Sage.
 # But $WORKTREE_DIRECTORY is not a git repository.
 # We make $WORKTREE_DIRECTORY a worktree whose index is at tag "new".
-# We then commit the current sources and set the tag "old". (This keeps all mtimes unchanged.)
-# Then we update worktree and index with "git checkout new".
-# (This keeps mtimes of unchanged files unchanged and mtimes of changed files newer than unchanged files.)
 if [ -L $WORKTREE_NAME ]; then
     rm -f $WORKTREE_NAME
 fi
 git worktree prune --verbose
 git worktree add --detach $WORKTREE_NAME
 rm -rf $WORKTREE_DIRECTORY/.git && mv $WORKTREE_NAME/.git $WORKTREE_DIRECTORY/
 rm -rf $WORKTREE_NAME && ln -s $WORKTREE_DIRECTORY $WORKTREE_NAME
+
+# Copy "new" .gitignore (in current directory) to the worktree if there is not one already.
 if [ ! -f $WORKTREE_NAME/.gitignore ]; then cp .gitignore $WORKTREE_NAME/; fi
-(cd $WORKTREE_NAME && git add -A && git commit --quiet --allow-empty -m "old" -a && git tag -f old && git checkout -f new && git clean -fd && git status)
+# We then commit the worktree and set the tag "old". (This keeps all modification times unchanged.)
+(cd $WORKTREE_NAME && git add -A && git commit --quiet --allow-empty -m "old" && git tag -f old)
+# Then we update the worktree and index with "git checkout new". (This keeps modification times of "old" files
+# and makes modification times of "new" files newer, which triggers incremental build.)
+(cd $WORKTREE_NAME && git checkout -f new)

.ci/write-dockerfile.sh

@@ -1,12 +1,23 @@
 #! /usr/bin/env bash
-set -e
-shopt -s extglob
 ##
 ## Write a Dockerfile for portability testing to stdout.
 ##
 ## This script needs to be run from SAGE_ROOT (root of the Sage repository).
 ## It is called by $SAGE_ROOT/tox.ini for all environments 'tox -e docker-...'
 ##
+## The generated Dockerfile needs Sage source tree directory as context,
+## and builds Sage docker images based on the source tree.
+##
+## Hence this is how to use it:
+##
+##   git worktree add source-tree
+##   docker build source-tree -f Dockerfile ...
+##   git worktree remove source-tree
+##
+## where we assume the current directory is SAGE_ROOT.
+##
+set -e
+shopt -s extglob
 ## Positional arguments:
 ##
 SYSTEM="${1:-debian}"
@@ -15,7 +26,7 @@ WITH_SYSTEM_SPKG="${3:-yes}"
 IGNORE_MISSING_SYSTEM_PACKAGES="${4:-no}"
 EXTRA_SAGE_PACKAGES="${5:-_bootstrap}"
 ##
-## Environment variables that take influence:
+## Environment variables that influence:
 ##
 ## - BOOTSTRAP
 ## - CONFIGURE_ARGS
@@ -274,32 +285,28 @@ cat <<EOF
 
 FROM with-system-packages AS bootstrapped
 #:bootstrapping:
-RUN rm -rf /new /sage/.git
-$ADD Makefile VERSION.txt COPYING.txt condarc.yml README.md bootstrap conftest.py configure_wrapper configure.ac sage .homebrew-build-env tox.ini .gitignore /new/
-$ADD config/config.rpath /new/config/config.rpath
-$ADD src/doc/bootstrap /new/src/doc/bootstrap
-$ADD src/bin /new/src/bin
-$ADD src/pyproject.toml src/requirements.txt.m4 src/setup.cfg.m4 src/VERSION.txt /new/src/
-$ADD m4 /new/m4
-$ADD pkgs /new/pkgs
-$ADD build /new/build
-$ADD .upstream.d /new/.upstream.d
-ADD .ci /.ci
-RUN if [ -d /sage ]; then \\
-        echo "### Incremental build from \$(cat /sage/VERSION.txt)" && \\
-        printf '/src/*\n!/src/doc/bootstrap\n!/src/bin\n!/src/*.m4\n!/src/*.toml\n!/src/VERSION.txt\n' >> /sage/.gitignore && \\
-        printf '/src/*\n!/src/doc/bootstrap\n!/src/bin\n!/src/*.m4\n!/src/*.toml\n!/src/VERSION.txt\n' >> /new/.gitignore && \\
-        if ! (cd /new && /.ci/retrofit-worktree.sh worktree-image /sage); then \\
-            echo "retrofit-worktree.sh failed, falling back to replacing /sage"; \\
-            for a in local logs; do \\
-                if [ -d /sage/\$a ]; then mv /sage/\$a /new/; fi; \\
-            done; \\
-            rm -rf /sage; \\
-            mv /new /sage; \\
-        fi; \\
-    else \\
-        mv /new /sage; \\
-    fi
+RUN rm -rf /source-tree
+$ADD . /source-tree
+RUN <<EOT
+rm -rf /source-tree/.git
+if [ -d /sage ]; then
+  BASE_VERSION=\$(cat /sage/VERSION.txt)
+  if (cd /source-tree && .ci/retrofit-worktree.sh worktree-image /sage); then
+    echo "### Starting incremental build from \$BASE_VERSION"
+  else
+    echo "retrofit-worktree.sh failed..."
+    rm -rf /sage
+    mv /source-tree /sage
+    echo "### Starting build from scratch"
+  fi
+else
+  git config --global user.name "ci-sage workflow"
+  git config --global user.email "[email protected]"
+  (cd /source-tree && git init && git add -A && git commit --quiet --allow-empty -m "new")
+  mv /source-tree /sage
+  echo "### Starting build from scratch"
+fi
+EOT
 WORKDIR /sage
 ARG BOOTSTRAP="${BOOTSTRAP-./bootstrap}"
 $RUN sh -x -c "\${BOOTSTRAP}"$ENDRUN$THEN_SAVE_STATUS
@@ -346,18 +353,7 @@ ENV MAKE="make -j\${NUMPROC}"
 ARG USE_MAKEFLAGS="-k V=0"
 ENV SAGE_CHECK=warn
 ENV SAGE_CHECK_PACKAGES="!cython,!python3,!cysignals,!linbox,!ppl,!cmake,!rpy2,!sage_sws2rst"
-$ADD .gitignore /new/.gitignore
-$ADD src /new/src
-RUN cd /new && rm -rf .git && \\
-    if /.ci/retrofit-worktree.sh worktree-pre /sage; then \\
-        cd /sage && touch configure build/make/Makefile; \\
-    else \\
-        echo "retrofit-worktree.sh failed, falling back to replacing /sage/src"; \\
-        rm -rf /sage/src; \\
-        mv src /sage/src; \\
-        cd /sage && ./bootstrap && ./config.status; \\
-    fi; \\
-    cd /sage && rm -rf .git; rm -rf /new || echo "(error ignored)"
+RUN cd /sage && touch configure build/make/Makefile
 ARG TARGETS="build"
 $RUN$CHECK_STATUS_THEN make SAGE_SPKG="sage-spkg -y -o" \${USE_MAKEFLAGS} \${TARGETS}$ENDRUN$THEN_SAVE_STATUS
 

.github/workflows/changelog_trigger.yml

@@ -1,4 +1,4 @@
-name: Trigger Changelog Generation
+name: Trigger changelog generation
 
 on:
   release:

.github/workflows/ci-linux-incremental.yml

@@ -81,27 +81,103 @@ jobs:
             sage-package metrics :all:
           fi
 
-  test:
+  standard:
     needs: [changed_files]
     uses: ./.github/workflows/docker.yml
     with:
-      # Build incrementally from published Docker image
+      tox_system_factors: >-
+        ["ubuntu-focal",
+         "ubuntu-noble",
+         "debian-bullseye",
+         "debian-bookworm",
+         "fedora-30",
+         "fedora-40",
+         "archlinux-latest",
+         "gentoo-python3.11",
+         "debian-bullseye-i386"]
+      tox_packages_factors: >-
+          ["standard"]
       incremental: true
       free_disk_space: true
       from_docker_repository: ghcr.io/sagemath/sage/
       from_docker_target: "with-targets"
       from_docker_tag: "dev"
       docker_targets: "with-targets"
       targets: "${{needs.changed_files.outputs.build_targets}} ci-build-with-fallback doc-html ptest-nodoc"
+      docker_push_repository: ghcr.io/${{ github.repository }}/
+      max_parallel: 8
+
+  minimal:
+    needs: [changed_files]
+    uses: ./.github/workflows/docker.yml
+    with:
       tox_system_factors: >-
         ["ubuntu-focal",
          "ubuntu-noble",
          "debian-bullseye",
          "debian-bookworm",
          "fedora-30",
-         "fedora-40",]
+         "fedora-40",
+         "archlinux-latest",
+         "gentoo-python3.11",
+         "debian-bullseye-i386"]
       tox_packages_factors: >-
-          ["standard",
-           "minimal"]
+          ["minimal"]
+      incremental: true
+      free_disk_space: true
+      from_docker_repository: ghcr.io/sagemath/sage/
+      from_docker_target: "with-targets"
+      from_docker_tag: "dev"
+      docker_targets: "with-targets"
+      targets: "${{needs.changed_files.outputs.build_targets}} ci-build-with-fallback doc-html ptest-nodoc"
       docker_push_repository: ghcr.io/${{ github.repository }}/
       max_parallel: 8
+
+  standard-constraints_pkgs:
+    # Turned off until fixed
+    if: false
+    needs: [changed_files]
+    uses: ./.github/workflows/docker.yml
+    with:
+      # Build incrementally from published Docker image
+      incremental: true
+      free_disk_space: true
+      from_docker_repository: ghcr.io/sagemath/sage/
+      from_docker_target: "with-targets-pre"
+      from_docker_tag: "dev"
+      docker_targets: "with-targets-pre"
+      targets_pre: "${{needs.changed_files.outputs.build_targets}} all-sage-local python3-ensure tox-ensure sagelib-tox-sagepython-constraints_pkgs-norequirements"
+      tox_system_factors: >-
+        ["ubuntu-focal",
+         "ubuntu-noble",
+         "debian-bookworm",
+         "fedora-40",
+         "debian-bullseye-i386"]
+      tox_packages_factors: >-
+          ["standard"]
+      docker_push_repository: ghcr.io/${{ github.repository }}/
+      max_parallel: 16
+
+  standard-sitepackages:
+    # Turned off until fixed
+    if: false
+    needs: [changed_files]
+    uses: ./.github/workflows/docker.yml
+    with:
+      # Build incrementally from published Docker image
+      incremental: true
+      free_disk_space: true
+      from_docker_repository: ghcr.io/sagemath/sage/
+      from_docker_target: "with-targets"
+      from_docker_tag: "dev"
+      docker_targets: "with-targets"
+      targets: "${{needs.changed_files.outputs.build_targets}} doc-html ptest-nodoc"
+      # Only test systems with a usable system python (>= 3.9)
+      # with only a small number of test failures as of 10.2.rc0
+      tox_system_factors: >-
+        ["gentoo-python3.11",
+         "archlinux-latest",
+         "fedora-40"]
+      tox_packages_factors: >-
+          ["standard-sitepackages"]
+      docker_push_repository: ghcr.io/${{ github.repository }}/