docker.yml

name: Workflow for Linux portability CI

on:
  workflow_call:
    inputs:
      targets_pre:
        default: all-sage-local
        type: string
      targets:
        default: build
        type: string
      targets_optional:
        default: ptest
        type: string
      tox_system_factors:
        description: 'Stringified JSON object listing tox system factors'
        type: string
        # 'tox -e update_docker_platforms' updates below
        default: >-
          [
           "ubuntu-focal",
           "ubuntu-jammy",
           "ubuntu-noble",
           "debian-bullseye",
           "debian-bookworm",
           "debian-trixie",
           "debian-sid",
           "linuxmint-21",
           "linuxmint-21.1",
           "linuxmint-21.2",
           "linuxmint-21.3",
           "linuxmint-22",
           "linuxmint-22.1",
           "fedora-36",
           "fedora-37",
           "fedora-38",
           "fedora-39",
           "fedora-40",
           "fedora-41",
           "centos-stream-9",
           "centos-stream-9-python3.12",
           "almalinux-8-python3.9",
           "almalinux-9-python3.11",
           "gentoo-python3.10",
           "gentoo-python3.11",
           "gentoo-python3.12",
           "archlinux-latest",
           "opensuse-15.5-gcc_11-python3.11",
           "opensuse-tumbleweed-python3.10",
          ]
        # 'tox -e update_docker_platforms' updates above
      tox_packages_factors:
        description: 'Stringified JSON object listing tox packages factors'
        type: string
        default: >-
          ["minimal",
           "standard",
           ]
      extra_sage_packages:
        description: 'Extra Sage packages to install as system packages'
        type: string
        default: ""
      max_parallel:
        type: number
        default: 30
      free_disk_space:
        default: false
        type: boolean
      timeout:
        description: 'Elapsed time (seconds) at which to kill the build'
        default: 20000
        type: number
      logs_artifact_postfix:
        description: 'Postfix (default: none) for logs artifact name'
        default: ""
        type: string
      #
      # Publishing to GitHub Packages
      #
      docker_push_repository:
        required: false
        type: string
      #
      # Incremental builds
      #
      docker_targets:
        default: "with-system-packages configured with-targets-pre with-targets with-targets-optional"
        type: string
      incremental:
        description: 'Whether to build Sage from scratch or from prebuilt Sage'
        default: false
        type: boolean
      from_docker_repository:
        required: false
        type: string
      from_docker_target:
        required: false
        type: string
      from_docker_tag:
        required: false
        default: "$BUILD_TAG"
        type: string
      #
      # For use in upstream CIs. sage_trac_* are now ignored and will be removed later.
      #
      upstream_artifact:
        required: false
        type: string
      sage_repo:
        required: false
        type: string
      sage_trac_git:
        required: false
        type: string
      sage_trac_ticket:
        required: false
        type: string
      sage_ref:
        required: false
        type: string

  workflow_dispatch:
    inputs:
      tox_system_factors:
        default: >-
          [ "ubuntu-jammy"]
        type: string
      tox_packages_factors:
        type: string
        default: >-
          ["standard"]
      targets_pre:
        default: "all-sage-local"
        type: string
      targets:
        default: "build"
        type: string
      targets_optional:
        default: "ptest"
        type: string
      docker_targets:
        default: "with-targets"
        type: string
      incremental:
        default: true
        type: boolean
      from_docker_repository:
        default: "ghcr.io/sagemath/sage/"
        type: string
      from_docker_target:
        default: "with-targets-pre"
        type: string
      from_docker_tag:
        default: "dev"
        type: string


jobs:
  linux:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      max-parallel: ${{ inputs.max_parallel || 10 }}
      matrix:
        tox_system_factor:   ${{ fromJson(inputs.tox_system_factors) }}
        tox_packages_factor: ${{ fromJson(inputs.tox_packages_factors) }}
    env:
      # See tox.ini to see how these environment variables define the
      # Dockerfile used to build the docker image, written by the
      # .ci/write-dockerfile.sh
      TOX_ENV:            "docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}${{ inputs.incremental && '-incremental' || '' }}"
      LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-tox-docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}${{ inputs.logs_artifact_postfix }}
      DOCKER_TARGETS:     ${{ inputs.docker_targets || github.event.inputs.docker_targets }}
      TARGETS_PRE:        ${{ inputs.targets_pre }}
      TARGETS:            ${{ inputs.targets }}
      TARGETS_OPTIONAL:   ${{ inputs.targets_optional }}
      FROM_DOCKER_REPOSITORY: ${{ inputs.from_docker_repository }}
      FROM_DOCKER_TARGET: ${{ inputs.from_docker_target }}
      FROM_DOCKER_TAG:    ${{ inputs.from_docker_tag }}
      EXTRA_CONFIGURE_ARGS: --enable-fat-binary
      EXTRA_SAGE_PACKAGES:  ${{ inputs.extra_sage_packages }}
    steps:
      - name: Maximize build disk space
        uses: easimon/maximize-build-space@v10
        with:
          # need space in /var for Docker images
          root-reserve-mb:      30000
          remove-dotnet:        true
          remove-android:       true
          remove-haskell:       true
          remove-codeql:        true
          remove-docker-images: true
        continue-on-error: true
        if: inputs.free_disk_space

      - name: Check out SageMath
        uses: actions/checkout@v4
        with:
          repository: ${{ inputs.sage_repo }}
          ref: ${{ inputs.sage_ref }}
          fetch-depth: 10000

      - name: Download upstream artifact
        uses: actions/download-artifact@v4
        with:
          path: upstream
          name: ${{ inputs.upstream_artifact }}
        if: inputs.upstream_artifact

      - name: Install test prerequisites
        run: |
          sudo DEBIAN_FRONTEND=noninteractive apt-get update
          sudo DEBIAN_FRONTEND=noninteractive apt-get install tox
          sudo apt-get clean
          df -h

      - name: Update Sage packages from upstream artifact
        run: |
          export PATH=$(pwd)/build/bin:$PATH
          # The sed command removes any lines containing 'upstream' from .dockerignore
          (cd upstream && bash -x update-pkgs.sh) && sed -i.bak '/upstream/d' .dockerignore
          if [ -r .ci/write-dockerfile.sh ]; then
            # Add 'ADD upstream upstream' before the line containing ':toolchain:' in the script
            echo "/:toolchain:/i ADD upstream upstream" | sed -i.bak -f - .ci/write-dockerfile.sh
          fi
          # Show the changes made to the repository for review
          git diff
        if: inputs.upstream_artifact

      - name: Try to login to ghcr.io
        if: (inputs.docker_push_repository == null || inputs.docker_push_repository != '')
        # https://docs.github.com/en/actions/reference/workflow-commands-for-github-actions#setting-an-environment-variable
        run: |
          TOKEN="${{ secrets.DOCKER_PKG_GITHUB_TOKEN }}"
          if [ -z "$TOKEN" ]; then
            TOKEN="${{ secrets.GITHUB_TOKEN }}"
          fi
          if echo "$TOKEN" | docker login ghcr.io -u ${{ github.actor }} --password-stdin; then
            if [ -z "${{ inputs.docker_push_repository }}" ]; then
              echo "DOCKER_PUSH_REPOSITORY=ghcr.io/${{ github.repository }}/" >> $GITHUB_ENV
            else
              echo "DOCKER_PUSH_REPOSITORY=$(echo ${{ inputs.docker_push_repository }} | tr "[:upper:]" "[:lower:]")" >> $GITHUB_ENV
            fi
            echo "DOCKER_CONFIG_FILE=$HOME/.docker/config.json" >> $GITHUB_ENV
          fi

      - name: Determine Docker tags to use
        run: |
          # This line needs to be run before the step "Merge CI fixes from sagemath/sage".
          #
          DOCKER_TAG="$(git describe --dirty --always)"
          echo "DOCKER_TAG=$DOCKER_TAG" >> $GITHUB_ENV
          #
          # From the docker documentation via .ci/update-env.sh:
          # "A tag name must be valid ASCII and may
          # contain lowercase and uppercase letters, digits, underscores, periods and
          # dashes. A tag name may not start with a period or a dash and may contain a
          # maximum of 128 characters."
          #
          EXTRA_DOCKER_TAGS=`echo $GITHUB_REF_NAME | tr -d '[:space:]' | tr -c '[:alnum:]_.-' '-' | sed 's/^[-.]*//' | cut -c1-128`
          shopt -s extglob
          case "$GITHUB_REF_NAME" in
            +([0-9]).+([0-9])?(.+([0-9])) )
              EXTRA_DOCKER_TAGS="latest dev $EXTRA_DOCKER_TAGS";;
            +([0-9]).+([0-9])?(.+([0-9]))?(.)@(a|alpha|b|beta|rc)+([0-9]) )
              EXTRA_DOCKER_TAGS="dev $EXTRA_DOCKER_TAGS";;
          esac
          echo "EXTRA_DOCKER_TAGS=$EXTRA_DOCKER_TAGS" >> $GITHUB_ENV

      - name: Merge CI fixes from sagemath/sage
        # This step needs to happen after the commit sha is put in DOCKER_TAG
        # so that multi-stage builds can work correctly.
        run: |
          .ci/merge-fixes.sh
        env:
          GH_TOKEN: ${{ github.token }}
          SAGE_CI_FIXES_FROM_REPOSITORIES: ${{ vars.SAGE_CI_FIXES_FROM_REPOSITORIES }}

      - name: Show disk space
        run: |
          df -h
        if: inputs.free_disk_space

      - name: Configure and build Sage distribution within a Docker container
        run: |
          # The first command below is a self-destruct sequence,
          # which preempts the GitHub Actions 6-hour job cancellation.
          #
          # Using "docker exec", we enter the temporary containers used by
          # "docker build" and kill the "make" processes of the Sage distribution.
          (
            sleep ${{ inputs.timeout || 20000 }}
            for container_id in $(docker ps -q); do
              #
              # The arcane "find" command is a replacement for "pkill make",
              # which we use because pkill is not installed in the "minimal" package
              # configuration on many platforms.
              #
              docker exec "$container_id" find /proc -maxdepth 2 -name cmdline \
                -exec bash -c 'grep -l "[m][a][k][e]" {} | cut -d/ -f3 | xargs --no-run-if-empty kill' \;
            done
          ) &
          # Set pipeline to fail if any command fails
          set -o pipefail
          #
          EXTRA_DOCKER_BUILD_ARGS="--build-arg NUMPROC=9 --build-arg USE_MAKEFLAGS=\"-k V=0 SAGE_NUM_THREADS=5\""
          #
          # The tox command starts to build and test Sage within a Docker container.
          #
          # After the build, the sed command strips away timestamps from
          # "docker build" (buildkit) such as "#25 1211.0" at the beginning of
          # each line. The timestamps are redundant because GH Actions provides
          # timestamps for each line already. Stripping the timestamps from
          # the beginnings of lines also allows GitHub Actions to recognize
          # workflow commands such as ::error etc. The other sed commands
          # annotate configuration notices/warnings/errors as GitHub
          # ::warning/::warning/::error respectively.
          #
          tox -e $TOX_ENV -- $TARGETS 2>&1 | sed -E --unbuffered "
            s/^#[0-9]+ [0-9]+[.][0-9]+ //
            /^configure: notice:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|
            /^configure: warning:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|
            /^configure: error:/s|^|::error file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|
          "

      - name: Copy logs from the Docker image or build container
        run: |
          mkdir -p "artifacts/$LOGS_ARTIFACT_NAME"
          cp -r .tox/$TOX_ENV/* "artifacts/$LOGS_ARTIFACT_NAME"
          rm -rf "artifacts/$LOGS_ARTIFACT_NAME"/{bin,lib,pyvenv.cfg}
        if: always()

      - name: Upload logs artifact
        uses: actions/upload-artifact@v4
        with:
          path: artifacts
          name: ${{ env.LOGS_ARTIFACT_NAME }}
        if: always()

      - name: Print out logs for immediate inspection
        # and markup the output with GitHub Actions logging commands
        run: |
          .github/workflows/scan-logs.sh "artifacts/$LOGS_ARTIFACT_NAME"
        if: always()

      - name: List Docker images
        run: |
          if [ -n "$DOCKER_PUSH_REPOSITORY" -a -f .tox/$TOX_ENV/Dockertags.pushed ]; then
             set -- $(cat .tox/$TOX_ENV/Dockertags.pushed)
             case $# in
                 1) images="image";  one_image="the image";;
                 *) images="images"; one_image="one of the images";;
             esac
             echo "::notice title=Docker $images pushed::Pushed $images $*)"
             echo
             echo "To pull $one_image and enter the container, type:"
             echo
             for TAG in $*; do
                 echo "  \$ docker run -it $TAG bash"
             done
             echo
             echo "To use $one_image as the base for an incremental build, type:"
             echo
             TOX_ENV_SANS_INCREMENTAL=${TOX_ENV/-incremental/}
             DOCKER_IMAGE=${TOX_ENV_SANS_INCREMENTAL#docker-}
             for TAG in $*; do
                 echo -n "  \$"
                 if [ "$DOCKER_PUSH_REPOSITORY" != "ghcr.io/sagemath/sage/" ]; then
                     echo -n " FROM_DOCKER_REPOSITORY=$DOCKER_PUSH_REPOSITORY"
                 fi
                 eval DOCKER_TARGET=\${TAG#*$DOCKER_IMAGE-}
                 DOCKER_TARGET=${DOCKER_TARGET%:*}
                 if [ "$DOCKER_TARGET" != "with-targets" ]; then
                     echo -n " FROM_DOCKER_TARGET=$DOCKER_TARGET"
                 fi
                 echo " FROM_DOCKER_TAG=${TAG#*:} tox -e $TOX_ENV_SANS_INCREMENTAL-incremental"
             done
          elif [ -n "$DOCKER_PUSH_REPOSITORY" -a -f .tox/$TOX_ENV/Dockertags ]; then
             echo "Unable to push Docker images to $DOCKER_PUSH_REPOSITORY."
             echo "This is normal in a pull request to sagemath/sage or to another user's repository."
             echo
             echo "If you need Docker images, "
             echo " - either run this GitHub Actions workflow in your repository fork"
             echo " - or use the method described in https://doc.sagemath.org/html/en/developer/portability_testing.html#automatic-docker-based-build-testing-using-tox"
          else
             echo "No Docker images created."
          fi
        if: ${{ always() && inputs.docker_push_repository }}