Skip to content

Commit

Permalink
Regression test jobs now specify the C++ standard version and (if app…
Browse files Browse the repository at this point in the history
…licable) the standard library to link with

* Add a new job to test clang with libc++ on ubuntu. (All other ubuntu jobs for gcc & clang link with libstdc++.)

* Add a new msvc job so that we now test `c++latest` and `c++20`.

* Job names now include more information:
<short os name> | <compiler> | <c++ std> | <std library> | <full os name>

* Reorder jobs so that GCC are first, then clang on ubuntu & macOS, and then msvc on Windows

e.g.
ubu-24 | g++-14 | c++2b | libstdc++ | ubuntu-24.04
mac-13 | clang++ | c++2b | default | macos-13
win-22 | cl.exe | c++latest | default | windows-2022
  • Loading branch information
bluetarpmedia committed May 23, 2024
1 parent b77ac91 commit 1202022
Show file tree
Hide file tree
Showing 2,618 changed files with 1,622 additions and 25 deletions.
57 changes: 48 additions & 9 deletions .github/workflows/regression-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ on:

jobs:
regression-tests:
name: Run on ${{ matrix.os }} using ${{ matrix.compiler }}
name: ${{ matrix.shortosname }} | ${{ matrix.compiler }} | ${{ matrix.cxx_std }} | ${{ matrix.stdlib }} | ${{ matrix.os }}
runs-on: ${{ matrix.os }}
env:
CXX: ${{ matrix.compiler }}
Expand All @@ -20,22 +20,61 @@ jobs:
fail-fast: false
matrix:
os: [ubuntu-24.04]
compiler: [g++-13, g++-14, clang++-18]
shortosname: [ubu-24]
compiler: [g++-14, g++-13]
cxx_std: [c++2b]
stdlib: [libstdc++]
include:
- os: ubuntu-20.04
shortosname: ubu-20
compiler: g++-10
cxx_std: c++20
stdlib: libstdc++
- os: ubuntu-24.04
shortosname: ubu-24
compiler: clang++-18
cxx_std: c++20
stdlib: libstdc++
- os: ubuntu-22.04
shortosname: ubu-22
compiler: clang++-15
cxx_std: c++20
stdlib: libstdc++
- os: ubuntu-22.04
shortosname: ubu-22
compiler: clang++-15
cxx_std: c++20
stdlib: libc++-15-dev
- os: ubuntu-20.04
shortosname: ubu-20
compiler: clang++-12
- os: ubuntu-20.04
compiler: g++-10
cxx_std: c++20
stdlib: libstdc++
- os: macos-14
shortosname: mac-14
compiler: clang++
cxx_std: c++2b
stdlib: default
- os: macos-13
shortosname: mac-13
compiler: clang++
cxx_std: c++2b
stdlib: default
- os: macos-13
shortosname: mac-13
compiler: clang++-15
- os: windows-latest
cxx_std: c++2b
stdlib: default
- os: windows-2022
shortosname: win-22
compiler: cl.exe
cxx_std: c++latest
stdlib: default
- os: windows-2022
shortosname: win-22
compiler: cl.exe
cxx_std: c++20
stdlib: default
steps:
- name: Checkout repo
uses: actions/checkout@v4
Expand All @@ -50,21 +89,21 @@ jobs:
if: startsWith(matrix.os, 'ubuntu') || startsWith(matrix.os, 'macos')
run: |
cd regression-tests
bash run-tests.sh -c ${{ matrix.compiler }} -l ${{ matrix.os }}
bash run-tests.sh -c ${{ matrix.compiler }} -s ${{ matrix.cxx_std }} -d ${{ matrix.stdlib }} -l ${{ matrix.os }}
- name: Run regression tests - Windows version
if: matrix.os == 'windows-latest'
run: |
"C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat" && ^
git config --local core.autocrlf false && ^
cd regression-tests && ^
bash run-tests.sh -c ${{ matrix.compiler }} -l ${{ matrix.os }}
bash run-tests.sh -c ${{ matrix.compiler }} -s ${{ matrix.cxx_std }} -d ${{ matrix.stdlib }} -l ${{ matrix.os }}
shell: cmd

- name: Upload patch
if: ${{ !cancelled() }}
uses: actions/upload-artifact@v4
with:
name: ${{ matrix.os }}-${{ matrix.compiler }}.patch
path: regression-tests/${{ matrix.os }}-${{ matrix.compiler }}.patch
name: ${{ matrix.os }}-${{ matrix.compiler }}-${{ matrix.cxx_std }}-${{ matrix.stdlib }}.patch
path: regression-tests/${{ matrix.os }}-${{ matrix.compiler }}-${{ matrix.cxx_std }}-${{ matrix.stdlib }}.patch
if-no-files-found: ignore
62 changes: 46 additions & 16 deletions regression-tests/run-tests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
usage() {
echo "Usage: $0 -c <compiler> [-l <run label>] [-t <tests to run>]"
echo " -c <compiler> The compiler to use for the test"
echo " -s <cxx_std> The C++ standard to compile with (e.g. 'c++20', 'c++2b', 'c++latest' depending on the compiler)"
echo " -d <stdlib> Clang-only: the C++ Standard Library to link with ('libstdc++', 'libc++', or 'default' for platform default)"
echo " -l <run label> The label to use in output patch file name"
echo " -t <tests to run> Runs only the provided, comma-separated tests (filenames including .cpp2)"
echo " If the argument is not used all tests are run"
Expand Down Expand Up @@ -46,7 +48,7 @@ check_file () {
git ls-files --error-unmatch "$file" > /dev/null 2>&1
untracked=$?

patch_file="$label$cxx_compiler.patch"
patch_file="${label}-${cxx_compiler}-${cxx_std}-${cxx_stdlib}.patch"

if [[ $untracked -eq 1 ]]; then
# Add the file to the index to be able to diff it...
Expand All @@ -67,14 +69,20 @@ check_file () {
fi
}

optstring="c:l:t:"
optstring="c:s:d:l:t:"
while getopts ${optstring} arg; do
case "${arg}" in
c)
cxx_compiler="${OPTARG}"
;;
s)
cxx_std="${OPTARG}"
;;
d)
cxx_stdlib="${OPTARG}"
;;
l)
label="${OPTARG}-"
label="${OPTARG}"
;;
t)
# Replace commas with spaces
Expand Down Expand Up @@ -125,8 +133,8 @@ expected_results_dir="test-results"
################
# Get the directory with the exec outputs and compilation command
if [[ "$cxx_compiler" == *"cl.exe"* ]]; then
compiler_cmd='cl.exe -nologo -std:c++latest -MD -EHsc -I ..\..\..\include -Fe:'
exec_out_dir="$expected_results_dir/msvc-2022"
compiler_cmd="cl.exe -nologo -std:${cxx_std} -MD -EHsc -I ..\..\..\include -Fe:"
exec_out_dir="$expected_results_dir/msvc-2022-${cxx_std}"
compiler_version=$(cl.exe)
else
# Verify the compiler command
Expand All @@ -136,7 +144,6 @@ else
exit 2
fi

cpp_std=c++2b
compiler_version=$("$cxx_compiler" --version)

if [[ "$compiler_version" == *"Apple clang version 14.0"* ||
Expand All @@ -148,16 +155,10 @@ else
exec_out_dir="$expected_results_dir/clang-12"
elif [[ "$compiler_version" == *"clang version 15.0"* ]]; then
exec_out_dir="$expected_results_dir/clang-15"
# c++2b causes starge issues on GitHub ubuntu runners
cpp_std="c++20"
elif [[ "$compiler_version" == *"clang version 18.1"* ]]; then
exec_out_dir="$expected_results_dir/clang-18"
# c++2b causes starge issues on GitHub ubuntu runners
cpp_std="c++20"
elif [[ "$compiler_version" == *"g++-10"* ]]; then
exec_out_dir="$expected_results_dir/gcc-10"
# GCC 10 does not support c++2b
cpp_std=c++20
elif [[ "$compiler_version" == *"g++-12"* ||
"$compiler_version" == *"g++-13"*
]]; then
Expand All @@ -169,15 +170,44 @@ else
exit 2
fi

compiler_cmd="$cxx_compiler -I../../../include -std=$cpp_std -pthread -o "
# Append the C++ standard (e.g. 'c++20') to the expected output dir name
exec_out_dir="${exec_out_dir}-${cxx_std}"

# Clang can link with either libstdc++ or libc++
# By default clang on ubuntu links with libstdc++ and on macOS links with libc++.
if [[ "$compiler_version" == *"clang"* ]]; then
if [[ "$cxx_stdlib" == "default" || "$cxx_stdlib" == "" ]]; then
cxx_stdlib_link_arg="" # Use compiler/platform default
elif [[ "$cxx_stdlib" == "libstdc++" ]]; then
cxx_stdlib_link_arg="-stdlib=libstdc++"
elif [[ "$cxx_stdlib" == *"libc++"* ]]; then

# Need to install the correct libc++ packages, e.g. `libc++-15-dev` and `libc++abi-15-dev` for clang 15.
# Our `cxx_stdlib` variable contains the `libc++-XX-dev` package name so we need to create the abi version.
cxx_stdlib_abi_package="${cxx_stdlib/libc++/libc++abi}"
printf "Installing packages: $cxx_stdlib $cxx_stdlib_abi_package\n\n"
sudo apt-get install -y $cxx_stdlib $cxx_stdlib_abi_package

cxx_stdlib_link_arg="-stdlib=libc++"
exec_out_dir="${exec_out_dir}-libcpp"
else
printf "Unhandled C++ Standard Library option:\n$cxx_stdlib\n\n"
exit 2
fi
else
cxx_stdlib_link_arg="" # Use compiler/platform default
fi

compiler_cmd="$cxx_compiler -I../../../include -std=$cxx_std $cxx_stdlib_link_arg -pthread -o "
printf "\ncompiler_cmd: $compiler_cmd\n\n"
fi

if [[ -d "$exec_out_dir" ]]; then
printf "Full compiler version for '$cxx_compiler':\n$compiler_version\n\n"

printf "Directory with reference compilation/execution files to use:\n$exec_out_dir\n\n"
else
printf "Directory with reference compilation/execution files not found for compiler: '$cxx_compiler'\n\n"
printf "Directory with reference compilation/execution files not found for compiler: '$cxx_compiler' at $exec_out_dir\n\n"
exit 2
fi

Expand All @@ -197,8 +227,8 @@ regression_test_link_obj=""
if [[ "$cxx_compiler" == *"cl.exe"* ]]; then
echo "Building std and std.compat modules"
(cd $exec_out_dir; \
cl.exe -nologo -std:c++latest -MD -EHsc -c "${VCToolsInstallDir}/modules/std.ixx";
cl.exe -nologo -std:c++latest -MD -EHsc -c "${VCToolsInstallDir}/modules/std.compat.ixx")
cl.exe -nologo -std:${cxx_std} -MD -EHsc -c "${VCToolsInstallDir}/modules/std.ixx";
cl.exe -nologo -std:${cxx_std} -MD -EHsc -c "${VCToolsInstallDir}/modules/std.compat.ixx")
regression_test_link_obj="std.obj std.compat.obj"
fi

Expand Down
File renamed without changes.
Loading

0 comments on commit 1202022

Please sign in to comment.