From 3e25e103fc30e080215d59abc7f5f8cc17527ccf Mon Sep 17 00:00:00 2001 From: Larry Gritz Date: Fri, 29 Nov 2024 11:07:05 -0800 Subject: [PATCH] ci: refactor using a single steps workflow The logic for the GHA steps was repeated across multiple workflows and platforms. Consolidate it all into a common "ci-steps" workflow, and have all the other workflows reference that when possible. This should hopefully make maintenance of the CI scripts easier in the future, with less redundancy of the step logic. The only major place where we deviate is that we have a separeately specified set of steps for the "old" VFX Platform 2021 and 2022 based tests (because we need to specify different container arguments to it). Signed-off-by: Larry Gritz --- .github/workflows/analysis.yml | 126 ++--- .github/workflows/ci-steps.yml | 205 ++++++++ .github/workflows/ci.yml | 498 +++++-------------- .github/workflows/docs.yml | 130 ++--- src/build-scripts/ci-startup.bash | 5 +- src/build-scripts/install_homebrew_deps.bash | 1 + 6 files changed, 417 insertions(+), 548 deletions(-) create mode 100644 .github/workflows/ci-steps.yml diff --git a/.github/workflows/analysis.yml b/.github/workflows/analysis.yml index 8184bbd689..0b35bca34e 100644 --- a/.github/workflows/analysis.yml +++ b/.github/workflows/analysis.yml @@ -16,10 +16,14 @@ on: - main - '*analysis*' - '*sonar*' - paths-ignore: - - '**.md' - - '**.rst' - - 'docs/**' + paths: + - '**' + - '!**.md' + - '!**.rst' + - '!**/analysis.yml' + - '!**/docs.yml' + - '!**.properties' + - '!docs/**' # Allow manual kicking off of the workflow from github.com workflow_dispatch: # Uncomment the following line if we want to run analysis on all PRs: @@ -40,6 +44,33 @@ jobs: # Exclude runs on forks, since only the main org has the SonarCloud # account credentials. if: github.repository == 'AcademySoftwareFoundation/OpenImageIO' + uses: ./.github/workflows/ci-steps.yml + with: + nametag: ${{ matrix.nametag || 'unnamed!' }} + runner: ${{ matrix.runner || 'ubuntu-latest' }} + container: ${{ matrix.container }} + cc_compiler: ${{ matrix.cc_compiler }} + cxx_compiler: ${{ matrix.cxx_compiler }} + cxx_std: ${{ matrix.cxx_std || '17' }} + build_type: ${{ matrix.build_type || 'Release' }} + depcmds: ${{ matrix.depcmds }} + extra_artifacts: ${{ matrix.extra_artifacts }} + fmt_ver: ${{ matrix.fmt_ver }} + opencolorio_ver: ${{ matrix.opencolorio_ver }} + openexr_ver: ${{ matrix.openexr_ver }} + pybind11_ver: ${{ matrix.pybind11_ver }} + python_ver: ${{ matrix.python_ver }} + setenvs: ${{ matrix.setenvs }} + simd: ${{ matrix.simd }} + skip_build: ${{ matrix.skip_build }} + skip_tests: ${{ matrix.skip_tests }} + abi_check: ${{ matrix.abi_check }} + build_docs: ${{ matrix.build_docs }} + generator: ${{ matrix.generator }} + ctest_args: ${{ matrix.ctest_args }} + ctest_test_timeout: ${{ matrix.ctest_test_timeout }} + coverage: ${{ matrix.coverage || '0' }} + sonar: ${{ matrix.sonar || '0' }} strategy: fail-fast: false matrix: @@ -47,13 +78,12 @@ jobs: - desc: sonar gcc9/C++14 py39 exr3.1 ocio2.2 nametag: static-analysis-sonar os: ubuntu-latest - container: aswf/ci-osl:2023-clang15 - vfxyear: 2023 + container: aswf/ci-osl:2024-clang17 cxx_std: 17 - python_ver: "3.10" + python_ver: "3.11" simd: "avx2,f16c" fmt_ver: 10.1.1 - pybind11_ver: v2.10.0 + pybind11_ver: v2.12.0 coverage: 1 # skip_tests: 1 sonar: 1 @@ -64,83 +94,3 @@ jobs: CMAKE_UNITY_BUILD=OFF CODECOV=1 CTEST_TEST_TIMEOUT=1200 - - runs-on: ${{ matrix.os }} - container: - image: ${{ matrix.container }} - env: - CXX: ${{matrix.cxx_compiler}} - CC: ${{matrix.cc_compiler}} - CMAKE_CXX_STANDARD: ${{matrix.cxx_std}} - USE_SIMD: ${{matrix.simd}} - FMT_VERSION: ${{matrix.fmt_ver}} - OPENEXR_VERSION: ${{matrix.openexr_ver}} - PYBIND11_VERSION: ${{matrix.pybind11_ver}} - PYTHON_VERSION: ${{matrix.python_ver}} - # DEBUG_CI: 1 - steps: - # We would like to use harden-runner, but it flags too many false - # positives, every time we download a dependency. We should use it only - # on CI runs where we are producing artifacts that users might rely on. - # - name: Harden Runner - # uses: step-security/harden-runner@248ae51c2e8cc9622ecf50685c8bf7150c6e8813 # v1.4.3 - # with: - # egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - with: - fetch-depth: '0' - - name: Prepare ccache timestamp - id: ccache_cache_keys - run: echo "date=`date -u +'%Y-%m-%dT%H:%M:%SZ'`" >> $GITHUB_OUTPUT - - name: ccache - id: ccache - uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0 - with: - path: /tmp/ccache - key: ${{github.job}}-${{matrix.nametag}}-${{steps.ccache_cache_keys.outputs.date}} - restore-keys: ${{github.job}}- - - name: Build setup - run: | - ${{matrix.setenvs}} - src/build-scripts/ci-startup.bash - - name: Dependencies - run: | - ${{matrix.depcmds}} - src/build-scripts/gh-installdeps.bash - - name: Install sonar-scanner and build-wrapper - if: matrix.sonar == '1' - uses: sonarsource/sonarcloud-github-c-cpp@e4882e1621ad2fb48dddfa48287411bed34789b1 # v2.0.2 - - name: Build - run: src/build-scripts/ci-build.bash - - name: Testsuite - if: matrix.skip_tests != '1' - run: src/build-scripts/ci-test.bash - - name: Code coverage - if: matrix.coverage == '1' - run: src/build-scripts/ci-coverage.bash - - name: Sonar-scanner - if: matrix.sonar == 1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} - run: | - which sonar-scanner - ls -l /__w/OpenImageIO/OpenImageIO/bw_output - echo "BUILD_OUTPUT_DIR is " "${{ env.BUILD_WRAPPER_OUT_DIR }}" - find . -name "*.gcov" -print - # sonar-scanner --define sonar.cfamily.build-wrapper-output="${{ env.BUILD_WRAPPER_OUT_DIR }}" - time sonar-scanner --define sonar.host.url="${{ env.SONAR_SERVER_URL }}" --define sonar.cfamily.build-wrapper-output="$BUILD_WRAPPER_OUT_DIR" --define sonar.cfamily.gcov.reportsPath="_coverage" --define sonar.cfamily.threads="$PARALLEL" - # Consult https://docs.sonarcloud.io/advanced-setup/ci-based-analysis/sonarscanner-cli/ for more information and options - - # - uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 # v4.3.0 - # if: failure() - # with: - # name: oiio-${{github.job}}-${{matrix.nametag}} - # path: | - # build/cmake-save - # build/testsuite/*/*.* - # !build/testsuite/oiio-images - # !build/testsuite/openexr-images - # !build/testsuite/fits-images - # !build/testsuite/j2kp4files_v1_5 - diff --git a/.github/workflows/ci-steps.yml b/.github/workflows/ci-steps.yml new file mode 100644 index 0000000000..8871fe41ac --- /dev/null +++ b/.github/workflows/ci-steps.yml @@ -0,0 +1,205 @@ +# Copyright Contributors to the OpenImageIO project. +# SPDX-License-Identifier: Apache-2.0 +# https://github.com/AcademySoftwareFoundation/OpenImageIO + +########################################################################## +# Common steps for all CI workflows +########################################################################## + +name: CI Steps + +on: + workflow_call: + # This inputs receive values via the "with:" section in ci_workflow.yml + inputs: + build: + type: string + runner: + type: string + container: + type: string + cc_compiler: + type: string + cxx_compiler: + type: string + cxx_std: + type: string + build_type: + type: string + depcmds: + type: string + extra_artifacts: + type: string + fmt_ver: + type: string + opencolorio_ver: + type: string + openexr_ver: + type: string + pybind11_ver: + type: string + python_ver: + type: string + setenvs: + type: string + simd: + type: string + skip_build: + type: string + skip_tests: + type: string + abi_check: + type: string + build_docs: + type: string + generator: + type: string + ctest_args: + type: string + ctest_test_timeout: + type: string + coverage: + type: string + sonar: + type: string + nametag: + type: string + +permissions: read-all + + +jobs: + + steps: + name: "${{inputs.cxx_compiler}} c++${{inputs.cxx_std}} py${{inputs.python_ver}}" + runs-on: ${{ inputs.runner }} + container: + image: ${{ inputs.container }} + + env: + CXX: ${{inputs.cxx_compiler}} + CC: ${{inputs.cc_compiler}} + CMAKE_BUILD_TYPE: ${{inputs.build_type}} + CMAKE_CXX_STANDARD: ${{inputs.cxx_std}} + CMAKE_GENERATOR: ${{inputs.generator}} + CTEST_ARGS: ${{inputs.ctest_args}} + CTEST_TEST_TIMEOUT: ${{inputs.ctest_test_timeout}} + USE_SIMD: ${{inputs.simd}} + FMT_VERSION: ${{inputs.fmt_ver}} + OPENCOLORIO_VERSION: ${{inputs.opencolorio_ver}} + OPENEXR_VERSION: ${{inputs.openexr_ver}} + PYBIND11_VERSION: ${{inputs.pybind11_ver}} + PYTHON_VERSION: ${{inputs.python_ver}} + ABI_CHECK: ${{inputs.abi_check}} + ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION: node16 + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true + + steps: + # - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + # # Note: can't upgrade to actions/checkout 4.0 because it needs newer + # # glibc than these containers have. + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + fetch-depth: '0' + - name: Prepare ccache timestamp + id: ccache_cache_keys + shell: bash + run: echo "date=`date -u +'%Y-%m-%dT%H:%M:%SZ'`" >> $GITHUB_OUTPUT + - name: ccache + id: ccache + uses: actions/cache@e12d46a63a90f2fae62d114769bbf2a179198b5c # v3.3.3 + # Note: can't upgrade to actions/cache 4.0 because it needs newer + # glibc than these containers have. + with: + path: ./ccache + key: ${{github.job}}-${{inputs.nametag}}-${{steps.ccache_cache_keys.outputs.date}} + restore-keys: ${{github.job}}- + save-always: true + - name: Setup Nuget.exe (Windows only) + if: runner.os == 'Windows' + uses: nuget/setup-nuget@a21f25cd3998bf370fde17e3f1b4c12c175172f9 # v2.0.0 + - name: Build setup + shell: bash + run: | + ${{inputs.setenvs}} + src/build-scripts/ci-startup.bash + - name: Dependencies + shell: bash + run: | + ${{inputs.depcmds}} + if [[ "$RUNNER_OS" == "Linux" ]]; then + src/build-scripts/gh-installdeps.bash + elif [[ "$RUNNER_OS" == "macOS" ]]; then + src/build-scripts/install_homebrew_deps.bash + elif [[ "$RUNNER_OS" == "Windows" ]]; then + src/build-scripts/gh-win-installdeps.bash + fi + - name: Install sonar-scanner and build-wrapper + if: inputs.sonar == '1' + uses: sonarsource/sonarcloud-github-c-cpp@e4882e1621ad2fb48dddfa48287411bed34789b1 # v2.0.2 + - name: Build + if: inputs.skip_build != '1' + shell: bash + run: src/build-scripts/ci-build.bash + - name: Testsuite + if: inputs.skip_tests != '1' + shell: bash + run: src/build-scripts/ci-test.bash + - name: Code coverage + if: inputs.coverage == '1' + run: src/build-scripts/ci-coverage.bash + - name: Sonar-scanner + if: inputs.sonar == 1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} + run: | + which sonar-scanner + ls -l /__w/OpenImageIO/OpenImageIO/bw_output + echo "BUILD_OUTPUT_DIR is " "${{ env.BUILD_WRAPPER_OUT_DIR }}" + find . -name "*.gcov" -print + # sonar-scanner --define sonar.cfamily.build-wrapper-output="${{ env.BUILD_WRAPPER_OUT_DIR }}" + time sonar-scanner --define sonar.host.url="${{ env.SONAR_SERVER_URL }}" --define sonar.cfamily.build-wrapper-output="$BUILD_WRAPPER_OUT_DIR" --define sonar.cfamily.gcov.reportsPath="_coverage" --define sonar.cfamily.threads="$PARALLEL" + # Consult https://docs.sonarcloud.io/advanced-setup/ci-based-analysis/sonarscanner-cli/ for more information and options + - name: Check out ABI standard + if: inputs.abi_check != '' + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + ref: ${{inputs.abi_check}} + path: abi_standard + - name: Build ABI standard + if: inputs.abi_check != '' + shell: bash + run: | + mkdir -p abi_standard/build + pushd abi_standard + src/build-scripts/ci-build.bash + popd + - name: Check ABI + if: inputs.abi_check != '' + shell: bash + run: | + src/build-scripts/ci-abicheck.bash ./build abi_standard/build libOpenImageIO libOpenImageIO_Util + - name: Build Docs + if: inputs.build_docs == '1' + shell: bash + run: | + cd src/doc + time make doxygen + time make sphinx + - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 + # Note: versions of upload-artifact >= 4.0 can't run on the ASWF + # containers because they need a newer glibc than they have. + if: ${{ failure() || inputs.build_docs == '1'}} + with: + name: oiio-${{github.job}}-${{inputs.nametag}} + path: | + build/cmake-save + build/compat_reports + build/sphinx + build/testsuite/*/*.* + !build/testsuite/oiio-images + !build/testsuite/openexr-images + !build/testsuite/fits-images + !build/testsuite/j2kp4files_v1_5 + ${{ inputs.extra_artifacts }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fd70afc747..954741225c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,18 +7,20 @@ name: CI on: push: # Skip jobs when only documentation files are changed - paths-ignore: - - '**.md' - - '**.rst' - - '**/analysis.yml' - - '**/docs.yml' - - '**.properties' - - 'docs/**' + paths: + - '**' + - '!**.md' + - '!**.rst' + - '!**/analysis.yml' + - '!**/docs.yml' + - '!**.properties' + - '!docs/**' pull_request: - paths-ignore: - - '**.md' - - '**.rst' - - 'docs/**' + paths: + - '**' + - '!**.md' + - '!**.rst' + - '!docs/**' schedule: # Full nightly build - cron: "0 8 * * *" @@ -32,12 +34,12 @@ permissions: read-all jobs: aswf-old: - name: "VFX${{matrix.vfxyear}} ${{matrix.desc}}" + name: "(old) ${{matrix.desc}}" strategy: fail-fast: false matrix: include: - - desc: gcc9/C++17 py3.7 exr3.1 ocio2.0 + - desc: VP2021 gcc9/C++17 py3.7 exr3.1 ocio2.0 nametag: linux-vfx2021 runner: ubuntu-latest container: aswf/ci-osl:2021 @@ -51,7 +53,7 @@ jobs: pybind11_ver: v2.7.0 setenvs: export PUGIXML_VERSION=v1.9 WEBP_VERSION=v1.1.0 USE_OPENVDB=0 FREETYPE_VERSION=VER-2-10-0 - - desc: clang10/C++17 avx2 exr3.1 ocio2.0 + - desc: VP2021 clang10/C++17 avx2 exr3.1 ocio2.0 nametag: linux-clang10-cpp14 runner: ubuntu-latest container: aswf/ci-osl:2021-clang10 @@ -67,7 +69,7 @@ jobs: fmt_ver: 8.1.1 setenvs: export USE_OPENVDB=0 FREETYPE_VERSION=VER-2-11-0 - - desc: gcc9/C++17 py39 exr3.1 ocio2.1 + - desc: VP2022 gcc9/C++17 py39 exr3.1 ocio2.1 nametag: linux-vfx2022 runner: ubuntu-latest container: aswf/ci-osl:2022-clang11 @@ -79,7 +81,7 @@ jobs: fmt_ver: 8.1.1 pybind11_ver: v2.9.0 setenvs: export FREETYPE_VERSION=VER-2-12-0 - - desc: clang13/C++17 py39 avx2 exr3.1 ocio2.1 + - desc: VP2022 clang13/C++17 py39 avx2 exr3.1 ocio2.1 nametag: linux-vfx2022-clang13 runner: ubuntu-latest container: aswf/ci-osl:2022-clang13 @@ -93,46 +95,6 @@ jobs: fmt_ver: 9.1.0 pybind11_ver: v2.8.1 setenvs: export FREETYPE_VERSION=VER-2-12-0 - - desc: icc/C++17 py3.9 exr3.1 ocio2.1 qt5.15 - nametag: linux-vfx2022-icc - runner: ubuntu-latest - container: aswf/ci-osl:2022 - vfxyear: 2022 - old_node: 1 - cxx_std: 17 - opencolorio_ver: v2.2.1 - python_ver: 3.9 - # simd: "avx2,f16c" - fmt_ver: 7.1.3 - # icc MUST use this older FMT version - pybind11_ver: v2.9.0 - setenvs: export USE_ICC=1 USE_OPENVDB=0 - OIIO_EXTRA_CPP_ARGS="-fp-model=precise" - FREETYPE_VERSION=VER-2-13-0 - DISABLE_libuhdr=1 - # For icc, use fp-model precise to eliminate needless LSB errors - # that make test results differ from other platforms. - - desc: icx/C++17 py3.9 exr3.1 ocio2.2 qt5.15 - nametag: linux-vfx2022-icx - runner: ubuntu-latest - container: aswf/ci-osl:2022 - vfxyear: 2022 - old_node: 1 - cc_compiler: icx - cxx_compiler: icpx - cxx_std: 17 - opencolorio_ver: v2.3.2 - python_ver: 3.9 - pybind11_ver: v2.9.0 - simd: "avx2,f16c" - setenvs: export USE_OPENVDB=0 - OPENCOLORIO_CXX=g++ - UHDR_CMAKE_C_COMPILER=gcc - UHDR_CMAKE_CXX_COMPILER=g++ - # OCIO doesn't build with icx, so we have to force the ocio build - # to use g++. - # Building libuhdr with icx results in test failures - # so we force using gcc/g++. - desc: oldest gcc9.3/C++17 py3.7 exr-3.1 # Oldest versions of the dependencies that we support. nametag: linux-oldest @@ -276,165 +238,147 @@ jobs: !build/testsuite/j2kp4files_v1_5 - aswf: - name: "VFX${{matrix.vfxyear}} ${{matrix.desc}}" + + configs: + name: "${{matrix.desc}}" + uses: ./.github/workflows/ci-steps.yml + with: + nametag: ${{ matrix.nametag || 'unnamed!' }} + runner: ${{ matrix.runner || 'ubuntu-latest' }} + container: ${{ matrix.container }} + cc_compiler: ${{ matrix.cc_compiler }} + cxx_compiler: ${{ matrix.cxx_compiler }} + cxx_std: ${{ matrix.cxx_std || '17' }} + build_type: ${{ matrix.build_type || 'Release' }} + depcmds: ${{ matrix.depcmds }} + extra_artifacts: ${{ matrix.extra_artifacts }} + fmt_ver: ${{ matrix.fmt_ver }} + opencolorio_ver: ${{ matrix.opencolorio_ver }} + openexr_ver: ${{ matrix.openexr_ver }} + pybind11_ver: ${{ matrix.pybind11_ver }} + python_ver: ${{ matrix.python_ver }} + setenvs: ${{ matrix.setenvs }} + simd: ${{ matrix.simd }} + skip_build: ${{ matrix.skip_build }} + skip_tests: ${{ matrix.skip_tests }} + abi_check: ${{ matrix.abi_check }} + build_docs: ${{ matrix.build_docs }} + generator: ${{ matrix.generator }} + ctest_args: ${{ matrix.ctest_args }} + ctest_test_timeout: ${{ matrix.ctest_test_timeout }} + coverage: ${{ matrix.coverage || 0 }} + sonar: ${{ matrix.sonar || 0 }} strategy: fail-fast: false matrix: include: - - desc: gcc11/C++17 py3.10 exr3.1 ocio2.2 + - desc: VFX2023 gcc11/C++17 py3.10 exr3.1 ocio2.2 nametag: linux-vfx2023 runner: ubuntu-latest container: aswftesting/ci-osl:2023-clang15 - vfxyear: 2023 - cxx_std: 17 python_ver: "3.10" simd: "avx2,f16c" fmt_ver: 10.1.1 pybind11_ver: v2.10.0 - - desc: gcc11/C++17 py3.11 exr3.2 ocio2.3 + - desc: VFX2023 icc/C++17 py3.10 exr3.1 ocio2.1 qt5.15 + nametag: linux-vfx2023-icc + runner: ubuntu-latest + container: aswf/ci-osl:2023 + opencolorio_ver: v2.2.1 + python_ver: "3.10" + # simd: "avx2,f16c" + fmt_ver: 7.1.3 + # icc MUST use this older FMT version + pybind11_ver: v2.9.0 + setenvs: export USE_ICC=1 USE_OPENVDB=0 + OIIO_EXTRA_CPP_ARGS="-fp-model=precise" + FREETYPE_VERSION=VER-2-13-0 + DISABLE_libuhdr=1 + # For icc, use fp-model precise to eliminate needless LSB errors + # that make test results differ from other platforms. + - desc: VFX2023 icx/C++17 py3.10 exr3.1 ocio2.2 qt5.15 + nametag: linux-vfx2023-icx + runner: ubuntu-latest + container: aswf/ci-osl:2023 + cc_compiler: icx + cxx_compiler: icpx + python_ver: "3.10" + pybind11_ver: v2.10.0 + simd: "avx2,f16c" + setenvs: export USE_OPENVDB=0 + xOPENCOLORIO_CXX=g++ + UHDR_CMAKE_C_COMPILER=gcc + UHDR_CMAKE_CXX_COMPILER=g++ + # Building libuhdr with icx results in test failures + # so we force using gcc/g++. + - desc: VFX2024 gcc11/C++17 py3.11 exr3.2 ocio2.3 nametag: linux-vfx2024 runner: ubuntu-latest container: aswftesting/ci-osl:2024-clang17 - vfxyear: 2024 - cxx_std: 17 python_ver: "3.11" simd: "avx2,f16c" fmt_ver: 10.1.1 pybind11_ver: v2.12.0 - - desc: sanitizers + - desc: VFX2024 clang/C++17 py3.11 exr3.2 ocio2.3 + nametag: linux-vfx2024 + runner: ubuntu-latest + container: aswftesting/ci-osl:2024-clang17 + cc_compiler: clang + cxx_compiler: clang++ + python_ver: "3.11" + simd: "avx2,f16c" + fmt_ver: 10.1.1 + pybind11_ver: v2.12.0 + - desc: VFX2024 sanitizers nametag: sanitizer runner: ubuntu-latest container: aswf/ci-osl:2024-clang17 - vfxyear: 2024 cc_compiler: clang cxx_compiler: clang++ - cxx_std: 17 + build_type: Debug opencolorio_ver: v2.3.2 python_ver: "3.11" + ctest_test_timeout: "1200" setenvs: export SANITIZE=address,undefined - OIIO_CMAKE_FLAGS="-DSANITIZE=address,undefined -DUSE_PYTHON=0" - CMAKE_BUILD_TYPE=Debug - CTEST_TEST_TIMEOUT=1200 + OIIO_CMAKE_FLAGS="-DSANITIZE=address,undefined -DOIIO_HARDENING=3 -DUSE_PYTHON=0" CTEST_EXCLUSIONS="broken|png-damaged" + # Test ABI stability. `abi_check` is the version or commit that we # believe is the current standard against which we don't want to # break the ABI. Basically, we will build that version as well as # the current one, and compare the resulting libraries. - - desc: abi check + - desc: ABI check nametag: abi-check runner: ubuntu-latest - container: aswftesting/ci-osl:2023-clang15 - vfxyear: 2023 - cxx_std: 17 - python_ver: "3.10" + container: aswf/ci-osl:2024-clang17 + build_type: RelWithDebInfo + python_ver: "3.11" simd: "avx2,f16c" - fmt_ver: 10.1.1 - pybind11_ver: v2.10.0 skip_tests: 1 # abi_check: v2.6.2.0 abi_check: 7929c81619ea8201065c475761546d51091f6773 setenvs: export OIIO_CMAKE_FLAGS="-DOIIO_BUILD_TOOLS=0 -DOIIO_BUILD_TESTS=0 -DUSE_PYTHON=0" USE_OPENCV=0 USE_FFMPEG=0 USE_PYTHON=0 USE_FREETYPE=0 - CMAKE_BUILD_TYPE=RelWithDebInfo - - runs-on: ${{ matrix.runner }} - container: - image: ${{ matrix.container }} - env: - CXX: ${{matrix.cxx_compiler}} - CC: ${{matrix.cc_compiler}} - CMAKE_CXX_STANDARD: ${{matrix.cxx_std}} - USE_SIMD: ${{matrix.simd}} - FMT_VERSION: ${{matrix.fmt_ver}} - OPENCOLORIO_VERSION: ${{matrix.opencolorio_ver}} - OPENEXR_VERSION: ${{matrix.openexr_ver}} - PYBIND11_VERSION: ${{matrix.pybind11_ver}} - PYTHON_VERSION: ${{matrix.python_ver}} - ABI_CHECK: ${{matrix.abi_check}} - steps: - # We would like to use harden-runner, but it flags too many false - # positives, every time we download a dependency. We should use it only - # on CI runs where we are producing artifacts that users might rely on. - # - name: Harden Runner - # uses: step-security/harden-runner@248ae51c2e8cc9622ecf50685c8bf7150c6e8813 # v1.4.3 - # with: - # egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs - - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 - # Note: can't upgrade to actions/checkout 4.0 because it needs newer - # glibc than these containers have. - - name: Prepare ccache timestamp - id: ccache_cache_keys - run: echo "date=`date -u +'%Y-%m-%dT%H:%M:%SZ'`" >> $GITHUB_OUTPUT - - name: ccache - id: ccache - uses: actions/cache@e12d46a63a90f2fae62d114769bbf2a179198b5c # v3.3.3 - # Note: can't upgrade to actions/cache 4.0 because it needs newer - # glibc than these containers have. - with: - path: /tmp/ccache - key: ${{github.job}}-${{matrix.nametag}}-${{steps.ccache_cache_keys.outputs.date}} - restore-keys: ${{github.job}}- - - name: Build setup - run: | - ${{matrix.setenvs}} - src/build-scripts/ci-startup.bash - - name: Dependencies - run: | - ${{matrix.depcmds}} - src/build-scripts/gh-installdeps.bash - - name: Build - if: matrix.skip_build != '1' - run: src/build-scripts/ci-build.bash - - name: Testsuite - if: matrix.skip_tests != '1' - run: src/build-scripts/ci-test.bash - - name: Check out ABI standard - if: matrix.abi_check != '' - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - with: - ref: ${{matrix.abi_check}} - path: abi_standard - - name: Build ABI standard - if: matrix.abi_check != '' - run: | - mkdir -p abi_standard/build - pushd abi_standard - src/build-scripts/ci-build.bash - popd - - name: Check ABI - if: matrix.abi_check != '' - run: | - src/build-scripts/ci-abicheck.bash ./build abi_standard/build libOpenImageIO libOpenImageIO_Util - - name: Build Docs - if: matrix.build_docs == '1' - run: | - cd src/doc - time make doxygen - time make sphinx - - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 - # Note: versions of upload-artifact >= 4.0 can't run on the ASWF - # containers because they need a newer glibc than they have. - if: ${{ failure() || matrix.build_docs == '1'}} - with: - name: oiio-${{github.job}}-${{matrix.nametag}} - path: | - build/cmake-save - build/compat_reports - build/sphinx - build/testsuite/*/*.* - !build/testsuite/oiio-images - !build/testsuite/openexr-images - !build/testsuite/fits-images - !build/testsuite/j2kp4files_v1_5 + # Test formatting. This test entry doesn't do a full build, it + # just runs clang-format on everything, and passes if nothing is + # misformatted. Upon failure, the build artifact will be the full + # source code with the formatting fixed (diffs will also appear in + # the console output). + - desc: "clang-format" + nametag: clang-format + runner: ubuntu-latest + cxx_std: 17 + extra_artifacts: "src/*.*" + openexr_ver: v3.1.13 + python_ver: "3.10" + skip_tests: 1 + setenvs: export BUILDTARGET=clang-format + OIIO_CMAKE_FLAGS=-DUSE_PYTHON=0 + LLVM_VERSION=17.0.6 LLVM_DISTRO_NAME=ubuntu-22.04 + SKIP_SYSTEM_DEPS_INSTALL=1 QT_VERSION=0 + OpenImageIO_OPTIONAL_DEPS=ALL - ubuntu: - name: "Ubuntu ${{matrix.desc}}" - strategy: - fail-fast: false - matrix: - include: - desc: latest releases gcc13 C++20 py3.12 avx2 exr3.3 ocio2.4 nametag: linux-latest-releases runner: ubuntu-24.04 @@ -475,7 +419,7 @@ jobs: PTEX_VERSION=main PUGIXML_VERSION=master WEBP_VERSION=main - OIIO_CMAKE_FLAGS="-DFORTIFY_SOURCE=2" + OIIO_CMAKE_FLAGS="-DOIIO_HARDENING=2" EXTRA_DEP_PACKAGES="python3.12-dev python3-numpy" USE_OPENVDB=0 SKIP_APT_GET_UPDATE=1 @@ -501,6 +445,7 @@ jobs: PTEX_VERSION=v2.4.2 PUGIXML_VERSION=v1.14 WEBP_VERSION=v1.4.0 + - desc: clang14 C++20 avx2 exr3.1 ocio2.1 nametag: linux-clang14 runner: ubuntu-20.04 @@ -521,14 +466,14 @@ jobs: runner: ubuntu-20.04 cxx_compiler: g++-9 cxx_std: 17 + build_type: Debug python_ver: 3.8 simd: sse4.2 openexr_ver: v3.1.13 pybind11_ver: v2.7.0 - setenvs: export CMAKE_BUILD_TYPE=Debug - PUGIXML_VERSION=v1.9 - CTEST_TEST_TIMEOUT=1200 - - desc: static libs gcc9 C++17 exr2.4 + ctest_test_timeout: 1200 + setenvs: export PUGIXML_VERSION=v1.9 + - desc: static libs gcc9 C++17 exr3.1 nametag: linux-static runner: ubuntu-20.04 cxx_compiler: g++-9 @@ -540,178 +485,48 @@ jobs: depcmds: | sudo rm -rf /usr/local/include/OpenEXR sudo rm -rf /usr/local/lib64/cmake/{IlmBase,OpenEXR} - # Test formatting. This test entry doesn't do a full build, it - # just runs clang-format on everything, and passes if nothing is - # misformatted. Upon failure, the build artifact will be the full - # source code with the formatting fixed (diffs will also appear in - # the console output). - - desc: "clang-format" - nametag: clang-format - runner: ubuntu-latest - cxx_std: 17 - extra_artifacts: "src/*.*" - openexr_ver: v3.1.13 - python_ver: "3.10" - skip_tests: 1 - setenvs: export BUILDTARGET=clang-format - OIIO_CMAKE_FLAGS=-DUSE_PYTHON=0 - LLVM_VERSION=17.0.6 LLVM_DISTRO_NAME=ubuntu-22.04 - SKIP_SYSTEM_DEPS_INSTALL=1 QT_VERSION=0 - xOpenImageIO_BUILD_MISSING_DEPS=missing - OpenImageIO_OPTIONAL_DEPS=ALL - - runs-on: ${{ matrix.runner }} - env: - CXX: ${{matrix.cxx_compiler}} - CC: ${{matrix.cc_compiler}} - CMAKE_CXX_STANDARD: ${{matrix.cxx_std}} - USE_SIMD: ${{matrix.simd}} - FMT_VERSION: ${{matrix.fmt_ver}} - OPENCOLORIO_VERSION: ${{matrix.opencolorio_ver}} - OPENEXR_VERSION: ${{matrix.openexr_ver}} - PYBIND11_VERSION: ${{matrix.pybind11_ver}} - PYTHON_VERSION: ${{matrix.python_ver}} - steps: - # We would like to use harden-runner, but it flags too many false - # positives, every time we download a dependency. We should use it only - # on CI runs where we are producing artifacts that users might rely on. - # - name: Harden Runner - # uses: step-security/harden-runner@248ae51c2e8cc9622ecf50685c8bf7150c6e8813 # v1.4.3 - # with: - # egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - - name: Prepare ccache timestamp - id: ccache_cache_keys - run: echo "date=`date -u +'%Y-%m-%dT%H:%M:%SZ'`" >> $GITHUB_OUTPUT - - name: ccache - id: ccache - uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0 - with: - path: /tmp/ccache - key: ${{github.job}}-${{matrix.nametag}}-${{ steps.ccache_cache_keys.outputs.date }} - restore-keys: ${{github.job}}- - save-always: true - - name: Build setup - run: | - ${{matrix.setenvs}} - src/build-scripts/ci-startup.bash - - name: Dependencies - run: | - ${{matrix.depcmds}} - src/build-scripts/gh-installdeps.bash - - name: Build - run: src/build-scripts/ci-build.bash - - name: Testsuite - if: matrix.skip_tests != '1' - run: src/build-scripts/ci-test.bash - - uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 # v4.3.0 - if: failure() - with: - name: oiio-${{github.job}}-${{matrix.nametag}} - path: | - build/cmake-save - build/testsuite/*/*.* - ${{ matrix.extra_artifacts }} - !build/testsuite/oiio-images - !build/testsuite/openexr-images - !build/testsuite/fits-images - !build/testsuite/j2kp4files_v1_5 - - macos: - name: "${{matrix.runner}} appleclang${{matrix.aclang}}/C++${{matrix.cxx_std}} py${{matrix.python_ver}} ${{matrix.desc}}" - strategy: - fail-fast: false - matrix: - include: - - desc: MacOS-13-Intel + # + # Mac Tests + # + - desc: MacOS-13-Intel aclang14/C++17/py3.11 runner: macos-13 nametag: macos13-py311 cc_compiler: clang cxx_compiler: clang++ cxx_std: 17 python_ver: "3.11" - aclang: 14 simd: sse4.2,avx2 + ctest_test_timeout: 1200 setenvs: export MACOSX_DEPLOYMENT_TARGET=12.0 - - desc: MacOS-14-ARM + - desc: MacOS-14-ARM aclang15/C++20/py3.12 runner: macos-14 nametag: macos14-arm-py312 cc_compiler: clang cxx_compiler: clang++ cxx_std: 20 python_ver: "3.12" - aclang: 15 - - desc: MacOS-15-ARM + ctest_test_timeout: 800 + - desc: MacOS-15-ARM aclang16/C++20/py3.13 runner: macos-15 nametag: macos15-arm-py313 cc_compiler: clang cxx_compiler: clang++ cxx_std: 20 python_ver: "3.13" - aclang: 16 - runs-on: ${{ matrix.runner }} - env: - CXX: ${{matrix.cxx_compiler}} - CC: ${{matrix.cc_compiler}} - CMAKE_CXX_STANDARD: ${{ matrix.cxx_std }} - PYTHON_VERSION: ${{ matrix.python_ver }} - USE_SIMD: ${{matrix.simd}} - CTEST_TEST_TIMEOUT: 800 - steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - - name: Prepare ccache timestamp - id: ccache_cache_keys - run: echo "date=`date -u +'%Y-%m-%dT%H:%M:%SZ'`" >> $GITHUB_OUTPUT - - name: ccache - id: ccache - uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0 - with: - path: /Users/runner/.ccache - key: ${{github.job}}-${{matrix.nametag}}-${{ steps.ccache_cache_keys.outputs.date }} - restore-keys: ${{github.job}}- - save-always: true - - name: Build setup - run: | - ${{matrix.setenvs}} - src/build-scripts/ci-startup.bash - - name: Dependencies - run: | - src/build-scripts/install_homebrew_deps.bash - ${{matrix.depcmds}} - - name: Build - run: | - export PYTHONPATH=/usr/local/lib/python${PYTHON_VERSION}/site-packages:$PYTHONPATH - src/build-scripts/ci-build.bash - - name: Testsuite - run: src/build-scripts/ci-test.bash - - uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 # v4.3.0 - if: failure() - with: - name: oiio-${{github.job}}-${{matrix.nametag}} - path: | - build/cmake-save - build/testsuite/*/*.* - !build/testsuite/oiio-images - !build/testsuite/openexr-images - !build/testsuite/fits-images - !build/testsuite/j2kp4files_v1_5 - + ctest_test_timeout: 800 - windows: - name: "${{matrix.desc}} VS${{matrix.vsver}}" - strategy: - fail-fast: false - matrix: - include: - - desc: windows-2019 + # + # Windows Tests + # + - desc: Windows-2019 VS2019 runner: windows-2019 vsver: 2019 generator: "Visual Studio 16 2019" # openexr_ver: v3.2.4 python_ver: 3.7 # simd: sse4.2 - - desc: windows-2022 + - desc: Windows-2022 VS2022 runner: windows-2022 vsver: 2022 generator: "Visual Studio 17 2022" @@ -719,40 +534,3 @@ jobs: python_ver: "3.9" # simd: sse4.2 # setenvs: export OpenImageIO_BUILD_MISSING_DEPS=none - runs-on: ${{ matrix.runner }} - env: - PYTHON_VERSION: ${{matrix.python_ver}} - CMAKE_GENERATOR: ${{matrix.generator}} - OPENEXR_VERSION: ${{matrix.openexr_ver}} - USE_SIMD: ${{matrix.simd}} - CTEST_ARGS: "--timeout 180 --repeat after-timeout:6" - steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - - name: Setup Nuget.exe - uses: nuget/setup-nuget@a21f25cd3998bf370fde17e3f1b4c12c175172f9 # v2.0.0 - - name: Build setup - shell: bash - run: | - ${{matrix.setenvs}} - src/build-scripts/ci-startup.bash - - name: Dependencies - shell: bash - run: src/build-scripts/gh-win-installdeps.bash - - name: Build - shell: bash - run: src/build-scripts/ci-build.bash - - name: Testsuite - shell: bash - run: src/build-scripts/ci-test.bash - - uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 # v4.3.0 - if: failure() - with: - name: oiio-${{github.job}}-VS${{matrix.vsver}} - path: | - build/cmake-save - build/testsuite/*/*.* - build/deps/dist/*/*.cmake - !build/testsuite/oiio-images - !build/testsuite/openexr-images - !build/testsuite/fits-images - !build/testsuite/j2kp4files_v1_5 diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index be5eb81eb0..9ff3d72c08 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -34,11 +34,43 @@ on: permissions: read-all +# Allow subsequent pushes to the same PR or REF to cancel any previous jobs. +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: + docs: + name: "Docs / ${{matrix.desc}}" + uses: ./.github/workflows/ci-steps.yml + with: + nametag: ${{ matrix.nametag || 'unnamed!' }} + runner: ${{ matrix.runner || 'ubuntu-latest' }} + container: ${{ matrix.container }} + cc_compiler: ${{ matrix.cc_compiler }} + cxx_compiler: ${{ matrix.cxx_compiler }} + cxx_std: ${{ matrix.cxx_std || '17' }} + build_type: ${{ matrix.build_type || 'Release' }} + depcmds: ${{ matrix.depcmds }} + extra_artifacts: ${{ matrix.extra_artifacts }} + fmt_ver: ${{ matrix.fmt_ver }} + opencolorio_ver: ${{ matrix.opencolorio_ver }} + openexr_ver: ${{ matrix.openexr_ver }} + pybind11_ver: ${{ matrix.pybind11_ver }} + python_ver: ${{ matrix.python_ver }} + setenvs: ${{ matrix.setenvs }} + simd: ${{ matrix.simd }} + skip_build: ${{ matrix.skip_build }} + skip_tests: ${{ matrix.skip_tests }} + abi_check: ${{ matrix.abi_check }} + build_docs: ${{ matrix.build_docs }} + generator: ${{ matrix.generator }} + ctest_args: ${{ matrix.ctest_args }} + ctest_test_timeout: ${{ matrix.ctest_test_timeout }} + coverage: ${{ matrix.coverage || '0' }} + sonar: ${{ matrix.sonar || '0' }} - aswf: - name: "Docs / VFX${{matrix.vfxyear}} ${{matrix.desc}}" strategy: fail-fast: false matrix: @@ -55,97 +87,3 @@ jobs: skip_tests: 1 setenvs: export EXTRA_DEP_PACKAGES="doxygen sphinx-doc" PIP_INSTALLS="sphinx breathe sphinx-tabs furo" - - runs-on: ${{ matrix.runner }} - container: - image: ${{ matrix.container }} - env: - CXX: ${{matrix.cxx_compiler}} - CC: ${{matrix.cc_compiler}} - CMAKE_CXX_STANDARD: ${{matrix.cxx_std}} - USE_SIMD: ${{matrix.simd}} - FMT_VERSION: ${{matrix.fmt_ver}} - OPENCOLORIO_VERSION: ${{matrix.opencolorio_ver}} - OPENEXR_VERSION: ${{matrix.openexr_ver}} - PYBIND11_VERSION: ${{matrix.pybind11_ver}} - PYTHON_VERSION: ${{matrix.python_ver}} - ABI_CHECK: ${{matrix.abi_check}} - ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION: node16 - ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true - steps: - # We would like to use harden-runner, but it flags too many false - # positives, every time we download a dependency. We should use it only - # on CI runs where we are producing artifacts that users might rely on. - # - name: Harden Runner - # uses: step-security/harden-runner@248ae51c2e8cc9622ecf50685c8bf7150c6e8813 # v1.4.3 - # with: - # egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs - - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 - # Note: can't upgrade to actions/checkout 4.0 because it needs newer - # glibc than these containers have. - - name: Prepare ccache timestamp - id: ccache_cache_keys - run: echo "date=`date -u +'%Y-%m-%dT%H:%M:%SZ'`" >> $GITHUB_OUTPUT - - name: ccache - id: ccache - uses: actions/cache@e12d46a63a90f2fae62d114769bbf2a179198b5c # v3.3.3 - # Note: can't upgrade to actions/cache 4.0 because it needs newer - # glibc than these containers have. - with: - path: /tmp/ccache - key: ${{github.job}}-${{matrix.nametag}}-${{steps.ccache_cache_keys.outputs.date}} - restore-keys: ${{github.job}}- - - name: Build setup - run: | - ${{matrix.setenvs}} - src/build-scripts/ci-startup.bash - - name: Dependencies - run: | - ${{matrix.depcmds}} - src/build-scripts/gh-installdeps.bash - - name: Build - if: matrix.skip_build != '1' - run: src/build-scripts/ci-build.bash - - name: Testsuite - if: matrix.skip_tests != '1' - run: src/build-scripts/ci-test.bash - - name: Check out ABI standard - if: matrix.abi_check != '' - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - with: - ref: ${{matrix.abi_check}} - path: abi_standard - - name: Build ABI standard - if: matrix.abi_check != '' - run: | - mkdir -p abi_standard/build - pushd abi_standard - src/build-scripts/ci-build.bash - popd - - name: Check ABI - if: matrix.abi_check != '' - run: | - src/build-scripts/ci-abicheck.bash ./build abi_standard/build libOpenImageIO libOpenImageIO_Util - - name: Build Docs - if: matrix.build_docs == '1' - run: | - cd src/doc - time make doxygen - time make sphinx - - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 - # Note: versions of upload-artifact >= 4.0 can't run on the ASWF - # containers because they need a newer glibc than they have. - if: ${{ failure() || matrix.build_docs == '1'}} - with: - name: oiio-${{github.job}}-${{matrix.nametag}} - path: | - build/cmake-save - build/compat_reports - build/sphinx - build/testsuite/*/*.* - !build/testsuite/oiio-images - !build/testsuite/openexr-images - !build/testsuite/fits-images - !build/testsuite/j2kp4files_v1_5 - - diff --git a/src/build-scripts/ci-startup.bash b/src/build-scripts/ci-startup.bash index 527f29d594..2307d4d2f8 100755 --- a/src/build-scripts/ci-startup.bash +++ b/src/build-scripts/ci-startup.bash @@ -12,10 +12,7 @@ export PATH=/usr/local/bin/_ccache:/usr/lib/ccache:$PATH export USE_CCACHE=${USE_CCACHE:=1} export CCACHE_CPP2= -export CCACHE_DIR=/tmp/ccache -if [[ "${RUNNER_OS}" == "macOS" ]] ; then - export CCACHE_DIR=$HOME/.ccache -fi +export CCACHE_DIR=$HOME/.ccache mkdir -p $CCACHE_DIR export OpenImageIO_ROOT=$PWD/dist diff --git a/src/build-scripts/install_homebrew_deps.bash b/src/build-scripts/install_homebrew_deps.bash index 0c0fd56aab..fb1c98424c 100755 --- a/src/build-scripts/install_homebrew_deps.bash +++ b/src/build-scripts/install_homebrew_deps.bash @@ -44,6 +44,7 @@ brew install --display-times -q freetype libraw dcmtk pybind11 numpy || true brew install --display-times -q ffmpeg libheif ptex || true brew install --display-times -q tbb || true brew install --display-times -q openvdb || true +brew install --display-times -q robin-map || true if [[ "${USE_OPENCV}" != "0" ]] && [[ "${INSTALL_OPENCV:=1}" != "0" ]] ; then brew install --display-times -q opencv || true fi