random-macosx-build.yml

name: MacOS build of a random missing solc version

on:
  push:
    # Action disabled. The filter excludes all the possible branch names.
    branches: ["!*"]

env:
  MIN_SUPPORTED_SOLIDITY_VERSION: 0.3.6
  TARGET_BRANCH: macosx-static-binaries
  COMMITTER_NAME: macosx build action
  COMMITTER_EMAIL: builds@ethereum.org

jobs:
  select-solc-version:
    runs-on: ubuntu-latest
    outputs:
      solidity-version: ${{ env.SOLIDITY_VERSION }}

    steps:
      - name: Install semver utility
        run: |
          npm install semver

      - name: Select version to build
        run: |
          github_tag_response=$(curl "https://api.github.com/repos/ethereum/solidity/git/refs/tags/")
          github_macosx_files_response=$(curl "https://api.github.com/repos/${GITHUB_REPOSITORY}/contents/macosx-amd64?ref=${TARGET_BRANCH}")

          available_tags=$(
            echo "$github_tag_response" |
            jq --raw-output '.[].ref' |
            sed -n 's|^refs/tags/v\([0-9.]\+\)$|\1|p'
          )
          eligible_tags=$(
            npx semver --range ">= ${MIN_SUPPORTED_SOLIDITY_VERSION}" $available_tags |
            sort
          )
          available_binaries=$(
            echo "$github_macosx_files_response" |
            jq --raw-output '.[].name' |
            sed -n 's/^solc-macosx-amd64-v\([0-9.]*\)+commit.[0-9a-f]\+$/\1/p' |
            sort
          )
          missing_versions="$(comm -23 <(echo "$eligible_tags") <(echo "$available_binaries"))"
          selected_version=$(echo "$missing_versions" | sort --random-sort | head --lines 1)

          echo "Missing Solidity versions:"
          echo "${missing_versions}"
          echo "Selected version: ${selected_version}"

          echo "SOLIDITY_VERSION=${selected_version}" >> $GITHUB_ENV

  build:
    runs-on: macos-10.15
    needs:
      - select-solc-version
    env:
      SOLIDITY_VERSION: ${{ needs.select-solc-version.outputs.solidity-version }}
    outputs:
      full-build-version: ${{ env.FULL_BUILD_VERSION }}

    if: "needs.select-solc-version.outputs.solidity-version"
    steps:
      - name: Check out Solidity source
        uses: actions/checkout@v2
        with:
          repository: 'ethereum/solidity'
          ref: v${{ env.SOLIDITY_VERSION }}
          path: solidity/
          fetch-depth: 0

      - name: Install semver utility
        run: |
          npm install --global semver

      - name: Install dependencies
        run: |
          cd solidity/

          if semver --range '>= 0.5.13' "$SOLIDITY_VERSION"; then
            test -f .circleci/osx_install_dependencies.sh

            # On 0.5.13 and higher we can just use the script for installing dependencies for CI
            .circleci/osx_install_dependencies.sh
          else
            test ! -e .circleci/osx_install_dependencies.sh

            brew unlink python
            brew install coreutils

            # Not all boost versions work with all solc versions. Through trial and error I found
            # out that boost 1.66.0 works correctly with every solc version since 0.4.0 and possibly
            # with earlier ones too.
            # NOTE: Homebrew currently provides only boost 1.74.0 so this needs to be built from source.
            boost_install_dir="${PWD}/boost/boost-1.66.0-install"
            echo "LOCAL_BOOST_INSTALL_DIR=${boost_install_dir}" >> $GITHUB_ENV
            mkdir -p ../boost/
            (
              cd ../boost/

              curl --location --remote-name https://dl.bintray.com/boostorg/release/1.66.0/source/boost_1_66_0.tar.bz2
              tar xf boost_1_66_0.tar.bz2
              cd boost_1_66_0/

              # Boost won't build on clang 11.4+ because of a bug in compiler version detection.
              # that results in unsupported flags being added. See https://github.com/boostorg/build/pull/560
              sed -i.bak 's|^[[:blank:]]*\(flags darwin\.compile\.c++ OPTIONS \$(condition) :\) -fcoalesce-templates ;[[:blank:]]*$|\1 ;|g' tools/build/src/tools/darwin.jam
              sed -i.bak 's|^[[:blank:]]*\(flags darwin\.compile OPTIONS \$(condition) :\) -Wno-long-double ;[[:blank:]]*$|\1 ;|g' tools/build/src/tools/darwin.jam

              # Options and workarounds based on boost.rb formula for Boost 1.65.0 from Homebrew
              ./bootstrap.sh \
                --prefix="${boost_install_dir}" \
                --libdir="${boost_install_dir}/lib" \
                --without-icu \
                --without-libraries=python,mpi \
                || { cat bootstrap.log; exit 1; }

              echo "using darwin : : clang++ ;" > user-config.jam
              ./b2 headers
              ./b2 install \
                --user-config=user-config.jam \
                --prefix="${boost_install_dir}" \
                --libdir="${boost_install_dir}/lib" \
                -d2 \
                -j3 \
                --layout=tagged \
                -sNO_LZMA=1 \
                link=shared,static \
                threading=multi \
                cxxflags=-std=c++11 \
                cxxflags=-stdlib=libc++ \
                linkflags=-stdlib=libc++
            )
            export DYLD_LIBRARY_PATH="${boost_install_dir}/lib:$DYLD_LIBRARY_PATH"

            # NOTE: As long as the build does not fail, the compiler should work with any Z3 version.
            # The prover might find slightly different answers but they should not be wrong.
            # SMT tests are tied to a specific Z3 version and might fail due to that but here
            # we're only running them when building versions later than 0.5.13 so it's not an issue.
            z3_version="4.8.5"
            z3_release="z3-${z3_version}-x64-osx-10.14.2"
            curl -O -L "https://github.com/Z3Prover/z3/releases/download/Z3-${z3_version}/${z3_release}.zip"
            unzip "${z3_release}.zip"
            rm "${z3_release}.zip"
            mv "${z3_release}/bin/libz3.a" /usr/local/lib
            mv "${z3_release}/bin/z3"      /usr/local/bin
            mv "${z3_release}/include/"*   /usr/local/include
            rm -r "${z3_release}"

            # Since 0.5.11 evmone can be used as an altenative to IPC tests
            if semver --range '>= 0.5.11' "$SOLIDITY_VERSION"; then
              test -f test/EVMHost.h

              evmone_version="0.1.0"
              evmone_package="evmone-0.1.0-darwin-x86_64"
              curl -O -L "https://github.com/ethereum/evmone/releases/download/v${evmone_version}/${evmone_package}.tar.gz"
              tar xzpf "${evmone_package}.tar.gz" -C /usr/local
              rm "${evmone_package}.tar.gz"
            else
              test ! -e test/EVMHost.h
            fi
          fi

      - name: Determine full build version
        run: |
          cd solidity/

          # Can't run get_version.sh because MacOS grep does not have the -P flag.
          test "$SOLIDITY_VERSION" = "$(sed -n 's/^set(PROJECT_VERSION "\([0-9.]*\)")$/\1/p' CMakeLists.txt)"

          last_commit_hash=$(git rev-parse --short=8 HEAD)
          full_build_version="v${SOLIDITY_VERSION}+commit.${last_commit_hash}"

          echo "LAST_COMMIT_HASH=${last_commit_hash}" >> $GITHUB_ENV
          echo "FULL_BUILD_VERSION=${full_build_version}" >> $GITHUB_ENV

      - name: Build
        run: |
          cd solidity/

          # In 0.3.6 boostTest.cpp has its own main() function which leads to "duplicate symbol '_main'" linker error
          # despite BOOST_TEST_NO_MAIN being defined. 0.4.0 does not have this problem so here we just backport that change.
          if semver --range '= 0.3.6' "$SOLIDITY_VERSION"; then
            # NOTE: Only the first commit should be necessary but one of its changes seems to
            # have been mistakenly included in the other and it won't compile on its own.
            git cherry-pick 1bc0320811ef2b213bda0629b702bffae5e2f925 # [PR #837] Cleanup of test suite init.
            git cherry-pick 53f68a155f071194fd779352d5997c03a6c387ed # [PR #837] Exponential sleep increase on mining failure.

            # The cherry picked commits only affect tests. Force the compiler to still identify itself
            # using the release commit so that it produces the same bytecode as the original release.
            echo "$LAST_COMMIT_HASH" > commit_hash.txt
          fi

          # Static linking with jsoncpp was introduced in 0.4.5
          if semver --range '>= 0.4.2 <= 0.4.4' "$SOLIDITY_VERSION"; then
            # The change can be backported from 0.4.5 and applies cleanly between 0.4.2 and 0.4.4.
            git cherry-pick 4bde0a2d36297c4b3fa17c7dac2bb1681e1e7f75 # [#1252] Build jsoncpp from source using jsoncpp.cmake script

            # The cherry picked commits only affect the build system. Force the compiler to still identify itself
            # using the release commit so that it produces the same bytecode as the original release.
            echo "$LAST_COMMIT_HASH" > commit_hash.txt
          elif semver --range '>= 0.3.6 <= 0.4.1' "$SOLIDITY_VERSION"; then
            # The commit won't apply cleanly before 0.4.2 due to conflicting changes in install_deps.sh and .travis.yml.
            # Those files don't affect our build so just reset the unmerged files.
            # NOTE: core.editor setting needs to be overridden for prevent git from asking us to edit the commit message.
            git cherry-pick 4bde0a2d36297c4b3fa17c7dac2bb1681e1e7f75 || true # [PR #1252] Build jsoncpp from source using jsoncpp.cmake script
            git reset -- scripts/install_deps.sh .travis.yml
            git checkout scripts/install_deps.sh .travis.yml
            git -c core.editor=/usr/bin/true cherry-pick --continue

            # The cherry picked commits only affect the build system. Force the compiler to still identify itself
            # using the release commit so that it produces the same bytecode as the original release.
            echo "$LAST_COMMIT_HASH" > commit_hash.txt
          fi

          # Between 0.3.6 and 0.4.16 deps/ was a submodule and contained parts of the cmake configuration
          if semver --range '>= 0.3.6 <= 0.4.16' "$SOLIDITY_VERSION"; then
            git submodule init
            git submodule update
          fi

          # Remove the -Werror flag. Unfortunately versions older than 0.6.1 do not compile without warnings
          # when using a recent clang version on MacOS 10.15.
          if semver --range '>= 0.3.6 <= 0.6.0' "$SOLIDITY_VERSION"; then
            sed -i.bak '/^[[:blank:]]*add_compile_options(-Werror)[[:blank:]]*$/d' cmake/EthCompilerSettings.cmake
          fi

          # Pre-0.5.0 versions were using wrong boost test header, resulting in the 'duplicate symbol' linker error in static builds.
          # See https://github.com/ethereum/solidity/pull/4572
          if semver --range '< 0.5.0' "$SOLIDITY_VERSION"; then
            sed -i.bak 's|^[[:blank:]]*#include <boost/test/included/unit_test\.hpp>[[:blank:]]*$|#include <boost/test/unit_test.hpp>|g' test/boostTest.cpp
          fi

          # Uncommitted changes result in .mod (or -mod in 0.4.0) being included in version string which affects
          # bytecode produced by the compiler. Committing them would change the commit hash so that's not
          # an option. We have to disable this if we want the output to match a release build.
          if semver --range '>= 0.4.0 <= 0.6.0' "$SOLIDITY_VERSION"; then
            sed -i.bak '/^[[:blank:]]*set(SOL_COMMIT_HASH \"\${SOL_COMMIT_HASH}[-.]mod\")[[:blank:]]*$/d' cmake/scripts/buildinfo.cmake
          fi

          # Starting with 0.4.16 there's no STATIC_LINKING option but there's SOLC_LINK_STATIC instead.
          if semver --range '>= 0.4.16' "$SOLIDITY_VERSION"; then
            static_linking_option="-DSOLC_LINK_STATIC=1"
          else
            static_linking_option="-DSTATIC_LINKING=1"
          fi

          mkdir -p build/
          echo -n > prerelease.txt
          cd build/

          if [[ $LOCAL_BOOST_INSTALL_DIR != "" ]]; then
            boost_options=(-DBOOST_ROOT:PATHNAME="$LOCAL_BOOST_INSTALL_DIR" -DBoost_NO_BOOST_CMAKE=TRUE -DBoost_NO_SYSTEM_PATHS=TRUE)
          else
            boost_options=()
          fi

          cmake .. "${static_linking_option}" -DCMAKE_BUILD_TYPE=Release -G "Unix Makefiles" "${boost_options[@]}"
          make -j 3 solc soltest

          # solfuzzer was added to the command-line tests in 0.4.10
          if semver --range '>= 0.4.10' "$SOLIDITY_VERSION"; then
            make -j 3 solfuzzer
          fi

      - name: Verify version reported by the binary
        run: |
          cd solidity/build/

          echo $(solc/solc --version)
          ! [[ $(solc/solc --version) =~ [-.]mod ]] || { echo "ERROR: Not a release build!"; exit 1; }

      - name: Upload solc as an artifact
        uses: actions/upload-artifact@v2
        with:
          # NOTE: The name is used for the .zip archive but the file inside is still called solc
          name: solc-macosx-amd64-${{ env.FULL_BUILD_VERSION }}
          path: solidity/build/solc/solc

      - name: List dynamic libraries used by the binary
        run: |
          cd solidity/build/solc/
          dynamic_libraries=$(otool -L solc)
          echo "${dynamic_libraries}"

          # The MacOS binaries are not completely static but there should be nothing dynamic
          # beyond the stuff we expect to be there.
          extra_dynamic_libraries=$(
            echo "${dynamic_libraries}" |
            sed '/^solc:$/d' |
            sed '/^[[:blank:]]*\/usr\/lib\/libSystem.B.dylib (.*)[[:blank:]]*$/d' |
            sed '/^[[:blank:]]*\/usr\/lib\/libc++.1.dylib (.*)[[:blank:]]*$/d'
          )
          test -z "$extra_dynamic_libraries"

      - name: Run tests
        run: |
          cd solidity/

          if semver --range '>= 0.4.24 <= 0.5.9' "$SOLIDITY_VERSION"; then
            # Between 0.4.24 and 0.5.9 tests.sh disables IPC tests on MacOS when it's running on CircleCI
            export CIRCLECI=1
          elif semver --range '= 0.5.10' "$SOLIDITY_VERSION"; then
            # In 0.5.10 IPC tests would no longer get disabled but were not actually removed until 0.5.11.
            # We need to patch the script to disable them or the it will will try to run a linux
            # version of aleth and hang.
            sed -i.bak 's|^IPC_ENABLED=true$|IPC_ENABLED=false|g' scripts/tests.sh
            sed -i.bak 's|\$IPC_FLAGS|--no-ipc|g' scripts/tests.sh
          fi

          if semver --range '= 0.5.0' "$SOLIDITY_VERSION"; then
            # Command-line tests were broken on MacOS in 0.5.0 (fixed in 0.5.1: #5419).
            # Part of the message about missing pragma was not being stripped correctly.
            sed -i.bak 's|s/ \\?Consider adding|s/ Consider adding|g' test/cmdlineTests.sh
          fi

          # Set $TERM to silence the errors from tput (tput: No value for $TERM and no -T specified)
          export TERM=dumb

          if semver --range '>= 0.4.24' "$SOLIDITY_VERSION"; then
            scripts/tests.sh
          elif semver --range '>= 0.4.11 < 0.4.24' "$SOLIDITY_VERSION"; then
            # In these versions tests.sh is hard-coded to run IPC tests but we can manually pass --no-ipc flag to soltest
            test/cmdlineTests.sh

            for optimize in "" "--optimize"; do
              if semver --range '< 0.4.21' "$SOLIDITY_VERSION"; then
                # --evm-version option has been introduced in 0.4.21
                build/test/soltest -- --testpath test/ "$optimize" --no-ipc
              else
                build/test/soltest -- --testpath test/ "$optimize" --evm-version homestead --no-ipc
                build/test/soltest -- --testpath test/ "$optimize" --evm-version byzantium --no-ipc
              fi

              if semver --range '>= 0.4.22 < 0.4.24' "$SOLIDITY_VERSION"; then
                # 0.4.22 started running tests against constantinople
                build/test/soltest -- --testpath test/ "$optimize" --evm-version constantinople --no-ipc
              fi
            done
          elif semver --range '>= 0.4.9 < 0.4.11' "$SOLIDITY_VERSION"; then
            # Before that IPC tests are hard-coded so we can only run commandline tests
            test/cmdlineTests.sh
          else
            # On older versions all we can do is a simple smoke test
            echo "contract C {}" > test.sol
            build/solc/solc test.sol
          fi

  push-to-repo:
    runs-on: ubuntu-latest
    needs:
      - select-solc-version
      - build
    env:
      SOLIDITY_VERSION: ${{ needs.select-solc-version.outputs.solidity-version }}
      FULL_BUILD_VERSION: ${{ needs.build.outputs.full-build-version }}

    if: "needs.select-solc-version.outputs.solidity-version"
    steps:
      - name: Remove unused software to free up space
        run: |
          # These are quick to delete and large enough to make a difference
          rm -rf /usr/share/swift/                         # 1.3 GB in 80 subdirs
          rm -rf /usr/local/lib/android/sdk/build-tools/   # 2.1 GB in 450 subdirs
          rm -rf /usr/share/dotnet/shared/                 # 5.3 GB in 350 subdirs
          rm -rf /usr/local/lib/android/sdk/ndk/           # 7.6 GB in 1500 subdirs

      - uses: actions/checkout@v2
        with:
          ref: ${{ env.TARGET_BRANCH }}
          path: 'solc-bin'

      - name: Fetch latest commits
        run: |
          cd solc-bin/
          # Repo is so large that often another run manages to push a commit while we're cloning.
          git pull origin "$TARGET_BRANCH"

      - name: Fail if the file is already in the repo
        run: |
          test ! -e "solc-bin/macosx-amd64/solc-macosx-amd64-${{ env.FULL_BUILD_VERSION }}"

      - name: Download MacOS artifact
        uses: actions/download-artifact@v2
        with:
          name: solc-macosx-amd64-${{ env.FULL_BUILD_VERSION }}

      - name: Set committer name and e-mail
        run: |
          cd solc-bin/
          git config --local user.name "$COMMITTER_NAME"
          git config --local user.email "$COMMITTER_EMAIL"

      - name: Commit and push the binary
        run: |
          cd solc-bin/
          chmod 0755 ../solc
          mv ../solc "macosx-amd64/solc-macosx-amd64-${{ env.FULL_BUILD_VERSION }}"
          git add --verbose "macosx-amd64/solc-macosx-amd64-${{ env.FULL_BUILD_VERSION }}"
          git commit --message "Solidity MacOS build ${{ env.FULL_BUILD_VERSION }}"
          git push origin HEAD