diff --git a/.cirrus.yml b/.cirrus.yml deleted file mode 100644 index e28649926e7..00000000000 --- a/.cirrus.yml +++ /dev/null @@ -1,21 +0,0 @@ -freebsd_instance: - image_family: freebsd-12-1 - -task: - install_script: | - IGNORE_OSVERSION=yes pkg update - pkg install -U -y git boost_build boost-libs unzip wget openssl cmake ninja - echo "using clang ;" > ~/user-config.jam - submodules_script: | - git submodule update --init --recursive - build_cmake_script: | - cmake -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_FLAGS="-Werror" -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=ON -Dbuild_examples=ON -Dbuild_tools=ON -Dpython-bindings=OFF -G Ninja . - cmake --build . --parallel 2 - ./test/test_primitives - tests_script: | - cd test - b2 -l250 warnings-as-errors=on warnings=all crypto=openssl deterministic-tests include=/usr/local/include library-path=/usr/local/lib - enum_if_script: | - cd test - b2 -l250 warnings-as-errors=on warnings=all crypto=openssl stage_enum_if stage_dependencies include=/usr/local/include library-path=/usr/local/lib - LD_LIBRARY_PATH=./dependencies ./enum_if diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml deleted file mode 100644 index fab43b28d98..00000000000 --- a/.github/workflows/android.yml +++ /dev/null @@ -1,117 +0,0 @@ -name: Android - -on: - push: - branches: [ RC_2_0 master ] - pull_request: - -env: - NDK_VERSION: "r21d" - OPENSSL_VERSION: "1.1.1i" - OPENSSL_OPTS: "no-deprecated no-shared no-makedepend -fvisibility=hidden -O3" - -jobs: - - android_arm32_build: - name: Build Android Arm 32bits - runs-on: ubuntu-20.04 - - steps: - - name: checkout - uses: actions/checkout@v2.3.3 - with: - submodules: true - - - name: install boost - run: | - git clone --depth=1 --branch=boost-1.75.0 https://github.com/boostorg/boost.git - cd boost - git submodule update --init --depth=1 - ./bootstrap.sh - ./b2 headers - cd .. - - - name: install ndk - run: | - wget -nv -O android-ndk.zip https://dl.google.com/android/repository/android-ndk-${NDK_VERSION}-linux-x86_64.zip - unzip -qq android-ndk.zip - export NDK=${PWD}/android-ndk-${NDK_VERSION} - ${NDK}/build/tools/make_standalone_toolchain.py --arch arm --api 19 --stl libc++ --install-dir android-toolchain - - - name: install openssl - run: | - export ANDROID_TOOLCHAIN=${PWD}/android-toolchain - export PATH=${ANDROID_TOOLCHAIN}/arm-linux-androideabi/bin:${PATH} - export CC=${ANDROID_TOOLCHAIN}/bin/arm-linux-androideabi-clang - wget -nv -O openssl.tar.gz https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz - tar xzf openssl.tar.gz - cd openssl-${OPENSSL_VERSION} - ./Configure linux-armv4 ${OPENSSL_OPTS} -march=armv7-a -mfpu=neon -fPIC --prefix=${PWD}/../openssl - make &> /dev/null - make install_sw &> /dev/null - cd .. - - - name: build library - run: | - export ANDROID_TOOLCHAIN=${PWD}/android-toolchain - export PATH=${ANDROID_TOOLCHAIN}/arm-linux-androideabi/bin:${PATH} - export BOOST_ROOT=${PWD}/boost - export OPENSSL_ROOT=${PWD}/openssl - echo "boost-build ${BOOST_ROOT}/tools/build/src ;" > boost-build.jam - echo "using clang-linux : arm : ${ANDROID_TOOLCHAIN}/bin/arm-linux-androideabi-clang++ - -fPIC - -march=armv7-a - -mfpu=neon ;" >>~/user-config.jam; - ${BOOST_ROOT}/b2 warnings-as-errors=on cxxstd=17 toolset=clang-linux-arm target-os=android link=static crypto=openssl openssl-include=${OPENSSL_ROOT}/include openssl-lib=${OPENSSL_ROOT}/lib - - android_arm32_webtorrent_build: - name: Build Android Arm 32bits with WebTorrent - runs-on: ubuntu-20.04 - - steps: - - name: checkout - uses: actions/checkout@v2.3.3 - with: - submodules: recursive - - - name: install boost - run: | - git clone --depth=1 --branch=boost-1.75.0 https://github.com/boostorg/boost.git - cd boost - git submodule update --init --depth=1 - ./bootstrap.sh - ./b2 headers - cd .. - - - name: install ndk - run: | - wget -nv -O android-ndk.zip https://dl.google.com/android/repository/android-ndk-${NDK_VERSION}-linux-x86_64.zip - unzip -qq android-ndk.zip - export NDK=${PWD}/android-ndk-${NDK_VERSION} - ${NDK}/build/tools/make_standalone_toolchain.py --arch arm --api 28 --stl libc++ --install-dir android-toolchain - - - name: install openssl - run: | - export ANDROID_TOOLCHAIN=${PWD}/android-toolchain - export PATH=${ANDROID_TOOLCHAIN}/arm-linux-androideabi/bin:${PATH} - export CC=${ANDROID_TOOLCHAIN}/bin/arm-linux-androideabi-clang - wget -nv -O openssl.tar.gz https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz - tar xzf openssl.tar.gz - cd openssl-${OPENSSL_VERSION} - ./Configure linux-armv4 ${OPENSSL_OPTS} -march=armv7-a -mfpu=neon -fPIC --prefix=${PWD}/../openssl - make &> /dev/null - make install_sw &> /dev/null - cd .. - - - name: build library - run: | - export ANDROID_TOOLCHAIN=${PWD}/android-toolchain - export PATH=${ANDROID_TOOLCHAIN}/arm-linux-androideabi/bin:${PATH} - export BOOST_ROOT=${PWD}/boost - export OPENSSL_ROOT=${PWD}/openssl - echo "boost-build ${BOOST_ROOT}/tools/build/src ;" > boost-build.jam - echo "using clang-linux : arm : ${ANDROID_TOOLCHAIN}/bin/arm-linux-androideabi-clang++ - -fPIC - -march=armv7-a - -mfpu=neon ;" >>~/user-config.jam; - ${BOOST_ROOT}/b2 warnings-as-errors=on cxxstd=17 toolset=clang-linux-arm target-os=android link=static crypto=openssl openssl-include=${OPENSSL_ROOT}/include openssl-lib=${OPENSSL_ROOT}/lib webtorrent=on diff --git a/.github/workflows/c.yml b/.github/workflows/c.yml deleted file mode 100644 index 4bcb49cc910..00000000000 --- a/.github/workflows/c.yml +++ /dev/null @@ -1,68 +0,0 @@ -name: C bindings - -on: - push: - branches: [ RC_1_2 RC_2_0 master ] - pull_request: - -jobs: - test: - name: build - runs-on: ${{ matrix.os }} - - strategy: - matrix: - os: [ubuntu-20.04, macos-latest, windows-2019 ] - - steps: - - uses: actions/checkout@v2 - with: - submodules: true - - - name: dependencies (MacOS) - if: runner.os == 'macOS' - run: | - brew install boost-build boost - - - name: update package lists (linux) - if: runner.os == 'Linux' - continue-on-error: true - run: | - sudo apt update - - - name: dependencies (linux) - if: runner.os == 'Linux' - run: | - sudo apt install libboost-tools-dev libboost-dev libboost-system-dev - - - name: install boost (windows) - if: runner.os == 'Windows' - shell: cmd - run: | - git clone --depth=1 --recurse-submodules -j10 --branch=boost-1.72.0 https://github.com/boostorg/boost.git - cd boost - bootstrap.bat - - - name: boost headers (windows) - if: runner.os == 'Windows' - shell: cmd - run: | - cd boost - .\b2 headers - - - name: build (windows) - if: runner.os == 'Windows' - shell: cmd - run: | - set BOOST_ROOT=%CD%\boost - set BOOST_BUILD_PATH=%BOOST_ROOT%\tools\build - set PATH=%BOOST_ROOT%;%PATH% - - cd bindings\c - b2 - - - name: build - if: runner.os != 'Windows' - run: | - cd bindings/c - b2 diff --git a/.github/workflows/cibuildwheel.yml b/.github/workflows/cibuildwheel.yml new file mode 100644 index 00000000000..e021bba6022 --- /dev/null +++ b/.github/workflows/cibuildwheel.yml @@ -0,0 +1,45 @@ +name: cibuildwheel + +on: + push: + branches: [ RC_1_2 RC_2_0 master ] + pull_request: + +jobs: + build_wheels: + name: cibuildwheel + runs-on: ${{ matrix.os }} + continue-on-error: true + + strategy: + matrix: + os: [ubuntu-20.04, macos-10.15, windows-2019] + pytag: [cp36, cp37, cp38, cp39] + + env: + CIBW_BUILD_VERBOSITY: 1 + CIBW_BUILD: ${{ matrix.pytag }}-* + + steps: + - uses: actions/checkout@v2 + with: + submodules: true + + # "outer" python, used only to invoke cihelper and cibuildwheel + - uses: actions/setup-python@v2 + + - name: install cihelper + run: | + python -m pip install ./cihelper + + - name: install cibuildwheel + run: | + python -m pip install cibuildwheel + + - name: run cibuildwheel + run: | + cihelper_cibw_run --output-dir wheelhouse + + - uses: actions/upload-artifact@v2 + with: + path: ./wheelhouse/*.whl diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml deleted file mode 100644 index 5f7d0e5dbae..00000000000 --- a/.github/workflows/docs.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: Documentation - -on: - push: - branches: [ RC_1_2 RC_2_0 master ] - pull_request: - -jobs: - - docs: - name: Docs - runs-on: ubuntu-20.04 - - steps: - - name: checkout - uses: actions/checkout@v2.3.3 - - - name: update package lists - continue-on-error: true - run: | - sudo apt update - - - name: install dependencies - run: | - sudo apt install python-docutils python-pygments python-pil gsfonts inkscape icoutils graphviz hunspell imagemagick - python -m pip install aafigure - ~/.local/bin/aafigure --version - - - name: spell-check - run: | - cd docs - make AAFIGURE=~/.local/bin/aafigure RST2HTML=rst2html spell-check html - - - name: build docs - run: | - cd docs - make AAFIGURE=~/.local/bin/aafigure RST2HTML=rst2html diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml deleted file mode 100644 index ad7f1242d01..00000000000 --- a/.github/workflows/linux.yml +++ /dev/null @@ -1,278 +0,0 @@ -name: Linux - -on: - push: - branches: [ RC_1_2 RC_2_0 master ] - pull_request: - -jobs: - - pre-commit: - # TODO: matrix across python version and os - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - submodules: recursive - - uses: actions/setup-python@v2 - with: - python-version: 3.6 - - uses: pre-commit/action@v2.0.0 - - build: - name: build - runs-on: ubuntu-20.04 - continue-on-error: true - - strategy: - matrix: - config: [ asio-debugging=on picker-debugging=on, extensions=off logging=off streaming=off super-seeding=off share-mode=off predictive-pieces=off dht=off alert-msg=off encryption=off mutable-torrents=off deprecated-functions=off, crypto=gcrypt, mmap-disk-io=off ] - - steps: - - name: checkout - uses: actions/checkout@v2.3.3 - with: - submodules: true - - - name: update package lists - continue-on-error: true - run: | - sudo apt update - - - name: install boost - run: | - sudo apt install libboost-tools-dev libboost-dev libboost-system-dev - echo "using gcc ;" >>~/user-config.jam - - - name: install gcrypt - if: ${{ contains(matrix.config, 'crypto=gcrypt') }} - run: sudo apt install libgcrypt20-dev - - - name: build library - run: | - b2 ${{ matrix.config }} cxxstd=17 - - - name: build examples - run: | - cd examples - b2 ${{ matrix.config }} - - - name: build tools - run: | - cd tools - b2 ${{ matrix.config }} warnings-as-errors=on - - - - fuzzers: - name: Fuzzers - runs-on: ubuntu-20.04 - continue-on-error: true - - steps: - - name: checkout - uses: actions/checkout@v2.3.3 - with: - submodules: true - - - name: update package lists - continue-on-error: true - run: | - sudo apt update - - - name: install boost - run: | - sudo apt install libboost-tools-dev libboost-dev libboost-system-dev - echo "using clang : 9 : clang++-9 ;" >>~/user-config.jam - - - name: build fuzzers - run: | - cd fuzzers - b2 clang cxxstd=17 warnings-as-errors=on - - - - check_headers: - name: check headers - runs-on: ubuntu-20.04 - - steps: - - name: checkout - uses: actions/checkout@v2.3.3 - with: - submodules: true - - - name: update package lists - continue-on-error: true - run: | - sudo apt update - - - name: install boost - run: | - sudo apt install libboost-tools-dev libboost-dev libboost-system-dev - echo "using gcc ;" >>~/user-config.jam - - - name: compile header files individually - run: | - b2 check-headers cxxstd=17 warnings-as-errors=on - - - - clang_tidy: - name: clang-tidy - runs-on: ubuntu-20.04 - - steps: - - name: checkout - uses: actions/checkout@v2.3.3 - with: - submodules: true - - - name: update package lists - continue-on-error: true - run: | - sudo apt update - - - name: install clang-tidy - run: sudo apt install clang-tidy libc++-dev - - - name: install boost - run: | - sudo apt install libboost-tools-dev libboost-dev - echo "using clang_tidy : : clang-tidy \"-checks=-clang-analyzer-core.*,-clang-analyzer-unix.*\" : -I/usr/local/clang-7.0.0/include/c++/v1 -stdlib=libc++ -stdlib=libc++ ;" >> ~/user-config.jam; - - - name: analyze - run: | - b2 -a clang_tidy - - - - test: - name: Tests - runs-on: ubuntu-20.04 - continue-on-error: true - - strategy: - matrix: - include: - - config: webtorrent=on address-sanitizer=norecover undefined-sanitizer=norecover crypto=openssl - - config: toolset=clang address-sanitizer=norecover undefined-sanitizer=norecover - - config: thread-sanitizer=norecover crypto=openssl release debug-symbols=on - - config: crypto=gnutls - - config: deprecated-functions=off - - steps: - - name: checkout - uses: actions/checkout@v2.3.3 - with: - submodules: recursive - - - name: update package lists - continue-on-error: true - run: | - sudo apt update - - - name: install gnutls - if: ${{ contains(matrix.config, 'crypto=gnutls') }} - run: | - sudo apt install libgnutls28-dev - - - name: install boost - run: | - sudo apt install libboost-tools-dev libboost-dev libboost-system-dev - pip install websockets - echo "using gcc ;" >>~/user-config.jam - echo "using clang : 9 : clang++-9 ;" >>~/user-config.jam - - - name: build and run tests - run: | - cd test - b2 ${{ matrix.config }} -l400 warnings-as-errors=on debug-iterators=on invariant-checks=full asserts=on deterministic-tests - - - name: run tests (flaky) - uses: nick-invision/retry@v2 - with: - timeout_minutes: 30 - retry_wait_seconds: 4 - max_attempts: 3 - command: (cd test; b2 ${{ matrix.config }} -l400 warnings-as-errors=on debug-iterators=on invariant-checks=full asserts=on) - - - - sim: - name: Simulations - runs-on: ubuntu-20.04 - - steps: - - name: checkout - uses: actions/checkout@v2.3.3 - with: - submodules: true - - - name: update package lists - continue-on-error: true - run: | - sudo apt update - - - name: install boost - run: | - sudo apt install libboost-tools-dev libboost-dev libboost-system-dev - echo "using gcc ;" >>~/user-config.jam - - - name: build and run simulations - run: | - cd simulation - b2 debug-iterators=on invariant-checks=full asserts=on picker-debugging=on - - - dist: - name: build dist - runs-on: ${{ matrix.os }} - - strategy: - matrix: - os: [ubuntu-20.04 ] - - steps: - - name: checkout - uses: actions/checkout@v2.3.3 - with: - submodules: true - - - name: update package lists - continue-on-error: true - run: | - sudo apt update - - - name: install dependencies - run: | - sudo apt install libboost-tools-dev libboost-python-dev libboost-dev libboost-system-dev - sudo apt install python-docutils python-pygments python-pil gsfonts inkscape icoutils graphviz hunspell imagemagick python3-setuptools - python3 -m pip install aafigure - echo "using gcc ;" >>~/user-config.jam - - - name: build tarball - run: AAFIGURE=~/.local/bin/aafigure RST2HTML=rst2html make dist - - - uses: actions/upload-artifact@v2 - with: - name: tarball - path: libtorrent-rasterbar-*.tar.gz - - - name: test-tarball (b2 install) - run: | - tar xvzf libtorrent-rasterbar-*.tar.gz - cd libtorrent-rasterbar-*/ - b2 install cxxstd=14 --prefix=test-install-root - cat test-install-root/share/pkgconfig/libtorrent-rasterbar.pc - - - name: test-tarball (b2 tests) - run: | - cd libtorrent-rasterbar-*/test - b2 testing.execute=off - b2 test_torrent_info - - - name: test-tarball (python bindings) - run: | - cd libtorrent-rasterbar-*/ - python3 bindings/python/setup.py build diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml deleted file mode 100644 index a328ee5d045..00000000000 --- a/.github/workflows/macos.yml +++ /dev/null @@ -1,124 +0,0 @@ -name: MacOS - -on: - push: - branches: [ RC_1_2 RC_2_0 master ] - pull_request: - -jobs: - - test: - name: Tests - runs-on: macos-latest - continue-on-error: true - - strategy: - matrix: - config: [ crypto=openssl webtorrent=on, deprecated-functions=off ] - - steps: - - name: checkout - uses: actions/checkout@v2.3.3 - with: - submodules: recursive - - - name: install boost - run: | - brew install boost-build boost - HOMEBREW_NO_INSTALL_CLEANUP=1 brew reinstall openssl@1.1 - pip3 install websockets - echo "using darwin ;" >>~/user-config.jam - - - name: build and run tests - run: (cd test; b2 ${{ matrix.config }} -l400 warnings-as-errors=on debug-iterators=on invariant-checks=full asserts=on deterministic-tests) - - - name: run tests (flaky) - uses: nick-invision/retry@v2 - with: - timeout_minutes: 30 - retry_wait_seconds: 1 - max_attempts: 3 - command: (cd test; b2 ${{ matrix.config }} -l400 warnings-as-errors=on debug-iterators=on invariant-checks=full asserts=on) - - - sim: - name: Simulations - runs-on: macos-latest - - steps: - - name: checkout - uses: actions/checkout@v2.3.3 - with: - submodules: true - - - name: install boost - run: | - brew install boost-build boost - echo "using darwin ;" >>~/user-config.jam - - - name: build and run simulations - run: | - cd simulation - b2 -l400 debug-iterators=on invariant-checks=full asserts=on - - - build: - name: Build - runs-on: macos-latest - continue-on-error: true - - strategy: - matrix: - config: [ crypto=openssl, release ] - - steps: - - name: checkout - uses: actions/checkout@v2.3.3 - with: - submodules: true - - - name: install boost - run: | - brew install openssl boost-build boost - echo "using darwin ;" >>~/user-config.jam - - - name: build library - run: | - b2 ${{ matrix.config }} -l400 warnings-as-errors=on cxxstd=14 - - - ios_build: - name: Build iOS - runs-on: macos-latest - continue-on-error: true - - steps: - - name: checkout - uses: actions/checkout@v2.3.3 - with: - submodules: true - - - name: install boost - run: | - brew install boost-build boost - echo "using darwin : ios_sim : clang++ : -Wno-deprecated-declarations - \"-isysroot /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneSimulator.platform/Developer/SDKs/iPhoneSimulator.sdk\" - -mios-simulator-version-min=7 - -fobjc-abi-version=2 - \"-isysroot /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneSimulator.platform/Developer/SDKs/iPhoneSimulator.sdk\" - -mios-simulator-version-min=7 - -fobjc-abi-version=2 ;" >>~/user-config.jam; - - echo "using darwin : ios : clang++ : -Wno-deprecated-declarations - \"-isysroot /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS.sdk\" - -mios-version-min=7 - \"-arch armv7\" - -fobjc-abi-version=2 - \"-isysroot /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS.sdk\" - -mios-version-min=7 - \"-arch armv7\" - -fobjc-abi-version=2 ;" >>~/user-config.jam; - - - name: build library - run: | - b2 -l400 warnings-as-errors=on cxxstd=14 darwin-ios darwin-ios_sim link=static diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml deleted file mode 100644 index bf5d2a265f9..00000000000 --- a/.github/workflows/python.yml +++ /dev/null @@ -1,90 +0,0 @@ -name: Python bindings - -on: - push: - branches: [ RC_1_2 RC_2_0 master ] - pull_request: - -jobs: - test: - name: build - runs-on: ${{ matrix.os }} - - strategy: - # on linux and mac, the boost libraries are fixed to a python version. - # since we install boost from repos on those systems, we must use the - # matching python versions - matrix: - include: - - os: ubuntu-20.04 - python-version: 3.8 - - os: macos-latest - python-version: 3.9 - - os: windows-2019 - python-version: 3.6 - - steps: - - uses: actions/checkout@v2 - with: - submodules: true - - # Note: - # - on mac and linux images, 'python' is python2 and 'python3' is - # python3 - # - on windows, neither 'python' nor 'python3' is in PATH by default - # - setup-python sets up PATH so 'python' and 'python3' point to the - # requested version on mac and linux, but on windows it only sets up - # 'python'. - - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: install tox - run: | - python3 -m pip install --upgrade pip - python3 -m pip install --upgrade tox - - - name: dependencies (MacOS) - if: runner.os == 'macOS' - run: | - brew install boost-build boost boost-python3 - - - name: update package lists (linux) - if: runner.os == 'Linux' - continue-on-error: true - run: | - sudo apt update - - - name: dependencies (linux) - if: runner.os == 'Linux' - run: | - sudo apt install libboost-tools-dev libboost-python-dev libboost-dev libboost-system-dev - - - name: install boost (windows) - if: runner.os == 'Windows' - shell: cmd - run: | - git clone --depth=1 --recurse-submodules -j10 --branch=boost-1.72.0 https://github.com/boostorg/boost.git - cd boost - bootstrap.bat - - - name: boost headers (windows) - if: runner.os == 'Windows' - shell: cmd - run: | - cd boost - .\b2 headers - - - name: build/test with tox (windows) - if: runner.os == 'Windows' - shell: cmd - run: | - set BOOST_ROOT=%CD%\boost - set PATH=%BOOST_ROOT%;%PATH% - set PYTHON_INTERPRETER=python - tox -e py - - - name: build/test with tox - if: runner.os != 'Windows' - run: | - tox -e py diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml deleted file mode 100644 index f55d0f46ed7..00000000000 --- a/.github/workflows/windows.yml +++ /dev/null @@ -1,171 +0,0 @@ -name: Windows - -on: - push: - branches: [ RC_1_2 RC_2_0 master ] - pull_request: - -defaults: - run: - shell: cmd - -jobs: - - tests: - name: Test - runs-on: windows-2019 - continue-on-error: true - - strategy: - matrix: - include: - - config: address-model=32 - - config: address-model=64 crypto=openssl openssl-lib=vcpkg\installed\x64-windows\lib\ openssl-include=vcpkg\installed\x64-windows\include - - config: release - - steps: - - name: checkout - uses: actions/checkout@v2.3.3 - with: - submodules: true - - # we only build with openssl on 64 bit. The github action images comes - # with openssl pre-installed, but only the 64 bit version and there's no - # naming convention to separate the two. When building the 32 bit version - # it appears it still picks up the 64 bit DLL, and fails. - - name: install openssl (64 bit) - if: ${{ contains(matrix.config, 'crypto=openssl') }} - uses: lukka/run-vcpkg@v6 - with: - vcpkgArguments: 'openssl' - vcpkgTriplet: 'x64-windows' - vcpkgDirectory: '${{ github.workspace }}/vcpkg' - vcpkgGitCommitId: 0bf3923f9fab4001c00f0f429682a0853b5749e0 - - - name: install boost - run: | - git clone --depth=1 --recurse-submodules -j10 --branch=boost-1.72.0 https://github.com/boostorg/boost.git - cd boost - bootstrap.bat - - - name: boost headers - run: | - cd boost - .\b2 headers - - - name: tests (deterministic) - run: | - set BOOST_ROOT=%CD%\boost - set PATH=%BOOST_ROOT%;%PATH% - set PYTHON_INTERPRETER=python - cd test - b2 -l400 warnings=all warnings-as-errors=on ${{ matrix.config }} deterministic-tests - - - - name: tests (flaky) - run: | - set BOOST_ROOT=%CD%\boost - set PATH=%BOOST_ROOT%;%PATH% - set PYTHON_INTERPRETER=python - cd test - set c=3 - :retry - if %c%==0 exit /B 1 - set /a c = %c% -1 - b2 -l400 warnings=all warnings-as-errors=on ${{ matrix.config }} - if %errorlevel%==0 exit /B 0 - if %c% gtr 0 goto retry - exit /B 1 - - simulations: - name: Simulations - runs-on: windows-2019 - - steps: - - name: checkout - uses: actions/checkout@v2.3.3 - with: - submodules: true - - - name: install boost - run: | - git clone --depth=1 --recurse-submodules -j10 --branch=boost-1.72.0 https://github.com/boostorg/boost.git - cd boost - bootstrap.bat - - - name: boost headers - run: | - cd boost - .\b2 headers - - # debug iterators are turned off here because msvc has issues with noexcept - # specifiers when debug iterators are enabled. Specifically, constructors that - # allocate memory are still marked as noexcept. That results in program - # termination - # the IOCP backend in asio appears to have an issue where it hangs under - # certain unexpected terminations (through exceptions) - - name: build sims - run: | - set BOOST_ROOT=%CD%\boost - set PATH=%BOOST_ROOT%;%PATH% - cd simulation - b2 --hash address-model=64 link=static debug-iterators=off invariant-checks=on crypto=built-in define=BOOST_ASIO_DISABLE_IOCP asserts=on testing.execute=off - - - name: run sims - run: | - set BOOST_ROOT=%CD%\boost - set PATH=%BOOST_ROOT%;%PATH% - cd simulation - b2 --hash -l500 address-model=64 link=static debug-iterators=off invariant-checks=on crypto=built-in define=BOOST_ASIO_DISABLE_IOCP asserts=on - - build: - name: Build - runs-on: windows-2019 - continue-on-error: true - - strategy: - matrix: - include: - - config: asio-debugging=on picker-debugging=on windows-version=vista - - config: windows-api=store windows-version=win10 - - config: deprecated-functions=off - - steps: - - name: checkout - uses: actions/checkout@v2.3.3 - with: - submodules: true - - - name: install boost - run: | - git clone --depth=1 --recurse-submodules -j10 --branch=boost-1.72.0 https://github.com/boostorg/boost.git - cd boost - bootstrap.bat - - - name: boost headers - run: | - cd boost - .\b2 headers - - - name: build library - run: | - set BOOST_ROOT=%CD%\boost - set PATH=%BOOST_ROOT%;%PATH% - b2 ${{ matrix.config }} cxxstd=17 warnings=all warnings-as-errors=on - - - name: build examples - if: ${{ ! contains(matrix.config, 'windows-api=store') }} - run: | - set BOOST_ROOT=%CD%\boost - set PATH=%BOOST_ROOT%;%PATH% - cd examples - b2 ${{ matrix.config }} warnings=all warnings-as-errors=on - - - name: build tools - if: ${{ ! contains(matrix.config, 'windows-api=store') }} - run: | - set BOOST_ROOT=%CD%\boost - set PATH=%BOOST_ROOT%;%PATH% - cd tools - b2 ${{ matrix.config }} warnings=all warnings-as-errors=on - diff --git a/appveyor.yml b/appveyor.yml deleted file mode 100644 index 6d2232abfc1..00000000000 --- a/appveyor.yml +++ /dev/null @@ -1,113 +0,0 @@ -version: "{build}" -branches: - only: - - master - - RC_2_0 - - RC_1_2 - - RC_1_1 -image: Visual Studio 2019 -clone_depth: 1 -environment: - matrix: - - variant: debug - compiler: gcc - model: 32 - crypto: openssl - ssl_lib: /usr/local/include - ssl_include: /usr/local/lib - lib: 1 - - cmake: 1 - crypto: openssl - ssl_root_dir: c:\OpenSSL-v111-Win64 - webtorrent: on - - variant: release - compiler: msvc - model: 64 - crypto: openssl - ssl_lib: c:\OpenSSL-v111-Win64\lib - ssl_include: c:\OpenSSL-v111-Win64\include - webtorrent: on - tests: 1 - -artifacts: - - path: bindings/python/dist/* - name: python-module - -install: - - git submodule update --init --recursive - - set ROOT_DIRECTORY=%CD% - - cd %ROOT_DIRECTORY% - - if not defined api ( set api="desktop" ) - - if not defined compiler ( set compiler="" ) - - if not defined linking ( set linking="shared" ) - - if not defined crypto ( set crypto=built-in ) - - if not defined webtorrent ( set webtorrent=off ) - - if not defined ssl_lib ( set ssl_lib=c:\ ) - - if not defined ssl_include ( set ssl_include=c:\ ) - - if not defined ssl_root_dir ( set ssl_root_dir=c:\ ) - - cd %ROOT_DIRECTORY% - - set BOOST_ROOT=c:\Libraries\boost_1_71_0 - - set BOOST_BUILD_PATH=%BOOST_ROOT%\tools\build - - echo %BOOST_ROOT% - - echo %BOOST_BUILD_PATH% - - set PATH=%PATH%;%BOOST_BUILD_PATH% - - ps: '"using msvc ;`nusing python : 3.6 : c:\\Python36-x64 : c:\\Python36-x64\\include : c:\\Python36-x64\\libs ;`n" | Set-Content $env:HOMEDRIVE\$env:HOMEPATH\user-config.jam' - - type %HOMEDRIVE%%HOMEPATH%\user-config.jam - - cd %ROOT_DIRECTORY% - - set PATH=c:\Python36-x64;%PATH% - - set PYTHON_INTERPRETER=c:\Python36-x64\python.exe - - python --version - - python -m pip install websockets - - echo %ROOT_DIRECTORY% - - cd %BOOST_BUILD_PATH% - - bootstrap.bat >nul - - cd %ROOT_DIRECTORY% - -build_script: - - # just the library - - cd %ROOT_DIRECTORY% - - if defined lib ( - b2.exe --hash openssl-lib=%ssl_lib% openssl-include=%ssl_include% warnings=all warnings-as-errors=on %compiler% address-model=%model% picker-debugging=on invariant-checks=full variant=%variant% link=shared crypto=%crypto% webtorrent=%webtorrent% asserts=on export-extra=on windows-api=%api% windows-version=win10 cxxstd=17 - ) - - # test - - cd %ROOT_DIRECTORY%\test - - if defined tests ( - b2.exe --hash openssl-lib=%ssl_lib% openssl-include=%ssl_include% warnings=all warnings-as-errors=on %compiler% address-model=%model% picker-debugging=on invariant-checks=full variant=%variant% link=shared crypto=%crypto% webtorrent=%webtorrent% asserts=on export-extra=on windows-api=%api% windows-version=win10 testing.execute=off - ) - - # python binding - - cd %ROOT_DIRECTORY%\bindings\python - # we use 64 bit python builds - # boost.python itself doesn't build warning free, so we can't build - # with warnings-as-errors - - if defined python ( - b2.exe --hash openssl-lib=%ssl_lib% openssl-include=%ssl_include% warnings=all %compiler% address-model=%model% picker-debugging=on invariant-checks=full variant=%variant% link=shared crypto=%crypto% webtorrent=%webtorrent% asserts=on export-extra=on windows-api=%api% windows-version=win10 libtorrent-link=shared stage_module stage_dependencies - ) - - if defined python_dist ( - c:\Python36-x64\python.exe setup.py bdist --format=msi - ) - - # minimal support for cmake build - # we need to build the boost libraries we use with C++17 - # and stage it for cmake to pick up - - if defined cmake ( - cd %BOOST_ROOT% && - b2.exe cxxstd=17 release --with-python --with-system --layout=system address-model=64 link=shared stage && - cd %ROOT_DIRECTORY% && - mkdir build && - cd build && - cmake -DBOOST_LIBRARYDIR=%BOOST_ROOT%\stage\lib -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=ON -Dbuild_examples=ON -Dbuild_tools=ON -Dwebtorrent=%webtorrent% -DOPENSSL_ROOT_DIR=%ssl_root_dir% -Dpython-bindings=%python% -Dboost-python-module-name="python" -Dskip-python-runtime-test=true -DPython_ADDITIONAL_VERSIONS="2.7" -G "Visual Studio 16 2019" -A x64 .. && - cmake --build . --config Release --parallel %NUMBER_OF_PROCESSORS% -- -verbosity:minimal - ) - -test_script: - - cd %ROOT_DIRECTORY%\test - - if defined tests ( - appveyor-retry b2.exe -l500 --hash openssl-lib=%ssl_lib% openssl-include=%ssl_include% warnings=all warnings-as-errors=on %compiler% address-model=%model% picker-debugging=on invariant-checks=full variant=%variant% link=shared crypto=%crypto% webtorrent=%webtorrent% asserts=on export-extra=on windows-api=%api% windows-version=win10 deterministic-tests - ) - - - if defined cmake ( - appveyor-retry ctest - ) diff --git a/cihelper/cihelper/__init__.py b/cihelper/cihelper/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/cihelper/cihelper/boost.py b/cihelper/cihelper/boost.py new file mode 100644 index 00000000000..49949b9ea27 --- /dev/null +++ b/cihelper/cihelper/boost.py @@ -0,0 +1,71 @@ +import contextlib +import logging +import os +import os.path +import pathlib +import sys +import tarfile +import urllib.request + +from . import common + +_LOG = logging.getLogger(__name__) + + +def get_version() -> str: + return os.environ.get("CIHELPER_BOOST_VERSION", "1.76.0") + + +def get_root() -> pathlib.Path: + # bootstrap.sh doesn't support being run from a dir with spaces + assert " " not in str(common.get_base()), common.get_base() + vunder = get_version().replace(".", "_") + return common.get_base() / f"boost_{vunder}" + + +def get_archive_url(suffix: str) -> str: + version = get_version() + vunder = version.replace(".", "_") + return ( + "https://boostorg.jfrog.io/artifactory/main/release/" + f"{version}/source/boost_{vunder}{suffix}" + ) + + +def download_and_extract() -> None: + # This is pure-python but slow + url = get_archive_url(".tar.bz2") + _LOG.debug("requesting: %s", url) + resp = urllib.request.urlopen(url) + tar = tarfile.open(mode="r|bz2", fileobj=resp) + with contextlib.closing(tar): + # will populate //boost_x_y_z + _LOG.debug("extracting to: %s", common.get_base()) + common.get_base().mkdir(parents=True, exist_ok=True) + tar.extractall(path=str(common.get_base())) + + +def bootstrap() -> None: + if sys.platform == "win32": + exe = get_root() / "bootstrap.bat" + else: + exe = get_root() / "bootstrap.sh" + common.check_call([exe], cwd=get_root()) + config = get_root() / "project-config.jam" + _LOG.info("contents of %s:\n%s", config, config.read_text()) + common.check_call( + [get_root() / "b2", f"-j{common.get_jobs()}", "headers"], cwd=get_root() + ) + + +def install() -> None: + _LOG.info("PATH = %r", os.environ["PATH"]) + if get_root().is_dir(): + return + download_and_extract() + bootstrap() + + +def install_cmd() -> None: + common.configure_logger() + install() diff --git a/cihelper/cihelper/cibw.py b/cihelper/cihelper/cibw.py new file mode 100644 index 00000000000..5363c290cad --- /dev/null +++ b/cihelper/cihelper/cibw.py @@ -0,0 +1,96 @@ +import logging +import os +import runpy +import shlex +import sys + +from . import common +from . import env + +_LOG = logging.getLogger(__name__) + + +def set_env(name: str, value: str) -> None: + _LOG.info("set %s = %r", name, value) + os.environ[name] = value + + +def append_inner_env(name: str, value: str) -> None: + existing = os.environ.get("CIBW_ENVIRONMENT", "") + set_env("CIBW_ENVIRONMENT", f"{existing} {name}={value}") + + +def setup_env() -> None: + if sys.platform == "linux": + # cibuildwheel will invoke docker. + + # Our homedir-based install paths will be different in the container. + # Set this *now*, so we calculate other paths correctly + set_env("CIHELPER_BASE", "/cihelper") + # *Also* set in CIBW_ENVIRONMENT so paths are calculated the same + # way inside docker + append_inner_env("CIHELPER_BASE", "/cihelper") + + # We'll need to install cihelper in the container + set_env( + "CIBW_BEFORE_ALL", + "python -m pip install ./cihelper && cihelper_boost_install", + ) + + # As of writing: + # - manylinux2010 has g++ ?? + # - manylinux2014 has g++ ?? + # - manylinux_2_24 has g++ 6.3.0 + set_env("CIBW_MANYLINUX_X86_64_IMAGE", "manylinux2014") + set_env("CIBW_MANYLINUX_I686_IMAGE", "manylinux2014") + + # Supply our env variables + for name, value in env.get().items(): + # We've already figured exact values we want. We *don't* want + # cibuildwheel to do any expansion. shlex-quote will defend against + # this. + append_inner_env(name, shlex.quote(value)) + + # Supply our PATH entries + path = list(env.get_path()) + if path: + value = os.pathsep.join(path + ["$PATH"]) + # We *do* want cibuildwheel to do expansion, for $PATH. I can't find a + # way to tell cibuildwheel to expand $PATH but not perform expansion + # on the other paths we're supplying; hopefully they don't contain + # variable names. + append_inner_env("PATH", f'"{value}"') + else: + # cibuildwheel will build the wheel nakedly. + # Do *not* install cihelper, because it's already installed (it's + # running this script!) + set_env("CIBW_BEFORE_ALL", "cihelper_boost_install") + + # cibuildwheel's environment variable expansion seems to stomp on \, so + # with CIBW_ENVIRONMENT=PATH="c:\foo;$PATH", PATH becomes "c:foo;...". + # Not sure if the intent is that we just manipulate PATH directly on + # Windows since we run nakedly. That seems the best answer for now + # so that's what we do. + for name, value in env.get().items(): + set_env(name, value) + path = list(env.get_path()) + if path: + set_env("PATH", os.pathsep.join(path + [os.environ["PATH"]])) + + # This would normally be in root files like setup.cfg or setup.py, but all + # our files are in bindings/python so cibuildwheel can't figure it out on + # its own + set_env("CIBW_PROJECT_REQUIRES_PYTHON", ">=3.6") + # Per https://cibuildwheel.readthedocs.io/en/stable/cpp_standards/, 10.14 + # is required for full C++17 support + set_env("MACOSX_DEPLOYMENT_TARGET", "10.14") + + +def run() -> None: + setup_env() + runpy.run_module("cibuildwheel", run_name="__main__") + + +def run_cmd() -> None: + common.configure_logger() + run() diff --git a/cihelper/cihelper/common.py b/cihelper/cihelper/common.py new file mode 100644 index 00000000000..185db1a8d27 --- /dev/null +++ b/cihelper/cihelper/common.py @@ -0,0 +1,53 @@ +import logging +import multiprocessing +import os +import pathlib +import subprocess +import sys +from typing import Sequence +from typing import Union + +import appdirs + +_LOG = logging.getLogger(__name__) + + +def configure_logger() -> None: + logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) + + +def get_jobs() -> int: + return multiprocessing.cpu_count() + 1 + + +_AnyPath = Union[str, os.PathLike] + + +# The real check_call has a huge list of parameters, just copy the ones we use +def check_call(cmd: Sequence[_AnyPath], *, cwd: _AnyPath) -> None: + _LOG.info("+ %s", cmd) + subprocess.check_call(cmd, cwd=cwd, stdout=sys.stdout, stderr=sys.stderr) + + +def get_appdirs() -> appdirs.AppDirs: + return appdirs.AppDirs(appname="cihelper") + + +def get_base() -> pathlib.Path: + base = os.environ.get("CIHELPER_BASE") + if base: + return pathlib.Path(base) + # boost's bootstrap.sh (and maybe others) break if run from a dir with + # spaces in the name. On OSX, user_data_dir includes "Application Support". + # work around this by trying several dirs. + dirs = get_appdirs() + for path in ( + dirs.user_data_dir, + dirs.user_cache_dir, + dirs.user_state_dir, + dirs.user_config_dir, + ): + if " " not in path: + return pathlib.Path(path) + # fallback to user_data_dir + return pathlib.Path(dirs.user_data_dir) diff --git a/cihelper/cihelper/env.py b/cihelper/cihelper/env.py new file mode 100644 index 00000000000..a19969390f7 --- /dev/null +++ b/cihelper/cihelper/env.py @@ -0,0 +1,14 @@ +from typing import Mapping +from typing import Sequence + +from . import boost + + +def get() -> Mapping[str, str]: + return { + "BOOST_ROOT": str(boost.get_root()), + } + + +def get_path() -> Sequence[str]: + return [str(boost.get_root())] diff --git a/cihelper/pyproject.toml b/cihelper/pyproject.toml new file mode 100644 index 00000000000..9e98b626978 --- /dev/null +++ b/cihelper/pyproject.toml @@ -0,0 +1,6 @@ +[project] +requires-python = ">=3.6" + +[build-system] +requires = ["setuptools", "wheel"] +build-backend = "setuptools.build_meta" diff --git a/cihelper/setup.cfg b/cihelper/setup.cfg new file mode 100644 index 00000000000..b906a5f34d1 --- /dev/null +++ b/cihelper/setup.cfg @@ -0,0 +1,15 @@ +[metadata] +name = cihelper +version = 0.0.1 +description = helpers for libtorrent CI + +[options] +python_requires = >=3.6 +packages = find: +install_requires = + appdirs + +[options.entry_points] +console_scripts = + cihelper_boost_install = cihelper.boost:install_cmd + cihelper_cibw_run = cihelper.cibw:run_cmd