From 379d281600b11842f9fd7495efa313077998082d Mon Sep 17 00:00:00 2001 From: Andrey Senyaev <76472231+asenyaev@users.noreply.github.com> Date: Fri, 26 Aug 2022 15:27:15 +0300 Subject: [PATCH] Build zlib from the source on Linux with the latest version (#715) * Added zlib build from the source on Linux, handling auditwheel issues * Removed unused variable * Removed bzip2 from dependencies for OpenCV Python packages --- .github/workflows/build_wheels_linux.yml | 10 +---- .github/workflows/build_wheels_linux_arm.yml | 12 +----- .github/workflows/build_wheels_macos.yml | 4 -- .github/workflows/build_wheels_macos_m1.yml | 10 ----- docker/manylinux2014/Dockerfile_aarch64 | 29 +++++++++----- docker/manylinux2014/Dockerfile_x86_64 | 29 +++++++++----- patch_auditwheel_whitelist.py | 40 ++++++++++++++++---- 7 files changed, 75 insertions(+), 59 deletions(-) diff --git a/.github/workflows/build_wheels_linux.yml b/.github/workflows/build_wheels_linux.yml index a8cdd34e..ab377325 100644 --- a/.github/workflows/build_wheels_linux.yml +++ b/.github/workflows/build_wheels_linux.yml @@ -35,14 +35,13 @@ jobs: env: ACTIONS_ALLOW_UNSECURE_COMMANDS: true REPO_DIR: . - BUILD_COMMIT: ${{ github.ref }} PROJECT_SPEC: opencv-python MB_PYTHON_VERSION: ${{ matrix.python-version }} TRAVIS_PYTHON_VERSION: ${{ matrix.python-version }} MB_ML_VER: 2014 TRAVIS_BUILD_DIR: ${{ github.workspace }} CONFIG_PATH: travis_config.sh - DOCKER_IMAGE: quay.io/opencv-ci/opencv-python-manylinux2014-x86-64:20220628 + DOCKER_IMAGE: quay.io/opencv-ci/opencv-python-manylinux2014-x86-64:20220824 USE_CCACHE: 0 UNICODE_WIDTH: 32 PLAT: x86_64 @@ -55,9 +54,6 @@ jobs: working-directory: ${{ github.workspace }} - name: Setup environment run: | - if [[ "${{ github.event.pull_request }}" == "true" ]]; then - echo "BUILD_COMMIT=${{ github.base_ref }}" >> $GITHUB_ENV - fi if [[ "${{ github.event_name }}" == "schedule" || "${{ github.event_name }}" == "workflow_dispatch" ]]; then echo "ENABLE_ROLLING=1" >> $GITHUB_ENV fi @@ -135,7 +131,6 @@ jobs: env: ACTIONS_ALLOW_UNSECURE_COMMANDS: true REPO_DIR: . - BUILD_COMMIT: ${{ github.ref }} PROJECT_SPEC: opencv-python PLAT: x86_64 MB_PYTHON_VERSION: ${{ matrix.python-version }} @@ -156,9 +151,6 @@ jobs: working-directory: ${{ github.workspace }} - name: Setup environment run: | - if [[ "${{ github.event.pull_request }}" == "true" ]]; then - echo "BUILD_COMMIT=${{ github.base_ref }}" >> $GITHUB_ENV - fi if [[ "${{ github.event_name }}" == "schedule" || "${{ github.event_name }}" == "workflow_dispatch" ]]; then echo "ENABLE_ROLLING=1" >> $GITHUB_ENV fi diff --git a/.github/workflows/build_wheels_linux_arm.yml b/.github/workflows/build_wheels_linux_arm.yml index c6b9b4c5..079efbe1 100644 --- a/.github/workflows/build_wheels_linux_arm.yml +++ b/.github/workflows/build_wheels_linux_arm.yml @@ -35,7 +35,6 @@ jobs: env: ACTIONS_ALLOW_UNSECURE_COMMANDS: true REPO_DIR: . - BUILD_COMMIT: ${{ github.ref }} PROJECT_SPEC: opencv-python MB_PYTHON_VERSION: ${{ matrix.python-version }} TRAVIS_PYTHON_VERSION: ${{ matrix.python-version }} @@ -43,7 +42,7 @@ jobs: MB_ML_VER: 2014 TRAVIS_BUILD_DIR: ${{ github.workspace }} CONFIG_PATH: travis_config.sh - DOCKER_IMAGE: quay.io/opencv-ci/opencv-python-manylinux2014-aarch64:20220628 + DOCKER_IMAGE: quay.io/opencv-ci/opencv-python-manylinux2014-aarch64:20220824 USE_CCACHE: 0 UNICODE_WIDTH: 32 SDIST: ${{ matrix.build_sdist || 0 }} @@ -55,9 +54,6 @@ jobs: working-directory: ${{ github.workspace }} - name: Setup environment run: | - if [[ "${{ github.event.pull_request }}" == "true" ]]; then - echo "BUILD_COMMIT=${{ github.base_ref }}" >> $GITHUB_ENV - fi if [[ "${{ github.event_name }}" == "schedule" || "${{ github.event_name }}" == "workflow_dispatch" ]]; then echo "ENABLE_ROLLING=1" >> $GITHUB_ENV fi @@ -97,7 +93,6 @@ jobs: ACTIONS_ALLOW_UNSECURE_COMMANDS: true MB_PYTHON_VERSION: ${{ matrix.python-version }} PLAT: aarch64 - BUILD_COMMIT: ${{ github.ref }} NP_TEST_DEP: numpy==1.19.4 NP_TEST_DEP_LATEST: numpy==1.21.4 CONFIG_PATH: travis_config.sh @@ -107,11 +102,6 @@ jobs: - name: Cleanup run: find . -mindepth 1 -delete working-directory: ${{ github.workspace }} - - name: Setup environment - run: | - if [[ "${{ github.event.pull_request }}" == "true" ]]; then - echo "BUILD_COMMIT=${{ github.base_ref }}" >> $GITHUB_ENV - fi - name: Checkout uses: actions/checkout@v3 with: diff --git a/.github/workflows/build_wheels_macos.yml b/.github/workflows/build_wheels_macos.yml index 1ac459cd..1c2a1614 100644 --- a/.github/workflows/build_wheels_macos.yml +++ b/.github/workflows/build_wheels_macos.yml @@ -35,7 +35,6 @@ jobs: env: ACTIONS_ALLOW_UNSECURE_COMMANDS: true REPO_DIR: . - BUILD_COMMIT: ${{ github.ref }} PROJECT_SPEC: opencv-python MB_PYTHON_VERSION: ${{ matrix.python-version }} TRAVIS_PYTHON_VERSION: ${{ matrix.python-version }} @@ -57,9 +56,6 @@ jobs: working-directory: ${{ github.workspace }} - name: Setup environment run: | - if [[ "${{ github.event.pull_request }}" == "true" ]]; then - echo "BUILD_COMMIT=${{ github.base_ref }}" >> $GITHUB_ENV - fi if [[ "${{ github.event_name }}" == "schedule" || "${{ github.event_name }}" == "workflow_dispatch" ]]; then echo "ENABLE_ROLLING=1" >> $GITHUB_ENV fi diff --git a/.github/workflows/build_wheels_macos_m1.yml b/.github/workflows/build_wheels_macos_m1.yml index 3d8f9b62..b51801ab 100644 --- a/.github/workflows/build_wheels_macos_m1.yml +++ b/.github/workflows/build_wheels_macos_m1.yml @@ -30,7 +30,6 @@ jobs: without_gui: [0, 1] build_sdist: [0] env: - BUILD_COMMIT: ${{ github.ref }} CI_BUILD: 1 SDIST: ${{ matrix.build_sdist || 0 }} ENABLE_HEADLESS: ${{ matrix.without_gui }} @@ -41,9 +40,6 @@ jobs: working-directory: ${{ github.workspace }} - name: Setup environment run: | - if [[ "${{ github.event.pull_request }}" == "true" ]]; then - echo "BUILD_COMMIT=${{ github.base_ref }}" >> $GITHUB_ENV - fi if [[ "${{ github.event_name }}" == "schedule" || "${{ github.event_name }}" == "workflow_dispatch" ]]; then echo "ENABLE_ROLLING=1" >> $GITHUB_ENV fi @@ -84,18 +80,12 @@ jobs: without_gui: [0, 1] build_sdist: [0] env: - BUILD_COMMIT: ${{ github.ref }} OPENCV_TEST_DATA_PATH: ${{ github.workspace }}/opencv_extra/testdata PYLINT_TEST_FILE: ${{ github.workspace }}/opencv/samples/python/squares.py steps: - name: Cleanup run: find . -mindepth 1 -delete working-directory: ${{ github.workspace }} - - name: Setup environment - run: | - if [[ "${{ github.event.pull_request }}" == "true" ]]; then - echo "BUILD_COMMIT=${{ github.base_ref }}" >> $GITHUB_ENV - fi - name: Checkout uses: actions/checkout@v3 with: diff --git a/docker/manylinux2014/Dockerfile_aarch64 b/docker/manylinux2014/Dockerfile_aarch64 index 2679cbbf..48ad3f98 100644 --- a/docker/manylinux2014/Dockerfile_aarch64 +++ b/docker/manylinux2014/Dockerfile_aarch64 @@ -1,4 +1,4 @@ -# Version: 20220628 +# Version: 20220824 # Image name: quay.io/opencv-ci/opencv-python-manylinux2014-aarch64 FROM quay.io/pypa/manylinux2014_aarch64:latest @@ -11,16 +11,28 @@ ARG NASM_VERSION=2.15.04 ARG OPENSSL_VERSION=1_1_1o ARG QT_VERSION=5.15.0 ARG YASM_VERSION=1.3.0 +ARG ZLIB_VERSION=1.2.12 ENV LD_LIBRARY_PATH /usr/local/lib:$LD_LIBRARY_PATH # epel-release need for aarch64 to get openblas packages -RUN yum install bzip2-devel curl-devel zlib-devel xcb-util-renderutil-devel xcb-util-devel xcb-util-image-devel xcb-util-keysyms-devel xcb-util-wm-devel mesa-libGL-devel libxkbcommon-devel libxkbcommon-x11-devel libXi-devel lapack-devel epel-release -y && \ +RUN yum install bzip2-devel curl-devel xcb-util-renderutil-devel xcb-util-devel xcb-util-image-devel xcb-util-keysyms-devel xcb-util-wm-devel mesa-libGL-devel libxkbcommon-devel libxkbcommon-x11-devel libXi-devel lapack-devel epel-release -y && \ yum install openblas-devel -y && \ cp /usr/include/lapacke/lapacke*.h /usr/include/ && \ curl https://raw.githubusercontent.com/xianyi/OpenBLAS/v0.3.3/cblas.h -o /usr/include/cblas.h && \ # libpng will be built from source - yum remove libpng -y + yum remove libpng zlib-devel bzip2-devel -y + +RUN mkdir ~/zlib_sources && \ + cd ~/zlib_sources && \ + curl -O -L https://www.zlib.net/zlib-${ZLIB_VERSION}.tar.gz && \ + tar -xf zlib-${ZLIB_VERSION}.tar.gz && \ + cd zlib-${ZLIB_VERSION} && \ + ./configure --prefix=/usr/local && \ + make && \ + make install && \ + cd .. && \ + rm -rf ~/zlib_sources RUN mkdir ~/libpng_sources && \ cd ~/libpng_sources && \ @@ -71,8 +83,8 @@ RUN mkdir ~/openssl_sources && \ RUN mkdir ~/nasm_sources && \ cd ~/nasm_sources && \ - curl -O -L http://www.nasm.us/pub/nasm/releasebuilds/${NASM_VERSION}/nasm-${NASM_VERSION}.tar.bz2 && \ - tar -xf nasm-${NASM_VERSION}.tar.bz2 && cd nasm-${NASM_VERSION} && ./autogen.sh && \ + curl -O -L http://www.nasm.us/pub/nasm/releasebuilds/${NASM_VERSION}/nasm-${NASM_VERSION}.tar.gz && \ + tar -xf nasm-${NASM_VERSION}.tar.gz && cd nasm-${NASM_VERSION} && ./autogen.sh && \ ./configure --prefix="/ffmpeg_build" --bindir="$HOME/bin" && \ make -j$(getconf _NPROCESSORS_ONLN) && \ make install && \ @@ -102,8 +114,8 @@ RUN mkdir ~/libvpx_sources && \ RUN mkdir ~/ffmpeg_sources && \ cd ~/ffmpeg_sources && \ - curl -O -L https://ffmpeg.org/releases/ffmpeg-${FFMPEG_VERSION}.tar.bz2 && \ - tar -xf ffmpeg-${FFMPEG_VERSION}.tar.bz2 && \ + curl -O -L https://ffmpeg.org/releases/ffmpeg-${FFMPEG_VERSION}.tar.gz && \ + tar -xf ffmpeg-${FFMPEG_VERSION}.tar.gz && \ cd ffmpeg-${FFMPEG_VERSION} && \ PATH=~/bin:$PATH && \ PKG_CONFIG_PATH="/ffmpeg_build/lib/pkgconfig" ./configure --prefix="/ffmpeg_build" --extra-cflags="-I/ffmpeg_build/include" --extra-ldflags="-L/ffmpeg_build/lib" --enable-openssl --enable-libvpx --enable-shared --enable-pic --bindir="$HOME/bin" && \ @@ -111,8 +123,7 @@ RUN mkdir ~/ffmpeg_sources && \ make install && \ echo "/ffmpeg_build/lib/" >> /etc/ld.so.conf && \ ldconfig && \ - rm -rf ~/ffmpeg_sources && \ - yum remove bzip2-devel -y + rm -rf ~/ffmpeg_sources RUN curl -O -L https://github.com/ccache/ccache/releases/download/v${CCACHE_VERSION}/ccache-${CCACHE_VERSION}.tar.gz && \ tar -xf ccache-${CCACHE_VERSION}.tar.gz && \ diff --git a/docker/manylinux2014/Dockerfile_x86_64 b/docker/manylinux2014/Dockerfile_x86_64 index 8bb77341..85fc22fd 100644 --- a/docker/manylinux2014/Dockerfile_x86_64 +++ b/docker/manylinux2014/Dockerfile_x86_64 @@ -1,4 +1,4 @@ -# Version: 20220628 +# Version: 20220824 # Image name: quay.io/opencv-ci/opencv-python-manylinux2014-x86-64 FROM quay.io/pypa/manylinux2014_x86_64:latest @@ -11,16 +11,28 @@ ARG NASM_VERSION=2.15.04 ARG OPENSSL_VERSION=1_1_1o ARG QT_VERSION=5.15.0 ARG YASM_VERSION=1.3.0 +ARG ZLIB_VERSION=1.2.12 ENV LD_LIBRARY_PATH /usr/local/lib:$LD_LIBRARY_PATH # epel-release need for aarch64 to get openblas packages -RUN yum install bzip2-devel curl-devel zlib-devel xcb-util-renderutil-devel xcb-util-devel xcb-util-image-devel xcb-util-keysyms-devel xcb-util-wm-devel mesa-libGL-devel libxkbcommon-devel libxkbcommon-x11-devel libXi-devel lapack-devel epel-release -y && \ +RUN yum install bzip2-devel curl-devel xcb-util-renderutil-devel xcb-util-devel xcb-util-image-devel xcb-util-keysyms-devel xcb-util-wm-devel mesa-libGL-devel libxkbcommon-devel libxkbcommon-x11-devel libXi-devel lapack-devel epel-release -y && \ yum install openblas-devel -y && \ cp /usr/include/lapacke/lapacke*.h /usr/include/ && \ curl https://raw.githubusercontent.com/xianyi/OpenBLAS/v0.3.3/cblas.h -o /usr/include/cblas.h && \ # libpng will be built from source - yum remove libpng -y + yum remove libpng zlib-devel bzip2-devel -y + +RUN mkdir ~/zlib_sources && \ + cd ~/zlib_sources && \ + curl -O -L https://www.zlib.net/zlib-${ZLIB_VERSION}.tar.gz && \ + tar -xf zlib-${ZLIB_VERSION}.tar.gz && \ + cd zlib-${ZLIB_VERSION} && \ + ./configure --prefix=/usr/local && \ + make && \ + make install && \ + cd .. && \ + rm -rf ~/zlib_sources RUN mkdir ~/libpng_sources && \ cd ~/libpng_sources && \ @@ -71,8 +83,8 @@ RUN mkdir ~/openssl_sources && \ RUN mkdir ~/nasm_sources && \ cd ~/nasm_sources && \ - curl -O -L http://www.nasm.us/pub/nasm/releasebuilds/${NASM_VERSION}/nasm-${NASM_VERSION}.tar.bz2 && \ - tar -xf nasm-${NASM_VERSION}.tar.bz2 && cd nasm-${NASM_VERSION} && ./autogen.sh && \ + curl -O -L http://www.nasm.us/pub/nasm/releasebuilds/${NASM_VERSION}/nasm-${NASM_VERSION}.tar.gz && \ + tar -xf nasm-${NASM_VERSION}.tar.gz && cd nasm-${NASM_VERSION} && ./autogen.sh && \ ./configure --prefix="/ffmpeg_build" --bindir="$HOME/bin" && \ make -j$(getconf _NPROCESSORS_ONLN) && \ make install && \ @@ -102,8 +114,8 @@ RUN mkdir ~/libvpx_sources && \ RUN mkdir ~/ffmpeg_sources && \ cd ~/ffmpeg_sources && \ - curl -O -L https://ffmpeg.org/releases/ffmpeg-${FFMPEG_VERSION}.tar.bz2 && \ - tar -xf ffmpeg-${FFMPEG_VERSION}.tar.bz2 && \ + curl -O -L https://ffmpeg.org/releases/ffmpeg-${FFMPEG_VERSION}.tar.gz && \ + tar -xf ffmpeg-${FFMPEG_VERSION}.tar.gz && \ cd ffmpeg-${FFMPEG_VERSION} && \ PATH=~/bin:$PATH && \ PKG_CONFIG_PATH="/ffmpeg_build/lib/pkgconfig" ./configure --prefix="/ffmpeg_build" --extra-cflags="-I/ffmpeg_build/include" --extra-ldflags="-L/ffmpeg_build/lib" --enable-openssl --enable-libvpx --enable-shared --enable-pic --bindir="$HOME/bin" && \ @@ -111,8 +123,7 @@ RUN mkdir ~/ffmpeg_sources && \ make install && \ echo "/ffmpeg_build/lib/" >> /etc/ld.so.conf && \ ldconfig && \ - rm -rf ~/ffmpeg_sources && \ - yum remove bzip2-devel -y + rm -rf ~/ffmpeg_sources RUN curl -O -L https://github.com/ccache/ccache/releases/download/v${CCACHE_VERSION}/ccache-${CCACHE_VERSION}.tar.gz && \ tar -xf ccache-${CCACHE_VERSION}.tar.gz && \ diff --git a/patch_auditwheel_whitelist.py b/patch_auditwheel_whitelist.py index b32da17a..150f97ca 100644 --- a/patch_auditwheel_whitelist.py +++ b/patch_auditwheel_whitelist.py @@ -3,13 +3,39 @@ from auditwheel import policy -policies = None -with open(join(dirname(abspath(policy.__file__)), "manylinux-policy.json")) as f: - policies = json.load(f) +def add_zlib_versions(): + with open(join(dirname(abspath(policy.__file__)), "manylinux-policy.json"), 'r') as manylinux_policy: + manylinux_policy_lines = manylinux_policy.readlines() -for p in policies: - p["lib_whitelist"].append("libxcb.so.1") + manylinux_policy_data = "" + for line in manylinux_policy_lines: + if "ZLIB" in line: + if len(line) > 22: + updated_line = line[:-2] + ', "1.2.9", "1.2.12"]' + else: + updated_line = line[:-2] + '"1.2.9", "1.2.12"]' + manylinux_policy_replacement = line.replace(line, updated_line) + else: + manylinux_policy_replacement = line + manylinux_policy_data = manylinux_policy_data + manylinux_policy_replacement -with open(join(dirname(abspath(policy.__file__)), "manylinux-policy.json"), "w") as f: - f.write(json.dumps(policies)) + with open(join(dirname(abspath(policy.__file__)), "manylinux-policy.json"), 'w') as manylinux_final_policy: + manylinux_final_policy.write(manylinux_policy_data) + +def add_whitelisted_libs(): + policies = None + + with open(join(dirname(abspath(policy.__file__)), "manylinux-policy.json")) as f: + policies = json.load(f) + + for p in policies: + p["lib_whitelist"].append("libxcb.so.1") + + with open(join(dirname(abspath(policy.__file__)), "manylinux-policy.json"), "w") as f: + f.write(json.dumps(policies)) + + +if __name__ == '__main__': + add_zlib_versions() + add_whitelisted_libs()