From 5ef2032e1971b2eb6fc4be4f0c05fecb3f790743 Mon Sep 17 00:00:00 2001 From: Armen Zambrano G Date: Tue, 7 Sep 2021 09:39:19 -0400 Subject: [PATCH 01/33] feat: Cross-compile macos arm64 wheels This uses [cibuildwheel](https://cibuildwheel.readthedocs.io/en/stable/) in order to cross-compile the arm64 wheels for Python 3.8, 3.9 & 3.10. Fixes #1286 --- .github/workflows/packages.yml | 37 ++++++++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 137e91c28..ccf13e515 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -3,7 +3,6 @@ name: Build packages on: - workflow_dispatch - jobs: build-sdist: strategy: @@ -51,16 +50,15 @@ jobs: --health-timeout 5s --health-retries 5 - build-manylinux: strategy: fail-fast: false matrix: include: - - {tag: manylinux2014, arch: x86_64} - - {tag: manylinux2014, arch: i686} - - {tag: manylinux_2_24, arch: aarch64} - - {tag: manylinux_2_24, arch: ppc64le} + - { tag: manylinux2014, arch: x86_64 } + - { tag: manylinux2014, arch: i686 } + - { tag: manylinux_2_24, arch: aarch64 } + - { tag: manylinux_2_24, arch: ppc64le } runs-on: ubuntu-20.04 steps: @@ -107,13 +105,12 @@ jobs: --health-timeout 5s --health-retries 5 - build-macos: runs-on: macos-10.15 strategy: fail-fast: false matrix: - python-version: ['3.6', '3.7', '3.8', '3.9'] + python-version: ["3.6", "3.7", "3.8", "3.9"] steps: - name: Checkout repos @@ -137,3 +134,27 @@ jobs: name: packages_macos path: | dist/*/*${{ matrix.platform }}.whl + + # This only uploads the arm64 wheels for Apple M1 silicon usage + build-macos-arm64: + runs-on: macos-10.15 + strategy: + fail-fast: false + name: "build-macos (3.8-3.10 - arm64)" + steps: + - uses: actions/checkout@v2 + # This builds all possible Python versions (currently, 3.8 to 3.10) + - uses: pypa/cibuildwheel@v2.1.1 + with: + output-dir: dist + # Read about options here https://cibuildwheel.readthedocs.io/en/stable/options + env: + CIBW_ARCHS_MACOS: arm64 + # Skip testing the arm64 builds + CIBW_TEST_SKIP: "*:arm64" + - name: Upload artifacts + uses: actions/upload-artifact@v2 + with: + name: packages_macos + path: | + dist/*/*macosx_11_0_arm64.whl From 96ebfa93e34bb354c26694e2e5c980decde31064 Mon Sep 17 00:00:00 2001 From: Armen Zambrano G Date: Tue, 7 Sep 2021 15:59:41 -0400 Subject: [PATCH 02/33] Adjust glob matching --- .github/workflows/packages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index ccf13e515..4137c40d4 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -157,4 +157,4 @@ jobs: with: name: packages_macos path: | - dist/*/*macosx_11_0_arm64.whl + dist/*macosx_11_0_arm64.whl From fe1165544c185b3e55158ec7779eef1a502bae6e Mon Sep 17 00:00:00 2001 From: Armen Zambrano G Date: Tue, 7 Sep 2021 16:20:21 -0400 Subject: [PATCH 03/33] Name the upload artifact correctly --- .github/workflows/packages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 4137c40d4..f47fb1394 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -155,6 +155,6 @@ jobs: - name: Upload artifacts uses: actions/upload-artifact@v2 with: - name: packages_macos + name: packages_macos_arm64 path: | dist/*macosx_11_0_arm64.whl From 8d55da40749e2a9b730d8d37054934e6769101d5 Mon Sep 17 00:00:00 2001 From: Armen Zambrano G Date: Wed, 8 Sep 2021 14:56:39 -0400 Subject: [PATCH 04/33] Try again --- .github/workflows/packages.yml | 34 ++++++++++++++++++++++++++++------ scripts/build/build_macos.sh | 26 ++++++++++---------------- setup.py | 3 ++- 3 files changed, 40 insertions(+), 23 deletions(-) diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index f47fb1394..482b06a15 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -2,6 +2,8 @@ name: Build packages on: - workflow_dispatch + # XXX: Temp change while working on PR; remove with prettier changes + - pull_request jobs: build-sdist: @@ -137,24 +139,44 @@ jobs: # This only uploads the arm64 wheels for Apple M1 silicon usage build-macos-arm64: - runs-on: macos-10.15 + runs-on: ${{ matrix.os }} strategy: fail-fast: false - name: "build-macos (3.8-3.10 - arm64)" + matrix: + os: [macos-10.15, macos-11.0] + name: "build-${{ matrix.os }} - arm64)" steps: - uses: actions/checkout@v2 + - name: Install pre-requisites + run: | + # brew install gnu-sed postgresql@13 + brew install postgresql@13 + brew services start postgresql + # gsed -i "s/^setup(name=\"psycopg2\"/setup(name=\"psycopg2-binary\"/" setup.py + # git diff + # This builds all possible Python versions (currently, 3.8 to 3.10) - uses: pypa/cibuildwheel@v2.1.1 with: output-dir: dist # Read about options here https://cibuildwheel.readthedocs.io/en/stable/options env: - CIBW_ARCHS_MACOS: arm64 - # Skip testing the arm64 builds - CIBW_TEST_SKIP: "*:arm64" + CIBW_ARCHS: arm64 + PACKAGE_NAME: psycopg2-binary + + - name: Some tests + run: | + git diff + ls -l dist + pip install psycopg2-binary --no-index -f dist/ + python -c "import psycopg2; print(psycopg2.__version__)" + python -c "import psycopg2; print(psycopg2.__libpq_version__)" + python -c "import psycopg2; print(psycopg2.extensions.libpq_version())" + python -c "import tests; tests.unittest.main(defaultTest='tests.test_suite')" + - name: Upload artifacts uses: actions/upload-artifact@v2 with: - name: packages_macos_arm64 + name: packages_${{ matrix.os }}_arm64 path: | dist/*macosx_11_0_arm64.whl diff --git a/scripts/build/build_macos.sh b/scripts/build/build_macos.sh index 0fe4b56ce..d2e1631f6 100755 --- a/scripts/build/build_macos.sh +++ b/scripts/build/build_macos.sh @@ -8,8 +8,8 @@ set -euo pipefail set -x -dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -prjdir="$( cd "${dir}/../.." && pwd )" +dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +prjdir="$(cd "${dir}/../.." && pwd)" brew install gnu-sed postgresql@13 @@ -17,13 +17,13 @@ brew install gnu-sed postgresql@13 brew services start postgresql for i in $(seq 10 -1 0); do - eval pg_isready && break - if [ $i == 0 ]; then - echo "PostgreSQL service not ready, giving up" - exit 1 - fi - echo "PostgreSQL service not ready, waiting a bit, attempts left: $i" - sleep 5 + eval pg_isready && break + if [ $i == 0 ]; then + echo "PostgreSQL service not ready, giving up" + exit 1 + fi + echo "PostgreSQL service not ready, waiting a bit, attempts left: $i" + sleep 5 done # Find psycopg version @@ -35,15 +35,9 @@ mkdir -p "$distdir" # Install required python packages pip install -U pip wheel delocate -# Replace the package name -if [[ "${PACKAGE_NAME:-}" ]]; then - gsed -i "s/^setup(name=\"psycopg2\"/setup(name=\"${PACKAGE_NAME}\"/" \ - "${prjdir}/setup.py" -fi - # Build the wheels wheeldir="${prjdir}/wheels" -pip wheel -w ${wheeldir} . +pip wheel --use-feature=in-tree-build -w ${wheeldir} . delocate-listdeps ${wheeldir}/*.whl # Check where is the libpq. I'm gonna kill it for testing diff --git a/setup.py b/setup.py index 8137ef630..60efb191f 100644 --- a/setup.py +++ b/setup.py @@ -45,6 +45,7 @@ # for a consistent versioning pattern. PSYCOPG_VERSION = '2.9.1' +PSYCOPG_NAME = os.environ.get("PACKAGE_NAME", "psycopg2") # note: if you are changing the list of supported Python version please fix @@ -545,7 +546,7 @@ def is_py_64(): print("failed to read readme: ignoring...") readme = __doc__ -setup(name="psycopg2", +setup(name=PSYCOPG_NAME, version=PSYCOPG_VERSION, author="Federico Di Gregorio", author_email="fog@initd.org", From 7e6f7d48e79ef87ad15af3db4eeefe17fb35b7b0 Mon Sep 17 00:00:00 2001 From: Armen Zambrano G Date: Wed, 8 Sep 2021 15:01:44 -0400 Subject: [PATCH 05/33] Undo prettier --- .github/workflows/packages.yml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 482b06a15..d207f6e06 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -2,8 +2,6 @@ name: Build packages on: - workflow_dispatch - # XXX: Temp change while working on PR; remove with prettier changes - - pull_request jobs: build-sdist: @@ -52,15 +50,16 @@ jobs: --health-timeout 5s --health-retries 5 + build-manylinux: strategy: fail-fast: false matrix: include: - - { tag: manylinux2014, arch: x86_64 } - - { tag: manylinux2014, arch: i686 } - - { tag: manylinux_2_24, arch: aarch64 } - - { tag: manylinux_2_24, arch: ppc64le } + - {tag: manylinux2014, arch: x86_64} + - {tag: manylinux2014, arch: i686} + - {tag: manylinux_2_24, arch: aarch64} + - {tag: manylinux_2_24, arch: ppc64le} runs-on: ubuntu-20.04 steps: @@ -112,7 +111,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.6", "3.7", "3.8", "3.9"] + python-version: ['3.6', '3.7', '3.8', '3.9'] steps: - name: Checkout repos From 224ccd8c766aab1071623c3571aedf39c6fcaf9e Mon Sep 17 00:00:00 2001 From: Armen Zambrano G Date: Wed, 8 Sep 2021 15:02:07 -0400 Subject: [PATCH 06/33] Undo prettier --- .github/workflows/packages.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index d207f6e06..7c53cde47 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -3,6 +3,7 @@ name: Build packages on: - workflow_dispatch + jobs: build-sdist: strategy: @@ -50,7 +51,7 @@ jobs: --health-timeout 5s --health-retries 5 - + build-manylinux: strategy: fail-fast: false @@ -106,6 +107,7 @@ jobs: --health-timeout 5s --health-retries 5 + build-macos: runs-on: macos-10.15 strategy: From b999a5c9ef13afeeea67063957a912fe733b5c59 Mon Sep 17 00:00:00 2001 From: Armen Zambrano G Date: Wed, 8 Sep 2021 15:09:30 -0400 Subject: [PATCH 07/33] We will run tests locally --- .github/workflows/packages.yml | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 7c53cde47..c8850efb0 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -138,23 +138,20 @@ jobs: path: | dist/*/*${{ matrix.platform }}.whl - # This only uploads the arm64 wheels for Apple M1 silicon usage + # This only uploads the arm64 wheels for Apple M1 Silicon usage build-macos-arm64: runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: os: [macos-10.15, macos-11.0] - name: "build-${{ matrix.os }} - arm64)" + name: "build-${{ matrix.os }} - arm64" steps: - uses: actions/checkout@v2 - name: Install pre-requisites run: | - # brew install gnu-sed postgresql@13 brew install postgresql@13 brew services start postgresql - # gsed -i "s/^setup(name=\"psycopg2\"/setup(name=\"psycopg2-binary\"/" setup.py - # git diff # This builds all possible Python versions (currently, 3.8 to 3.10) - uses: pypa/cibuildwheel@v2.1.1 @@ -163,18 +160,9 @@ jobs: # Read about options here https://cibuildwheel.readthedocs.io/en/stable/options env: CIBW_ARCHS: arm64 + # This allows substitution within setup.py PACKAGE_NAME: psycopg2-binary - - name: Some tests - run: | - git diff - ls -l dist - pip install psycopg2-binary --no-index -f dist/ - python -c "import psycopg2; print(psycopg2.__version__)" - python -c "import psycopg2; print(psycopg2.__libpq_version__)" - python -c "import psycopg2; print(psycopg2.extensions.libpq_version())" - python -c "import tests; tests.unittest.main(defaultTest='tests.test_suite')" - - name: Upload artifacts uses: actions/upload-artifact@v2 with: From f3c696d22e454600727f55087f55dcfa3e2d16bf Mon Sep 17 00:00:00 2001 From: Armen Zambrano G Date: Wed, 8 Sep 2021 15:19:17 -0400 Subject: [PATCH 08/33] Script to test wheel on Apple Sillicon --- scripts/build/test_macos_arm64.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100755 scripts/build/test_macos_arm64.sh diff --git a/scripts/build/test_macos_arm64.sh b/scripts/build/test_macos_arm64.sh new file mode 100755 index 000000000..fdfe0be77 --- /dev/null +++ b/scripts/build/test_macos_arm64.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +# Test macos arm64 wheel packages from Github actions. +# It only makes sense to run this script from an Apple Silicon device +# +# From Github's Actions tab, choose the "Build packages" run and +# look for artifacts at the bottom. Download "packages_macos_arm64" +# and call this script with the path to it +set -euo pipefail +set -x + +url= \ No newline at end of file From 6d99aa2f29a240c3ba110a33644fdb6af6ed87c8 Mon Sep 17 00:00:00 2001 From: Armen Zambrano G Date: Thu, 9 Sep 2021 09:20:37 -0400 Subject: [PATCH 09/33] Upload repaired_wheels --- .github/workflows/packages.yml | 21 +++++++++++++-------- scripts/build/build_macos.sh | 10 +++++++--- scripts/build/test_macos_arm64.sh | 13 ++++++++++++- 3 files changed, 32 insertions(+), 12 deletions(-) diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index c8850efb0..cbba2288c 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -140,32 +140,37 @@ jobs: # This only uploads the arm64 wheels for Apple M1 Silicon usage build-macos-arm64: - runs-on: ${{ matrix.os }} + runs-on: macos-11.0 strategy: fail-fast: false - matrix: - os: [macos-10.15, macos-11.0] - name: "build-${{ matrix.os }} - arm64" + name: "build-macos-11 - arm64" steps: - uses: actions/checkout@v2 - name: Install pre-requisites + # This skips updating brew when installing a package + env: HOMEBREW_NO_AUTO_UPDATE=1 run: | brew install postgresql@13 brew services start postgresql # This builds all possible Python versions (currently, 3.8 to 3.10) - uses: pypa/cibuildwheel@v2.1.1 - with: - output-dir: dist # Read about options here https://cibuildwheel.readthedocs.io/en/stable/options env: CIBW_ARCHS: arm64 + # This changes the default in order to place the wheel in "repaired_wheels" + CIBW_REPAIR_WHEEL_COMMAND: delocate-listdeps {wheel} && delocate-wheel --require-archs {delocate_archs} -w repaired_wheels {wheel} # This allows substitution within setup.py PACKAGE_NAME: psycopg2-binary + + - run: | + ls -l + ls -l dist + ls -l repaired_wheels - name: Upload artifacts uses: actions/upload-artifact@v2 with: - name: packages_${{ matrix.os }}_arm64 + name: packages_macos_arm64 path: | - dist/*macosx_11_0_arm64.whl + repaired_wheels/*.whl diff --git a/scripts/build/build_macos.sh b/scripts/build/build_macos.sh index d2e1631f6..b21aa8ad0 100755 --- a/scripts/build/build_macos.sh +++ b/scripts/build/build_macos.sh @@ -33,7 +33,9 @@ distdir="${prjdir}/dist/psycopg2-$version" mkdir -p "$distdir" # Install required python packages -pip install -U pip wheel delocate +pip install -U pip wheel +# Running it after installing wheel will save building the wheel +pip install -U delocate # Build the wheels wheeldir="${prjdir}/wheels" @@ -46,8 +48,10 @@ if [[ -z "${LIBPQ:-}" ]]; then fi delocate-wheel ${wheeldir}/*.whl -# https://github.com/MacPython/wiki/wiki/Spinning-wheels#question-will-pip-give-me-a-broken-wheel -delocate-addplat --rm-orig -x 10_9 -x 10_10 ${wheeldir}/*.whl +if [[ $(uname -m) != 'arm64' ]]; then + # https://github.com/MacPython/wiki/wiki/Spinning-wheels#question-will-pip-give-me-a-broken-wheel + delocate-addplat --rm-orig -x 10_9 -x 10_10 ${wheeldir}/*.whl +fi cp ${wheeldir}/*.whl ${distdir} # kill the libpq to make sure tests don't depend on it diff --git a/scripts/build/test_macos_arm64.sh b/scripts/build/test_macos_arm64.sh index fdfe0be77..e327bd756 100755 --- a/scripts/build/test_macos_arm64.sh +++ b/scripts/build/test_macos_arm64.sh @@ -9,4 +9,15 @@ set -euo pipefail set -x -url= \ No newline at end of file +tempdir=$(mktemp -d /tmp/psycopg2.$$) +venv="$tempdir/venv" +unzip "$1" -d "$tempdir" + +# XXX: It would be ideal to run this script against various Python versions +python3 -m venv "$venv" +source "$venv/bin/activate" +pip install psycopg2-binary --no-index -f dist +python -c "import psycopg2; print(psycopg2.__version__)" +python -c "import psycopg2; print(psycopg2.__libpq_version__)" +python -c "import psycopg2; print(psycopg2.extensions.libpq_version())" +python -c "import tests; tests.unittest.main(defaultTest='tests.test_suite')" From ec074269f2d8132d766534ad226bb75df02b2245 Mon Sep 17 00:00:00 2001 From: Armen Zambrano G Date: Thu, 9 Sep 2021 09:25:38 -0400 Subject: [PATCH 10/33] Formatting --- .github/workflows/packages.yml | 4 ++-- scripts/build/build_macos.sh | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index cbba2288c..bc27155be 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -148,7 +148,7 @@ jobs: - uses: actions/checkout@v2 - name: Install pre-requisites # This skips updating brew when installing a package - env: HOMEBREW_NO_AUTO_UPDATE=1 + env: HOMEBREW_NO_AUTO_UPDATE=1 run: | brew install postgresql@13 brew services start postgresql @@ -162,7 +162,7 @@ jobs: CIBW_REPAIR_WHEEL_COMMAND: delocate-listdeps {wheel} && delocate-wheel --require-archs {delocate_archs} -w repaired_wheels {wheel} # This allows substitution within setup.py PACKAGE_NAME: psycopg2-binary - + - run: | ls -l ls -l dist diff --git a/scripts/build/build_macos.sh b/scripts/build/build_macos.sh index b21aa8ad0..16c6be9ad 100755 --- a/scripts/build/build_macos.sh +++ b/scripts/build/build_macos.sh @@ -8,8 +8,8 @@ set -euo pipefail set -x -dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -prjdir="$(cd "${dir}/../.." && pwd)" +dir="$(cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)" +prjdir="$( cd "${dir}/../.." && pwd )" brew install gnu-sed postgresql@13 @@ -17,13 +17,13 @@ brew install gnu-sed postgresql@13 brew services start postgresql for i in $(seq 10 -1 0); do - eval pg_isready && break - if [ $i == 0 ]; then + eval pg_isready && break + if [ $i == 0 ]; then echo "PostgreSQL service not ready, giving up" exit 1 - fi - echo "PostgreSQL service not ready, waiting a bit, attempts left: $i" - sleep 5 + fi + echo "PostgreSQL service not ready, waiting a bit, attempts left: $i" + sleep 5 done # Find psycopg version From c551f2e02e4160c39b771e6c37115295e078da57 Mon Sep 17 00:00:00 2001 From: Armen Zambrano G Date: Thu, 9 Sep 2021 09:34:11 -0400 Subject: [PATCH 11/33] More formatting --- scripts/build/build_macos.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/build/build_macos.sh b/scripts/build/build_macos.sh index 16c6be9ad..ccb4b2f08 100755 --- a/scripts/build/build_macos.sh +++ b/scripts/build/build_macos.sh @@ -19,8 +19,8 @@ brew services start postgresql for i in $(seq 10 -1 0); do eval pg_isready && break if [ $i == 0 ]; then - echo "PostgreSQL service not ready, giving up" - exit 1 + echo "PostgreSQL service not ready, giving up" + exit 1 fi echo "PostgreSQL service not ready, waiting a bit, attempts left: $i" sleep 5 From 1f379d0e15356afae6d6a8103532105f1083206d Mon Sep 17 00:00:00 2001 From: Armen Zambrano G Date: Thu, 9 Sep 2021 09:36:54 -0400 Subject: [PATCH 12/33] Fix --- .github/workflows/packages.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index bc27155be..c3b0d1cbb 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -141,14 +141,13 @@ jobs: # This only uploads the arm64 wheels for Apple M1 Silicon usage build-macos-arm64: runs-on: macos-11.0 - strategy: - fail-fast: false name: "build-macos-11 - arm64" steps: - uses: actions/checkout@v2 - name: Install pre-requisites # This skips updating brew when installing a package - env: HOMEBREW_NO_AUTO_UPDATE=1 + env: + HOMEBREW_NO_AUTO_UPDATE: 1 run: | brew install postgresql@13 brew services start postgresql From e4d5059eefb9438c3088a1c7db6d0ea99adcb2e1 Mon Sep 17 00:00:00 2001 From: Armen Zambrano G Date: Thu, 9 Sep 2021 09:46:13 -0400 Subject: [PATCH 13/33] Create directory --- .github/workflows/packages.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index c3b0d1cbb..f8cc2e4ab 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -144,13 +144,13 @@ jobs: name: "build-macos-11 - arm64" steps: - uses: actions/checkout@v2 - - name: Install pre-requisites + - name: Start postgresql service # This skips updating brew when installing a package env: HOMEBREW_NO_AUTO_UPDATE: 1 run: | - brew install postgresql@13 brew services start postgresql + mkdir repaired_wheels # This builds all possible Python versions (currently, 3.8 to 3.10) - uses: pypa/cibuildwheel@v2.1.1 From 618496e0514a15e0e54fd4916de9ccf686cc9c1f Mon Sep 17 00:00:00 2001 From: Armen Zambrano G Date: Thu, 9 Sep 2021 09:53:53 -0400 Subject: [PATCH 14/33] Use absolute path --- .github/workflows/packages.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index f8cc2e4ab..6e13d4663 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -158,7 +158,10 @@ jobs: env: CIBW_ARCHS: arm64 # This changes the default in order to place the wheel in "repaired_wheels" - CIBW_REPAIR_WHEEL_COMMAND: delocate-listdeps {wheel} && delocate-wheel --require-archs {delocate_archs} -w repaired_wheels {wheel} + # The -w argument requires an absolute path + CIBW_REPAIR_WHEEL_COMMAND: | + delocate-listdeps {wheel} && + delocate-wheel --require-archs {delocate_archs} -w ${{ github.workspace}}/repaired_wheels {wheel} # This allows substitution within setup.py PACKAGE_NAME: psycopg2-binary From 2c9c31b49abb956bc006ded9b59786bc201a9358 Mon Sep 17 00:00:00 2001 From: Armen Zambrano G Date: Thu, 9 Sep 2021 10:01:20 -0400 Subject: [PATCH 15/33] Try again --- .github/workflows/packages.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 6e13d4663..67c83acc4 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -141,7 +141,7 @@ jobs: # This only uploads the arm64 wheels for Apple M1 Silicon usage build-macos-arm64: runs-on: macos-11.0 - name: "build-macos-11 - arm64" + name: "build-macos (3.8-3.10, arm64)" steps: - uses: actions/checkout@v2 - name: Start postgresql service @@ -159,9 +159,7 @@ jobs: CIBW_ARCHS: arm64 # This changes the default in order to place the wheel in "repaired_wheels" # The -w argument requires an absolute path - CIBW_REPAIR_WHEEL_COMMAND: | - delocate-listdeps {wheel} && - delocate-wheel --require-archs {delocate_archs} -w ${{ github.workspace}}/repaired_wheels {wheel} + CIBW_REPAIR_WHEEL_COMMAND: "delocate-listdeps {wheel} && delocate-wheel --require-archs {delocate_archs} -w ${{ github.workspace}}/repaired_wheels {wheel}" # This allows substitution within setup.py PACKAGE_NAME: psycopg2-binary From 81764681a02b35445a3e2efef85b2a518ed1b89f Mon Sep 17 00:00:00 2001 From: Armen Zambrano G Date: Thu, 9 Sep 2021 10:09:17 -0400 Subject: [PATCH 16/33] Try again --- .github/workflows/packages.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 67c83acc4..ef161d7e8 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -150,7 +150,6 @@ jobs: HOMEBREW_NO_AUTO_UPDATE: 1 run: | brew services start postgresql - mkdir repaired_wheels # This builds all possible Python versions (currently, 3.8 to 3.10) - uses: pypa/cibuildwheel@v2.1.1 @@ -159,18 +158,17 @@ jobs: CIBW_ARCHS: arm64 # This changes the default in order to place the wheel in "repaired_wheels" # The -w argument requires an absolute path - CIBW_REPAIR_WHEEL_COMMAND: "delocate-listdeps {wheel} && delocate-wheel --require-archs {delocate_archs} -w ${{ github.workspace}}/repaired_wheels {wheel}" + # CIBW_REPAIR_WHEEL_COMMAND: "delocate-listdeps {wheel} && delocate-wheel --require-archs {delocate_archs} -w ${{ github.workspace}}/repaired_wheels {wheel}" # This allows substitution within setup.py PACKAGE_NAME: psycopg2-binary - run: | ls -l ls -l dist - ls -l repaired_wheels + ls -l wheelhouse - name: Upload artifacts uses: actions/upload-artifact@v2 with: name: packages_macos_arm64 - path: | - repaired_wheels/*.whl + path: ./wheelhouse/*.whl From d780f646fb9f1a9084ce745606af5c64ee71493b Mon Sep 17 00:00:00 2001 From: Armen Zambrano G Date: Thu, 9 Sep 2021 10:16:35 -0400 Subject: [PATCH 17/33] Faster --- .github/workflows/packages.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index ef161d7e8..c2f76444e 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -156,15 +156,11 @@ jobs: # Read about options here https://cibuildwheel.readthedocs.io/en/stable/options env: CIBW_ARCHS: arm64 - # This changes the default in order to place the wheel in "repaired_wheels" - # The -w argument requires an absolute path - # CIBW_REPAIR_WHEEL_COMMAND: "delocate-listdeps {wheel} && delocate-wheel --require-archs {delocate_archs} -w ${{ github.workspace}}/repaired_wheels {wheel}" + CIBW_BUILD: cp39-macosx_arm64 # This allows substitution within setup.py PACKAGE_NAME: psycopg2-binary - run: | - ls -l - ls -l dist ls -l wheelhouse - name: Upload artifacts From 32058519a61c8089766c421d1a783a7bd3db446b Mon Sep 17 00:00:00 2001 From: Armen Zambrano G Date: Thu, 9 Sep 2021 10:35:44 -0400 Subject: [PATCH 18/33] Use CIBW_REPAIR again --- .github/workflows/m1.yml | 39 ++++++++++++++++++++++++++++++++++ .github/workflows/packages.yml | 4 ---- 2 files changed, 39 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/m1.yml diff --git a/.github/workflows/m1.yml b/.github/workflows/m1.yml new file mode 100644 index 000000000..c4673e749 --- /dev/null +++ b/.github/workflows/m1.yml @@ -0,0 +1,39 @@ +--- +name: Build packages +on: + - pull_request + + +jobs: + # This only uploads the arm64 wheels for Apple M1 Silicon usage + build-macos-arm64: + runs-on: macos-11.0 + name: "build-macos (3.8-3.10, arm64)" + steps: + - uses: actions/checkout@v2 + - name: Start postgresql service + # This skips updating brew when installing a package + env: + HOMEBREW_NO_AUTO_UPDATE: 1 + run: | + brew services start postgresql + + # This builds all possible Python versions (currently, 3.8 to 3.10) + - uses: pypa/cibuildwheel@v2.1.1 + # Read about options here https://cibuildwheel.readthedocs.io/en/stable/options + env: + CIBW_ARCHS: arm64 + # XXX: temp: faster + CIBW_BUILD: cp39-macosx_arm64 + CIBW_REPAIR_WHEEL_COMMAND: "delocate-listdeps {wheel} && delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel}" + # This allows substitution within setup.py + PACKAGE_NAME: psycopg2-binary + + - run: | + ls -l + + - name: Upload artifacts + uses: actions/upload-artifact@v2 + with: + name: packages_macos_arm64 + path: ./wheelhouse/*.whl diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index c2f76444e..ff3ae21c7 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -156,13 +156,9 @@ jobs: # Read about options here https://cibuildwheel.readthedocs.io/en/stable/options env: CIBW_ARCHS: arm64 - CIBW_BUILD: cp39-macosx_arm64 # This allows substitution within setup.py PACKAGE_NAME: psycopg2-binary - - run: | - ls -l wheelhouse - - name: Upload artifacts uses: actions/upload-artifact@v2 with: From b67480098bac155b5d7a2def4f0df5878b08ac66 Mon Sep 17 00:00:00 2001 From: Armen Zambrano G Date: Thu, 9 Sep 2021 10:39:55 -0400 Subject: [PATCH 19/33] Try with push --- .github/workflows/m1.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/m1.yml b/.github/workflows/m1.yml index c4673e749..30c4cf484 100644 --- a/.github/workflows/m1.yml +++ b/.github/workflows/m1.yml @@ -1,7 +1,7 @@ --- name: Build packages on: - - pull_request + - push jobs: @@ -26,6 +26,8 @@ jobs: # XXX: temp: faster CIBW_BUILD: cp39-macosx_arm64 CIBW_REPAIR_WHEEL_COMMAND: "delocate-listdeps {wheel} && delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel}" + # It silences a warning since we can't test on Apple Silicon runners + CIBW_TEST_SKIP: "*-macosx_arm64" # This allows substitution within setup.py PACKAGE_NAME: psycopg2-binary From 89974ac39a96d749bb51e4fff17ea8418d96e047 Mon Sep 17 00:00:00 2001 From: Armen Zambrano G Date: Thu, 9 Sep 2021 10:40:34 -0400 Subject: [PATCH 20/33] New name --- .github/workflows/m1.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/m1.yml b/.github/workflows/m1.yml index 30c4cf484..4ce96acc3 100644 --- a/.github/workflows/m1.yml +++ b/.github/workflows/m1.yml @@ -1,5 +1,5 @@ --- -name: Build packages +name: Build packages (M1) on: - push From af32975af07eb2f3994806096c65156f0f95f701 Mon Sep 17 00:00:00 2001 From: Armen Zambrano G Date: Thu, 9 Sep 2021 10:48:43 -0400 Subject: [PATCH 21/33] Verbosity --- .github/workflows/m1.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/m1.yml b/.github/workflows/m1.yml index 4ce96acc3..5c12c6061 100644 --- a/.github/workflows/m1.yml +++ b/.github/workflows/m1.yml @@ -25,6 +25,7 @@ jobs: CIBW_ARCHS: arm64 # XXX: temp: faster CIBW_BUILD: cp39-macosx_arm64 + CIBW_BUILD_VERBOSITY: 3 CIBW_REPAIR_WHEEL_COMMAND: "delocate-listdeps {wheel} && delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel}" # It silences a warning since we can't test on Apple Silicon runners CIBW_TEST_SKIP: "*-macosx_arm64" From ad58ed7d4cf48fe15b9b3841e09d52a489301f5e Mon Sep 17 00:00:00 2001 From: Armen Zambrano G Date: Thu, 9 Sep 2021 11:01:20 -0400 Subject: [PATCH 22/33] Use armenzg action --- .github/workflows/m1.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/m1.yml b/.github/workflows/m1.yml index 5c12c6061..8c809df6b 100644 --- a/.github/workflows/m1.yml +++ b/.github/workflows/m1.yml @@ -19,14 +19,12 @@ jobs: brew services start postgresql # This builds all possible Python versions (currently, 3.8 to 3.10) - - uses: pypa/cibuildwheel@v2.1.1 + - uses: armenzg/cibuildwheel # Read about options here https://cibuildwheel.readthedocs.io/en/stable/options env: CIBW_ARCHS: arm64 # XXX: temp: faster CIBW_BUILD: cp39-macosx_arm64 - CIBW_BUILD_VERBOSITY: 3 - CIBW_REPAIR_WHEEL_COMMAND: "delocate-listdeps {wheel} && delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel}" # It silences a warning since we can't test on Apple Silicon runners CIBW_TEST_SKIP: "*-macosx_arm64" # This allows substitution within setup.py From 87d4afc1bbf3fdb06d168739b86837b589cecb92 Mon Sep 17 00:00:00 2001 From: Armen Zambrano G Date: Thu, 9 Sep 2021 11:02:39 -0400 Subject: [PATCH 23/33] Try main --- .github/workflows/m1.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/m1.yml b/.github/workflows/m1.yml index 8c809df6b..102bfb8ef 100644 --- a/.github/workflows/m1.yml +++ b/.github/workflows/m1.yml @@ -19,7 +19,7 @@ jobs: brew services start postgresql # This builds all possible Python versions (currently, 3.8 to 3.10) - - uses: armenzg/cibuildwheel + - uses: armenzg/cibuildwheel@main # Read about options here https://cibuildwheel.readthedocs.io/en/stable/options env: CIBW_ARCHS: arm64 From eba27a50b5ce0288c1964da4b806af318002e918 Mon Sep 17 00:00:00 2001 From: Armen Zambrano G Date: Thu, 9 Sep 2021 11:28:38 -0400 Subject: [PATCH 24/33] Try build script --- .github/workflows/m1.yml | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/.github/workflows/m1.yml b/.github/workflows/m1.yml index 102bfb8ef..276f538db 100644 --- a/.github/workflows/m1.yml +++ b/.github/workflows/m1.yml @@ -38,3 +38,38 @@ jobs: with: name: packages_macos_arm64 path: ./wheelhouse/*.whl + +build-macos: + runs-on: macos-10.15 + strategy: + fail-fast: false + matrix: + python-version: ['3.8'] + + steps: + - name: Checkout repos + uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Build packages + run: ./scripts/build/build_macos.sh + env: + PACKAGE_NAME: psycopg2-binary + PSYCOPG2_TESTDB: postgres + PSYCOPG2_TEST_FAST: 1 + + - run: | + ls -l + ls -l wheels + ls -l dist + + - name: Upload artifacts + uses: actions/upload-artifact@v2 + with: + name: packages_macos + path: | + dist/*/*${{ matrix.platform }}.whl From 9fee3079b5fd255c169edbba44062c9580bb8558 Mon Sep 17 00:00:00 2001 From: Armen Zambrano G Date: Thu, 9 Sep 2021 11:30:21 -0400 Subject: [PATCH 25/33] Try again --- .github/workflows/m1.yml | 2 +- scripts/build/build_macos.sh | 45 +++++++++++++++++++----------------- 2 files changed, 25 insertions(+), 22 deletions(-) diff --git a/.github/workflows/m1.yml b/.github/workflows/m1.yml index 276f538db..d9211f829 100644 --- a/.github/workflows/m1.yml +++ b/.github/workflows/m1.yml @@ -39,7 +39,7 @@ jobs: name: packages_macos_arm64 path: ./wheelhouse/*.whl -build-macos: + build-macos: runs-on: macos-10.15 strategy: fail-fast: false diff --git a/scripts/build/build_macos.sh b/scripts/build/build_macos.sh index ccb4b2f08..b974c867b 100755 --- a/scripts/build/build_macos.sh +++ b/scripts/build/build_macos.sh @@ -8,8 +8,8 @@ set -euo pipefail set -x -dir="$(cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)" -prjdir="$( cd "${dir}/../.." && pwd )" +dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +prjdir="$(cd "${dir}/../.." && pwd)" brew install gnu-sed postgresql@13 @@ -17,13 +17,13 @@ brew install gnu-sed postgresql@13 brew services start postgresql for i in $(seq 10 -1 0); do - eval pg_isready && break - if [ $i == 0 ]; then - echo "PostgreSQL service not ready, giving up" - exit 1 - fi - echo "PostgreSQL service not ready, waiting a bit, attempts left: $i" - sleep 5 + eval pg_isready && break + if [ $i == 0 ]; then + echo "PostgreSQL service not ready, giving up" + exit 1 + fi + echo "PostgreSQL service not ready, waiting a bit, attempts left: $i" + sleep 5 done # Find psycopg version @@ -57,21 +57,24 @@ cp ${wheeldir}/*.whl ${distdir} # kill the libpq to make sure tests don't depend on it mv "$LIBPQ" "${LIBPQ}-bye" -# Install and test the built wheel -pip install ${PACKAGE_NAME:-psycopg2} --no-index -f "$distdir" +# XXX: Move this code into a test script +if [[ $(uname -m) != 'arm64' ]]; then + # Install and test the built wheel + pip install ${PACKAGE_NAME:-psycopg2} --no-index -f "$distdir" -# Print psycopg and libpq versions -python -c "import psycopg2; print(psycopg2.__version__)" -python -c "import psycopg2; print(psycopg2.__libpq_version__)" -python -c "import psycopg2; print(psycopg2.extensions.libpq_version())" + # Print psycopg and libpq versions + python -c "import psycopg2; print(psycopg2.__version__)" + python -c "import psycopg2; print(psycopg2.__libpq_version__)" + python -c "import psycopg2; print(psycopg2.extensions.libpq_version())" -# fail if we are not using the expected libpq library -# Disabled as we just use what's available on the system on macOS -# if [[ "${WANT_LIBPQ:-}" ]]; then -# python -c "import psycopg2, sys; sys.exit(${WANT_LIBPQ} != psycopg2.extensions.libpq_version())" -# fi + # fail if we are not using the expected libpq library + # Disabled as we just use what's available on the system on macOS + # if [[ "${WANT_LIBPQ:-}" ]]; then + # python -c "import psycopg2, sys; sys.exit(${WANT_LIBPQ} != psycopg2.extensions.libpq_version())" + # fi -python -c "import tests; tests.unittest.main(defaultTest='tests.test_suite')" + python -c "import tests; tests.unittest.main(defaultTest='tests.test_suite')" +fi # just because I'm a boy scout mv "${LIBPQ}-bye" "$LIBPQ" From 9fc985b3da5d6f5aa7d1adfd7acb2d68a79cbe76 Mon Sep 17 00:00:00 2001 From: Armen Date: Thu, 9 Sep 2021 16:12:50 -0400 Subject: [PATCH 26/33] Small change --- scripts/build/build_macos.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/build/build_macos.sh b/scripts/build/build_macos.sh index b974c867b..9f47a590d 100755 --- a/scripts/build/build_macos.sh +++ b/scripts/build/build_macos.sh @@ -54,11 +54,11 @@ if [[ $(uname -m) != 'arm64' ]]; then fi cp ${wheeldir}/*.whl ${distdir} -# kill the libpq to make sure tests don't depend on it -mv "$LIBPQ" "${LIBPQ}-bye" - # XXX: Move this code into a test script if [[ $(uname -m) != 'arm64' ]]; then + # kill the libpq to make sure tests don't depend on it + mv "$LIBPQ" "${LIBPQ}-bye" + # Install and test the built wheel pip install ${PACKAGE_NAME:-psycopg2} --no-index -f "$distdir" @@ -74,7 +74,7 @@ if [[ $(uname -m) != 'arm64' ]]; then # fi python -c "import tests; tests.unittest.main(defaultTest='tests.test_suite')" -fi -# just because I'm a boy scout -mv "${LIBPQ}-bye" "$LIBPQ" + # just because I'm a boy scout + mv "${LIBPQ}-bye" "$LIBPQ" +fi From 8adcdcb1f7d41fc6fb3b2a42f33ffc6c9027aacf Mon Sep 17 00:00:00 2001 From: Armen Date: Thu, 9 Sep 2021 16:23:59 -0400 Subject: [PATCH 27/33] One more try --- .github/workflows/m1.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/m1.yml b/.github/workflows/m1.yml index d9211f829..b406c3157 100644 --- a/.github/workflows/m1.yml +++ b/.github/workflows/m1.yml @@ -3,7 +3,6 @@ name: Build packages (M1) on: - push - jobs: # This only uploads the arm64 wheels for Apple M1 Silicon usage build-macos-arm64: @@ -23,7 +22,7 @@ jobs: # Read about options here https://cibuildwheel.readthedocs.io/en/stable/options env: CIBW_ARCHS: arm64 - # XXX: temp: faster + # XXX: temp: only one version to build CIBW_BUILD: cp39-macosx_arm64 # It silences a warning since we can't test on Apple Silicon runners CIBW_TEST_SKIP: "*-macosx_arm64" @@ -32,6 +31,7 @@ jobs: - run: | ls -l + ls -l wheelhouse - name: Upload artifacts uses: actions/upload-artifact@v2 @@ -40,11 +40,11 @@ jobs: path: ./wheelhouse/*.whl build-macos: - runs-on: macos-10.15 + runs-on: macos-11.0 strategy: fail-fast: false matrix: - python-version: ['3.8'] + python-version: "3.8" steps: - name: Checkout repos @@ -61,7 +61,9 @@ jobs: PACKAGE_NAME: psycopg2-binary PSYCOPG2_TESTDB: postgres PSYCOPG2_TEST_FAST: 1 - + ARCHFLAGS: "-arch arm64" + _PYTHON_HOST_PLATFORM: macosx-11.0-arm64 + - run: | ls -l ls -l wheels From 4913603536997bed79cef080d0a5ebf7b32074da Mon Sep 17 00:00:00 2001 From: Armen Date: Thu, 9 Sep 2021 16:26:23 -0400 Subject: [PATCH 28/33] Less changes --- .github/workflows/m1.yml | 1 + .github/workflows/packages.yml | 40 +++++----------------------------- 2 files changed, 6 insertions(+), 35 deletions(-) diff --git a/.github/workflows/m1.yml b/.github/workflows/m1.yml index b406c3157..77da06094 100644 --- a/.github/workflows/m1.yml +++ b/.github/workflows/m1.yml @@ -1,4 +1,5 @@ --- +# This is temporary; We will need to move to packages.yml name: Build packages (M1) on: - push diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index ff3ae21c7..ac2d658d0 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -3,7 +3,6 @@ name: Build packages on: - workflow_dispatch - jobs: build-sdist: strategy: @@ -51,16 +50,15 @@ jobs: --health-timeout 5s --health-retries 5 - build-manylinux: strategy: fail-fast: false matrix: include: - - {tag: manylinux2014, arch: x86_64} - - {tag: manylinux2014, arch: i686} - - {tag: manylinux_2_24, arch: aarch64} - - {tag: manylinux_2_24, arch: ppc64le} + - { tag: manylinux2014, arch: x86_64 } + - { tag: manylinux2014, arch: i686 } + - { tag: manylinux_2_24, arch: aarch64 } + - { tag: manylinux_2_24, arch: ppc64le } runs-on: ubuntu-20.04 steps: @@ -107,13 +105,12 @@ jobs: --health-timeout 5s --health-retries 5 - build-macos: runs-on: macos-10.15 strategy: fail-fast: false matrix: - python-version: ['3.6', '3.7', '3.8', '3.9'] + python-version: ["3.6", "3.7", "3.8", "3.9"] steps: - name: Checkout repos @@ -137,30 +134,3 @@ jobs: name: packages_macos path: | dist/*/*${{ matrix.platform }}.whl - - # This only uploads the arm64 wheels for Apple M1 Silicon usage - build-macos-arm64: - runs-on: macos-11.0 - name: "build-macos (3.8-3.10, arm64)" - steps: - - uses: actions/checkout@v2 - - name: Start postgresql service - # This skips updating brew when installing a package - env: - HOMEBREW_NO_AUTO_UPDATE: 1 - run: | - brew services start postgresql - - # This builds all possible Python versions (currently, 3.8 to 3.10) - - uses: pypa/cibuildwheel@v2.1.1 - # Read about options here https://cibuildwheel.readthedocs.io/en/stable/options - env: - CIBW_ARCHS: arm64 - # This allows substitution within setup.py - PACKAGE_NAME: psycopg2-binary - - - name: Upload artifacts - uses: actions/upload-artifact@v2 - with: - name: packages_macos_arm64 - path: ./wheelhouse/*.whl From de4fe1ea4b55c84e2f86f250acbde410942cef78 Mon Sep 17 00:00:00 2001 From: Armen Date: Thu, 9 Sep 2021 16:33:18 -0400 Subject: [PATCH 29/33] Only Python 3.8 --- .github/workflows/m1.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/m1.yml b/.github/workflows/m1.yml index 77da06094..cac18c696 100644 --- a/.github/workflows/m1.yml +++ b/.github/workflows/m1.yml @@ -42,10 +42,6 @@ jobs: build-macos: runs-on: macos-11.0 - strategy: - fail-fast: false - matrix: - python-version: "3.8" steps: - name: Checkout repos @@ -54,7 +50,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v2 with: - python-version: ${{ matrix.python-version }} + python-version: 3.8 - name: Build packages run: ./scripts/build/build_macos.sh From e8ddb29c2dfb2ddd871c1142a1a5d65825fcbb5e Mon Sep 17 00:00:00 2001 From: Armen Date: Thu, 9 Sep 2021 16:42:03 -0400 Subject: [PATCH 30/33] A tiny bit more --- scripts/build/build_macos.sh | 35 ++++++++++++++++------------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/scripts/build/build_macos.sh b/scripts/build/build_macos.sh index 9f47a590d..453152ac0 100755 --- a/scripts/build/build_macos.sh +++ b/scripts/build/build_macos.sh @@ -54,27 +54,24 @@ if [[ $(uname -m) != 'arm64' ]]; then fi cp ${wheeldir}/*.whl ${distdir} -# XXX: Move this code into a test script -if [[ $(uname -m) != 'arm64' ]]; then - # kill the libpq to make sure tests don't depend on it - mv "$LIBPQ" "${LIBPQ}-bye" +# kill the libpq to make sure tests don't depend on it +[[ -z "${LIBPQ:-}" ]] && mv "$LIBPQ" "${LIBPQ}-bye" - # Install and test the built wheel - pip install ${PACKAGE_NAME:-psycopg2} --no-index -f "$distdir" +# Install and test the built wheel +pip install ${PACKAGE_NAME:-psycopg2} --no-index -f "$distdir" - # Print psycopg and libpq versions - python -c "import psycopg2; print(psycopg2.__version__)" - python -c "import psycopg2; print(psycopg2.__libpq_version__)" - python -c "import psycopg2; print(psycopg2.extensions.libpq_version())" +# Print psycopg and libpq versions +python -c "import psycopg2; print(psycopg2.__version__)" +python -c "import psycopg2; print(psycopg2.__libpq_version__)" +python -c "import psycopg2; print(psycopg2.extensions.libpq_version())" - # fail if we are not using the expected libpq library - # Disabled as we just use what's available on the system on macOS - # if [[ "${WANT_LIBPQ:-}" ]]; then - # python -c "import psycopg2, sys; sys.exit(${WANT_LIBPQ} != psycopg2.extensions.libpq_version())" - # fi +# fail if we are not using the expected libpq library +# Disabled as we just use what's available on the system on macOS +# if [[ "${WANT_LIBPQ:-}" ]]; then +# python -c "import psycopg2, sys; sys.exit(${WANT_LIBPQ} != psycopg2.extensions.libpq_version())" +# fi - python -c "import tests; tests.unittest.main(defaultTest='tests.test_suite')" +python -c "import tests; tests.unittest.main(defaultTest='tests.test_suite')" - # just because I'm a boy scout - mv "${LIBPQ}-bye" "$LIBPQ" -fi +# just because I'm a boy scout +[[ -z "${LIBPQ:-}" ]] && mv "${LIBPQ}-bye" "$LIBPQ" From 57383fb2e570d4c65962029b70dc641350cf1e17 Mon Sep 17 00:00:00 2001 From: Armen Zambrano G Date: Fri, 10 Sep 2021 12:08:34 -0400 Subject: [PATCH 31/33] New script --- .github/workflows/m1.yml | 2 ++ scripts/build/build_macos_arm64.sh | 12 ++++++++++++ 2 files changed, 14 insertions(+) create mode 100755 scripts/build/build_macos_arm64.sh diff --git a/.github/workflows/m1.yml b/.github/workflows/m1.yml index d9211f829..dc357902d 100644 --- a/.github/workflows/m1.yml +++ b/.github/workflows/m1.yml @@ -61,6 +61,8 @@ jobs: PACKAGE_NAME: psycopg2-binary PSYCOPG2_TESTDB: postgres PSYCOPG2_TEST_FAST: 1 + ARCHFLAGS: "-arch arm64" + _PYTHON_HOST_PLATFORM: "macosx-11.0-arm64" - run: | ls -l diff --git a/scripts/build/build_macos_arm64.sh b/scripts/build/build_macos_arm64.sh new file mode 100755 index 000000000..08b6470a4 --- /dev/null +++ b/scripts/build/build_macos_arm64.sh @@ -0,0 +1,12 @@ +#!/bin/bash +# +# Script to generate arm64 wheels on an Apple Silicon host +# +# TODO: File issue on why cross-compiling is not working +python3 -m venv .venv +source .venv/bin/activate +pip install -U pip wheel +pip install cibuildwheel +# You will be prompted for your password in order to install Python under /Applications +PACKAGE_NAME=psycopg2-binary +CIBW_BUILD=cp38-macosx_arm64 cibuildwheel . --output-dir wheelhouse --platform macos 2>&1 From 415eaee5514e810336e4a4d4d68c03feaafd45e8 Mon Sep 17 00:00:00 2001 From: Armen Zambrano G Date: Fri, 10 Sep 2021 12:54:45 -0400 Subject: [PATCH 32/33] WIP --- scripts/build/build_macos_arm64.sh | 72 +++++++++++++++++++++++++++--- 1 file changed, 65 insertions(+), 7 deletions(-) diff --git a/scripts/build/build_macos_arm64.sh b/scripts/build/build_macos_arm64.sh index 08b6470a4..068b2e62f 100755 --- a/scripts/build/build_macos_arm64.sh +++ b/scripts/build/build_macos_arm64.sh @@ -3,10 +3,68 @@ # Script to generate arm64 wheels on an Apple Silicon host # # TODO: File issue on why cross-compiling is not working -python3 -m venv .venv -source .venv/bin/activate -pip install -U pip wheel -pip install cibuildwheel -# You will be prompted for your password in order to install Python under /Applications -PACKAGE_NAME=psycopg2-binary -CIBW_BUILD=cp38-macosx_arm64 cibuildwheel . --output-dir wheelhouse --platform macos 2>&1 +set -euo pipefail + +create-venv() { + python3 -m venv "${1}" + pip install -U pip wheel +} + +tempdir=$(mktemp -d /tmp/psycopg2.$$) + +echo "Everything will be created under ${tempdir}" + +# create-venv "${tempdir}/venv_cibuildwheel" cibuildwheel +# source "${tempdir}/venv_cibuildwheel/bin/activate" +# pip install cibuildwheel +# export PACKAGE_NAME=psycopg2-binary +# # When you see "Repairing wheel" you will see this output when executing in an Apple Silicon +# # /opt/homebrew/Cellar/openssl@1.1/1.1.1l/lib/libcrypto.1.1.dylib +# # /opt/homebrew/Cellar/openssl@1.1/1.1.1l/lib/libssl.1.1.dylib +# # /opt/homebrew/Cellar/postgresql/13.4/lib/libpq.5.13.dylib +# # If you don't specify --output-dir you will be prompted for sudo +# CIBW_BUILD=cp38-macosx_arm64 cibuildwheel . --output-dir "${tempdir}/wheelhouse" --platform macos 2>&1 +# # This package should be close to 2MB rather than 140KB +# du -h wheelhouse/*.whl +# deactivate + +create-venv "${tempdir}/venv_delocate" +source "${tempdir}/venv_delocate/bin/activate" +pip install delocate +export PACKAGE_NAME=psycopg2-binary +# I'm making the steps close to what cibuildwheel does +pip wheel --use-feature=in-tree-build --wheel-dir "${tempdir}/built_wheel" --no-deps . +set -x +delocate-listdeps "${tempdir}/built_wheel/*.whl" +delocate-wheel --require-archs arm64 \ + -w "${tempdir}/repaired_wheel" \ + "${tempdir}/built_wheel/*.whl" +# This should be around 140KB +du -h "${tempdir}/built_wheel/*.whl" || exit 0 +# This package should be close to 2MB rather than 140KB +du -h "${tempdir}/repaired_wheel/*" +deactivate + +# Start the database for testing +# brew services start postgresql + +# for i in $(seq 10 -1 0); do +# eval pg_isready && break +# if [ $i == 0 ]; then +# echo "PostgreSQL service not ready, giving up" +# exit 1 +# fi +# echo "PostgreSQL service not ready, waiting a bit, attempts left: $i" +# sleep 5 +# done +# create-venv "${tempdir}/venv_test_package" +# source "${tempdir}/venv_test_package/bin/activate" +# export PSYCOPG2_TESTDB=postgres +# export PSYCOPG2_TEST_FAST=1 +# # pip install psycopg2-binary --no-index -f "${tempdir}/wheelhouse" +# pip install psycopg2-binary --no-index -f wheelhouse +# python -c "import psycopg2; print(psycopg2.__version__)" +# python -c "import psycopg2; print(psycopg2.__libpq_version__)" +# python -c "import psycopg2; print(psycopg2.extensions.libpq_version())" +# python -c "import tests; tests.unittest.main(defaultTest='tests.test_suite')" +# deactivate From 30cbc683fa66def5a8b25e5d784df86da645c6cf Mon Sep 17 00:00:00 2001 From: Armen Date: Fri, 10 Sep 2021 13:54:47 -0400 Subject: [PATCH 33/33] Changes for Intel --- scripts/build/build_macos_arm64.sh | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/scripts/build/build_macos_arm64.sh b/scripts/build/build_macos_arm64.sh index 068b2e62f..09535125a 100755 --- a/scripts/build/build_macos_arm64.sh +++ b/scripts/build/build_macos_arm64.sh @@ -7,42 +7,51 @@ set -euo pipefail create-venv() { python3 -m venv "${1}" + source "${1}/bin/activate" pip install -U pip wheel } tempdir=$(mktemp -d /tmp/psycopg2.$$) +export PACKAGE_NAME=psycopg2-binary +brew install postgresql echo "Everything will be created under ${tempdir}" +# These are necessary for an Intel machine (from cibuildwheel macos.py code) +export ARCHFLAGS="-arch arm64" +export _PYTHON_HOST_PLATFORM="macosx-11.0-arm64" +export SDKROOT="/Library/Developer/CommandLineTools/SDKs/MacOSX11.sdk" + # create-venv "${tempdir}/venv_cibuildwheel" cibuildwheel # source "${tempdir}/venv_cibuildwheel/bin/activate" # pip install cibuildwheel -# export PACKAGE_NAME=psycopg2-binary # # When you see "Repairing wheel" you will see this output when executing in an Apple Silicon # # /opt/homebrew/Cellar/openssl@1.1/1.1.1l/lib/libcrypto.1.1.dylib # # /opt/homebrew/Cellar/openssl@1.1/1.1.1l/lib/libssl.1.1.dylib # # /opt/homebrew/Cellar/postgresql/13.4/lib/libpq.5.13.dylib # # If you don't specify --output-dir you will be prompted for sudo -# CIBW_BUILD=cp38-macosx_arm64 cibuildwheel . --output-dir "${tempdir}/wheelhouse" --platform macos 2>&1 +# set -x +# CIBW_BUILD=cp38-macosx_arm64 cibuildwheel . --output-dir "${tempdir}/wheelhouse" --platform macos +# set +x # # This package should be close to 2MB rather than 140KB -# du -h wheelhouse/*.whl +# du -h "${tempdir}/wheelhouse/psycopg2_binary-2.9.1-cp38-cp38-macosx_11_0_arm64.whl" # deactivate create-venv "${tempdir}/venv_delocate" source "${tempdir}/venv_delocate/bin/activate" pip install delocate -export PACKAGE_NAME=psycopg2-binary +set -x # I'm making the steps close to what cibuildwheel does pip wheel --use-feature=in-tree-build --wheel-dir "${tempdir}/built_wheel" --no-deps . -set -x -delocate-listdeps "${tempdir}/built_wheel/*.whl" +delocate-listdeps "${tempdir}/built_wheel/psycopg2_binary-2.9.1-cp38-cp38-macosx_11_0_arm64.whl" delocate-wheel --require-archs arm64 \ -w "${tempdir}/repaired_wheel" \ - "${tempdir}/built_wheel/*.whl" + "${tempdir}/built_wheel/psycopg2_binary-2.9.1-cp38-cp38-macosx_11_0_arm64.whl" +set +x # This should be around 140KB -du -h "${tempdir}/built_wheel/*.whl" || exit 0 +du -h "${tempdir}/built_wheel/psycopg2_binary-2.9.1-cp38-cp38-macosx_11_0_arm64.whl" || exit 0 # This package should be close to 2MB rather than 140KB -du -h "${tempdir}/repaired_wheel/*" +du -h "${tempdir}/repaired_wheel/psycopg2_binary-2.9.1-cp38-cp38-macosx_11_0_arm64.whl" deactivate # Start the database for testing