WIP - manylinux2014 policy tests

.travis.yml

@@ -1,11 +1,14 @@
+os: linux
 language: python
-dist: xenial
+
 matrix:
   include:
   - python: "3.5"
   - python: "3.6"
   - python: "3.7"
   - python: "3.8"
+  - language: c
+    arch: arm64
   - python: "3.7"
     env: LINTER=1
 
@@ -17,6 +20,7 @@ notifications:
   email: false
 
 before_install:
+  - if [ "$(uname -m)" == "aarch64" ]; then sudo apt-get install -y python3-pip python3-venv && python3 -m venv ./.venv && source ./.venv/bin/activate; fi
   - pip install --upgrade pip setuptools
 
 install:

test-requirements.txt

@@ -1,7 +1,6 @@
 pytest>=3.4
 pytest-cov
 jsonschema
-numpy
 pypatchelf
 flake8
 pretend

tests/integration/test_bundled_cffi.py

@@ -1,6 +1,9 @@
+import platform
+import pytest
 from auditwheel.wheel_abi import analyze_wheel_abi
 
 
+@pytest.mark.skipif(platform.machine() != 'x86_64', reason='only supported on x86_64')
 def test_analyze_wheel_abi():
     winfo = analyze_wheel_abi('tests/integration/cffi-1.5.0-cp27-none-linux_x86_64.whl')
     external_libs = winfo.external_refs['manylinux1_x86_64']['libs']

tests/integration/test_bundled_pypy_snappy.py

@@ -1,6 +1,9 @@
+import platform
+import pytest
 from auditwheel.wheel_abi import analyze_wheel_abi
 
 
+@pytest.mark.skipif(platform.machine() != 'x86_64', reason='only supported on x86_64')
 def test_analyze_wheel_abi_pypy_cffi():
     winfo = analyze_wheel_abi(
         'tests/integration/python_snappy-0.5.2-pp260-pypy_41-linux_x86_64.whl')

tests/integration/test_manylinux.py

@@ -9,27 +9,36 @@
 import sys
 import logging
 import zipfile
-from auditwheel.policy import get_priority_by_name
+from auditwheel.policy import get_priority_by_name, get_arch_name
 from elftools.elf.elffile import ELFFile
 
 
 logger = logging.getLogger(__name__)
 
 
 ENCODING = 'utf-8'
-MANYLINUX1_IMAGE_ID = 'quay.io/pypa/manylinux1_x86_64'
-MANYLINUX2010_IMAGE_ID = 'quay.io/pypa/manylinux2010_x86_64'
-MANYLINUX_IMAGES = {
-    'manylinux1': MANYLINUX1_IMAGE_ID,
-    'manylinux2010': MANYLINUX2010_IMAGE_ID,
-}
+PLATFORM = get_arch_name()
+MANYLINUX1_IMAGE_ID = 'quay.io/pypa/manylinux1_{}'.format(PLATFORM)
+MANYLINUX2010_IMAGE_ID = 'quay.io/pypa/manylinux2010_{}'.format(PLATFORM)
+MANYLINUX2014_IMAGE_ID = 'quay.io/pypa/manylinux2014_{}:latest'.format(PLATFORM)
+if PLATFORM in {'i686', 'x86_64'}:
+    MANYLINUX_IMAGES = {
+        'manylinux1': MANYLINUX1_IMAGE_ID,
+        'manylinux2010': MANYLINUX2010_IMAGE_ID,
+        'manylinux2014': MANYLINUX2014_IMAGE_ID,
+    }
+else:
+    MANYLINUX_IMAGES = {
+        'manylinux2014': MANYLINUX2014_IMAGE_ID,
+    }
 DOCKER_CONTAINER_NAME = 'auditwheel-test-manylinux'
 PYTHON_MAJ_MIN = [str(i) for i in sys.version_info[:2]]
 PYTHON_ABI = 'cp{0}-cp{0}{1}'.format(''.join(PYTHON_MAJ_MIN), 'm' if sys.version_info.minor < 8 else '')
 PYTHON_IMAGE_ID = 'python:' + '.'.join(PYTHON_MAJ_MIN)
 DEVTOOLSET = {
     'manylinux1': 'devtoolset-2',
     'manylinux2010': 'devtoolset-8',
+    'manylinux2014': 'devtoolset-8',
 }
 PATH_DIRS = [
     '/opt/python/{}/bin'.format(PYTHON_ABI),
@@ -44,7 +53,7 @@
 PATH = {k: ':'.join(PATH_DIRS).format(devtoolset=v)
         for k, v in DEVTOOLSET.items()}
 WHEEL_CACHE_FOLDER = op.expanduser('~/.cache/auditwheel_tests')
-ORIGINAL_NUMPY_WHEEL = 'numpy-1.16.0-{}-linux_x86_64.whl'.format(PYTHON_ABI)
+ORIGINAL_NUMPY_WHEEL = 'numpy-1.16.5-{}-linux_{}.whl'.format(PYTHON_ABI, PLATFORM)
 ORIGINAL_SIX_WHEEL = 'six-1.11.0-py2.py3-none-any.whl'
 
 
@@ -169,9 +178,10 @@ def any_manylinux_container(any_manylinux_img, io_folder):
             env[key] = os.environ[key]
 
     with docker_container_ctx(manylinux_img, io_folder, env) as container:
-        yield policy, container
+        yield '{}_{}'.format(policy, PLATFORM), container
 
 
+@pytest.mark.xfail(condition=(PLATFORM == 'aarch64'), reason='numpy build fails on aarch64', strict=True)
 def test_build_repair_numpy(any_manylinux_container, docker_python, io_folder):
     # Integration test: repair numpy built from scratch
 
@@ -190,7 +200,7 @@ def test_build_repair_numpy(any_manylinux_container, docker_python, io_folder):
         # This part of the build is independent of the auditwheel code-base
         # so it's safe to put it in cache.
         docker_exec(manylinux_ctr,
-            'pip wheel -w /io --no-binary=:all: numpy==1.16.0')
+            'pip wheel -w /io --no-binary=:all: numpy==1.16.5')
         os.makedirs(op.join(WHEEL_CACHE_FOLDER, policy), exist_ok=True)
         shutil.copy2(op.join(io_folder, ORIGINAL_NUMPY_WHEEL),
                      op.join(WHEEL_CACHE_FOLDER, policy, ORIGINAL_NUMPY_WHEEL))
@@ -201,19 +211,19 @@ def test_build_repair_numpy(any_manylinux_container, docker_python, io_folder):
 
     # Repair the wheel using the manylinux container
     repair_command = (
-        'auditwheel repair --plat {policy}_x86_64 -w /io /io/{orig_wheel}'
+        'auditwheel repair --plat {policy} -w /io /io/{orig_wheel}'
     ).format(policy=policy, orig_wheel=orig_wheel)
     docker_exec(manylinux_ctr, repair_command)
     filenames = os.listdir(io_folder)
 
     assert len(filenames) == 2
     repaired_wheels = [fn for fn in filenames if 'manylinux' in fn]
-    assert repaired_wheels == ['numpy-1.16.0-{}-{}_x86_64.whl'.format(PYTHON_ABI, policy)]
+    assert repaired_wheels == ['numpy-1.16.5-{}-{}.whl'.format(PYTHON_ABI, policy)]
     repaired_wheel = repaired_wheels[0]
     output = docker_exec(manylinux_ctr, 'auditwheel show /io/' + repaired_wheel)
     assert (
-        'numpy-1.16.0-{abi}-{policy}_x86_64.whl is consistent'
-        ' with the following platform tag: "{policy}_x86_64"'
+        'numpy-1.16.5-{abi}-{policy}.whl is consistent'
+        ' with the following platform tag: "{policy}"'
     ).format(abi=PYTHON_ABI, policy=policy) in output.replace('\n', ' ')
 
     # Check that the repaired numpy wheel can be installed and executed
@@ -253,20 +263,20 @@ def test_build_wheel_with_binary_executable(any_manylinux_container, docker_pyth
 
     # Repair the wheel using the appropriate manylinux container
     repair_command = (
-        'auditwheel repair --plat {policy}_x86_64 -w /io /io/{orig_wheel}'
+        'auditwheel repair --plat {policy} -w /io /io/{orig_wheel}'
     ).format(policy=policy, orig_wheel=orig_wheel)
     docker_exec(manylinux_ctr, repair_command)
     filenames = os.listdir(io_folder)
     assert len(filenames) == 2
     repaired_wheels = [fn for fn in filenames if policy in fn]
     # Wheel picks up newer symbols when built in manylinux2010
-    expected_wheel_name = 'testpackage-0.0.1-py3-none-%s_x86_64.whl' % policy
+    expected_wheel_name = 'testpackage-0.0.1-py3-none-{}.whl'.format(policy)
     assert repaired_wheels == [expected_wheel_name]
     repaired_wheel = repaired_wheels[0]
     output = docker_exec(manylinux_ctr, 'auditwheel show /io/' + repaired_wheel)
     assert (
-        'testpackage-0.0.1-py3-none-{policy}_x86_64.whl is consistent'
-        ' with the following platform tag: "{policy}_x86_64"'
+        'testpackage-0.0.1-py3-none-{policy}.whl is consistent'
+        ' with the following platform tag: "{policy}"'
     ).format(policy=policy) in output.replace('\n', ' ')
 
     docker_exec(docker_python, 'pip install /io/' + repaired_wheel)
@@ -306,12 +316,12 @@ def test_build_wheel_with_image_dependencies(with_dependency, any_manylinux_cont
 
     repair_command = \
         'LD_LIBRARY_PATH=/auditwheel_src/tests/integration/testdependencies:$LD_LIBRARY_PATH '\
-        'auditwheel -v repair --plat {policy}_x86_64 -w /io /io/{orig_wheel}'
+        'auditwheel -v repair --plat {policy} -w /io /io/{orig_wheel}'
 
-    policy_priority = get_priority_by_name(policy + '_x86_64')
+    policy_priority = get_priority_by_name(policy)
     older_policies = \
         [p for p in MANYLINUX_IMAGES.keys()
-         if policy_priority < get_priority_by_name(p + '_x86_64')]
+         if policy_priority < get_priority_by_name('{}_{}'.format(p, PLATFORM))]
     for target_policy in older_policies:
         # we shall fail to repair the wheel when targeting an older policy than
         # the one matching the image
@@ -330,13 +340,13 @@ def test_build_wheel_with_image_dependencies(with_dependency, any_manylinux_cont
     assert len(filenames) == 2
     repaired_wheels = [fn for fn in filenames if policy in fn]
     expected_wheel_name = \
-        'testdependencies-0.0.1-{}-{}_x86_64.whl'.format(PYTHON_ABI, policy)
+        'testdependencies-0.0.1-{}-{}.whl'.format(PYTHON_ABI, policy)
     assert repaired_wheels == [expected_wheel_name]
     repaired_wheel = repaired_wheels[0]
     output = docker_exec(manylinux_ctr, 'auditwheel show /io/' + repaired_wheel)
     assert (
-        'testdependencies-0.0.1-{abi}-{policy}_x86_64.whl is consistent'
-        ' with the following platform tag: "{policy}_x86_64"'
+        'testdependencies-0.0.1-{abi}-{policy}.whl is consistent'
+        ' with the following platform tag: "{policy}"'
     ).format(abi=PYTHON_ABI, policy=policy) in output.replace('\n', ' ')
 
     # check the original wheel with a dependency was not compliant
@@ -345,12 +355,12 @@ def test_build_wheel_with_image_dependencies(with_dependency, any_manylinux_cont
     if with_dependency == '1':
         assert (
             '{orig_wheel} is consistent with the following platform tag: '
-            '"linux_x86_64"'
-        ).format(orig_wheel=orig_wheel) in output.replace('\n', ' ')
+            '"linux_{platform}"'
+        ).format(orig_wheel=orig_wheel, platform=PLATFORM) in output.replace('\n', ' ')
     else:
         assert (
             '{orig_wheel} is consistent with the following platform tag: '
-            '"{policy}_x86_64"'
+            '"{policy}"'
         ).format(orig_wheel=orig_wheel, policy=policy) in output.replace('\n', ' ')
 
     docker_exec(docker_python, 'pip install /io/' + repaired_wheel)
@@ -381,18 +391,19 @@ def test_build_repair_pure_wheel(any_manylinux_container, io_folder):
 
     # Repair the wheel using the manylinux container
     repair_command = (
-        'auditwheel repair --plat {policy}_x86_64 -w /io /io/{orig_wheel}'
+        'auditwheel repair --plat {policy} -w /io /io/{orig_wheel}'
     ).format(policy=policy, orig_wheel=orig_wheel)
     docker_exec(manylinux_ctr, repair_command)
     filenames = os.listdir(io_folder)
     assert len(filenames) == 1  # no new wheels
     assert filenames == [ORIGINAL_SIX_WHEEL]
 
     output = docker_exec(manylinux_ctr, 'auditwheel show /io/' + filenames[0])
+    expected = 'manylinux1' if PLATFORM in {'x86_64', 'i686'} else 'manylinux2014'
     assert ''.join([
         ORIGINAL_SIX_WHEEL,
         ' is consistent with the following platform tag: ',
-        '"manylinux1_x86_64".  ',
+        '"{}_{}".  '.format(expected, PLATFORM),
         'The wheel references no external versioned symbols from system- ',
         'provided shared libraries.  ',
         'The wheel requires no external shared libraries! :)',
@@ -429,13 +440,13 @@ def test_build_wheel_depending_on_library_with_rpath(any_manylinux_container, do
         tags = {t.entry.d_tag for t in dynamic.iter_tags()}
         assert "DT_{}".format(dtag.upper()) in tags
     filenames = os.listdir(io_folder)
-    assert filenames == ['testrpath-0.0.1-{}-linux_x86_64.whl'.format(PYTHON_ABI)]
+    assert filenames == ['testrpath-0.0.1-{}-linux_{}.whl'.format(PYTHON_ABI, PLATFORM)]
     orig_wheel = filenames[0]
     assert 'manylinux' not in orig_wheel
 
     # Repair the wheel using the appropriate manylinux container
     repair_command = (
-        'auditwheel repair --plat {policy}_x86_64 -w /io /io/{orig_wheel}'
+        'auditwheel repair --plat {policy} -w /io /io/{orig_wheel}'
     ).format(policy=policy, orig_wheel=orig_wheel)
     docker_exec(
         manylinux_ctr,
@@ -445,15 +456,19 @@ def test_build_wheel_depending_on_library_with_rpath(any_manylinux_container, do
     repaired_wheels = [fn for fn in filenames if policy in fn]
     # Wheel picks up newer symbols when built in manylinux2010
     expected_wheel_name = (
-        'testrpath-0.0.1-{abi}-{policy}_x86_64.whl'
+        'testrpath-0.0.1-{abi}-{policy}.whl'
     ).format(abi=PYTHON_ABI, policy=policy)
     assert expected_wheel_name in repaired_wheels
     repaired_wheel = expected_wheel_name
     output = docker_exec(manylinux_ctr, 'auditwheel show /io/' + repaired_wheel)
+    if PLATFORM in {'x86_64', 'i686'}:
+        expect = 'manylinux1_{}'.format(PLATFORM)
+    else:
+        expect = 'manylinux2014_{}'.format(PLATFORM)
     assert (
-        'testrpath-0.0.1-{abi}-{policy}_x86_64.whl is consistent'
-        ' with the following platform tag: "manylinux1_x86_64"'
-    ).format(abi=PYTHON_ABI, policy=policy) in output.replace('\n', ' ')
+        'testrpath-0.0.1-{abi}-{policy}.whl is consistent'
+        ' with the following platform tag: "{expect}"'
+    ).format(abi=PYTHON_ABI, policy=policy, expect=expect) in output.replace('\n', ' ')
 
     docker_exec(docker_python, 'pip install /io/' + repaired_wheel)
     output = docker_exec(

tests/integration/test_pyfpe.py

@@ -1,6 +1,9 @@
+import platform
+import pytest
 from auditwheel.wheel_abi import analyze_wheel_abi
 
 
+@pytest.mark.skipif(platform.machine() != 'x86_64', reason='only supported on x86_64')
 def test_analyze_wheel_abi():
     winfo = analyze_wheel_abi('tests/integration/fpewheel-0.0.0-cp35-cp35m-linux_x86_64.whl')
     assert winfo.sym_tag == 'manylinux1_x86_64'  # for external symbols, it could get manylinux1

tests/integration/testdependencies/dependency.c

@@ -3,7 +3,9 @@
 
 int dep_run()
 {
-#if defined(__GLIBC_PREREQ) && __GLIBC_PREREQ(2, 10)
+#if defined(__GLIBC_PREREQ) && __GLIBC_PREREQ(2, 17)
+    return (int)secure_getenv("NON_EXISTING_ENV_VARIABLE");
+#elif defined(__GLIBC_PREREQ) && __GLIBC_PREREQ(2, 10)
     return malloc_info(0, stdout);
 #else
     return 0;

tests/integration/testdependencies/testdependencies.c

@@ -15,6 +15,8 @@ run(PyObject *self, PyObject *args)
 
 #ifdef WITH_DEPENDENCY
     res = dep_run();
+#elif defined(__GLIBC_PREREQ) && __GLIBC_PREREQ(2, 17)
+    res = (int)secure_getenv("NON_EXISTING_ENV_VARIABLE");
 #elif defined(__GLIBC_PREREQ) && __GLIBC_PREREQ(2, 10)
     res = malloc_info(0, stdout);
 #else

tests/unit/test_policy.py

@@ -35,14 +35,22 @@ def test_64bits_arch_name(machine_mock, reported_arch, expected_arch):
 class TestPolicyAccess:
 
     def test_get_by_priority(self):
-        assert get_policy_name(100) == 'manylinux1_x86_64'
-        assert get_policy_name(0) == 'linux_x86_64'
+        _arch = get_arch_name()
+        assert get_policy_name(80) == 'manylinux2014_{}'.format(_arch)
+        if _arch in {'x86_64', 'i686'}:
+            assert get_policy_name(90) == 'manylinux2010_{}'.format(_arch)
+            assert get_policy_name(100) == 'manylinux1_{}'.format(_arch)
+        assert get_policy_name(0) == 'linux_{}'.format(_arch)
 
     def test_get_by_priority_missing(self):
         assert get_policy_name(101) is None
 
     def test_get_by_name(self):
-        assert get_priority_by_name("manylinux1_x86_64") == 100
+        _arch = get_arch_name()
+        assert get_priority_by_name("manylinux2014_{}".format(_arch)) == 80
+        if _arch in {'x86_64', 'i686'}:
+            assert get_priority_by_name("manylinux2010_{}".format(_arch)) == 90
+            assert get_priority_by_name("manylinux1_{}".format(_arch)) == 100
 
     def test_get_by_name_missing(self):
         assert get_priority_by_name("nosuchpolicy") is None