-
Notifications
You must be signed in to change notification settings - Fork 3k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Use manylinux2010 image to build linux python wheels (#1282)
* Update cuda for python wheels * Update cuda for python wheels * Update cuda for python wheels * Update azure-pipelines-py-packaging.yml * Update to cuda 10 * Only test win gpu * Update cuda for python wheels * Use manylinux2010 image to build linux python wheels Allow wheels built to truly be compliant with a manylinux policy
- Loading branch information
Showing
17 changed files
with
327 additions
and
145 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,10 @@ | ||
| #------------------------------------------------------------------------- | ||
| # Copyright (c) Microsoft Corporation. All rights reserved. | ||
| # Licensed under the MIT License. | ||
| #-------------------------------------------------------------------------- | ||
|
|
||
| # This file can be modified by setup.py when building a manylinux2010 wheel | ||
| # When modified, it will preload some libraries needed for the python C extension | ||
| # Do not remove or move the following comment | ||
|
|
||
| # LD_PRELOAD_BEGIN_MARK |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -3,9 +3,14 @@ | |
| # Licensed under the MIT License. | ||
| #-------------------------------------------------------------------------- | ||
|
|
||
| from setuptools import setup, find_packages | ||
| from os import path, getcwd | ||
| from setuptools import setup, find_packages, Extension | ||
| from distutils import log as logger | ||
| from distutils.command.build_ext import build_ext as _build_ext | ||
| from glob import glob | ||
| from os import path, getcwd, environ, remove | ||
| from shutil import copyfile | ||
| import platform | ||
| import subprocess | ||
| import sys | ||
| import datetime | ||
|
|
||
|
|
@@ -38,12 +43,69 @@ | |
| nightly_build = True | ||
| sys.argv.remove('--nightly_build') | ||
|
|
||
| is_manylinux2010 = False | ||
| if environ.get('AUDITWHEEL_PLAT', None) == 'manylinux2010_x86_64': | ||
| is_manylinux2010 = True | ||
|
|
||
|
|
||
| class build_ext(_build_ext): | ||
| def build_extension(self, ext): | ||
| dest_file = self.get_ext_fullpath(ext.name) | ||
| logger.info('copying %s -> %s', ext.sources[0], dest_file) | ||
| copyfile(ext.sources[0], dest_file) | ||
|
|
||
|
|
||
| try: | ||
| from wheel.bdist_wheel import bdist_wheel as _bdist_wheel | ||
| class bdist_wheel(_bdist_wheel): | ||
| def finalize_options(self): | ||
| _bdist_wheel.finalize_options(self) | ||
| self.root_is_pure = False | ||
| if not is_manylinux2010: | ||
| self.root_is_pure = False | ||
|
|
||
| def _rewrite_ld_preload(self, to_preload): | ||
| with open('onnxruntime/capi/_ld_preload.py', 'rt') as f: | ||
| ld_preload = f.read().splitlines() | ||
| with open('onnxruntime/capi/_ld_preload.py', 'wt') as f: | ||
| for line in ld_preload: | ||
| f.write(line) | ||
| f.write('\n') | ||
| if 'LD_PRELOAD_BEGIN_MARK' in line: | ||
| break | ||
| if len(to_preload) > 0: | ||
| f.write('from ctypes import CDLL, RTLD_GLOBAL\n') | ||
| for library in to_preload: | ||
| f.write('_{} = CDLL("{}", mode=RTLD_GLOBAL)\n'.format(library.split('.')[0], library)) | ||
|
|
||
| def run(self): | ||
| if is_manylinux2010: | ||
| source = 'onnxruntime/capi/onnxruntime_pybind11_state.so' | ||
| dest = 'onnxruntime/capi/onnxruntime_pybind11_state_manylinux2010.so' | ||
| logger.info('copying %s -> %s', source, dest) | ||
| copyfile(source, dest) | ||
| result = subprocess.run(['patchelf', '--print-needed', dest], check=True, stdout=subprocess.PIPE, universal_newlines=True) | ||
| cuda_dependencies = ['libcublas.so', 'libcudnn.so', 'libcudart.so'] | ||
| to_preload = [] | ||
| args = ['patchelf', '--debug'] | ||
| for line in result.stdout.split('\n'): | ||
| for dependency in cuda_dependencies: | ||
| if dependency in line: | ||
| to_preload.append(line) | ||
| args.extend(['--remove-needed', line]) | ||
| args.append(dest) | ||
| if len(to_preload) > 0: | ||
| subprocess.run(args, check=True, stdout=subprocess.PIPE) | ||
| self._rewrite_ld_preload(to_preload) | ||
| _bdist_wheel.run(self) | ||
| if is_manylinux2010: | ||
| file = glob(path.join(self.dist_dir, '*linux*.whl'))[0] | ||
| logger.info('repairing %s for manylinux2010', file) | ||
| try: | ||
| subprocess.run(['auditwheel', 'repair', '--plat', 'manylinux2010_x86_64', '-w', self.dist_dir, file], check=True, stdout=subprocess.PIPE) | ||
| finally: | ||
| logger.info('removing %s', file) | ||
| remove(file) | ||
|
|
||
| except ImportError: | ||
| bdist_wheel = None | ||
|
|
||
|
|
@@ -57,7 +119,18 @@ def finalize_options(self): | |
| else: | ||
| libs = ['onnxruntime_pybind11_state.pyd', 'mkldnn.dll', 'mklml.dll', 'libiomp5md.dll'] | ||
|
|
||
| data = [path.join('capi', x) for x in libs if path.isfile(path.join('onnxruntime', 'capi', x))] | ||
| if is_manylinux2010: | ||
| data = [] | ||
| ext_modules = [ | ||
| Extension( | ||
| 'onnxruntime.capi.onnxruntime_pybind11_state', | ||
| ['onnxruntime/capi/onnxruntime_pybind11_state_manylinux2010.so'], | ||
| ), | ||
| ] | ||
| else: | ||
| data = [path.join('capi', x) for x in libs if path.isfile(path.join('onnxruntime', 'capi', x))] | ||
| ext_modules = [] | ||
|
|
||
|
|
||
| python_modules_list = list() | ||
| if '--use_openvino' in sys.argv: | ||
|
|
@@ -98,14 +171,15 @@ def finalize_options(self): | |
| long_description=long_description, | ||
| author='Microsoft Corporation', | ||
| author_email='[email protected]', | ||
| cmdclass={'bdist_wheel': bdist_wheel}, | ||
| cmdclass={'bdist_wheel': bdist_wheel, 'build_ext': build_ext}, | ||
| license="MIT License", | ||
| packages=['onnxruntime', | ||
| 'onnxruntime.backend', | ||
| 'onnxruntime.capi', | ||
| 'onnxruntime.datasets', | ||
| 'onnxruntime.tools', | ||
| ], | ||
| ext_modules=ext_modules, | ||
| package_data={ | ||
| 'onnxruntime': data + examples + extra, | ||
| }, | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.