From 8f26959322b906b4545ac2869faa01b0f6abeb65 Mon Sep 17 00:00:00 2001 From: Roy Williams Date: Thu, 25 Feb 2021 14:09:06 -0500 Subject: [PATCH] Make libprotobuf symbols local on OSX @gnossen gave a great overview in https://github.com/grpc/grpc/pull/24992 of the overall problem. If a python process using both protobuf _and_ another native library linking in libprotobuf frequently can cause crashes. This seems to frequently affect tensorflow as well: tensorflow/tensorflow#8394, tensorflow/tensorflow#9525 (comment) tensorflow/tensorflow#24976, tensorflow/tensorflow#35573, https://github.com/tensorflow/tensorflow/blob/v2.0.0/tensorflow/contrib/makefile/rename_protobuf.sh, tensorflow/tensorflow#16104 Testing locally this fixes both crashes when linking in multiple versions of protobuf and fixes `DescriptorPool` clashes as well (e.g. Python and Native code import different versions of the same message). --- python/setup.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/python/setup.py b/python/setup.py index f97ce4804af76..56728fd2e7abb 100755 --- a/python/setup.py +++ b/python/setup.py @@ -192,6 +192,18 @@ def get_option_from_sys_argv(option_str): extra_compile_args = [] + message_extra_link_args = None + api_implementation_link_args = None + if "darwin" in sys.platform: + if sys.version_info[0] == 2: + message_init_symbol = 'init_message' + api_implementation_init_symbol = 'init_api_implementation' + else: + message_init_symbol = 'PyInit__message' + api_implementation_init_symbol = 'PyInit__api_implementation' + message_extra_link_args = ['-Wl,-exported_symbol,_%s' % message_init_symbol] + api_implementation_link_args = ['-Wl,-exported_symbol,_%s' % api_implementation_init_symbol] + if sys.platform != 'win32': extra_compile_args.append('-Wno-write-strings') extra_compile_args.append('-Wno-invalid-offsetof') @@ -242,6 +254,7 @@ def get_option_from_sys_argv(option_str): include_dirs=[".", "../src"], libraries=libraries, extra_objects=extra_objects, + extra_link_args=message_extra_link_args, library_dirs=['../src/.libs'], extra_compile_args=extra_compile_args, ), @@ -249,6 +262,7 @@ def get_option_from_sys_argv(option_str): "google.protobuf.internal._api_implementation", glob.glob('google/protobuf/internal/api_implementation.cc'), extra_compile_args=extra_compile_args + ['-DPYTHON_PROTO2_CPP_IMPL_V2'], + extra_link_args=api_implementation_link_args, ), ]) os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] = 'cpp'