[tensorflow] fix macOS build errors caused by numpy and bazel upgrades (

#19363) * Revert "incorporate changes from microsoft:master" * Revert "Revert "incorporate changes from microsoft:master"" * issue targeted in newer numpy version * work-around for broken numpy libs on macOS * temporarily add debug code to analyze CI failures * again temporary debug code * fix linkage command on macOS (broken by Bazel upgrade) * fix regex for macOS linker command * remove debug code, bump version * x-add-version seems to require a separate commit... * Fix misspelled "acceleration". * x-add-version Co-authored-by: jgehw <[email protected]> Co-authored-by: Billy Robert O'Neal III <[email protected]>
microsoft · Sep 1, 2021 · 7734e8b · 7734e8b
1 parent adc1caf
commit 7734e8b
Show file tree

Hide file tree

Showing 10 changed files with 57 additions and 54 deletions.
diff --git a/ports/tensorflow-common/convert_lib_params_macos.py b/ports/tensorflow-common/convert_lib_params_macos.py
@@ -0,0 +1,18 @@
+import sys
+
+version = sys.argv[1]
+lib_suffix = "" if len(sys.argv) < 3 else sys.argv[2]
+
+with open(f"libtensorflow{lib_suffix}.{version}.a-2.params", "w") as f_out:
+    parts = []
+    with open(f"libtensorflow_framework.{version}.dylib-2.params", "r") as f_in:
+        for line in f_in:
+            if line.startswith("-Wl,-force_load,"):
+                f_out.write(line[16:])
+                parts.append(line[16:])
+    parts = set(parts)
+    with open(f"libtensorflow{lib_suffix}.{version}.dylib-2.params", "r") as f_in:
+        for line in f_in:
+            if line.startswith("-Wl,-force_load,"):
+                if line[16:] not in parts:
+                    f_out.write(line[16:])
diff --git a/ports/tensorflow-common/generate_static_link_cmd_linux.py b/ports/tensorflow-common/generate_static_link_cmd_linux.py
@@ -4,7 +4,7 @@
 lib_suffix = "" if len(sys.argv) < 3 else sys.argv[2]
 with open(sys.argv[1], "r") as f_in:
     with open("static_link.sh", "w") as f_out:
-        p_cd = re.compile("^\\((cd .*) && \\\\$")
+        p_cd = re.compile(r"^\((cd .*) && \\$")
         p_linker = re.compile(fr"^\s*(.+)gcc.+(@bazel-out\S+libtensorflow{lib_suffix}\.so\.\d\.\d\.\d-2\.params).*")
         f_out.write("#!/bin/bash\n# note: ar/binutils version 2.27 required to support output files > 4GB\n")
         env = []

diff --git a/ports/tensorflow-common/generate_static_link_cmd_macos.py b/ports/tensorflow-common/generate_static_link_cmd_macos.py
@@ -4,44 +4,20 @@
 lib_suffix = "" if len(sys.argv) < 3 else sys.argv[2]
 with open(sys.argv[1], "r") as f_in:
     with open("static_link.sh", "w") as f_out:
-        p_cd = re.compile("^\\((cd .*) && \\\\$")
-        p_linker1 = re.compile(fr"^.*cc_wrapper.sh.+-shared.+-o (bazel-out\S+libtensorflow{lib_suffix}\.\d\.\d\.\d\.dylib)")
-        p_linker2 = re.compile("^.*cc_wrapper.sh.+-shared.+-o (bazel-out\\S+libtensorflow_framework\\.\\d\\.\\d\\.\\d\\.dylib)")
+        p_cd = re.compile(r"^\((cd .*) && \\$")
+        p_linker = re.compile(fr"^\s*.+cc_wrapper.sh.+(@bazel-out\S+libtensorflow{lib_suffix}\.\d\.\d\.\d\.dylib-2\.params).*")
         f_out.write("#!/bin/bash\n# note: ar/binutils version 2.27 required to support output files > 4GB\n")
         env = []
-        parts = None
         for line in f_in:
             if line.startswith("(cd"):
                 # new command, reset
                 env = [line]
             else:
-                m1 = p_linker1.match(line)
-                m2 = p_linker2.match(line)
+                m1 = p_linker.match(line)
                 if m1:
-                    tokens = line.split()
-                    if parts is None:
-                        parts = [t[16:] for t in tokens if t.startswith("-Wl,-force_load,")]
-                    else:
-                        m = p_cd.match(env[0])
-                        f_out.write(m.group(1) + "\n")
-                        tmp = [t[16:] for t in tokens if t.startswith("-Wl,-force_load,")]
-                        old = set(parts)
-                        parts += [t for t in tmp if t not in old]
-                        line = f"libtool -static -o {m1.group(1).replace('.dylib', '.a')} {' '.join(parts)}\n"
-                        f_out.write(line)
-                        break
-                elif m2 and len(env) > 6:
-                    tokens = line.split()
-                    if parts is None:
-                        parts = [t[16:] for t in tokens if t.startswith("-Wl,-force_load,")]
-                    else:
-                        m = p_cd.match(env[0])
-                        f_out.write(m.group(1) + "\n")
-                        tmp = [t[16:] for t in tokens if t.startswith("-Wl,-force_load,")]
-                        old = set(parts)
-                        parts += [t for t in tmp if t not in old]
-                        line = f"libtool -static -o {m2.group(1).replace('_framework', lib_suffix).replace('.dylib', '.a')} {' '.join(parts)}\n"
-                        f_out.write(line)
-                        break
+                    m2 = p_cd.match(env[0])
+                    f_out.write(m2.group(1) + "\n")
+                    line = f'"/usr/bin/libtool" -static -o {m1.group(1)[1:-9].replace(".dylib", ".a")} {m1.group(1).replace(".dylib", ".a")}\n'
+                    f_out.write(line)
                 else:
                     env.append(line)
diff --git a/ports/tensorflow-common/generate_static_link_cmd_windows.py b/ports/tensorflow-common/generate_static_link_cmd_windows.py
@@ -5,7 +5,7 @@
 lib_suffix = "" if len(sys.argv) < 3 else sys.argv[2]
 with open(sys.argv[1], "r") as f_in:
     with open("static_link.bat", "w") as f_out:
-        p_setenv = re.compile("^\s*(SET .+=.*)$")
+        p_setenv = re.compile(r"^\s*(SET .+=.*)$")
         p_linker = re.compile(fr".+link\.exe.+tensorflow{lib_suffix}\.dll-2\.params.*")
         env = []
         for line in f_in:

diff --git a/ports/tensorflow-common/portfile.cmake b/ports/tensorflow-common/portfile.cmake
@@ -3,6 +3,7 @@ set(VCPKG_POLICY_EMPTY_INCLUDE_FOLDER enabled)
 set(TENSORFLOW_FILES
     "${CMAKE_CURRENT_LIST_DIR}/change-macros-for-static-lib.patch"
     "${CMAKE_CURRENT_LIST_DIR}/convert_lib_params_linux.py"
+    "${CMAKE_CURRENT_LIST_DIR}/convert_lib_params_macos.py"
     "${CMAKE_CURRENT_LIST_DIR}/convert_lib_params_windows.py"
     "${CMAKE_CURRENT_LIST_DIR}/fix-build-error.patch"
     "${CMAKE_CURRENT_LIST_DIR}/fix-linux-build.patch"

diff --git a/ports/tensorflow-common/tensorflow-common.cmake b/ports/tensorflow-common/tensorflow-common.cmake
@@ -45,7 +45,16 @@ else()
 	get_filename_component(PYTHON3_DIR "${PYTHON3}" DIRECTORY)
 	vcpkg_add_to_path(PREPEND ${PYTHON3_DIR})
 
-	vcpkg_execute_required_process(COMMAND ${PYTHON3} -m pip install --user -U numpy WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR} LOGNAME prerequesits-pip-${TARGET_TRIPLET})
+	if(VCPKG_TARGET_IS_OSX)
+		# acceleration libs currently broken on macOS => force numpy user space reinstall without BLAS/LAPACK/ATLAS
+		# remove this work-around again, i.e. default to "else" branch, once acceleration libs are fixed upstream
+		set(ENV{BLAS} "None")
+		set(ENV{LAPACK} "None")
+		set(ENV{ATLAS} "None")
+		vcpkg_execute_required_process(COMMAND ${PYTHON3} -m pip install --user -U --force-reinstall numpy WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR} LOGNAME prerequesits-pip-${TARGET_TRIPLET})
+	else()
+		vcpkg_execute_required_process(COMMAND ${PYTHON3} -m pip install --user -U numpy WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR} LOGNAME prerequesits-pip-${TARGET_TRIPLET})
+	endif()
 	vcpkg_execute_required_process(COMMAND ${PYTHON3} -c "import site; print(site.getusersitepackages())" WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR} LOGNAME prerequesits-pypath-${TARGET_TRIPLET} OUTPUT_VARIABLE PYTHON_LIB_PATH)
 endif()
 set(ENV{PYTHON_BIN_PATH} "${PYTHON3}")
@@ -265,20 +274,18 @@ foreach(BUILD_TYPE dbg rel)
 				LOGNAME build-${TARGET_TRIPLET}-${BUILD_TYPE}
 			)
 		endif()
-		if(NOT VCPKG_TARGET_IS_OSX)
-			if(VCPKG_TARGET_IS_WINDOWS)
-				vcpkg_execute_build_process(
-					COMMAND ${PYTHON3} "${CMAKE_CURRENT_LIST_DIR}/convert_lib_params_${PLATFORM_SUFFIX}.py" ${TF_LIB_SUFFIX}
-					WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-${BUILD_TYPE}/bazel-bin/tensorflow
-					LOGNAME postbuild1-${TARGET_TRIPLET}-${BUILD_TYPE}
-				)
-			else()
-				vcpkg_execute_build_process(
-					COMMAND ${PYTHON3} "${CMAKE_CURRENT_LIST_DIR}/convert_lib_params_${PLATFORM_SUFFIX}.py" ${TF_VERSION} ${TF_LIB_SUFFIX}
-					WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-${BUILD_TYPE}/bazel-bin/tensorflow
-					LOGNAME postbuild1-${TARGET_TRIPLET}-${BUILD_TYPE}
-				)
-			endif()
+		if(VCPKG_TARGET_IS_WINDOWS)
+			vcpkg_execute_build_process(
+				COMMAND ${PYTHON3} "${CMAKE_CURRENT_LIST_DIR}/convert_lib_params_${PLATFORM_SUFFIX}.py" ${TF_LIB_SUFFIX}
+				WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-${BUILD_TYPE}/bazel-bin/tensorflow
+				LOGNAME postbuild1-${TARGET_TRIPLET}-${BUILD_TYPE}
+			)
+		else()
+			vcpkg_execute_build_process(
+				COMMAND ${PYTHON3} "${CMAKE_CURRENT_LIST_DIR}/convert_lib_params_${PLATFORM_SUFFIX}.py" ${TF_VERSION} ${TF_LIB_SUFFIX}
+				WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-${BUILD_TYPE}/bazel-bin/tensorflow
+				LOGNAME postbuild1-${TARGET_TRIPLET}-${BUILD_TYPE}
+			)
 		endif()
 		# for some reason stdout of bazel ends up in stderr, so use err log file in the following command
 		vcpkg_execute_build_process(

diff --git a/ports/tensorflow-common/vcpkg.json b/ports/tensorflow-common/vcpkg.json
@@ -1,7 +1,7 @@
 {
   "name": "tensorflow-common",
   "version-semver": "2.4.1",
-  "port-version": 3,
+  "port-version": 4,
   "description": "This meta package holds common files for the C [tensorflow] and the C++ [tensorflow-cc] API version of TensorFlow but is not installable on its own.",
   "homepage": "https://github.com/tensorflow/tensorflow"
 }
diff --git a/scripts/ci.baseline.txt b/scripts/ci.baseline.txt
@@ -1757,7 +1757,3 @@ dimcli:x64-windows-static=fail
 
 # cppgraphqlgen triggers an ICE on Apple Clang that comes with MacOS 11.
 cppgraphqlgen:x64-osx=fail
-
-# Changes in Python have broken tensorflow on our osx hardware
-tensorflow:x64-osx=fail
-tensorflow-cc:x64-osx=fail
diff --git a/versions/baseline.json b/versions/baseline.json
@@ -6274,7 +6274,7 @@
     },
     "tensorflow-common": {
       "baseline": "2.4.1",
-      "port-version": 3
+      "port-version": 4
     },
     "tensorpipe": {
       "baseline": "2021-04-26",

diff --git a/versions/t-/tensorflow-common.json b/versions/t-/tensorflow-common.json
@@ -1,5 +1,10 @@
 {
   "versions": [
+    {
+      "git-tree": "3d84b641f07ef269a213e0382e719a0c338f4ed7",
+      "version-semver": "2.4.1",
+      "port-version": 4
+    },
     {
       "git-tree": "dd652b405ef53658c13af438e8414110f2977520",
       "version-semver": "2.4.1",