[tensorflow] fix macOS build errors caused by numpy and bazel upgrades (#19363)

* Revert "incorporate changes from microsoft:master"

* Revert "Revert "incorporate changes from microsoft:master""

* issue targeted in newer numpy version

* work-around for broken numpy libs on macOS

* temporarily add debug code to analyze CI failures

* again temporary debug code

* fix linkage command on macOS (broken by Bazel upgrade)

* fix regex for macOS linker command

* remove debug code, bump version

* x-add-version seems to require a separate commit...

* Fix misspelled "acceleration".

* x-add-version

Co-authored-by: jgehw <Joachim_Gehweiler@McAfee.com>
Co-authored-by: Billy Robert O'Neal III <bion@microsoft.com>

7 files changed, 51 insertions, 49 deletions

diff --git a/ports/tensorflow-common/convert_lib_params_macos.py b/ports/tensorflow-common/convert_lib_params_macos.py
new file mode 100644
index 000000000..b3c419363
--- /dev/null
+++ b/ports/tensorflow-common/convert_lib_params_macos.py
@@ -0,0 +1,18 @@
+import sys

+

+version = sys.argv[1]

+lib_suffix = "" if len(sys.argv) < 3 else sys.argv[2]

+

+with open(f"libtensorflow{lib_suffix}.{version}.a-2.params", "w") as f_out:

+    parts = []

+    with open(f"libtensorflow_framework.{version}.dylib-2.params", "r") as f_in:

+        for line in f_in:

+            if line.startswith("-Wl,-force_load,"):

+                f_out.write(line[16:])

+                parts.append(line[16:])

+    parts = set(parts)

+    with open(f"libtensorflow{lib_suffix}.{version}.dylib-2.params", "r") as f_in:

+        for line in f_in:

+            if line.startswith("-Wl,-force_load,"):

+                if line[16:] not in parts:

+                    f_out.write(line[16:])

diff --git a/ports/tensorflow-common/generate_static_link_cmd_linux.py b/ports/tensorflow-common/generate_static_link_cmd_linux.py
index 34c8db134..7b1755215 100644
--- a/ports/tensorflow-common/generate_static_link_cmd_linux.py
+++ b/ports/tensorflow-common/generate_static_link_cmd_linux.py
@@ -4,7 +4,7 @@ import sys
 lib_suffix = "" if len(sys.argv) < 3 else sys.argv[2]

 with open(sys.argv[1], "r") as f_in:

     with open("static_link.sh", "w") as f_out:

-        p_cd = re.compile("^\\((cd .*) && \\\\$")

+        p_cd = re.compile(r"^\((cd .*) && \\$")

         p_linker = re.compile(fr"^\s*(.+)gcc.+(@bazel-out\S+libtensorflow{lib_suffix}\.so\.\d\.\d\.\d-2\.params).*")

         f_out.write("#!/bin/bash\n# note: ar/binutils version 2.27 required to support output files > 4GB\n")

         env = []

diff --git a/ports/tensorflow-common/generate_static_link_cmd_macos.py b/ports/tensorflow-common/generate_static_link_cmd_macos.py
index bb8f6faa2..5894c99b0 100644
--- a/ports/tensorflow-common/generate_static_link_cmd_macos.py
+++ b/ports/tensorflow-common/generate_static_link_cmd_macos.py
@@ -4,44 +4,20 @@ import sys
 lib_suffix = "" if len(sys.argv) < 3 else sys.argv[2]

 with open(sys.argv[1], "r") as f_in:

     with open("static_link.sh", "w") as f_out:

-        p_cd = re.compile("^\\((cd .*) && \\\\$")

-        p_linker1 = re.compile(fr"^.*cc_wrapper.sh.+-shared.+-o (bazel-out\S+libtensorflow{lib_suffix}\.\d\.\d\.\d\.dylib)")

-        p_linker2 = re.compile("^.*cc_wrapper.sh.+-shared.+-o (bazel-out\\S+libtensorflow_framework\\.\\d\\.\\d\\.\\d\\.dylib)")

+        p_cd = re.compile(r"^\((cd .*) && \\$")

+        p_linker = re.compile(fr"^\s*.+cc_wrapper.sh.+(@bazel-out\S+libtensorflow{lib_suffix}\.\d\.\d\.\d\.dylib-2\.params).*")

         f_out.write("#!/bin/bash\n# note: ar/binutils version 2.27 required to support output files > 4GB\n")

         env = []

-        parts = None

         for line in f_in:

             if line.startswith("(cd"):

                 # new command, reset

                 env = [line]

             else:

-                m1 = p_linker1.match(line)

-                m2 = p_linker2.match(line)

+                m1 = p_linker.match(line)

                 if m1:

-                    tokens = line.split()

-                    if parts is None:

-                        parts = [t[16:] for t in tokens if t.startswith("-Wl,-force_load,")]

-                    else:

-                        m = p_cd.match(env[0])

-                        f_out.write(m.group(1) + "\n")

-                        tmp = [t[16:] for t in tokens if t.startswith("-Wl,-force_load,")]

-                        old = set(parts)

-                        parts += [t for t in tmp if t not in old]

-                        line = f"libtool -static -o {m1.group(1).replace('.dylib', '.a')} {' '.join(parts)}\n"

-                        f_out.write(line)

-                        break

-                elif m2 and len(env) > 6:

-                    tokens = line.split()

-                    if parts is None:

-                        parts = [t[16:] for t in tokens if t.startswith("-Wl,-force_load,")]

-                    else:

-                        m = p_cd.match(env[0])

-                        f_out.write(m.group(1) + "\n")

-                        tmp = [t[16:] for t in tokens if t.startswith("-Wl,-force_load,")]

-                        old = set(parts)

-                        parts += [t for t in tmp if t not in old]

-                        line = f"libtool -static -o {m2.group(1).replace('_framework', lib_suffix).replace('.dylib', '.a')} {' '.join(parts)}\n"

-                        f_out.write(line)

-                        break

+                    m2 = p_cd.match(env[0])

+                    f_out.write(m2.group(1) + "\n")

+                    line = f'"/usr/bin/libtool" -static -o {m1.group(1)[1:-9].replace(".dylib", ".a")} {m1.group(1).replace(".dylib", ".a")}\n'

+                    f_out.write(line)

                 else:

                     env.append(line)

diff --git a/ports/tensorflow-common/generate_static_link_cmd_windows.py b/ports/tensorflow-common/generate_static_link_cmd_windows.py
index 780334ad9..84ec9eede 100644
--- a/ports/tensorflow-common/generate_static_link_cmd_windows.py
+++ b/ports/tensorflow-common/generate_static_link_cmd_windows.py
@@ -5,7 +5,7 @@ import sys
 lib_suffix = "" if len(sys.argv) < 3 else sys.argv[2]

 with open(sys.argv[1], "r") as f_in:

     with open("static_link.bat", "w") as f_out:

-        p_setenv = re.compile("^\s*(SET .+=.*)$")

+        p_setenv = re.compile(r"^\s*(SET .+=.*)$")

         p_linker = re.compile(fr".+link\.exe.+tensorflow{lib_suffix}\.dll-2\.params.*")

         env = []

         for line in f_in:

diff --git a/ports/tensorflow-common/portfile.cmake b/ports/tensorflow-common/portfile.cmake
index e505c4430..edd171aa4 100644
--- a/ports/tensorflow-common/portfile.cmake
+++ b/ports/tensorflow-common/portfile.cmake
@@ -3,6 +3,7 @@ set(VCPKG_POLICY_EMPTY_INCLUDE_FOLDER enabled)
 set(TENSORFLOW_FILES
     "${CMAKE_CURRENT_LIST_DIR}/change-macros-for-static-lib.patch"
     "${CMAKE_CURRENT_LIST_DIR}/convert_lib_params_linux.py"
+    "${CMAKE_CURRENT_LIST_DIR}/convert_lib_params_macos.py"
     "${CMAKE_CURRENT_LIST_DIR}/convert_lib_params_windows.py"
     "${CMAKE_CURRENT_LIST_DIR}/fix-build-error.patch"
     "${CMAKE_CURRENT_LIST_DIR}/fix-linux-build.patch"
diff --git a/ports/tensorflow-common/tensorflow-common.cmake b/ports/tensorflow-common/tensorflow-common.cmake
index 776cb9c00..403f32c6c 100644
--- a/ports/tensorflow-common/tensorflow-common.cmake
+++ b/ports/tensorflow-common/tensorflow-common.cmake
@@ -45,7 +45,16 @@ else()
 	get_filename_component(PYTHON3_DIR "${PYTHON3}" DIRECTORY)
 	vcpkg_add_to_path(PREPEND ${PYTHON3_DIR})
 
-	vcpkg_execute_required_process(COMMAND ${PYTHON3} -m pip install --user -U numpy WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR} LOGNAME prerequesits-pip-${TARGET_TRIPLET})
+	if(VCPKG_TARGET_IS_OSX)
+		# acceleration libs currently broken on macOS => force numpy user space reinstall without BLAS/LAPACK/ATLAS
+		# remove this work-around again, i.e. default to "else" branch, once acceleration libs are fixed upstream
+		set(ENV{BLAS} "None")
+		set(ENV{LAPACK} "None")
+		set(ENV{ATLAS} "None")
+		vcpkg_execute_required_process(COMMAND ${PYTHON3} -m pip install --user -U --force-reinstall numpy WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR} LOGNAME prerequesits-pip-${TARGET_TRIPLET})
+	else()
+		vcpkg_execute_required_process(COMMAND ${PYTHON3} -m pip install --user -U numpy WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR} LOGNAME prerequesits-pip-${TARGET_TRIPLET})
+	endif()
 	vcpkg_execute_required_process(COMMAND ${PYTHON3} -c "import site; print(site.getusersitepackages())" WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR} LOGNAME prerequesits-pypath-${TARGET_TRIPLET} OUTPUT_VARIABLE PYTHON_LIB_PATH)
 endif()
 set(ENV{PYTHON_BIN_PATH} "${PYTHON3}")
@@ -265,20 +274,18 @@ foreach(BUILD_TYPE dbg rel)
 				LOGNAME build-${TARGET_TRIPLET}-${BUILD_TYPE}
 			)
 		endif()
-		if(NOT VCPKG_TARGET_IS_OSX)
-			if(VCPKG_TARGET_IS_WINDOWS)
-				vcpkg_execute_build_process(
-					COMMAND ${PYTHON3} "${CMAKE_CURRENT_LIST_DIR}/convert_lib_params_${PLATFORM_SUFFIX}.py" ${TF_LIB_SUFFIX}
-					WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-${BUILD_TYPE}/bazel-bin/tensorflow
-					LOGNAME postbuild1-${TARGET_TRIPLET}-${BUILD_TYPE}
-				)
-			else()
-				vcpkg_execute_build_process(
-					COMMAND ${PYTHON3} "${CMAKE_CURRENT_LIST_DIR}/convert_lib_params_${PLATFORM_SUFFIX}.py" ${TF_VERSION} ${TF_LIB_SUFFIX}
-					WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-${BUILD_TYPE}/bazel-bin/tensorflow
-					LOGNAME postbuild1-${TARGET_TRIPLET}-${BUILD_TYPE}
-				)
-			endif()
+		if(VCPKG_TARGET_IS_WINDOWS)
+			vcpkg_execute_build_process(
+				COMMAND ${PYTHON3} "${CMAKE_CURRENT_LIST_DIR}/convert_lib_params_${PLATFORM_SUFFIX}.py" ${TF_LIB_SUFFIX}
+				WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-${BUILD_TYPE}/bazel-bin/tensorflow
+				LOGNAME postbuild1-${TARGET_TRIPLET}-${BUILD_TYPE}
+			)
+		else()
+			vcpkg_execute_build_process(
+				COMMAND ${PYTHON3} "${CMAKE_CURRENT_LIST_DIR}/convert_lib_params_${PLATFORM_SUFFIX}.py" ${TF_VERSION} ${TF_LIB_SUFFIX}
+				WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-${BUILD_TYPE}/bazel-bin/tensorflow
+				LOGNAME postbuild1-${TARGET_TRIPLET}-${BUILD_TYPE}
+			)
 		endif()
 		# for some reason stdout of bazel ends up in stderr, so use err log file in the following command
 		vcpkg_execute_build_process(
diff --git a/ports/tensorflow-common/vcpkg.json b/ports/tensorflow-common/vcpkg.json
index d97e2c228..b0dfa7959 100644
--- a/ports/tensorflow-common/vcpkg.json
+++ b/ports/tensorflow-common/vcpkg.json
@@ -1,7 +1,7 @@
 {
   "name": "tensorflow-common",
   "version-semver": "2.4.1",
-  "port-version": 3,
+  "port-version": 4,
   "description": "This meta package holds common files for the C [tensorflow] and the C++ [tensorflow-cc] API version of TensorFlow but is not installable on its own.",
   "homepage": "https://github.com/tensorflow/tensorflow"
 }