[icu] Enable parallel builds (#6695)

* Add VCPKG_NUM_LOGICAL_CORES

* break out logic that retries running a command several times into its own function

* Parallelize icu

3 files changed, 153 insertions, 119 deletions

diff --git a/scripts/cmake/vcpkg_build_cmake.cmake b/scripts/cmake/vcpkg_build_cmake.cmake
index 3db6597b4..b110d2cc9 100644
--- a/scripts/cmake/vcpkg_build_cmake.cmake
+++ b/scripts/cmake/vcpkg_build_cmake.cmake
@@ -60,10 +60,6 @@ function(vcpkg_build_cmake)
         set(TARGET_PARAM)
     endif()
 
-    if(_bc_DISABLE_PARALLEL)
-        set(PARALLEL_ARG ${NO_PARALLEL_ARG})
-    endif()
-
     foreach(BUILDTYPE "debug" "release")
         if(NOT DEFINED VCPKG_BUILD_TYPE OR VCPKG_BUILD_TYPE STREQUAL BUILDTYPE)
             if(BUILDTYPE STREQUAL "debug")
@@ -75,8 +71,6 @@ function(vcpkg_build_cmake)
             endif()
 
             message(STATUS "Building ${TARGET_TRIPLET}-${SHORT_BUILDTYPE}")
-            set(LOGPREFIX "${CURRENT_BUILDTREES_DIR}/${_bc_LOGFILE_ROOT}-${TARGET_TRIPLET}-${SHORT_BUILDTYPE}")
-            set(LOGS)
 
             if(_bc_ADD_BIN_TO_PATH)
                 set(_BACKUP_ENV_PATH "$ENV{PATH}")
@@ -91,121 +85,22 @@ function(vcpkg_build_cmake)
                     set(ENV{PATH} "${CURRENT_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}/bin${_PATHSEP}$ENV{PATH}")
                 endif()
             endif()
-            execute_process(
-                COMMAND ${CMAKE_COMMAND} --build . --config ${CONFIG} ${TARGET_PARAM} -- ${BUILD_ARGS} ${PARALLEL_ARG}
-                OUTPUT_FILE "${LOGPREFIX}-out.log"
-                ERROR_FILE "${LOGPREFIX}-err.log"
-                RESULT_VARIABLE error_code
-                WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-${SHORT_BUILDTYPE})
-            if(error_code)
-                file(READ "${LOGPREFIX}-out.log" out_contents)
-                file(READ "${LOGPREFIX}-err.log" err_contents)
-
-                if(out_contents)
-                    list(APPEND LOGS "${LOGPREFIX}-out.log")
-                endif()
-                if(err_contents)
-                    list(APPEND LOGS "${LOGPREFIX}-err.log")
-                endif()
-
-                if(out_contents MATCHES "LINK : fatal error LNK1102:" OR out_contents MATCHES " fatal error C1060: ")
-                    # The linker ran out of memory during execution. We will try continuing once more, with parallelism disabled.
-                    message(STATUS "Restarting Build ${TARGET_TRIPLET}-${SHORT_BUILDTYPE} without parallelism because memory exceeded")
-                    execute_process(
-                        COMMAND ${CMAKE_COMMAND} --build . --config ${CONFIG} ${TARGET_PARAM} -- ${BUILD_ARGS} ${NO_PARALLEL_ARG}
-                        OUTPUT_FILE "${LOGPREFIX}-out-1.log"
-                        ERROR_FILE "${LOGPREFIX}-err-1.log"
-                        RESULT_VARIABLE error_code
-                        WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-${SHORT_BUILDTYPE})
-
-                    if(error_code)
-                        file(READ "${LOGPREFIX}-out-1.log" out_contents)
-                        file(READ "${LOGPREFIX}-err-1.log" err_contents)
-
-                        if(out_contents)
-                            list(APPEND LOGS "${LOGPREFIX}-out-1.log")
-                        endif()
-                        if(err_contents)
-                            list(APPEND LOGS "${LOGPREFIX}-err-1.log")
-                        endif()
-                    endif()
-                elseif(out_contents MATCHES ": No such file or directory")
-                    # WSL workaround - WSL occassionally fails with no such file or directory. Detect if we are running in WSL and restart.
-                    execute_process(COMMAND "uname" "-r"
-                        OUTPUT_VARIABLE UNAME_R ERROR_VARIABLE UNAME_R
-                        OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_STRIP_TRAILING_WHITESPACE)
-
-                    if (UNAME_R MATCHES "Microsoft")
-                        set(ITERATION 0)
-                        while (ITERATION LESS 10 AND out_contents MATCHES ": No such file or directory")
-                            MATH(EXPR ITERATION "${ITERATION}+1")
-                            message(STATUS "Restarting Build ${TARGET_TRIPLET}-${SHORT_BUILDTYPE} because of wsl subsystem issue. Iteration: ${ITERATION}")
-                            execute_process(
-                                COMMAND ${CMAKE_COMMAND} --build . --config ${CONFIG} ${TARGET_PARAM} -- ${BUILD_ARGS}
-                                OUTPUT_FILE "${LOGPREFIX}-out-${ITERATION}.log"
-                                ERROR_FILE "${LOGPREFIX}-err-${ITERATION}.log"
-                                RESULT_VARIABLE error_code
-                                WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-${SHORT_BUILDTYPE})
 
-                            if(error_code)
-                                file(READ "${LOGPREFIX}-out-${ITERATION}.log" out_contents)
-                                file(READ "${LOGPREFIX}-err-${ITERATION}.log" err_contents)
-
-                                if(out_contents)
-                                    list(APPEND LOGS "${LOGPREFIX}-out-${ITERATION}.log")
-                                endif()
-                                if(err_contents)
-                                    list(APPEND LOGS "${LOGPREFIX}-err-${ITERATION}.log")
-                                endif()
-                            else()
-                                break()
-                            endif()
-                        endwhile()
-                    endif()
-                elseif(out_contents MATCHES "mt : general error c101008d: " OR out_contents MATCHES "mt.exe : general error c101008d: ")
-                    # Antivirus workaround - occasionally files are locked and cause mt.exe to fail
-                    set(ITERATION 0)
-                    while (ITERATION LESS 3 AND (out_contents MATCHES "mt : general error c101008d: " OR out_contents MATCHES "mt.exe : general error c101008d: "))
-                        MATH(EXPR ITERATION "${ITERATION}+1")
-                        message(STATUS "Restarting Build ${TARGET_TRIPLET}-${SHORT_BUILDTYPE} because of mt.exe file locking issue. Iteration: ${ITERATION}")
-                        execute_process(
-                            COMMAND ${CMAKE_COMMAND} --build . --config ${CONFIG} ${TARGET_PARAM} -- ${BUILD_ARGS} ${PARALLEL_ARG}
-                            OUTPUT_FILE "${LOGPREFIX}-out-${ITERATION}.log"
-                            ERROR_FILE "${LOGPREFIX}-err-${ITERATION}.log"
-                            RESULT_VARIABLE error_code
-                            WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-${SHORT_BUILDTYPE})
-
-                        if(error_code)
-                            file(READ "${LOGPREFIX}-out-${ITERATION}.log" out_contents)
-                            file(READ "${LOGPREFIX}-err-${ITERATION}.log" err_contents)
-
-                            if(out_contents)
-                                list(APPEND LOGS "${LOGPREFIX}-out-${ITERATION}.log")
-                            endif()
-                            if(err_contents)
-                                list(APPEND LOGS "${LOGPREFIX}-err-${ITERATION}.log")
-                            endif()
-                        else()
-                            break()
-                        endif()
-                    endwhile()
-                endif()
-
-                if(error_code)
-                    set(STRINGIFIED_LOGS)
-                    foreach(LOG ${LOGS})
-                        file(TO_NATIVE_PATH "${LOG}" NATIVE_LOG)
-                        list(APPEND STRINGIFIED_LOGS "    ${NATIVE_LOG}\n")
-                    endforeach()
-                    set(_eb_COMMAND ${CMAKE_COMMAND} --build . --config ${CONFIG} ${TARGET_PARAM} -- ${BUILD_ARGS} ${PARALLEL_ARG})
-                    set(_eb_WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-${SHORT_BUILDTYPE})
-                    message(FATAL_ERROR
-                        "  Command failed: ${_eb_COMMAND}\n"
-                        "  Working Directory: ${_eb_WORKING_DIRECTORY}\n"
-                        "  See logs for more information:\n"
-                        ${STRINGIFIED_LOGS})
-                endif()
+            if (_bc_DISABLE_PARALLEL)
+                vcpkg_execute_build_process(
+                    COMMAND ${CMAKE_COMMAND} --build . --config ${CONFIG} ${TARGET_PARAM} -- ${BUILD_ARGS} ${NO_PARALLEL_ARG}
+                    WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-${SHORT_BUILDTYPE}
+                    LOGNAME "${_bc_LOGFILE_ROOT}-${TARGET_TRIPLET}-${SHORT_BUILDTYPE}"
+                )
+            else()
+                vcpkg_execute_build_process(
+                    COMMAND ${CMAKE_COMMAND} --build . --config ${CONFIG} ${TARGET_PARAM} -- ${BUILD_ARGS} ${PARALLEL_ARG}
+                    NO_PARALLEL_COMMAND ${CMAKE_COMMAND} --build . --config ${CONFIG} ${TARGET_PARAM} -- ${BUILD_ARGS} ${NO_PARALLEL_ARG}
+                    WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-${SHORT_BUILDTYPE}
+                    LOGNAME "${_bc_LOGFILE_ROOT}-${TARGET_TRIPLET}-${SHORT_BUILDTYPE}"
+                )
             endif()
+
             if(_bc_ADD_BIN_TO_PATH)
                 set(ENV{PATH} "${_BACKUP_ENV_PATH}")
             endif()
diff --git a/scripts/cmake/vcpkg_common_functions.cmake b/scripts/cmake/vcpkg_common_functions.cmake
index d66fc5eff..b99f8bee8 100644
--- a/scripts/cmake/vcpkg_common_functions.cmake
+++ b/scripts/cmake/vcpkg_common_functions.cmake
@@ -7,6 +7,7 @@ include(vcpkg_extract_source_archive)
 include(vcpkg_extract_source_archive_ex)
 include(vcpkg_execute_required_process)
 include(vcpkg_execute_required_process_repeat)
+include(vcpkg_execute_build_process)
 include(vcpkg_find_acquire_program)
 include(vcpkg_fixup_cmake_targets)
 include(vcpkg_from_github)
diff --git a/scripts/cmake/vcpkg_execute_build_process.cmake b/scripts/cmake/vcpkg_execute_build_process.cmake
new file mode 100644
index 000000000..f02aa296d
--- /dev/null
+++ b/scripts/cmake/vcpkg_execute_build_process.cmake
@@ -0,0 +1,138 @@
+## # vcpkg_execute_build_process
+##
+## Execute a required build process
+##
+## ## Usage
+## ```cmake
+## vcpkg_execute_build_process(
+##     COMMAND <cmd> [<args>...]
+##     [NO_PARALLEL_COMMAND <cmd> [<args>...]]
+##     WORKING_DIRECTORY </path/to/dir>
+##     LOGNAME <log_name>)
+## )
+## ```
+## ## Parameters
+## ### COMMAND
+## The command to be executed, along with its arguments.
+##
+## ### NO_PARALLEL_COMMAND
+## Optional parameter which specifies a non-parallel command to attempt if a
+## failure potentially due to parallelism is detected.
+##
+## ### WORKING_DIRECTORY
+## The directory to execute the command in.
+##
+## ### LOGNAME
+## The prefix to use for the log files.
+##
+## This should be a unique name for different triplets so that the logs don't
+## conflict when building multiple at once.
+##
+## ## Examples
+##
+## * [icu](https://github.com/Microsoft/vcpkg/blob/master/ports/icu/portfile.cmake)
+function(vcpkg_execute_build_process)
+    cmake_parse_arguments(_ebp "" "WORKING_DIRECTORY;LOGNAME" "COMMAND;NO_PARALLEL_COMMAND" ${ARGN})
+
+    set(LOG_OUT "${CURRENT_BUILDTREES_DIR}/${_ebp_LOGNAME}-out.log")
+    set(LOG_ERR "${CURRENT_BUILDTREES_DIR}/${_ebp_LOGNAME}-err.log")
+
+    execute_process(
+        COMMAND ${_ebp_COMMAND}
+        WORKING_DIRECTORY ${_ebp_WORKING_DIRECTORY}
+        OUTPUT_FILE ${LOG_OUT}
+        ERROR_FILE ${LOG_ERR}
+        RESULT_VARIABLE error_code
+    )
+
+    if(error_code)
+        file(READ ${LOG_OUT} out_contents)
+        file(READ ${LOG_ERR} err_contents)
+
+        if(out_contents)
+            list(APPEND LOGS ${LOG_OUT})
+        endif()
+        if(err_contents)
+            list(APPEND LOGS ${LOG_ERR})
+        endif()
+
+        if(out_contents MATCHES "LINK : fatal error LNK1102:" OR out_contents MATCHES " fatal error C1060: "
+           OR err_contents MATCHES "LINK : fatal error LNK1102:" OR err_contents MATCHES " fatal error C1060: ")
+            # The linker ran out of memory during execution. We will try continuing once more, with parallelism disabled.
+            message(STATUS "Restarting Build without parallelism because memory exceeded")
+            set(LOG_OUT "${CURRENT_BUILDTREES_DIR}/${_ebp_LOGNAME}-out-1.log")
+            set(LOG_ERR "${CURRENT_BUILDTREES_DIR}/${_ebp_LOGNAME}-err-1.log")
+
+            if(${_ebp_NO_PARALLEL_COMMAND})
+                execute_process(
+                    COMMAND ${_ebp_NO_PARALLEL_COMMAND}
+                    WORKING_DIRECTORY ${_ebp_WORKING_DIRECTORY}
+                    OUTPUT_FILE ${LOG_OUT}
+                    ERROR_FILE ${LOG_ERR}
+                    RESULT_VARIABLE error_code
+                )
+            else()
+                execute_process(
+                    COMMAND ${_ebp_COMMAND}
+                    WORKING_DIRECTORY ${_ebp_WORKING_DIRECTORY}
+                    OUTPUT_FILE ${LOG_OUT}
+                    ERROR_FILE ${LOG_ERR}
+                    RESULT_VARIABLE error_code
+                )
+            endif()
+
+            if(error_code)
+                file(READ ${LOG_OUT} out_contents)
+                file(READ ${LOG_ERR} err_contents)
+
+                if(out_contents)
+                    list(APPEND LOGS ${LOG_OUT})
+                endif()
+                if(err_contents)
+                    list(APPEND LOGS ${LOG_ERR})
+                endif()
+            endif()
+        elseif(out_contents MATCHES "mt : general error c101008d: " OR out_contents MATCHES "mt.exe : general error c101008d: ")
+            # Antivirus workaround - occasionally files are locked and cause mt.exe to fail
+            message(STATUS "mt.exe has failed. This may be the result of anti-virus. Disabling anti-virus on the buildtree folder may improve build speed")
+            set(ITERATION 0)
+            while (ITERATION LESS 3 AND (out_contents MATCHES "mt : general error c101008d: " OR out_contents MATCHES "mt.exe : general error c101008d: "))
+                MATH(EXPR ITERATION "${ITERATION}+1")
+                message(STATUS "Restarting Build ${TARGET_TRIPLET}-${SHORT_BUILDTYPE} because of mt.exe file locking issue. Iteration: ${ITERATION}")
+                execute_process(
+                    COMMAND ${_ebp_COMMAND}
+                    OUTPUT_FILE "${LOGPREFIX}-out-${ITERATION}.log"
+                    ERROR_FILE "${LOGPREFIX}-err-${ITERATION}.log"
+                    RESULT_VARIABLE error_code
+                    WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-${SHORT_BUILDTYPE})
+
+                if(error_code)
+                    file(READ "${LOGPREFIX}-out-${ITERATION}.log" out_contents)
+                    file(READ "${LOGPREFIX}-err-${ITERATION}.log" err_contents)
+
+                    if(out_contents)
+                        list(APPEND LOGS "${LOGPREFIX}-out-${ITERATION}.log")
+                    endif()
+                    if(err_contents)
+                        list(APPEND LOGS "${LOGPREFIX}-err-${ITERATION}.log")
+                    endif()
+                else()
+                    break()
+                endif()
+            endwhile()
+        endif()
+
+        if(error_code)
+            set(STRINGIFIED_LOGS)
+            foreach(LOG ${LOGS})
+                file(TO_NATIVE_PATH "${LOG}" NATIVE_LOG)
+                list(APPEND STRINGIFIED_LOGS "    ${NATIVE_LOG}\n")
+            endforeach()
+            message(FATAL_ERROR
+                "  Command failed: ${_ebp_COMMAND}\n"
+                "  Working Directory: ${_ebp_WORKING_DIRECTORY}\n"
+                "  See logs for more information:\n"
+                ${STRINGIFIED_LOGS})
+        endif(error_code)
+    endif(error_code)
+endfunction(vcpkg_execute_build_process)