[vcpkg-build-cmake] Build at IDLE priority. Try to detect out-of-memory issues in the linker, and restart the build once.

3 files changed, 100 insertions, 29 deletions

diff --git a/scripts/cmake/vcpkg_build_cmake.cmake b/scripts/cmake/vcpkg_build_cmake.cmake
index 0b4bbd211..548e6cf46 100644
--- a/scripts/cmake/vcpkg_build_cmake.cmake
+++ b/scripts/cmake/vcpkg_build_cmake.cmake
@@ -33,10 +33,13 @@ function(vcpkg_build_cmake)
         set(_bc_LOGFILE_ROOT "build")
     endif()
 
+    set(PARALLEL_ARG)
+    set(NO_PARALLEL_ARG)
+
     if(_VCPKG_CMAKE_GENERATOR MATCHES "Ninja")
         set(BUILD_ARGS "-v") # verbose output
         if (_bc_DISABLE_PARALLEL)
-            list(APPEND BUILD_ARGS "-j1")
+            set(NO_PARALLEL_ARG "-j1")
         endif()
     elseif(_VCPKG_CMAKE_GENERATOR MATCHES "Visual Studio")
         set(BUILD_ARGS
@@ -44,7 +47,7 @@ function(vcpkg_build_cmake)
             "/p:UseIntelMKL=No"
         )
         if (NOT _bc_DISABLE_PARALLEL)
-            list(APPEND BUILD_ARGS "/m")
+            set(PARALLEL_ARG "/m")
         endif()
     elseif(_VCPKG_CMAKE_GENERATOR MATCHES "NMake")
         # No options are currently added for nmake builds
@@ -58,23 +61,79 @@ function(vcpkg_build_cmake)
         set(TARGET_PARAM)
     endif()
 
-    if(NOT DEFINED VCPKG_BUILD_TYPE OR VCPKG_BUILD_TYPE STREQUAL "release")
-        message(STATUS "Build ${TARGET_TRIPLET}-rel")
-        vcpkg_execute_required_process(
-            COMMAND ${CMAKE_COMMAND} --build . --config Release ${TARGET_PARAM} -- ${BUILD_ARGS}
-            WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel
-            LOGNAME ${_bc_LOGFILE_ROOT}-${TARGET_TRIPLET}-rel
-        )
-        message(STATUS "Build ${TARGET_TRIPLET}-rel done")
-    endif()
+    foreach(BUILDTYPE "release" "debug")
+        if(NOT DEFINED VCPKG_BUILD_TYPE OR VCPKG_BUILD_TYPE STREQUAL BUILDTYPE)
+            if(BUILDTYPE STREQUAL "debug")
+                set(SHORT_BUILDTYPE "dbg")
+            else()
+                set(SHORT_BUILDTYPE "rel")
+            endif()
 
-    if(NOT DEFINED VCPKG_BUILD_TYPE OR VCPKG_BUILD_TYPE STREQUAL "debug")
-        message(STATUS "Build ${TARGET_TRIPLET}-dbg")
-        vcpkg_execute_required_process(
-            COMMAND ${CMAKE_COMMAND} --build . --config Debug ${TARGET_PARAM} -- ${BUILD_ARGS}
-            WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-dbg
-            LOGNAME ${_bc_LOGFILE_ROOT}-${TARGET_TRIPLET}-dbg
-        )
-        message(STATUS "Build ${TARGET_TRIPLET}-dbg done")
-    endif()
+            message(STATUS "Build ${TARGET_TRIPLET}-${SHORT_BUILDTYPE}")
+            set(LOGPREFIX "${CURRENT_BUILDTREES_DIR}/${_bc_LOGFILE_ROOT}-${TARGET_TRIPLET}-${SHORT_BUILDTYPE}")
+            set(LOGS)
+
+            if(BUILDTYPE STREQUAL "release")
+                set(CONFIG "Release")
+            else()
+                set(CONFIG "Debug")
+            endif()
+
+            execute_process(
+                COMMAND ${CMAKE_COMMAND} --build . --config Release ${TARGET_PARAM} -- ${BUILD_ARGS} ${PARALLEL_ARG}
+                OUTPUT_FILE "${LOGPREFIX}-out.log"
+                ERROR_FILE "${LOGPREFIX}-err.log"
+                RESULT_VARIABLE error_code
+                WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-${SHORT_BUILDTYPE})
+            if(error_code)
+                file(READ "${LOGPREFIX}-out.log" out_contents)
+                file(READ "${LOGPREFIX}-err.log" err_contents)
+
+                if(out_contents)
+                    list(APPEND LOGS "${LOGPREFIX}-out.log")
+                endif()
+                if(err_contents)
+                    list(APPEND LOGS "${LOGPREFIX}-err.log")
+                endif()
+
+                if(out_contents MATCHES "LINK : fatal error LNK1102:" OR out_contents MATCHES " fatal error C1060: ")
+                    # The linker ran out of memory during execution. We will try continuing once more, with parallelism disabled.
+                    execute_process(
+                        COMMAND ${CMAKE_COMMAND} --build . --config Release ${TARGET_PARAM} -- ${BUILD_ARGS} ${NO_PARALLEL_ARG}
+                        OUTPUT_FILE "${LOGPREFIX}-out-1.log"
+                        ERROR_FILE "${LOGPREFIX}-err-1.log"
+                        RESULT_VARIABLE error_code
+                        WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-${SHORT_BUILDTYPE})
+
+                    if(error_code)
+                        file(READ "${LOGPREFIX}-out-1.log" out_contents)
+                        file(READ "${LOGPREFIX}-err-1.log" err_contents)
+
+                        if(out_contents)
+                            list(APPEND LOGS "${LOGPREFIX}-out-1.log")
+                        endif()
+                        if(err_contents)
+                            list(APPEND LOGS "${LOGPREFIX}-err-1.log")
+                        endif()
+                    endif()
+                endif()
+
+                if(error_code)
+                    set(STRINGIFIED_LOGS)
+                    foreach(LOG ${LOGS})
+                        file(TO_NATIVE_PATH "${LOG}" NATIVE_LOG)
+                        list(APPEND STRINGIFIED_LOGS "    ${NATIVE_LOG}\n")
+                    endforeach()
+                    set(_eb_COMMAND ${CMAKE_COMMAND} --build . --config Release ${TARGET_PARAM} -- ${BUILD_ARGS} ${NO_PARALLEL_ARG})
+                    set(_eb_WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-${SHORT_BUILDTYPE})
+                    message(FATAL_ERROR
+                        "  Command failed: ${_eb_COMMAND}\n"
+                        "  Working Directory: ${_eb_WORKING_DIRECTORY}\n"
+                        "  See logs for more information:\n"
+                        ${STRINGIFIED_LOGS})
+                endif()
+            endif()
+            message(STATUS "Build ${TARGET_TRIPLET}-${SHORT_BUILDTYPE} done")
+        endif()
+    endforeach()
 endfunction()
diff --git a/scripts/cmake/vcpkg_execute_required_process.cmake b/scripts/cmake/vcpkg_execute_required_process.cmake
index 7c4907016..5b8922c14 100644
--- a/scripts/cmake/vcpkg_execute_required_process.cmake
+++ b/scripts/cmake/vcpkg_execute_required_process.cmake
@@ -30,22 +30,34 @@
 ## * [qt5](https://github.com/Microsoft/vcpkg/blob/master/ports/qt5/portfile.cmake)
 function(vcpkg_execute_required_process)
     cmake_parse_arguments(vcpkg_execute_required_process "" "WORKING_DIRECTORY;LOGNAME" "COMMAND" ${ARGN})
-    #debug_message("vcpkg_execute_required_process(${vcpkg_execute_required_process_COMMAND})")
+    set(LOG_OUT "${CURRENT_BUILDTREES_DIR}/${vcpkg_execute_required_process_LOGNAME}-out.log")
+    set(LOG_ERR "${CURRENT_BUILDTREES_DIR}/${vcpkg_execute_required_process_LOGNAME}-err.log")
     execute_process(
         COMMAND ${vcpkg_execute_required_process_COMMAND}
-        OUTPUT_FILE ${CURRENT_BUILDTREES_DIR}/${vcpkg_execute_required_process_LOGNAME}-out.log
-        ERROR_FILE ${CURRENT_BUILDTREES_DIR}/${vcpkg_execute_required_process_LOGNAME}-err.log
+        OUTPUT_FILE ${LOG_OUT}
+        ERROR_FILE ${LOG_ERR}
         RESULT_VARIABLE error_code
         WORKING_DIRECTORY ${vcpkg_execute_required_process_WORKING_DIRECTORY})
-    #debug_message("error_code=${error_code}")
     if(error_code)
-        file(TO_NATIVE_PATH "${CURRENT_BUILDTREES_DIR}/${vcpkg_execute_required_process_LOGNAME}-out.log" NATIVE_LOG_OUT)
-        file(TO_NATIVE_PATH "${CURRENT_BUILDTREES_DIR}/${vcpkg_execute_required_process_LOGNAME}-err.log" NATIVE_LOG_ERR)
+        set(LOGS)
+        file(READ "${LOG_OUT}" out_contents)
+        file(READ "${LOG_ERR}" err_contents)
+        if(out_contents)
+            list(APPEND LOGS "${LOG_OUT}")
+        endif()
+        if(err_contents)
+            list(APPEND LOGS "${LOG_ERR}")
+        endif()
+        set(STRINGIFIED_LOGS)
+        foreach(LOG ${LOGS})
+            file(TO_NATIVE_PATH "${LOG}" NATIVE_LOG)
+            list(APPEND STRINGIFIED_LOGS "    ${NATIVE_LOG}\n")
+        endforeach()
         message(FATAL_ERROR
             "  Command failed: ${vcpkg_execute_required_process_COMMAND}\n"
             "  Working Directory: ${vcpkg_execute_required_process_WORKING_DIRECTORY}\n"
             "  See logs for more information:\n"
-            "    ${NATIVE_LOG_OUT}\n"
-            "    ${NATIVE_LOG_ERR}\n")
+            ${STRINGIFIED_LOGS}
+        )
     endif()
 endfunction()
diff --git a/toolsrc/src/vcpkg/base/system.cpp b/toolsrc/src/vcpkg/base/system.cpp
index 625ee6ce0..84721bf6e 100644
--- a/toolsrc/src/vcpkg/base/system.cpp
+++ b/toolsrc/src/vcpkg/base/system.cpp
@@ -220,7 +220,7 @@ namespace vcpkg::System
                                                 nullptr,
                                                 nullptr,
                                                 FALSE,
-                                                BELOW_NORMAL_PRIORITY_CLASS | CREATE_UNICODE_ENVIRONMENT,
+                                                IDLE_PRIORITY_CLASS | CREATE_UNICODE_ENVIRONMENT,
                                                 env_cstr.data(),
                                                 nullptr,
                                                 &startup_info,