[vcpkg] Enable binary caching by default (#12370)

* [vcpkg] Enable binary caching by default. Support `x-` migration.

Fix passing multiple copies of single parameter arguments.

* [vcpkg] Handle x- prefixes for general arguments

* [vcpkg] Fix #12285 and improve documentation of default binary cache path

* [vcpkg] Revert x- prefix homogenization for per-command arguments

* [vcpkg] Only use accelerated compiler detection for Windows Desktop + Ninja. Improve breadcrumbs for users encountering issues.

* [vcpkg] Fix compiler tracking not pre-downloading Ninja. Fix compiler tracking not looking in -err.log.

* [vcpkg] Update toolsrc/src/vcpkg/binarycaching.cpp

Co-authored-by: Billy O'Neal <bion@microsoft.com>

* [vcpkg] Format

Co-authored-by: Robert Schumacher <roschuma@microsoft.com>
Co-authored-by: Billy O'Neal <bion@microsoft.com>

7 files changed, 109 insertions, 41 deletions

diff --git a/scripts/detect_compiler/CMakeLists.txt b/scripts/detect_compiler/CMakeLists.txt
index 379a87d03..ea27db2b9 100644
--- a/scripts/detect_compiler/CMakeLists.txt
+++ b/scripts/detect_compiler/CMakeLists.txt
@@ -1,12 +1,14 @@
 cmake_minimum_required(VERSION 3.10)

 project(detect_compiler NONE)

 

-set(CMAKE_C_COMPILER_WORKS 1)

-set(CMAKE_C_COMPILER_ID_RUN 1)

-set(CMAKE_C_COMPILER_FORCED 1)

-set(CMAKE_CXX_COMPILER_WORKS 1)

-set(CMAKE_CXX_COMPILER_ID_RUN 1)

-set(CMAKE_CXX_COMPILER_FORCED 1)

+if(CMAKE_GENERATOR STREQUAL "Ninja" AND CMAKE_SYSTEM_NAME STREQUAL "Windows")

+    set(CMAKE_C_COMPILER_WORKS 1)

+    set(CMAKE_C_COMPILER_ID_RUN 1)

+    set(CMAKE_C_COMPILER_FORCED 1)

+    set(CMAKE_CXX_COMPILER_WORKS 1)

+    set(CMAKE_CXX_COMPILER_ID_RUN 1)

+    set(CMAKE_CXX_COMPILER_FORCED 1)

+endif()

 

 enable_language(C)

 enable_language(CXX)

diff --git a/scripts/detect_compiler/portfile.cmake b/scripts/detect_compiler/portfile.cmake
index 16c760706..4f68faea4 100644
--- a/scripts/detect_compiler/portfile.cmake
+++ b/scripts/detect_compiler/portfile.cmake
@@ -2,6 +2,8 @@ set(LOGS
     ${CURRENT_BUILDTREES_DIR}/config-${TARGET_TRIPLET}-out.log

     ${CURRENT_BUILDTREES_DIR}/config-${TARGET_TRIPLET}-rel-out.log

     ${CURRENT_BUILDTREES_DIR}/config-${TARGET_TRIPLET}-dbg-out.log

+    ${CURRENT_BUILDTREES_DIR}/config-${TARGET_TRIPLET}-rel-err.log

+    ${CURRENT_BUILDTREES_DIR}/config-${TARGET_TRIPLET}-dbg-err.log

 )

 

 foreach(LOG IN LISTS LOGS)

@@ -22,7 +24,5 @@ foreach(LOG IN LISTS LOGS)
     if(EXISTS ${LOG})

         file(READ "${LOG}" _contents)

         message("${_contents}")

-        return()

     endif()

 endforeach()

-message(FATAL_ERROR "Could read logs: ${LOGS}")

diff --git a/toolsrc/include/vcpkg/vcpkgcmdarguments.h b/toolsrc/include/vcpkg/vcpkgcmdarguments.h
index 6973311b0..8cf0680b9 100644
--- a/toolsrc/include/vcpkg/vcpkgcmdarguments.h
+++ b/toolsrc/include/vcpkg/vcpkgcmdarguments.h
@@ -135,7 +135,7 @@ namespace vcpkg
         constexpr static StringLiteral OVERLAY_TRIPLETS_ARG = "overlay-triplets";
         std::vector<std::string> overlay_triplets;
 
-        constexpr static StringLiteral BINARY_SOURCES_ARG = "x-binarysource";
+        constexpr static StringLiteral BINARY_SOURCES_ARG = "binarysource";
         std::vector<std::string> binary_sources;
 
         constexpr static StringLiteral DEBUG_SWITCH = "debug";
@@ -166,8 +166,8 @@ namespace vcpkg
         constexpr static StringLiteral MANIFEST_MODE_FEATURE = "manifests";
         Optional<bool> manifest_mode = nullopt;
 
-        bool binary_caching_enabled() const { return binary_caching.value_or(false); }
-        bool compiler_tracking_enabled() const { return compiler_tracking.value_or(false); }
+        bool binary_caching_enabled() const { return binary_caching.value_or(true); }
+        bool compiler_tracking_enabled() const { return compiler_tracking.value_or(true); }
 
         std::string command;
         std::vector<std::string> command_arguments;
diff --git a/toolsrc/src/vcpkg/base/system.cpp b/toolsrc/src/vcpkg/base/system.cpp
index 6fdfeb6a7..63674963e 100644
--- a/toolsrc/src/vcpkg/base/system.cpp
+++ b/toolsrc/src/vcpkg/base/system.cpp
@@ -135,7 +135,20 @@ namespace vcpkg
         static ExpectedS<fs::path> s_home = []() -> ExpectedS<fs::path> {
             auto maybe_home = System::get_environment_variable("LOCALAPPDATA");
             if (!maybe_home.has_value() || maybe_home.get()->empty())
-                return {"unable to read %LOCALAPPDATA%", ExpectedRightTag{}};
+            {
+                // Consult %APPDATA% as a workaround for Service accounts
+                // Microsoft/vcpkg#12285
+                maybe_home = System::get_environment_variable("APPDATA");
+                if (!maybe_home.has_value() || maybe_home.get()->empty())
+                {
+                    return {"unable to read %LOCALAPPDATA% or %APPDATA%", ExpectedRightTag{}};
+                }
+
+                auto p = fs::u8path(*maybe_home.get()).parent_path();
+                p /= "Local";
+                if (!p.is_absolute()) return {"%APPDATA% was not an absolute path", ExpectedRightTag{}};
+                return {std::move(p), ExpectedLeftTag{}};
+            }
 
             auto p = fs::u8path(*maybe_home.get());
             if (!p.is_absolute()) return {"%LOCALAPPDATA% was not an absolute path", ExpectedRightTag{}};
diff --git a/toolsrc/src/vcpkg/binarycaching.cpp b/toolsrc/src/vcpkg/binarycaching.cpp
index 1ceeb6a77..0a0b100dc 100644
--- a/toolsrc/src/vcpkg/binarycaching.cpp
+++ b/toolsrc/src/vcpkg/binarycaching.cpp
@@ -660,6 +660,22 @@ ExpectedS<std::unique_ptr<IBinaryProvider>> vcpkg::create_binary_provider_from_c
 }

 namespace

 {

+    const ExpectedS<fs::path>& default_cache_path()

+    {

+        static auto cachepath = System::get_platform_cache_home().then([](fs::path p) -> ExpectedS<fs::path> {

+            p /= fs::u8path("vcpkg/archives");

+            if (p.is_absolute())

+            {

+                return {std::move(p), expected_left_tag};

+            }

+            else

+            {

+                return {"default path was not absolute: " + p.u8string(), expected_right_tag};

+            }

+        });

+        return cachepath;

+    }

+

     struct State

     {

         bool m_cleared = false;

@@ -856,17 +872,11 @@ namespace
                                      segments[0].first);

                 }

 

-                auto&& maybe_home = System::get_platform_cache_home();

+                const auto& maybe_home = default_cache_path();

                 if (!maybe_home.has_value()) return add_error(maybe_home.error(), segments[0].first);

 

-                auto p = *maybe_home.get();

-                p /= fs::u8path("vcpkg/archives");

-                if (!p.is_absolute())

-                {

-                    return add_error("default path was not absolute: " + p.u8string(), segments[0].first);

-                }

-

-                handle_readwrite(state->archives_to_read, state->archives_to_write, std::move(p), segments, 1);

+                handle_readwrite(

+                    state->archives_to_read, state->archives_to_write, fs::path(*maybe_home.get()), segments, 1);

             }

             else

             {

@@ -986,38 +996,44 @@ std::string vcpkg::generate_nuspec(const VcpkgPaths& paths,
 

 void vcpkg::help_topic_binary_caching(const VcpkgPaths&)

 {

-    System::print2(

-        System::Color::warning,

-        "** The following help documentation covers an experimental feature that will change at any time **\n\n");

-

     HelpTableFormatter tbl;

-    tbl.text(

-        "Vcpkg can cache compiled packages to accelerate restoration on a single machine or across the network."

-        " This functionality is currently disabled by default and must be enabled by either passing `--binarycaching` "

-        "to every vcpkg command line or setting the environment variable `VCPKG_FEATURE_FLAGS` to `binarycaching`.");

+    tbl.text("Vcpkg can cache compiled packages to accelerate restoration on a single machine or across the network."

+             " This functionality is currently enabled by default and can be disabled by either passing "

+             "`--no-binarycaching` to every vcpkg command line or setting the environment variable "

+             "`VCPKG_FEATURE_FLAGS` to `-binarycaching`.");

     tbl.blank();

     tbl.blank();

     tbl.text(

-        "Once caching is enabled, it can be further configured by either passing `--x-binarysource=<source>` options "

+        "Once caching is enabled, it can be further configured by either passing `--binarysource=<source>` options "

         "to every command line or setting the `VCPKG_BINARY_SOURCES` environment variable to a set of sources (Ex: "

         "\"<source>;<source>;...\"). Command line sources are interpreted after environment sources.");

     tbl.blank();

     tbl.blank();

     tbl.header("Valid source strings");

     tbl.format("clear", "Removes all previous sources");

-    tbl.format("default[,upload]", "Adds the default file-based source location (~/.vcpkg/archives).");

-    tbl.format("files,<path>[,upload]", "Adds a custom file-based source location.");

-    tbl.format("nuget,<uri>[,upload]",

+    tbl.format("default[,<rw>]", "Adds the default file-based location.");

+    tbl.format("files,<path>[,<rw>]", "Adds a custom file-based location.");

+    tbl.format("nuget,<uri>[,<rw>]",

                "Adds a NuGet-based source; equivalent to the `-Source` parameter of the NuGet CLI.");

-    tbl.format("nugetconfig,<path>[,upload]",

+    tbl.format("nugetconfig,<path>[,<rw>]",

                "Adds a NuGet-config-file-based source; equivalent to the `-Config` parameter of the NuGet CLI. This "

                "config should specify `defaultPushSource` for uploads.");

     tbl.format("interactive", "Enables interactive credential management for some source types");

     tbl.blank();

-    tbl.text("The `upload` optional parameter for certain source strings controls whether on-demand builds will be "

-             "uploaded to that remote.");

-

+    tbl.text("The `<rw>` optional parameter for certain strings controls whether they will be consulted for "

+             "downloading binaries and whether on-demand builds will be uploaded to that remote. It can be specified "

+             "as 'read', 'write', or 'readwrite'.");

+    tbl.blank();

     System::print2(tbl.m_str);

+    const auto& maybe_cachepath = default_cache_path();

+    if (auto p = maybe_cachepath.get())

+    {

+        auto p_preferred = *p;

+        System::print2(

+            "\nBased on your system settings, the default path to store binaries is\n    ",

+            p_preferred.make_preferred().u8string(),

+            "\n\nThis consults %LOCALAPPDATA%/%APPDATA% on Windows and $XDG_CACHE_HOME or $HOME on other platforms.");

+    }

 }

 

 std::string vcpkg::generate_nuget_packages_config(const Dependencies::ActionPlan& action)

diff --git a/toolsrc/src/vcpkg/build.cpp b/toolsrc/src/vcpkg/build.cpp
index 994ba441e..be8004875 100644
--- a/toolsrc/src/vcpkg/build.cpp
+++ b/toolsrc/src/vcpkg/build.cpp
@@ -460,6 +460,11 @@ namespace vcpkg::Build
         System::print2("Detecting compiler hash for triplet ", triplet, "...\n");

         auto buildpath = paths.buildtrees / "detect_compiler";

 

+#if !defined(_WIN32)

+        // TODO: remove when vcpkg.exe is in charge for acquiring tools. Change introduced in vcpkg v0.0.107.

+        // bootstrap should have already downloaded ninja, but making sure it is present in case it was deleted.

+        vcpkg::Util::unused(paths.get_tool_exe(Tools::NINJA));

+#endif

         std::vector<System::CMakeVariable> cmake_args{

             {"CURRENT_PORT_DIR", paths.scripts / "detect_compiler"},

             {"CURRENT_BUILDTREES_DIR", buildpath},

@@ -501,6 +506,14 @@ namespace vcpkg::Build
             env);

         out_file.close();

 

+        if (compiler_hash.empty())

+        {

+            Debug::print("Compiler information tracking can be disabled by passing --",

+                         VcpkgCmdArguments::FEATURE_FLAGS_ARG,

+                         "=-",

+                         VcpkgCmdArguments::COMPILER_TRACKING_FEATURE,

+                         "\n");

+        }

         Checks::check_exit(VCPKG_LINE_INFO,

                            !compiler_hash.empty(),

                            "Error occured while detecting compiler information. Pass `--debug` for more information.");

diff --git a/toolsrc/src/vcpkg/vcpkgcmdarguments.cpp b/toolsrc/src/vcpkg/vcpkgcmdarguments.cpp
index 6cb723001..6cfbd43f8 100644
--- a/toolsrc/src/vcpkg/vcpkgcmdarguments.cpp
+++ b/toolsrc/src/vcpkg/vcpkgcmdarguments.cpp
@@ -182,13 +182,17 @@ namespace vcpkg
     static bool try_parse_argument_as_option(StringView option, StringView argument, T& place, F parser)
     {
         // remove the first two '-'s
-        const auto arg = argument.substr(2);
+        auto arg = argument.substr(2);
         if (arg.size() <= option.size() + 1)
         {
             // it is impossible for this argument to be this option
             return false;
         }
 
+        if (Strings::starts_with(arg, "x-") && !Strings::starts_with(option, "x-"))
+        {
+            arg = arg.substr(2);
+        }
         if (Strings::starts_with(arg, option) && arg.byte_at_index(option.size()) == '=')
         {
             parser(arg.substr(option.size() + 1), option, place);
@@ -198,21 +202,33 @@ namespace vcpkg
         return false;
     }
 
+    static bool equals_modulo_experimental(StringView arg, StringView option)
+    {
+        if (Strings::starts_with(arg, "x-") && !Strings::starts_with(option, "x-"))
+        {
+            return arg.substr(2) == option;
+        }
+        else
+        {
+            return arg == option;
+        }
+    }
+
     // returns true if this does parse this argument as this option
     // REQUIRES: Strings::starts_with(argument, "--");
     template<class T>
     static bool try_parse_argument_as_switch(StringView option, StringView argument, T& place)
     {
         // remove the first two '-'s
-        const auto arg = argument.substr(2);
+        auto arg = argument.substr(2);
 
-        if (arg == option)
+        if (equals_modulo_experimental(arg, option))
         {
             parse_switch(true, option, place);
             return true;
         }
 
-        if (Strings::starts_with(arg, "no-") && arg.substr(3) == option)
+        if (Strings::starts_with(arg, "no-") && equals_modulo_experimental(arg.substr(3), option))
         {
             parse_switch(false, option, place);
             return true;
@@ -451,6 +467,12 @@ namespace vcpkg
                             System::Color::error, "Error: The option '%s' must be passed an argument.\n", option.name);
                         failed = true;
                     }
+                    else if (value.size() > 1)
+                    {
+                        System::printf(
+                            System::Color::error, "Error: The option '%s' can only be passed once.\n", option.name);
+                        failed = true;
+                    }
                     else
                     {
                         output.settings.emplace(option.name, value.front());
@@ -588,6 +610,8 @@ namespace vcpkg
         table.format("", "(default: " + format_environment_variable("VCPKG_DEFAULT_TRIPLET") + ')');
         table.format(opt(OVERLAY_PORTS_ARG, "=", "<path>"), "Specify directories to be used when searching for ports");
         table.format(opt(OVERLAY_TRIPLETS_ARG, "=", "<path>"), "Specify directories containing triplets files");
+        table.format(opt(BINARY_SOURCES_ARG, "=", "<path>"),
+                     "Add sources for binary caching. See 'vcpkg help binarycaching'");
         table.format(opt(DOWNLOADS_ROOT_DIR_ARG, "=", "<path>"), "Specify the downloads root directory");
         table.format("", "(default: " + format_environment_variable("VCPKG_DOWNLOADS") + ')');
         table.format(opt(VCPKG_ROOT_DIR_ARG, " ", "<path>"), "Specify the vcpkg root directory");