[vcpkg] Enable reentrant vcpkg calls (#13751)

* [vcpkg] Enable recursive vcpkg calls

Via envvars VCPKG_COMMAND and VCPKG_X_RECURSIVE_DATA. Child processes can call vcpkg via "$VCPKG_COMMAND <args>" in limited internal circumstances.

* [vcpkg] Address CR comments

* [vcpkg] Do not move through Optional<&> into Optional<T>

Co-authored-by: Robert Schumacher <roschuma@microsoft.com>

11 files changed, 147 insertions, 82 deletions

diff --git a/toolsrc/include/vcpkg/base/optional.h b/toolsrc/include/vcpkg/base/optional.h
index 54e370dff..caacf815e 100644
--- a/toolsrc/include/vcpkg/base/optional.h
+++ b/toolsrc/include/vcpkg/base/optional.h
@@ -29,6 +29,24 @@ namespace vcpkg
             constexpr OptionalStorage() noexcept : m_is_present(false), m_inactive() { }
             constexpr OptionalStorage(const T& t) : m_is_present(true), m_t(t) { }
             constexpr OptionalStorage(T&& t) : m_is_present(true), m_t(std::move(t)) { }
+            template<class U, class = std::enable_if_t<!std::is_reference<U>::value>>
+            constexpr explicit OptionalStorage(Optional<U>&& t) : m_is_present(false), m_inactive()
+            {
+                if (auto p = t.get())
+                {
+                    m_is_present = true;
+                    new (&m_t) T(std::move(*p));
+                }
+            }
+            template<class U>
+            constexpr explicit OptionalStorage(const Optional<U>& t) : m_is_present(false), m_inactive()
+            {
+                if (auto p = t.get())
+                {
+                    m_is_present = true;
+                    new (&m_t) T(*p);
+                }
+            }
 
             ~OptionalStorage() noexcept
             {
diff --git a/toolsrc/include/vcpkg/base/system.h b/toolsrc/include/vcpkg/base/system.h
index 2340728fd..8e45f6538 100644
--- a/toolsrc/include/vcpkg/base/system.h
+++ b/toolsrc/include/vcpkg/base/system.h
@@ -8,6 +8,7 @@
 namespace vcpkg::System
 {
     Optional<std::string> get_environment_variable(ZStringView varname) noexcept;
+    void set_environment_variable(ZStringView varname, Optional<ZStringView> value) noexcept;
 
     const ExpectedS<fs::path>& get_home_dir() noexcept;
 
diff --git a/toolsrc/include/vcpkg/vcpkgcmdarguments.h b/toolsrc/include/vcpkg/vcpkgcmdarguments.h
index 3901000e3..eaf5cc3eb 100644
--- a/toolsrc/include/vcpkg/vcpkgcmdarguments.h
+++ b/toolsrc/include/vcpkg/vcpkgcmdarguments.h
@@ -174,10 +174,13 @@ namespace vcpkg
         constexpr static StringLiteral REGISTRIES_FEATURE = "registries";
         Optional<bool> registries_feature = nullopt;
 
+        constexpr static StringLiteral RECURSIVE_DATA_ENV = "VCPKG_X_RECURSIVE_DATA";
+
         bool binary_caching_enabled() const { return binary_caching.value_or(true); }
         bool compiler_tracking_enabled() const { return compiler_tracking.value_or(true); }
         bool registries_enabled() const { return registries_feature.value_or(false); }
         bool output_json() const { return json.value_or(false); }
+        bool is_recursive_invocation() const { return m_is_recursive_invocation; }
 
         std::string command;
         std::vector<std::string> command_arguments;
@@ -192,6 +195,7 @@ namespace vcpkg
         void track_feature_flag_metrics() const;
 
     private:
+        bool m_is_recursive_invocation = false;
         std::unordered_set<std::string> command_switches;
         std::unordered_map<std::string, std::vector<std::string>> command_options;
     };
diff --git a/toolsrc/src/vcpkg-test/optional.cpp b/toolsrc/src/vcpkg-test/optional.cpp
index f3c61387f..520867f2c 100644
--- a/toolsrc/src/vcpkg-test/optional.cpp
+++ b/toolsrc/src/vcpkg-test/optional.cpp
@@ -58,6 +58,27 @@ TEST_CASE ("ref conversion", "[optional]")
     REQUIRE(cref_1.get() == &x);
 }
 
+TEST_CASE ("value conversion", "[optional]")
+{
+    using vcpkg::Optional;
+
+    Optional<long> j = 1;
+    Optional<int> i = j;
+    Optional<const char*> cstr = "hello, world!";
+    Optional<std::string> cppstr = cstr;
+
+    std::vector<int> v{1, 2, 3};
+    Optional<std::vector<int>&> o_v(v);
+    REQUIRE(o_v.has_value());
+    REQUIRE(o_v.get()->size() == 3);
+    Optional<std::vector<int>> o_w(std::move(o_v));
+    REQUIRE(o_w.has_value());
+    REQUIRE(o_w.get()->size() == 3);
+    // Moving from Optional<&> should not move the underlying object
+    REQUIRE(o_v.has_value());
+    REQUIRE(o_v.get()->size() == 3);
+}
+
 TEST_CASE ("common_projection", "[optional]")
 {
     using vcpkg::Util::common_projection;
diff --git a/toolsrc/src/vcpkg-test/system.cpp b/toolsrc/src/vcpkg-test/system.cpp
index dec7b5769..e7f8aca8f 100644
--- a/toolsrc/src/vcpkg-test/system.cpp
+++ b/toolsrc/src/vcpkg-test/system.cpp
@@ -23,39 +23,11 @@ using vcpkg::Checks::check_exit;
 using vcpkg::System::CPUArchitecture;
 using vcpkg::System::get_environment_variable;
 using vcpkg::System::guess_visual_studio_prompt_target_architecture;
+using vcpkg::System::set_environment_variable;
 using vcpkg::System::to_cpu_architecture;
 
 namespace
 {
-    void set_environment_variable(ZStringView varname, Optional<std::string> value)
-    {
-#if defined(_WIN32)
-        const auto w_varname = vcpkg::Strings::to_utf16(varname);
-        const auto w_varcstr = w_varname.c_str();
-        BOOL exit_code;
-        if (value)
-        {
-            const auto w_value = vcpkg::Strings::to_utf16(value.value_or_exit(VCPKG_LINE_INFO));
-            exit_code = SetEnvironmentVariableW(w_varcstr, w_value.c_str());
-        }
-        else
-        {
-            exit_code = SetEnvironmentVariableW(w_varcstr, nullptr);
-        }
-
-        check_exit(VCPKG_LINE_INFO, exit_code != 0);
-#else  // ^^^ defined(_WIN32) / !defined(_WIN32) vvv
-        if (auto v = value.get())
-        {
-            check_exit(VCPKG_LINE_INFO, setenv(varname.c_str(), v->c_str(), 1) == 0);
-        }
-        else
-        {
-            check_exit(VCPKG_LINE_INFO, unsetenv(varname.c_str()) == 0);
-        }
-#endif // defined(_WIN32)
-    }
-
     struct environment_variable_resetter
     {
         explicit environment_variable_resetter(ZStringView varname_)
diff --git a/toolsrc/src/vcpkg.cpp b/toolsrc/src/vcpkg.cpp
index 7e9b5a044..876449fb8 100644
--- a/toolsrc/src/vcpkg.cpp
+++ b/toolsrc/src/vcpkg.cpp
@@ -200,6 +200,7 @@ int main(const int argc, const char* const* const argv)
 
     System::initialize_global_job_object();
 #endif
+    System::set_environment_variable("VCPKG_COMMAND", fs::generic_u8string(System::get_exe_path_of_current_process()));
 
     Checks::register_global_shutdown_handler([]() {
         const auto elapsed_us_inner = GlobalState::timer.lock()->microseconds();
diff --git a/toolsrc/src/vcpkg/base/system.cpp b/toolsrc/src/vcpkg/base/system.cpp
index cbb50fe27..9429752be 100644
--- a/toolsrc/src/vcpkg/base/system.cpp
+++ b/toolsrc/src/vcpkg/base/system.cpp
@@ -107,6 +107,34 @@ namespace vcpkg
 #endif // defined(_WIN32)
     }
 
+    void System::set_environment_variable(ZStringView varname, Optional<ZStringView> value) noexcept
+    {
+#if defined(_WIN32)
+        const auto w_varname = Strings::to_utf16(varname);
+        const auto w_varcstr = w_varname.c_str();
+        BOOL exit_code;
+        if (auto v = value.get())
+        {
+            exit_code = SetEnvironmentVariableW(w_varcstr, Strings::to_utf16(*v).c_str());
+        }
+        else
+        {
+            exit_code = SetEnvironmentVariableW(w_varcstr, nullptr);
+        }
+
+        Checks::check_exit(VCPKG_LINE_INFO, exit_code != 0);
+#else  // ^^^ defined(_WIN32) / !defined(_WIN32) vvv
+        if (auto v = value.get())
+        {
+            Checks::check_exit(VCPKG_LINE_INFO, setenv(varname.c_str(), v->c_str(), 1) == 0);
+        }
+        else
+        {
+            Checks::check_exit(VCPKG_LINE_INFO, unsetenv(varname.c_str()) == 0);
+        }
+#endif // defined(_WIN32)
+    }
+
     const ExpectedS<fs::path>& System::get_home_dir() noexcept
     {
         static ExpectedS<fs::path> s_home = []() -> ExpectedS<fs::path> {
diff --git a/toolsrc/src/vcpkg/base/system.process.cpp b/toolsrc/src/vcpkg/base/system.process.cpp
index a0cdf0365..16b37dff7 100644
--- a/toolsrc/src/vcpkg/base/system.process.cpp
+++ b/toolsrc/src/vcpkg/base/system.process.cpp
@@ -289,7 +289,6 @@ namespace vcpkg
             L"USERDOMAIN_ROAMINGPROFILE",
             L"USERNAME",
             L"USERPROFILE",
-            L"VCPKG_DISABLE_METRICS",
             L"windir",
             // Enables proxy information to be passed to Curl, the underlying download library in cmake.exe
             L"http_proxy",
diff --git a/toolsrc/src/vcpkg/build.cpp b/toolsrc/src/vcpkg/build.cpp
index 6952b5c3b..da2094cff 100644
--- a/toolsrc/src/vcpkg/build.cpp
+++ b/toolsrc/src/vcpkg/build.cpp
@@ -319,6 +319,17 @@ namespace vcpkg::Build
                     env[env_var] = env_val.value_or_exit(VCPKG_LINE_INFO);
                 }
             }
+            static constexpr StringLiteral s_extra_vars[] = {
+                "VCPKG_COMMAND",
+                "VCPKG_FORCE_SYSTEM_BINARIES",
+                VcpkgCmdArguments::RECURSIVE_DATA_ENV,
+            };
+
+            for (auto var : s_extra_vars)
+            {
+                auto val = System::get_environment_variable(var);
+                if (auto p_val = val.get()) env.emplace(var, *p_val);
+            }
 
             return {env};
         });
diff --git a/toolsrc/src/vcpkg/metrics.cpp b/toolsrc/src/vcpkg/metrics.cpp
index 241531df3..709fc828c 100644
--- a/toolsrc/src/vcpkg/metrics.cpp
+++ b/toolsrc/src/vcpkg/metrics.cpp
@@ -258,23 +258,6 @@ namespace vcpkg::Metrics
 #endif
         ;
 
-    // for child vcpkg processes, we also want to disable metrics
-    static void set_vcpkg_disable_metrics_environment_variable(bool disabled)
-    {
-#if defined(_WIN32)
-        SetEnvironmentVariableW(L"VCPKG_DISABLE_METRICS", disabled ? L"1" : nullptr);
-#else
-        if (disabled)
-        {
-            setenv("VCPKG_DISABLE_METRICS", "1", true);
-        }
-        else
-        {
-            unsetenv("VCPKG_DISABLE_METRICS");
-        }
-#endif
-    }
-
     std::string get_MAC_user()
     {
 #if defined(_WIN32)
@@ -323,11 +306,7 @@ namespace vcpkg::Metrics
 
     void Metrics::set_print_metrics(bool should_print_metrics) { g_should_print_metrics = should_print_metrics; }
 
-    void Metrics::set_disabled(bool disabled)
-    {
-        set_vcpkg_disable_metrics_environment_variable(disabled);
-        g_metrics_disabled = disabled;
-    }
+    void Metrics::set_disabled(bool disabled) { g_metrics_disabled = disabled; }
 
     bool Metrics::metrics_enabled()
     {
diff --git a/toolsrc/src/vcpkg/vcpkgcmdarguments.cpp b/toolsrc/src/vcpkg/vcpkgcmdarguments.cpp
index bd428e8b3..af755408c 100644
--- a/toolsrc/src/vcpkg/vcpkgcmdarguments.cpp
+++ b/toolsrc/src/vcpkg/vcpkgcmdarguments.cpp
@@ -629,8 +629,26 @@ namespace vcpkg
         table.format(opt(JSON_SWITCH, "", ""), "(Experimental) Request JSON output");
     }
 
+    static void from_env(ZStringView var, std::unique_ptr<std::string>& dst)
+    {
+        if (dst) return;
+
+        auto maybe_val = System::get_environment_variable(var);
+        if (auto val = maybe_val.get())
+        {
+            dst = std::make_unique<std::string>(std::move(*val));
+        }
+    }
+
     void VcpkgCmdArguments::imbue_from_environment()
     {
+        static bool s_reentrancy_guard = false;
+        Checks::check_exit(VCPKG_LINE_INFO,
+                           !s_reentrancy_guard,
+                           "VcpkgCmdArguments::imbue_from_environment() modifies global state and thus may only be "
+                           "called once per process.");
+        s_reentrancy_guard = true;
+
         if (!disable_metrics)
         {
             const auto vcpkg_disable_metrics_env = System::get_environment_variable(DISABLE_METRICS_ENV);
@@ -640,14 +658,10 @@ namespace vcpkg
             }
         }
 
-        if (!triplet)
-        {
-            const auto vcpkg_default_triplet_env = System::get_environment_variable(TRIPLET_ENV);
-            if (const auto unpacked = vcpkg_default_triplet_env.get())
-            {
-                triplet = std::make_unique<std::string>(*unpacked);
-            }
-        }
+        from_env(TRIPLET_ENV, triplet);
+        from_env(VCPKG_ROOT_DIR_ENV, vcpkg_root_dir);
+        from_env(DOWNLOADS_ROOT_DIR_ENV, downloads_root_dir);
+        from_env(DEFAULT_VISUAL_STUDIO_PATH_ENV, default_visual_studio_path);
 
         {
             const auto vcpkg_overlay_ports_env = System::get_environment_variable(OVERLAY_PORTS_ENV);
@@ -657,7 +671,6 @@ namespace vcpkg
                 overlay_ports.insert(std::end(overlay_ports), std::begin(overlays), std::end(overlays));
             }
         }
-
         {
             const auto vcpkg_overlay_triplets_env = System::get_environment_variable(OVERLAY_TRIPLETS_ENV);
             if (const auto unpacked = vcpkg_overlay_triplets_env.get())
@@ -666,37 +679,53 @@ namespace vcpkg
                 overlay_triplets.insert(std::end(overlay_triplets), std::begin(triplets), std::end(triplets));
             }
         }
-
-        if (!vcpkg_root_dir)
         {
-            const auto vcpkg_root_env = System::get_environment_variable(VCPKG_ROOT_DIR_ENV);
-            if (const auto unpacked = vcpkg_root_env.get())
+            const auto vcpkg_feature_flags_env = System::get_environment_variable(FEATURE_FLAGS_ENV);
+            if (const auto v = vcpkg_feature_flags_env.get())
             {
-                vcpkg_root_dir = std::make_unique<std::string>(*unpacked);
+                auto flags = Strings::split(*v, ',');
+                parse_feature_flags(flags, *this);
             }
         }
 
-        if (!downloads_root_dir)
         {
-            const auto vcpkg_downloads_env = vcpkg::System::get_environment_variable(DOWNLOADS_ROOT_DIR_ENV);
-            if (const auto unpacked = vcpkg_downloads_env.get())
+            auto maybe_vcpkg_recursive_data = System::get_environment_variable(RECURSIVE_DATA_ENV);
+            if (auto vcpkg_recursive_data = maybe_vcpkg_recursive_data.get())
             {
-                downloads_root_dir = std::make_unique<std::string>(*unpacked);
-            }
-        }
+                m_is_recursive_invocation = true;
 
-        const auto vcpkg_feature_flags_env = System::get_environment_variable(FEATURE_FLAGS_ENV);
-        if (const auto v = vcpkg_feature_flags_env.get())
-        {
-            auto flags = Strings::split(*v, ',');
-            parse_feature_flags(flags, *this);
-        }
+                auto rec_doc = Json::parse(*vcpkg_recursive_data).value_or_exit(VCPKG_LINE_INFO).first;
+                const auto& obj = rec_doc.object();
 
-        {
-            const auto vcpkg_visual_studio_path_env = System::get_environment_variable(DEFAULT_VISUAL_STUDIO_PATH_ENV);
-            if (const auto unpacked = vcpkg_visual_studio_path_env.get())
+                if (auto entry = obj.get(DOWNLOADS_ROOT_DIR_ENV))
+                {
+                    downloads_root_dir = std::make_unique<std::string>(entry->string().to_string());
+                }
+
+                if (obj.get(DISABLE_METRICS_ENV))
+                {
+                    disable_metrics = true;
+                }
+
+                // Setting the recursive data to 'poison' prevents more than one level of recursion because
+                // Json::parse() will fail.
+                System::set_environment_variable(RECURSIVE_DATA_ENV, "poison");
+            }
+            else
             {
-                default_visual_studio_path = std::make_unique<std::string>(*unpacked);
+                Json::Object obj;
+                if (downloads_root_dir)
+                {
+                    obj.insert(DOWNLOADS_ROOT_DIR_ENV, Json::Value::string(*downloads_root_dir.get()));
+                }
+
+                if (disable_metrics)
+                {
+                    obj.insert(DISABLE_METRICS_ENV, Json::Value::boolean(true));
+                }
+
+                System::set_environment_variable(RECURSIVE_DATA_ENV,
+                                                 Json::stringify(obj, Json::JsonStyle::with_spaces(0)));
             }
         }
     }
@@ -904,4 +933,6 @@ namespace vcpkg
     constexpr StringLiteral VcpkgCmdArguments::COMPILER_TRACKING_FEATURE;
     constexpr StringLiteral VcpkgCmdArguments::MANIFEST_MODE_FEATURE;
     constexpr StringLiteral VcpkgCmdArguments::REGISTRIES_FEATURE;
+
+    constexpr StringLiteral VcpkgCmdArguments::RECURSIVE_DATA_ENV;
 }