[vcpkg] Add experimental x-azblob binary provider (#13626)

* [vcpkg] Add experimental x-azblob binary provider

* [vcpkg] Test azblob storage provider in CI

* [vcpkg] Address some CR comments from #13639

* [vcpkg] Fixup azure-pipelines

* [vcpkg] Fix regression where the downloaded package is purged before decompressing

* [vcpkg] Further refactor vcpkg::Downloads

* [vcpkg] Enable OSX for x-azblob testing

* [vcpkg] Reduce diff against master

* [vcpkg] Extract Downloads::details::split_uri_view

* [vcpkg] Address PR comments

* [vcpkg] Add testing and metrics for x-azblob

* [vcpkg] Add docs for x-azblob

This includes a note that it is currently experimental

* [vcpkg] Address CR comments

* [vcpkg] Revert pipeline changes except OSX to minimize disruption

Co-authored-by: Robert Schumacher <roschuma@microsoft.com>
Co-authored-by: Billy Robert O'Neal III <bion@microsoft.com>

8 files changed, 98 insertions, 38 deletions

diff --git a/toolsrc/include/vcpkg/base/downloads.h b/toolsrc/include/vcpkg/base/downloads.h
index 61c792488..bf97c0650 100644
--- a/toolsrc/include/vcpkg/base/downloads.h
+++ b/toolsrc/include/vcpkg/base/downloads.h
@@ -1,16 +1,41 @@
 #pragma once
 
 #include <vcpkg/base/files.h>
+#include <vcpkg/base/optional.h>
+#include <vcpkg/base/view.h>
 
 namespace vcpkg::Downloads
 {
+    namespace details
+    {
+        struct SplitURIView
+        {
+            StringView scheme;
+            Optional<StringView> authority;
+            StringView path_query_fragment;
+        };
+
+        // e.g. {"https","//example.org", "/index.html"}
+        Optional<SplitURIView> split_uri_view(StringView uri);
+    }
+
     void verify_downloaded_file_hash(const Files::Filesystem& fs,
                                      const std::string& url,
                                      const fs::path& path,
                                      const std::string& sha512);
 
+    // Returns url that was successfully downloaded from
+    std::string download_file(Files::Filesystem& fs,
+                              View<std::string> urls,
+                              const fs::path& download_path,
+                              const std::string& sha512);
+
     void download_file(Files::Filesystem& fs,
                        const std::string& url,
                        const fs::path& download_path,
                        const std::string& sha512);
+
+    std::vector<int> download_files(Files::Filesystem& fs, View<std::pair<std::string, fs::path>> url_pairs);
+    int put_file(const Files::Filesystem&, StringView url, const fs::path& file);
+    std::vector<int> url_heads(View<std::string> urls);
 }
diff --git a/toolsrc/include/vcpkg/base/fwd/lockguarded.h b/toolsrc/include/vcpkg/base/fwd/lockguarded.h
new file mode 100644
index 000000000..e7e88b5a1
--- /dev/null
+++ b/toolsrc/include/vcpkg/base/fwd/lockguarded.h
@@ -0,0 +1,10 @@
+#pragma once
+
+namespace vcpkg::Util
+{
+    template<class T>
+    struct LockGuardPtr;
+
+    template<class T>
+    struct LockGuarded;
+}
diff --git a/toolsrc/include/vcpkg/base/lockguarded.h b/toolsrc/include/vcpkg/base/lockguarded.h
new file mode 100644
index 000000000..e573ec7b0
--- /dev/null
+++ b/toolsrc/include/vcpkg/base/lockguarded.h
@@ -0,0 +1,35 @@
+#pragma once
+
+#include <vcpkg/base/fwd/lockguarded.h>
+
+#include <mutex>
+
+namespace vcpkg::Util
+{
+    template<class T>
+    struct LockGuarded
+    {
+        friend struct LockGuardPtr<T>;
+
+        LockGuardPtr<T> lock() { return *this; }
+
+    private:
+        std::mutex m_mutex;
+        T m_t;
+    };
+
+    template<class T>
+    struct LockGuardPtr
+    {
+        T& operator*() { return m_ptr; }
+        T* operator->() { return &m_ptr; }
+
+        T* get() { return &m_ptr; }
+
+        LockGuardPtr(LockGuarded<T>& sync) : m_lock(sync.m_mutex), m_ptr(sync.m_t) { }
+
+    private:
+        std::lock_guard<std::mutex> m_lock;
+        T& m_ptr;
+    };
+}
diff --git a/toolsrc/include/vcpkg/base/util.h b/toolsrc/include/vcpkg/base/util.h
index 0453ac20a..ca5104c23 100644
--- a/toolsrc/include/vcpkg/base/util.h
+++ b/toolsrc/include/vcpkg/base/util.h
@@ -5,7 +5,6 @@
 #include <algorithm>
 #include <functional>
 #include <map>
-#include <mutex>
 #include <type_traits>
 #include <unordered_map>
 #include <utility>
@@ -222,36 +221,6 @@ namespace vcpkg::Util
         ~ResourceBase() = default;
     };
 
-    template<class T>
-    struct LockGuardPtr;
-
-    template<class T>
-    struct LockGuarded
-    {
-        friend struct LockGuardPtr<T>;
-
-        LockGuardPtr<T> lock() { return *this; }
-
-    private:
-        std::mutex m_mutex;
-        T m_t;
-    };
-
-    template<class T>
-    struct LockGuardPtr
-    {
-        T& operator*() { return m_ptr; }
-        T* operator->() { return &m_ptr; }
-
-        T* get() { return &m_ptr; }
-
-        LockGuardPtr(LockGuarded<T>& sync) : m_lock(sync.m_mutex), m_ptr(sync.m_t) { }
-
-    private:
-        std::unique_lock<std::mutex> m_lock;
-        T& m_ptr;
-    };
-
     namespace Enum
     {
         template<class E>
diff --git a/toolsrc/include/vcpkg/binarycaching.h b/toolsrc/include/vcpkg/binarycaching.h
index 9e77bc763..5a127cb5c 100644
--- a/toolsrc/include/vcpkg/binarycaching.h
+++ b/toolsrc/include/vcpkg/binarycaching.h
@@ -8,8 +8,12 @@
 
 #include <vcpkg/packagespec.h>
 
+#include <unordered_map>
+
 namespace vcpkg
 {
+    struct MergeBinaryProviders;
+
     enum class RestoreResult
     {
         missing,
@@ -20,18 +24,32 @@ namespace vcpkg
     struct IBinaryProvider
     {
         virtual ~IBinaryProvider() = default;
-        /// Gives the BinaryProvider an opportunity to batch any downloading or server communication for executing
-        /// `plan`.
-        virtual void prefetch(const VcpkgPaths& paths, const Dependencies::ActionPlan& plan) = 0;
+
         /// Attempts to restore the package referenced by `action` into the packages directory.
         virtual RestoreResult try_restore(const VcpkgPaths& paths, const Dependencies::InstallPlanAction& action) = 0;
+
         /// Called upon a successful build of `action`
         virtual void push_success(const VcpkgPaths& paths, const Dependencies::InstallPlanAction& action) = 0;
-        /// Requests the result of `try_restore()` without actually downloading the package. Used by CI to determine
-        /// missing packages.
-        virtual RestoreResult precheck(const VcpkgPaths& paths, const Dependencies::InstallPlanAction& action) = 0;
+
+        /// <summary>Gives the BinaryProvider an opportunity to batch any downloading or server communication for
+        /// executing `plan`.</summary>
+        /// <remarks>Must only be called once for a given binary provider instance</remarks>
+        /// <param name="actions">InOut vector of actions to be prefetched</param>
+        virtual void prefetch(const VcpkgPaths& paths,
+                              std::vector<const Dependencies::InstallPlanAction*>& actions) = 0;
+
+        /// <summary>Requests the result of <c>try_restore()</c> without actually downloading the package. Used by CI to
+        /// determine missing packages.</summary>
+        /// <param name="results_map">InOut map to track the restored packages. Should be initialized to
+        /// <c>{&amp;action, RestoreResult::missing}</c> for all install actions</param>
+        virtual void precheck(
+            const VcpkgPaths& paths,
+            std::unordered_map<const Dependencies::InstallPlanAction*, RestoreResult>& results_map) = 0;
     };
 
+    std::unordered_map<const Dependencies::InstallPlanAction*, RestoreResult> binary_provider_precheck(
+        const VcpkgPaths& paths, const Dependencies::ActionPlan& plan, IBinaryProvider& provider);
+
     IBinaryProvider& null_binary_provider();
 
     ExpectedS<std::unique_ptr<IBinaryProvider>> create_binary_provider_from_configs(View<std::string> args);
diff --git a/toolsrc/include/vcpkg/dependencies.h b/toolsrc/include/vcpkg/dependencies.h
index 5e46a6194..1ea81c185 100644
--- a/toolsrc/include/vcpkg/dependencies.h
+++ b/toolsrc/include/vcpkg/dependencies.h
@@ -61,6 +61,7 @@ namespace vcpkg::Dependencies
         std::string displayname() const;
         const std::string& public_abi() const;
         bool has_package_abi() const;
+        Optional<const std::string&> package_abi() const;
         const Build::PreBuildInfo& pre_build_info(LineInfo linfo) const;
 
         PackageSpec spec;
diff --git a/toolsrc/include/vcpkg/globalstate.h b/toolsrc/include/vcpkg/globalstate.h
index 35ca71dbb..44c34f25c 100644
--- a/toolsrc/include/vcpkg/globalstate.h
+++ b/toolsrc/include/vcpkg/globalstate.h
@@ -1,7 +1,8 @@
 #pragma once
 
+#include <vcpkg/base/fwd/lockguarded.h>
+
 #include <vcpkg/base/chrono.h>
-#include <vcpkg/base/util.h>
 
 #include <atomic>
 #include <string>
diff --git a/toolsrc/include/vcpkg/metrics.h b/toolsrc/include/vcpkg/metrics.h
index 3ab1bc758..b1a5e6104 100644
--- a/toolsrc/include/vcpkg/metrics.h
+++ b/toolsrc/include/vcpkg/metrics.h
@@ -1,6 +1,7 @@
 #pragma once
 
 #include <vcpkg/base/files.h>
+#include <vcpkg/base/lockguarded.h>
 #include <vcpkg/base/util.h>
 
 #include <string>