[vcpkg] Add support for VCPKG_BINARY_SOURCES and --x-binarysource=<> (#10476)

* [vcpkg] Add support for VCPKG_BINARY_SOURCES and --binarysource=<>

* [vcpkg] Rename --binarysource to --x-binarysource to denote internal/experimental

* [vcpkg] Address review comments & add tests for BinaryConfigParser

* [vcpkg] Replace do {} while(1); with for(;;)

Avoids conditional expresion is constant warnings

* [vcpkg] Invert if/else

* [vcpkg] Fix warning in export.prefab.cpp

* [vcpkg] Resolve merge regressions

12 files changed, 559 insertions, 120 deletions

diff --git a/toolsrc/src/vcpkg-test/binaryconfigparser.cpp b/toolsrc/src/vcpkg-test/binaryconfigparser.cpp
new file mode 100644
index 000000000..910d0836f
--- /dev/null
+++ b/toolsrc/src/vcpkg-test/binaryconfigparser.cpp
@@ -0,0 +1,183 @@
+#include <catch2/catch.hpp>
+#include <vcpkg/binarycaching.h>
+
+using namespace vcpkg;
+
+#if defined(_WIN32)
+#define ABSOLUTE_PATH "C:\\foo"
+#else
+#define ABSOLUTE_PATH "/foo"
+#endif
+
+TEST_CASE ("BinaryConfigParser empty", "[binaryconfigparser]")
+{
+    auto parsed = create_binary_provider_from_configs_pure("", {});
+    REQUIRE(parsed.has_value());
+}
+
+TEST_CASE ("BinaryConfigParser unacceptable provider", "[binaryconfigparser]")
+{
+    auto parsed = create_binary_provider_from_configs_pure("unacceptable", {});
+    REQUIRE(!parsed.has_value());
+}
+
+TEST_CASE ("BinaryConfigParser files provider", "[binaryconfigparser]")
+{
+    {
+        auto parsed = create_binary_provider_from_configs_pure("files", {});
+        REQUIRE(!parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("files,relative-path", {});
+        REQUIRE(!parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("files,C:foo", {});
+        REQUIRE(!parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("files," ABSOLUTE_PATH, {});
+        REQUIRE(parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("files," ABSOLUTE_PATH ",nonsense", {});
+        REQUIRE(!parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("files," ABSOLUTE_PATH ",upload", {});
+        REQUIRE(parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("files," ABSOLUTE_PATH ",upload,extra", {});
+        REQUIRE(!parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("files,,upload", {});
+        REQUIRE(!parsed.has_value());
+    }
+}
+
+TEST_CASE ("BinaryConfigParser default provider", "[binaryconfigparser]")
+{
+    {
+        auto parsed = create_binary_provider_from_configs_pure("default", {});
+        REQUIRE(parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("default,nonsense", {});
+        REQUIRE(!parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("default,upload", {});
+        REQUIRE(parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("default,upload,extra", {});
+        REQUIRE(!parsed.has_value());
+    }
+}
+
+TEST_CASE ("BinaryConfigParser clear provider", "[binaryconfigparser]")
+{
+    {
+        auto parsed = create_binary_provider_from_configs_pure("clear", {});
+        REQUIRE(parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("clear,upload", {});
+        REQUIRE(!parsed.has_value());
+    }
+}
+
+TEST_CASE ("BinaryConfigParser multiple providers", "[binaryconfigparser]")
+{
+    {
+        auto parsed = create_binary_provider_from_configs_pure("clear;default", {});
+        REQUIRE(parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("clear;default,upload", {});
+        REQUIRE(parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("clear;default,upload;clear;clear", {});
+        REQUIRE(parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("clear;files,relative;default", {});
+        REQUIRE(!parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure(";;;clear;;;;", {});
+        REQUIRE(parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure(";;;,;;;;", {});
+        REQUIRE(!parsed.has_value());
+    }
+}
+
+TEST_CASE ("BinaryConfigParser escaping", "[binaryconfigparser]")
+{
+    {
+        auto parsed = create_binary_provider_from_configs_pure(";;;;;;;`", {});
+        REQUIRE(!parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure(";;;;;;;`defaul`t", {});
+        REQUIRE(parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("files," ABSOLUTE_PATH "`", {});
+        REQUIRE(!parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("files," ABSOLUTE_PATH "`,", {});
+        REQUIRE(parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("files," ABSOLUTE_PATH "``", {});
+        REQUIRE(parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("files," ABSOLUTE_PATH "```", {});
+        REQUIRE(!parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("files," ABSOLUTE_PATH "````", {});
+        REQUIRE(parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("files," ABSOLUTE_PATH ",", {});
+        REQUIRE(!parsed.has_value());
+    }
+}
+
+TEST_CASE ("BinaryConfigParser args", "[binaryconfigparser]")
+{
+    {
+        auto parsed =
+            create_binary_provider_from_configs_pure("files," ABSOLUTE_PATH, std::vector<std::string>{"clear"});
+        REQUIRE(parsed.has_value());
+    }
+    {
+        auto parsed =
+            create_binary_provider_from_configs_pure("files," ABSOLUTE_PATH, std::vector<std::string>{"clear;default"});
+        REQUIRE(parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("files," ABSOLUTE_PATH,
+                                                               std::vector<std::string>{"clear;default,"});
+        REQUIRE(!parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("files," ABSOLUTE_PATH,
+                                                               std::vector<std::string>{"clear", "clear;default,"});
+        REQUIRE(!parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("files," ABSOLUTE_PATH,
+                                                               std::vector<std::string>{"clear", "clear"});
+        REQUIRE(parsed.has_value());
+    }
+}
diff --git a/toolsrc/src/vcpkg/base/system.cpp b/toolsrc/src/vcpkg/base/system.cpp
index f49632cac..60787727e 100644
--- a/toolsrc/src/vcpkg/base/system.cpp
+++ b/toolsrc/src/vcpkg/base/system.cpp
@@ -105,6 +105,19 @@ namespace vcpkg
 #endif // defined(_WIN32)
     }
 
+    ExpectedS<std::string> System::get_home_dir() noexcept
+    {
+#ifdef _WIN32
+        auto maybe_home = System::get_environment_variable("USERPROFILE");
+        if (!maybe_home.has_value() || maybe_home.get()->empty())
+            return {"unable to read %USERPROFILE%", ExpectedRightTag{}};
+#else
+        auto maybe_home = System::get_environment_variable("HOME");
+        if (!maybe_home.has_value() || maybe_home.get()->empty()) return {"unable to read $HOME", ExpectedRightTag{}};
+#endif
+        return {std::move(*maybe_home.get()), ExpectedLeftTag{}};
+    }
+
 #if defined(_WIN32)
     static bool is_string_keytype(const DWORD hkey_type)
     {
diff --git a/toolsrc/src/vcpkg/base/system.process.cpp b/toolsrc/src/vcpkg/base/system.process.cpp
index 2bb61455e..33d44ab6c 100644
--- a/toolsrc/src/vcpkg/base/system.process.cpp
+++ b/toolsrc/src/vcpkg/base/system.process.cpp
@@ -492,7 +492,7 @@ namespace vcpkg
 
         std::wstring out_env;
 
-        while (1)
+        for (;;)
         {
             auto eq = std::find(it, e, '=');
             if (eq == e) break;
diff --git a/toolsrc/src/vcpkg/binarycaching.cpp b/toolsrc/src/vcpkg/binarycaching.cpp
index c8e77d938..87738c039 100644
--- a/toolsrc/src/vcpkg/binarycaching.cpp
+++ b/toolsrc/src/vcpkg/binarycaching.cpp
@@ -2,6 +2,7 @@
 

 #include <vcpkg/base/checks.h>

 #include <vcpkg/base/files.h>

+#include <vcpkg/base/parse.h>

 #include <vcpkg/base/system.print.h>

 #include <vcpkg/base/system.process.h>

 #include <vcpkg/binarycaching.h>

@@ -60,6 +61,10 @@ namespace
 

     struct ArchivesBinaryProvider : IBinaryProvider

     {

+        ArchivesBinaryProvider(std::vector<fs::path>&& read_dirs, std::vector<fs::path>&& write_dirs)

+            : m_read_dirs(std::move(read_dirs)), m_write_dirs(std::move(write_dirs))

+        {

+        }

         ~ArchivesBinaryProvider() = default;

         void prefetch() override {}

         RestoreResult try_restore(const VcpkgPaths& paths, const Dependencies::InstallPlanAction& action) override

@@ -68,121 +73,136 @@ namespace
             auto& spec = action.spec;

             auto& fs = paths.get_filesystem();

             std::error_code ec;

-            const fs::path archives_root_dir = paths.root / "archives";

-            const std::string archive_name = abi_tag + ".zip";

-            const fs::path archive_subpath = fs::u8path(abi_tag.substr(0, 2)) / archive_name;

-            const fs::path archive_path = archives_root_dir / archive_subpath;

-            const fs::path archive_tombstone_path = archives_root_dir / "fail" / archive_subpath;

-            if (fs.exists(archive_path))

+            for (auto&& archives_root_dir : m_read_dirs)

             {

-                System::print2("Using cached binary package: ", archive_path.u8string(), "\n");

+                const std::string archive_name = abi_tag + ".zip";

+                const fs::path archive_subpath = fs::u8path(abi_tag.substr(0, 2)) / archive_name;

+                const fs::path archive_path = archives_root_dir / archive_subpath;

+                if (fs.exists(archive_path))

+                {

+                    System::print2("Using cached binary package: ", archive_path.u8string(), "\n");

 

-                int archive_result = decompress_archive(paths, spec, archive_path).exit_code;

+                    int archive_result = decompress_archive(paths, spec, archive_path).exit_code;

 

-                if (archive_result != 0)

-                {

-                    System::print2("Failed to decompress archive package\n");

-                    if (action.build_options.purge_decompress_failure == Build::PurgeDecompressFailure::NO)

+                    if (archive_result == 0)

                     {

-                        return RestoreResult::build_failed;

+                        return RestoreResult::success;

                     }

                     else

                     {

-                        System::print2("Purging bad archive\n");

-                        fs.remove(archive_path, ec);

+                        System::print2("Failed to decompress archive package\n");

+                        if (action.build_options.purge_decompress_failure == Build::PurgeDecompressFailure::NO)

+                        {

+                            return RestoreResult::build_failed;

+                        }

+                        else

+                        {

+                            System::print2("Purging bad archive\n");

+                            fs.remove(archive_path, ec);

+                        }

                     }

                 }

-                else

-                {

-                    return RestoreResult::success;

-                }

             }

-

-            if (fs.exists(archive_tombstone_path))

+            for (auto&& archives_root_dir : m_read_dirs)

             {

-                if (action.build_options.fail_on_tombstone == Build::FailOnTombstone::YES)

+                const std::string archive_name = abi_tag + ".zip";

+                const fs::path archive_subpath = fs::u8path(abi_tag.substr(0, 2)) / archive_name;

+                const fs::path archive_tombstone_path = archives_root_dir / "fail" / archive_subpath;

+                if (fs.exists(archive_tombstone_path))

                 {

-                    System::print2("Found failure tombstone: ", archive_tombstone_path.u8string(), "\n");

-                    return RestoreResult::build_failed;

+                    if (action.build_options.fail_on_tombstone == Build::FailOnTombstone::YES)

+                    {

+                        System::print2("Found failure tombstone: ", archive_tombstone_path.u8string(), "\n");

+                        return RestoreResult::build_failed;

+                    }

+                    else

+                    {

+                        System::print2(System::Color::warning,

+                                       "Found failure tombstone: ",

+                                       archive_tombstone_path.u8string(),

+                                       "\n");

+                    }

                 }

                 else

                 {

-                    System::print2(

-                        System::Color::warning, "Found failure tombstone: ", archive_tombstone_path.u8string(), "\n");

+                    const fs::path archive_path = archives_root_dir / archive_subpath;

+                    System::printf("Could not locate cached archive: %s\n", archive_path.u8string());

                 }

             }

-            else

-            {

-                System::printf("Could not locate cached archive: %s\n", archive_path.u8string());

-            }

-

             return RestoreResult::missing;

         }

         void push_success(const VcpkgPaths& paths, const Dependencies::InstallPlanAction& action) override

         {

+            if (m_write_dirs.empty()) return;

             const auto& abi_tag = action.package_abi.value_or_exit(VCPKG_LINE_INFO);

             auto& spec = action.spec;

             auto& fs = paths.get_filesystem();

-            const fs::path archives_root_dir = paths.root / "archives";

-            const std::string archive_name = abi_tag + ".zip";

-            const fs::path archive_subpath = fs::u8path(abi_tag.substr(0, 2)) / archive_name;

-            const fs::path archive_path = archives_root_dir / archive_subpath;

-

             const auto tmp_archive_path = paths.buildtrees / spec.name() / (spec.triplet().to_string() + ".zip");

-

             compress_directory(paths, paths.package_dir(spec), tmp_archive_path);

 

-            fs.create_directories(archive_path.parent_path(), ignore_errors);

-            std::error_code ec;

-            fs.rename_or_copy(tmp_archive_path, archive_path, ".tmp", ec);

-            if (ec)

+            for (auto&& m_directory : m_write_dirs)

             {

-                System::printf(System::Color::warning,

-                               "Failed to store binary cache %s: %s\n",

-                               archive_path.u8string(),

-                               ec.message());

+                const fs::path& archives_root_dir = m_directory;

+                const std::string archive_name = abi_tag + ".zip";

+                const fs::path archive_subpath = fs::u8path(abi_tag.substr(0, 2)) / archive_name;

+                const fs::path archive_path = archives_root_dir / archive_subpath;

+

+                fs.create_directories(archive_path.parent_path(), ignore_errors);

+                std::error_code ec;

+                if (m_write_dirs.size() > 1)

+                    fs.copy_file(tmp_archive_path, archive_path, fs::copy_options::overwrite_existing, ec);

+                else

+                    fs.rename_or_copy(tmp_archive_path, archive_path, ".tmp", ec);

+                if (ec)

+                {

+                    System::printf(System::Color::warning,

+                                   "Failed to store binary cache %s: %s\n",

+                                   archive_path.u8string(),

+                                   ec.message());

+                }

+                else

+                    System::printf("Stored binary cache: %s\n", archive_path.u8string());

             }

-            else

-                System::printf("Stored binary cache: %s\n", archive_path.u8string());

+            if (m_write_dirs.size() > 1) fs.remove(tmp_archive_path, ignore_errors);

         }

         void push_failure(const VcpkgPaths& paths, const std::string& abi_tag, const PackageSpec& spec) override

         {

+            if (m_write_dirs.empty()) return;

             auto& fs = paths.get_filesystem();

             std::error_code ec;

-            const fs::path archives_root_dir = paths.root / "archives";

-            const std::string archive_name = abi_tag + ".zip";

-            const fs::path archive_subpath = fs::u8path(abi_tag.substr(0, 2)) / archive_name;

-            const fs::path archive_path = archives_root_dir / archive_subpath;

-            const fs::path archive_tombstone_path = archives_root_dir / "fail" / archive_subpath;

-            const fs::path abi_package_dir = paths.package_dir(spec) / "share" / spec.name();

-            const fs::path abi_file_in_package = paths.package_dir(spec) / "share" / spec.name() / "vcpkg_abi_info.txt";

-

-            if (!fs.exists(archive_tombstone_path))

+            for (auto&& m_directory : m_write_dirs)

             {

-                // Build failed, store all failure logs in the tombstone.

-                const auto tmp_log_path = paths.buildtrees / spec.name() / "tmp_failure_logs";

-                const auto tmp_log_path_destination = tmp_log_path / spec.name();

-                const auto tmp_failure_zip = paths.buildtrees / spec.name() / "failure_logs.zip";

-                fs.create_directories(tmp_log_path_destination, ignore_errors);

-

-                for (auto& log_file : fs::stdfs::directory_iterator(paths.buildtrees / spec.name()))

+                const fs::path& archives_root_dir = m_directory;

+                const std::string archive_name = abi_tag + ".zip";

+                const fs::path archive_subpath = fs::u8path(abi_tag.substr(0, 2)) / archive_name;

+                const fs::path archive_tombstone_path = archives_root_dir / "fail" / archive_subpath;

+                if (!fs.exists(archive_tombstone_path))

                 {

-                    if (log_file.path().extension() == ".log")

+                    // Build failed, store all failure logs in the tombstone.

+                    const auto tmp_log_path = paths.buildtrees / spec.name() / "tmp_failure_logs";

+                    const auto tmp_log_path_destination = tmp_log_path / spec.name();

+                    const auto tmp_failure_zip = paths.buildtrees / spec.name() / "failure_logs.zip";

+                    fs.create_directories(tmp_log_path_destination, ignore_errors);

+

+                    for (auto& log_file : fs::stdfs::directory_iterator(paths.buildtrees / spec.name()))

                     {

-                        fs.copy_file(log_file.path(),

-                                     tmp_log_path_destination / log_file.path().filename(),

-                                     fs::copy_options::none,

-                                     ec);

+                        if (log_file.path().extension() == ".log")

+                        {

+                            fs.copy_file(log_file.path(),

+                                         tmp_log_path_destination / log_file.path().filename(),

+                                         fs::copy_options::none,

+                                         ec);

+                        }

                     }

-                }

 

-                compress_directory(paths, tmp_log_path, paths.buildtrees / spec.name() / "failure_logs.zip");

+                    compress_directory(paths, tmp_log_path, paths.buildtrees / spec.name() / "failure_logs.zip");

 

-                fs.create_directories(archive_tombstone_path.parent_path(), ignore_errors);

-                fs.rename_or_copy(tmp_failure_zip, archive_tombstone_path, ".tmp", ec);

+                    fs.create_directories(archive_tombstone_path.parent_path(), ignore_errors);

+                    fs.rename_or_copy(tmp_failure_zip, archive_tombstone_path, ".tmp", ec);

 

-                // clean up temporary directory

-                fs.remove_all(tmp_log_path, VCPKG_LINE_INFO);

+                    // clean up temporary directory

+                    fs.remove_all(tmp_log_path, VCPKG_LINE_INFO);

+                }

             }

         }

         RestoreResult precheck(const VcpkgPaths& paths,

@@ -192,35 +212,205 @@ namespace
             const auto& abi_tag = action.package_abi.value_or_exit(VCPKG_LINE_INFO);

             auto& fs = paths.get_filesystem();

             std::error_code ec;

-            const fs::path archives_root_dir = paths.root / "archives";

-            const std::string archive_name = abi_tag + ".zip";

-            const fs::path archive_subpath = fs::u8path(abi_tag.substr(0, 2)) / archive_name;

-            const fs::path archive_path = archives_root_dir / archive_subpath;

-            const fs::path archive_tombstone_path = archives_root_dir / "fail" / archive_subpath;

+            for (auto&& archives_root_dir : m_read_dirs)

+            {

+                const std::string archive_name = abi_tag + ".zip";

+                const fs::path archive_subpath = fs::u8path(abi_tag.substr(0, 2)) / archive_name;

+                const fs::path archive_path = archives_root_dir / archive_subpath;

+

+                if (fs.exists(archive_path))

+                {

+                    return RestoreResult::success;

+                }

+            }

+            for (auto&& archives_root_dir : m_read_dirs)

+            {

+                const std::string archive_name = abi_tag + ".zip";

+                const fs::path archive_subpath = fs::u8path(abi_tag.substr(0, 2)) / archive_name;

+                const fs::path archive_tombstone_path = archives_root_dir / "fail" / archive_subpath;

+

+                if (purge_tombstones)

+                {

+                    fs.remove(archive_tombstone_path, ec); // Ignore error

+                }

+                else if (fs.exists(archive_tombstone_path))

+                {

+                    if (action.build_options.fail_on_tombstone == Build::FailOnTombstone::YES)

+                    {

+                        return RestoreResult::build_failed;

+                    }

+                }

+            }

+            return RestoreResult::missing;

+        }

+

+        std::vector<fs::path> m_read_dirs, m_write_dirs;

+    };

+}

+

+ExpectedS<std::unique_ptr<IBinaryProvider>> vcpkg::create_binary_provider_from_configs(const VcpkgPaths& paths,

+                                                                                       View<std::string> args)

+{

+    std::string env_string = System::get_environment_variable("VCPKG_BINARY_SOURCES").value_or("");

+

+    // Preserve existing behavior until CI can be updated

+    // TODO: remove

+    if (args.size() == 0 && env_string.empty())

+    {

+        auto p = paths.root / fs::u8path("archives");

+        return {std::make_unique<ArchivesBinaryProvider>(std::vector<fs::path>{p}, std::vector<fs::path>{p})};

+    }

 

-            if (fs.exists(archive_path))

+    return create_binary_provider_from_configs_pure(env_string, args);

+}

+ExpectedS<std::unique_ptr<IBinaryProvider>> vcpkg::create_binary_provider_from_configs_pure(

+    const std::string& env_string, View<std::string> args)

+{

+    struct BinaryConfigParser : Parse::ParserBase

+    {

+        using Parse::ParserBase::ParserBase;

+

+        std::vector<fs::path> archives_to_read;

+        std::vector<fs::path> archives_to_write;

+

+        void parse()

+        {

+            while (!at_eof())

             {

-                return RestoreResult::success;

+                std::vector<std::pair<SourceLoc, std::string>> segments;

+

+                for (;;)

+                {

+                    SourceLoc loc = cur_loc();

+                    std::string segment;

+                    for (;;)

+                    {

+                        auto n = match_until([](char32_t ch) { return ch == ',' || ch == '`' || ch == ';'; });

+                        Strings::append(segment, n);

+                        auto ch = cur();

+                        if (ch == '\0' || ch == ',' || ch == ';')

+                            break;

+                        else if (ch == '`')

+                        {

+                            ch = next();

+                            if (ch == '\0')

+                                add_error("unexpected eof: trailing unescaped backticks (`) are not allowed");

+                            else

+                                Unicode::utf8_append_code_point(segment, ch);

+                            next();

+                        }

+                        else

+                            Checks::unreachable(VCPKG_LINE_INFO);

+                    }

+                    segments.emplace_back(std::move(loc), std::move(segment));

+

+                    auto ch = cur();

+                    if (ch == '\0' || ch == ';')

+                        break;

+                    else if (ch == ',')

+                    {

+                        next();

+                        continue;

+                    }

+                    else

+                        Checks::unreachable(VCPKG_LINE_INFO);

+                }

+

+                if (segments.size() != 1 || !segments[0].second.empty()) handle_segments(std::move(segments));

+                segments.clear();

+                if (get_error()) return;

+                if (cur() == ';') next();

             }

+        }

 

-            if (purge_tombstones)

+        void handle_segments(std::vector<std::pair<SourceLoc, std::string>>&& segments)

+        {

+            if (segments.empty()) return;

+            if (segments[0].second == "clear")

             {

-                fs.remove(archive_tombstone_path, ec); // Ignore error

+                if (segments.size() != 1)

+                    return add_error("unexpected arguments: binary config 'clear' does not take arguments",

+                                     segments[1].first);

+                archives_to_read.clear();

+                archives_to_write.clear();

             }

-            else if (fs.exists(archive_tombstone_path))

+            else if (segments[0].second == "files")

             {

-                if (action.build_options.fail_on_tombstone == Build::FailOnTombstone::YES)

+                if (segments.size() < 2)

+                    return add_error("expected arguments: binary config 'files' requires at least a path argument",

+                                     segments[0].first);

+

+                auto p = fs::u8path(segments[1].second);

+                if (!p.is_absolute())

+                    return add_error("expected arguments: path arguments for binary config strings must be absolute",

+                                     segments[1].first);

+

+                if (segments.size() > 3)

                 {

-                    return RestoreResult::build_failed;

+                    return add_error("unexpected arguments: binary config 'files' does not take more than 2 arguments",

+                                     segments[3].first);

                 }

+                else if (segments.size() == 3)

+                {

+                    if (segments[2].second != "upload")

+                    {

+                        return add_error("unexpected arguments: binary config 'files' can only accept 'upload' as "

+                                         "a second argument",

+                                         segments[2].first);

+                    }

+                    else

+                    {

+                        archives_to_write.push_back(p);

+                    }

+                }

+                archives_to_read.push_back(std::move(p));

             }

+            else if (segments[0].second == "default")

+            {

+                if (segments.size() > 2)

+                    return add_error("unexpected arguments: binary config 'default' does not take more than 1 argument",

+                                     segments[0].first);

 

-            return RestoreResult::missing;

+                auto maybe_home = System::get_home_dir();

+                if (!maybe_home.has_value()) return add_error(maybe_home.error(), segments[0].first);

+

+                auto p = fs::u8path(maybe_home.value_or_exit(VCPKG_LINE_INFO)) / fs::u8path(".vcpkg/archives");

+                if (!p.is_absolute())

+                    return add_error("default path was not absolute: " + p.u8string(), segments[0].first);

+                if (segments.size() == 2)

+                {

+                    if (segments[1].second != "upload")

+                    {

+                        return add_error(

+                            "unexpected arguments: binary config 'default' can only accept 'upload' as an argument",

+                            segments[1].first);

+                    }

+                    else

+                    {

+                        archives_to_write.push_back(p);

+                    }

+                }

+                archives_to_read.push_back(std::move(p));

+            }

+            else

+            {

+                return add_error("unknown binary provider type: valid providers are 'clear', 'default', and 'files'",

+                                 segments[0].first);

+            }

         }

     };

-}

 

-std::unique_ptr<vcpkg::IBinaryProvider> vcpkg::create_archives_provider()

-{

-    return std::make_unique<ArchivesBinaryProvider>();

+    BinaryConfigParser env_parser(env_string, "VCPKG_BINARY_SOURCES");

+    env_parser.parse();

+    if (auto err = env_parser.get_error()) return err->format();

+    for (auto&& arg : args)

+    {

+        BinaryConfigParser arg_parser(arg, "<command>");

+        arg_parser.parse();

+        if (auto err = arg_parser.get_error()) return err->format();

+        Util::Vectors::append(&env_parser.archives_to_read, arg_parser.archives_to_read);

+        Util::Vectors::append(&env_parser.archives_to_write, arg_parser.archives_to_write);

+    }

+    return {std::make_unique<ArchivesBinaryProvider>(std::move(env_parser.archives_to_read),

+                                                     std::move(env_parser.archives_to_write))};

 }

diff --git a/toolsrc/src/vcpkg/build.cpp b/toolsrc/src/vcpkg/build.cpp
index b3168075d..5d40e8c24 100644
--- a/toolsrc/src/vcpkg/build.cpp
+++ b/toolsrc/src/vcpkg/build.cpp
@@ -39,6 +39,7 @@ namespace vcpkg::Build::Command
     void perform_and_exit_ex(const FullPackageSpec& full_spec,

                              const SourceControlFileLocation& scfl,

                              const PathsPortFileProvider& provider,

+                             IBinaryProvider& binaryprovider,

                              const VcpkgPaths& paths)

     {

         auto var_provider_storage = CMakeVars::make_triplet_cmake_var_provider(paths);

@@ -93,7 +94,7 @@ namespace vcpkg::Build::Command
         action->build_options = build_package_options;

 

         const auto build_timer = Chrono::ElapsedTimer::create_started();

-        const auto result = Build::build_package(paths, *action, create_archives_provider().get(), status_db);

+        const auto result = Build::build_package(paths, *action, binaryprovider, status_db);

         System::print2("Elapsed time for package ", spec, ": ", build_timer, '\n');

 

         if (result.code == BuildResult::CASCADED_DUE_TO_MISSING_DEPENDENCIES)

@@ -135,6 +136,9 @@ namespace vcpkg::Build::Command
         const ParsedArguments options = args.parse_arguments(COMMAND_STRUCTURE);

         std::string first_arg = args.command_arguments.at(0);

 

+        auto binaryprovider =

+            create_binary_provider_from_configs(paths, args.binarysources).value_or_exit(VCPKG_LINE_INFO);

+

         const FullPackageSpec spec = Input::check_and_get_full_package_spec(

             std::move(first_arg), default_triplet, COMMAND_STRUCTURE.example_text);

 

@@ -146,7 +150,7 @@ namespace vcpkg::Build::Command
 

         Checks::check_exit(VCPKG_LINE_INFO, scfl != nullptr, "Error: Couldn't find port '%s'", port_name);

 

-        perform_and_exit_ex(spec, *scfl, provider, paths);

+        perform_and_exit_ex(spec, *scfl, provider, *binaryprovider, paths);

     }

 }

 

@@ -820,10 +824,10 @@ namespace vcpkg::Build
 

     ExtendedBuildResult build_package(const VcpkgPaths& paths,

                                       const Dependencies::InstallPlanAction& action,

-                                      IBinaryProvider* binaries_provider,

+                                      IBinaryProvider& binaries_provider,

                                       const StatusParagraphs& status_db)

     {

-        auto binary_caching_enabled = binaries_provider && action.build_options.binary_caching == BinaryCaching::YES;

+        auto binary_caching_enabled = action.build_options.binary_caching == BinaryCaching::YES;

 

         auto& fs = paths.get_filesystem();

         auto& spec = action.spec;

@@ -872,7 +876,7 @@ namespace vcpkg::Build
         const fs::path abi_file_in_package = paths.package_dir(spec) / "share" / spec.name() / "vcpkg_abi_info.txt";

         if (binary_caching_enabled)

         {

-            auto restore = binaries_provider->try_restore(paths, action);

+            auto restore = binaries_provider.try_restore(paths, action);

             if (restore == RestoreResult::build_failed)

                 return BuildResult::BUILD_FAILED;

             else if (restore == RestoreResult::success)

@@ -895,12 +899,12 @@ namespace vcpkg::Build
 

         if (binary_caching_enabled && result.code == BuildResult::SUCCEEDED)

         {

-            binaries_provider->push_success(paths, action);

+            binaries_provider.push_success(paths, action);

         }

         else if (binary_caching_enabled &&

                  (result.code == BuildResult::BUILD_FAILED || result.code == BuildResult::POST_BUILD_CHECKS_FAILED))

         {

-            binaries_provider->push_failure(paths, action.package_abi.value_or_exit(VCPKG_LINE_INFO), spec);

+            binaries_provider.push_failure(paths, action.package_abi.value_or_exit(VCPKG_LINE_INFO), spec);

         }

 

         return result;

diff --git a/toolsrc/src/vcpkg/commands.buildexternal.cpp b/toolsrc/src/vcpkg/commands.buildexternal.cpp
index 07ad7b6b1..c9ac0158a 100644
--- a/toolsrc/src/vcpkg/commands.buildexternal.cpp
+++ b/toolsrc/src/vcpkg/commands.buildexternal.cpp
@@ -1,5 +1,6 @@
 #include "pch.h"
 
+#include <vcpkg/binarycaching.h>
 #include <vcpkg/build.h>
 #include <vcpkg/cmakevars.h>
 #include <vcpkg/commands.h>
@@ -20,6 +21,9 @@ namespace vcpkg::Commands::BuildExternal
     {
         const ParsedArguments options = args.parse_arguments(COMMAND_STRUCTURE);
 
+        auto binaryprovider =
+            create_binary_provider_from_configs(paths, args.binarysources).value_or_exit(VCPKG_LINE_INFO);
+
         const FullPackageSpec spec = Input::check_and_get_full_package_spec(
             std::string(args.command_arguments.at(0)), default_triplet, COMMAND_STRUCTURE.example_text);
         Input::check_triplet(spec.package_spec.triplet(), paths);
@@ -33,6 +37,7 @@ namespace vcpkg::Commands::BuildExternal
         Checks::check_exit(
             VCPKG_LINE_INFO, maybe_scfl.has_value(), "could not load control file for %s", spec.package_spec.name());
 
-        Build::Command::perform_and_exit_ex(spec, maybe_scfl.value_or_exit(VCPKG_LINE_INFO), provider, paths);
+        Build::Command::perform_and_exit_ex(
+            spec, maybe_scfl.value_or_exit(VCPKG_LINE_INFO), provider, *binaryprovider, paths);
     }
 }
diff --git a/toolsrc/src/vcpkg/commands.ci.cpp b/toolsrc/src/vcpkg/commands.ci.cpp
index f21366a36..0ea23cd82 100644
--- a/toolsrc/src/vcpkg/commands.ci.cpp
+++ b/toolsrc/src/vcpkg/commands.ci.cpp
@@ -256,7 +256,8 @@ namespace vcpkg::Commands::CI
         const PortFileProvider::PortFileProvider& provider,
         const CMakeVars::CMakeVarProvider& var_provider,
         const std::vector<FullPackageSpec>& specs,
-        const bool purge_tombstones)
+        const bool purge_tombstones,
+        IBinaryProvider& binaryprovider)
     {
         auto ret = std::make_unique<UnknownCIPortsResults>();
 
@@ -306,8 +307,6 @@ namespace vcpkg::Commands::CI
 
         Build::compute_all_abis(paths, action_plan, var_provider, {});
 
-        auto binaryprovider = create_archives_provider();
-
         std::string stdout_buffer;
         for (auto&& action : action_plan.install_actions)
         {
@@ -322,7 +321,7 @@ namespace vcpkg::Commands::CI
                 p->build_options = build_options;
             }
 
-            auto precheck_result = binaryprovider->precheck(paths, action, purge_tombstones);
+            auto precheck_result = binaryprovider.precheck(paths, action, purge_tombstones);
             bool b_will_build = false;
 
             std::string state;
@@ -393,6 +392,9 @@ namespace vcpkg::Commands::CI
             System::print2(System::Color::warning, "Warning: Running ci without binary caching!\n");
         }
 
+        auto binaryprovider =
+            create_binary_provider_from_configs(paths, args.binarysources).value_or_exit(VCPKG_LINE_INFO);
+
         const ParsedArguments options = args.parse_arguments(COMMAND_STRUCTURE);
 
         std::set<std::string> exclusions_set;
@@ -458,8 +460,13 @@ namespace vcpkg::Commands::CI
                 return FullPackageSpec{spec, std::move(default_features)};
             });
 
-            auto split_specs = find_unknown_ports_for_ci(
-                paths, exclusions_set, provider, var_provider, all_default_full_specs, purge_tombstones);
+            auto split_specs = find_unknown_ports_for_ci(paths,
+                                                         exclusions_set,
+                                                         provider,
+                                                         var_provider,
+                                                         all_default_full_specs,
+                                                         purge_tombstones,
+                                                         *binaryprovider);
             PortFileProvider::MapPortFileProvider new_default_provider(split_specs->default_feature_provider);
 
             Dependencies::CreateInstallPlanOptions serialize_options;
@@ -503,7 +510,8 @@ namespace vcpkg::Commands::CI
             else
             {
                 auto collection_timer = Chrono::ElapsedTimer::create_started();
-                auto summary = Install::perform(action_plan, Install::KeepGoing::YES, paths, status_db, var_provider);
+                auto summary = Install::perform(
+                    action_plan, Install::KeepGoing::YES, paths, status_db, *binaryprovider, var_provider);
                 auto collection_time_elapsed = collection_timer.elapsed();
 
                 // Adding results for ports that were built or pulled from an archive
diff --git a/toolsrc/src/vcpkg/commands.setinstalled.cpp b/toolsrc/src/vcpkg/commands.setinstalled.cpp
index 344892fe8..2ef1b5bd2 100644
--- a/toolsrc/src/vcpkg/commands.setinstalled.cpp
+++ b/toolsrc/src/vcpkg/commands.setinstalled.cpp
@@ -1,13 +1,14 @@
 #include "pch.h"
 
 #include <vcpkg/base/system.print.h>
+#include <vcpkg/binarycaching.h>
 #include <vcpkg/commands.h>
 #include <vcpkg/globalstate.h>
 #include <vcpkg/help.h>
 #include <vcpkg/input.h>
 #include <vcpkg/install.h>
-#include <vcpkg/remove.h>
 #include <vcpkg/portfileprovider.h>
+#include <vcpkg/remove.h>
 #include <vcpkg/vcpkglib.h>
 
 namespace vcpkg::Commands::SetInstalled
@@ -35,6 +36,9 @@ namespace vcpkg::Commands::SetInstalled
             Input::check_triplet(spec.package_spec.triplet(), paths);
         }
 
+        auto binaryprovider =
+            create_binary_provider_from_configs(paths, args.binarysources).value_or_exit(VCPKG_LINE_INFO);
+
         const Build::BuildPackageOptions install_plan_options = {
             Build::UseHeadVersion::NO,
             Build::AllowDownloads::YES,
@@ -47,7 +51,6 @@ namespace vcpkg::Commands::SetInstalled
             Build::FailOnTombstone::NO,
         };
 
-
         PortFileProvider::PathsPortFileProvider provider(paths, args.overlay_ports.get());
         auto cmake_vars = CMakeVars::make_triplet_cmake_var_provider(paths);
 
@@ -66,7 +69,8 @@ namespace vcpkg::Commands::SetInstalled
 
         std::set<std::string> all_abis;
 
-        for (const auto& action : action_plan.install_actions) {
+        for (const auto& action : action_plan.install_actions)
+        {
             all_abis.insert(action.package_abi.value_or_exit(VCPKG_LINE_INFO));
         }
 
@@ -101,7 +105,8 @@ namespace vcpkg::Commands::SetInstalled
 
         Dependencies::print_plan(real_action_plan, true);
 
-        const auto summary = Install::perform(real_action_plan, Install::KeepGoing::NO, paths, status_db, *cmake_vars);
+        const auto summary =
+            Install::perform(real_action_plan, Install::KeepGoing::NO, paths, status_db, *binaryprovider, *cmake_vars);
 
         System::print2("\nTotal elapsed time: ", summary.total_elapsed_time, "\n\n");
 
diff --git a/toolsrc/src/vcpkg/commands.upgrade.cpp b/toolsrc/src/vcpkg/commands.upgrade.cpp
index fbce1a421..112a5b6d6 100644
--- a/toolsrc/src/vcpkg/commands.upgrade.cpp
+++ b/toolsrc/src/vcpkg/commands.upgrade.cpp
@@ -1,5 +1,6 @@
 #include "pch.h"
 
+#include <vcpkg/binarycaching.h>
 #include <vcpkg/commands.h>
 #include <vcpkg/dependencies.h>
 #include <vcpkg/globalstate.h>
@@ -41,6 +42,9 @@ namespace vcpkg::Commands::Upgrade
         const bool no_dry_run = Util::Sets::contains(options.switches, OPTION_NO_DRY_RUN);
         const KeepGoing keep_going = to_keep_going(Util::Sets::contains(options.switches, OPTION_KEEP_GOING));
 
+        auto binaryprovider =
+            create_binary_provider_from_configs(paths, args.binarysources).value_or_exit(VCPKG_LINE_INFO);
+
         StatusParagraphs status_db = database_load_check(paths);
 
         // Load ports from ports dirs
@@ -184,7 +188,7 @@ namespace vcpkg::Commands::Upgrade
         var_provider.load_tag_vars(action_plan, provider);
 
         const Install::InstallSummary summary =
-            Install::perform(action_plan, keep_going, paths, status_db, var_provider);
+            Install::perform(action_plan, keep_going, paths, status_db, *binaryprovider, var_provider);
 
         System::print2("\nTotal elapsed time: ", summary.total_elapsed_time, "\n\n");
 
diff --git a/toolsrc/src/vcpkg/export.prefab.cpp b/toolsrc/src/vcpkg/export.prefab.cpp
index 30a3ffa5a..3200159d5 100644
--- a/toolsrc/src/vcpkg/export.prefab.cpp
+++ b/toolsrc/src/vcpkg/export.prefab.cpp
@@ -254,8 +254,10 @@ namespace vcpkg::Export::Prefab
     {
         auto provider = CMakeVars::make_triplet_cmake_var_provider(paths);
 
-        auto build_info = build_info_from_triplet(paths, provider, default_triplet);
-        Checks::check_exit(VCPKG_LINE_INFO, is_supported(build_info), "Currenty supported on android triplets");
+        {
+            auto build_info = build_info_from_triplet(paths, provider, default_triplet);
+            Checks::check_exit(VCPKG_LINE_INFO, is_supported(build_info), "Currenty supported on android triplets");
+        }
 
         std::vector<VcpkgPaths::TripletFile> available_triplets = paths.get_available_triplets();
 
diff --git a/toolsrc/src/vcpkg/install.cpp b/toolsrc/src/vcpkg/install.cpp
index 1b9b7db34..951e40278 100644
--- a/toolsrc/src/vcpkg/install.cpp
+++ b/toolsrc/src/vcpkg/install.cpp
@@ -300,7 +300,7 @@ namespace vcpkg::Install
     static ExtendedBuildResult perform_install_plan_action(const VcpkgPaths& paths,
                                                     InstallPlanAction& action,
                                                     StatusParagraphs& status_db,
-                                                    IBinaryProvider* binaries_provider)
+                                                    IBinaryProvider& binaries_provider)
     {
         const InstallPlanType& plan_type = action.plan_type;
         const std::string display_name = action.spec.to_string();
@@ -425,6 +425,7 @@ namespace vcpkg::Install
                            const KeepGoing keep_going,
                            const VcpkgPaths& paths,
                            StatusParagraphs& status_db,
+                           IBinaryProvider& binaryprovider,
                            const CMakeVars::CMakeVarProvider& var_provider)
     {
         std::vector<SpecSummary> results;
@@ -457,16 +458,15 @@ namespace vcpkg::Install
         for (auto&& action : action_plan.already_installed)
         {
             results.emplace_back(action.spec, &action);
-            results.back().build_result = perform_install_plan_action(paths, action, status_db, nullptr);
+            results.back().build_result = perform_install_plan_action(paths, action, status_db, binaryprovider);
         }
 
         Build::compute_all_abis(paths, action_plan, var_provider, status_db);
 
-        auto binary_provider = create_archives_provider();
         for (auto&& action : action_plan.install_actions)
         {
             with_tracking(action.spec, [&]() {
-                auto result = perform_install_plan_action(paths, action, status_db, binary_provider.get());
+                auto result = perform_install_plan_action(paths, action, status_db, binaryprovider);
 
                 if (result.code != BuildResult::SUCCEEDED && keep_going == KeepGoing::NO)
                 {
@@ -638,6 +638,9 @@ namespace vcpkg::Install
         // input sanitization
         const ParsedArguments options = args.parse_arguments(COMMAND_STRUCTURE);
 
+        auto binaryprovider =
+            create_binary_provider_from_configs(paths, args.binarysources).value_or_exit(VCPKG_LINE_INFO);
+
         const std::vector<FullPackageSpec> specs = Util::fmap(args.command_arguments, [&](auto&& arg) {
             return Input::check_and_get_full_package_spec(
                 std::string(arg), default_triplet, COMMAND_STRUCTURE.example_text);
@@ -731,7 +734,7 @@ namespace vcpkg::Install
                     {
                         const auto vs_prompt_view = to_zstring_view(vs_prompt);
                         System::print2(vcpkg::System::Color::warning,
-                                        "warning: vcpkg appears to be in a Visual Studio prompt targeting ",
+                                       "warning: vcpkg appears to be in a Visual Studio prompt targeting ",
                                        vs_prompt_view,
                                        " but is installing packages for ",
                                        common_triplet.to_string(),
@@ -755,7 +758,8 @@ namespace vcpkg::Install
             Checks::exit_success(VCPKG_LINE_INFO);
         }
 
-        const InstallSummary summary = perform(action_plan, keep_going, paths, status_db, var_provider);
+        const InstallSummary summary =
+            perform(action_plan, keep_going, paths, status_db, *binaryprovider, var_provider);
 
         System::print2("\nTotal elapsed time: ", summary.total_elapsed_time, "\n\n");
 
diff --git a/toolsrc/src/vcpkg/vcpkgcmdarguments.cpp b/toolsrc/src/vcpkg/vcpkgcmdarguments.cpp
index d8deae566..4c49d1108 100644
--- a/toolsrc/src/vcpkg/vcpkgcmdarguments.cpp
+++ b/toolsrc/src/vcpkg/vcpkgcmdarguments.cpp
@@ -79,6 +79,21 @@ namespace vcpkg
         option_field->emplace_back(std::move(new_value));
     }
 
+    static void parse_cojoined_multivalue(std::string new_value,
+                                          const std::string& option_name,
+                                          std::vector<std::string>& option_field)
+    {
+        if (new_value.empty())
+        {
+            System::print2(System::Color::error, "Error: expected value after ", option_name, '\n');
+            Metrics::g_metrics.lock()->track_property("error", "error option name");
+            Help::print_usage();
+            Checks::exit_fail(VCPKG_LINE_INFO);
+        }
+
+        option_field.emplace_back(std::move(new_value));
+    }
+
     VcpkgCmdArguments VcpkgCmdArguments::create_from_command_line(const int argc,
                                                                   const CommandLineCharType* const* const argv)
     {
@@ -176,6 +191,12 @@ namespace vcpkg
                         arg.substr(sizeof("--overlay-triplets=") - 1), "--overlay-triplets", args.overlay_triplets);
                     continue;
                 }
+                if (Strings::starts_with(arg, "--x-binarysource="))
+                {
+                    parse_cojoined_multivalue(
+                        arg.substr(sizeof("--x-binarysource=") - 1), "--x-binarysource", args.binarysources);
+                    continue;
+                }
                 if (arg == "--debug")
                 {
                     parse_switch(true, "debug", args.debug);