Merge pull request #10032 from ras0219-msft/dev/roschuma/compute-all-abis

[vcpkg] Compute ABIs upfront instead of just-in-time

6 files changed, 360 insertions, 231 deletions

diff --git a/toolsrc/src/vcpkg/binarycaching.cpp b/toolsrc/src/vcpkg/binarycaching.cpp
new file mode 100644
index 000000000..af4a879ca
--- /dev/null
+++ b/toolsrc/src/vcpkg/binarycaching.cpp
@@ -0,0 +1,226 @@
+#include "pch.h"

+

+#include <vcpkg/base/checks.h>

+#include <vcpkg/base/files.h>

+#include <vcpkg/base/system.print.h>

+#include <vcpkg/base/system.process.h>

+#include <vcpkg/binarycaching.h>

+#include <vcpkg/build.h>

+#include <vcpkg/dependencies.h>

+

+using namespace vcpkg;

+

+namespace

+{

+    static System::ExitCodeAndOutput decompress_archive(const VcpkgPaths& paths,

+                                                        const PackageSpec& spec,

+                                                        const fs::path& archive_path)

+    {

+        auto& fs = paths.get_filesystem();

+

+        auto pkg_path = paths.package_dir(spec);

+        fs.remove_all(pkg_path, VCPKG_LINE_INFO);

+        std::error_code ec;

+        fs.create_directories(pkg_path, ec);

+        auto files = fs.get_files_non_recursive(pkg_path);

+        Checks::check_exit(VCPKG_LINE_INFO, files.empty(), "unable to clear path: %s", pkg_path.u8string());

+

+#if defined(_WIN32)

+        auto&& seven_zip_exe = paths.get_tool_exe(Tools::SEVEN_ZIP);

+        auto cmd = Strings::format(

+            R"("%s" x "%s" -o"%s" -y)", seven_zip_exe.u8string(), archive_path.u8string(), pkg_path.u8string());

+#else

+        auto cmd = Strings::format(R"(unzip -qq "%s" "-d%s")", archive_path.u8string(), pkg_path.u8string());

+#endif

+        return System::cmd_execute_and_capture_output(cmd, System::get_clean_environment());

+    }

+

+    // Compress the source directory into the destination file.

+    static void compress_directory(const VcpkgPaths& paths, const fs::path& source, const fs::path& destination)

+    {

+        auto& fs = paths.get_filesystem();

+

+        std::error_code ec;

+

+        fs.remove(destination, ec);

+        Checks::check_exit(

+            VCPKG_LINE_INFO, !fs.exists(destination), "Could not remove file: %s", destination.u8string());

+#if defined(_WIN32)

+        auto&& seven_zip_exe = paths.get_tool_exe(Tools::SEVEN_ZIP);

+

+        System::cmd_execute_and_capture_output(

+            Strings::format(

+                R"("%s" a "%s" "%s\*")", seven_zip_exe.u8string(), destination.u8string(), source.u8string()),

+            System::get_clean_environment());

+#else

+        System::cmd_execute_clean(

+            Strings::format(R"(cd '%s' && zip --quiet -r '%s' *)", source.u8string(), destination.u8string()));

+#endif

+    }

+

+    struct ArchivesBinaryProvider : IBinaryProvider

+    {

+        ~ArchivesBinaryProvider() = default;

+        void prefetch() override {}

+        RestoreResult try_restore(const VcpkgPaths& paths, const Dependencies::InstallPlanAction& action) override

+        {

+            const auto& abi_tag = action.package_abi.value_or_exit(VCPKG_LINE_INFO);

+            auto& spec = action.spec;

+            auto& fs = paths.get_filesystem();

+            std::error_code ec;

+            const fs::path archives_root_dir = paths.root / "archives";

+            const std::string archive_name = abi_tag + ".zip";

+            const fs::path archive_subpath = fs::u8path(abi_tag.substr(0, 2)) / archive_name;

+            const fs::path archive_path = archives_root_dir / archive_subpath;

+            const fs::path archive_tombstone_path = archives_root_dir / "fail" / archive_subpath;

+            if (fs.exists(archive_path))

+            {

+                System::print2("Using cached binary package: ", archive_path.u8string(), "\n");

+

+                int archive_result = decompress_archive(paths, spec, archive_path).exit_code;

+

+                if (archive_result != 0)

+                {

+                    System::print2("Failed to decompress archive package\n");

+                    if (action.build_options.purge_decompress_failure == Build::PurgeDecompressFailure::NO)

+                    {

+                        return RestoreResult::build_failed;

+                    }

+                    else

+                    {

+                        System::print2("Purging bad archive\n");

+                        fs.remove(archive_path, ec);

+                    }

+                }

+                else

+                {

+                    return RestoreResult::success;

+                }

+            }

+

+            if (fs.exists(archive_tombstone_path))

+            {

+                if (action.build_options.fail_on_tombstone == Build::FailOnTombstone::YES)

+                {

+                    System::print2("Found failure tombstone: ", archive_tombstone_path.u8string(), "\n");

+                    return RestoreResult::build_failed;

+                }

+                else

+                {

+                    System::print2(

+                        System::Color::warning, "Found failure tombstone: ", archive_tombstone_path.u8string(), "\n");

+                }

+            }

+            else

+            {

+                System::printf("Could not locate cached archive: %s\n", archive_path.u8string());

+            }

+

+            return RestoreResult::missing;

+        }

+        void push_success(const VcpkgPaths& paths, const Dependencies::InstallPlanAction& action) override

+        {

+            const auto& abi_tag = action.package_abi.value_or_exit(VCPKG_LINE_INFO);

+            auto& spec = action.spec;

+            auto& fs = paths.get_filesystem();

+            std::error_code ec;

+            const fs::path archives_root_dir = paths.root / "archives";

+            const std::string archive_name = abi_tag + ".zip";

+            const fs::path archive_subpath = fs::u8path(abi_tag.substr(0, 2)) / archive_name;

+            const fs::path archive_path = archives_root_dir / archive_subpath;

+

+            const auto tmp_archive_path = paths.buildtrees / spec.name() / (spec.triplet().to_string() + ".zip");

+

+            compress_directory(paths, paths.package_dir(spec), tmp_archive_path);

+

+            fs.create_directories(archive_path.parent_path(), ec);

+            fs.rename_or_copy(tmp_archive_path, archive_path, ".tmp", ec);

+            if (ec)

+            {

+                System::printf(System::Color::warning,

+                               "Failed to store binary cache %s: %s\n",

+                               archive_path.u8string(),

+                               ec.message());

+            }

+            else

+                System::printf("Stored binary cache: %s\n", archive_path.u8string());

+        }

+        void push_failure(const VcpkgPaths& paths, const std::string& abi_tag, const PackageSpec& spec) override

+        {

+            auto& fs = paths.get_filesystem();

+            std::error_code ec;

+            const fs::path archives_root_dir = paths.root / "archives";

+            const std::string archive_name = abi_tag + ".zip";

+            const fs::path archive_subpath = fs::u8path(abi_tag.substr(0, 2)) / archive_name;

+            const fs::path archive_path = archives_root_dir / archive_subpath;

+            const fs::path archive_tombstone_path = archives_root_dir / "fail" / archive_subpath;

+            const fs::path abi_package_dir = paths.package_dir(spec) / "share" / spec.name();

+            const fs::path abi_file_in_package = paths.package_dir(spec) / "share" / spec.name() / "vcpkg_abi_info.txt";

+

+            if (!fs.exists(archive_tombstone_path))

+            {

+                // Build failed, store all failure logs in the tombstone.

+                const auto tmp_log_path = paths.buildtrees / spec.name() / "tmp_failure_logs";

+                const auto tmp_log_path_destination = tmp_log_path / spec.name();

+                const auto tmp_failure_zip = paths.buildtrees / spec.name() / "failure_logs.zip";

+                fs.create_directories(tmp_log_path_destination, ec);

+

+                for (auto& log_file : fs::stdfs::directory_iterator(paths.buildtrees / spec.name()))

+                {

+                    if (log_file.path().extension() == ".log")

+                    {

+                        fs.copy_file(log_file.path(),

+                                     tmp_log_path_destination / log_file.path().filename(),

+                                     fs::stdfs::copy_options::none,

+                                     ec);

+                    }

+                }

+

+                compress_directory(paths, tmp_log_path, paths.buildtrees / spec.name() / "failure_logs.zip");

+

+                fs.create_directories(archive_tombstone_path.parent_path(), ec);

+                fs.rename_or_copy(tmp_failure_zip, archive_tombstone_path, ".tmp", ec);

+

+                // clean up temporary directory

+                fs.remove_all(tmp_log_path, VCPKG_LINE_INFO);

+            }

+        }

+        RestoreResult precheck(const VcpkgPaths& paths,

+                               const Dependencies::InstallPlanAction& action,

+                               bool purge_tombstones) override

+        {

+            const auto& abi_tag = action.package_abi.value_or_exit(VCPKG_LINE_INFO);

+            auto& fs = paths.get_filesystem();

+            std::error_code ec;

+            const fs::path archives_root_dir = paths.root / "archives";

+            const std::string archive_name = abi_tag + ".zip";

+            const fs::path archive_subpath = fs::u8path(abi_tag.substr(0, 2)) / archive_name;

+            const fs::path archive_path = archives_root_dir / archive_subpath;

+            const fs::path archive_tombstone_path = archives_root_dir / "fail" / archive_subpath;

+

+            if (fs.exists(archive_path))

+            {

+                return RestoreResult::success;

+            }

+

+            if (purge_tombstones)

+            {

+                fs.remove(archive_tombstone_path, ec); // Ignore error

+            }

+            else if (fs.exists(archive_tombstone_path))

+            {

+                if (action.build_options.fail_on_tombstone == Build::FailOnTombstone::YES)

+                {

+                    return RestoreResult::build_failed;

+                }

+            }

+

+            return RestoreResult::missing;

+        }

+    };

+}

+

+std::unique_ptr<vcpkg::IBinaryProvider> vcpkg::create_archives_provider()

+{

+    return std::make_unique<ArchivesBinaryProvider>();

+}

diff --git a/toolsrc/src/vcpkg/build.cpp b/toolsrc/src/vcpkg/build.cpp
index f6c310415..81997e79c 100644
--- a/toolsrc/src/vcpkg/build.cpp
+++ b/toolsrc/src/vcpkg/build.cpp
@@ -12,6 +12,7 @@
 #include <vcpkg/base/system.process.h>

 #include <vcpkg/base/util.h>

 

+#include <vcpkg/binarycaching.h>

 #include <vcpkg/build.h>

 #include <vcpkg/commands.h>

 #include <vcpkg/dependencies.h>

@@ -38,11 +39,8 @@ namespace vcpkg::Build::Command
     void perform_and_exit_ex(const FullPackageSpec& full_spec,

                              const SourceControlFileLocation& scfl,

                              const PathsPortFileProvider& provider,

-                             const ParsedArguments& options,

                              const VcpkgPaths& paths)

     {

-        vcpkg::Util::unused(options);

-

         auto var_provider_storage = CMakeVars::make_triplet_cmake_var_provider(paths);

         auto& var_provider = *var_provider_storage;

         var_provider.load_dep_info_vars(std::array<PackageSpec, 1>{full_spec.package_spec});

@@ -95,7 +93,7 @@ namespace vcpkg::Build::Command
         action->build_options = build_package_options;

 

         const auto build_timer = Chrono::ElapsedTimer::create_started();

-        const auto result = Build::build_package(paths, *action, var_provider, status_db);

+        const auto result = Build::build_package(paths, *action, create_archives_provider().get(), status_db);

         System::print2("Elapsed time for package ", spec, ": ", build_timer, '\n');

 

         if (result.code == BuildResult::CASCADED_DUE_TO_MISSING_DEPENDENCIES)

@@ -148,7 +146,7 @@ namespace vcpkg::Build::Command
 

         Checks::check_exit(VCPKG_LINE_INFO, scfl != nullptr, "Error: Couldn't find port '%s'", port_name);

 

-        perform_and_exit_ex(spec, *scfl, provider, options, paths);

+        perform_and_exit_ex(spec, *scfl, provider, paths);

     }

 }

 

@@ -464,11 +462,10 @@ namespace vcpkg::Build
         return hash;

     }

 

-    static ExtendedBuildResult do_build_package(const VcpkgPaths& paths,

-                                                const PreBuildInfo& pre_build_info,

-                                                const std::string& abi_tag,

-                                                const Dependencies::InstallPlanAction& action)

+    static ExtendedBuildResult do_build_package(const VcpkgPaths& paths, const Dependencies::InstallPlanAction& action)

     {

+        const auto& pre_build_info = *action.pre_build_info.value_or_exit(VCPKG_LINE_INFO).get();

+

         auto& fs = paths.get_filesystem();

         auto&& scfl = action.source_control_file_location.value_or_exit(VCPKG_LINE_INFO);

 

@@ -530,7 +527,11 @@ namespace vcpkg::Build
         {

             std::error_code err;

             fs.create_directory(buildpath, err);

-            Checks::check_exit(VCPKG_LINE_INFO, !err.value(), "Failed to create directory '%s', code: %d", buildpath.u8string(), err.value());

+            Checks::check_exit(VCPKG_LINE_INFO,

+                               !err.value(),

+                               "Failed to create directory '%s', code: %d",

+                               buildpath.u8string(),

+                               err.value());

         }

         auto stdoutlog = buildpath / ("stdout-" + action.spec.triplet().canonical_name() + ".log");

         std::ofstream out_file(stdoutlog.native().c_str(), std::ios::out | std::ios::binary | std::ios::trunc);

@@ -584,8 +585,11 @@ namespace vcpkg::Build
         auto find_itr = action.feature_dependencies.find("core");

         Checks::check_exit(VCPKG_LINE_INFO, find_itr != action.feature_dependencies.end());

 

-        std::unique_ptr<BinaryControlFile> bcf = create_binary_control_file(

-            *scfl.source_control_file->core_paragraph, triplet, build_info, abi_tag, std::move(find_itr->second));

+        std::unique_ptr<BinaryControlFile> bcf = create_binary_control_file(*scfl.source_control_file->core_paragraph,

+                                                                            triplet,

+                                                                            build_info,

+                                                                            action.public_abi(),

+                                                                            std::move(find_itr->second));

 

         if (error_count != 0)

         {

@@ -611,11 +615,9 @@ namespace vcpkg::Build
     }

 

     static ExtendedBuildResult do_build_package_and_clean_buildtrees(const VcpkgPaths& paths,

-                                                                     const PreBuildInfo& pre_build_info,

-                                                                     const std::string& abi_tag,

                                                                      const Dependencies::InstallPlanAction& action)

     {

-        auto result = do_build_package(paths, pre_build_info, abi_tag, action);

+        auto result = do_build_package(paths, action);

 

         if (action.build_options.clean_buildtrees == CleanBuildtrees::YES)

         {

@@ -638,12 +640,12 @@ namespace vcpkg::Build
 

     Optional<AbiTagAndFile> compute_abi_tag(const VcpkgPaths& paths,

                                             const Dependencies::InstallPlanAction& action,

-                                            const PreBuildInfo& pre_build_info,

                                             Span<const AbiEntry> dependency_abis)

     {

         auto& fs = paths.get_filesystem();

         Triplet triplet = action.spec.triplet();

         const std::string& name = action.spec.name();

+        const auto& pre_build_info = *action.pre_build_info.value_or_exit(VCPKG_LINE_INFO);

 

         std::vector<AbiEntry> abi_tag_entries(dependency_abis.begin(), dependency_abis.end());

 

@@ -757,75 +759,81 @@ namespace vcpkg::Build
         return nullopt;

     }

 

-    static System::ExitCodeAndOutput decompress_archive(const VcpkgPaths& paths,

-                                                        const PackageSpec& spec,

-                                                        const fs::path& archive_path)

-    {

-        auto& fs = paths.get_filesystem();

-

-        auto pkg_path = paths.package_dir(spec);

-        fs.remove_all(pkg_path, VCPKG_LINE_INFO);

-        std::error_code ec;

-        fs.create_directories(pkg_path, ec);

-        auto files = fs.get_files_non_recursive(pkg_path);

-        Checks::check_exit(VCPKG_LINE_INFO, files.empty(), "unable to clear path: %s", pkg_path.u8string());

-

-#if defined(_WIN32)

-        auto&& seven_zip_exe = paths.get_tool_exe(Tools::SEVEN_ZIP);

-        auto cmd = Strings::format(

-            R"("%s" x "%s" -o"%s" -y)", seven_zip_exe.u8string(), archive_path.u8string(), pkg_path.u8string());

-#else

-        auto cmd = Strings::format(R"(unzip -qq "%s" "-d%s")", archive_path.u8string(), pkg_path.u8string());

-#endif

-        return System::cmd_execute_and_capture_output(cmd, System::get_clean_environment());

-    }

-

-    // Compress the source directory into the destination file.

-    static void compress_directory(const VcpkgPaths& paths, const fs::path& source, const fs::path& destination)

+    void compute_all_abis(const VcpkgPaths& paths,

+                          Dependencies::ActionPlan& action_plan,

+                          const CMakeVars::CMakeVarProvider& var_provider,

+                          const StatusParagraphs& status_db)

     {

-        auto& fs = paths.get_filesystem();

-

-        std::error_code ec;

-

-        fs.remove(destination, ec);

-        Checks::check_exit(

-            VCPKG_LINE_INFO, !fs.exists(destination), "Could not remove file: %s", destination.u8string());

-#if defined(_WIN32)

-        auto&& seven_zip_exe = paths.get_tool_exe(Tools::SEVEN_ZIP);

+        using Dependencies::InstallPlanAction;

+        for (auto it = action_plan.install_actions.begin(); it != action_plan.install_actions.end(); ++it)

+        {

+            auto& action = *it;

+            std::vector<AbiEntry> dependency_abis;

+            if (!Util::Enum::to_bool(action.build_options.only_downloads))

+            {

+                for (auto&& pspec : action.package_dependencies)

+                {

+                    if (pspec == action.spec) continue;

 

-        System::cmd_execute_and_capture_output(

-            Strings::format(

-                R"("%s" a "%s" "%s\*")", seven_zip_exe.u8string(), destination.u8string(), source.u8string()),

-            System::get_clean_environment());

-#else

-        System::cmd_execute_clean(

-            Strings::format(R"(cd '%s' && zip --quiet -r '%s' *)", source.u8string(), destination.u8string()));

-#endif

-    }

+                    auto pred = [&](const InstallPlanAction& ipa) { return ipa.spec == pspec; };

+                    auto it2 = std::find_if(action_plan.install_actions.begin(), it, pred);

+                    if (it2 == it)

+                    {

+                        // Finally, look in current installed

+                        auto status_it = status_db.find(pspec);

+                        if (status_it == status_db.end())

+                        {

+                            Checks::exit_with_message(

+                                VCPKG_LINE_INFO, "Failed to find dependency abi for %s -> %s", action.spec, pspec);

+                        }

+                        else

+                        {

+                            dependency_abis.emplace_back(AbiEntry{pspec.name(), status_it->get()->package.abi});

+                        }

+                    }

+                    else

+                    {

+                        dependency_abis.emplace_back(AbiEntry{pspec.name(), it2->public_abi()});

+                    }

+                }

+            }

 

-    static void compress_archive(const VcpkgPaths& paths, const PackageSpec& spec, const fs::path& destination)

-    {

-        compress_directory(paths, paths.package_dir(spec), destination);

+            action.pre_build_info = std::make_unique<PreBuildInfo>(

+                paths, action.spec.triplet(), var_provider.get_tag_vars(action.spec).value_or_exit(VCPKG_LINE_INFO));

+            auto maybe_abi_tag_and_file = compute_abi_tag(paths, action, dependency_abis);

+            if (auto p = maybe_abi_tag_and_file.get())

+            {

+                action.abi_tag_file = std::move(p->tag_file);

+                action.package_abi = std::move(p->tag);

+            }

+            else

+            {

+                action.package_abi = "";

+            }

+        }

     }

 

     ExtendedBuildResult build_package(const VcpkgPaths& paths,

                                       const Dependencies::InstallPlanAction& action,

-                                      const CMakeVars::CMakeVarProvider& var_provider,

+                                      IBinaryProvider* binaries_provider,

                                       const StatusParagraphs& status_db)

     {

+        auto binary_caching_enabled = binaries_provider && action.build_options.binary_caching == BinaryCaching::YES;

+

         auto& fs = paths.get_filesystem();

         Triplet triplet = action.spec.triplet();

+        auto& spec = action.spec;

         const std::string& name = action.source_control_file_location.value_or_exit(VCPKG_LINE_INFO)

                                       .source_control_file->core_paragraph->name;

 

         std::vector<FeatureSpec> missing_fspecs;

         for (const auto& kv : action.feature_dependencies)

         {

-            for (const FeatureSpec& spec : kv.second)

+            for (const FeatureSpec& fspec : kv.second)

             {

-                if (!(status_db.is_installed(spec) || spec.name() == name))

+                if (!(status_db.is_installed(fspec) || spec.name() == name))

                 {

-                    missing_fspecs.emplace_back(spec);

+                    missing_fspecs.emplace_back(fspec);

                 }

             }

         }

@@ -835,8 +843,6 @@ namespace vcpkg::Build
             return {BuildResult::CASCADED_DUE_TO_MISSING_DEPENDENCIES, std::move(missing_fspecs)};

         }

 

-        const PackageSpec spec(name, triplet);

-

         std::vector<AbiEntry> dependency_abis;

         for (auto&& pspec : action.package_dependencies)

         {

@@ -850,129 +856,47 @@ namespace vcpkg::Build
                 AbiEntry{status_it->get()->package.spec.name(), status_it->get()->package.abi});

         }

 

-        const std::unordered_map<std::string, std::string>& cmake_vars =

-            var_provider.get_tag_vars(spec).value_or_exit(VCPKG_LINE_INFO);

-        const PreBuildInfo pre_build_info(paths, triplet, cmake_vars);

-

-        auto maybe_abi_tag_and_file = compute_abi_tag(paths, action, pre_build_info, dependency_abis);

-        if (!maybe_abi_tag_and_file)

+        if (!action.abi_tag_file)

         {

-            return do_build_package_and_clean_buildtrees(

-                paths, pre_build_info, pre_build_info.public_abi_override.value_or(AbiTagAndFile{}.tag), action);

+            return do_build_package_and_clean_buildtrees(paths, action);

         }

 

+        auto& abi_file = *action.abi_tag_file.get();

+

         std::error_code ec;

-        const auto abi_tag_and_file = maybe_abi_tag_and_file.get();

-        const fs::path archives_root_dir = paths.root / "archives";

-        const std::string archive_name = abi_tag_and_file->tag + ".zip";

-        const fs::path archive_subpath = fs::u8path(abi_tag_and_file->tag.substr(0, 2)) / archive_name;

-        const fs::path archive_path = archives_root_dir / archive_subpath;

-        const fs::path archive_tombstone_path = archives_root_dir / "fail" / archive_subpath;

         const fs::path abi_package_dir = paths.package_dir(spec) / "share" / spec.name();

         const fs::path abi_file_in_package = paths.package_dir(spec) / "share" / spec.name() / "vcpkg_abi_info.txt";

-

-        if (action.build_options.binary_caching == BinaryCaching::YES)

+        if (binary_caching_enabled)

         {

-            if (fs.exists(archive_path))

+            auto restore = binaries_provider->try_restore(paths, action);

+            if (restore == RestoreResult::build_failed)

+                return BuildResult::BUILD_FAILED;

+            else if (restore == RestoreResult::success)

             {

-                System::print2("Using cached binary package: ", archive_path.u8string(), "\n");

-

-                int archive_result = decompress_archive(paths, spec, archive_path).exit_code;

-

-                if (archive_result != 0)

-                {

-                    System::print2("Failed to decompress archive package\n");

-                    if (action.build_options.purge_decompress_failure == PurgeDecompressFailure::NO)

-                    {

-                        return BuildResult::BUILD_FAILED;

-                    }

-                    else

-                    {

-                        System::print2("Purging bad archive\n");

-                        fs.remove(archive_path, ec);

-                    }

-                }

-                else

-                {

-                    auto maybe_bcf = Paragraphs::try_load_cached_package(paths, spec);

-                    auto bcf = std::make_unique<BinaryControlFile>(std::move(maybe_bcf).value_or_exit(VCPKG_LINE_INFO));

-                    return {BuildResult::SUCCEEDED, std::move(bcf)};

-                }

+                auto maybe_bcf = Paragraphs::try_load_cached_package(paths, spec);

+                auto bcf = std::make_unique<BinaryControlFile>(std::move(maybe_bcf).value_or_exit(VCPKG_LINE_INFO));

+                return {BuildResult::SUCCEEDED, std::move(bcf)};

             }

-

-            if (fs.exists(archive_tombstone_path))

+            else

             {

-                if (action.build_options.fail_on_tombstone == FailOnTombstone::YES)

-                {

-                    System::print2("Found failure tombstone: ", archive_tombstone_path.u8string(), "\n");

-                    return BuildResult::BUILD_FAILED;

-                }

-                else

-                {

-                    System::print2(

-                        System::Color::warning, "Found failure tombstone: ", archive_tombstone_path.u8string(), "\n");

-                }

+                // missing package, proceed to build.

             }

-

-            System::printf("Could not locate cached archive: %s\n", archive_path.u8string());

         }

 

-        ExtendedBuildResult result = do_build_package_and_clean_buildtrees(

-            paths, pre_build_info, pre_build_info.public_abi_override.value_or(abi_tag_and_file->tag), action);

+        ExtendedBuildResult result = do_build_package_and_clean_buildtrees(paths, action);

 

         fs.create_directories(abi_package_dir, ec);

-        Checks::check_exit(VCPKG_LINE_INFO, !ec, "Coud not create directory %s", abi_package_dir.u8string());

-        fs.copy_file(abi_tag_and_file->tag_file, abi_file_in_package, fs::stdfs::copy_options::none, ec);

+        fs.copy_file(abi_file, abi_file_in_package, fs::stdfs::copy_options::none, ec);

         Checks::check_exit(VCPKG_LINE_INFO, !ec, "Could not copy into file: %s", abi_file_in_package.u8string());

 

-        if (action.build_options.binary_caching == BinaryCaching::YES && result.code == BuildResult::SUCCEEDED)

+        if (binary_caching_enabled && result.code == BuildResult::SUCCEEDED)

         {

-            const auto tmp_archive_path = paths.buildtrees / spec.name() / (spec.triplet().to_string() + ".zip");

-

-            compress_archive(paths, spec, tmp_archive_path);

-

-            fs.create_directories(archive_path.parent_path(), ec);

-            fs.rename_or_copy(tmp_archive_path, archive_path, ".tmp", ec);

-            if (ec)

-            {

-                System::printf(System::Color::warning,

-                               "Failed to store binary cache %s: %s\n",

-                               archive_path.u8string(),

-                               ec.message());

-            }

-            else

-                System::printf("Stored binary cache: %s\n", archive_path.u8string());

+            binaries_provider->push_success(paths, action);

         }

-        else if (action.build_options.binary_caching == BinaryCaching::YES &&

+        else if (binary_caching_enabled &&

                  (result.code == BuildResult::BUILD_FAILED || result.code == BuildResult::POST_BUILD_CHECKS_FAILED))

         {

-            if (!fs.exists(archive_tombstone_path))

-            {

-                // Build failed, store all failure logs in the tombstone.

-                const auto tmp_log_path = paths.buildtrees / spec.name() / "tmp_failure_logs";

-                const auto tmp_log_path_destination = tmp_log_path / spec.name();

-                const auto tmp_failure_zip = paths.buildtrees / spec.name() / "failure_logs.zip";

-                fs.create_directories(tmp_log_path_destination, ec);

-

-                for (auto& log_file : fs::stdfs::directory_iterator(paths.buildtrees / spec.name()))

-                {

-                    if (log_file.path().extension() == ".log")

-                    {

-                        fs.copy_file(log_file.path(),

-                                     tmp_log_path_destination / log_file.path().filename(),

-                                     fs::stdfs::copy_options::none,

-                                     ec);

-                    }

-                }

-

-                compress_directory(paths, tmp_log_path, paths.buildtrees / spec.name() / "failure_logs.zip");

-

-                fs.create_directories(archive_tombstone_path.parent_path(), ec);

-                fs.rename_or_copy(tmp_failure_zip, archive_tombstone_path, ".tmp", ec);

-

-                // clean up temporary directory

-                fs.remove_all(tmp_log_path, VCPKG_LINE_INFO);

-            }

+            binaries_provider->push_failure(paths, action.package_abi.value_or_exit(VCPKG_LINE_INFO), spec);

         }

 

         return result;

diff --git a/toolsrc/src/vcpkg/commands.buildexternal.cpp b/toolsrc/src/vcpkg/commands.buildexternal.cpp
index 4a69be879..07ad7b6b1 100644
--- a/toolsrc/src/vcpkg/commands.buildexternal.cpp
+++ b/toolsrc/src/vcpkg/commands.buildexternal.cpp
@@ -33,6 +33,6 @@ namespace vcpkg::Commands::BuildExternal
         Checks::check_exit(
             VCPKG_LINE_INFO, maybe_scfl.has_value(), "could not load control file for %s", spec.package_spec.name());
 
-        Build::Command::perform_and_exit_ex(spec, maybe_scfl.value_or_exit(VCPKG_LINE_INFO), provider, options, paths);
+        Build::Command::perform_and_exit_ex(spec, maybe_scfl.value_or_exit(VCPKG_LINE_INFO), provider, paths);
     }
 }
diff --git a/toolsrc/src/vcpkg/commands.ci.cpp b/toolsrc/src/vcpkg/commands.ci.cpp
index 2fbe78b42..f21366a36 100644
--- a/toolsrc/src/vcpkg/commands.ci.cpp
+++ b/toolsrc/src/vcpkg/commands.ci.cpp
@@ -6,6 +6,7 @@
 #include <vcpkg/base/stringliteral.h>
 #include <vcpkg/base/system.h>
 #include <vcpkg/base/util.h>
+#include <vcpkg/binarycaching.h>
 #include <vcpkg/build.h>
 #include <vcpkg/commands.h>
 #include <vcpkg/dependencies.h>
@@ -220,7 +221,7 @@ namespace vcpkg::Commands::CI
         std::map<PackageSpec, Build::BuildResult> known;
         std::map<PackageSpec, std::vector<std::string>> features;
         std::unordered_map<std::string, SourceControlFileLocation> default_feature_provider;
-        std::map<PackageSpec, std::string> abi_tag_map;
+        std::map<PackageSpec, std::string> abi_map;
     };
 
     static bool supported_for_triplet(const CMakeVars::CMakeVarProvider& var_provider,
@@ -259,8 +260,6 @@ namespace vcpkg::Commands::CI
     {
         auto ret = std::make_unique<UnknownCIPortsResults>();
 
-        auto& fs = paths.get_filesystem();
-
         std::set<PackageSpec> will_fail;
 
         const Build::BuildPackageOptions build_options = {
@@ -294,9 +293,7 @@ namespace vcpkg::Commands::CI
         std::vector<FullPackageSpec> install_specs;
         for (auto&& install_action : action_plan.install_actions)
         {
-            install_specs.emplace_back(FullPackageSpec{
-                install_action.spec,
-                std::vector<std::string>{install_action.feature_list.begin(), install_action.feature_list.end()}});
+            install_specs.emplace_back(FullPackageSpec{install_action.spec, install_action.feature_list});
         }
 
         var_provider.load_tag_vars(install_specs, provider);
@@ -306,59 +303,26 @@ namespace vcpkg::Commands::CI
         Checks::check_exit(VCPKG_LINE_INFO,
                            action_plan.already_installed.empty(),
                            "Cannot use CI command with packages already installed.");
+
+        Build::compute_all_abis(paths, action_plan, var_provider, {});
+
+        auto binaryprovider = create_archives_provider();
+
         std::string stdout_buffer;
         for (auto&& action : action_plan.install_actions)
         {
             auto p = &action;
-            // determine abi tag
-            std::string abi;
+            ret->abi_map.emplace(action.spec, action.package_abi.value_or_exit(VCPKG_LINE_INFO));
+            ret->features.emplace(action.spec, action.feature_list);
             if (auto scfl = p->source_control_file_location.get())
             {
                 auto emp = ret->default_feature_provider.emplace(p->spec.name(), *scfl);
                 emp.first->second.source_control_file->core_paragraph->default_features = p->feature_list;
 
-                auto triplet = p->spec.triplet();
-
                 p->build_options = build_options;
-
-                auto dependency_abis =
-                    Util::fmap(p->package_dependencies, [&](const PackageSpec& spec) -> Build::AbiEntry {
-                        auto it = ret->abi_tag_map.find(spec);
-
-                        if (it == ret->abi_tag_map.end())
-                            return {spec.name(), ""};
-                        else
-                            return {spec.name(), it->second};
-                    });
-
-                const auto pre_build_info = Build::PreBuildInfo(
-                    paths, triplet, var_provider.get_tag_vars(p->spec).value_or_exit(VCPKG_LINE_INFO));
-
-                auto maybe_tag_and_file = Build::compute_abi_tag(paths, *p, pre_build_info, dependency_abis);
-                if (auto tag_and_file = maybe_tag_and_file.get())
-                {
-                    abi = tag_and_file->tag;
-                    ret->abi_tag_map.emplace(p->spec, abi);
-                }
-            }
-            else if (auto ipv = p->installed_package.get())
-            {
-                abi = ipv->core->package.abi;
-                if (!abi.empty()) ret->abi_tag_map.emplace(p->spec, abi);
-            }
-
-            auto archives_root_dir = paths.root / "archives";
-            auto archive_name = abi + ".zip";
-            auto archive_subpath = fs::u8path(abi.substr(0, 2)) / archive_name;
-            auto archive_path = archives_root_dir / archive_subpath;
-            auto archive_tombstone_path = archives_root_dir / "fail" / archive_subpath;
-
-            if (purge_tombstones)
-            {
-                std::error_code ec;
-                fs.remove(archive_tombstone_path, ec); // Ignore error
             }
 
+            auto precheck_result = binaryprovider->precheck(paths, action, purge_tombstones);
             bool b_will_build = false;
 
             std::string state;
@@ -385,12 +349,12 @@ namespace vcpkg::Commands::CI
                 ret->known.emplace(p->spec, BuildResult::CASCADED_DUE_TO_MISSING_DEPENDENCIES);
                 will_fail.emplace(p->spec);
             }
-            else if (fs.exists(archive_path))
+            else if (precheck_result == RestoreResult::success)
             {
                 state = "pass";
                 ret->known.emplace(p->spec, BuildResult::SUCCEEDED);
             }
-            else if (fs.exists(archive_tombstone_path))
+            else if (precheck_result == RestoreResult::build_failed)
             {
                 state = "fail";
                 ret->known.emplace(p->spec, BuildResult::BUILD_FAILED);
@@ -403,7 +367,11 @@ namespace vcpkg::Commands::CI
             }
 
             Strings::append(stdout_buffer,
-                            Strings::format("%40s: %1s %8s: %s\n", p->spec, (b_will_build ? "*" : " "), state, abi));
+                            Strings::format("%40s: %1s %8s: %s\n",
+                                            p->spec,
+                                            (b_will_build ? "*" : " "),
+                                            state,
+                                            action.package_abi.value_or_exit(VCPKG_LINE_INFO)));
             if (stdout_buffer.size() > 2048)
             {
                 System::print2(stdout_buffer);
@@ -466,7 +434,6 @@ namespace vcpkg::Commands::CI
         };
 
         std::vector<std::map<PackageSpec, BuildResult>> all_known_results;
-        std::map<PackageSpec, std::string> abi_tag_map;
 
         XunitTestResults xunitTestResults;
 
@@ -542,28 +509,27 @@ namespace vcpkg::Commands::CI
                 // Adding results for ports that were built or pulled from an archive
                 for (auto&& result : summary.results)
                 {
-                    auto& port_features = split_specs->features[result.spec];
+                    auto& port_features = split_specs->features.at(result.spec);
                     split_specs->known.erase(result.spec);
                     xunitTestResults.add_test_results(result.spec.to_string(),
                                                       result.build_result.code,
                                                       result.timing,
-                                                      split_specs->abi_tag_map.at(result.spec),
+                                                      split_specs->abi_map.at(result.spec),
                                                       port_features);
                 }
 
                 // Adding results for ports that were not built because they have known states
                 for (auto&& port : split_specs->known)
                 {
-                    auto& port_features = split_specs->features[port.first];
+                    auto& port_features = split_specs->features.at(port.first);
                     xunitTestResults.add_test_results(port.first.to_string(),
                                                       port.second,
                                                       Chrono::ElapsedTime{},
-                                                      split_specs->abi_tag_map.at(port.first),
+                                                      split_specs->abi_map.at(port.first),
                                                       port_features);
                 }
 
                 all_known_results.emplace_back(std::move(split_specs->known));
-                abi_tag_map.insert(split_specs->abi_tag_map.begin(), split_specs->abi_tag_map.end());
 
                 results.push_back({triplet, std::move(summary)});
 
diff --git a/toolsrc/src/vcpkg/dependencies.cpp b/toolsrc/src/vcpkg/dependencies.cpp
index beb92f4b0..07aad4d22 100644
--- a/toolsrc/src/vcpkg/dependencies.cpp
+++ b/toolsrc/src/vcpkg/dependencies.cpp
@@ -403,6 +403,15 @@ namespace vcpkg::Dependencies
         const std::string features = Strings::join(",", feature_list);
         return Strings::format("%s[%s]:%s", this->spec.name(), features, this->spec.triplet());
     }
+    const std::string& InstallPlanAction::public_abi() const
+    {
+        if (auto p = pre_build_info.get())
+        {
+            if (auto q = p->get()->public_abi_override.get()) return *q;
+        }
+        if (auto p = installed_package.get()) return p->core->package.abi;
+        return package_abi.value_or_exit(VCPKG_LINE_INFO);
+    }
 
     bool InstallPlanAction::compare_by_name(const InstallPlanAction* left, const InstallPlanAction* right)
     {
diff --git a/toolsrc/src/vcpkg/install.cpp b/toolsrc/src/vcpkg/install.cpp
index 3d6e4c2d9..5084ac918 100644
--- a/toolsrc/src/vcpkg/install.cpp
+++ b/toolsrc/src/vcpkg/install.cpp
@@ -4,6 +4,7 @@
 #include <vcpkg/base/hash.h>
 #include <vcpkg/base/system.print.h>
 #include <vcpkg/base/util.h>
+#include <vcpkg/binarycaching.h>
 #include <vcpkg/build.h>
 #include <vcpkg/cmakevars.h>
 #include <vcpkg/commands.h>
@@ -299,7 +300,7 @@ namespace vcpkg::Install
     ExtendedBuildResult perform_install_plan_action(const VcpkgPaths& paths,
                                                     InstallPlanAction& action,
                                                     StatusParagraphs& status_db,
-                                                    const CMakeVars::CMakeVarProvider& var_provider)
+                                                    IBinaryProvider* binaries_provider)
     {
         const InstallPlanType& plan_type = action.plan_type;
         const std::string display_name = action.spec.to_string();
@@ -339,7 +340,7 @@ namespace vcpkg::Install
             else
                 System::printf("Building package %s...\n", display_name_with_features);
 
-            auto result = Build::build_package(paths, action, var_provider, status_db);
+            auto result = Build::build_package(paths, action, binaries_provider, status_db);
 
             if (BuildResult::DOWNLOADED == result.code)
             {
@@ -456,13 +457,16 @@ namespace vcpkg::Install
         for (auto&& action : action_plan.already_installed)
         {
             results.emplace_back(action.spec, &action);
-            results.back().build_result = perform_install_plan_action(paths, action, status_db, var_provider);
+            results.back().build_result = perform_install_plan_action(paths, action, status_db, nullptr);
         }
 
+        Build::compute_all_abis(paths, action_plan, var_provider, status_db);
+
+        auto binary_provider = create_archives_provider();
         for (auto&& action : action_plan.install_actions)
         {
             with_tracking(action.spec, [&]() {
-                auto result = perform_install_plan_action(paths, action, status_db, var_provider);
+                auto result = perform_install_plan_action(paths, action, status_db, binary_provider.get());
 
                 if (result.code != BuildResult::SUCCEEDED && keep_going == KeepGoing::NO)
                 {