[vcpkg] Extract binary caching functionality into separate interface

4 files changed, 235 insertions, 150 deletions

diff --git a/toolsrc/src/vcpkg/binarycaching.cpp b/toolsrc/src/vcpkg/binarycaching.cpp
new file mode 100644
index 000000000..bbf565c9c
--- /dev/null
+++ b/toolsrc/src/vcpkg/binarycaching.cpp
@@ -0,0 +1,206 @@
+#include <vcpkg/base/checks.h>

+#include <vcpkg/base/files.h>

+#include <vcpkg/base/system.print.h>

+#include <vcpkg/base/system.process.h>

+#include <vcpkg/binarycaching.h>

+#include <vcpkg/build.h>

+#include <vcpkg/dependencies.h>

+

+using namespace vcpkg;

+

+namespace

+{

+    static System::ExitCodeAndOutput decompress_archive(const VcpkgPaths& paths,

+                                                        const PackageSpec& spec,

+                                                        const fs::path& archive_path)

+    {

+        auto& fs = paths.get_filesystem();

+

+        auto pkg_path = paths.package_dir(spec);

+        fs.remove_all(pkg_path, VCPKG_LINE_INFO);

+        std::error_code ec;

+        fs.create_directories(pkg_path, ec);

+        auto files = fs.get_files_non_recursive(pkg_path);

+        Checks::check_exit(VCPKG_LINE_INFO, files.empty(), "unable to clear path: %s", pkg_path.u8string());

+

+#if defined(_WIN32)

+        auto&& seven_zip_exe = paths.get_tool_exe(Tools::SEVEN_ZIP);

+        auto cmd = Strings::format(

+            R"("%s" x "%s" -o"%s" -y)", seven_zip_exe.u8string(), archive_path.u8string(), pkg_path.u8string());

+#else

+        auto cmd = Strings::format(R"(unzip -qq "%s" "-d%s")", archive_path.u8string(), pkg_path.u8string());

+#endif

+        return System::cmd_execute_and_capture_output(cmd, System::get_clean_environment());

+    }

+

+    // Compress the source directory into the destination file.

+    static void compress_directory(const VcpkgPaths& paths, const fs::path& source, const fs::path& destination)

+    {

+        auto& fs = paths.get_filesystem();

+

+        std::error_code ec;

+

+        fs.remove(destination, ec);

+        Checks::check_exit(

+            VCPKG_LINE_INFO, !fs.exists(destination), "Could not remove file: %s", destination.u8string());

+#if defined(_WIN32)

+        auto&& seven_zip_exe = paths.get_tool_exe(Tools::SEVEN_ZIP);

+

+        System::cmd_execute_and_capture_output(

+            Strings::format(

+                R"("%s" a "%s" "%s\*")", seven_zip_exe.u8string(), destination.u8string(), source.u8string()),

+            System::get_clean_environment());

+#else

+        System::cmd_execute_clean(

+            Strings::format(R"(cd '%s' && zip --quiet -r '%s' *)", source.u8string(), destination.u8string()));

+#endif

+    }

+

+    static void compress_archive(const VcpkgPaths& paths, const PackageSpec& spec, const fs::path& destination)

+    {

+        compress_directory(paths, paths.package_dir(spec), destination);

+    }

+

+    struct ArchivesBinaryProvider : IBinaryProvider

+    {

+        ~ArchivesBinaryProvider() = default;

+        void prefetch() override {}

+        RestoreResult try_restore(const VcpkgPaths& paths,

+                                  const PackageSpec& spec,

+                                  const Build::AbiTagAndFile& abi_tag_and_file,

+                                  const Build::BuildPackageOptions& build_options) override

+        {

+            auto& fs = paths.get_filesystem();

+            std::error_code ec;

+            const fs::path archives_root_dir = paths.root / "archives";

+            const std::string archive_name = abi_tag_and_file.tag + ".zip";

+            const fs::path archive_subpath = fs::u8path(abi_tag_and_file.tag.substr(0, 2)) / archive_name;

+            const fs::path archive_path = archives_root_dir / archive_subpath;

+            const fs::path archive_tombstone_path = archives_root_dir / "fail" / archive_subpath;

+            const fs::path abi_package_dir = paths.package_dir(spec) / "share" / spec.name();

+            const fs::path abi_file_in_package = paths.package_dir(spec) / "share" / spec.name() / "vcpkg_abi_info.txt";

+            if (fs.exists(archive_path))

+            {

+                System::print2("Using cached binary package: ", archive_path.u8string(), "\n");

+

+                int archive_result = decompress_archive(paths, spec, archive_path).exit_code;

+

+                if (archive_result != 0)

+                {

+                    System::print2("Failed to decompress archive package\n");

+                    if (build_options.purge_decompress_failure == Build::PurgeDecompressFailure::NO)

+                    {

+                        return RestoreResult::BUILD_FAILED;

+                    }

+                    else

+                    {

+                        System::print2("Purging bad archive\n");

+                        fs.remove(archive_path, ec);

+                    }

+                }

+                else

+                {

+                    return RestoreResult::SUCCESS;

+                }

+            }

+

+            if (fs.exists(archive_tombstone_path))

+            {

+                if (build_options.fail_on_tombstone == Build::FailOnTombstone::YES)

+                {

+                    System::print2("Found failure tombstone: ", archive_tombstone_path.u8string(), "\n");

+                    return RestoreResult::BUILD_FAILED;

+                }

+                else

+                {

+                    System::print2(

+                        System::Color::warning, "Found failure tombstone: ", archive_tombstone_path.u8string(), "\n");

+                }

+            }

+            else

+            {

+                System::printf("Could not locate cached archive: %s\n", archive_path.u8string());

+            }

+

+            return RestoreResult::MISSING;

+        }

+        void push_success(const VcpkgPaths& paths,

+                          const Build::AbiTagAndFile& abi_tag_and_file,

+                          const Dependencies::InstallPlanAction& action) override

+        {

+            auto& spec = action.spec;

+            auto& fs = paths.get_filesystem();

+            std::error_code ec;

+            const fs::path archives_root_dir = paths.root / "archives";

+            const std::string archive_name = abi_tag_and_file.tag + ".zip";

+            const fs::path archive_subpath = fs::u8path(abi_tag_and_file.tag.substr(0, 2)) / archive_name;

+            const fs::path archive_path = archives_root_dir / archive_subpath;

+            const fs::path archive_tombstone_path = archives_root_dir / "fail" / archive_subpath;

+            const fs::path abi_package_dir = paths.package_dir(spec) / "share" / spec.name();

+            const fs::path abi_file_in_package = paths.package_dir(spec) / "share" / spec.name() / "vcpkg_abi_info.txt";

+

+            const auto tmp_archive_path = paths.buildtrees / spec.name() / (spec.triplet().to_string() + ".zip");

+

+            compress_archive(paths, spec, tmp_archive_path);

+

+            fs.create_directories(archive_path.parent_path(), ec);

+            fs.rename_or_copy(tmp_archive_path, archive_path, ".tmp", ec);

+            if (ec)

+            {

+                System::printf(System::Color::warning,

+                               "Failed to store binary cache %s: %s\n",

+                               archive_path.u8string(),

+                               ec.message());

+            }

+            else

+                System::printf("Stored binary cache: %s\n", archive_path.u8string());

+        }

+        void push_failure(const VcpkgPaths& paths,

+                          const Build::AbiTagAndFile& abi_tag_and_file,

+                          const PackageSpec& spec) override

+        {

+            auto& fs = paths.get_filesystem();

+            std::error_code ec;

+            const fs::path archives_root_dir = paths.root / "archives";

+            const std::string archive_name = abi_tag_and_file.tag + ".zip";

+            const fs::path archive_subpath = fs::u8path(abi_tag_and_file.tag.substr(0, 2)) / archive_name;

+            const fs::path archive_path = archives_root_dir / archive_subpath;

+            const fs::path archive_tombstone_path = archives_root_dir / "fail" / archive_subpath;

+            const fs::path abi_package_dir = paths.package_dir(spec) / "share" / spec.name();

+            const fs::path abi_file_in_package = paths.package_dir(spec) / "share" / spec.name() / "vcpkg_abi_info.txt";

+

+            if (!fs.exists(archive_tombstone_path))

+            {

+                // Build failed, store all failure logs in the tombstone.

+                const auto tmp_log_path = paths.buildtrees / spec.name() / "tmp_failure_logs";

+                const auto tmp_log_path_destination = tmp_log_path / spec.name();

+                const auto tmp_failure_zip = paths.buildtrees / spec.name() / "failure_logs.zip";

+                fs.create_directories(tmp_log_path_destination, ec);

+

+                for (auto& log_file : fs::stdfs::directory_iterator(paths.buildtrees / spec.name()))

+                {

+                    if (log_file.path().extension() == ".log")

+                    {

+                        fs.copy_file(log_file.path(),

+                                     tmp_log_path_destination / log_file.path().filename(),

+                                     fs::stdfs::copy_options::none,

+                                     ec);

+                    }

+                }

+

+                compress_directory(paths, tmp_log_path, paths.buildtrees / spec.name() / "failure_logs.zip");

+

+                fs.create_directories(archive_tombstone_path.parent_path(), ec);

+                fs.rename_or_copy(tmp_failure_zip, archive_tombstone_path, ".tmp", ec);

+

+                // clean up temporary directory

+                fs.remove_all(tmp_log_path, VCPKG_LINE_INFO);

+            }

+        }

+    };

+}

+

+std::unique_ptr<vcpkg::IBinaryProvider> vcpkg::create_archives_provider()

+{

+    return std::make_unique<ArchivesBinaryProvider>();

+}

diff --git a/toolsrc/src/vcpkg/build.cpp b/toolsrc/src/vcpkg/build.cpp
index b4b8a3485..7ec6ff60e 100644
--- a/toolsrc/src/vcpkg/build.cpp
+++ b/toolsrc/src/vcpkg/build.cpp
@@ -12,6 +12,7 @@
 #include <vcpkg/base/system.process.h>

 #include <vcpkg/base/util.h>

 

+#include <vcpkg/binarycaching.h>

 #include <vcpkg/build.h>

 #include <vcpkg/commands.h>

 #include <vcpkg/dependencies.h>

@@ -38,11 +39,8 @@ namespace vcpkg::Build::Command
     void perform_and_exit_ex(const FullPackageSpec& full_spec,

                              const SourceControlFileLocation& scfl,

                              const PathsPortFileProvider& provider,

-                             const ParsedArguments& options,

                              const VcpkgPaths& paths)

     {

-        vcpkg::Util::unused(options);

-

         CMakeVars::TripletCMakeVarProvider var_provider(paths);

         var_provider.load_dep_info_vars(std::array<PackageSpec, 1>{full_spec.package_spec});

         var_provider.load_tag_vars(std::array<FullPackageSpec, 1>{full_spec}, provider);

@@ -94,7 +92,8 @@ namespace vcpkg::Build::Command
         action->build_options = build_package_options;

 

         const auto build_timer = Chrono::ElapsedTimer::create_started();

-        const auto result = Build::build_package(paths, *action, var_provider, status_db);

+        const auto result =

+            Build::build_package(paths, *action, var_provider, create_archives_provider().get(), status_db);

         System::print2("Elapsed time for package ", spec, ": ", build_timer, '\n');

 

         if (result.code == BuildResult::CASCADED_DUE_TO_MISSING_DEPENDENCIES)

@@ -147,7 +146,7 @@ namespace vcpkg::Build::Command
 

         Checks::check_exit(VCPKG_LINE_INFO, scfl != nullptr, "Error: Couldn't find port '%s'", port_name);

 

-        perform_and_exit_ex(spec, *scfl, provider, options, paths);

+        perform_and_exit_ex(spec, *scfl, provider, paths);

     }

 }

 

@@ -737,62 +736,14 @@ namespace vcpkg::Build
         return nullopt;

     }

 

-    static System::ExitCodeAndOutput decompress_archive(const VcpkgPaths& paths,

-                                                        const PackageSpec& spec,

-                                                        const fs::path& archive_path)

-    {

-        auto& fs = paths.get_filesystem();

-

-        auto pkg_path = paths.package_dir(spec);

-        fs.remove_all(pkg_path, VCPKG_LINE_INFO);

-        std::error_code ec;

-        fs.create_directories(pkg_path, ec);

-        auto files = fs.get_files_non_recursive(pkg_path);

-        Checks::check_exit(VCPKG_LINE_INFO, files.empty(), "unable to clear path: %s", pkg_path.u8string());

-

-#if defined(_WIN32)

-        auto&& seven_zip_exe = paths.get_tool_exe(Tools::SEVEN_ZIP);

-        auto cmd = Strings::format(

-            R"("%s" x "%s" -o"%s" -y)", seven_zip_exe.u8string(), archive_path.u8string(), pkg_path.u8string());

-#else

-        auto cmd = Strings::format(R"(unzip -qq "%s" "-d%s")", archive_path.u8string(), pkg_path.u8string());

-#endif

-        return System::cmd_execute_and_capture_output(cmd, System::get_clean_environment());

-    }

-

-    // Compress the source directory into the destination file.

-    static void compress_directory(const VcpkgPaths& paths, const fs::path& source, const fs::path& destination)

-    {

-        auto& fs = paths.get_filesystem();

-

-        std::error_code ec;

-

-        fs.remove(destination, ec);

-        Checks::check_exit(

-            VCPKG_LINE_INFO, !fs.exists(destination), "Could not remove file: %s", destination.u8string());

-#if defined(_WIN32)

-        auto&& seven_zip_exe = paths.get_tool_exe(Tools::SEVEN_ZIP);

-

-        System::cmd_execute_and_capture_output(

-            Strings::format(

-                R"("%s" a "%s" "%s\*")", seven_zip_exe.u8string(), destination.u8string(), source.u8string()),

-            System::get_clean_environment());

-#else

-        System::cmd_execute_clean(

-            Strings::format(R"(cd '%s' && zip --quiet -r '%s' *)", source.u8string(), destination.u8string()));

-#endif

-    }

-

-    static void compress_archive(const VcpkgPaths& paths, const PackageSpec& spec, const fs::path& destination)

-    {

-        compress_directory(paths, paths.package_dir(spec), destination);

-    }

-

     ExtendedBuildResult build_package(const VcpkgPaths& paths,

                                       const Dependencies::InstallPlanAction& action,

                                       const CMakeVars::CMakeVarProvider& var_provider,

+                                      IBinaryProvider* binaries_provider,

                                       const StatusParagraphs& status_db)

     {

+        auto binary_caching_enabled = binaries_provider && action.build_options.binary_caching == BinaryCaching::YES;

+

         auto& fs = paths.get_filesystem();

         Triplet triplet = action.spec.triplet();

         const std::string& name = action.source_control_file_location.value_or_exit(VCPKG_LINE_INFO)

@@ -843,116 +794,40 @@ namespace vcpkg::Build
 

         std::error_code ec;

         const auto abi_tag_and_file = maybe_abi_tag_and_file.get();

-        const fs::path archives_root_dir = paths.root / "archives";

-        const std::string archive_name = abi_tag_and_file->tag + ".zip";

-        const fs::path archive_subpath = fs::u8path(abi_tag_and_file->tag.substr(0, 2)) / archive_name;

-        const fs::path archive_path = archives_root_dir / archive_subpath;

-        const fs::path archive_tombstone_path = archives_root_dir / "fail" / archive_subpath;

         const fs::path abi_package_dir = paths.package_dir(spec) / "share" / spec.name();

         const fs::path abi_file_in_package = paths.package_dir(spec) / "share" / spec.name() / "vcpkg_abi_info.txt";

-

-        if (action.build_options.binary_caching == BinaryCaching::YES)

+        if (binary_caching_enabled)

         {

-            if (fs.exists(archive_path))

+            auto restore = binaries_provider->try_restore(paths, spec, *abi_tag_and_file, action.build_options);

+            if (restore == RestoreResult::BUILD_FAILED)

+                return BuildResult::BUILD_FAILED;

+            else if (restore == RestoreResult::SUCCESS)

             {

-                System::print2("Using cached binary package: ", archive_path.u8string(), "\n");

-

-                int archive_result = decompress_archive(paths, spec, archive_path).exit_code;

-

-                if (archive_result != 0)

-                {

-                    System::print2("Failed to decompress archive package\n");

-                    if (action.build_options.purge_decompress_failure == PurgeDecompressFailure::NO)

-                    {

-                        return BuildResult::BUILD_FAILED;

-                    }

-                    else

-                    {

-                        System::print2("Purging bad archive\n");

-                        fs.remove(archive_path, ec);

-                    }

-                }

-                else

-                {

-                    auto maybe_bcf = Paragraphs::try_load_cached_package(paths, spec);

-                    auto bcf = std::make_unique<BinaryControlFile>(std::move(maybe_bcf).value_or_exit(VCPKG_LINE_INFO));

-                    return {BuildResult::SUCCEEDED, std::move(bcf)};

-                }

+                auto maybe_bcf = Paragraphs::try_load_cached_package(paths, spec);

+                auto bcf = std::make_unique<BinaryControlFile>(std::move(maybe_bcf).value_or_exit(VCPKG_LINE_INFO));

+                return {BuildResult::SUCCEEDED, std::move(bcf)};

             }

-

-            if (fs.exists(archive_tombstone_path))

+            else

             {

-                if (action.build_options.fail_on_tombstone == FailOnTombstone::YES)

-                {

-                    System::print2("Found failure tombstone: ", archive_tombstone_path.u8string(), "\n");

-                    return BuildResult::BUILD_FAILED;

-                }

-                else

-                {

-                    System::print2(

-                        System::Color::warning, "Found failure tombstone: ", archive_tombstone_path.u8string(), "\n");

-                }

+                // missing package, proceed to build.

             }

-

-            System::printf("Could not locate cached archive: %s\n", archive_path.u8string());

         }

 

         ExtendedBuildResult result = do_build_package_and_clean_buildtrees(

             paths, pre_build_info, pre_build_info.public_abi_override.value_or(abi_tag_and_file->tag), action);

 

         fs.create_directories(abi_package_dir, ec);

-        Checks::check_exit(VCPKG_LINE_INFO, !ec, "Coud not create directory %s", abi_package_dir.u8string());

         fs.copy_file(abi_tag_and_file->tag_file, abi_file_in_package, fs::stdfs::copy_options::none, ec);

         Checks::check_exit(VCPKG_LINE_INFO, !ec, "Could not copy into file: %s", abi_file_in_package.u8string());

 

-        if (action.build_options.binary_caching == BinaryCaching::YES && result.code == BuildResult::SUCCEEDED)

+        if (binary_caching_enabled && result.code == BuildResult::SUCCEEDED)

         {

-            const auto tmp_archive_path = paths.buildtrees / spec.name() / (spec.triplet().to_string() + ".zip");

-

-            compress_archive(paths, spec, tmp_archive_path);

-

-            fs.create_directories(archive_path.parent_path(), ec);

-            fs.rename_or_copy(tmp_archive_path, archive_path, ".tmp", ec);

-            if (ec)

-            {

-                System::printf(System::Color::warning,

-                               "Failed to store binary cache %s: %s\n",

-                               archive_path.u8string(),

-                               ec.message());

-            }

-            else

-                System::printf("Stored binary cache: %s\n", archive_path.u8string());

+            binaries_provider->push_success(paths, *abi_tag_and_file, action);

         }

-        else if (action.build_options.binary_caching == BinaryCaching::YES &&

+        else if (binary_caching_enabled &&

                  (result.code == BuildResult::BUILD_FAILED || result.code == BuildResult::POST_BUILD_CHECKS_FAILED))

         {

-            if (!fs.exists(archive_tombstone_path))

-            {

-                // Build failed, store all failure logs in the tombstone.

-                const auto tmp_log_path = paths.buildtrees / spec.name() / "tmp_failure_logs";

-                const auto tmp_log_path_destination = tmp_log_path / spec.name();

-                const auto tmp_failure_zip = paths.buildtrees / spec.name() / "failure_logs.zip";

-                fs.create_directories(tmp_log_path_destination, ec);

-

-                for (auto& log_file : fs::stdfs::directory_iterator(paths.buildtrees / spec.name()))

-                {

-                    if (log_file.path().extension() == ".log")

-                    {

-                        fs.copy_file(log_file.path(),

-                                     tmp_log_path_destination / log_file.path().filename(),

-                                     fs::stdfs::copy_options::none,

-                                     ec);

-                    }

-                }

-

-                compress_directory(paths, tmp_log_path, paths.buildtrees / spec.name() / "failure_logs.zip");

-

-                fs.create_directories(archive_tombstone_path.parent_path(), ec);

-                fs.rename_or_copy(tmp_failure_zip, archive_tombstone_path, ".tmp", ec);

-

-                // clean up temporary directory

-                fs.remove_all(tmp_log_path, VCPKG_LINE_INFO);

-            }

+            binaries_provider->push_failure(paths, *abi_tag_and_file, spec);

         }

 

         return result;

diff --git a/toolsrc/src/vcpkg/commands.buildexternal.cpp b/toolsrc/src/vcpkg/commands.buildexternal.cpp
index 4a69be879..07ad7b6b1 100644
--- a/toolsrc/src/vcpkg/commands.buildexternal.cpp
+++ b/toolsrc/src/vcpkg/commands.buildexternal.cpp
@@ -33,6 +33,6 @@ namespace vcpkg::Commands::BuildExternal
         Checks::check_exit(
             VCPKG_LINE_INFO, maybe_scfl.has_value(), "could not load control file for %s", spec.package_spec.name());
 
-        Build::Command::perform_and_exit_ex(spec, maybe_scfl.value_or_exit(VCPKG_LINE_INFO), provider, options, paths);
+        Build::Command::perform_and_exit_ex(spec, maybe_scfl.value_or_exit(VCPKG_LINE_INFO), provider, paths);
     }
 }
diff --git a/toolsrc/src/vcpkg/install.cpp b/toolsrc/src/vcpkg/install.cpp
index e2a8ce4c5..1cd29284c 100644
--- a/toolsrc/src/vcpkg/install.cpp
+++ b/toolsrc/src/vcpkg/install.cpp
@@ -4,6 +4,7 @@
 #include <vcpkg/base/hash.h>
 #include <vcpkg/base/system.print.h>
 #include <vcpkg/base/util.h>
+#include <vcpkg/binarycaching.h>
 #include <vcpkg/build.h>
 #include <vcpkg/cmakevars.h>
 #include <vcpkg/commands.h>
@@ -299,6 +300,7 @@ namespace vcpkg::Install
     ExtendedBuildResult perform_install_plan_action(const VcpkgPaths& paths,
                                                     InstallPlanAction& action,
                                                     StatusParagraphs& status_db,
+                                                    IBinaryProvider* binaries_provider,
                                                     const CMakeVars::CMakeVarProvider& var_provider)
     {
         const InstallPlanType& plan_type = action.plan_type;
@@ -339,7 +341,7 @@ namespace vcpkg::Install
             else
                 System::printf("Building package %s...\n", display_name_with_features);
 
-            auto result = Build::build_package(paths, action, var_provider, status_db);
+            auto result = Build::build_package(paths, action, var_provider, binaries_provider, status_db);
 
             if (BuildResult::DOWNLOADED == result.code)
             {
@@ -456,13 +458,15 @@ namespace vcpkg::Install
         for (auto&& action : action_plan.already_installed)
         {
             results.emplace_back(action.spec, &action);
-            results.back().build_result = perform_install_plan_action(paths, action, status_db, var_provider);
+            results.back().build_result = perform_install_plan_action(paths, action, status_db, nullptr, var_provider);
         }
 
+        auto binary_provider = create_archives_provider();
         for (auto&& action : action_plan.install_actions)
         {
             with_tracking(action.spec, [&]() {
-                auto result = perform_install_plan_action(paths, action, status_db, var_provider);
+                auto result =
+                    perform_install_plan_action(paths, action, status_db, binary_provider.get(), var_provider);
 
                 if (result.code != BuildResult::SUCCEEDED && keep_going == KeepGoing::NO)
                 {