Download Mode (#7950)

* [portfile functions] Override execute_process() to accept ALLOW_IN_DOWNLOAD_MODE option

* [vcpkg install] Set VCPKG_DOWNLOAD_MODE when using --only-downloads option

* [vcpkg_find_acquire_program] Allow in Download Mode

* Don't stop when build fails for a package

* Download sources for all packages in dependency graph

* Improve output messages

* Enable acquiring MSYS packages in download mode

* Documentation

* Update documentation

* execute_process() always fails on Download Mode

* Regenerate docs and fix formatting

* Run clang-format

* Use _execute_process on vcpkg_from_<source> helpers

* Fix calls to _execute_process() when not in Download Mode

5 files changed, 50 insertions, 11 deletions

diff --git a/toolsrc/src/vcpkg/build.cpp b/toolsrc/src/vcpkg/build.cpp
index 9245ddfbc..e5c00b929 100644
--- a/toolsrc/src/vcpkg/build.cpp
+++ b/toolsrc/src/vcpkg/build.cpp
@@ -53,6 +53,7 @@ namespace vcpkg::Build::Command
         const Build::BuildPackageOptions build_package_options{

             Build::UseHeadVersion::NO,

             Build::AllowDownloads::YES,

+            Build::OnlyDownloads::NO,

             Build::CleanBuildtrees::NO,

             Build::CleanPackages::NO,

             Build::CleanDownloads::NO,

@@ -407,6 +408,11 @@ namespace vcpkg::Build
             {"VCPKG_CONCURRENCY", std::to_string(get_concurrency())},

         };

 

+        if (Util::Enum::to_bool(config.build_package_options.only_downloads))

+        {

+            variables.push_back({"VCPKG_DOWNLOAD_MODE", "true"});

+        }

+

         if (!System::get_environment_variable("VCPKG_FORCE_SYSTEM_BINARIES").has_value())

         {

             variables.push_back({"GIT", git_exe_path});

@@ -528,6 +534,7 @@ namespace vcpkg::Build
                                                 const BuildPackageConfig& config)

     {

         auto& fs = paths.get_filesystem();

+

 #if defined(_WIN32)

         const fs::path& powershell_exe_path = paths.get_tool_exe("powershell-core");

         if (!fs.exists(powershell_exe_path.parent_path() / "powershell.exe"))

@@ -559,6 +566,14 @@ namespace vcpkg::Build
 #else

         const int return_code = System::cmd_execute_clean(command, env);

 #endif

+        // With the exception of empty packages, builds in "Download Mode" always result in failure.

+        if (config.build_package_options.only_downloads == Build::OnlyDownloads::YES)

+        {

+            // TODO: Capture executed command output and evaluate whether the failure was intended.

+            // If an unintended error occurs then return a BuildResult::DOWNLOAD_FAILURE status.

+            return BuildResult::DOWNLOADED;

+        }

+

         const auto buildtimeus = timer.microseconds();

         const auto spec_string = spec.to_string();

 

@@ -793,20 +808,23 @@ namespace vcpkg::Build
         const std::string& name = config.scf.core_paragraph->name;

 

         std::vector<FeatureSpec> required_fspecs = compute_required_feature_specs(config, status_db);

-        std::vector<FeatureSpec> required_fspecs_copy = required_fspecs;

 

         // extract out the actual package ids

         auto dep_pspecs = Util::fmap(required_fspecs, [](FeatureSpec const& fspec) { return fspec.spec(); });

         Util::sort_unique_erase(dep_pspecs);

 

         // Find all features that aren't installed. This mutates required_fspecs.

-        Util::erase_remove_if(required_fspecs, [&](FeatureSpec const& fspec) {

-            return status_db.is_installed(fspec) || fspec.name() == name;

-        });

-

-        if (!required_fspecs.empty())

+        // Skip this validation when running in Download Mode.

+        if (config.build_package_options.only_downloads != Build::OnlyDownloads::YES)

         {

-            return {BuildResult::CASCADED_DUE_TO_MISSING_DEPENDENCIES, std::move(required_fspecs)};

+            Util::erase_remove_if(required_fspecs, [&](FeatureSpec const& fspec) {

+                return status_db.is_installed(fspec) || fspec.name() == name;

+            });

+

+            if (!required_fspecs.empty())

+            {

+                return {BuildResult::CASCADED_DUE_TO_MISSING_DEPENDENCIES, std::move(required_fspecs)};

+            }

         }

 

         const PackageSpec spec =

@@ -817,7 +835,10 @@ namespace vcpkg::Build
         // dep_pspecs was not destroyed

         for (auto&& pspec : dep_pspecs)

         {

-            if (pspec == spec) continue;

+            if (pspec == spec || Util::Enum::to_bool(config.build_package_options.only_downloads))

+            {

+                continue;

+            }

             const auto status_it = status_db.find_installed(pspec);

             Checks::check_exit(VCPKG_LINE_INFO, status_it != status_db.end());

             dependency_abis.emplace_back(

@@ -949,6 +970,7 @@ namespace vcpkg::Build
         static const std::string POST_BUILD_CHECKS_FAILED_STRING = "POST_BUILD_CHECKS_FAILED";

         static const std::string CASCADED_DUE_TO_MISSING_DEPENDENCIES_STRING = "CASCADED_DUE_TO_MISSING_DEPENDENCIES";

         static const std::string EXCLUDED_STRING = "EXCLUDED";

+        static const std::string DOWNLOADED_STRING = "DOWNLOADED";

 

         switch (build_result)

         {

@@ -959,6 +981,7 @@ namespace vcpkg::Build
             case BuildResult::FILE_CONFLICTS: return FILE_CONFLICTS_STRING;

             case BuildResult::CASCADED_DUE_TO_MISSING_DEPENDENCIES: return CASCADED_DUE_TO_MISSING_DEPENDENCIES_STRING;

             case BuildResult::EXCLUDED: return EXCLUDED_STRING;

+            case BuildResult::DOWNLOADED: return DOWNLOADED_STRING;

             default: Checks::unreachable(VCPKG_LINE_INFO);

         }

     }

diff --git a/toolsrc/src/vcpkg/commands.ci.cpp b/toolsrc/src/vcpkg/commands.ci.cpp
index f0f162f5c..6e0a71adf 100644
--- a/toolsrc/src/vcpkg/commands.ci.cpp
+++ b/toolsrc/src/vcpkg/commands.ci.cpp
@@ -212,6 +212,7 @@ namespace vcpkg::Commands::CI
         const Build::BuildPackageOptions build_options = {
             Build::UseHeadVersion::NO,
             Build::AllowDownloads::YES,
+            Build::OnlyDownloads::NO,
             Build::CleanBuildtrees::YES,
             Build::CleanPackages::YES,
             Build::CleanDownloads::NO,
@@ -356,6 +357,7 @@ namespace vcpkg::Commands::CI
         const Build::BuildPackageOptions install_plan_options = {
             Build::UseHeadVersion::NO,
             Build::AllowDownloads::YES,
+            Build::OnlyDownloads::NO,
             Build::CleanBuildtrees::YES,
             Build::CleanPackages::YES,
             Build::CleanDownloads::NO,
diff --git a/toolsrc/src/vcpkg/commands.upgrade.cpp b/toolsrc/src/vcpkg/commands.upgrade.cpp
index 1e64b2eb6..b1dbf6194 100644
--- a/toolsrc/src/vcpkg/commands.upgrade.cpp
+++ b/toolsrc/src/vcpkg/commands.upgrade.cpp
@@ -154,6 +154,7 @@ namespace vcpkg::Commands::Upgrade
         const Build::BuildPackageOptions install_plan_options = {
             Build::UseHeadVersion::NO,
             Build::AllowDownloads::YES,
+            Build::OnlyDownloads::NO,
             Build::CleanBuildtrees::NO,
             Build::CleanPackages::NO,
             Build::CleanDownloads::NO,
diff --git a/toolsrc/src/vcpkg/export.cpp b/toolsrc/src/vcpkg/export.cpp
index 5ceb47adf..349d9aefd 100644
--- a/toolsrc/src/vcpkg/export.cpp
+++ b/toolsrc/src/vcpkg/export.cpp
@@ -73,6 +73,7 @@ namespace vcpkg::Export
         static constexpr Build::BuildPackageOptions BUILD_OPTIONS = {
             Build::UseHeadVersion::NO,
             Build::AllowDownloads::YES,
+            Build::OnlyDownloads::NO,
             Build::CleanBuildtrees::NO,
             Build::CleanPackages::NO,
             Build::CleanDownloads::NO,
diff --git a/toolsrc/src/vcpkg/install.cpp b/toolsrc/src/vcpkg/install.cpp
index 1812f1624..009965887 100644
--- a/toolsrc/src/vcpkg/install.cpp
+++ b/toolsrc/src/vcpkg/install.cpp
@@ -342,6 +342,13 @@ namespace vcpkg::Install
                 return Build::build_package(paths, build_config, status_db);
             }();
 
+            if (BuildResult::DOWNLOADED == result.code)
+            {
+                System::print2(
+                    System::Color::success, "Downloaded sources for package ", display_name_with_features, "\n");
+                return result;
+            }
+
             if (result.code != Build::BuildResult::SUCCEEDED)
             {
                 System::print2(System::Color::error, Build::create_error_message(result.code, action.spec), "\n");
@@ -467,16 +474,18 @@ namespace vcpkg::Install
     static constexpr StringLiteral OPTION_DRY_RUN = "--dry-run";
     static constexpr StringLiteral OPTION_USE_HEAD_VERSION = "--head";
     static constexpr StringLiteral OPTION_NO_DOWNLOADS = "--no-downloads";
+    static constexpr StringLiteral OPTION_ONLY_DOWNLOADS = "--only-downloads";
     static constexpr StringLiteral OPTION_RECURSE = "--recurse";
     static constexpr StringLiteral OPTION_KEEP_GOING = "--keep-going";
     static constexpr StringLiteral OPTION_XUNIT = "--x-xunit";
     static constexpr StringLiteral OPTION_USE_ARIA2 = "--x-use-aria2";
     static constexpr StringLiteral OPTION_CLEAN_AFTER_BUILD = "--clean-after-build";
 
-    static constexpr std::array<CommandSwitch, 7> INSTALL_SWITCHES = {{
+    static constexpr std::array<CommandSwitch, 8> INSTALL_SWITCHES = {{
         {OPTION_DRY_RUN, "Do not actually build or install"},
         {OPTION_USE_HEAD_VERSION, "Install the libraries on the command line using the latest upstream sources"},
         {OPTION_NO_DOWNLOADS, "Do not download new sources"},
+        {OPTION_ONLY_DOWNLOADS, "Download sources but don't build packages"},
         {OPTION_RECURSE, "Allow removal of packages as part of installation"},
         {OPTION_KEEP_GOING, "Continue installing packages on failure"},
         {OPTION_USE_ARIA2, "Use aria2 to perform download tasks"},
@@ -631,10 +640,12 @@ namespace vcpkg::Install
         const bool dry_run = Util::Sets::contains(options.switches, OPTION_DRY_RUN);
         const bool use_head_version = Util::Sets::contains(options.switches, (OPTION_USE_HEAD_VERSION));
         const bool no_downloads = Util::Sets::contains(options.switches, (OPTION_NO_DOWNLOADS));
+        const bool only_downloads = Util::Sets::contains(options.switches, (OPTION_ONLY_DOWNLOADS));
         const bool is_recursive = Util::Sets::contains(options.switches, (OPTION_RECURSE));
         const bool use_aria2 = Util::Sets::contains(options.switches, (OPTION_USE_ARIA2));
         const bool clean_after_build = Util::Sets::contains(options.switches, (OPTION_CLEAN_AFTER_BUILD));
-        const KeepGoing keep_going = to_keep_going(Util::Sets::contains(options.switches, OPTION_KEEP_GOING));
+        const KeepGoing keep_going =
+            to_keep_going(Util::Sets::contains(options.switches, OPTION_KEEP_GOING) || only_downloads);
 
         auto& fs = paths.get_filesystem();
 
@@ -647,11 +658,12 @@ namespace vcpkg::Install
         const Build::BuildPackageOptions install_plan_options = {
             Util::Enum::to_enum<Build::UseHeadVersion>(use_head_version),
             Util::Enum::to_enum<Build::AllowDownloads>(!no_downloads),
+            Util::Enum::to_enum<Build::OnlyDownloads>(only_downloads),
             clean_after_build ? Build::CleanBuildtrees::YES : Build::CleanBuildtrees::NO,
             clean_after_build ? Build::CleanPackages::YES : Build::CleanPackages::NO,
             clean_after_build ? Build::CleanDownloads::YES : Build::CleanDownloads::NO,
             download_tool,
-            GlobalState::g_binary_caching ? Build::BinaryCaching::YES : Build::BinaryCaching::NO,
+            (GlobalState::g_binary_caching && !only_downloads) ? Build::BinaryCaching::YES : Build::BinaryCaching::NO,
             Build::FailOnTombstone::NO,
         };