[vcpkg] Implementation of --x-binarysource=nuget (and friends) (#12058)

* [vcpkg] Initial implementation of --x-binarysource=nuget

* [vcpkg] Remove double-double quoting of CMake arguments

* [vcpkg] Update nuget.exe to 5.5.1 to support Azure DevOps Artifacts

* [vcpkg] Enable batching of NuGet server calls with prefetch(). Add `interactive` binarysource.

* [vcpkg] Add `nugetconfig` binary source

* [vcpkg] Short circuit querying remote NuGet servers once all refs are found

* [vcpkg] Add experimental help for binary caching

* [vcpkg] Improved NuGet cache package descriptions and version formatting

* [vcpkg] Rename `CmdLineBuilder::build()` to extract()

* [vcpkg-help] Ascii-betize help topics

* [vcpkg] Add tests for cmdlinebuilder. Improve handling of quotes and slashes.

* [vcpkg] Addressing code review comments

* [vcpkg] Add tests for vcpkg::reformat_version()

* [vcpkg] Added test for vcpkg::generate_nuspec()

* [vcpkg] Add tests for vcpkg::XmlSerializer

* [vcpkg] Addressed code review comment

* [vcpkg] Add test for vcpkg::Strings::find_first_of

* [vcpkg] Fix machine-specific paths in test for vcpkg::generate_nuspec()

Co-authored-by: Robert Schumacher <roschuma@microsoft.com>

4 files changed, 265 insertions, 19 deletions

diff --git a/toolsrc/src/vcpkg-test/binarycaching.cpp b/toolsrc/src/vcpkg-test/binarycaching.cpp
new file mode 100644
index 000000000..817b85e03
--- /dev/null
+++ b/toolsrc/src/vcpkg-test/binarycaching.cpp
@@ -0,0 +1,138 @@
+#include <catch2/catch.hpp>
+#include <vcpkg/binarycaching.private.h>
+#include <vcpkg/base/files.h>
+#include <vcpkg/vcpkgcmdarguments.h>
+#include <vcpkg/sourceparagraph.h>
+#include <vcpkg/paragraphs.h>
+#include <vcpkg/dependencies.h>
+#include <string>
+
+using namespace vcpkg;
+
+TEST_CASE ("reformat_version semver-ish", "[reformat_version]")
+{
+    REQUIRE(reformat_version("0.0.0", "abitag") == "0.0.0-abitag");
+    REQUIRE(reformat_version("1.0.1", "abitag") == "1.0.1-abitag");
+    REQUIRE(reformat_version("1.01.000", "abitag") == "1.1.0-abitag");
+    REQUIRE(reformat_version("1.2", "abitag") == "1.2.0-abitag");
+    REQUIRE(reformat_version("v52", "abitag") == "52.0.0-abitag");
+    REQUIRE(reformat_version("v09.01.02", "abitag") == "9.1.2-abitag");
+    REQUIRE(reformat_version("1.1.1q", "abitag") == "1.1.1-abitag");
+    REQUIRE(reformat_version("1", "abitag") == "1.0.0-abitag");
+}
+
+TEST_CASE ("reformat_version date", "[reformat_version]")
+{
+    REQUIRE(reformat_version("2020-06-26", "abitag") == "2020.6.26-abitag");
+    REQUIRE(reformat_version("20-06-26", "abitag") == "0.0.0-abitag");
+    REQUIRE(reformat_version("2020-06-26-release", "abitag") == "2020.6.26-abitag");
+    REQUIRE(reformat_version("2020-06-26000", "abitag") == "2020.6.26-abitag");
+}
+
+TEST_CASE ("reformat_version generic", "[reformat_version]")
+{
+    REQUIRE(reformat_version("apr", "abitag") == "0.0.0-abitag");
+    REQUIRE(reformat_version("", "abitag") == "0.0.0-abitag");
+}
+
+TEST_CASE ("generate_nuspec", "[generate_nuspec]")
+{
+    auto& fsWrapper = Files::get_real_filesystem();
+    VcpkgCmdArguments args = VcpkgCmdArguments::create_from_arg_sequence(nullptr, nullptr);
+    args.packages_root_dir = std::make_unique<std::string>("/");
+    VcpkgPaths paths(fsWrapper, args);
+
+    auto pghs = Paragraphs::parse_paragraphs(R"(
+Source: zlib2
+Version: 1.5
+Build-Depends: zlib
+Description: a spiffy compression library wrapper
+
+Feature: a
+Description: a feature
+
+Feature: b
+Description: enable bzip capabilities
+Build-Depends: bzip
+)",
+                                             "<testdata>");
+    REQUIRE(pghs.has_value());
+    auto maybe_scf = SourceControlFile::parse_control_file(fs::path(), std::move(*pghs.get()));
+    REQUIRE(maybe_scf.has_value());
+    SourceControlFileLocation scfl{std::move(*maybe_scf.get()), fs::path()};
+
+    Dependencies::InstallPlanAction ipa(PackageSpec{"zlib2", Triplet::X64_WINDOWS},
+                                        scfl,
+                                        Dependencies::RequestType::USER_REQUESTED,
+                                        {{"a", {}}, {"b", {}}});
+
+    ipa.abi_info = Build::AbiInfo{};
+    ipa.abi_info.get()->package_abi = "packageabi";
+    std::string tripletabi("tripletabi");
+    ipa.abi_info.get()->triplet_abi = tripletabi;
+
+    NugetReference ref(ipa);
+
+    REQUIRE(ref.nupkg_filename() == "zlib2_x64-windows.1.5.0-packageabi.nupkg");
+
+    auto nuspec = generate_nuspec(paths, ipa, ref);
+#ifdef _WIN32
+#define PKGPATH "C:\\zlib2_x64-windows\\**"
+#else
+#define PKGPATH "/zlib2_x64-windows/**"
+#endif
+    std::string expected = R"(<package>
+  <metadata>
+    <id>zlib2_x64-windows</id>
+    <version>1.5.0-packageabi</version>
+    <authors>vcpkg</authors>
+    <description>NOT FOR DIRECT USE. Automatically generated cache package.
+
+a spiffy compression library wrapper
+
+Version: 1.5
+Triplet/Compiler hash: tripletabi
+Features: a, b
+Dependencies:
+</description>
+    <packageTypes><packageType name="vcpkg"/></packageTypes>
+    </metadata>
+  <files><file src=")" PKGPATH R"(" target=""/></files>
+  </package>
+)";
+    auto expected_lines = Strings::split(expected, '\n');
+    auto nuspec_lines = Strings::split(nuspec, '\n');
+    for (size_t i = 0; i < expected_lines.size() && i < nuspec_lines.size(); ++i)
+    {
+        INFO("on line: " << i);
+        REQUIRE(nuspec_lines[i] == expected_lines[i]);
+    }
+    REQUIRE(nuspec_lines.size() == expected_lines.size());
+}
+
+TEST_CASE ("XmlSerializer", "[XmlSerializer]")
+{
+    XmlSerializer xml;
+    xml.open_tag("a");
+    xml.open_tag("b");
+    xml.simple_tag("c", "d");
+    xml.close_tag("b");
+    xml.text("escaping: & < > \" '");
+
+    REQUIRE(xml.buf == R"(<a><b><c>d</c></b>escaping: &amp; &lt; &gt; &quot; &apos;)");
+
+    xml = XmlSerializer();
+    xml.emit_declaration();
+    xml.start_complex_open_tag("a").text_attr("b", "<").text_attr("c", "  ").finish_self_closing_complex_tag();
+    REQUIRE(xml.buf == R"(<?xml version="1.0" encoding="utf-8"?><a b="&lt;" c="  "/>)");
+
+    xml = XmlSerializer();
+    xml.start_complex_open_tag("a").finish_complex_open_tag();
+    REQUIRE(xml.buf == R"(<a>)");
+
+    xml = XmlSerializer();
+    xml.line_break();
+    xml.open_tag("a").line_break().line_break();
+    xml.close_tag("a").line_break().line_break();
+    REQUIRE(xml.buf == "\n<a>\n  \n  </a>\n\n");
+}
\ No newline at end of file
diff --git a/toolsrc/src/vcpkg-test/binaryconfigparser.cpp b/toolsrc/src/vcpkg-test/binaryconfigparser.cpp
index 910d0836f..53e19a8f6 100644
--- a/toolsrc/src/vcpkg-test/binaryconfigparser.cpp
+++ b/toolsrc/src/vcpkg-test/binaryconfigparser.cpp
@@ -57,6 +57,78 @@ TEST_CASE ("BinaryConfigParser files provider", "[binaryconfigparser]")
     }
 }
 
+TEST_CASE ("BinaryConfigParser nuget source provider", "[binaryconfigparser]")
+{
+    {
+        auto parsed = create_binary_provider_from_configs_pure("nuget", {});
+        REQUIRE(!parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("nuget,relative-path", {});
+        REQUIRE(parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("nuget,http://example.org/", {});
+        REQUIRE(parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("nuget," ABSOLUTE_PATH, {});
+        REQUIRE(parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("nuget," ABSOLUTE_PATH ",nonsense", {});
+        REQUIRE(!parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("nuget," ABSOLUTE_PATH ",upload", {});
+        REQUIRE(parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("nuget," ABSOLUTE_PATH ",upload,extra", {});
+        REQUIRE(!parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("nuget,,upload", {});
+        REQUIRE(!parsed.has_value());
+    }
+}
+
+TEST_CASE ("BinaryConfigParser nuget config provider", "[binaryconfigparser]")
+{
+    {
+        auto parsed = create_binary_provider_from_configs_pure("nugetconfig", {});
+        REQUIRE(!parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("nugetconfig,relative-path", {});
+        REQUIRE(!parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("nugetconfig,http://example.org/", {});
+        REQUIRE(!parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("nugetconfig," ABSOLUTE_PATH, {});
+        REQUIRE(parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("nugetconfig," ABSOLUTE_PATH ",nonsense", {});
+        REQUIRE(!parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("nugetconfig," ABSOLUTE_PATH ",upload", {});
+        REQUIRE(parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("nugetconfig," ABSOLUTE_PATH ",upload,extra", {});
+        REQUIRE(!parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("nugetconfig,,upload", {});
+        REQUIRE(!parsed.has_value());
+    }
+}
+
 TEST_CASE ("BinaryConfigParser default provider", "[binaryconfigparser]")
 {
     {
@@ -89,6 +161,18 @@ TEST_CASE ("BinaryConfigParser clear provider", "[binaryconfigparser]")
     }
 }
 
+TEST_CASE ("BinaryConfigParser interactive provider", "[binaryconfigparser]")
+{
+    {
+        auto parsed = create_binary_provider_from_configs_pure("interactive", {});
+        REQUIRE(parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("interactive,upload", {});
+        REQUIRE(!parsed.has_value());
+    }
+}
+
 TEST_CASE ("BinaryConfigParser multiple providers", "[binaryconfigparser]")
 {
     {
diff --git a/toolsrc/src/vcpkg-test/strings.cpp b/toolsrc/src/vcpkg-test/strings.cpp
index eb1f5697f..3172b557f 100644
--- a/toolsrc/src/vcpkg-test/strings.cpp
+++ b/toolsrc/src/vcpkg-test/strings.cpp
@@ -41,3 +41,13 @@ TEST_CASE ("split by char", "[strings]")
     REQUIRE(split("    hello  world    ", ' ') == result_t{"hello", "world"});
     REQUIRE(split("no delimiters", ',') == result_t{"no delimiters"});
 }
+
+TEST_CASE ("find_first_of", "[strings]")
+{
+    using vcpkg::Strings::find_first_of;
+    REQUIRE(find_first_of("abcdefg", "hij") == std::string());
+    REQUIRE(find_first_of("abcdefg", "a") == std::string("abcdefg"));
+    REQUIRE(find_first_of("abcdefg", "g") == std::string("g"));
+    REQUIRE(find_first_of("abcdefg", "bg") == std::string("bcdefg"));
+    REQUIRE(find_first_of("abcdefg", "gb") == std::string("bcdefg"));
+}
diff --git a/toolsrc/src/vcpkg-test/system.cpp b/toolsrc/src/vcpkg-test/system.cpp
index 6e87f9b3e..8b44f8f24 100644
--- a/toolsrc/src/vcpkg-test/system.cpp
+++ b/toolsrc/src/vcpkg-test/system.cpp
@@ -7,16 +7,17 @@
 #include <vcpkg/base/zstringview.h>
 #include <vcpkg/base/strings.h>
 #include <vcpkg/base/system.h>
+#include <vcpkg/base/system.process.h>
 
+using vcpkg::nullopt;
 using vcpkg::Optional;
 using vcpkg::StringView;
 using vcpkg::ZStringView;
 using vcpkg::Checks::check_exit;
+using vcpkg::System::CPUArchitecture;
 using vcpkg::System::get_environment_variable;
-using vcpkg::System::to_cpu_architecture;
 using vcpkg::System::guess_visual_studio_prompt_target_architecture;
-using vcpkg::nullopt;
-using vcpkg::System::CPUArchitecture;
+using vcpkg::System::to_cpu_architecture;
 
 namespace
 {
@@ -126,30 +127,43 @@ TEST_CASE ("guess_visual_studio_prompt", "[system]")
     set_environment_variable("VSCMD_ARG_TGT_ARCH", nullopt);
     CHECK(!guess_visual_studio_prompt_target_architecture().has_value());
     set_environment_variable("VSCMD_ARG_TGT_ARCH", "x86");
-    CHECK(guess_visual_studio_prompt_target_architecture()
-        .value_or_exit(VCPKG_LINE_INFO) == CPUArchitecture::X86);
+    CHECK(guess_visual_studio_prompt_target_architecture().value_or_exit(VCPKG_LINE_INFO) == CPUArchitecture::X86);
     set_environment_variable("VSCMD_ARG_TGT_ARCH", "x64");
-    CHECK(guess_visual_studio_prompt_target_architecture()
-        .value_or_exit(VCPKG_LINE_INFO) == CPUArchitecture::X64);
+    CHECK(guess_visual_studio_prompt_target_architecture().value_or_exit(VCPKG_LINE_INFO) == CPUArchitecture::X64);
     set_environment_variable("VSCMD_ARG_TGT_ARCH", "arm");
-    CHECK(guess_visual_studio_prompt_target_architecture()
-        .value_or_exit(VCPKG_LINE_INFO) == CPUArchitecture::ARM);
+    CHECK(guess_visual_studio_prompt_target_architecture().value_or_exit(VCPKG_LINE_INFO) == CPUArchitecture::ARM);
     set_environment_variable("VSCMD_ARG_TGT_ARCH", "arm64");
-    CHECK(guess_visual_studio_prompt_target_architecture()
-        .value_or_exit(VCPKG_LINE_INFO) == CPUArchitecture::ARM64);
+    CHECK(guess_visual_studio_prompt_target_architecture().value_or_exit(VCPKG_LINE_INFO) == CPUArchitecture::ARM64);
 
     // check that apparent "nested" prompts defer to "vsdevcmd"
     set_environment_variable("VCINSTALLDIR", "anything");
-    CHECK(guess_visual_studio_prompt_target_architecture()
-        .value_or_exit(VCPKG_LINE_INFO) == CPUArchitecture::ARM64);
+    CHECK(guess_visual_studio_prompt_target_architecture().value_or_exit(VCPKG_LINE_INFO) == CPUArchitecture::ARM64);
     set_environment_variable("VSCMD_ARG_TGT_ARCH", nullopt);
     set_environment_variable("Platform", nullopt);
-    CHECK(guess_visual_studio_prompt_target_architecture()
-        .value_or_exit(VCPKG_LINE_INFO) == CPUArchitecture::X86);
+    CHECK(guess_visual_studio_prompt_target_architecture().value_or_exit(VCPKG_LINE_INFO) == CPUArchitecture::X86);
     set_environment_variable("Platform", "x86");
-    CHECK(guess_visual_studio_prompt_target_architecture()
-        .value_or_exit(VCPKG_LINE_INFO) == CPUArchitecture::X86);
+    CHECK(guess_visual_studio_prompt_target_architecture().value_or_exit(VCPKG_LINE_INFO) == CPUArchitecture::X86);
     set_environment_variable("Platform", "x64");
-    CHECK(guess_visual_studio_prompt_target_architecture()
-        .value_or_exit(VCPKG_LINE_INFO) == CPUArchitecture::X64);
+    CHECK(guess_visual_studio_prompt_target_architecture().value_or_exit(VCPKG_LINE_INFO) == CPUArchitecture::X64);
+}
+
+TEST_CASE ("cmdlinebuilder", "[system]")
+{
+    using vcpkg::System::CmdLineBuilder;
+
+    CmdLineBuilder cmd;
+    cmd.path_arg(fs::u8path("relative/path.exe"));
+    cmd.string_arg("abc");
+    cmd.string_arg("hello world!");
+    cmd.string_arg("|");
+    cmd.string_arg(";");
+    REQUIRE(cmd.extract() == "relative/path.exe abc \"hello world!\" \"|\" \";\"");
+
+    cmd.path_arg(fs::u8path("trailing\\slash\\"));
+    cmd.string_arg("inner\"quotes");
+#ifdef _WIN32
+    REQUIRE(cmd.extract() == "\"trailing\\slash\\\\\" \"inner\\\"quotes\"");
+#else
+    REQUIRE(cmd.extract() == "\"trailing\\\\slash\\\\\" \"inner\\\"quotes\"");
+#endif
 }