[vcpkg] Implement versions db generator (#13777)

* [vcpkg] Add script to generate ports versions history

* [vcpkg] Fix formatting

* Fetch port versions from commit ID

* Use global --x-json switch

* Use --no-checkout when cloning secondary instance

* Clone from local repository instead of from GitHub

* Use CmdLineBuilder to build git commands

* Use CmdLineBuilder and reduce repeated code

* Fetch version at baseline and code cleanup

* Guess version scheme from old CONTROL files

* Rename version db generator script

* Simplify x-history json output

* Use CONTROL/manifest parsers on x-history

* Use git-tree instaed of commit-id

* Remove 'ports' field from root object

* Clean up code

* More code cleanup

* Improve port version detection

* Improve generator logging

* Do not ignore parsing errors in CONTROL files

* PR review comments in Python script

* Fix subprocess.run() calls

* Make `canonicalize()` return error instead of terminating

* [vcpkg] Add tests for new test_parse_control_file paths

* Remove unnecessary std::move() calls

* Fix formatting

* Python formatting

Co-authored-by: Robert Schumacher <roschuma@microsoft.com>

3 files changed, 56 insertions, 6 deletions

diff --git a/toolsrc/src/vcpkg-test/binarycaching.cpp b/toolsrc/src/vcpkg-test/binarycaching.cpp
index 33f63e2ea..cb0d7cb99 100644
--- a/toolsrc/src/vcpkg-test/binarycaching.cpp
+++ b/toolsrc/src/vcpkg-test/binarycaching.cpp
@@ -75,7 +75,7 @@ Build-Depends: bzip
 )",
                                              "<testdata>");
     REQUIRE(pghs.has_value());
-    auto maybe_scf = SourceControlFile::parse_control_file(fs::path(), std::move(*pghs.get()));
+    auto maybe_scf = SourceControlFile::parse_control_file(fs::u8string(fs::path()), std::move(*pghs.get()));
     REQUIRE(maybe_scf.has_value());
     SourceControlFileLocation scfl{std::move(*maybe_scf.get()), fs::path()};
 
@@ -255,7 +255,7 @@ Description: a spiffy compression library wrapper
 )",
                                              "<testdata>");
     REQUIRE(pghs.has_value());
-    auto maybe_scf = SourceControlFile::parse_control_file(fs::path(), std::move(*pghs.get()));
+    auto maybe_scf = SourceControlFile::parse_control_file(fs::u8string(fs::path()), std::move(*pghs.get()));
     REQUIRE(maybe_scf.has_value());
     SourceControlFileLocation scfl{std::move(*maybe_scf.get()), fs::path()};
     plan.install_actions.push_back(Dependencies::InstallPlanAction());
@@ -278,7 +278,7 @@ Description: a spiffy compression library wrapper
 )",
                                               "<testdata>");
     REQUIRE(pghs2.has_value());
-    auto maybe_scf2 = SourceControlFile::parse_control_file(fs::path(), std::move(*pghs2.get()));
+    auto maybe_scf2 = SourceControlFile::parse_control_file(fs::u8string(fs::path()), std::move(*pghs2.get()));
     REQUIRE(maybe_scf2.has_value());
     SourceControlFileLocation scfl2{std::move(*maybe_scf2.get()), fs::path()};
     plan.install_actions.push_back(Dependencies::InstallPlanAction());
diff --git a/toolsrc/src/vcpkg-test/manifests.cpp b/toolsrc/src/vcpkg-test/manifests.cpp
index 75b14771f..5ec3f160a 100644
--- a/toolsrc/src/vcpkg-test/manifests.cpp
+++ b/toolsrc/src/vcpkg-test/manifests.cpp
@@ -331,9 +331,9 @@ TEST_CASE ("Serialize all the ports", "[manifests]")
             auto pghs = Paragraphs::parse_paragraphs(contents, fs::u8string(control));
             REQUIRE(pghs);
 
-            scfs.push_back(std::move(
-                *SourceControlFile::parse_control_file(control, std::move(pghs).value_or_exit(VCPKG_LINE_INFO))
-                     .value_or_exit(VCPKG_LINE_INFO)));
+            scfs.push_back(std::move(*SourceControlFile::parse_control_file(
+                                          fs::u8string(control), std::move(pghs).value_or_exit(VCPKG_LINE_INFO))
+                                          .value_or_exit(VCPKG_LINE_INFO)));
         }
         else if (fs.exists(manifest))
         {
diff --git a/toolsrc/src/vcpkg-test/paragraph.cpp b/toolsrc/src/vcpkg-test/paragraph.cpp
index 05ba8cfba..356ee88f8 100644
--- a/toolsrc/src/vcpkg-test/paragraph.cpp
+++ b/toolsrc/src/vcpkg-test/paragraph.cpp
@@ -51,6 +51,38 @@ TEST_CASE ("SourceParagraph construct minimum", "[paragraph]")
     REQUIRE(pgh.core_paragraph->dependencies.size() == 0);
 }
 
+TEST_CASE ("SourceParagraph construct invalid", "[paragraph]")
+{
+    auto m_pgh = test_parse_control_file({{
+        {"Source", "zlib"},
+        {"Version", "1.2.8"},
+        {"Build-Depends", "1.2.8"},
+    }});
+
+    REQUIRE(!m_pgh.has_value());
+
+    m_pgh = test_parse_control_file({{
+        {"Source", "zlib"},
+        {"Version", "1.2.8"},
+        {"Default-Features", "1.2.8"},
+    }});
+
+    REQUIRE(!m_pgh.has_value());
+
+    m_pgh = test_parse_control_file({
+        {
+            {"Source", "zlib"},
+            {"Version", "1.2.8"},
+        },
+        {
+            {"Feature", "a"},
+            {"Build-Depends", "1.2.8"},
+        },
+    });
+
+    REQUIRE(!m_pgh.has_value());
+}
+
 TEST_CASE ("SourceParagraph construct maximum", "[paragraph]")
 {
     auto m_pgh = test_parse_control_file({{
@@ -76,6 +108,24 @@ TEST_CASE ("SourceParagraph construct maximum", "[paragraph]")
     REQUIRE(pgh.core_paragraph->default_features[0] == "df");
 }
 
+TEST_CASE ("SourceParagraph construct feature", "[paragraph]")
+{
+    auto m_pgh = test_parse_control_file({
+        {
+            {"Source", "s"},
+            {"Version", "v"},
+        },
+        {{"Feature", "f"}, {"Description", "d2"}, {"Build-Depends", "bd2"}},
+    });
+    REQUIRE(m_pgh.has_value());
+    auto& pgh = **m_pgh.get();
+
+    REQUIRE(pgh.feature_paragraphs.size() == 1);
+    REQUIRE(pgh.feature_paragraphs[0]->name == "f");
+    REQUIRE(pgh.feature_paragraphs[0]->description == std::vector<std::string>{"d2"});
+    REQUIRE(pgh.feature_paragraphs[0]->dependencies.size() == 1);
+}
+
 TEST_CASE ("SourceParagraph two dependencies", "[paragraph]")
 {
     auto m_pgh = test_parse_control_file({{