[vcpkg] Implement versions db generator (#13777)

* [vcpkg] Add script to generate ports versions history

* [vcpkg] Fix formatting

* Fetch port versions from commit ID

* Use global --x-json switch

* Use --no-checkout when cloning secondary instance

* Clone from local repository instead of from GitHub

* Use CmdLineBuilder to build git commands

* Use CmdLineBuilder and reduce repeated code

* Fetch version at baseline and code cleanup

* Guess version scheme from old CONTROL files

* Rename version db generator script

* Simplify x-history json output

* Use CONTROL/manifest parsers on x-history

* Use git-tree instaed of commit-id

* Remove 'ports' field from root object

* Clean up code

* More code cleanup

* Improve port version detection

* Improve generator logging

* Do not ignore parsing errors in CONTROL files

* PR review comments in Python script

* Fix subprocess.run() calls

* Make `canonicalize()` return error instead of terminating

* [vcpkg] Add tests for new test_parse_control_file paths

* Remove unnecessary std::move() calls

* Fix formatting

* Python formatting

Co-authored-by: Robert Schumacher <roschuma@microsoft.com>

12 files changed, 533 insertions, 101 deletions

diff --git a/scripts/generatePortVersionsDb.py b/scripts/generatePortVersionsDb.py
new file mode 100644
index 000000000..cefd61e1c
--- /dev/null
+++ b/scripts/generatePortVersionsDb.py
@@ -0,0 +1,71 @@
+import os

+import os.path

+import sys

+import subprocess

+import json

+import time

+

+from subprocess import CalledProcessError

+from json.decoder import JSONDecodeError

+from pathlib import Path

+

+

+SCRIPT_DIRECTORY = os.path.dirname(os.path.abspath(__file__))

+

+

+def get_current_git_ref():

+    output = subprocess.run(['git.exe', '-C', SCRIPT_DIRECTORY, 'rev-parse', '--verify', 'HEAD'], 

+        capture_output=True, 

+        encoding='utf-8')

+    if output.returncode == 0:

+        return output.stdout.strip()

+    print(f"Failed to get git ref:", output.stderr.strip(), file=sys.stderr)

+    return None

+

+

+def generate_port_versions_db(ports_path, db_path, revision):

+    start_time = time.time()

+    port_names = [item for item in os.listdir(ports_path) if os.path.isdir(os.path.join(ports_path, item))]

+    total_count = len(port_names)

+    for counter, port_name in enumerate(port_names):

+        containing_dir = os.path.join(db_path, f'{port_name[0]}-')

+        os.makedirs(containing_dir, exist_ok=True)

+        output_filepath = os.path.join(containing_dir, f'{port_name}.json')

+        if not os.path.exists(output_filepath):            

+            output = subprocess.run(

+                [os.path.join(SCRIPT_DIRECTORY, '../vcpkg'), 'x-history', port_name, '--x-json'], 

+                capture_output=True, encoding='utf-8')

+            if output.returncode == 0:

+                try:

+                    versions_object = json.loads(output.stdout)

+                    with open(output_filepath, 'w') as output_file:

+                        json.dump(versions_object, output_file)

+                except JSONDecodeError:

+                    print(f'Maformed JSON from vcpkg x-history {port_name}: ', output.stdout.strip(), file=sys.stderr)

+            else:

+                print(f'x-history {port_name} failed: ', output.stdout.strip(), file=sys.stderr)

+        # This should be replaced by a progress bar

+        if counter > 0 and counter % 100 == 0:

+            elapsed_time = time.time() - start_time

+            print(f'Processed {counter} out of {total_count}. Elapsed time: {elapsed_time:.2f} seconds')

+    rev_file = os.path.join(db_path, revision)

+    Path(rev_file).touch()

+    elapsed_time = time.time() - start_time

+    print(f'Processed {total_count} total ports. Elapsed time: {elapsed_time:.2f} seconds')

+    

+

+def main(ports_path, db_path):

+    revision = get_current_git_ref()

+    if not revision:

+        print('Couldn\'t fetch current Git revision', file=sys.stderr)

+        sys.exit(1)

+    rev_file = os.path.join(db_path, revision)

+    if os.path.exists(rev_file):

+        print(f'Database files already exist for commit {revision}')

+        sys.exit(0)

+    generate_port_versions_db(ports_path=ports_path, db_path=db_path, revision=revision)

+

+

+if __name__ == "__main__":

+    main(ports_path=os.path.join(SCRIPT_DIRECTORY, '../ports'), 

+        db_path=os.path.join(SCRIPT_DIRECTORY, '../port_versions'))

diff --git a/toolsrc/include/vcpkg/paragraphs.h b/toolsrc/include/vcpkg/paragraphs.h
index f4ea36ede..0293f0733 100644
--- a/toolsrc/include/vcpkg/paragraphs.h
+++ b/toolsrc/include/vcpkg/paragraphs.h
@@ -17,12 +17,18 @@ namespace vcpkg::Paragraphs
 
     ExpectedS<Paragraph> parse_single_paragraph(const std::string& str, const std::string& origin);
     ExpectedS<Paragraph> get_single_paragraph(const Files::Filesystem& fs, const fs::path& control_path);
+
     ExpectedS<std::vector<Paragraph>> get_paragraphs(const Files::Filesystem& fs, const fs::path& control_path);
+    ExpectedS<std::vector<Paragraph>> get_paragraphs_text(const std::string& text, const std::string& origin);
+
     ExpectedS<std::vector<Paragraph>> parse_paragraphs(const std::string& str, const std::string& origin);
 
     bool is_port_directory(const Files::Filesystem& fs, const fs::path& path);
 
     Parse::ParseExpected<SourceControlFile> try_load_port(const Files::Filesystem& fs, const fs::path& path);
+    Parse::ParseExpected<SourceControlFile> try_load_port_text(const std::string& text,
+                                                               const std::string& origin,
+                                                               bool is_manifest);
 
     ExpectedS<BinaryControlFile> try_load_cached_package(const VcpkgPaths& paths, const PackageSpec& spec);
 
diff --git a/toolsrc/include/vcpkg/sourceparagraph.h b/toolsrc/include/vcpkg/sourceparagraph.h
index 21e5a4ef9..d88c0b0fd 100644
--- a/toolsrc/include/vcpkg/sourceparagraph.h
+++ b/toolsrc/include/vcpkg/sourceparagraph.h
@@ -91,11 +91,14 @@ namespace vcpkg
             return ret;
         }
 
+        static Parse::ParseExpected<SourceControlFile> parse_manifest_object(const std::string& origin,
+                                                                             const Json::Object& object);
+
         static Parse::ParseExpected<SourceControlFile> parse_manifest_file(const fs::path& path_to_manifest,
                                                                            const Json::Object& object);
 
         static Parse::ParseExpected<SourceControlFile> parse_control_file(
-            const fs::path& path_to_control, std::vector<Parse::Paragraph>&& control_paragraphs);
+            const std::string& origin, std::vector<Parse::Paragraph>&& control_paragraphs);
 
         // Always non-null in non-error cases
         std::unique_ptr<SourceParagraph> core_paragraph;
diff --git a/toolsrc/include/vcpkg/versions.h b/toolsrc/include/vcpkg/versions.h
index f0304c788..331bf7f8b 100644
--- a/toolsrc/include/vcpkg/versions.h
+++ b/toolsrc/include/vcpkg/versions.h
@@ -1,14 +1,12 @@
 #pragma once
 
-#include <vcpkg/fwd/vcpkgpaths.h>
-
 namespace vcpkg::Versions
 {
     enum class Scheme
     {
-        String,
         Relaxed,
         Semver,
-        Date
+        Date,
+        String
     };
 }
diff --git a/toolsrc/src/vcpkg-test/binarycaching.cpp b/toolsrc/src/vcpkg-test/binarycaching.cpp
index 33f63e2ea..cb0d7cb99 100644
--- a/toolsrc/src/vcpkg-test/binarycaching.cpp
+++ b/toolsrc/src/vcpkg-test/binarycaching.cpp
@@ -75,7 +75,7 @@ Build-Depends: bzip
 )",
                                              "<testdata>");
     REQUIRE(pghs.has_value());
-    auto maybe_scf = SourceControlFile::parse_control_file(fs::path(), std::move(*pghs.get()));
+    auto maybe_scf = SourceControlFile::parse_control_file(fs::u8string(fs::path()), std::move(*pghs.get()));
     REQUIRE(maybe_scf.has_value());
     SourceControlFileLocation scfl{std::move(*maybe_scf.get()), fs::path()};
 
@@ -255,7 +255,7 @@ Description: a spiffy compression library wrapper
 )",
                                              "<testdata>");
     REQUIRE(pghs.has_value());
-    auto maybe_scf = SourceControlFile::parse_control_file(fs::path(), std::move(*pghs.get()));
+    auto maybe_scf = SourceControlFile::parse_control_file(fs::u8string(fs::path()), std::move(*pghs.get()));
     REQUIRE(maybe_scf.has_value());
     SourceControlFileLocation scfl{std::move(*maybe_scf.get()), fs::path()};
     plan.install_actions.push_back(Dependencies::InstallPlanAction());
@@ -278,7 +278,7 @@ Description: a spiffy compression library wrapper
 )",
                                               "<testdata>");
     REQUIRE(pghs2.has_value());
-    auto maybe_scf2 = SourceControlFile::parse_control_file(fs::path(), std::move(*pghs2.get()));
+    auto maybe_scf2 = SourceControlFile::parse_control_file(fs::u8string(fs::path()), std::move(*pghs2.get()));
     REQUIRE(maybe_scf2.has_value());
     SourceControlFileLocation scfl2{std::move(*maybe_scf2.get()), fs::path()};
     plan.install_actions.push_back(Dependencies::InstallPlanAction());
diff --git a/toolsrc/src/vcpkg-test/manifests.cpp b/toolsrc/src/vcpkg-test/manifests.cpp
index 75b14771f..5ec3f160a 100644
--- a/toolsrc/src/vcpkg-test/manifests.cpp
+++ b/toolsrc/src/vcpkg-test/manifests.cpp
@@ -331,9 +331,9 @@ TEST_CASE ("Serialize all the ports", "[manifests]")
             auto pghs = Paragraphs::parse_paragraphs(contents, fs::u8string(control));
             REQUIRE(pghs);
 
-            scfs.push_back(std::move(
-                *SourceControlFile::parse_control_file(control, std::move(pghs).value_or_exit(VCPKG_LINE_INFO))
-                     .value_or_exit(VCPKG_LINE_INFO)));
+            scfs.push_back(std::move(*SourceControlFile::parse_control_file(
+                                          fs::u8string(control), std::move(pghs).value_or_exit(VCPKG_LINE_INFO))
+                                          .value_or_exit(VCPKG_LINE_INFO)));
         }
         else if (fs.exists(manifest))
         {
diff --git a/toolsrc/src/vcpkg-test/paragraph.cpp b/toolsrc/src/vcpkg-test/paragraph.cpp
index 05ba8cfba..356ee88f8 100644
--- a/toolsrc/src/vcpkg-test/paragraph.cpp
+++ b/toolsrc/src/vcpkg-test/paragraph.cpp
@@ -51,6 +51,38 @@ TEST_CASE ("SourceParagraph construct minimum", "[paragraph]")
     REQUIRE(pgh.core_paragraph->dependencies.size() == 0);
 }
 
+TEST_CASE ("SourceParagraph construct invalid", "[paragraph]")
+{
+    auto m_pgh = test_parse_control_file({{
+        {"Source", "zlib"},
+        {"Version", "1.2.8"},
+        {"Build-Depends", "1.2.8"},
+    }});
+
+    REQUIRE(!m_pgh.has_value());
+
+    m_pgh = test_parse_control_file({{
+        {"Source", "zlib"},
+        {"Version", "1.2.8"},
+        {"Default-Features", "1.2.8"},
+    }});
+
+    REQUIRE(!m_pgh.has_value());
+
+    m_pgh = test_parse_control_file({
+        {
+            {"Source", "zlib"},
+            {"Version", "1.2.8"},
+        },
+        {
+            {"Feature", "a"},
+            {"Build-Depends", "1.2.8"},
+        },
+    });
+
+    REQUIRE(!m_pgh.has_value());
+}
+
 TEST_CASE ("SourceParagraph construct maximum", "[paragraph]")
 {
     auto m_pgh = test_parse_control_file({{
@@ -76,6 +108,24 @@ TEST_CASE ("SourceParagraph construct maximum", "[paragraph]")
     REQUIRE(pgh.core_paragraph->default_features[0] == "df");
 }
 
+TEST_CASE ("SourceParagraph construct feature", "[paragraph]")
+{
+    auto m_pgh = test_parse_control_file({
+        {
+            {"Source", "s"},
+            {"Version", "v"},
+        },
+        {{"Feature", "f"}, {"Description", "d2"}, {"Build-Depends", "bd2"}},
+    });
+    REQUIRE(m_pgh.has_value());
+    auto& pgh = **m_pgh.get();
+
+    REQUIRE(pgh.feature_paragraphs.size() == 1);
+    REQUIRE(pgh.feature_paragraphs[0]->name == "f");
+    REQUIRE(pgh.feature_paragraphs[0]->description == std::vector<std::string>{"d2"});
+    REQUIRE(pgh.feature_paragraphs[0]->dependencies.size() == 1);
+}
+
 TEST_CASE ("SourceParagraph two dependencies", "[paragraph]")
 {
     auto m_pgh = test_parse_control_file({{
diff --git a/toolsrc/src/vcpkg/commands.format-manifest.cpp b/toolsrc/src/vcpkg/commands.format-manifest.cpp
index 1be6ecbeb..8d4be1ce9 100644
--- a/toolsrc/src/vcpkg/commands.format-manifest.cpp
+++ b/toolsrc/src/vcpkg/commands.format-manifest.cpp
@@ -80,8 +80,8 @@ namespace
                            paragraphs.error());
             return {};
         }
-        auto scf_res =
-            SourceControlFile::parse_control_file(control_path, std::move(paragraphs).value_or_exit(VCPKG_LINE_INFO));
+        auto scf_res = SourceControlFile::parse_control_file(fs::u8string(control_path),
+                                                             std::move(paragraphs).value_or_exit(VCPKG_LINE_INFO));
         if (!scf_res)
         {
             System::printf(System::Color::error, "Failed to parse control file: %s\n", control_path_string);
diff --git a/toolsrc/src/vcpkg/commands.porthistory.cpp b/toolsrc/src/vcpkg/commands.porthistory.cpp
index 4313ab647..aa011b999 100644
--- a/toolsrc/src/vcpkg/commands.porthistory.cpp
+++ b/toolsrc/src/vcpkg/commands.porthistory.cpp
@@ -1,77 +1,257 @@
+#include <vcpkg/base/json.h>
 #include <vcpkg/base/system.print.h>
 #include <vcpkg/base/system.process.h>
 #include <vcpkg/base/util.h>
 
 #include <vcpkg/commands.porthistory.h>
 #include <vcpkg/help.h>
+#include <vcpkg/paragraphs.h>
 #include <vcpkg/tools.h>
 #include <vcpkg/vcpkgcmdarguments.h>
 #include <vcpkg/vcpkgpaths.h>
+#include <vcpkg/versions.h>
 
 namespace vcpkg::Commands::PortHistory
 {
-    struct PortControlVersion
+    namespace
     {
-        std::string commit_id;
-        std::string version;
-        std::string date;
-    };
+        struct HistoryVersion
+        {
+            std::string port_name;
+            std::string git_tree;
+            std::string commit_id;
+            std::string commit_date;
+            std::string version_string;
+            std::string version;
+            int port_version;
+            Versions::Scheme scheme;
+        };
 
-    static System::ExitCodeAndOutput run_git_command(const VcpkgPaths& paths, const std::string& cmd)
-    {
-        const fs::path& git_exe = paths.get_tool_exe(Tools::GIT);
-        const fs::path dot_git_dir = paths.root / ".git";
+        const System::ExitCodeAndOutput run_git_command_inner(const VcpkgPaths& paths,
+                                                              const fs::path& dot_git_directory,
+                                                              const fs::path& working_directory,
+                                                              const std::string& cmd)
+        {
+            const fs::path& git_exe = paths.get_tool_exe(Tools::GIT);
 
-        const std::string full_cmd =
-            Strings::format(R"("%s" --git-dir="%s" %s)", fs::u8string(git_exe), fs::u8string(dot_git_dir), cmd);
+            System::CmdLineBuilder builder;
+            builder.path_arg(git_exe)
+                .string_arg(Strings::concat("--git-dir=", fs::u8string(dot_git_directory)))
+                .string_arg(Strings::concat("--work-tree=", fs::u8string(working_directory)));
+            const std::string full_cmd = Strings::concat(builder.extract(), " ", cmd);
 
-        auto output = System::cmd_execute_and_capture_output(full_cmd);
-        Checks::check_exit(VCPKG_LINE_INFO, output.exit_code == 0, "Failed to run command: %s", full_cmd);
-        return output;
-    }
+            const auto output = System::cmd_execute_and_capture_output(full_cmd);
+            return output;
+        }
 
-    static std::string get_version_from_commit(const VcpkgPaths& paths,
-                                               const std::string& commit_id,
-                                               const std::string& port_name)
-    {
-        const std::string cmd = Strings::format(R"(show %s:ports/%s/CONTROL)", commit_id, port_name);
-        auto output = run_git_command(paths, cmd);
+        const System::ExitCodeAndOutput run_git_command(const VcpkgPaths& paths, const std::string& cmd)
+        {
+            const fs::path& work_dir = paths.root;
+            const fs::path dot_git_dir = paths.root / ".git";
+
+            return run_git_command_inner(paths, dot_git_dir, work_dir, cmd);
+        }
 
-        const auto version = Strings::find_at_most_one_enclosed(output.output, "\nVersion: ", "\n");
-        const auto port_version = Strings::find_at_most_one_enclosed(output.output, "\nPort-Version: ", "\n");
-        Checks::check_exit(VCPKG_LINE_INFO, version.has_value(), "CONTROL file does not have a 'Version' field");
-        if (auto pv = port_version.get())
+        bool is_date(const std::string& version_string)
         {
-            return Strings::format("%s#%s", version.get()->to_string(), pv->to_string());
+            // The date regex is not complete, it matches strings that look like dates,
+            // e.g.: 2020-99-99.
+            //
+            // The regex has two capture groups:
+            // * Date: "^([0-9]{4,}[-][0-9]{2}[-][0-9]{2})", it matches strings that resemble YYYY-MM-DD.
+            //         It does not validate that MM <= 12, or that DD is possible with the given MM.
+            //         YYYY should be AT LEAST 4 digits, for some kind of "future proofing".
+            std::regex re("^([0-9]{4,}[-][0-9]{2}[-][0-9]{2})((?:[.|-][0-9a-zA-Z]+)*)$");
+            return std::regex_match(version_string, re);
         }
 
-        return version.get()->to_string();
-    }
+        bool is_date_without_tags(const std::string& version_string)
+        {
+            std::regex re("^([0-9]{4,}[-][0-9]{2}[-][0-9]{2})$");
+            return std::regex_match(version_string, re);
+        }
 
-    static std::vector<PortControlVersion> read_versions_from_log(const VcpkgPaths& paths, const std::string& port_name)
-    {
-        const std::string cmd =
-            Strings::format(R"(log --format="%%H %%cd" --date=short --left-only -- ports/%s/.)", port_name);
-        auto output = run_git_command(paths, cmd);
-
-        auto commits = Util::fmap(
-            Strings::split(output.output, '\n'), [](const std::string& line) -> auto {
-                auto parts = Strings::split(line, ' ');
-                return std::make_pair(parts[0], parts[1]);
-            });
-
-        std::vector<PortControlVersion> ret;
-        std::string last_version;
-        for (auto&& commit_date_pair : commits)
+        bool is_semver(const std::string& version_string)
         {
-            const std::string version = get_version_from_commit(paths, commit_date_pair.first, port_name);
-            if (last_version != version)
+            // This is the "official" SemVer regex, taken from:
+            // https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string
+            std::regex re("^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*"
+                          ")(?:\\.(?:0|["
+                          "1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$");
+            return std::regex_match(version_string, re);
+        }
+
+        bool is_semver_relaxed(const std::string& version_string)
+        {
+            std::regex re("^(?:[0-9a-zA-Z]+)\\.(?:[0-9a-zA-Z]+)\\.(?:[0-9a-zA-Z]+)(?:[\\.|-|\\+][0-9a-zA-Z]+)*$");
+            return std::regex_match(version_string, re);
+        }
+
+        const Versions::Scheme guess_version_scheme(const std::string& version_string)
+        {
+            if (is_date(version_string))
+            {
+                return Versions::Scheme::Date;
+            }
+
+            if (is_semver(version_string) || is_semver_relaxed(version_string))
             {
-                ret.emplace_back(PortControlVersion{commit_date_pair.first, version, commit_date_pair.second});
-                last_version = version;
+                return Versions::Scheme::Relaxed;
             }
+
+            return Versions::Scheme::String;
+        }
+
+        std::pair<std::string, int> clean_version_string(const std::string& version_string,
+                                                         int port_version,
+                                                         bool from_manifest)
+        {
+            // Manifest files and ports that use the `Port-Version` field are assumed to have a clean version string
+            // already.
+            if (from_manifest || port_version > 0)
+            {
+                return std::make_pair(version_string, port_version);
+            }
+
+            std::string clean_version = version_string;
+            int clean_port_version = 0;
+
+            const auto index = version_string.find_last_of('-');
+            if (index != std::string::npos)
+            {
+                // Very lazy check to keep date versions untouched
+                if (!is_date_without_tags(version_string))
+                {
+                    auto maybe_port_version = version_string.substr(index + 1);
+                    clean_version.resize(index);
+
+                    try
+                    {
+                        clean_port_version = std::stoi(maybe_port_version);
+                    }
+                    catch (std::exception&)
+                    {
+                        // If not convertible to int consider last fragment as part of version string
+                        clean_version = version_string;
+                    }
+                }
+            }
+
+            return std::make_pair(clean_version, clean_port_version);
+        }
+
+        vcpkg::Optional<HistoryVersion> get_version_from_text(const std::string& text,
+                                                              const std::string& git_tree,
+                                                              const std::string& commit_id,
+                                                              const std::string& commit_date,
+                                                              const std::string& port_name,
+                                                              bool is_manifest)
+        {
+            auto res = Paragraphs::try_load_port_text(text, Strings::concat(commit_id, ":", port_name), is_manifest);
+            if (const auto& maybe_scf = res.get())
+            {
+                if (const auto& scf = maybe_scf->get())
+                {
+                    // TODO: Get clean version name and port version
+                    const auto version_string = scf->core_paragraph->version;
+                    const auto clean_version =
+                        clean_version_string(version_string, scf->core_paragraph->port_version, is_manifest);
+
+                    // SCF to HistoryVersion
+                    return HistoryVersion{
+                        port_name,
+                        git_tree,
+                        commit_id,
+                        commit_date,
+                        Strings::concat(clean_version.first, "#", std::to_string(clean_version.second)),
+                        clean_version.first,
+                        clean_version.second,
+                        guess_version_scheme(clean_version.first)};
+                }
+            }
+
+            return nullopt;
+        }
+
+        vcpkg::Optional<HistoryVersion> get_version_from_commit(const VcpkgPaths& paths,
+                                                                const std::string& commit_id,
+                                                                const std::string& commit_date,
+                                                                const std::string& port_name)
+        {
+            const std::string rev_parse_cmd = Strings::format("rev-parse %s:ports/%s", commit_id, port_name);
+            auto rev_parse_output = run_git_command(paths, rev_parse_cmd);
+            if (rev_parse_output.exit_code == 0)
+            {
+                // Remove newline character
+                const auto git_tree = Strings::trim(std::move(rev_parse_output.output));
+
+                // Do we have a manifest file?
+                const std::string manifest_cmd = Strings::format(R"(show %s:vcpkg.json)", git_tree, port_name);
+                auto manifest_output = run_git_command(paths, manifest_cmd);
+                if (manifest_output.exit_code == 0)
+                {
+                    return get_version_from_text(
+                        manifest_output.output, git_tree, commit_id, commit_date, port_name, true);
+                }
+
+                const std::string cmd = Strings::format(R"(show %s:CONTROL)", git_tree, commit_id, port_name);
+                auto control_output = run_git_command(paths, cmd);
+
+                if (control_output.exit_code == 0)
+                {
+                    return get_version_from_text(
+                        control_output.output, git_tree, commit_id, commit_date, port_name, false);
+                }
+            }
+
+            return nullopt;
+        }
+
+        std::vector<HistoryVersion> read_versions_from_log(const VcpkgPaths& paths, const std::string& port_name)
+        {
+            // log --format="%H %cd" --date=short --left-only -- ports/{port_name}/.
+            System::CmdLineBuilder builder;
+            builder.string_arg("log");
+            builder.string_arg("--format=%H %cd");
+            builder.string_arg("--date=short");
+            builder.string_arg("--left-only");
+            builder.string_arg("--"); // Begin pathspec
+            builder.string_arg(Strings::format("ports/%s/.", port_name));
+            const auto output = run_git_command(paths, builder.extract());
+
+            auto commits = Util::fmap(
+                Strings::split(output.output, '\n'), [](const std::string& line) -> auto {
+                    auto parts = Strings::split(line, ' ');
+                    return std::make_pair(parts[0], parts[1]);
+                });
+
+            std::vector<HistoryVersion> ret;
+            std::string last_version;
+            for (auto&& commit_date_pair : commits)
+            {
+                auto maybe_version =
+                    get_version_from_commit(paths, commit_date_pair.first, commit_date_pair.second, port_name);
+                if (maybe_version.has_value())
+                {
+                    const auto version = maybe_version.value_or_exit(VCPKG_LINE_INFO);
+
+                    // Keep latest port with the current version string
+                    if (last_version != version.version_string)
+                    {
+                        last_version = version.version_string;
+                        ret.emplace_back(version);
+                    }
+                }
+                // NOTE: Uncomment this code if you're looking for edge cases to patch in the generation.
+                //       Otherwise, x-history simply skips "bad" versions, which is OK behavior.
+                // else
+                //{
+                //    Checks::exit_with_message(VCPKG_LINE_INFO, "Failed to get version from %s:%s",
+                //    commit_date_pair.first, port_name);
+                //}
+            }
+            return ret;
         }
-        return ret;
     }
 
     const CommandStructure COMMAND_STRUCTURE = {
@@ -84,13 +264,47 @@ namespace vcpkg::Commands::PortHistory
 
     void perform_and_exit(const VcpkgCmdArguments& args, const VcpkgPaths& paths)
     {
-        (void)args.parse_arguments(COMMAND_STRUCTURE);
+        const ParsedArguments options = args.parse_arguments(COMMAND_STRUCTURE);
+
         std::string port_name = args.command_arguments.at(0);
-        std::vector<PortControlVersion> versions = read_versions_from_log(paths, port_name);
-        System::print2("             version          date    vcpkg commit\n");
-        for (auto&& version : versions)
+        std::vector<HistoryVersion> versions = read_versions_from_log(paths, port_name);
+
+        if (args.output_json())
         {
-            System::printf("%20.20s    %s    %s\n", version.version, version.date, version.commit_id);
+            Json::Array versions_json;
+            for (auto&& version : versions)
+            {
+                Json::Object object;
+                object.insert("git-tree", Json::Value::string(version.git_tree));
+                switch (version.scheme)
+                {
+                    case Versions::Scheme::Semver: // falls through
+                    case Versions::Scheme::Relaxed:
+                        object.insert("version", Json::Value::string(version.version));
+                        break;
+                    case Versions::Scheme::Date:
+                        object.insert("version-date", Json::Value::string(version.version));
+                        break;
+                    case Versions::Scheme::String: // falls through
+                    default: object.insert("version-string", Json::Value::string(version.version)); break;
+                }
+                object.insert("port-version", Json::Value::integer(version.port_version));
+                versions_json.push_back(std::move(object));
+            }
+
+            Json::Object root;
+            root.insert("versions", versions_json);
+
+            auto json_string = Json::stringify(root, vcpkg::Json::JsonStyle::with_spaces(2));
+            System::printf("%s\n", json_string);
+        }
+        else
+        {
+            System::print2("             version          date    vcpkg commit\n");
+            for (auto&& version : versions)
+            {
+                System::printf("%20.20s    %s    %s\n", version.version_string, version.commit_date, version.commit_id);
+            }
         }
         Checks::exit_success(VCPKG_LINE_INFO);
     }
diff --git a/toolsrc/src/vcpkg/paragraphs.cpp b/toolsrc/src/vcpkg/paragraphs.cpp
index 90073317c..ba28acccf 100644
--- a/toolsrc/src/vcpkg/paragraphs.cpp
+++ b/toolsrc/src/vcpkg/paragraphs.cpp
@@ -241,6 +241,11 @@ namespace vcpkg::Paragraphs
         return contents.error().message();
     }
 
+    ExpectedS<std::vector<Paragraph>> get_paragraphs_text(const std::string& text, const std::string& origin)
+    {
+        return parse_paragraphs(text, origin);
+    }
+
     ExpectedS<std::vector<Paragraph>> get_paragraphs(const Files::Filesystem& fs, const fs::path& control_path)
     {
         const Expected<std::string> contents = fs.read_contents(control_path);
@@ -262,36 +267,73 @@ namespace vcpkg::Paragraphs
         return fs.exists(path / fs::u8path("CONTROL")) || fs.exists(path / fs::u8path("vcpkg.json"));
     }
 
-    static ParseExpected<SourceControlFile> try_load_manifest(const Files::Filesystem& fs,
-                                                              const std::string& port_name,
-                                                              const fs::path& path_to_manifest,
-                                                              std::error_code& ec)
+    static ParseExpected<SourceControlFile> try_load_manifest_object(
+        const std::string& origin,
+        const ExpectedT<std::pair<vcpkg::Json::Value, vcpkg::Json::JsonStyle>, std::unique_ptr<Parse::IParseError>>&
+            res)
     {
         auto error_info = std::make_unique<ParseControlErrorInfo>();
-        auto res = Json::parse_file(fs, path_to_manifest, ec);
-        if (ec) return error_info;
-
         if (auto val = res.get())
         {
             if (val->first.is_object())
             {
-                return SourceControlFile::parse_manifest_file(path_to_manifest, val->first.object());
+                return SourceControlFile::parse_manifest_object(origin, val->first.object());
             }
             else
             {
-                error_info->name = port_name;
+                error_info->name = origin;
                 error_info->error = "Manifest files must have a top-level object";
                 return error_info;
             }
         }
         else
         {
-            error_info->name = port_name;
+            error_info->name = origin;
             error_info->error = res.error()->format();
             return error_info;
         }
     }
 
+    static ParseExpected<SourceControlFile> try_load_manifest_text(const std::string& text, const std::string& origin)
+    {
+        auto res = Json::parse(text);
+        return try_load_manifest_object(origin, res);
+    }
+
+    static ParseExpected<SourceControlFile> try_load_manifest(const Files::Filesystem& fs,
+                                                              const std::string& port_name,
+                                                              const fs::path& path_to_manifest,
+                                                              std::error_code& ec)
+    {
+        (void)port_name;
+
+        auto error_info = std::make_unique<ParseControlErrorInfo>();
+        auto res = Json::parse_file(fs, path_to_manifest, ec);
+        if (ec) return error_info;
+
+        return try_load_manifest_object(fs::u8string(path_to_manifest), res);
+    }
+
+    ParseExpected<SourceControlFile> try_load_port_text(const std::string& text,
+                                                        const std::string& origin,
+                                                        bool is_manifest)
+    {
+        if (is_manifest)
+        {
+            return try_load_manifest_text(text, origin);
+        }
+
+        ExpectedS<std::vector<Paragraph>> pghs = get_paragraphs_text(text, origin);
+        if (auto vector_pghs = pghs.get())
+        {
+            return SourceControlFile::parse_control_file(origin, std::move(*vector_pghs));
+        }
+        auto error_info = std::make_unique<ParseControlErrorInfo>();
+        error_info->name = fs::u8string(origin);
+        error_info->error = pghs.error();
+        return error_info;
+    }
+
     ParseExpected<SourceControlFile> try_load_port(const Files::Filesystem& fs, const fs::path& path)
     {
         const auto path_to_manifest = path / fs::u8path("vcpkg.json");
@@ -318,7 +360,7 @@ namespace vcpkg::Paragraphs
         ExpectedS<std::vector<Paragraph>> pghs = get_paragraphs(fs, path_to_control);
         if (auto vector_pghs = pghs.get())
         {
-            return SourceControlFile::parse_control_file(path_to_control, std::move(*vector_pghs));
+            return SourceControlFile::parse_control_file(fs::u8string(path_to_control), std::move(*vector_pghs));
         }
         auto error_info = std::make_unique<ParseControlErrorInfo>();
         error_info->name = fs::u8string(path.filename());
diff --git a/toolsrc/src/vcpkg/sourceparagraph.cpp b/toolsrc/src/vcpkg/sourceparagraph.cpp
index 802e6d6a3..74858a738 100644
--- a/toolsrc/src/vcpkg/sourceparagraph.cpp
+++ b/toolsrc/src/vcpkg/sourceparagraph.cpp
@@ -221,7 +221,7 @@ namespace vcpkg
 
                 fpgh.extra_info.sort_keys();
             }
-            void operator()(SourceControlFile& scf) const
+            [[nodiscard]] std::unique_ptr<ParseControlErrorInfo> operator()(SourceControlFile& scf) const
             {
                 (*this)(*scf.core_paragraph);
                 std::for_each(scf.feature_paragraphs.begin(), scf.feature_paragraphs.end(), *this);
@@ -231,20 +231,21 @@ namespace vcpkg
                     std::adjacent_find(scf.feature_paragraphs.begin(), scf.feature_paragraphs.end(), FeatureEqual{});
                 if (adjacent_equal != scf.feature_paragraphs.end())
                 {
-                    Checks::exit_with_message(VCPKG_LINE_INFO,
-                                              R"(Multiple features with the same name for port %s: %s
+                    auto error_info = std::make_unique<ParseControlErrorInfo>();
+                    error_info->name = scf.core_paragraph->name;
+                    error_info->error = Strings::format(R"(Multiple features with the same name for port %s: %s
     This is invalid; please make certain that features have distinct names.)",
-                                              scf.core_paragraph->name,
-                                              (*adjacent_equal)->name);
+                                                        scf.core_paragraph->name,
+                                                        (*adjacent_equal)->name);
+                    return error_info;
                 }
+                return nullptr;
             }
         } canonicalize{};
     }
 
-    static ParseExpected<SourceParagraph> parse_source_paragraph(const fs::path& path_to_control, Paragraph&& fields)
+    static ParseExpected<SourceParagraph> parse_source_paragraph(const std::string& origin, Paragraph&& fields)
     {
-        auto origin = fs::u8string(path_to_control);
-
         ParagraphParser parser(std::move(fields));
 
         auto spgh = std::make_unique<SourceParagraph>();
@@ -276,10 +277,35 @@ namespace vcpkg
         TextRowCol textrowcol;
         std::string buf;
         parser.optional_field(SourceParagraphFields::BUILD_DEPENDS, {buf, textrowcol});
-        spgh->dependencies = parse_dependencies_list(buf, origin, textrowcol).value_or_exit(VCPKG_LINE_INFO);
+
+        auto maybe_dependencies = parse_dependencies_list(buf, origin, textrowcol);
+        if (maybe_dependencies.has_value())
+        {
+            spgh->dependencies = maybe_dependencies.value_or_exit(VCPKG_LINE_INFO);
+        }
+        else
+        {
+            auto error_info = std::make_unique<ParseControlErrorInfo>();
+            error_info->name = origin;
+            error_info->error = maybe_dependencies.error();
+            return error_info;
+        }
+
         buf.clear();
         parser.optional_field(SourceParagraphFields::DEFAULT_FEATURES, {buf, textrowcol});
-        spgh->default_features = parse_default_features_list(buf, origin, textrowcol).value_or_exit(VCPKG_LINE_INFO);
+
+        auto maybe_default_features = parse_default_features_list(buf, origin, textrowcol);
+        if (maybe_default_features.has_value())
+        {
+            spgh->default_features = maybe_default_features.value_or_exit(VCPKG_LINE_INFO);
+        }
+        else
+        {
+            auto error_info = std::make_unique<ParseControlErrorInfo>();
+            error_info->name = origin;
+            error_info->error = maybe_default_features.error();
+            return error_info;
+        }
 
         auto supports_expr = parser.optional_field(SourceParagraphFields::SUPPORTS);
         if (!supports_expr.empty())
@@ -304,9 +330,8 @@ namespace vcpkg
             return spgh;
     }
 
-    static ParseExpected<FeatureParagraph> parse_feature_paragraph(const fs::path& path_to_control, Paragraph&& fields)
+    static ParseExpected<FeatureParagraph> parse_feature_paragraph(const std::string& origin, Paragraph&& fields)
     {
-        auto origin = fs::u8string(path_to_control);
         ParagraphParser parser(std::move(fields));
 
         auto fpgh = std::make_unique<FeatureParagraph>();
@@ -315,9 +340,19 @@ namespace vcpkg
         fpgh->description = Strings::split(parser.required_field(SourceParagraphFields::DESCRIPTION), '\n');
         trim_all(fpgh->description);
 
-        fpgh->dependencies =
-            parse_dependencies_list(parser.optional_field(SourceParagraphFields::BUILD_DEPENDS), origin)
-                .value_or_exit(VCPKG_LINE_INFO);
+        auto maybe_dependencies =
+            parse_dependencies_list(parser.optional_field(SourceParagraphFields::BUILD_DEPENDS), origin);
+        if (maybe_dependencies.has_value())
+        {
+            fpgh->dependencies = maybe_dependencies.value_or_exit(VCPKG_LINE_INFO);
+        }
+        else
+        {
+            auto error_info = std::make_unique<ParseControlErrorInfo>();
+            error_info->name = origin;
+            error_info->error = maybe_dependencies.error();
+            return error_info;
+        }
 
         auto err = parser.error_info(fpgh->name.empty() ? origin : fpgh->name);
         if (err)
@@ -327,18 +362,18 @@ namespace vcpkg
     }
 
     ParseExpected<SourceControlFile> SourceControlFile::parse_control_file(
-        const fs::path& path_to_control, std::vector<Parse::Paragraph>&& control_paragraphs)
+        const std::string& origin, std::vector<Parse::Paragraph>&& control_paragraphs)
     {
         if (control_paragraphs.size() == 0)
         {
             auto ret = std::make_unique<Parse::ParseControlErrorInfo>();
-            ret->name = fs::u8string(path_to_control);
+            ret->name = origin;
             return ret;
         }
 
         auto control_file = std::make_unique<SourceControlFile>();
 
-        auto maybe_source = parse_source_paragraph(path_to_control, std::move(control_paragraphs.front()));
+        auto maybe_source = parse_source_paragraph(origin, std::move(control_paragraphs.front()));
         if (const auto source = maybe_source.get())
             control_file->core_paragraph = std::move(*source);
         else
@@ -348,14 +383,17 @@ namespace vcpkg
 
         for (auto&& feature_pgh : control_paragraphs)
         {
-            auto maybe_feature = parse_feature_paragraph(path_to_control, std::move(feature_pgh));
+            auto maybe_feature = parse_feature_paragraph(origin, std::move(feature_pgh));
             if (const auto feature = maybe_feature.get())
                 control_file->feature_paragraphs.emplace_back(std::move(*feature));
             else
                 return std::move(maybe_feature).error();
         }
 
-        canonicalize(*control_file);
+        if (auto maybe_error = canonicalize(*control_file))
+        {
+            return std::move(maybe_error);
+        }
         return control_file;
     }
 
@@ -854,7 +892,10 @@ namespace vcpkg
             r.optional_object_field(
                 obj, FEATURES, control_file->feature_paragraphs, FeaturesFieldDeserializer::instance);
 
-            canonicalize(*control_file);
+            if (auto maybe_error = canonicalize(*control_file))
+            {
+                Checks::exit_with_message(VCPKG_LINE_INFO, maybe_error->error);
+            }
             return std::move(control_file);
         }
 
@@ -879,8 +920,8 @@ namespace vcpkg
     constexpr StringLiteral ManifestDeserializer::DEFAULT_FEATURES;
     constexpr StringLiteral ManifestDeserializer::SUPPORTS;
 
-    Parse::ParseExpected<SourceControlFile> SourceControlFile::parse_manifest_file(const fs::path& path_to_manifest,
-                                                                                   const Json::Object& manifest)
+    Parse::ParseExpected<SourceControlFile> SourceControlFile::parse_manifest_object(const std::string& origin,
+                                                                                     const Json::Object& manifest)
     {
         Json::Reader reader;
 
@@ -889,7 +930,7 @@ namespace vcpkg
         if (!reader.errors().empty())
         {
             auto err = std::make_unique<ParseControlErrorInfo>();
-            err->name = fs::u8string(path_to_manifest);
+            err->name = origin;
             err->other_errors = std::move(reader.errors());
             return std::move(err);
         }
@@ -903,6 +944,12 @@ namespace vcpkg
         }
     }
 
+    Parse::ParseExpected<SourceControlFile> SourceControlFile::parse_manifest_file(const fs::path& path_to_manifest,
+                                                                                   const Json::Object& manifest)
+    {
+        return parse_manifest_object(fs::u8string(path_to_manifest), manifest);
+    }
+
     void print_error_message(Span<const std::unique_ptr<Parse::ParseControlErrorInfo>> error_info_list)
     {
         Checks::check_exit(VCPKG_LINE_INFO, error_info_list.size() > 0);
diff --git a/toolsrc/windows-bootstrap/vcpkg.vcxproj b/toolsrc/windows-bootstrap/vcpkg.vcxproj
index 259d1a330..5f897bc67 100644
--- a/toolsrc/windows-bootstrap/vcpkg.vcxproj
+++ b/toolsrc/windows-bootstrap/vcpkg.vcxproj
@@ -267,6 +267,7 @@
     <ClInclude Include="..\include\vcpkg\vcpkgcmdarguments.h" />

     <ClInclude Include="..\include\vcpkg\vcpkglib.h" />

     <ClInclude Include="..\include\vcpkg\vcpkgpaths.h" />

+    <ClInclude Include="..\include\vcpkg\versions.h" />

     <ClInclude Include="..\include\vcpkg\versiont.h" />

     <ClInclude Include="..\include\vcpkg\visualstudio.h" />

   </ItemGroup>