[vcpkg format-manifest] Add convert-control flag (#12471)

* [vcpkg format-manifest] initial convert-control attempt

TODO: manifest comments! we should keep $directives

* Finalize x-format-manifest

First, fix Json::parse -- "\c", for any c, was incorrectly parsed.
It would emit the escaped character, and then parse the character, so
that `\b` would give you { '\b', 'b' }.

Second, canonicalize source paragraphs as we're parsing them. This found
an error in qt5 -- The `declarative` feature was listed twice, and we
now catch it, so I removed the second paragraph.

Add PlatformExpression::complexity to allow ordering platform
expressions in a somewhat reasonable way.

Notes:
  - We allow `all_modules` as a feature name for back-compat with
    paraview
  - In order to actually convert CONTROL to vcpkg.json, we'd need to
    rename the qt5 `default` feature.
  - We need to add support for $directives in x-format-manifest

* fix qt5 port

* format

* fix compile

* fix tests for canonicalization

* Clean up code

* add error message for nothing to format

* add extra_info field

* add `const X&` overloads for `Object::insert[_or_replace]`

* fix compile

* simple CRs

* add tests

* format

* Fix mosquitto port file

also unmerge a line

* fail the tests on malformed manifest

* fix format_all

* fix coroutine port-version

* format manifests

10 files changed, 987 insertions, 107 deletions

diff --git a/toolsrc/src/vcpkg-test/json.cpp b/toolsrc/src/vcpkg-test/json.cpp
index 90a3537ab..0b9941861 100644
--- a/toolsrc/src/vcpkg-test/json.cpp
+++ b/toolsrc/src/vcpkg-test/json.cpp
@@ -80,6 +80,49 @@ TEST_CASE ("JSON parse strings", "[json]")
     REQUIRE(res.get()->first.string() == grin);
 }
 
+TEST_CASE ("JSON parse strings with escapes", "[json]")
+{
+    auto res = Json::parse(R"("\t")");
+    REQUIRE(res);
+    REQUIRE(res.get()->first.is_string());
+    REQUIRE(res.get()->first.string() == "\t");
+
+    res = Json::parse(R"("\\")");
+    REQUIRE(res);
+    REQUIRE(res.get()->first.is_string());
+    REQUIRE(res.get()->first.string() == "\\");
+
+    res = Json::parse(R"("\/")");
+    REQUIRE(res);
+    REQUIRE(res.get()->first.is_string());
+    REQUIRE(res.get()->first.string() == "/");
+
+    res = Json::parse(R"("\b")");
+    REQUIRE(res);
+    REQUIRE(res.get()->first.is_string());
+    REQUIRE(res.get()->first.string() == "\b");
+
+    res = Json::parse(R"("\f")");
+    REQUIRE(res);
+    REQUIRE(res.get()->first.is_string());
+    REQUIRE(res.get()->first.string() == "\f");
+
+    res = Json::parse(R"("\n")");
+    REQUIRE(res);
+    REQUIRE(res.get()->first.is_string());
+    REQUIRE(res.get()->first.string() == "\n");
+
+    res = Json::parse(R"("\r")");
+    REQUIRE(res);
+    REQUIRE(res.get()->first.is_string());
+    REQUIRE(res.get()->first.string() == "\r");
+
+    res = Json::parse(R"("This is a \"test\", hopefully it worked")");
+    REQUIRE(res);
+    REQUIRE(res.get()->first.is_string());
+    REQUIRE(res.get()->first.string() == R"(This is a "test", hopefully it worked)");
+}
+
 TEST_CASE ("JSON parse integers", "[json]")
 {
     auto res = Json::parse("0");
diff --git a/toolsrc/src/vcpkg-test/manifests.cpp b/toolsrc/src/vcpkg-test/manifests.cpp
index 108b2d0c5..9fc041a19 100644
--- a/toolsrc/src/vcpkg-test/manifests.cpp
+++ b/toolsrc/src/vcpkg-test/manifests.cpp
@@ -68,10 +68,10 @@ TEST_CASE ("manifest construct maximum", "[manifests]")
                 "name": "iroh",
                 "description": "zuko's uncle",
                 "dependencies": [
+                    "firebending",
                     {
                         "name": "tea"
                     },
-                    "firebending",
                     {
                         "name": "order.white-lotus",
                         "features": [ "the-ancient-ways" ],
@@ -105,18 +105,20 @@ TEST_CASE ("manifest construct maximum", "[manifests]")
     REQUIRE(pgh.feature_paragraphs[0]->description.size() == 1);
     REQUIRE(pgh.feature_paragraphs[0]->description[0] == "zuko's uncle");
     REQUIRE(pgh.feature_paragraphs[0]->dependencies.size() == 3);
-    REQUIRE(pgh.feature_paragraphs[0]->dependencies[0].name == "tea");
-    REQUIRE(pgh.feature_paragraphs[0]->dependencies[1].name == "firebending");
-    REQUIRE(pgh.feature_paragraphs[0]->dependencies[2].name == "order.white-lotus");
-    REQUIRE(pgh.feature_paragraphs[0]->dependencies[2].features.size() == 1);
-    REQUIRE(pgh.feature_paragraphs[0]->dependencies[2].features[0] == "the-ancient-ways");
-    REQUIRE_FALSE(pgh.feature_paragraphs[0]->dependencies[2].platform.evaluate(
+    REQUIRE(pgh.feature_paragraphs[0]->dependencies[0].name == "firebending");
+
+    REQUIRE(pgh.feature_paragraphs[0]->dependencies[1].name == "order.white-lotus");
+    REQUIRE(pgh.feature_paragraphs[0]->dependencies[1].features.size() == 1);
+    REQUIRE(pgh.feature_paragraphs[0]->dependencies[1].features[0] == "the-ancient-ways");
+    REQUIRE_FALSE(pgh.feature_paragraphs[0]->dependencies[1].platform.evaluate(
         {{"VCPKG_CMAKE_SYSTEM_NAME", ""}, {"VCPKG_TARGET_ARCHITECTURE", "arm"}}));
-    REQUIRE(pgh.feature_paragraphs[0]->dependencies[2].platform.evaluate(
+    REQUIRE(pgh.feature_paragraphs[0]->dependencies[1].platform.evaluate(
         {{"VCPKG_CMAKE_SYSTEM_NAME", ""}, {"VCPKG_TARGET_ARCHITECTURE", "x86"}}));
-    REQUIRE(pgh.feature_paragraphs[0]->dependencies[2].platform.evaluate(
+    REQUIRE(pgh.feature_paragraphs[0]->dependencies[1].platform.evaluate(
         {{"VCPKG_CMAKE_SYSTEM_NAME", "Linux"}, {"VCPKG_TARGET_ARCHITECTURE", "x86"}}));
 
+    REQUIRE(pgh.feature_paragraphs[0]->dependencies[2].name == "tea");
+
     REQUIRE(pgh.feature_paragraphs[1]->name == "zuko");
     REQUIRE(pgh.feature_paragraphs[1]->description.size() == 2);
     REQUIRE(pgh.feature_paragraphs[1]->description[0] == "son of the fire lord");
@@ -134,8 +136,8 @@ TEST_CASE ("SourceParagraph manifest two dependencies", "[manifests]")
     auto& pgh = **m_pgh.get();
 
     REQUIRE(pgh.core_paragraph->dependencies.size() == 2);
-    REQUIRE(pgh.core_paragraph->dependencies[0].name == "z");
-    REQUIRE(pgh.core_paragraph->dependencies[1].name == "openssl");
+    REQUIRE(pgh.core_paragraph->dependencies[0].name == "openssl");
+    REQUIRE(pgh.core_paragraph->dependencies[1].name == "z");
 }
 
 TEST_CASE ("SourceParagraph manifest three dependencies", "[manifests]")
@@ -149,9 +151,10 @@ TEST_CASE ("SourceParagraph manifest three dependencies", "[manifests]")
     auto& pgh = **m_pgh.get();
 
     REQUIRE(pgh.core_paragraph->dependencies.size() == 3);
-    REQUIRE(pgh.core_paragraph->dependencies[0].name == "z");
-    REQUIRE(pgh.core_paragraph->dependencies[1].name == "openssl");
-    REQUIRE(pgh.core_paragraph->dependencies[2].name == "xyz");
+    // should be ordered
+    REQUIRE(pgh.core_paragraph->dependencies[0].name == "openssl");
+    REQUIRE(pgh.core_paragraph->dependencies[1].name == "xyz");
+    REQUIRE(pgh.core_paragraph->dependencies[2].name == "z");
 }
 
 TEST_CASE ("SourceParagraph manifest construct qualified dependencies", "[manifests]")
@@ -239,3 +242,50 @@ TEST_CASE ("SourceParagraph manifest empty supports", "[manifests]")
                                      true);
     REQUIRE_FALSE(m_pgh.has_value());
 }
+
+TEST_CASE ("Serialize all the ports", "[manifests]")
+{
+    std::vector<std::string> args_list = {"x-format-manifest"};
+    auto& fs = Files::get_real_filesystem();
+    auto args = VcpkgCmdArguments::create_from_arg_sequence(args_list.data(), args_list.data() + args_list.size());
+    auto paths = VcpkgPaths{fs, args};
+
+    std::vector<SourceControlFile> scfs;
+
+    for (auto dir : fs::directory_iterator(paths.ports))
+    {
+        const auto control = dir / fs::u8path("CONTROL");
+        const auto manifest = dir / fs::u8path("vcpkg.json");
+        if (fs.exists(control))
+        {
+            auto contents = fs.read_contents(control, VCPKG_LINE_INFO);
+            auto pghs = Paragraphs::parse_paragraphs(contents, control.u8string());
+            REQUIRE(pghs);
+
+            scfs.push_back(std::move(
+                *SourceControlFile::parse_control_file(control, std::move(pghs).value_or_exit(VCPKG_LINE_INFO))
+                     .value_or_exit(VCPKG_LINE_INFO)));
+        }
+        else if (fs.exists(manifest))
+        {
+            std::error_code ec;
+            auto contents = Json::parse_file(fs, manifest, ec);
+            REQUIRE_FALSE(ec);
+            REQUIRE(contents);
+
+            auto scf = SourceControlFile::parse_manifest_file(manifest,
+                                                              contents.value_or_exit(VCPKG_LINE_INFO).first.object());
+            REQUIRE(scf);
+
+            scfs.push_back(std::move(*scf.value_or_exit(VCPKG_LINE_INFO)));
+        }
+    }
+
+    for (auto& scf : scfs)
+    {
+        auto serialized = serialize_manifest(scf);
+        auto serialized_scf = SourceControlFile::parse_manifest_file({}, serialized).value_or_exit(VCPKG_LINE_INFO);
+
+        REQUIRE(*serialized_scf == scf);
+    }
+}
diff --git a/toolsrc/src/vcpkg-test/paragraph.cpp b/toolsrc/src/vcpkg-test/paragraph.cpp
index 2638a3900..05ba8cfba 100644
--- a/toolsrc/src/vcpkg-test/paragraph.cpp
+++ b/toolsrc/src/vcpkg-test/paragraph.cpp
@@ -87,8 +87,9 @@ TEST_CASE ("SourceParagraph two dependencies", "[paragraph]")
     auto& pgh = **m_pgh.get();
 
     REQUIRE(pgh.core_paragraph->dependencies.size() == 2);
-    REQUIRE(pgh.core_paragraph->dependencies[0].name == "z");
-    REQUIRE(pgh.core_paragraph->dependencies[1].name == "openssl");
+    // should be ordered
+    REQUIRE(pgh.core_paragraph->dependencies[0].name == "openssl");
+    REQUIRE(pgh.core_paragraph->dependencies[1].name == "z");
 }
 
 TEST_CASE ("SourceParagraph three dependencies", "[paragraph]")
@@ -102,9 +103,10 @@ TEST_CASE ("SourceParagraph three dependencies", "[paragraph]")
     auto& pgh = **m_pgh.get();
 
     REQUIRE(pgh.core_paragraph->dependencies.size() == 3);
-    REQUIRE(pgh.core_paragraph->dependencies[0].name == "z");
-    REQUIRE(pgh.core_paragraph->dependencies[1].name == "openssl");
-    REQUIRE(pgh.core_paragraph->dependencies[2].name == "xyz");
+    // should be ordered
+    REQUIRE(pgh.core_paragraph->dependencies[0].name == "openssl");
+    REQUIRE(pgh.core_paragraph->dependencies[1].name == "xyz");
+    REQUIRE(pgh.core_paragraph->dependencies[2].name == "z");
 }
 
 TEST_CASE ("SourceParagraph construct qualified dependencies", "[paragraph]")
diff --git a/toolsrc/src/vcpkg/base/json.cpp b/toolsrc/src/vcpkg/base/json.cpp
index 75b917523..440bc6745 100644
--- a/toolsrc/src/vcpkg/base/json.cpp
+++ b/toolsrc/src/vcpkg/base/json.cpp
@@ -37,8 +37,11 @@ namespace vcpkg::Json
             ValueImpl(ValueKindConstant<VK::Integer> vk, int64_t i) : tag(vk), integer(i) { }
             ValueImpl(ValueKindConstant<VK::Number> vk, double d) : tag(vk), number(d) { }
             ValueImpl(ValueKindConstant<VK::String> vk, std::string&& s) : tag(vk), string(std::move(s)) { }
+            ValueImpl(ValueKindConstant<VK::String> vk, const std::string& s) : tag(vk), string(s) { }
             ValueImpl(ValueKindConstant<VK::Array> vk, Array&& arr) : tag(vk), array(std::move(arr)) { }
+            ValueImpl(ValueKindConstant<VK::Array> vk, const Array& arr) : tag(vk), array(arr) { }
             ValueImpl(ValueKindConstant<VK::Object> vk, Object&& obj) : tag(vk), object(std::move(obj)) { }
+            ValueImpl(ValueKindConstant<VK::Object> vk, const Object& obj) : tag(vk), object(obj) { }
 
             ValueImpl& operator=(ValueImpl&& other) noexcept
             {
@@ -171,23 +174,65 @@ namespace vcpkg::Json
     Value::Value() noexcept = default;
     Value::Value(Value&&) noexcept = default;
     Value& Value::operator=(Value&&) noexcept = default;
-    Value::~Value() = default;
 
-    Value Value::clone() const noexcept
+    Value::Value(const Value& other)
     {
-        switch (kind())
+        switch (other.kind())
         {
-            case ValueKind::Null: return Value::null(nullptr);
-            case ValueKind::Boolean: return Value::boolean(boolean());
-            case ValueKind::Integer: return Value::integer(integer());
-            case ValueKind::Number: return Value::number(number());
-            case ValueKind::String: return Value::string(string());
-            case ValueKind::Array: return Value::array(array().clone());
-            case ValueKind::Object: return Value::object(object().clone());
-            default: Checks::exit_fail(VCPKG_LINE_INFO);
+            case ValueKind::Null: return; // default construct underlying_
+            case ValueKind::Boolean:
+                underlying_.reset(new ValueImpl(ValueKindConstant<VK::Boolean>(), other.underlying_->boolean));
+                break;
+            case ValueKind::Integer:
+                underlying_.reset(new ValueImpl(ValueKindConstant<VK::Integer>(), other.underlying_->integer));
+                break;
+            case ValueKind::Number:
+                underlying_.reset(new ValueImpl(ValueKindConstant<VK::Number>(), other.underlying_->number));
+                break;
+            case ValueKind::String:
+                underlying_.reset(new ValueImpl(ValueKindConstant<VK::String>(), other.underlying_->string));
+                break;
+            case ValueKind::Array:
+                underlying_.reset(new ValueImpl(ValueKindConstant<VK::Array>(), other.underlying_->array));
+                break;
+            case ValueKind::Object:
+                underlying_.reset(new ValueImpl(ValueKindConstant<VK::Object>(), other.underlying_->object));
+                break;
+            default: Checks::unreachable(VCPKG_LINE_INFO);
         }
     }
 
+    Value& Value::operator=(const Value& other)
+    {
+        switch (other.kind())
+        {
+            case ValueKind::Null: underlying_.reset(); break;
+            case ValueKind::Boolean:
+                underlying_.reset(new ValueImpl(ValueKindConstant<VK::Boolean>(), other.underlying_->boolean));
+                break;
+            case ValueKind::Integer:
+                underlying_.reset(new ValueImpl(ValueKindConstant<VK::Integer>(), other.underlying_->integer));
+                break;
+            case ValueKind::Number:
+                underlying_.reset(new ValueImpl(ValueKindConstant<VK::Number>(), other.underlying_->number));
+                break;
+            case ValueKind::String:
+                underlying_.reset(new ValueImpl(ValueKindConstant<VK::String>(), other.underlying_->string));
+                break;
+            case ValueKind::Array:
+                underlying_.reset(new ValueImpl(ValueKindConstant<VK::Array>(), other.underlying_->array));
+                break;
+            case ValueKind::Object:
+                underlying_.reset(new ValueImpl(ValueKindConstant<VK::Object>(), other.underlying_->object));
+                break;
+            default: Checks::unreachable(VCPKG_LINE_INFO);
+        }
+
+        return *this;
+    }
+
+    Value::~Value() = default;
+
     Value Value::null(std::nullptr_t) noexcept { return Value(); }
     Value Value::boolean(bool b) noexcept
     {
@@ -225,25 +270,43 @@ namespace vcpkg::Json
         val.underlying_ = std::make_unique<ValueImpl>(ValueKindConstant<VK::Array>(), std::move(arr));
         return val;
     }
+    Value Value::array(const Array& arr) noexcept
+    {
+        Value val;
+        val.underlying_ = std::make_unique<ValueImpl>(ValueKindConstant<VK::Array>(), arr);
+        return val;
+    }
     Value Value::object(Object&& obj) noexcept
     {
         Value val;
         val.underlying_ = std::make_unique<ValueImpl>(ValueKindConstant<VK::Object>(), std::move(obj));
         return val;
     }
-    // } struct Value
-    // struct Array {
-    Array Array::clone() const noexcept
+    Value Value::object(const Object& obj) noexcept
+    {
+        Value val;
+        val.underlying_ = std::make_unique<ValueImpl>(ValueKindConstant<VK::Object>(), obj);
+        return val;
+    }
+
+    bool operator==(const Value& lhs, const Value& rhs)
     {
-        Array arr;
-        arr.underlying_.reserve(size());
-        for (const auto& el : *this)
+        if (lhs.kind() != rhs.kind()) return false;
+
+        switch (lhs.kind())
         {
-            arr.underlying_.push_back(el.clone());
+            case ValueKind::Null: return true;
+            case ValueKind::Boolean: return lhs.underlying_->boolean == rhs.underlying_->boolean;
+            case ValueKind::Integer: return lhs.underlying_->integer == rhs.underlying_->integer;
+            case ValueKind::Number: return lhs.underlying_->number == rhs.underlying_->number;
+            case ValueKind::String: return lhs.underlying_->string == rhs.underlying_->string;
+            case ValueKind::Array: return lhs.underlying_->string == rhs.underlying_->string;
+            case ValueKind::Object: return lhs.underlying_->string == rhs.underlying_->string;
+            default: Checks::unreachable(VCPKG_LINE_INFO);
         }
-        return arr;
     }
-
+    // } struct Value
+    // struct Array {
     Value& Array::push_back(Value&& value)
     {
         underlying_.push_back(std::move(value));
@@ -265,6 +328,7 @@ namespace vcpkg::Json
     {
         return insert_before(it, Value::array(std::move(arr))).array();
     }
+    bool operator==(const Array& lhs, const Array& rhs) { return lhs.underlying_ == rhs.underlying_; }
     // } struct Array
     // struct Object {
     Value& Object::insert(std::string key, Value&& value)
@@ -273,14 +337,29 @@ namespace vcpkg::Json
         underlying_.push_back({std::move(key), std::move(value)});
         return underlying_.back().second;
     }
+    Value& Object::insert(std::string key, const Value& value)
+    {
+        vcpkg::Checks::check_exit(VCPKG_LINE_INFO, !contains(key));
+        underlying_.push_back({std::move(key), value});
+        return underlying_.back().second;
+    }
     Array& Object::insert(std::string key, Array&& value)
     {
         return insert(std::move(key), Value::array(std::move(value))).array();
     }
+    Array& Object::insert(std::string key, const Array& value)
+    {
+        return insert(std::move(key), Value::array(value)).array();
+    }
     Object& Object::insert(std::string key, Object&& value)
     {
         return insert(std::move(key), Value::object(std::move(value))).object();
     }
+    Object& Object::insert(std::string key, const Object& value)
+    {
+        return insert(std::move(key), Value::object(value)).object();
+    }
+
     Value& Object::insert_or_replace(std::string key, Value&& value)
     {
         auto v = get(key);
@@ -295,14 +374,36 @@ namespace vcpkg::Json
             return underlying_.back().second;
         }
     }
+    Value& Object::insert_or_replace(std::string key, const Value& value)
+    {
+        auto v = get(key);
+        if (v)
+        {
+            *v = value;
+            return *v;
+        }
+        else
+        {
+            underlying_.push_back({std::move(key), std::move(value)});
+            return underlying_.back().second;
+        }
+    }
     Array& Object::insert_or_replace(std::string key, Array&& value)
     {
         return insert_or_replace(std::move(key), Value::array(std::move(value))).array();
     }
+    Array& Object::insert_or_replace(std::string key, const Array& value)
+    {
+        return insert_or_replace(std::move(key), Value::array(value)).array();
+    }
     Object& Object::insert_or_replace(std::string key, Object&& value)
     {
         return insert_or_replace(std::move(key), Value::object(std::move(value))).object();
     }
+    Object& Object::insert_or_replace(std::string key, const Object& value)
+    {
+        return insert_or_replace(std::move(key), Value::object(value)).object();
+    }
 
     auto Object::internal_find_key(StringView key) const noexcept -> underlying_t::const_iterator
     {
@@ -350,16 +451,14 @@ namespace vcpkg::Json
         }
     }
 
-    Object Object::clone() const noexcept
+    void Object::sort_keys()
     {
-        Object obj;
-        obj.underlying_.reserve(size());
-        for (const auto& el : *this)
-        {
-            obj.insert(el.first.to_string(), el.second.clone());
-        }
-        return obj;
+        std::sort(underlying_.begin(), underlying_.end(), [](const value_type& lhs, const value_type& rhs) {
+            return lhs.first < rhs.first;
+        });
     }
+
+    bool operator==(const Object& lhs, const Object& rhs) { return lhs.underlying_ == rhs.underlying_; }
     // } struct Object
 
     // auto parse() {
@@ -411,7 +510,7 @@ namespace vcpkg::Json
                 }
                 else
                 {
-                    vcpkg::Checks::exit_fail(VCPKG_LINE_INFO);
+                    vcpkg::Checks::unreachable(VCPKG_LINE_INFO);
                 }
             }
 
@@ -448,14 +547,14 @@ namespace vcpkg::Json
 
                 switch (current)
                 {
-                    case '"': return '"';
-                    case '\\': return '\\';
-                    case '/': return '/';
-                    case 'b': return '\b';
-                    case 'f': return '\f';
-                    case 'n': return '\n';
-                    case 'r': return '\r';
-                    case 't': return '\t';
+                    case '"': next(); return '"';
+                    case '\\': next(); return '\\';
+                    case '/': next(); return '/';
+                    case 'b': next(); return '\b';
+                    case 'f': next(); return '\f';
+                    case 'n': next(); return '\n';
+                    case 'r': next(); return '\r';
+                    case 't': next(); return '\t';
                     case 'u':
                     {
                         char16_t code_unit = 0;
@@ -657,7 +756,7 @@ namespace vcpkg::Json
                         rest = U"ull";
                         val = Value::null(nullptr);
                         break;
-                    default: vcpkg::Checks::exit_fail(VCPKG_LINE_INFO);
+                    default: vcpkg::Checks::unreachable(VCPKG_LINE_INFO);
                 }
 
                 for (const char32_t* rest_it = rest; *rest_it != '\0'; ++rest_it)
diff --git a/toolsrc/src/vcpkg/base/strings.cpp b/toolsrc/src/vcpkg/base/strings.cpp
index 44fc3ebd1..19b9384e6 100644
--- a/toolsrc/src/vcpkg/base/strings.cpp
+++ b/toolsrc/src/vcpkg/base/strings.cpp
@@ -146,6 +146,13 @@ std::string Strings::trim(std::string&& s)
     return std::move(s);
 }
 
+StringView Strings::trim(StringView sv)
+{
+    auto last = std::find_if_not(sv.rbegin(), sv.rend(), details::is_space).base();
+    auto first = std::find_if_not(sv.begin(), sv.end(), details::is_space);
+    return StringView(first, last);
+}
+
 void Strings::trim_all_and_remove_whitespace_strings(std::vector<std::string>* strings)
 {
     for (std::string& s : *strings)
diff --git a/toolsrc/src/vcpkg/commands.format-manifest.cpp b/toolsrc/src/vcpkg/commands.format-manifest.cpp
index d7bd46359..d0eced7ea 100644
--- a/toolsrc/src/vcpkg/commands.format-manifest.cpp
+++ b/toolsrc/src/vcpkg/commands.format-manifest.cpp
@@ -6,13 +6,181 @@
 #include <vcpkg/base/system.debug.h>
 
 #include <vcpkg/commands.format-manifest.h>
+#include <vcpkg/paragraphs.h>
 #include <vcpkg/portfileprovider.h>
+#include <vcpkg/sourceparagraph.h>
+
+namespace
+{
+    using namespace vcpkg;
+
+    struct ToWrite
+    {
+        SourceControlFile scf;
+        fs::path file_to_write;
+        fs::path original_path;
+        std::string original_source;
+    };
+
+    Optional<ToWrite> read_manifest(Files::Filesystem& fs, fs::path&& manifest_path)
+    {
+        auto path_string = manifest_path.u8string();
+        Debug::print("Reading ", path_string, "\n");
+        auto contents = fs.read_contents(manifest_path, VCPKG_LINE_INFO);
+        auto parsed_json_opt = Json::parse(contents, manifest_path);
+        if (!parsed_json_opt.has_value())
+        {
+            System::printf(
+                System::Color::error, "Failed to parse %s: %s\n", path_string, parsed_json_opt.error()->format());
+            return nullopt;
+        }
+
+        const auto& parsed_json = parsed_json_opt.value_or_exit(VCPKG_LINE_INFO).first;
+        if (!parsed_json.is_object())
+        {
+            System::printf(System::Color::error, "The file %s is not an object\n", path_string);
+            return nullopt;
+        }
+
+        auto scf = SourceControlFile::parse_manifest_file(manifest_path, parsed_json.object());
+        if (!scf.has_value())
+        {
+            System::printf(System::Color::error, "Failed to parse manifest file: %s\n", path_string);
+            print_error_message(scf.error());
+            return nullopt;
+        }
+
+        return ToWrite{
+            std::move(*scf.value_or_exit(VCPKG_LINE_INFO)),
+            manifest_path,
+            manifest_path,
+            std::move(contents),
+        };
+    }
+
+    Optional<ToWrite> read_control_file(Files::Filesystem& fs, fs::path&& control_path)
+    {
+        std::error_code ec;
+        auto control_path_string = control_path.u8string();
+        Debug::print("Reading ", control_path_string, "\n");
+
+        auto manifest_path = control_path.parent_path();
+        manifest_path /= fs::u8path("vcpkg.json");
+
+        auto contents = fs.read_contents(control_path, VCPKG_LINE_INFO);
+        auto paragraphs = Paragraphs::parse_paragraphs(contents, control_path_string);
+
+        if (!paragraphs)
+        {
+            System::printf(System::Color::error,
+                           "Failed to read paragraphs from %s: %s\n",
+                           control_path_string,
+                           paragraphs.error());
+            return {};
+        }
+        auto scf_res =
+            SourceControlFile::parse_control_file(control_path, std::move(paragraphs).value_or_exit(VCPKG_LINE_INFO));
+        if (!scf_res)
+        {
+            System::printf(System::Color::error, "Failed to parse control file: %s\n", control_path_string);
+            print_error_message(scf_res.error());
+            return {};
+        }
+
+        return ToWrite{
+            std::move(*scf_res.value_or_exit(VCPKG_LINE_INFO)),
+            manifest_path,
+            control_path,
+            std::move(contents),
+        };
+    }
+
+    void write_file(Files::Filesystem& fs, const ToWrite& data)
+    {
+        auto original_path_string = data.original_path.u8string();
+        auto file_to_write_string = data.file_to_write.u8string();
+        if (data.file_to_write == data.original_path)
+        {
+            Debug::print("Formatting ", file_to_write_string, "\n");
+        }
+        else
+        {
+            Debug::print("Converting ", file_to_write_string, " -> ", original_path_string, "\n");
+        }
+        auto res = serialize_manifest(data.scf);
+
+        auto check = SourceControlFile::parse_manifest_file(fs::path{}, res);
+        if (!check)
+        {
+            System::printf(System::Color::error,
+                           R"([correctness check] Failed to parse serialized manifest file of %s
+Please open an issue at https://github.com/microsoft/vcpkg, with the following output:
+Error:)",
+                           data.scf.core_paragraph->name);
+            print_error_message(check.error());
+            Checks::exit_with_message(VCPKG_LINE_INFO,
+                                      R"(
+=== Serialized manifest file ===
+%s
+)",
+                                      Json::stringify(res, {}));
+        }
+
+        auto check_scf = std::move(check).value_or_exit(VCPKG_LINE_INFO);
+        if (*check_scf != data.scf)
+        {
+            Checks::exit_with_message(
+                VCPKG_LINE_INFO,
+                R"([correctness check] The serialized manifest SCF was different from the original SCF.
+Please open an issue at https://github.com/microsoft/vcpkg, with the following output:
+
+=== Original File ===
+%s
+
+=== Serialized File ===
+%s
+
+=== Original SCF ===
+%s
+
+=== Serialized SCF ===
+%s
+)",
+                data.original_source,
+                Json::stringify(res, {}),
+                Json::stringify(serialize_debug_manifest(data.scf), {}),
+                Json::stringify(serialize_debug_manifest(*check_scf), {}));
+        }
+
+        // the manifest scf is correct
+        std::error_code ec;
+        fs.write_contents(data.file_to_write, Json::stringify(res, {}), ec);
+        if (ec)
+        {
+            Checks::exit_with_message(
+                VCPKG_LINE_INFO, "Failed to write manifest file %s: %s\n", file_to_write_string, ec.message());
+        }
+        if (data.original_path != data.file_to_write)
+        {
+            fs.remove(data.original_path, ec);
+            if (ec)
+            {
+                Checks::exit_with_message(
+                    VCPKG_LINE_INFO, "Failed to remove control file %s: %s\n", original_path_string, ec.message());
+            }
+        }
+    }
+}
 
 namespace vcpkg::Commands::FormatManifest
 {
     static constexpr StringLiteral OPTION_ALL = "all";
+    static constexpr StringLiteral OPTION_CONVERT_CONTROL = "convert-control";
 
-    const CommandSwitch FORMAT_SWITCHES[] = {{OPTION_ALL, "Format all ports' manifest files."}};
+    const CommandSwitch FORMAT_SWITCHES[] = {
+        {OPTION_ALL, "Format all ports' manifest files."},
+        {OPTION_CONVERT_CONTROL, "Convert CONTROL files to manifest files."},
+    };
 
     const CommandStructure COMMAND_STRUCTURE = {
         create_example_string(R"###(x-format-manifest --all)###"),
@@ -26,23 +194,35 @@ namespace vcpkg::Commands::FormatManifest
     {
         auto parsed_args = args.parse_arguments(COMMAND_STRUCTURE);
 
-        std::vector<fs::path> files_to_format;
-
         auto& fs = paths.get_filesystem();
         bool has_error = false;
 
-        if (Util::Sets::contains(parsed_args.switches, OPTION_ALL))
+        const bool format_all = Util::Sets::contains(parsed_args.switches, OPTION_ALL);
+        const bool convert_control = Util::Sets::contains(parsed_args.switches, OPTION_CONVERT_CONTROL);
+
+        if (!format_all && convert_control)
         {
-            for (const auto& dir : fs::directory_iterator(paths.ports))
-            {
-                auto manifest_path = dir.path() / fs::u8path("vcpkg.json");
-                if (fs.exists(manifest_path))
-                {
-                    files_to_format.push_back(std::move(manifest_path));
-                }
-            }
+            System::print2(System::Color::warning, R"(x-format-manifest was passed '--convert-control' without '--all'.
+    This doesn't do anything:
+    we will automatically convert all control files passed explicitly.)");
+        }
+
+        if (!format_all && args.command_arguments.empty())
+        {
+            Checks::exit_with_message(
+                VCPKG_LINE_INFO,
+                "No files to format; please pass either --all, or the explicit files to format or convert.");
         }
 
+        std::vector<ToWrite> to_write;
+
+        const auto add_file = [&to_write, &has_error](Optional<ToWrite>&& opt) {
+            if (auto t = opt.get())
+                to_write.push_back(std::move(*t));
+            else
+                has_error = true;
+        };
+
         for (const auto& arg : args.command_arguments)
         {
             auto path = fs::u8path(arg);
@@ -50,40 +230,47 @@ namespace vcpkg::Commands::FormatManifest
             {
                 path = paths.original_cwd / path;
             }
-            files_to_format.push_back(std::move(path));
-        }
-
-        for (const auto& path : files_to_format)
-        {
-            std::error_code ec;
-            Debug::print("Formatting ", path.u8string(), "\n");
-            auto parsed_json_opt = Json::parse_file(fs, path, ec);
-            if (ec)
-            {
-                System::printf(System::Color::error, "Failed to read %s: %s\n", path.u8string(), ec.message());
-                has_error = true;
-            }
 
-            if (auto pr = parsed_json_opt.get())
+            if (path.filename() == fs::u8path("CONTROL"))
             {
-                fs.write_contents(path, Json::stringify(pr->first, Json::JsonStyle{}), ec);
+                add_file(read_control_file(fs, std::move(path)));
             }
             else
             {
-                System::printf(System::Color::error,
-                               "Failed to parse %s: %s\n",
-                               path.u8string(),
-                               parsed_json_opt.error()->format());
-                has_error = true;
+                add_file(read_manifest(fs, std::move(path)));
             }
+        }
 
-            if (ec)
+        if (format_all)
+        {
+            for (const auto& dir : fs::directory_iterator(paths.ports))
             {
-                System::printf(System::Color::error, "Failed to write %s: %s\n", path.u8string(), ec.message());
-                has_error = true;
+                auto control_path = dir.path() / fs::u8path("CONTROL");
+                auto manifest_path = dir.path() / fs::u8path("vcpkg.json");
+                auto manifest_exists = fs.exists(manifest_path);
+                auto control_exists = fs.exists(control_path);
+
+                Checks::check_exit(VCPKG_LINE_INFO,
+                                   !manifest_exists || !control_exists,
+                                   "Both a manifest file and a CONTROL file exist in port directory: %s",
+                                   dir.path().u8string());
+
+                if (manifest_exists)
+                {
+                    add_file(read_manifest(fs, std::move(manifest_path)));
+                }
+                if (convert_control && control_exists)
+                {
+                    add_file(read_control_file(fs, std::move(control_path)));
+                }
             }
         }
 
+        for (auto const& el : to_write)
+        {
+            write_file(fs, el);
+        }
+
         if (has_error)
         {
             Checks::exit_fail(VCPKG_LINE_INFO);
diff --git a/toolsrc/src/vcpkg/metrics.cpp b/toolsrc/src/vcpkg/metrics.cpp
index 7ce4646e2..49c4f0d0c 100644
--- a/toolsrc/src/vcpkg/metrics.cpp
+++ b/toolsrc/src/vcpkg/metrics.cpp
@@ -180,11 +180,11 @@ namespace vcpkg::Metrics
 
         std::string format_event_data_template() const
         {
-            auto props_plus_buildtimes = properties.clone();
+            auto props_plus_buildtimes = properties;
             if (buildtime_names.size() > 0)
             {
-                props_plus_buildtimes.insert("buildnames_1", Json::Value::array(buildtime_names.clone()));
-                props_plus_buildtimes.insert("buildtimes", Json::Value::array(buildtime_times.clone()));
+                props_plus_buildtimes.insert("buildnames_1", buildtime_names);
+                props_plus_buildtimes.insert("buildtimes", buildtime_times);
             }
 
             Json::Array arr = Json::Array();
@@ -233,9 +233,9 @@ namespace vcpkg::Metrics
 
                 base_data.insert("ver", Json::Value::integer(2));
                 base_data.insert("name", Json::Value::string("commandline_test7"));
-                base_data.insert("properties", Json::Value::object(std::move(props_plus_buildtimes)));
-                base_data.insert("measurements", Json::Value::object(measurements.clone()));
-                base_data.insert("feature-flags", Json::Value::object(feature_flags.clone()));
+                base_data.insert("properties", std::move(props_plus_buildtimes));
+                base_data.insert("measurements", measurements);
+                base_data.insert("feature-flags", feature_flags);
             }
 
             return Json::stringify(arr, vcpkg::Json::JsonStyle());
diff --git a/toolsrc/src/vcpkg/packagespec.cpp b/toolsrc/src/vcpkg/packagespec.cpp
index d4c293b46..c9b6091b8 100644
--- a/toolsrc/src/vcpkg/packagespec.cpp
+++ b/toolsrc/src/vcpkg/packagespec.cpp
@@ -93,8 +93,6 @@ namespace vcpkg
         return left.name() == right.name() && left.triplet() == right.triplet();
     }
 
-    bool operator!=(const PackageSpec& left, const PackageSpec& right) { return !(left == right); }
-
     ExpectedS<Features> Features::from_string(const std::string& name)
     {
         return parse_qualified_specifier(name).then([&](ParsedQualifiedSpecifier&& pqs) -> ExpectedS<Features> {
@@ -262,4 +260,15 @@ namespace vcpkg
         parser.skip_tabs_spaces();
         return ret;
     }
+
+    bool operator==(const Dependency& lhs, const Dependency& rhs)
+    {
+        if (lhs.name != rhs.name) return false;
+        if (lhs.features != rhs.features) return false;
+        if (!structurally_equal(lhs.platform, rhs.platform)) return false;
+        if (lhs.extra_info != rhs.extra_info) return false;
+
+        return true;
+    }
+    bool operator!=(const Dependency& lhs, const Dependency& rhs);
 }
diff --git a/toolsrc/src/vcpkg/platform-expression.cpp b/toolsrc/src/vcpkg/platform-expression.cpp
index fadb548c1..03bc81367 100644
--- a/toolsrc/src/vcpkg/platform-expression.cpp
+++ b/toolsrc/src/vcpkg/platform-expression.cpp
@@ -7,6 +7,7 @@
 
 #include <vcpkg/platform-expression.h>
 
+#include <numeric>
 #include <string>
 #include <vector>
 
@@ -415,6 +416,28 @@ namespace vcpkg::PlatformExpression
         return Visitor{context, override_ctxt}.visit(*this->underlying_);
     }
 
+    int Expr::complexity() const
+    {
+        if (is_empty()) return 0;
+
+        struct Impl
+        {
+            int operator()(const std::unique_ptr<detail::ExprImpl>& expr) const { return (*this)(*expr); }
+            int operator()(const detail::ExprImpl& expr) const
+            {
+                if (expr.kind == ExprKind::identifier) return 1;
+
+                if (expr.kind == ExprKind::op_not) return 1 + (*this)(expr.exprs.at(0));
+
+                return 1 + std::accumulate(expr.exprs.begin(), expr.exprs.end(), 0, [](int acc, const auto& el) {
+                           return acc + Impl{}(el);
+                       });
+            }
+        };
+
+        return Impl{}(underlying_);
+    }
+
     ExpectedS<Expr> parse_platform_expression(StringView expression, MultipleBinaryOperators multiple_binary_operators)
     {
         auto parser = ExpressionParser(expression, multiple_binary_operators);
@@ -429,4 +452,100 @@ namespace vcpkg::PlatformExpression
             return res;
         }
     }
+
+    bool structurally_equal(const Expr& lhs, const Expr& rhs)
+    {
+        struct Impl
+        {
+            bool operator()(const std::unique_ptr<detail::ExprImpl>& lhs,
+                            const std::unique_ptr<detail::ExprImpl>& rhs) const
+            {
+                return (*this)(*lhs, *rhs);
+            }
+            bool operator()(const detail::ExprImpl& lhs, const detail::ExprImpl& rhs) const
+            {
+                if (lhs.kind != rhs.kind) return false;
+
+                if (lhs.kind == ExprKind::identifier)
+                {
+                    return lhs.identifier == rhs.identifier;
+                }
+                else
+                {
+                    const auto& exprs_l = lhs.exprs;
+                    const auto& exprs_r = rhs.exprs;
+                    return std::equal(exprs_l.begin(), exprs_l.end(), exprs_r.begin(), exprs_r.end(), *this);
+                }
+            }
+        };
+
+        if (lhs.is_empty())
+        {
+            return rhs.is_empty();
+        }
+        if (rhs.is_empty())
+        {
+            return false;
+        }
+        return Impl{}(lhs.underlying_, rhs.underlying_);
+    }
+
+    int compare(const Expr& lhs, const Expr& rhs)
+    {
+        auto lhs_platform_complexity = lhs.complexity();
+        auto rhs_platform_complexity = lhs.complexity();
+
+        if (lhs_platform_complexity < rhs_platform_complexity) return -1;
+        if (rhs_platform_complexity < lhs_platform_complexity) return 1;
+
+        auto lhs_platform = to_string(lhs);
+        auto rhs_platform = to_string(rhs);
+
+        if (lhs_platform.size() < rhs_platform.size()) return -1;
+        if (rhs_platform.size() < lhs_platform.size()) return 1;
+
+        auto platform_cmp = lhs_platform.compare(rhs_platform);
+        if (platform_cmp < 0) return -1;
+        if (platform_cmp > 0) return 1;
+
+        return 0;
+    }
+
+    std::string to_string(const Expr& expr)
+    {
+        struct Impl
+        {
+            std::string operator()(const std::unique_ptr<detail::ExprImpl>& expr) const
+            {
+                return (*this)(*expr, false);
+            }
+            std::string operator()(const detail::ExprImpl& expr, bool outer) const
+            {
+                const char* join = nullptr;
+                switch (expr.kind)
+                {
+                    case ExprKind::identifier: return expr.identifier;
+                    case ExprKind::op_and: join = " & "; break;
+                    case ExprKind::op_or: join = " | "; break;
+                    case ExprKind::op_not: return Strings::format("!%s", (*this)(expr.exprs.at(0)));
+                    default: Checks::unreachable(VCPKG_LINE_INFO);
+                }
+
+                if (outer)
+                {
+                    return Strings::join(join, expr.exprs, *this);
+                }
+                else
+                {
+                    return Strings::format("(%s)", Strings::join(join, expr.exprs, *this));
+                }
+            }
+        };
+
+        if (expr.is_empty())
+        {
+            return std::string{};
+        }
+        return Impl{}(*expr.underlying_, true);
+    }
 }
diff --git a/toolsrc/src/vcpkg/sourceparagraph.cpp b/toolsrc/src/vcpkg/sourceparagraph.cpp
index 8c86f1677..8b16d0846 100644
--- a/toolsrc/src/vcpkg/sourceparagraph.cpp
+++ b/toolsrc/src/vcpkg/sourceparagraph.cpp
@@ -16,6 +16,57 @@ namespace vcpkg
 {
     using namespace vcpkg::Parse;
 
+    template<class Lhs, class Rhs>
+    static bool paragraph_equal(const Lhs& lhs, const Rhs& rhs)
+    {
+        return std::equal(
+            lhs.begin(), lhs.end(), rhs.begin(), rhs.end(), [](const std::string& lhs, const std::string& rhs) {
+                return Strings::trim(StringView(lhs)) == Strings::trim(StringView(rhs));
+            });
+    }
+
+    bool operator==(const SourceParagraph& lhs, const SourceParagraph& rhs)
+    {
+        if (lhs.name != rhs.name) return false;
+        if (lhs.version != rhs.version) return false;
+        if (lhs.port_version != rhs.port_version) return false;
+        if (!paragraph_equal(lhs.description, rhs.description)) return false;
+        if (!paragraph_equal(lhs.maintainers, rhs.maintainers)) return false;
+        if (lhs.homepage != rhs.homepage) return false;
+        if (lhs.documentation != rhs.documentation) return false;
+        if (lhs.dependencies != rhs.dependencies) return false;
+        if (lhs.default_features != rhs.default_features) return false;
+        if (lhs.license != rhs.license) return false;
+
+        if (lhs.type != rhs.type) return false;
+        if (!structurally_equal(lhs.supports_expression, rhs.supports_expression)) return false;
+
+        if (lhs.extra_info != rhs.extra_info) return false;
+
+        return true;
+    }
+
+    bool operator==(const FeatureParagraph& lhs, const FeatureParagraph& rhs)
+    {
+        if (lhs.name != rhs.name) return false;
+        if (lhs.dependencies != rhs.dependencies) return false;
+        if (!paragraph_equal(lhs.description, rhs.description)) return false;
+        if (lhs.extra_info != rhs.extra_info) return false;
+
+        return true;
+    }
+
+    bool operator==(const SourceControlFile& lhs, const SourceControlFile& rhs)
+    {
+        if (*lhs.core_paragraph != *rhs.core_paragraph) return false;
+        return std::equal(lhs.feature_paragraphs.begin(),
+                          lhs.feature_paragraphs.end(),
+                          rhs.feature_paragraphs.begin(),
+                          rhs.feature_paragraphs.end(),
+                          [](const std::unique_ptr<FeatureParagraph>& lhs,
+                             const std::unique_ptr<FeatureParagraph>& rhs) { return *lhs == *rhs; });
+    }
+
     namespace SourceParagraphFields
     {
         static const std::string BUILD_DEPENDS = "Build-Depends";
@@ -197,6 +248,117 @@ namespace vcpkg
         }
     }
 
+    namespace
+    {
+        constexpr static struct Canonicalize
+        {
+            struct FeatureLess
+            {
+                bool operator()(const std::unique_ptr<FeatureParagraph>& lhs,
+                                const std::unique_ptr<FeatureParagraph>& rhs) const
+                {
+                    return (*this)(*lhs, *rhs);
+                }
+                bool operator()(const FeatureParagraph& lhs, const FeatureParagraph& rhs) const
+                {
+                    return lhs.name < rhs.name;
+                }
+            };
+            struct FeatureEqual
+            {
+                bool operator()(const std::unique_ptr<FeatureParagraph>& lhs,
+                                const std::unique_ptr<FeatureParagraph>& rhs) const
+                {
+                    return (*this)(*lhs, *rhs);
+                }
+                bool operator()(const FeatureParagraph& lhs, const FeatureParagraph& rhs) const
+                {
+                    return lhs.name == rhs.name;
+                }
+            };
+
+            // assume canonicalized feature list
+            struct DependencyLess
+            {
+                bool operator()(const std::unique_ptr<Dependency>& lhs, const std::unique_ptr<Dependency>& rhs) const
+                {
+                    return (*this)(*lhs, *rhs);
+                }
+                bool operator()(const Dependency& lhs, const Dependency& rhs) const
+                {
+                    auto cmp = lhs.name.compare(rhs.name);
+                    if (cmp < 0) return true;
+                    if (cmp > 0) return false;
+
+                    // same dependency name
+
+                    // order by platform string:
+                    auto platform_cmp = compare(lhs.platform, rhs.platform);
+                    if (platform_cmp < 0) return true;
+                    if (platform_cmp > 0) return false;
+
+                    // then order by features
+                    // smaller list first, then lexicographical
+                    if (lhs.features.size() < rhs.features.size()) return true;
+                    if (rhs.features.size() < lhs.features.size()) return false;
+
+                    // then finally order by feature list
+                    if (std::lexicographical_compare(
+                            lhs.features.begin(), lhs.features.end(), rhs.features.begin(), rhs.features.end()))
+                    {
+                        return true;
+                    }
+                    return false;
+                }
+            };
+
+            template<class T>
+            void operator()(std::unique_ptr<T>& ptr) const
+            {
+                (*this)(*ptr);
+            }
+
+            void operator()(Dependency& dep) const
+            {
+                std::sort(dep.features.begin(), dep.features.end());
+                dep.extra_info.sort_keys();
+            }
+            void operator()(SourceParagraph& spgh) const
+            {
+                std::for_each(spgh.dependencies.begin(), spgh.dependencies.end(), *this);
+                std::sort(spgh.dependencies.begin(), spgh.dependencies.end(), DependencyLess{});
+
+                std::sort(spgh.default_features.begin(), spgh.default_features.end());
+
+                spgh.extra_info.sort_keys();
+            }
+            void operator()(FeatureParagraph& fpgh) const
+            {
+                std::for_each(fpgh.dependencies.begin(), fpgh.dependencies.end(), *this);
+                std::sort(fpgh.dependencies.begin(), fpgh.dependencies.end(), DependencyLess{});
+
+                fpgh.extra_info.sort_keys();
+            }
+            void operator()(SourceControlFile& scf) const
+            {
+                (*this)(*scf.core_paragraph);
+                std::for_each(scf.feature_paragraphs.begin(), scf.feature_paragraphs.end(), *this);
+                std::sort(scf.feature_paragraphs.begin(), scf.feature_paragraphs.end(), FeatureLess{});
+
+                auto adjacent_equal =
+                    std::adjacent_find(scf.feature_paragraphs.begin(), scf.feature_paragraphs.end(), FeatureEqual{});
+                if (adjacent_equal != scf.feature_paragraphs.end())
+                {
+                    Checks::exit_with_message(VCPKG_LINE_INFO,
+                                              R"(Multiple features with the same name for port %s: %s
+    This is invalid; please make certain that features have distinct names.)",
+                                              scf.core_paragraph->name,
+                                              (*adjacent_equal)->name);
+                }
+            }
+        } canonicalize{};
+    }
+
     static ParseExpected<SourceParagraph> parse_source_paragraph(const fs::path& path_to_control, Paragraph&& fields)
     {
         auto origin = path_to_control.u8string();
@@ -311,6 +473,7 @@ namespace vcpkg
                 return std::move(maybe_feature).error();
         }
 
+        canonicalize(*control_file);
         return control_file;
     }
 
@@ -351,6 +514,21 @@ namespace vcpkg
         StringView type_name_;
     };
 
+    struct NaturalNumberField : Json::VisitorCrtpBase<NaturalNumberField>
+    {
+        using type = int;
+        StringView type_name() { return "a natural number"; }
+
+        Optional<int> visit_integer(Json::Reader&, StringView, int64_t value)
+        {
+            if (value > std::numeric_limits<int>::max() || value < 0)
+            {
+                return nullopt;
+            }
+            return static_cast<int>(value);
+        }
+    };
+
     struct BooleanField : Json::VisitorCrtpBase<BooleanField>
     {
         using type = bool;
@@ -424,6 +602,12 @@ namespace vcpkg
             // strings with uppercase letters from the basic check
             static const std::regex RESERVED = std::regex(R"(prn|aux|nul|con|(lpt|com)[1-9]|core|default)");
 
+            // back-compat
+            if (sv == "all_modules")
+            {
+                return true;
+            }
+
             if (!std::regex_match(sv.begin(), sv.end(), BASIC_IDENTIFIER))
             {
                 return false; // we're not even in the shape of an identifier
@@ -651,10 +835,10 @@ namespace vcpkg
         using type = Dependency;
         StringView type_name() { return "a dependency"; }
 
-        constexpr static StringView NAME = "name";
-        constexpr static StringView FEATURES = "features";
-        constexpr static StringView DEFAULT_FEATURES = "default-features";
-        constexpr static StringView PLATFORM = "platform";
+        constexpr static StringLiteral NAME = "name";
+        constexpr static StringLiteral FEATURES = "features";
+        constexpr static StringLiteral DEFAULT_FEATURES = "default-features";
+        constexpr static StringLiteral PLATFORM = "platform";
         constexpr static StringView KNOWN_FIELDS[] = {NAME, FEATURES, DEFAULT_FEATURES, PLATFORM};
 
         Optional<Dependency> visit_string(Json::Reader&, StringView, StringView sv)
@@ -680,6 +864,15 @@ namespace vcpkg
             }
 
             Dependency dep;
+
+            for (const auto& el : obj)
+            {
+                if (Strings::starts_with(el.first, "$"))
+                {
+                    dep.extra_info.insert_or_replace(el.first.to_string(), el.second);
+                }
+            }
+
             r.required_object_field(type_name(), obj, NAME, dep.name, PackageNameField{});
             r.optional_object_field(
                 obj, FEATURES, dep.features, ArrayField<IdentifierField>{"an array of identifiers", AllowEmpty::Yes});
@@ -702,14 +895,31 @@ namespace vcpkg
         using type = std::unique_ptr<FeatureParagraph>;
         StringView type_name() { return "a feature"; }
 
-        constexpr static StringView NAME = "name";
-        constexpr static StringView DESCRIPTION = "description";
-        constexpr static StringView DEPENDENCIES = "dependencies";
+        constexpr static StringLiteral NAME = "name";
+        constexpr static StringLiteral DESCRIPTION = "description";
+        constexpr static StringLiteral DEPENDENCIES = "dependencies";
+        constexpr static StringView KNOWN_FIELDS[] = {NAME, DESCRIPTION, DEPENDENCIES};
 
         Optional<std::unique_ptr<FeatureParagraph>> visit_object(Json::Reader& r, StringView, const Json::Object& obj)
         {
+            {
+                auto extra_fields = invalid_json_fields(obj, KNOWN_FIELDS);
+                if (!extra_fields.empty())
+                {
+                    r.error().add_extra_fields(type_name().to_string(), std::move(extra_fields));
+                }
+            }
+
             auto feature = std::make_unique<FeatureParagraph>();
 
+            for (const auto& el : obj)
+            {
+                if (Strings::starts_with(el.first, "$"))
+                {
+                    feature->extra_info.insert_or_replace(el.first.to_string(), el.second);
+                }
+            }
+
             r.required_object_field(type_name(), obj, NAME, feature->name, IdentifierField{});
             r.required_object_field(type_name(), obj, DESCRIPTION, feature->description, ParagraphField{});
             r.optional_object_field(obj,
@@ -768,11 +978,21 @@ namespace vcpkg
         control_file->core_paragraph = std::make_unique<SourceParagraph>();
 
         auto& spgh = control_file->core_paragraph;
+        spgh->type = Type{Type::PORT};
+
+        for (const auto& el : manifest)
+        {
+            if (Strings::starts_with(el.first, "$"))
+            {
+                spgh->extra_info.insert_or_replace(el.first.to_string(), el.second);
+            }
+        }
 
         constexpr static StringView type_name = "vcpkg.json";
         visit.required_object_field(type_name, manifest, ManifestFields::NAME, spgh->name, IdentifierField{});
         visit.required_object_field(
             type_name, manifest, ManifestFields::VERSION, spgh->version, StringField{"a version"});
+        visit.optional_object_field(manifest, ManifestFields::PORT_VERSION, spgh->port_version, NaturalNumberField{});
         visit.optional_object_field(manifest, ManifestFields::MAINTAINERS, spgh->maintainers, ParagraphField{});
         visit.optional_object_field(manifest, ManifestFields::DESCRIPTION, spgh->description, ParagraphField{});
         visit.optional_object_field(manifest, ManifestFields::HOMEPAGE, spgh->homepage, StringField{"a url"});
@@ -806,6 +1026,7 @@ namespace vcpkg
             return std::make_unique<ParseControlErrorInfo>(std::move(err.pcei));
         }
 
+        canonicalize(*control_file);
         return std::move(control_file);
     }
 
@@ -845,4 +1066,147 @@ namespace vcpkg
         }
         return ret;
     }
+
+    static Json::Object serialize_manifest_impl(const SourceControlFile& scf, bool debug)
+    {
+        auto serialize_paragraph =
+            [&](Json::Object& obj, StringLiteral name, const std::vector<std::string>& pgh, bool always = false) {
+                if (!debug)
+                {
+                    if (pgh.empty())
+                    {
+                        if (always)
+                        {
+                            obj.insert(name, Json::Array());
+                        }
+                        return;
+                    }
+                    if (pgh.size() == 1)
+                    {
+                        obj.insert(name, Json::Value::string(pgh.front()));
+                        return;
+                    }
+                }
+
+                auto& arr = obj.insert(name, Json::Array());
+                for (const auto& s : pgh)
+                {
+                    arr.push_back(Json::Value::string(s));
+                }
+            };
+        auto serialize_optional_array =
+            [&](Json::Object& obj, StringLiteral name, const std::vector<std::string>& pgh) {
+                if (pgh.empty() && !debug) return;
+
+                auto& arr = obj.insert(name, Json::Array());
+                for (const auto& s : pgh)
+                {
+                    arr.push_back(Json::Value::string(s));
+                }
+            };
+        auto serialize_optional_string = [&](Json::Object& obj, StringLiteral name, const std::string& s) {
+            if (!s.empty() || debug)
+            {
+                obj.insert(name, Json::Value::string(s));
+            }
+        };
+        auto serialize_dependency = [&](Json::Array& arr, const Dependency& dep) {
+            if (dep.features.empty() && dep.platform.is_empty() && dep.extra_info.is_empty())
+            {
+                arr.push_back(Json::Value::string(dep.name));
+            }
+            else
+            {
+                auto& dep_obj = arr.push_back(Json::Object());
+                for (const auto& el : dep.extra_info)
+                {
+                    dep_obj.insert(el.first.to_string(), el.second);
+                }
+
+                dep_obj.insert(DependencyField::NAME, Json::Value::string(dep.name));
+
+                auto features_copy = dep.features;
+                auto core_it = std::find(features_copy.begin(), features_copy.end(), "core");
+                if (core_it != features_copy.end())
+                {
+                    dep_obj.insert(DependencyField::DEFAULT_FEATURES, Json::Value::boolean(false));
+                    features_copy.erase(core_it);
+                }
+
+                serialize_optional_array(dep_obj, DependencyField::FEATURES, features_copy);
+                serialize_optional_string(dep_obj, DependencyField::PLATFORM, to_string(dep.platform));
+            }
+        };
+
+        Json::Object obj;
+
+        for (const auto& el : scf.core_paragraph->extra_info)
+        {
+            obj.insert(el.first.to_string(), el.second);
+        }
+
+        obj.insert(ManifestFields::NAME, Json::Value::string(scf.core_paragraph->name));
+        obj.insert(ManifestFields::VERSION, Json::Value::string(scf.core_paragraph->version));
+
+        if (scf.core_paragraph->port_version != 0 || debug)
+        {
+            obj.insert(ManifestFields::PORT_VERSION, Json::Value::integer(scf.core_paragraph->port_version));
+        }
+
+        serialize_paragraph(obj, ManifestFields::MAINTAINERS, scf.core_paragraph->maintainers);
+        serialize_paragraph(obj, ManifestFields::DESCRIPTION, scf.core_paragraph->description);
+
+        serialize_optional_string(obj, ManifestFields::HOMEPAGE, scf.core_paragraph->homepage);
+        serialize_optional_string(obj, ManifestFields::DOCUMENTATION, scf.core_paragraph->documentation);
+        serialize_optional_string(obj, ManifestFields::LICENSE, scf.core_paragraph->license);
+        serialize_optional_string(obj, ManifestFields::SUPPORTS, to_string(scf.core_paragraph->supports_expression));
+
+        if (!scf.core_paragraph->dependencies.empty() || debug)
+        {
+            auto& deps = obj.insert(ManifestFields::DEPENDENCIES, Json::Array());
+
+            for (const auto& dep : scf.core_paragraph->dependencies)
+            {
+                serialize_dependency(deps, dep);
+            }
+        }
+
+        serialize_optional_array(obj, ManifestFields::DEFAULT_FEATURES, scf.core_paragraph->default_features);
+
+        if (!scf.feature_paragraphs.empty() || debug)
+        {
+            auto& arr = obj.insert(ManifestFields::FEATURES, Json::Array());
+            for (const auto& feature : scf.feature_paragraphs)
+            {
+                auto& feature_obj = arr.push_back(Json::Object());
+                for (const auto& el : feature->extra_info)
+                {
+                    feature_obj.insert(el.first.to_string(), el.second);
+                }
+
+                feature_obj.insert(FeatureField::NAME, Json::Value::string(feature->name));
+                serialize_paragraph(feature_obj, FeatureField::DESCRIPTION, feature->description, true);
+
+                if (!feature->dependencies.empty() || debug)
+                {
+                    auto& deps = feature_obj.insert(FeatureField::DEPENDENCIES, Json::Array());
+                    for (const auto& dep : feature->dependencies)
+                    {
+                        serialize_dependency(deps, dep);
+                    }
+                }
+            }
+        }
+
+        if (debug)
+        {
+            obj.insert("TYPE", Json::Value::string(Type::to_string(scf.core_paragraph->type)));
+        }
+
+        return obj;
+    }
+
+    Json::Object serialize_debug_manifest(const SourceControlFile& scf) { return serialize_manifest_impl(scf, true); }
+
+    Json::Object serialize_manifest(const SourceControlFile& scf) { return serialize_manifest_impl(scf, false); }
 }