[vcpkg] Registries MVP (#13038)

18 files changed, 642 insertions, 90 deletions

diff --git a/toolsrc/include/vcpkg/base/files.h b/toolsrc/include/vcpkg/base/files.h
index e0a555024..08bbe7906 100644
--- a/toolsrc/include/vcpkg/base/files.h
+++ b/toolsrc/include/vcpkg/base/files.h
@@ -157,7 +157,7 @@ namespace vcpkg::Files
         /// <summary>Read text lines from a file</summary>
         /// <remarks>Lines will have up to one trailing carriage-return character stripped (CRLF)</remarks>
         virtual Expected<std::vector<std::string>> read_lines(const fs::path& file_path) const = 0;
-        virtual fs::path find_file_recursively_up(const fs::path& starting_dir, const std::string& filename) const = 0;
+        virtual fs::path find_file_recursively_up(const fs::path& starting_dir, const fs::path& filename) const = 0;
         virtual std::vector<fs::path> get_files_recursive(const fs::path& dir) const = 0;
         virtual std::vector<fs::path> get_files_non_recursive(const fs::path& dir) const = 0;
         void write_lines(const fs::path& file_path, const std::vector<std::string>& lines, LineInfo linfo);
diff --git a/toolsrc/include/vcpkg/base/fwd/json.h b/toolsrc/include/vcpkg/base/fwd/json.h
new file mode 100644
index 000000000..7071103c1
--- /dev/null
+++ b/toolsrc/include/vcpkg/base/fwd/json.h
@@ -0,0 +1,44 @@
+#pragma once
+
+#include <vcpkg/base/fwd/optional.h>
+#include <vcpkg/base/fwd/span.h>
+#include <vcpkg/base/fwd/stringview.h>
+
+namespace vcpkg::Json
+{
+    struct JsonStyle;
+    enum class ValueKind : int;
+    struct Value;
+    struct Object;
+    struct Array;
+
+    struct ReaderError;
+    struct BasicReaderError;
+    struct Reader;
+
+    // This is written all the way out so that one can include a subclass in a header
+    template<class Type>
+    struct IDeserializer
+    {
+        using type = Type;
+        virtual StringView type_name() const = 0;
+
+        virtual Span<const StringView> valid_fields() const;
+
+        virtual Optional<Type> visit_null(Reader&, StringView);
+        virtual Optional<Type> visit_boolean(Reader&, StringView, bool);
+        virtual Optional<Type> visit_integer(Reader& r, StringView field_name, int64_t i);
+        virtual Optional<Type> visit_number(Reader&, StringView, double);
+        virtual Optional<Type> visit_string(Reader&, StringView, StringView);
+        virtual Optional<Type> visit_array(Reader&, StringView, const Array&);
+        virtual Optional<Type> visit_object(Reader&, StringView, const Object&);
+
+    protected:
+        IDeserializer() = default;
+        IDeserializer(const IDeserializer&) = default;
+        IDeserializer& operator=(const IDeserializer&) = default;
+        IDeserializer(IDeserializer&&) = default;
+        IDeserializer& operator=(IDeserializer&&) = default;
+        virtual ~IDeserializer() = default;
+    };
+}
diff --git a/toolsrc/include/vcpkg/base/fwd/optional.h b/toolsrc/include/vcpkg/base/fwd/optional.h
new file mode 100644
index 000000000..2c3a7ea6c
--- /dev/null
+++ b/toolsrc/include/vcpkg/base/fwd/optional.h
@@ -0,0 +1,7 @@
+#pragma once
+
+namespace vcpkg
+{
+    template<class T>
+    struct Optional;
+}
diff --git a/toolsrc/include/vcpkg/base/fwd/span.h b/toolsrc/include/vcpkg/base/fwd/span.h
new file mode 100644
index 000000000..af78aa642
--- /dev/null
+++ b/toolsrc/include/vcpkg/base/fwd/span.h
@@ -0,0 +1,7 @@
+#pragma once
+
+namespace vcpkg
+{
+    template<class T>
+    struct Span;
+}
diff --git a/toolsrc/include/vcpkg/base/fwd/stringview.h b/toolsrc/include/vcpkg/base/fwd/stringview.h
new file mode 100644
index 000000000..e297fd43b
--- /dev/null
+++ b/toolsrc/include/vcpkg/base/fwd/stringview.h
@@ -0,0 +1,6 @@
+#pragma once
+
+namespace vcpkg
+{
+    struct StringView;
+}
diff --git a/toolsrc/include/vcpkg/base/json.h b/toolsrc/include/vcpkg/base/json.h
index a4ad44ebd..3539f2df4 100644
--- a/toolsrc/include/vcpkg/base/json.h
+++ b/toolsrc/include/vcpkg/base/json.h
@@ -1,5 +1,7 @@
 #pragma once
 
+#include <vcpkg/base/fwd/json.h>
+
 #include <vcpkg/base/expected.h>
 #include <vcpkg/base/files.h>
 #include <vcpkg/base/parse.h>
@@ -64,10 +66,7 @@ namespace vcpkg::Json
         int indent = 2;
     };
 
-    struct Array;
-    struct Object;
-
-    enum class ValueKind
+    enum class ValueKind : int
     {
         Null,
         Boolean,
@@ -81,7 +80,6 @@ namespace vcpkg::Json
     namespace impl
     {
         struct ValueImpl;
-        struct SyntaxErrorImpl;
     }
 
     struct Value
@@ -289,6 +287,53 @@ namespace vcpkg::Json
         underlying_t underlying_;
     };
 
+    VCPKG_MSVC_WARNING(push);
+    VCPKG_MSVC_WARNING(disable : 4505);
+
+    template<class Type>
+    Span<const StringView> IDeserializer<Type>::valid_fields() const
+    {
+        return {};
+    }
+
+    template<class Type>
+    Optional<Type> IDeserializer<Type>::visit_null(Reader&, StringView)
+    {
+        return nullopt;
+    }
+    template<class Type>
+    Optional<Type> IDeserializer<Type>::visit_boolean(Reader&, StringView, bool)
+    {
+        return nullopt;
+    }
+    template<class Type>
+    Optional<Type> IDeserializer<Type>::visit_integer(Reader& r, StringView field_name, int64_t i)
+    {
+        return this->visit_number(r, field_name, static_cast<double>(i));
+    }
+    template<class Type>
+    Optional<Type> IDeserializer<Type>::visit_number(Reader&, StringView, double)
+    {
+        return nullopt;
+    }
+    template<class Type>
+    Optional<Type> IDeserializer<Type>::visit_string(Reader&, StringView, StringView)
+    {
+        return nullopt;
+    }
+    template<class Type>
+    Optional<Type> IDeserializer<Type>::visit_array(Reader&, StringView, const Array&)
+    {
+        return nullopt;
+    }
+    template<class Type>
+    Optional<Type> IDeserializer<Type>::visit_object(Reader&, StringView, const Object&)
+    {
+        return nullopt;
+    }
+
+    VCPKG_MSVC_WARNING(pop);
+
     struct ReaderError
     {
         virtual void add_missing_field(std::string&& type, std::string&& key) = 0;
@@ -299,6 +344,37 @@ namespace vcpkg::Json
         virtual ~ReaderError() = default;
     };
 
+    struct BasicReaderError : ReaderError
+    {
+        virtual void add_missing_field(std::string&& type, std::string&& key) override
+        {
+            missing_fields.emplace_back(std::move(type), std::move(key));
+        }
+        virtual void add_expected_type(std::string&& key, std::string&& expected_type) override
+        {
+            expected_types.emplace_back(std::move(key), std::move(expected_type));
+        }
+        virtual void add_extra_fields(std::string&& type, std::vector<std::string>&& fields) override
+        {
+            extra_fields.emplace_back(std::move(type), std::move(fields));
+        }
+        virtual void add_mutually_exclusive_fields(std::string&& type, std::vector<std::string>&& fields) override
+        {
+            mutually_exclusive_fields.emplace_back(std::move(type), std::move(fields));
+        }
+
+        bool has_error() const
+        {
+            return !missing_fields.empty() || !expected_types.empty() || !extra_fields.empty() ||
+                   !mutually_exclusive_fields.empty();
+        }
+
+        std::vector<std::pair<std::string, std::string>> missing_fields;
+        std::vector<std::pair<std::string, std::string>> expected_types;
+        std::vector<std::pair<std::string, std::vector<std::string>>> extra_fields;
+        std::vector<std::pair<std::string, std::vector<std::string>>> mutually_exclusive_fields;
+    };
+
     struct Reader
     {
         explicit Reader(ReaderError* err) : err(err) { }
@@ -308,30 +384,31 @@ namespace vcpkg::Json
     private:
         ReaderError* err;
 
-        template<class Visitor>
-        using VisitorType = typename std::remove_reference_t<Visitor>::type;
-
-        template<class Visitor>
-        Optional<VisitorType<Visitor>> internal_visit(const Value& value, StringView key, Visitor& visitor)
+        template<class Type>
+        Optional<Type> internal_visit(const Value& value, StringView key, IDeserializer<Type>& visitor)
         {
             switch (value.kind())
             {
-                using VK = Json::ValueKind;
-                case VK::Null: return visitor.visit_null(*this, key);
-                case VK::Boolean: return visitor.visit_boolean(*this, key, value.boolean());
-                case VK::Integer: return visitor.visit_integer(*this, key, value.integer());
-                case VK::Number: return visitor.visit_number(*this, key, value.number());
-                case VK::String: return visitor.visit_string(*this, key, value.string());
-                case VK::Array: return visitor.visit_array(*this, key, value.array());
-                case VK::Object: return visitor.visit_object(*this, key, value.object());
+                case ValueKind::Null: return visitor.visit_null(*this, key);
+                case ValueKind::Boolean: return visitor.visit_boolean(*this, key, value.boolean());
+                case ValueKind::Integer: return visitor.visit_integer(*this, key, value.integer());
+                case ValueKind::Number: return visitor.visit_number(*this, key, value.number());
+                case ValueKind::String: return visitor.visit_string(*this, key, value.string());
+                case ValueKind::Array: return visitor.visit_array(*this, key, value.array());
+                case ValueKind::Object:
+                {
+                    const auto& obj = value.object();
+                    check_for_unexpected_fields(obj, visitor.valid_fields(), visitor.type_name());
+                    return visitor.visit_object(*this, key, value.object());
+                }
             }
 
             vcpkg::Checks::exit_fail(VCPKG_LINE_INFO);
         }
 
         // returns whether the field was found, not whether it was valid
-        template<class Visitor>
-        bool internal_field(const Object& obj, StringView key, VisitorType<Visitor>& place, Visitor& visitor)
+        template<class Type>
+        bool internal_field(const Object& obj, StringView key, Type& place, IDeserializer<Type>& visitor)
         {
             auto value = obj.get(key);
             if (!value)
@@ -339,7 +416,7 @@ namespace vcpkg::Json
                 return false;
             }
 
-            Optional<VisitorType<Visitor>> opt = internal_visit(*value, key, visitor);
+            Optional<Type> opt = internal_visit(*value, key, visitor);
 
             if (auto val = opt.get())
             {
@@ -353,27 +430,115 @@ namespace vcpkg::Json
             return true;
         }
 
+        static std::vector<std::string> invalid_json_fields(const Json::Object& obj,
+                                                            Span<const StringView> known_fields) noexcept
+        {
+            const auto field_is_unknown = [known_fields](StringView sv) {
+                // allow directives
+                if (sv.size() != 0 && *sv.begin() == '$')
+                {
+                    return false;
+                }
+                return std::find(known_fields.begin(), known_fields.end(), sv) == known_fields.end();
+            };
+
+            std::vector<std::string> res;
+            for (const auto& kv : obj)
+            {
+                if (field_is_unknown(kv.first))
+                {
+                    res.push_back(kv.first.to_string());
+                }
+            }
+
+            return res;
+        }
+
+        // checks that an object doesn't contain any fields which both:
+        // * don't start with a `$`
+        // * are not in `valid_fields`
+        // if known_fields.empty(), then it's treated as if all field names are valid
+        void check_for_unexpected_fields(const Object& obj, Span<const StringView> valid_fields, StringView type_name)
+        {
+            if (valid_fields.size() == 0)
+            {
+                return;
+            }
+
+            auto extra_fields = invalid_json_fields(obj, valid_fields);
+            if (!extra_fields.empty())
+            {
+                error().add_extra_fields(type_name.to_string(), std::move(extra_fields));
+            }
+        }
+
     public:
-        template<class Visitor>
+        template<class Type>
         void required_object_field(
-            StringView type, const Object& obj, StringView key, VisitorType<Visitor>& place, Visitor&& visitor)
+            StringView type, const Object& obj, StringView key, Type& place, IDeserializer<Type>& visitor)
         {
             if (!internal_field(obj, key, place, visitor))
             {
                 err->add_missing_field(type.to_string(), key.to_string());
             }
         }
+        template<class Type>
+        void required_object_field(
+            StringView type, const Object& obj, StringView key, Type& place, IDeserializer<Type>&& visitor)
+        {
+            required_object_field(type, obj, key, place, visitor);
+        }
 
-        template<class Visitor>
-        void optional_object_field(const Object& obj, StringView key, VisitorType<Visitor>& place, Visitor&& visitor)
+        // returns whether key \in obj
+        template<class Type>
+        bool optional_object_field(const Object& obj, StringView key, Type& place, IDeserializer<Type>& visitor)
+        {
+            return internal_field(obj, key, place, visitor);
+        }
+        template<class Type>
+        bool optional_object_field(const Object& obj, StringView key, Type& place, IDeserializer<Type>&& visitor)
         {
-            internal_field(obj, key, place, visitor);
+            return optional_object_field(obj, key, place, visitor);
         }
 
-        template<class Visitor>
-        Optional<std::vector<VisitorType<Visitor>>> array_elements(const Array& arr, StringView key, Visitor&& visitor)
+        template<class Type>
+        Optional<Type> visit_value(const Value& value, StringView key, IDeserializer<Type>& visitor)
+        {
+            return internal_visit(value, key, visitor);
+        }
+        template<class Type>
+        Optional<Type> visit_value(const Value& value, StringView key, IDeserializer<Type>&& visitor)
         {
-            std::vector<VisitorType<Visitor>> result;
+            return visit_value(value, key, visitor);
+        }
+
+        template<class Type>
+        Optional<Type> visit_value(const Array& value, StringView key, IDeserializer<Type>& visitor)
+        {
+            return visitor.visit_array(*this, key, value);
+        }
+        template<class Type>
+        Optional<Type> visit_value(const Array& value, StringView key, IDeserializer<Type>&& visitor)
+        {
+            return visit_value(value, key, visitor);
+        }
+
+        template<class Type>
+        Optional<Type> visit_value(const Object& value, StringView key, IDeserializer<Type>& visitor)
+        {
+            check_for_unexpected_fields(value, visitor.valid_fields(), visitor.type_name());
+            return visitor.visit_object(*this, key, value);
+        }
+        template<class Type>
+        Optional<Type> visit_value(const Object& value, StringView key, IDeserializer<Type>&& visitor)
+        {
+            return visit_value(value, key, visitor);
+        }
+
+        template<class Type>
+        Optional<std::vector<Type>> array_elements(const Array& arr, StringView key, IDeserializer<Type>& visitor)
+        {
+            std::vector<Type> result;
             for (const auto& el : arr)
             {
                 auto opt = internal_visit(el, key, visitor);
@@ -388,42 +553,161 @@ namespace vcpkg::Json
             }
             return std::move(result);
         }
+        template<class Type>
+        Optional<std::vector<Type>> array_elements(const Array& arr, StringView key, IDeserializer<Type>&& visitor)
+        {
+            return array_elements(arr, key, visitor);
+        }
+    };
+
+    struct StringDeserializer final : IDeserializer<std::string>
+    {
+        virtual StringView type_name() const override { return type_name_; }
+        virtual Optional<std::string> visit_string(Reader&, StringView, StringView sv) override
+        {
+            return sv.to_string();
+        }
+
+        explicit StringDeserializer(StringView type_name_) : type_name_(type_name_) { }
+
+    private:
+        StringView type_name_;
+    };
+
+    struct PathDeserializer final : IDeserializer<fs::path>
+    {
+        virtual StringView type_name() const override { return "a path"; }
+        virtual Optional<fs::path> visit_string(Reader&, StringView, StringView sv) override { return fs::u8path(sv); }
+    };
+
+    struct NaturalNumberDeserializer final : IDeserializer<int>
+    {
+        virtual StringView type_name() const override { return "a natural number"; }
+
+        virtual Optional<int> visit_integer(Reader&, StringView, int64_t value) override
+        {
+            if (value > std::numeric_limits<int>::max() || value < 0)
+            {
+                return nullopt;
+            }
+            return static_cast<int>(value);
+        }
+    };
+
+    struct BooleanDeserializer final : IDeserializer<bool>
+    {
+        virtual StringView type_name() const override { return "a boolean"; }
+
+        virtual Optional<bool> visit_boolean(Reader&, StringView, bool b) override { return b; }
+    };
+
+    enum class AllowEmpty : bool
+    {
+        No,
+        Yes,
     };
 
-    // Warning: NEVER use this type except as a CRTP base
     template<class Underlying>
-    struct VisitorCrtpBase
+    struct ArrayDeserializer final : IDeserializer<std::vector<typename Underlying::type>>
+    {
+        using typename IDeserializer<std::vector<typename Underlying::type>>::type;
+
+        virtual StringView type_name() const override { return type_name_; }
+
+        ArrayDeserializer(StringView type_name_, AllowEmpty allow_empty, Underlying&& t = {})
+            : type_name_(type_name_), underlying_visitor_(static_cast<Underlying&&>(t)), allow_empty_(allow_empty)
+        {
+        }
+
+        virtual Optional<type> visit_array(Reader& r, StringView key, const Array& arr) override
+        {
+            if (allow_empty_ == AllowEmpty::No && arr.size() == 0)
+            {
+                return nullopt;
+            }
+            return r.array_elements(arr, key, underlying_visitor_);
+        }
+
+    private:
+        StringView type_name_;
+        Underlying underlying_visitor_;
+        AllowEmpty allow_empty_;
+    };
+
+    struct ParagraphDeserializer final : IDeserializer<std::vector<std::string>>
     {
-        // the following function must be defined by the Underlying class
-        // const char* type_name();
+        virtual StringView type_name() const override { return "a string or array of strings"; }
 
-        // We do this auto dance since function bodies are checked _after_ typedefs in the superclass,
-        // but function declarations are checked beforehand. Therefore, we can get C++ to use this typedef
-        // only once the function bodies are checked by returning `auto` and specifying the
-        // return type in the function body.
-        auto visit_null(Reader&, StringView) { return Optional<typename Underlying::type>(nullopt); }
-        auto visit_boolean(Reader&, StringView, bool) { return Optional<typename Underlying::type>(nullopt); }
-        auto visit_integer(Reader& r, StringView field_name, int64_t i)
+        virtual Optional<std::vector<std::string>> visit_string(Reader&, StringView, StringView sv) override
         {
-            return static_cast<Underlying&>(*this).visit_number(r, field_name, static_cast<double>(i));
+            std::vector<std::string> out;
+            out.push_back(sv.to_string());
+            return out;
+        }
+
+        virtual Optional<std::vector<std::string>> visit_array(Reader& r, StringView key, const Array& arr) override
+        {
+            return r.array_elements(arr, key, StringDeserializer{"a string"});
+        }
+    };
+
+    struct IdentifierDeserializer final : Json::IDeserializer<std::string>
+    {
+        virtual StringView type_name() const override { return "an identifier"; }
+
+        // [a-z0-9]+(-[a-z0-9]+)*, plus not any of {prn, aux, nul, con, lpt[1-9], com[1-9], core, default}
+        static bool is_ident(StringView sv);
+
+        virtual Optional<std::string> visit_string(Json::Reader&, StringView, StringView sv) override
+        {
+            if (is_ident(sv))
+            {
+                return sv.to_string();
+            }
+            else
+            {
+                return nullopt;
+            }
         }
-        auto visit_number(Reader&, StringView, double) { return Optional<typename Underlying::type>(nullopt); }
-        auto visit_string(Reader&, StringView, StringView) { return Optional<typename Underlying::type>(nullopt); }
-        auto visit_array(Reader&, StringView, const Json::Array&)
+    };
+
+    struct PackageNameDeserializer final : Json::IDeserializer<std::string>
+    {
+        virtual StringView type_name() const override { return "a package name"; }
+
+        static bool is_package_name(StringView sv)
         {
-            return Optional<typename Underlying::type>(nullopt);
+            if (sv.size() == 0)
+            {
+                return false;
+            }
+
+            for (const auto& ident : Strings::split(sv, '.'))
+            {
+                if (!IdentifierDeserializer::is_ident(ident))
+                {
+                    return false;
+                }
+            }
+
+            return true;
         }
-        auto visit_object(Reader&, StringView, const Json::Object&)
+
+        virtual Optional<std::string> visit_string(Json::Reader&, StringView, StringView sv) override
         {
-            return Optional<typename Underlying::type>(nullopt);
+            if (!is_package_name(sv))
+            {
+                return nullopt;
+            }
+            return sv.to_string();
         }
-        // we can't make the SMFs protected because of an issue with /permissive mode
     };
 
     ExpectedT<std::pair<Value, JsonStyle>, std::unique_ptr<Parse::IParseError>> parse_file(
         const Files::Filesystem&, const fs::path&, std::error_code& ec) noexcept;
     ExpectedT<std::pair<Value, JsonStyle>, std::unique_ptr<Parse::IParseError>> parse(
         StringView text, const fs::path& filepath = {}) noexcept;
+    std::pair<Value, JsonStyle> parse_file(vcpkg::LineInfo linfo, const Files::Filesystem&, const fs::path&) noexcept;
 
     std::string stringify(const Value&, JsonStyle style);
     std::string stringify(const Object&, JsonStyle style);
diff --git a/toolsrc/include/vcpkg/base/optional.h b/toolsrc/include/vcpkg/base/optional.h
index 091cd498e..54e370dff 100644
--- a/toolsrc/include/vcpkg/base/optional.h
+++ b/toolsrc/include/vcpkg/base/optional.h
@@ -1,5 +1,7 @@
 #pragma once
 
+#include <vcpkg/base/fwd/optional.h>
+
 #include <vcpkg/base/lineinfo.h>
 #include <vcpkg/base/pragmas.h>
 
@@ -15,6 +17,9 @@ namespace vcpkg
 
     const static constexpr NullOpt nullopt{0};
 
+    template<class T>
+    struct Optional;
+
     namespace details
     {
         template<class T, bool B = std::is_copy_constructible<T>::value>
@@ -167,6 +172,7 @@ namespace vcpkg
         {
             constexpr OptionalStorage() noexcept : m_t(nullptr) { }
             constexpr OptionalStorage(T& t) : m_t(&t) { }
+            constexpr OptionalStorage(Optional<T>& t) : m_t(t.get()) { }
 
             constexpr bool has_value() const { return m_t != nullptr; }
 
@@ -176,6 +182,24 @@ namespace vcpkg
             T* m_t;
         };
 
+        template<class T, bool B>
+        struct OptionalStorage<const T&, B>
+        {
+            constexpr OptionalStorage() noexcept : m_t(nullptr) { }
+            constexpr OptionalStorage(const T& t) : m_t(&t) { }
+            constexpr OptionalStorage(const Optional<T>& t) : m_t(t.get()) { }
+            constexpr OptionalStorage(const Optional<const T>& t) : m_t(t.get()) { }
+            constexpr OptionalStorage(Optional<T>&& t) = delete;
+            constexpr OptionalStorage(Optional<const T>&& t) = delete;
+
+            constexpr bool has_value() const { return m_t != nullptr; }
+
+            const T& value() const { return *this->m_t; }
+
+        private:
+            const T* m_t;
+        };
+
         // Note: implemented in checks.cpp to cut the header dependency
         void exit_if_null(bool b, const LineInfo& line_info);
     }
@@ -325,27 +349,4 @@ namespace vcpkg
         if (auto p = o.get()) return t != *p;
         return true;
     }
-
-    template<class Container, class Projection>
-    auto common_projection(const Container& input, Projection proj)
-        -> Optional<std::decay_t<decltype(proj(*(input.begin())))>>
-    {
-        const auto last = input.end();
-        auto first = input.begin();
-        if (first == last)
-        {
-            return nullopt;
-        }
-
-        const auto& prototype = proj(*first);
-        while (++first != last)
-        {
-            if (prototype != proj(*first))
-            {
-                return nullopt;
-            }
-        }
-
-        return prototype;
-    }
 }
diff --git a/toolsrc/include/vcpkg/base/stringview.h b/toolsrc/include/vcpkg/base/stringview.h
index 405142c36..9238990fe 100644
--- a/toolsrc/include/vcpkg/base/stringview.h
+++ b/toolsrc/include/vcpkg/base/stringview.h
@@ -1,5 +1,7 @@
 #pragma once
 
+#include <vcpkg/base/fwd/stringview.h>
+
 #include <vcpkg/base/optional.h>
 
 #include <limits>
diff --git a/toolsrc/include/vcpkg/base/util.h b/toolsrc/include/vcpkg/base/util.h
index 11807b249..0453ac20a 100644
--- a/toolsrc/include/vcpkg/base/util.h
+++ b/toolsrc/include/vcpkg/base/util.h
@@ -1,5 +1,7 @@
 #pragma once
 
+#include <vcpkg/base/optional.h>
+
 #include <algorithm>
 #include <functional>
 #include <map>
@@ -63,7 +65,10 @@ namespace vcpkg::Util
     }
 
     template<class Range, class Func>
-    using FmapOut = std::remove_reference_t<decltype(std::declval<Func&>()(*std::declval<Range>().begin()))>;
+    using FmapRefOut = decltype(std::declval<Func&>()(*std::declval<Range>().begin()));
+
+    template<class Range, class Func>
+    using FmapOut = std::decay_t<FmapRefOut<Range, Func>>;
 
     template<class Range, class Func, class Out = FmapOut<Range, Func>>
     std::vector<Out> fmap(Range&& xs, Func&& f)
@@ -77,6 +82,28 @@ namespace vcpkg::Util
         return ret;
     }
 
+    template<class Range, class Proj, class Out = FmapRefOut<Range, Proj>>
+    Optional<Out> common_projection(Range&& input, Proj&& proj)
+    {
+        const auto last = input.end();
+        auto first = input.begin();
+        if (first == last)
+        {
+            return nullopt;
+        }
+
+        Out prototype = proj(*first);
+        while (++first != last)
+        {
+            if (prototype != proj(*first))
+            {
+                return nullopt;
+            }
+        }
+
+        return prototype;
+    }
+
     template<class Cont, class Func>
     using FmapFlattenOut = std::decay_t<decltype(*begin(std::declval<Func>()(*begin(std::declval<Cont>()))))>;
 
diff --git a/toolsrc/include/vcpkg/configuration.h b/toolsrc/include/vcpkg/configuration.h
new file mode 100644
index 000000000..44440fa04
--- /dev/null
+++ b/toolsrc/include/vcpkg/configuration.h
@@ -0,0 +1,41 @@
+#pragma once
+
+#include <vcpkg/fwd/configuration.h>
+#include <vcpkg/fwd/vcpkgcmdarguments.h>
+
+#include <vcpkg/base/json.h>
+
+#include <vcpkg/registries.h>
+
+namespace vcpkg
+{
+    struct Configuration
+    {
+        // This member is set up via two different configuration options,
+        // `registries` and `default_registry`. The fall back logic is
+        // taken care of in RegistrySet.
+        RegistrySet registry_set;
+    };
+
+    struct ConfigurationDeserializer final : Json::IDeserializer<Configuration>
+    {
+        virtual StringView type_name() const override { return "a configuration object"; }
+
+        constexpr static StringLiteral DEFAULT_REGISTRY = "default-registry";
+        constexpr static StringLiteral REGISTRIES = "registries";
+        virtual Span<const StringView> valid_fields() const override
+        {
+            constexpr static StringView t[] = {DEFAULT_REGISTRY, REGISTRIES};
+            return t;
+        }
+
+        virtual Optional<Configuration> visit_object(Json::Reader& r, StringView, const Json::Object& obj) override;
+
+        ConfigurationDeserializer(const VcpkgCmdArguments& args);
+
+    private:
+        bool print_json;
+
+        bool registries_enabled;
+    };
+}
diff --git a/toolsrc/include/vcpkg/fwd/configuration.h b/toolsrc/include/vcpkg/fwd/configuration.h
new file mode 100644
index 000000000..5fa9cee34
--- /dev/null
+++ b/toolsrc/include/vcpkg/fwd/configuration.h
@@ -0,0 +1,6 @@
+#pragma once
+
+namespace vcpkg
+{
+    struct Configuration;
+}
diff --git a/toolsrc/include/vcpkg/fwd/registries.h b/toolsrc/include/vcpkg/fwd/registries.h
new file mode 100644
index 000000000..7352c429d
--- /dev/null
+++ b/toolsrc/include/vcpkg/fwd/registries.h
@@ -0,0 +1,8 @@
+#pragma once
+
+namespace vcpkg
+{
+    struct RegistryImpl;
+    struct Registry;
+    struct RegistrySet;
+}
diff --git a/toolsrc/include/vcpkg/paragraphs.h b/toolsrc/include/vcpkg/paragraphs.h
index 25d037591..95726f913 100644
--- a/toolsrc/include/vcpkg/paragraphs.h
+++ b/toolsrc/include/vcpkg/paragraphs.h
@@ -17,23 +17,33 @@ namespace vcpkg::Paragraphs
 
     bool is_port_directory(const Files::Filesystem& fs, const fs::path& path);
 
-    Parse::ParseExpected<SourceControlFile> try_load_manifest(const Files::Filesystem& fs,
-                                                              const std::string& port_name,
-                                                              const fs::path& path_to_manifest,
-                                                              std::error_code& ec);
-
     Parse::ParseExpected<SourceControlFile> try_load_port(const Files::Filesystem& fs, const fs::path& path);
 
     ExpectedS<BinaryControlFile> try_load_cached_package(const VcpkgPaths& paths, const PackageSpec& spec);
 
     struct LoadResults
     {
-        std::vector<std::unique_ptr<SourceControlFile>> paragraphs;
+        std::vector<SourceControlFileLocation> paragraphs;
         std::vector<std::unique_ptr<Parse::ParseControlErrorInfo>> errors;
     };
 
-    LoadResults try_load_all_ports(const Files::Filesystem& fs, const fs::path& ports_dir);
-
-    std::vector<std::unique_ptr<SourceControlFile>> load_all_ports(const Files::Filesystem& fs,
-                                                                   const fs::path& ports_dir);
+    // this allows one to pass this around as an overload set to stuff like `Util::fmap`,
+    // as opposed to making it a function
+    constexpr struct
+    {
+        const std::string& operator()(const SourceControlFileLocation* loc) const
+        {
+            return (*this)(*loc->source_control_file);
+        }
+        const std::string& operator()(const SourceControlFileLocation& loc) const
+        {
+            return (*this)(*loc.source_control_file);
+        }
+        const std::string& operator()(const SourceControlFile& scf) const { return scf.core_paragraph->name; }
+    } get_name_of_control_file;
+
+    LoadResults try_load_all_registry_ports(const VcpkgPaths& paths);
+
+    std::vector<SourceControlFileLocation> load_all_registry_ports(const VcpkgPaths& paths);
+    std::vector<SourceControlFileLocation> load_overlay_ports(const VcpkgPaths& paths, const fs::path& dir);
 }
diff --git a/toolsrc/include/vcpkg/portfileprovider.h b/toolsrc/include/vcpkg/portfileprovider.h
index e4b5d9cb5..d3fa0b51f 100644
--- a/toolsrc/include/vcpkg/portfileprovider.h
+++ b/toolsrc/include/vcpkg/portfileprovider.h
@@ -26,14 +26,13 @@ namespace vcpkg::PortFileProvider
 
     struct PathsPortFileProvider : Util::ResourceBase, PortFileProvider
     {
-        explicit PathsPortFileProvider(const vcpkg::VcpkgPaths& paths,
-                                       const std::vector<std::string>& ports_dirs_paths);
+        explicit PathsPortFileProvider(const vcpkg::VcpkgPaths& paths, const std::vector<std::string>& overlay_ports);
         ExpectedS<const SourceControlFileLocation&> get_control_file(const std::string& src_name) const override;
         std::vector<const SourceControlFileLocation*> load_all_control_files() const override;
 
     private:
-        Files::Filesystem& filesystem;
-        std::vector<fs::path> ports_dirs;
+        const VcpkgPaths& paths;
+        std::vector<fs::path> overlay_ports;
         mutable std::unordered_map<std::string, SourceControlFileLocation> cache;
     };
 }
diff --git a/toolsrc/include/vcpkg/registries.h b/toolsrc/include/vcpkg/registries.h
new file mode 100644
index 000000000..78c055dae
--- /dev/null
+++ b/toolsrc/include/vcpkg/registries.h
@@ -0,0 +1,97 @@
+#pragma once
+
+#include <vcpkg/base/fwd/json.h>
+
+#include <vcpkg/base/files.h>
+#include <vcpkg/base/optional.h>
+
+#include <vcpkg/vcpkgpaths.h>
+
+#include <memory>
+#include <string>
+#include <system_error>
+#include <vector>
+
+namespace vcpkg
+{
+    struct RegistryImpl
+    {
+        virtual fs::path get_registry_root(const VcpkgPaths& paths) const = 0;
+
+        virtual ~RegistryImpl() = default;
+    };
+
+    struct Registry
+    {
+        // requires: static_cast<bool>(implementation)
+        Registry(std::vector<std::string>&& packages, std::unique_ptr<RegistryImpl>&& implementation);
+
+        Registry(std::vector<std::string>&&, std::nullptr_t) = delete;
+
+        // always ordered lexicographically
+        Span<const std::string> packages() const { return packages_; }
+        const RegistryImpl& implementation() const { return *implementation_; }
+
+        static std::unique_ptr<RegistryImpl> builtin_registry();
+
+    private:
+        std::vector<std::string> packages_;
+        std::unique_ptr<RegistryImpl> implementation_;
+    };
+
+    struct RegistryImplDeserializer : Json::IDeserializer<std::unique_ptr<RegistryImpl>>
+    {
+        constexpr static StringLiteral KIND = "kind";
+        constexpr static StringLiteral PATH = "path";
+
+        constexpr static StringLiteral KIND_BUILTIN = "builtin";
+        constexpr static StringLiteral KIND_DIRECTORY = "directory";
+
+        virtual StringView type_name() const override;
+        virtual Span<const StringView> valid_fields() const override;
+
+        virtual Optional<std::unique_ptr<RegistryImpl>> visit_null(Json::Reader&, StringView) override;
+        virtual Optional<std::unique_ptr<RegistryImpl>> visit_object(Json::Reader&,
+                                                                     StringView,
+                                                                     const Json::Object&) override;
+    };
+
+    struct RegistryDeserializer final : Json::IDeserializer<Registry>
+    {
+        constexpr static StringLiteral PACKAGES = "packages";
+
+        virtual StringView type_name() const override;
+        virtual Span<const StringView> valid_fields() const override;
+
+        virtual Optional<Registry> visit_object(Json::Reader&, StringView, const Json::Object&) override;
+    };
+
+    // this type implements the registry fall back logic from the registries RFC:
+    // A port name maps to one of the non-default registries if that registry declares
+    // that it is the registry for that port name, else it maps to the default registry
+    // if that registry exists; else, there is no registry for a port.
+    // The way one sets this up is via the `"registries"` and `"default_registry"`
+    // configuration fields.
+    struct RegistrySet
+    {
+        RegistrySet();
+
+        // finds the correct registry for the port name
+        // Returns the null pointer if there is no registry set up for that name
+        const RegistryImpl* registry_for_port(StringView port_name) const;
+
+        Span<const Registry> registries() const { return registries_; }
+
+        const RegistryImpl* default_registry() const { return default_registry_.get(); }
+
+        // TODO: figure out how to get this to return an error (or maybe it should be a warning?)
+        void add_registry(Registry&& r);
+        void set_default_registry(std::unique_ptr<RegistryImpl>&& r);
+        void set_default_registry(std::nullptr_t r);
+
+    private:
+        std::unique_ptr<RegistryImpl> default_registry_;
+        std::vector<Registry> registries_;
+    };
+
+}
diff --git a/toolsrc/include/vcpkg/sourceparagraph.h b/toolsrc/include/vcpkg/sourceparagraph.h
index ac3699ca9..d5069adde 100644
--- a/toolsrc/include/vcpkg/sourceparagraph.h
+++ b/toolsrc/include/vcpkg/sourceparagraph.h
@@ -1,7 +1,8 @@
 #pragma once
 
+#include <vcpkg/base/fwd/json.h>
+
 #include <vcpkg/base/expected.h>
-#include <vcpkg/base/json.h>
 #include <vcpkg/base/span.h>
 #include <vcpkg/base/system.h>
 #include <vcpkg/base/system.print.h>
diff --git a/toolsrc/include/vcpkg/vcpkgcmdarguments.h b/toolsrc/include/vcpkg/vcpkgcmdarguments.h
index 3e956c83e..626e598be 100644
--- a/toolsrc/include/vcpkg/vcpkgcmdarguments.h
+++ b/toolsrc/include/vcpkg/vcpkgcmdarguments.h
@@ -170,9 +170,12 @@ namespace vcpkg
         Optional<bool> compiler_tracking = nullopt;
         constexpr static StringLiteral MANIFEST_MODE_FEATURE = "manifests";
         Optional<bool> manifest_mode = nullopt;
+        constexpr static StringLiteral REGISTRIES_FEATURE = "registries";
+        Optional<bool> registries_feature = nullopt;
 
         bool binary_caching_enabled() const { return binary_caching.value_or(true); }
         bool compiler_tracking_enabled() const { return compiler_tracking.value_or(true); }
+        bool registries_enabled() const { return registries_feature.value_or(false); }
         bool output_json() const { return json.value_or(false); }
 
         std::string command;
diff --git a/toolsrc/include/vcpkg/vcpkgpaths.h b/toolsrc/include/vcpkg/vcpkgpaths.h
index 2262543d9..ae2ee3823 100644
--- a/toolsrc/include/vcpkg/vcpkgpaths.h
+++ b/toolsrc/include/vcpkg/vcpkgpaths.h
@@ -1,5 +1,10 @@
 #pragma once
 
+#include <vcpkg/base/fwd/json.h>
+
+#include <vcpkg/fwd/configuration.h>
+#include <vcpkg/fwd/registries.h>
+#include <vcpkg/fwd/vcpkgcmdarguments.h>
 #include <vcpkg/fwd/vcpkgpaths.h>
 
 #include <vcpkg/base/cache.h>
@@ -45,7 +50,6 @@ namespace vcpkg
     }
 
     struct BinaryParagraph;
-    struct VcpkgCmdArguments;
     struct PackageSpec;
     struct Triplet;
 
@@ -77,6 +81,7 @@ namespace vcpkg
         fs::path original_cwd;
         fs::path root;
         fs::path manifest_root_dir;
+        fs::path config_root_dir;
         fs::path buildtrees;
         fs::path downloads;
         fs::path packages;
@@ -102,6 +107,10 @@ namespace vcpkg
         const fs::path& get_tool_exe(const std::string& tool) const;
         const std::string& get_tool_version(const std::string& tool) const;
 
+        Optional<const Json::Object&> get_manifest() const;
+        Optional<const Json::JsonStyle&> get_manifest_style() const;
+        const Configuration& get_configuration() const;
+
         /// <summary>Retrieve a toolset matching a VS version</summary>
         /// <remarks>
         ///   Valid version strings are "v120", "v140", "v141", and "". Empty string gets the latest.
@@ -112,7 +121,7 @@ namespace vcpkg
 
         const System::Environment& get_action_env(const Build::AbiInfo& abi_info) const;
         const std::string& get_triplet_info(const Build::AbiInfo& abi_info) const;
-        bool manifest_mode_enabled() const { return !manifest_root_dir.empty(); }
+        bool manifest_mode_enabled() const { return get_manifest().has_value(); }
 
         void track_feature_flag_metrics() const;