[vcpkg registries] support versions (#15114)

* [vcpkg registries] support versions

This PR merges the Registries changes and the versioning changes, so that one can use both at the same time.

There is one major difference between this PR and the RFC (#13590), which is that instead of version files looking like:

```json
[
  ...
]
```

version files look like:

```
{
  "versions": [
    ...
  ]
}
```

this is to support interop between this PR and existing demos and the like;
fixing this, along with perhaps renaming `port_versions` to `port-versions` should be done after this is merged,
should be a trivial change.

11 files changed, 64 insertions, 105 deletions

diff --git a/toolsrc/include/vcpkg/base/files.h b/toolsrc/include/vcpkg/base/files.h
index a6cc06c19..69ec41fe3 100644
--- a/toolsrc/include/vcpkg/base/files.h
+++ b/toolsrc/include/vcpkg/base/files.h
@@ -239,6 +239,12 @@ namespace vcpkg::Files
         virtual void current_path(const fs::path& path, std::error_code&) = 0;
         void current_path(const fs::path& path, LineInfo li);
 
+        // if the path does not exist, then (try_|)take_exclusive_file_lock attempts to create the file
+        // (but not any path members above the file itself)
+        // in other words, if `/a/b` is a directory, and you're attempting to lock `/a/b/c`,
+        // then these lock functions create `/a/b/c` if it doesn't exist;
+        // however, if `/a/b` doesn't exist, then the functions will fail.
+
         // waits forever for the file lock
         virtual fs::SystemHandle take_exclusive_file_lock(const fs::path& path, std::error_code&) = 0;
         // waits, at most, 1.5 seconds, for the file lock
diff --git a/toolsrc/include/vcpkg/base/jsonreader.h b/toolsrc/include/vcpkg/base/jsonreader.h
index bea60e7ce..da086fa02 100644
--- a/toolsrc/include/vcpkg/base/jsonreader.h
+++ b/toolsrc/include/vcpkg/base/jsonreader.h
@@ -75,13 +75,13 @@ namespace vcpkg::Json
         };
         std::vector<Path> m_path;
 
+    public:
         // checks that an object doesn't contain any fields which both:
         // * don't start with a `$`
         // * are not in `valid_fields`
         // if known_fields.empty(), then it's treated as if all field names are valid
         void check_for_unexpected_fields(const Object& obj, View<StringView> valid_fields, StringView type_name);
 
-    public:
         template<class Type>
         void required_object_field(
             StringView type, const Object& obj, StringView key, Type& place, IDeserializer<Type>& visitor)
diff --git a/toolsrc/include/vcpkg/configuration.h b/toolsrc/include/vcpkg/configuration.h
index 4cba88fe5..6d6a9b1f4 100644
--- a/toolsrc/include/vcpkg/configuration.h
+++ b/toolsrc/include/vcpkg/configuration.h
@@ -1,6 +1,10 @@
 #pragma once
 
 #include <vcpkg/fwd/configuration.h>
+#include <vcpkg/fwd/vcpkgcmdarguments.h>
+
+#include <vcpkg/base/files.h>
+#include <vcpkg/base/json.h>
 
 #include <vcpkg/registries.h>
 
@@ -12,5 +16,10 @@ namespace vcpkg
         // `registries` and `default_registry`. The fall back logic is
         // taken care of in RegistrySet.
         RegistrySet registry_set;
+
+        void validate_feature_flags(const FeatureFlagSettings& flags);
     };
+
+    std::unique_ptr<Json::IDeserializer<Configuration>> make_configuration_deserializer(
+        const fs::path& config_directory);
 }
diff --git a/toolsrc/include/vcpkg/configurationdeserializer.h b/toolsrc/include/vcpkg/configurationdeserializer.h
deleted file mode 100644
index d6b239356..000000000
--- a/toolsrc/include/vcpkg/configurationdeserializer.h
+++ /dev/null
@@ -1,66 +0,0 @@
-#pragma once
-
-#include <vcpkg/base/fwd/json.h>
-
-#include <vcpkg/fwd/vcpkgcmdarguments.h>
-
-#include <vcpkg/base/json.h>
-#include <vcpkg/base/optional.h>
-#include <vcpkg/base/stringliteral.h>
-#include <vcpkg/base/stringview.h>
-#include <vcpkg/base/view.h>
-
-#include <vcpkg/configuration.h>
-#include <vcpkg/registries.h>
-
-namespace vcpkg
-{
-    struct RegistryImplDeserializer : Json::IDeserializer<std::unique_ptr<RegistryImpl>>
-    {
-        constexpr static StringLiteral KIND = "kind";
-        constexpr static StringLiteral PATH = "path";
-
-        constexpr static StringLiteral KIND_BUILTIN = "builtin";
-        constexpr static StringLiteral KIND_FILESYSTEM = "filesystem";
-
-        virtual StringView type_name() const override;
-        virtual View<StringView> valid_fields() const override;
-
-        virtual Optional<std::unique_ptr<RegistryImpl>> visit_null(Json::Reader&) override;
-        virtual Optional<std::unique_ptr<RegistryImpl>> visit_object(Json::Reader&, const Json::Object&) override;
-
-        static RegistryImplDeserializer instance;
-    };
-
-    struct RegistryDeserializer final : Json::IDeserializer<Registry>
-    {
-        constexpr static StringLiteral PACKAGES = "packages";
-
-        virtual StringView type_name() const override;
-        virtual View<StringView> valid_fields() const override;
-
-        virtual Optional<Registry> visit_object(Json::Reader&, const Json::Object&) override;
-    };
-
-    struct ConfigurationDeserializer final : Json::IDeserializer<Configuration>
-    {
-        virtual StringView type_name() const override { return "a configuration object"; }
-
-        constexpr static StringLiteral DEFAULT_REGISTRY = "default-registry";
-        constexpr static StringLiteral REGISTRIES = "registries";
-        virtual View<StringView> valid_fields() const override
-        {
-            constexpr static StringView t[] = {DEFAULT_REGISTRY, REGISTRIES};
-            return t;
-        }
-
-        virtual Optional<Configuration> visit_object(Json::Reader& r, const Json::Object& obj) override;
-
-        ConfigurationDeserializer(const VcpkgCmdArguments& args);
-
-    private:
-        bool print_json;
-
-        bool registries_enabled;
-    };
-}
diff --git a/toolsrc/include/vcpkg/fwd/registries.h b/toolsrc/include/vcpkg/fwd/registries.h
index 7352c429d..73783cc6b 100644
--- a/toolsrc/include/vcpkg/fwd/registries.h
+++ b/toolsrc/include/vcpkg/fwd/registries.h
@@ -2,7 +2,8 @@
 
 namespace vcpkg
 {
-    struct RegistryImpl;
+    struct RegistryEntry;
+    struct RegistryImplementation;
     struct Registry;
     struct RegistrySet;
 }
diff --git a/toolsrc/include/vcpkg/portfileprovider.h b/toolsrc/include/vcpkg/portfileprovider.h
index b93c58e8c..fde28b0df 100644
--- a/toolsrc/include/vcpkg/portfileprovider.h
+++ b/toolsrc/include/vcpkg/portfileprovider.h
@@ -5,6 +5,7 @@
 #include <vcpkg/base/expected.h>
 #include <vcpkg/base/util.h>
 
+#include <vcpkg/registries.h>
 #include <vcpkg/sourceparagraph.h>
 #include <vcpkg/versions.h>
 
@@ -41,21 +42,19 @@ namespace vcpkg::PortFileProvider
 
     struct IVersionedPortfileProvider
     {
-        virtual const std::vector<vcpkg::Versions::VersionSpec>& get_port_versions(StringView port_name) const = 0;
+        virtual View<VersionT> get_port_versions(StringView port_name) const = 0;
         virtual ~IVersionedPortfileProvider() = default;
 
         virtual ExpectedS<const SourceControlFileLocation&> get_control_file(
-            const vcpkg::Versions::VersionSpec& version_spec) const = 0;
+            const Versions::VersionSpec& version_spec) const = 0;
     };
 
     struct IBaselineProvider
     {
-        virtual ~IBaselineProvider() = default;
-
         virtual Optional<VersionT> get_baseline_version(StringView port_name) const = 0;
+        virtual ~IBaselineProvider() = default;
     };
 
-    std::unique_ptr<IBaselineProvider> make_baseline_provider(const vcpkg::VcpkgPaths& paths);
-    std::unique_ptr<IBaselineProvider> make_baseline_provider(const vcpkg::VcpkgPaths& paths, StringView baseline);
-    std::unique_ptr<IVersionedPortfileProvider> make_versioned_portfile_provider(const vcpkg::VcpkgPaths& paths);
+    std::unique_ptr<IBaselineProvider> make_baseline_provider(const VcpkgPaths&);
+    std::unique_ptr<IVersionedPortfileProvider> make_versioned_portfile_provider(const VcpkgPaths&);
 }
diff --git a/toolsrc/include/vcpkg/registries.h b/toolsrc/include/vcpkg/registries.h
index 1f8c0f393..006bc789d 100644
--- a/toolsrc/include/vcpkg/registries.h
+++ b/toolsrc/include/vcpkg/registries.h
@@ -1,8 +1,11 @@
 #pragma once
 
+#include <vcpkg/fwd/configuration.h>
+#include <vcpkg/fwd/registries.h>
 #include <vcpkg/fwd/vcpkgpaths.h>
 
 #include <vcpkg/base/files.h>
+#include <vcpkg/base/jsonreader.h>
 #include <vcpkg/base/stringview.h>
 #include <vcpkg/base/view.h>
 
@@ -17,40 +20,45 @@ namespace vcpkg
 {
     struct RegistryEntry
     {
-        // returns fs::path() if version doesn't exist
-        virtual fs::path get_port_directory(const VcpkgPaths& paths, const VersionT& version) const = 0;
+        virtual View<VersionT> get_port_versions() const = 0;
+
+        virtual ExpectedS<fs::path> get_path_to_version(const VcpkgPaths& paths, const VersionT& version) const = 0;
 
         virtual ~RegistryEntry() = default;
     };
 
-    struct RegistryImpl
+    struct RegistryImplementation
     {
         // returns nullptr if the port doesn't exist
         virtual std::unique_ptr<RegistryEntry> get_port_entry(const VcpkgPaths& paths, StringView port_name) const = 0;
+
         // appends the names of the ports to the out parameter
+        // may result in duplicated port names; make sure to Util::sort_unique_erase at the end
         virtual void get_all_port_names(std::vector<std::string>& port_names, const VcpkgPaths& paths) const = 0;
 
         virtual Optional<VersionT> get_baseline_version(const VcpkgPaths& paths, StringView port_name) const = 0;
 
-        virtual ~RegistryImpl() = default;
+        virtual ~RegistryImplementation() = default;
     };
 
     struct Registry
     {
         // requires: static_cast<bool>(implementation)
-        Registry(std::vector<std::string>&& packages, std::unique_ptr<RegistryImpl>&& implementation);
+        Registry(std::vector<std::string>&& packages, std::unique_ptr<RegistryImplementation>&& implementation);
 
         Registry(std::vector<std::string>&&, std::nullptr_t) = delete;
 
         // always ordered lexicographically
         View<std::string> packages() const { return packages_; }
-        const RegistryImpl& implementation() const { return *implementation_; }
+        const RegistryImplementation& implementation() const { return *implementation_; }
+
+        static std::unique_ptr<RegistryImplementation> builtin_registry(std::string&& baseline = {});
 
-        static std::unique_ptr<RegistryImpl> builtin_registry();
+        friend RegistrySet; // for experimental_set_builtin_registry_baseline
 
     private:
         std::vector<std::string> packages_;
-        std::unique_ptr<RegistryImpl> implementation_;
+        std::unique_ptr<RegistryImplementation> implementation_;
     };
 
     // this type implements the registry fall back logic from the registries RFC:
@@ -65,20 +73,34 @@ namespace vcpkg
 
         // finds the correct registry for the port name
         // Returns the null pointer if there is no registry set up for that name
-        const RegistryImpl* registry_for_port(StringView port_name) const;
+        const RegistryImplementation* registry_for_port(StringView port_name) const;
+        Optional<VersionT> baseline_for_port(const VcpkgPaths& paths, StringView port_name) const;
 
         View<Registry> registries() const { return registries_; }
 
-        const RegistryImpl* default_registry() const { return default_registry_.get(); }
+        const RegistryImplementation* default_registry() const { return default_registry_.get(); }
 
         // TODO: figure out how to get this to return an error (or maybe it should be a warning?)
         void add_registry(Registry&& r);
-        void set_default_registry(std::unique_ptr<RegistryImpl>&& r);
+        void set_default_registry(std::unique_ptr<RegistryImplementation>&& r);
         void set_default_registry(std::nullptr_t r);
 
+        // this exists in order to allow versioning and registries to be developed and tested separately
+        void experimental_set_builtin_registry_baseline(StringView baseline) const;
+
+        // returns whether the registry set has any modifications to the default
+        // (i.e., whether `default_registry` was set, or `registries` had any entries)
+        // for checking against the registry feature flag.
+        bool has_modifications() const;
+
     private:
-        std::unique_ptr<RegistryImpl> default_registry_;
+        std::unique_ptr<RegistryImplementation> default_registry_;
         std::vector<Registry> registries_;
     };
 
+    std::unique_ptr<Json::IDeserializer<std::unique_ptr<RegistryImplementation>>>
+    get_registry_implementation_deserializer(const fs::path& configuration_directory);
+
+    std::unique_ptr<Json::IDeserializer<std::vector<Registry>>> get_registry_array_deserializer(
+        const fs::path& configuration_directory);
 }
diff --git a/toolsrc/include/vcpkg/sourceparagraph.h b/toolsrc/include/vcpkg/sourceparagraph.h
index ee07826bb..50417b8ad 100644
--- a/toolsrc/include/vcpkg/sourceparagraph.h
+++ b/toolsrc/include/vcpkg/sourceparagraph.h
@@ -116,8 +116,8 @@ namespace vcpkg
     Json::Object serialize_debug_manifest(const SourceControlFile& scf);
 
     /// <summary>
-    /// Full metadata of a package: core and other features. As well as the location the SourceControlFile was
-    /// loaded from.
+    /// Full metadata of a package: core and other features,
+    /// as well as the port directory the SourceControlFile was loaded from
     /// </summary>
     struct SourceControlFileLocation
     {
diff --git a/toolsrc/include/vcpkg/vcpkgcmdarguments.h b/toolsrc/include/vcpkg/vcpkgcmdarguments.h
index f18f4843c..2677e6811 100644
--- a/toolsrc/include/vcpkg/vcpkgcmdarguments.h
+++ b/toolsrc/include/vcpkg/vcpkgcmdarguments.h
@@ -134,6 +134,8 @@ namespace vcpkg
         std::unique_ptr<std::string> scripts_root_dir;
         constexpr static StringLiteral BUILTIN_PORTS_ROOT_DIR_ARG = "x-builtin-ports-root";
         std::unique_ptr<std::string> builtin_ports_root_dir;
+        constexpr static StringLiteral BUILTIN_PORT_VERSIONS_DIR_ARG = "x-builtin-port-versions-dir";
+        std::unique_ptr<std::string> builtin_port_versions_dir;
 
         constexpr static StringLiteral DEFAULT_VISUAL_STUDIO_PATH_ENV = "VCPKG_VISUAL_STUDIO_PATH";
         std::unique_ptr<std::string> default_visual_studio_path;
diff --git a/toolsrc/include/vcpkg/vcpkgpaths.h b/toolsrc/include/vcpkg/vcpkgpaths.h
index e71f28b0a..1f562c151 100644
--- a/toolsrc/include/vcpkg/vcpkgpaths.h
+++ b/toolsrc/include/vcpkg/vcpkgpaths.h
@@ -92,6 +92,7 @@ namespace vcpkg
         fs::path scripts;
         fs::path prefab;
         fs::path builtin_ports;
+        fs::path builtin_port_versions;
 
         fs::path tools;
         fs::path buildsystems;
@@ -116,7 +117,7 @@ namespace vcpkg
         const fs::path& get_tool_exe(const std::string& tool) const;
         const std::string& get_tool_version(const std::string& tool) const;
 
-        // Git manipulation
+        // Git manipulation in the vcpkg directory
         fs::path git_checkout_baseline(Files::Filesystem& filesystem, StringView commit_sha) const;
         fs::path git_checkout_port(Files::Filesystem& filesystem, StringView port_name, StringView git_tree) const;
         ExpectedS<std::string> git_show(const std::string& treeish, const fs::path& dot_git_dir) const;
diff --git a/toolsrc/include/vcpkg/versiondeserializers.h b/toolsrc/include/vcpkg/versiondeserializers.h
index 2efe340d7..02696d39b 100644
--- a/toolsrc/include/vcpkg/versiondeserializers.h
+++ b/toolsrc/include/vcpkg/versiondeserializers.h
@@ -10,13 +10,6 @@
 
 namespace vcpkg
 {
-    struct VersionDbEntry
-    {
-        VersionT version;
-        Versions::Scheme scheme = Versions::Scheme::String;
-        std::string git_tree;
-    };
-
     Json::IDeserializer<VersionT>& get_versiont_deserializer_instance();
     std::unique_ptr<Json::IDeserializer<std::string>> make_version_deserializer(StringLiteral type_name);
 
@@ -44,12 +37,4 @@ namespace vcpkg
                                    const std::string& version,
                                    int port_version,
                                    bool always_emit_port_version = false);
-
-    ExpectedS<std::map<std::string, VersionT, std::less<>>> parse_baseline_file(Files::Filesystem& fs,
-                                                                                StringView baseline_name,
-                                                                                const fs::path& baseline_file_path);
-
-    ExpectedS<std::vector<VersionDbEntry>> parse_versions_file(Files::Filesystem& fs,
-                                                               StringView port_name,
-                                                               const fs::path& versions_file_path);
 }