[vcpkg] Fix regression in error messages with registries/versioning (#15709)

* [vcpkg] Fix regression in error messages with registries/versioning

* [vcpkg] Clean up redundant re-parsing with try_load_overlay_port

* [vcpkg] Deduplicate PathsPortfileProvider

* [vcpkg] Restore uses of Checks::exit_maybe_upgrade after merge

5 files changed, 309 insertions, 418 deletions

diff --git a/toolsrc/src/vcpkg-test/dependencies.cpp b/toolsrc/src/vcpkg-test/dependencies.cpp
index 8f936abc3..c6241ebe0 100644
--- a/toolsrc/src/vcpkg-test/dependencies.cpp
+++ b/toolsrc/src/vcpkg-test/dependencies.cpp
@@ -78,6 +78,11 @@ struct MockVersionedPortfileProvider : PortFileProvider::IVersionedPortfileProvi
         }
         return it2->second;
     }
+
+    virtual void load_all_control_files(std::map<std::string, const SourceControlFileLocation*>&) const override
+    {
+        Checks::unreachable(VCPKG_LINE_INFO);
+    }
 };
 
 using Versions::Constraint;
@@ -190,6 +195,11 @@ struct MockOverlayProvider : PortFileProvider::IOverlayProvider, Util::ResourceB
         return it->second;
     }
 
+    virtual void load_all_control_files(std::map<std::string, const SourceControlFileLocation*>&) const override
+    {
+        Checks::unreachable(VCPKG_LINE_INFO);
+    }
+
 private:
     std::map<std::string, SourceControlFileLocation, std::less<>> mappings;
 };
diff --git a/toolsrc/src/vcpkg/commands.portsdiff.cpp b/toolsrc/src/vcpkg/commands.portsdiff.cpp
index 38a1c91cb..18e3a9d91 100644
--- a/toolsrc/src/vcpkg/commands.portsdiff.cpp
+++ b/toolsrc/src/vcpkg/commands.portsdiff.cpp
@@ -104,8 +104,7 @@ namespace vcpkg::Commands::PortsDiff
         System::cmd_execute_and_capture_output(cmd, System::get_clean_environment());
         System::cmd_execute_and_capture_output(System::Command(git_exe).string_arg("reset"),
                                                System::get_clean_environment());
-        const auto ports_at_commit =
-            Paragraphs::load_overlay_ports(paths, temp_checkout_path / ports_dir_name_as_string);
+        const auto ports_at_commit = Paragraphs::load_overlay_ports(fs, temp_checkout_path / ports_dir_name_as_string);
         std::map<std::string, VersionT> names_and_versions;
         for (auto&& port : ports_at_commit)
         {
diff --git a/toolsrc/src/vcpkg/paragraphs.cpp b/toolsrc/src/vcpkg/paragraphs.cpp
index 2d28c49d7..dc93ee2db 100644
--- a/toolsrc/src/vcpkg/paragraphs.cpp
+++ b/toolsrc/src/vcpkg/paragraphs.cpp
@@ -353,6 +353,7 @@ namespace vcpkg::Paragraphs
                 error_info->name = fs::u8string(path.filename());
                 error_info->error = Strings::format(
                     "Failed to load manifest file for port: %s\n", fs::u8string(path_to_manifest), ec.message());
+                return error_info;
             }
 
             return res;
@@ -491,13 +492,10 @@ namespace vcpkg::Paragraphs
         return std::move(results.paragraphs);
     }
 
-    std::vector<SourceControlFileLocation> load_overlay_ports(const VcpkgPaths& paths, const fs::path& directory)
+    std::vector<SourceControlFileLocation> load_overlay_ports(const Files::Filesystem& fs, const fs::path& directory)
     {
         LoadResults ret;
 
-        std::vector<std::string> port_names;
-
-        const auto& fs = paths.get_filesystem();
         auto port_dirs = fs.get_files_non_recursive(directory);
         Util::sort(port_dirs);
 
diff --git a/toolsrc/src/vcpkg/portfileprovider.cpp b/toolsrc/src/vcpkg/portfileprovider.cpp
index d2a37c4b3..47aa2976c 100644
--- a/toolsrc/src/vcpkg/portfileprovider.cpp
+++ b/toolsrc/src/vcpkg/portfileprovider.cpp
@@ -57,251 +57,37 @@ namespace vcpkg::PortFileProvider
         return Util::fmap(ports, [](auto&& kvpair) -> const SourceControlFileLocation* { return &kvpair.second; });
     }
 
-    PathsPortFileProvider::PathsPortFileProvider(const VcpkgPaths& paths_,
-                                                 const std::vector<std::string>& overlay_ports_)
-        : paths(paths_)
+    PathsPortFileProvider::PathsPortFileProvider(const VcpkgPaths& paths, const std::vector<std::string>& overlay_ports)
+        : m_baseline(make_baseline_provider(paths))
+        , m_versioned(make_versioned_portfile_provider(paths))
+        , m_overlay(make_overlay_provider(paths, overlay_ports))
     {
-        auto& fs = paths.get_filesystem();
-        for (auto&& overlay_path : overlay_ports_)
-        {
-            if (!overlay_path.empty())
-            {
-                auto overlay = fs::u8path(overlay_path);
-                if (overlay.is_absolute())
-                {
-                    overlay = fs.canonical(VCPKG_LINE_INFO, overlay);
-                }
-                else
-                {
-                    overlay = fs.canonical(VCPKG_LINE_INFO, paths.original_cwd / overlay);
-                }
-
-                Debug::print("Using overlay: ", fs::u8string(overlay), "\n");
-
-                Checks::check_exit(
-                    VCPKG_LINE_INFO, fs.exists(overlay), "Error: Path \"%s\" does not exist", fs::u8string(overlay));
-
-                Checks::check_exit(VCPKG_LINE_INFO,
-                                   fs::is_directory(fs.status(VCPKG_LINE_INFO, overlay)),
-                                   "Error: Path \"%s\" must be a directory",
-                                   overlay.string());
-
-                overlay_ports.emplace_back(overlay);
-            }
-        }
-    }
-
-    static std::unique_ptr<OverlayRegistryEntry> try_load_overlay_port(const Files::Filesystem& fs,
-                                                                       View<fs::path> overlay_ports,
-                                                                       const std::string& spec)
-    {
-        for (auto&& ports_dir : overlay_ports)
-        {
-            // Try loading individual port
-            if (Paragraphs::is_port_directory(fs, ports_dir))
-            {
-                auto maybe_scf = Paragraphs::try_load_port(fs, ports_dir);
-                if (auto scfp = maybe_scf.get())
-                {
-                    auto& scf = *scfp;
-                    if (scf->core_paragraph->name == spec)
-                    {
-                        return std::make_unique<OverlayRegistryEntry>(fs::path(ports_dir), scf->to_versiont());
-                    }
-                }
-                else
-                {
-                    print_error_message(maybe_scf.error());
-                    Checks::exit_maybe_upgrade(
-                        VCPKG_LINE_INFO, "Error: Failed to load port %s from %s", spec, fs::u8string(ports_dir));
-                }
-
-                continue;
-            }
-
-            auto ports_spec = ports_dir / fs::u8path(spec);
-            if (Paragraphs::is_port_directory(fs, ports_spec))
-            {
-                auto found_scf = Paragraphs::try_load_port(fs, ports_spec);
-                if (auto scfp = found_scf.get())
-                {
-                    auto& scf = *scfp;
-                    if (scf->core_paragraph->name == spec)
-                    {
-                        return std::make_unique<OverlayRegistryEntry>(std::move(ports_spec), scf->to_versiont());
-                    }
-                    Checks::exit_maybe_upgrade(VCPKG_LINE_INFO,
-                                               "Error: Failed to load port from %s: names did not match: '%s' != '%s'",
-                                               fs::u8string(ports_spec),
-                                               spec,
-                                               scf->core_paragraph->name);
-                }
-                else
-                {
-                    print_error_message(found_scf.error());
-                    Checks::exit_maybe_upgrade(
-                        VCPKG_LINE_INFO, "Error: Failed to load port %s from %s", spec, fs::u8string(ports_dir));
-                }
-            }
-        }
-        return nullptr;
-    }
-
-    static std::pair<std::unique_ptr<RegistryEntry>, Optional<VersionT>> try_load_registry_port_and_baseline(
-        const VcpkgPaths& paths, const std::string& spec)
-    {
-        if (auto registry = paths.get_configuration().registry_set.registry_for_port(spec))
-        {
-            auto entry = registry->get_port_entry(paths, spec);
-            auto maybe_baseline = registry->get_baseline_version(paths, spec);
-            if (entry)
-            {
-                if (!maybe_baseline)
-                {
-                    if (entry->get_port_versions().size() == 1)
-                    {
-                        maybe_baseline = entry->get_port_versions()[0];
-                    }
-                }
-                return {std::move(entry), std::move(maybe_baseline)};
-            }
-            else
-            {
-                Debug::print("Failed to find port `", spec, "` in registry: no entry found.\n");
-            }
-        }
-        else
-        {
-            Debug::print("Failed to find registry for port: `", spec, "`.\n");
-        }
-
-        return {nullptr, nullopt};
     }
 
     ExpectedS<const SourceControlFileLocation&> PathsPortFileProvider::get_control_file(const std::string& spec) const
     {
-        auto cache_it = cache.find(spec);
-        if (cache_it == cache.end())
+        auto maybe_scfl = m_overlay->get_control_file(spec);
+        if (auto scfl = maybe_scfl.get())
         {
-            const auto& fs = paths.get_filesystem();
-
-            std::unique_ptr<RegistryEntry> port;
-            VersionT port_version;
-
-            auto maybe_overlay_port = try_load_overlay_port(fs, overlay_ports, spec);
-            if (maybe_overlay_port)
-            {
-                port_version = maybe_overlay_port->version;
-                port = std::move(maybe_overlay_port);
-            }
-            else
-            {
-                auto maybe_registry_port = try_load_registry_port_and_baseline(paths, spec);
-                port = std::move(maybe_registry_port.first);
-                if (auto version = maybe_registry_port.second.get())
-                {
-                    port_version = std::move(*version);
-                }
-                else if (port)
-                {
-                    return std::string("No baseline version available.");
-                }
-            }
-
-            if (port)
-            {
-                auto maybe_port_path = port->get_path_to_version(paths, port_version);
-                if (!maybe_port_path.has_value())
-                {
-                    return std::move(maybe_port_path.error());
-                }
-                auto port_path = std::move(maybe_port_path).value_or_exit(VCPKG_LINE_INFO);
-
-                auto maybe_scfl = Paragraphs::try_load_port(fs, port_path);
-                if (auto p = maybe_scfl.get())
-                {
-                    auto maybe_error = (*p)->check_against_feature_flags(port_path, paths.get_feature_flags());
-                    if (maybe_error) return std::move(*maybe_error.get());
-
-                    cache_it =
-                        cache.emplace(spec, SourceControlFileLocation{std::move(*p), std::move(port_path)}).first;
-                }
-                else
-                {
-                    return Strings::format("Error: when loading port `%s` from directory `%s`:\n%s\n",
-                                           spec,
-                                           fs::u8string(port_path),
-                                           maybe_scfl.error()->error);
-                }
-            }
+            return *scfl;
         }
-
-        if (cache_it == cache.end())
+        auto maybe_baseline = m_baseline->get_baseline_version(spec);
+        if (auto baseline = maybe_baseline.get())
         {
-            return std::string("Port definition not found");
+            return m_versioned->get_control_file({spec, *baseline});
         }
         else
         {
-            return cache_it->second;
+            return Strings::concat("Error: unable to get baseline for port ", spec);
         }
     }
 
     std::vector<const SourceControlFileLocation*> PathsPortFileProvider::load_all_control_files() const
     {
-        // Reload cache with ports contained in all ports_dirs
-        cache.clear();
-        std::vector<const SourceControlFileLocation*> ret;
-
-        for (const fs::path& ports_dir : overlay_ports)
-        {
-            // Try loading individual port
-            if (Paragraphs::is_port_directory(paths.get_filesystem(), ports_dir))
-            {
-                auto maybe_scf = Paragraphs::try_load_port(paths.get_filesystem(), ports_dir);
-                if (auto scf = maybe_scf.get())
-                {
-                    auto port_name = scf->get()->core_paragraph->name;
-                    if (cache.find(port_name) == cache.end())
-                    {
-                        auto scfl = SourceControlFileLocation{std::move(*scf), ports_dir};
-                        auto it = cache.emplace(std::move(port_name), std::move(scfl));
-                        ret.emplace_back(&it.first->second);
-                    }
-                }
-                else
-                {
-                    print_error_message(maybe_scf.error());
-                    Checks::exit_maybe_upgrade(
-                        VCPKG_LINE_INFO, "Error: Failed to load port from %s", fs::u8string(ports_dir));
-                }
-                continue;
-            }
-
-            // Try loading all ports inside ports_dir
-            auto found_scfls = Paragraphs::load_overlay_ports(paths, ports_dir);
-            for (auto&& scfl : found_scfls)
-            {
-                auto port_name = scfl.source_control_file->core_paragraph->name;
-                if (cache.find(port_name) == cache.end())
-                {
-                    auto it = cache.emplace(std::move(port_name), std::move(scfl));
-                    ret.emplace_back(&it.first->second);
-                }
-            }
-        }
-
-        auto all_ports = Paragraphs::load_all_registry_ports(paths);
-        for (auto&& scfl : all_ports)
-        {
-            auto port_name = scfl.source_control_file->core_paragraph->name;
-            if (cache.find(port_name) == cache.end())
-            {
-                auto it = cache.emplace(port_name, std::move(scfl));
-                ret.emplace_back(&it.first->second);
-            }
-        }
-
-        return ret;
+        std::map<std::string, const SourceControlFileLocation*> m;
+        m_overlay->load_all_control_files(m);
+        m_versioned->load_all_control_files(m);
+        return Util::fmap(m, [](const auto& p) { return p.second; });
     }
 
     namespace
@@ -334,104 +120,158 @@ namespace vcpkg::PortFileProvider
         {
             VersionedPortfileProviderImpl(const VcpkgPaths& paths_) : paths(paths_) { }
 
-            virtual View<VersionT> get_port_versions(StringView port_name) const override
+            const ExpectedS<std::unique_ptr<RegistryEntry>>& entry(StringView name) const
             {
-                auto entry_it = m_entry_cache.find(port_name.to_string());
-                if (entry_it != m_entry_cache.end())
-                {
-                    return entry_it->second->get_port_versions();
-                }
-
-                auto entry = try_load_registry_port_and_baseline(paths, port_name.to_string());
-                if (!entry.first)
+                auto entry_it = m_entry_cache.find(name);
+                if (entry_it == m_entry_cache.end())
                 {
-                    Checks::exit_maybe_upgrade(
-                        VCPKG_LINE_INFO, "Error: Could not find a definition for port %s", port_name);
+                    if (auto reg = paths.get_configuration().registry_set.registry_for_port(name))
+                    {
+                        if (auto entry = reg->get_port_entry(paths, name))
+                        {
+                            entry_it = m_entry_cache.emplace(name.to_string(), std::move(entry)).first;
+                        }
+                        else
+                        {
+                            entry_it =
+                                m_entry_cache
+                                    .emplace(name.to_string(),
+                                             Strings::concat("Error: Could not find a definition for port ", name))
+                                    .first;
+                        }
+                    }
+                    else
+                    {
+                        entry_it = m_entry_cache
+                                       .emplace(name.to_string(),
+                                                Strings::concat("Error: no registry configured for port ", name))
+                                       .first;
+                    }
                 }
-                auto it = m_entry_cache.emplace(port_name.to_string(), std::move(entry.first));
-                return it.first->second->get_port_versions();
+                return entry_it->second;
             }
 
-            ExpectedS<const SourceControlFileLocation&> get_control_file(const VersionSpec& version_spec) const override
+            virtual View<VersionT> get_port_versions(StringView port_name) const override
             {
-                auto cache_it = m_control_cache.find(version_spec);
-                if (cache_it != m_control_cache.end())
-                {
-                    return cache_it->second;
-                }
+                return entry(port_name).value_or_exit(VCPKG_LINE_INFO)->get_port_versions();
+            }
 
-                auto entry_it = m_entry_cache.find(version_spec.port_name);
-                if (entry_it == m_entry_cache.end())
+            ExpectedS<std::unique_ptr<SourceControlFileLocation>> load_control_file(
+                const VersionSpec& version_spec) const
+            {
+                const auto& maybe_ent = entry(version_spec.port_name);
+                if (auto ent = maybe_ent.get())
                 {
-                    auto reg_for_port =
-                        paths.get_configuration().registry_set.registry_for_port(version_spec.port_name);
-
-                    if (!reg_for_port)
+                    auto maybe_path = ent->get()->get_path_to_version(paths, version_spec.version);
+                    if (auto path = maybe_path.get())
                     {
-                        return Strings::format("Error: no registry set up for port %s", version_spec.port_name);
+                        auto maybe_control_file = Paragraphs::try_load_port(paths.get_filesystem(), *path);
+                        if (auto scf = maybe_control_file.get())
+                        {
+                            if (scf->get()->core_paragraph->name == version_spec.port_name)
+                            {
+                                return std::make_unique<SourceControlFileLocation>(std::move(*scf), std::move(*path));
+                            }
+                            else
+                            {
+                                return Strings::format("Error: Failed to load port from %s: names did "
+                                                       "not match: '%s' != '%s'",
+                                                       fs::u8string(*path),
+                                                       version_spec.port_name,
+                                                       scf->get()->core_paragraph->name);
+                            }
+                        }
+                        else
+                        {
+                            // This should change to a soft error when ParseExpected is eliminated.
+                            print_error_message(maybe_control_file.error());
+                            Checks::exit_maybe_upgrade(VCPKG_LINE_INFO,
+                                                       "Error: Failed to load port %s from %s",
+                                                       version_spec.port_name,
+                                                       fs::u8string(*path));
+                        }
+                    }
+                    else
+                    {
+                        return maybe_path.error();
                     }
-
-                    auto entry = reg_for_port->get_port_entry(paths, version_spec.port_name);
-                    entry_it = m_entry_cache.emplace(version_spec.port_name, std::move(entry)).first;
                 }
+                return maybe_ent.error();
+            }
 
-                auto maybe_path = entry_it->second->get_path_to_version(paths, version_spec.version);
-                if (!maybe_path.has_value())
+            virtual ExpectedS<const SourceControlFileLocation&> get_control_file(
+                const VersionSpec& version_spec) const override
+            {
+                auto it = m_control_cache.find(version_spec);
+                if (it == m_control_cache.end())
                 {
-                    return std::move(maybe_path).error();
+                    it = m_control_cache.emplace(version_spec, load_control_file(version_spec)).first;
                 }
-                auto& port_directory = *maybe_path.get();
+                return it->second.map([](const auto& x) -> const SourceControlFileLocation& { return *x.get(); });
+            }
 
-                auto maybe_control_file = Paragraphs::try_load_port(paths.get_filesystem(), port_directory);
-                if (auto scf = maybe_control_file.get())
+            virtual void load_all_control_files(
+                std::map<std::string, const SourceControlFileLocation*>& out) const override
+            {
+                auto all_ports = Paragraphs::load_all_registry_ports(paths);
+                for (auto&& scfl : all_ports)
                 {
-                    if (scf->get()->core_paragraph->name == version_spec.port_name)
-                    {
-                        return m_control_cache
-                            .emplace(version_spec,
-                                     SourceControlFileLocation{std::move(*scf), std::move(port_directory)})
-                            .first->second;
-                    }
-                    return Strings::format("Error: Failed to load port from %s: names did not match: '%s' != '%s'",
-                                           fs::u8string(port_directory),
-                                           version_spec.port_name,
-                                           scf->get()->core_paragraph->name);
+                    auto port_name = scfl.source_control_file->core_paragraph->name;
+                    auto version = scfl.source_control_file->core_paragraph->to_versiont();
+                    auto it = m_control_cache
+                                  .emplace(VersionSpec{std::move(port_name), std::move(version)},
+                                           std::make_unique<SourceControlFileLocation>(std::move(scfl)))
+                                  .first;
+                    Checks::check_exit(VCPKG_LINE_INFO, it->second.has_value());
+                    out.emplace(it->first.port_name, it->second.get()->get());
                 }
-
-                print_error_message(maybe_control_file.error());
-                return Strings::format(
-                    "Error: Failed to load port %s from %s", version_spec.port_name, fs::u8string(port_directory));
             }
 
         private:
             const VcpkgPaths& paths; // TODO: remove this data member
-            mutable std::unordered_map<VersionSpec, SourceControlFileLocation, VersionSpecHasher> m_control_cache;
-            mutable std::map<std::string, std::unique_ptr<RegistryEntry>, std::less<>> m_entry_cache;
+            mutable std::
+                unordered_map<VersionSpec, ExpectedS<std::unique_ptr<SourceControlFileLocation>>, VersionSpecHasher>
+                    m_control_cache;
+            mutable std::map<std::string, ExpectedS<std::unique_ptr<RegistryEntry>>, std::less<>> m_entry_cache;
         };
 
         struct OverlayProviderImpl : IOverlayProvider, Util::ResourceBase
         {
             OverlayProviderImpl(const VcpkgPaths& paths, View<std::string> overlay_ports)
-                : paths(paths), m_overlay_ports(Util::fmap(overlay_ports, [&paths](const std::string& s) -> fs::path {
+                : m_fs(paths.get_filesystem())
+                , m_overlay_ports(Util::fmap(overlay_ports, [&paths](const std::string& s) -> fs::path {
                     return Files::combine(paths.original_cwd, fs::u8path(s));
                 }))
             {
+                for (auto&& overlay : m_overlay_ports)
+                {
+                    auto s_overlay = fs::u8string(overlay);
+                    Debug::print("Using overlay: ", s_overlay, "\n");
+
+                    Checks::check_exit(VCPKG_LINE_INFO,
+                                       fs::is_directory(m_fs.status(VCPKG_LINE_INFO, overlay)),
+                                       "Error: Overlay path \"%s\" must exist and must be a directory",
+                                       s_overlay);
+                }
             }
 
-            virtual Optional<const SourceControlFileLocation&> get_control_file(StringView port_name) const override
+            Optional<SourceControlFileLocation> load_port(StringView port_name) const
             {
-                auto it = m_overlay_cache.find(port_name);
-                if (it == m_overlay_cache.end())
+                auto s_port_name = port_name.to_string();
+
+                for (auto&& ports_dir : m_overlay_ports)
                 {
-                    auto s_port_name = port_name.to_string();
-                    auto maybe_overlay = try_load_overlay_port(paths.get_filesystem(), m_overlay_ports, s_port_name);
-                    Optional<SourceControlFileLocation> v;
-                    if (maybe_overlay)
+                    // Try loading individual port
+                    if (Paragraphs::is_port_directory(m_fs, ports_dir))
                     {
-                        auto maybe_scf = Paragraphs::try_load_port(paths.get_filesystem(), maybe_overlay->path);
-                        if (auto scf = maybe_scf.get())
+                        auto maybe_scf = Paragraphs::try_load_port(m_fs, ports_dir);
+                        if (auto scfp = maybe_scf.get())
                         {
-                            v = SourceControlFileLocation{std::move(*scf), maybe_overlay->path};
+                            auto& scf = *scfp;
+                            if (scf->core_paragraph->name == port_name)
+                            {
+                                return SourceControlFileLocation{std::move(scf), fs::path(ports_dir)};
+                            }
                         }
                         else
                         {
@@ -439,16 +279,94 @@ namespace vcpkg::PortFileProvider
                             Checks::exit_maybe_upgrade(VCPKG_LINE_INFO,
                                                        "Error: Failed to load port %s from %s",
                                                        port_name,
-                                                       fs::u8string(maybe_overlay->path));
+                                                       fs::u8string(ports_dir));
+                        }
+
+                        continue;
+                    }
+
+                    auto ports_spec = ports_dir / fs::u8path(port_name);
+                    if (Paragraphs::is_port_directory(m_fs, ports_spec))
+                    {
+                        auto found_scf = Paragraphs::try_load_port(m_fs, ports_spec);
+                        if (auto scfp = found_scf.get())
+                        {
+                            auto& scf = *scfp;
+                            if (scf->core_paragraph->name == port_name)
+                            {
+                                return SourceControlFileLocation{std::move(scf), std::move(ports_spec)};
+                            }
+                            Checks::exit_maybe_upgrade(
+                                VCPKG_LINE_INFO,
+                                "Error: Failed to load port from %s: names did not match: '%s' != '%s'",
+                                fs::u8string(ports_spec),
+                                port_name,
+                                scf->core_paragraph->name);
+                        }
+                        else
+                        {
+                            print_error_message(found_scf.error());
+                            Checks::exit_maybe_upgrade(VCPKG_LINE_INFO,
+                                                       "Error: Failed to load port %s from %s",
+                                                       port_name,
+                                                       fs::u8string(ports_dir));
                         }
                     }
-                    it = m_overlay_cache.emplace(std::move(s_port_name), std::move(v)).first;
+                }
+                return nullopt;
+            }
+
+            virtual Optional<const SourceControlFileLocation&> get_control_file(StringView port_name) const override
+            {
+                auto it = m_overlay_cache.find(port_name);
+                if (it == m_overlay_cache.end())
+                {
+                    it = m_overlay_cache.emplace(port_name.to_string(), load_port(port_name)).first;
                 }
                 return it->second;
             }
 
+            virtual void load_all_control_files(
+                std::map<std::string, const SourceControlFileLocation*>& out) const override
+            {
+                for (auto&& ports_dir : m_overlay_ports)
+                {
+                    // Try loading individual port
+                    if (Paragraphs::is_port_directory(m_fs, ports_dir))
+                    {
+                        auto maybe_scf = Paragraphs::try_load_port(m_fs, ports_dir);
+                        if (auto scfp = maybe_scf.get())
+                        {
+                            SourceControlFileLocation scfl{std::move(*scfp), fs::path(ports_dir)};
+                            auto name = scfl.source_control_file->core_paragraph->name;
+                            auto it = m_overlay_cache.emplace(std::move(name), std::move(scfl)).first;
+                            Checks::check_exit(VCPKG_LINE_INFO, it->second.get());
+                            out.emplace(it->first, it->second.get());
+                        }
+                        else
+                        {
+                            print_error_message(maybe_scf.error());
+                            Checks::exit_maybe_upgrade(
+                                VCPKG_LINE_INFO, "Error: Failed to load port from %s", fs::u8string(ports_dir));
+                        }
+
+                        continue;
+                    }
+
+                    // Try loading all ports inside ports_dir
+                    auto found_scfls = Paragraphs::load_overlay_ports(m_fs, ports_dir);
+                    for (auto&& scfl : found_scfls)
+                    {
+                        auto name = scfl.source_control_file->core_paragraph->name;
+                        auto it = m_overlay_cache.emplace(std::move(name), std::move(scfl)).first;
+                        Checks::check_exit(VCPKG_LINE_INFO, it->second.get());
+                        out.emplace(it->first, it->second.get());
+                    }
+                }
+            }
+
         private:
-            const VcpkgPaths& paths;
+            const Files::Filesystem& m_fs;
             const std::vector<fs::path> m_overlay_ports;
             mutable std::map<std::string, Optional<SourceControlFileLocation>, std::less<>> m_overlay_cache;
         };
diff --git a/toolsrc/src/vcpkg/registries.cpp b/toolsrc/src/vcpkg/registries.cpp
index 4f6c8c0c2..44a6314ab 100644
--- a/toolsrc/src/vcpkg/registries.cpp
+++ b/toolsrc/src/vcpkg/registries.cpp
@@ -27,8 +27,6 @@ namespace
     // when `BuiltinRegistryEntry` is using a port tree, it uses the scfl
     struct GitRegistryEntry final : RegistryEntry
     {
-        explicit GitRegistryEntry(std::string&& port_name) : port_name(port_name) { }
-
         View<VersionT> get_port_versions() const override { return port_versions; }
         ExpectedS<fs::path> get_path_to_version(const VcpkgPaths&, const VersionT& version) const override;
 
@@ -103,36 +101,24 @@ namespace
 
     struct BuiltinRegistryEntry final : RegistryEntry
     {
-        explicit BuiltinRegistryEntry(std::unique_ptr<GitRegistryEntry>&& entry)
-            : git_entry(std::move(entry)), scfl(nullptr)
-        {
-        }
-        explicit BuiltinRegistryEntry(std::unique_ptr<SourceControlFileLocation>&& scfl_)
-            : git_entry(nullptr), scfl(std::move(scfl_)), scfl_version(scfl->to_versiont())
-        {
-        }
-
-        View<VersionT> get_port_versions() const override
+        View<VersionT> get_port_versions() const override { return {&version, 1}; }
+        ExpectedS<fs::path> get_path_to_version(const VcpkgPaths&, const VersionT& v) const override
         {
-            if (git_entry)
+            if (v == version)
             {
-                return git_entry->port_versions;
-            }
-            else
-            {
-                return {&scfl_version, 1};
+                return path;
             }
+            return {Strings::format("Error: no version entry for %s at version %s.\n"
+                                    "We are currently using the version in the ports tree (%s).",
+                                    name,
+                                    v.to_string(),
+                                    version.to_string()),
+                    expected_right_tag};
         }
-        ExpectedS<fs::path> get_path_to_version(const VcpkgPaths&, const VersionT& version) const override;
-
-        // exactly one of these two shall be null
 
-        // if we find a versions.json, this shall be non-null and BuiltinRegistryEntry uses git_entry's implementation
-        std::unique_ptr<GitRegistryEntry> git_entry;
-        // otherwise, if we don't find a versions.json,
-        // we fall back to just using the version in the ports directory, and this is the non-null one
-        std::unique_ptr<SourceControlFileLocation> scfl;
-        VersionT scfl_version; // this exists so that we can return a pointer to it
+        std::string name;
+        fs::path path;
+        VersionT version;
     };
 
     struct FilesystemRegistryEntry final : RegistryEntry
@@ -153,7 +139,10 @@ namespace
 
     struct BuiltinRegistry final : RegistryImplementation
     {
-        BuiltinRegistry(std::string&& baseline) : m_baseline_identifier(std::move(baseline)) { }
+        BuiltinRegistry(std::string&& baseline) : m_baseline_identifier(std::move(baseline))
+        {
+            Debug::print("BuiltinRegistry initialized with: \"", m_baseline_identifier, "\"\n");
+        }
 
         std::unique_ptr<RegistryEntry> get_port_entry(const VcpkgPaths& paths, StringView port_name) const override;
 
@@ -205,7 +194,7 @@ namespace
     };
 
     fs::path relative_path_to_versions(StringView port_name);
-    ExpectedS<std::vector<VersionDbEntry>> load_versions_file(Files::Filesystem& fs,
+    ExpectedS<std::vector<VersionDbEntry>> load_versions_file(const Files::Filesystem& fs,
                                                               VersionDbType vdb,
                                                               const fs::path& port_versions,
                                                               StringView port_name,
@@ -247,31 +236,34 @@ namespace
     // { BuiltinRegistry::RegistryImplementation
     std::unique_ptr<RegistryEntry> BuiltinRegistry::get_port_entry(const VcpkgPaths& paths, StringView port_name) const
     {
-        auto versions_path = paths.builtin_registry_versions / relative_path_to_versions(port_name);
-        if (!m_baseline_identifier.empty() && paths.get_filesystem().exists(versions_path))
+        const auto& fs = paths.get_filesystem();
+        if (!m_baseline_identifier.empty())
         {
-            auto maybe_version_entries = load_versions_file(
-                paths.get_filesystem(), VersionDbType::Git, paths.builtin_registry_versions, port_name);
-            Checks::check_maybe_upgrade(
-                VCPKG_LINE_INFO, maybe_version_entries.has_value(), "Error: " + maybe_version_entries.error());
-            auto version_entries = std::move(maybe_version_entries).value_or_exit(VCPKG_LINE_INFO);
-
-            auto res =
-                std::make_unique<BuiltinRegistryEntry>(std::make_unique<GitRegistryEntry>(port_name.to_string()));
-            auto gre = res->git_entry.get();
-            for (auto&& version_entry : version_entries)
+            auto versions_path = paths.builtin_registry_versions / relative_path_to_versions(port_name);
+            if (fs.exists(versions_path))
             {
-                gre->port_versions.push_back(version_entry.version);
-                gre->git_trees.push_back(version_entry.git_tree);
+                auto maybe_version_entries =
+                    load_versions_file(fs, VersionDbType::Git, paths.builtin_registry_versions, port_name);
+                Checks::check_maybe_upgrade(
+                    VCPKG_LINE_INFO, maybe_version_entries.has_value(), "Error: " + maybe_version_entries.error());
+                auto version_entries = std::move(maybe_version_entries).value_or_exit(VCPKG_LINE_INFO);
+
+                auto gre = std::make_unique<GitRegistryEntry>();
+                gre->port_name = port_name.to_string();
+                for (auto&& version_entry : version_entries)
+                {
+                    gre->port_versions.push_back(version_entry.version);
+                    gre->git_trees.push_back(version_entry.git_tree);
+                }
+                return gre;
             }
-            return res;
         }
 
         // Fall back to current available version
         auto port_directory = paths.builtin_ports_directory() / fs::u8path(port_name);
-        if (paths.get_filesystem().exists(port_directory))
+        if (fs.exists(port_directory))
         {
-            auto found_scf = Paragraphs::try_load_port(paths.get_filesystem(), port_directory);
+            auto found_scf = Paragraphs::try_load_port(fs, port_directory);
             if (auto scfp = found_scf.get())
             {
                 auto& scf = *scfp;
@@ -283,8 +275,11 @@ namespace
 
                 if (scf->core_paragraph->name == port_name)
                 {
-                    return std::make_unique<BuiltinRegistryEntry>(
-                        std::make_unique<SourceControlFileLocation>(std::move(scf), std::move(port_directory)));
+                    auto res = std::make_unique<BuiltinRegistryEntry>();
+                    res->version = scf->core_paragraph->to_versiont();
+                    res->path = std::move(port_directory);
+                    res->name = std::move(scf->core_paragraph->name);
+                    return res;
                 }
                 Checks::exit_maybe_upgrade(VCPKG_LINE_INFO,
                                            "Error: Failed to load port from %s: names did not match: '%s' != '%s'",
@@ -355,7 +350,6 @@ namespace
     }
     Optional<VersionT> BuiltinRegistry::get_baseline_version(const VcpkgPaths& paths, StringView port_name) const
     {
-        Debug::print("Baseline version: \"", m_baseline_identifier, "\"\n");
         if (!m_baseline_identifier.empty())
         {
             const auto& baseline = m_baseline.get(
@@ -366,32 +360,39 @@ namespace
             {
                 return it->second;
             }
+            return nullopt;
         }
-        else
+
+        // if a baseline is not specified, use the ports directory version
+        auto port_path = paths.builtin_ports_directory() / fs::u8path(port_name);
+        auto maybe_scf = Paragraphs::try_load_port(paths.get_filesystem(), port_path);
+        if (auto pscf = maybe_scf.get())
         {
-            // if a baseline is not specified, use the ports directory version
-            auto maybe_scf = Paragraphs::try_load_port(paths.get_filesystem(),
-                                                       paths.builtin_ports_directory() / fs::u8path(port_name));
-            if (auto pscf = maybe_scf.get())
-            {
-                auto& scf = *pscf;
-                return scf->to_versiont();
-            }
-            Debug::print("Failed to load port `", port_name, "` from the ports tree: ", maybe_scf.error()->error, "\n");
+            auto& scf = *pscf;
+            return scf->to_versiont();
         }
-        return nullopt;
+        print_error_message(maybe_scf.error());
+        Checks::exit_maybe_upgrade(VCPKG_LINE_INFO, "Error: failed to load port from %s", fs::u8string(port_path));
     }
 
     void BuiltinRegistry::get_all_port_names(std::vector<std::string>& out, const VcpkgPaths& paths) const
     {
-        if (!m_baseline_identifier.empty() && paths.get_filesystem().exists(paths.builtin_registry_versions))
+        const auto& fs = paths.get_filesystem();
+
+        if (!m_baseline_identifier.empty() && fs.exists(paths.builtin_registry_versions))
         {
             load_all_port_names_from_registry_versions(out, paths, paths.builtin_registry_versions);
         }
-
-        for (auto port_directory : fs::directory_iterator(paths.builtin_ports_directory()))
+        std::error_code ec;
+        fs::directory_iterator dir_it(paths.builtin_ports_directory(), ec);
+        Checks::check_exit(VCPKG_LINE_INFO,
+                           !ec,
+                           "Error: failed while enumerating the builtin ports directory %s: %s",
+                           fs::u8string(paths.builtin_ports_directory()),
+                           ec.message());
+        for (auto port_directory : dir_it)
         {
-            if (!fs::is_directory(paths.get_filesystem().status(VCPKG_LINE_INFO, port_directory))) continue;
+            if (!fs::is_directory(fs.status(VCPKG_LINE_INFO, port_directory))) continue;
             auto filename = fs::u8string(port_directory.path().filename());
             if (filename == ".DS_Store") continue;
             out.push_back(filename);
@@ -475,7 +476,8 @@ namespace
             VCPKG_LINE_INFO, maybe_version_entries.has_value(), "Error: " + maybe_version_entries.error());
         auto version_entries = std::move(maybe_version_entries).value_or_exit(VCPKG_LINE_INFO);
 
-        auto res = std::make_unique<GitRegistryEntry>(port_name.to_string());
+        auto res = std::make_unique<GitRegistryEntry>();
+        res->port_name = port_name.to_string();
         for (auto&& version_entry : version_entries)
         {
             res->port_versions.push_back(version_entry.version);
@@ -581,49 +583,6 @@ namespace
 
     // { RegistryEntry
 
-    // { BuiltinRegistryEntry::RegistryEntry
-    ExpectedS<fs::path> BuiltinRegistryEntry::get_path_to_version(const VcpkgPaths& paths,
-                                                                  const VersionT& version) const
-    {
-        if (git_entry)
-        {
-            auto it = std::find(git_entry->port_versions.begin(), git_entry->port_versions.end(), version);
-            if (it == git_entry->port_versions.end())
-            {
-                return {
-                    Strings::concat("Error: No version entry for ",
-                                    git_entry->port_name,
-                                    " at version ",
-                                    version,
-                                    ". This may be fixed by updating vcpkg to the latest master via `git "
-                                    "pull`.\nAvailable versions:\n",
-                                    Strings::join("",
-                                                  git_entry->port_versions,
-                                                  [](const VersionT& v) { return Strings::concat("    ", v, "\n"); }),
-                                    "\nSee `vcpkg help versioning` for more information."),
-                    expected_right_tag};
-            }
-
-            const auto& git_tree = git_entry->git_trees[it - git_entry->port_versions.begin()];
-            return paths.git_checkout_port(git_entry->port_name, git_tree, paths.root / fs::u8path(".git"));
-        }
-
-        if (scfl_version == version)
-        {
-            return scfl->source_location;
-        }
-
-        auto& name = scfl->source_control_file->core_paragraph->name;
-        return Strings::format(
-            "Error: no version entry for %s at version %s.\n"
-            "We are currently using the version in the ports tree (%s), since no %s.json was found in /versions.",
-            name,
-            version.to_string(),
-            scfl->to_versiont().to_string(),
-            name);
-    }
-    // } BuiltinRegistryEntry::RegistryEntry
-
     // { FilesystemRegistryEntry::RegistryEntry
     ExpectedS<fs::path> FilesystemRegistryEntry::get_path_to_version(const VcpkgPaths&, const VersionT& version) const
     {
@@ -642,11 +601,23 @@ namespace
         auto it = std::find(port_versions.begin(), port_versions.end(), version);
         if (it == port_versions.end())
         {
-            return Strings::concat("Error: No version entry for ", port_name, " at version ", version, ".");
+            // This message suggests that the user updates vcpkg -- this is appropriate for the builtin registry for now
+            // but needs tweaking for external git registries
+            return {Strings::concat("Error: No version entry for ",
+                                    port_name,
+                                    " at version ",
+                                    version,
+                                    ". This may be fixed by updating vcpkg to the latest master via `git "
+                                    "pull`.\nAvailable versions:\n",
+                                    Strings::join("",
+                                                  port_versions,
+                                                  [](const VersionT& v) { return Strings::concat("    ", v, "\n"); }),
+                                    "\nSee `vcpkg help versioning` for more information."),
+                    expected_right_tag};
         }
 
         const auto& git_tree = git_trees[it - port_versions.begin()];
-        return paths.git_checkout_object_from_remote_registry(git_tree);
+        return paths.git_checkout_port(port_name, git_tree, paths.root / fs::u8path(".git"));
     }
     // } GitRegistryEntry::RegistryEntry
 
@@ -956,7 +927,7 @@ namespace
         return fs::u8path({port_name.byte_at_index(0), '-'}) / port_filename;
     }
 
-    ExpectedS<std::vector<VersionDbEntry>> load_versions_file(Files::Filesystem& fs,
+    ExpectedS<std::vector<VersionDbEntry>> load_versions_file(const Files::Filesystem& fs,
                                                               VersionDbType type,
                                                               const fs::path& registry_versions,
                                                               StringView port_name,
@@ -1090,18 +1061,13 @@ namespace vcpkg
             "an array of registries", RegistryDeserializer(configuration_directory));
     }
 
-    std::unique_ptr<RegistryImplementation> Registry::builtin_registry(std::string&& baseline)
-    {
-        return std::make_unique<BuiltinRegistry>(std::move(baseline));
-    }
-
     Registry::Registry(std::vector<std::string>&& packages, std::unique_ptr<RegistryImplementation>&& impl)
         : packages_(std::move(packages)), implementation_(std::move(impl))
     {
         Checks::check_exit(VCPKG_LINE_INFO, implementation_ != nullptr);
     }
 
-    RegistrySet::RegistrySet() : default_registry_(Registry::builtin_registry()), registries_() { }
+    RegistrySet::RegistrySet() : default_registry_(std::make_unique<BuiltinRegistry>("")) { }
 
     const RegistryImplementation* RegistrySet::registry_for_port(StringView name) const
     {