[vcpkg] Add `versions` feature flag and version field manifest parsing (#14079)

* [vcpkg] Add `versions` feature flag and version field manifest parsing

* Introduce FeatureFlagSettings struct to more easily access feature flags throughout the program
* To avoid users accidentally starting to write "version" instead of "version-string" in their manifests, vcpkg explicitly detects and prevents usage of ports with schemes other than "String"
* Drive-by fix of copiable SourceControlFileLocation and an exposed use-after-move bug

This code is largely extracted from PR #13777

Co-authored-by: Victor Romero <romerosanchezv@gmail.com>

* [vcpkg] Address CR Comments. Fix test crash on Linux.

Co-authored-by: Robert Schumacher <roschuma@microsoft.com>
Co-authored-by: Victor Romero <romerosanchezv@gmail.com>

9 files changed, 189 insertions, 43 deletions

diff --git a/toolsrc/src/vcpkg-test/manifests.cpp b/toolsrc/src/vcpkg-test/manifests.cpp
index 77f4eb440..be087584e 100644
--- a/toolsrc/src/vcpkg-test/manifests.cpp
+++ b/toolsrc/src/vcpkg-test/manifests.cpp
@@ -39,6 +39,7 @@ static Parse::ParseExpected<SourceControlFile> test_parse_manifest(StringView sv
     {
         print_error_message(res.error());
     }
+    REQUIRE(res.has_value() == !expect_fail);
     return res;
 }
 
@@ -59,6 +60,57 @@ TEST_CASE ("manifest construct minimum", "[manifests]")
     REQUIRE(pgh.core_paragraph->dependencies.empty());
 }
 
+TEST_CASE ("manifest versioning", "[manifests]")
+{
+    std::tuple<StringLiteral, Versions::Scheme, StringLiteral> data[] = {
+        {R"json({
+    "name": "zlib",
+    "version-string": "abcd"
+}
+)json",
+         Versions::Scheme::String,
+         "abcd"},
+        {R"json({
+    "name": "zlib",
+    "version-date": "2020-01-01"
+}
+)json",
+         Versions::Scheme::Date,
+         "2020-01-01"},
+        {R"json({
+    "name": "zlib",
+    "version": "1.2.3.4.5"
+}
+)json",
+         Versions::Scheme::Relaxed,
+         "1.2.3.4.5"},
+        {R"json({
+    "name": "zlib",
+    "version-semver": "1.2.3-rc3"
+}
+)json",
+         Versions::Scheme::Semver,
+         "1.2.3-rc3"},
+    };
+    for (auto v : data)
+    {
+        auto m_pgh = test_parse_manifest(std::get<0>(v));
+
+        REQUIRE(m_pgh.has_value());
+        auto& pgh = **m_pgh.get();
+        REQUIRE(Json::stringify(serialize_manifest(pgh), Json::JsonStyle::with_spaces(4)) == std::get<0>(v));
+        REQUIRE(pgh.core_paragraph->version_scheme == std::get<1>(v));
+        REQUIRE(pgh.core_paragraph->version == std::get<2>(v));
+    }
+
+    test_parse_manifest(R"json({
+        "name": "zlib",
+        "version-string": "abcd",
+        "version-semver": "1.2.3-rc3"
+    })json",
+                        true);
+}
+
 TEST_CASE ("manifest construct maximum", "[manifests]")
 {
     auto m_pgh = test_parse_manifest(R"json({
diff --git a/toolsrc/src/vcpkg-test/plan.cpp b/toolsrc/src/vcpkg-test/plan.cpp
index 55c6740da..363a2d9b4 100644
--- a/toolsrc/src/vcpkg-test/plan.cpp
+++ b/toolsrc/src/vcpkg-test/plan.cpp
@@ -389,9 +389,9 @@ TEST_CASE ("basic feature test 8", "[plan]")
     auto spec_c_64 = FullPackageSpec{spec_map.emplace("c", "a[a1]"), {"core"}};
 
     spec_map.triplet = Test::X86_WINDOWS;
-    auto spec_a_86 = FullPackageSpec{spec_map.emplace("a", "b", {{"a1", ""}}), {"core"}};
-    auto spec_b_86 = FullPackageSpec{spec_map.emplace("b")};
-    auto spec_c_86 = FullPackageSpec{spec_map.emplace("c", "a[a1]"), {"core"}};
+    auto spec_a_86 = FullPackageSpec{PackageSpec{"a", Test::X86_WINDOWS}};
+    auto spec_b_86 = FullPackageSpec{PackageSpec{"b", Test::X86_WINDOWS}};
+    auto spec_c_86 = FullPackageSpec{PackageSpec{"c", Test::X86_WINDOWS}};
 
     PortFileProvider::MapPortFileProvider map_port{spec_map.map};
     MockCMakeVarProvider var_provider;
diff --git a/toolsrc/src/vcpkg-test/util.cpp b/toolsrc/src/vcpkg-test/util.cpp
index 083861bc2..73ff9db5c 100644
--- a/toolsrc/src/vcpkg-test/util.cpp
+++ b/toolsrc/src/vcpkg-test/util.cpp
@@ -108,8 +108,10 @@ namespace vcpkg::Test
 
     PackageSpec PackageSpecMap::emplace(vcpkg::SourceControlFileLocation&& scfl)
     {
-        map.emplace(scfl.source_control_file->core_paragraph->name, std::move(scfl));
-        return {scfl.source_control_file->core_paragraph->name, triplet};
+        const auto& name = scfl.source_control_file->core_paragraph->name;
+        REQUIRE(map.find(name) == map.end());
+        map.emplace(name, std::move(scfl));
+        return {name, triplet};
     }
 
     static AllowSymlinks internal_can_create_symlinks() noexcept
diff --git a/toolsrc/src/vcpkg/commands.ci.cpp b/toolsrc/src/vcpkg/commands.ci.cpp
index 8fe1bf32e..fe41860be 100644
--- a/toolsrc/src/vcpkg/commands.ci.cpp
+++ b/toolsrc/src/vcpkg/commands.ci.cpp
@@ -338,7 +338,7 @@ namespace vcpkg::Commands::CI
                 ret->features.emplace(action.spec, action.feature_list);
                 if (auto scfl = p->source_control_file_location.get())
                 {
-                    auto emp = ret->default_feature_provider.emplace(p->spec.name(), *scfl);
+                    auto emp = ret->default_feature_provider.emplace(p->spec.name(), scfl->clone());
                     emp.first->second.source_control_file->core_paragraph->default_features = p->feature_list;
 
                     p->build_options = vcpkg::Build::default_build_package_options;
diff --git a/toolsrc/src/vcpkg/dependencies.cpp b/toolsrc/src/vcpkg/dependencies.cpp
index bbea346a6..21af23c0b 100644
--- a/toolsrc/src/vcpkg/dependencies.cpp
+++ b/toolsrc/src/vcpkg/dependencies.cpp
@@ -684,10 +684,15 @@ namespace vcpkg::Dependencies
         std::vector<FeatureSpec> feature_specs;
         for (const FullPackageSpec& spec : specs)
         {
-            const SourceControlFileLocation* scfl = port_provider.get_control_file(spec.package_spec.name()).get();
+            auto maybe_scfl = port_provider.get_control_file(spec.package_spec.name());
 
-            Checks::check_exit(
-                VCPKG_LINE_INFO, scfl, "Error: Cannot find definition for package `%s`.", spec.package_spec.name());
+            Checks::check_exit(VCPKG_LINE_INFO,
+                               maybe_scfl.has_value(),
+                               "Error: while loading port `%s`: %s",
+                               spec.package_spec.name(),
+                               maybe_scfl.error());
+
+            const SourceControlFileLocation* scfl = maybe_scfl.get();
 
             const std::vector<std::string> all_features =
                 Util::fmap(scfl->source_control_file->feature_paragraphs,
diff --git a/toolsrc/src/vcpkg/portfileprovider.cpp b/toolsrc/src/vcpkg/portfileprovider.cpp
index 0e065eff7..aaeda0bb1 100644
--- a/toolsrc/src/vcpkg/portfileprovider.cpp
+++ b/toolsrc/src/vcpkg/portfileprovider.cpp
@@ -5,6 +5,7 @@
 #include <vcpkg/portfileprovider.h>
 #include <vcpkg/registries.h>
 #include <vcpkg/sourceparagraph.h>
+#include <vcpkg/vcpkgcmdarguments.h>
 
 namespace vcpkg::PortFileProvider
 {
@@ -59,16 +60,10 @@ namespace vcpkg::PortFileProvider
         }
     }
 
-    ExpectedS<const SourceControlFileLocation&> PathsPortFileProvider::get_control_file(const std::string& spec) const
+    static Optional<SourceControlFileLocation> try_load_overlay_port(const Files::Filesystem& fs,
+                                                                     View<fs::path> overlay_ports,
+                                                                     const std::string& spec)
     {
-        auto cache_it = cache.find(spec);
-        if (cache_it != cache.end())
-        {
-            return cache_it->second;
-        }
-
-        const auto& fs = paths.get_filesystem();
-
         for (auto&& ports_dir : overlay_ports)
         {
             // Try loading individual port
@@ -79,10 +74,7 @@ namespace vcpkg::PortFileProvider
                 {
                     if (scf->get()->core_paragraph->name == spec)
                     {
-                        auto it = cache.emplace(std::piecewise_construct,
-                                                std::forward_as_tuple(spec),
-                                                std::forward_as_tuple(std::move(*scf), ports_dir));
-                        return it.first->second;
+                        return SourceControlFileLocation{std::move(*scf), ports_dir};
                     }
                 }
                 else
@@ -103,10 +95,7 @@ namespace vcpkg::PortFileProvider
                 {
                     if (scf->get()->core_paragraph->name == spec)
                     {
-                        auto it = cache.emplace(std::piecewise_construct,
-                                                std::forward_as_tuple(std::move(spec)),
-                                                std::forward_as_tuple(std::move(*scf), std::move(ports_spec)));
-                        return it.first->second;
+                        return SourceControlFileLocation{std::move(*scf), std::move(ports_spec)};
                     }
                     Checks::exit_with_message(VCPKG_LINE_INFO,
                                               "Error: Failed to load port from %s: names did not match: '%s' != '%s'",
@@ -122,7 +111,12 @@ namespace vcpkg::PortFileProvider
                 }
             }
         }
+        return nullopt;
+    }
 
+    static Optional<SourceControlFileLocation> try_load_registry_port(const VcpkgPaths& paths, const std::string& spec)
+    {
+        const auto& fs = paths.get_filesystem();
         if (auto registry = paths.get_configuration().registry_set.registry_for_port(spec))
         {
             auto registry_root = registry->get_registry_root(paths);
@@ -135,10 +129,7 @@ namespace vcpkg::PortFileProvider
                 {
                     if (scf->get()->core_paragraph->name == spec)
                     {
-                        auto it = cache.emplace(std::piecewise_construct,
-                                                std::forward_as_tuple(spec),
-                                                std::forward_as_tuple(std::move(*scf), std::move(port_directory)));
-                        return it.first->second;
+                        return SourceControlFileLocation{std::move(*scf), std::move(port_directory)};
                     }
                     Checks::exit_with_message(VCPKG_LINE_INFO,
                                               "Error: Failed to load port from %s: names did not match: '%s' != '%s'",
@@ -154,8 +145,44 @@ namespace vcpkg::PortFileProvider
                 }
             }
         }
+        return nullopt;
+    }
 
-        return std::string("Port definition not found");
+    ExpectedS<const SourceControlFileLocation&> PathsPortFileProvider::get_control_file(const std::string& spec) const
+    {
+        auto cache_it = cache.find(spec);
+        if (cache_it == cache.end())
+        {
+            const auto& fs = paths.get_filesystem();
+            auto maybe_port = try_load_overlay_port(fs, overlay_ports, spec);
+            if (!maybe_port)
+            {
+                maybe_port = try_load_registry_port(paths, spec);
+            }
+            if (auto p = maybe_port.get())
+            {
+                cache_it = cache.emplace(spec, std::move(*p)).first;
+            }
+        }
+
+        if (cache_it == cache.end())
+        {
+            return std::string("Port definition not found");
+        }
+        else
+        {
+            if (!paths.get_feature_flags().versions)
+            {
+                if (cache_it->second.source_control_file->core_paragraph->version_scheme != Versions::Scheme::String)
+                {
+                    return Strings::concat(
+                        "Port definition rejected because the `",
+                        VcpkgCmdArguments::VERSIONS_FEATURE,
+                        "` feature flag is disabled.\nThis can be fixed by using the \"version-string\" field.");
+                }
+            }
+            return cache_it->second;
+        }
     }
 
     std::vector<const SourceControlFileLocation*> PathsPortFileProvider::load_all_control_files() const
diff --git a/toolsrc/src/vcpkg/sourceparagraph.cpp b/toolsrc/src/vcpkg/sourceparagraph.cpp
index a01f05b28..802e6d6a3 100644
--- a/toolsrc/src/vcpkg/sourceparagraph.cpp
+++ b/toolsrc/src/vcpkg/sourceparagraph.cpp
@@ -28,6 +28,7 @@ namespace vcpkg
     {
         if (lhs.name != rhs.name) return false;
         if (lhs.version != rhs.version) return false;
+        if (lhs.version_scheme != rhs.version_scheme) return false;
         if (lhs.port_version != rhs.port_version) return false;
         if (!paragraph_equal(lhs.description, rhs.description)) return false;
         if (!paragraph_equal(lhs.maintainers, rhs.maintainers)) return false;
@@ -735,7 +736,13 @@ namespace vcpkg
         virtual StringView type_name() const override { return "a manifest"; }
 
         constexpr static StringLiteral NAME = "name";
-        constexpr static StringLiteral VERSION = "version-string";
+
+        // Default is a relaxed semver-like version
+        constexpr static StringLiteral VERSION_RELAXED = "version";
+        constexpr static StringLiteral VERSION_SEMVER = "version-semver";
+        constexpr static StringLiteral VERSION_DATE = "version-date";
+        // Legacy version string, accepts arbitrary string values.
+        constexpr static StringLiteral VERSION_STRING = "version-string";
 
         constexpr static StringLiteral PORT_VERSION = "port-version";
         constexpr static StringLiteral MAINTAINERS = "maintainers";
@@ -753,8 +760,10 @@ namespace vcpkg
         {
             static const StringView t[] = {
                 NAME,
-                VERSION,
-
+                VERSION_STRING,
+                VERSION_RELAXED,
+                VERSION_SEMVER,
+                VERSION_DATE,
                 PORT_VERSION,
                 MAINTAINERS,
                 DESCRIPTION,
@@ -788,12 +797,41 @@ namespace vcpkg
                 }
             }
 
-            static Json::StringDeserializer version_deserializer{"a version"};
+            static Json::StringDeserializer version_exact_deserializer{"an exact version string"};
+            static Json::StringDeserializer version_relaxed_deserializer{"a relaxed version string"};
+            static Json::StringDeserializer version_semver_deserializer{"a semantic version string"};
+            static Json::StringDeserializer version_date_deserializer{"a date version string"};
             static Json::StringDeserializer url_deserializer{"a url"};
 
             constexpr static StringView type_name = "vcpkg.json";
             r.required_object_field(type_name, obj, NAME, spgh->name, Json::IdentifierDeserializer::instance);
-            r.required_object_field(type_name, obj, VERSION, spgh->version, version_deserializer);
+            bool has_exact = r.optional_object_field(obj, VERSION_STRING, spgh->version, version_exact_deserializer);
+            bool has_relax = r.optional_object_field(obj, VERSION_RELAXED, spgh->version, version_relaxed_deserializer);
+            bool has_semver = r.optional_object_field(obj, VERSION_SEMVER, spgh->version, version_semver_deserializer);
+            bool has_date = r.optional_object_field(obj, VERSION_DATE, spgh->version, version_date_deserializer);
+            int num_versions = (int)has_exact + (int)has_relax + (int)has_semver + (int)has_date;
+            if (num_versions == 0)
+            {
+                r.add_generic_error(type_name, "expected a versioning field (example: ", VERSION_RELAXED, ")");
+            }
+            else if (num_versions > 1)
+            {
+                r.add_generic_error(type_name, "expected only one versioning field");
+            }
+            else
+            {
+                if (has_exact)
+                    spgh->version_scheme = Versions::Scheme::String;
+                else if (has_relax)
+                    spgh->version_scheme = Versions::Scheme::Relaxed;
+                else if (has_semver)
+                    spgh->version_scheme = Versions::Scheme::Semver;
+                else if (has_date)
+                    spgh->version_scheme = Versions::Scheme::Date;
+                else
+                    Checks::unreachable(VCPKG_LINE_INFO);
+            }
+
             r.optional_object_field(obj, PORT_VERSION, spgh->port_version, Json::NaturalNumberDeserializer::instance);
             r.optional_object_field(obj, MAINTAINERS, spgh->maintainers, Json::ParagraphDeserializer::instance);
             r.optional_object_field(obj, DESCRIPTION, spgh->description, Json::ParagraphDeserializer::instance);
@@ -825,8 +863,10 @@ namespace vcpkg
     ManifestDeserializer ManifestDeserializer::instance;
 
     constexpr StringLiteral ManifestDeserializer::NAME;
-    constexpr StringLiteral ManifestDeserializer::VERSION;
-
+    constexpr StringLiteral ManifestDeserializer::VERSION_STRING;
+    constexpr StringLiteral ManifestDeserializer::VERSION_RELAXED;
+    constexpr StringLiteral ManifestDeserializer::VERSION_SEMVER;
+    constexpr StringLiteral ManifestDeserializer::VERSION_DATE;
     constexpr StringLiteral ManifestDeserializer::PORT_VERSION;
     constexpr StringLiteral ManifestDeserializer::MAINTAINERS;
     constexpr StringLiteral ManifestDeserializer::DESCRIPTION;
@@ -1067,7 +1107,17 @@ namespace vcpkg
         }
 
         obj.insert(ManifestDeserializer::NAME, Json::Value::string(scf.core_paragraph->name));
-        obj.insert(ManifestDeserializer::VERSION, Json::Value::string(scf.core_paragraph->version));
+        auto version_field = [&] {
+            switch (scf.core_paragraph->version_scheme)
+            {
+                case Versions::Scheme::String: return ManifestDeserializer::VERSION_STRING;
+                case Versions::Scheme::Semver: return ManifestDeserializer::VERSION_SEMVER;
+                case Versions::Scheme::Relaxed: return ManifestDeserializer::VERSION_RELAXED;
+                case Versions::Scheme::Date: return ManifestDeserializer::VERSION_DATE;
+                default: Checks::unreachable(VCPKG_LINE_INFO);
+            }
+        }();
+        obj.insert(version_field, Json::Value::string(scf.core_paragraph->version));
 
         if (scf.core_paragraph->port_version != 0 || debug)
         {
diff --git a/toolsrc/src/vcpkg/vcpkgcmdarguments.cpp b/toolsrc/src/vcpkg/vcpkgcmdarguments.cpp
index 84d4526b1..6c7bbd5c3 100644
--- a/toolsrc/src/vcpkg/vcpkgcmdarguments.cpp
+++ b/toolsrc/src/vcpkg/vcpkgcmdarguments.cpp
@@ -51,6 +51,7 @@ namespace vcpkg
             {VcpkgCmdArguments::MANIFEST_MODE_FEATURE, args.manifest_mode},
             {VcpkgCmdArguments::COMPILER_TRACKING_FEATURE, args.compiler_tracking},
             {VcpkgCmdArguments::REGISTRIES_FEATURE, args.registries_feature},
+            {VcpkgCmdArguments::VERSIONS_FEATURE, args.versions_feature},
         };
 
         for (const auto& desc : flag_descriptions)
@@ -765,6 +766,7 @@ namespace vcpkg
             {MANIFEST_MODE_FEATURE, manifest_mode},
             {COMPILER_TRACKING_FEATURE, compiler_tracking},
             {REGISTRIES_FEATURE, registries_feature},
+            {VERSIONS_FEATURE, versions_feature},
         };
 
         for (const auto& flag : flags)
@@ -790,6 +792,7 @@ namespace vcpkg
             {BINARY_CACHING_FEATURE, binary_caching_enabled()},
             {COMPILER_TRACKING_FEATURE, compiler_tracking_enabled()},
             {REGISTRIES_FEATURE, registries_enabled()},
+            {VERSIONS_FEATURE, versions_enabled()},
         };
 
         for (const auto& flag : flags)
@@ -931,6 +934,6 @@ namespace vcpkg
     constexpr StringLiteral VcpkgCmdArguments::COMPILER_TRACKING_FEATURE;
     constexpr StringLiteral VcpkgCmdArguments::MANIFEST_MODE_FEATURE;
     constexpr StringLiteral VcpkgCmdArguments::REGISTRIES_FEATURE;
-
     constexpr StringLiteral VcpkgCmdArguments::RECURSIVE_DATA_ENV;
+    constexpr StringLiteral VcpkgCmdArguments::VERSIONS_FEATURE;
 }
diff --git a/toolsrc/src/vcpkg/vcpkgpaths.cpp b/toolsrc/src/vcpkg/vcpkgpaths.cpp
index cc51c6a2f..bd3e208ff 100644
--- a/toolsrc/src/vcpkg/vcpkgpaths.cpp
+++ b/toolsrc/src/vcpkg/vcpkgpaths.cpp
@@ -186,8 +186,11 @@ namespace vcpkg
     {
         struct VcpkgPathsImpl
         {
-            VcpkgPathsImpl(Files::Filesystem& fs, bool compiler_tracking)
-                : fs_ptr(&fs), m_tool_cache(get_tool_cache()), m_env_cache(compiler_tracking)
+            VcpkgPathsImpl(Files::Filesystem& fs, FeatureFlagSettings ff_settings)
+                : fs_ptr(&fs)
+                , m_tool_cache(get_tool_cache())
+                , m_env_cache(ff_settings.compiler_tracking)
+                , m_ff_settings(ff_settings)
             {
             }
 
@@ -209,11 +212,13 @@ namespace vcpkg
             Optional<std::pair<Json::Object, Json::JsonStyle>> m_manifest_doc;
             fs::path m_manifest_path;
             Configuration m_config;
+
+            FeatureFlagSettings m_ff_settings;
         };
     }
 
     VcpkgPaths::VcpkgPaths(Files::Filesystem& filesystem, const VcpkgCmdArguments& args)
-        : m_pimpl(std::make_unique<details::VcpkgPathsImpl>(filesystem, args.compiler_tracking_enabled()))
+        : m_pimpl(std::make_unique<details::VcpkgPathsImpl>(filesystem, args.feature_flag_settings()))
     {
         original_cwd = filesystem.current_path(VCPKG_LINE_INFO);
 #if defined(_WIN32)
@@ -564,6 +569,8 @@ If you wish to silence this error and use classic mode, you can:
 
     Files::Filesystem& VcpkgPaths::get_filesystem() const { return *m_pimpl->fs_ptr; }
 
+    const FeatureFlagSettings& VcpkgPaths::get_feature_flags() const { return m_pimpl->m_ff_settings; }
+
     void VcpkgPaths::track_feature_flag_metrics() const
     {
         struct