[vcpkg manifests] fix some issues (#12227)

Fixes #12190
Fixes #12208
Fixes #12234
Fixes #12286
Fixes #12315
Fixes #12186
Fixes #12331
Fixes googleapis/google-cloud-cpp#4487

11 files changed, 173 insertions, 57 deletions

diff --git a/toolsrc/src/vcpkg.cpp b/toolsrc/src/vcpkg.cpp
index b6f0782fa..003ae1024 100644
--- a/toolsrc/src/vcpkg.cpp
+++ b/toolsrc/src/vcpkg.cpp
@@ -250,6 +250,7 @@ int main(const int argc, const char* const* const argv)
                        "Warning: passed either --printmetrics or --no-printmetrics, but metrics are disabled.\n");
     }
 
+    args.debug_print_feature_flags();
     args.track_feature_flag_metrics();
 
     if (Debug::g_debugging)
diff --git a/toolsrc/src/vcpkg/base/files.cpp b/toolsrc/src/vcpkg/base/files.cpp
index 551938990..f119edace 100644
--- a/toolsrc/src/vcpkg/base/files.cpp
+++ b/toolsrc/src/vcpkg/base/files.cpp
@@ -874,15 +874,22 @@ namespace vcpkg::Files
             fs::stdfs::current_path(path, ec);
         }
 
-        virtual fs::SystemHandle try_take_exclusive_file_lock(const fs::path& path, std::error_code& ec) override
+        struct TakeExclusiveFileLockHelper
         {
-            fs::SystemHandle res;
+            fs::SystemHandle& res;
+            const fs::path::string_type& native;
+            TakeExclusiveFileLockHelper(fs::SystemHandle& res, const fs::path::string_type& native)
+                : res(res), native(native)
+            {
+            }
 
-            const auto system_file_name = path.native();
 #if defined(WIN32)
-            constexpr static auto busy_error = ERROR_BUSY;
-            const auto system_try_take_file_lock = [&] {
-                auto handle = CreateFileW(system_file_name.c_str(),
+            void assign_busy_error(std::error_code& ec) { ec.assign(ERROR_BUSY, std::system_category()); }
+
+            bool operator()(std::error_code& ec)
+            {
+                ec.clear();
+                auto handle = CreateFileW(native.c_str(),
                                           GENERIC_READ,
                                           0 /* no sharing */,
                                           nullptr /* no security attributes */,
@@ -901,31 +908,77 @@ namespace vcpkg::Files
 
                 res.system_handle = reinterpret_cast<intptr_t>(handle);
                 return true;
-            };
+            }
 #else // ^^^ WIN32 / !WIN32 vvv
-            constexpr static auto busy_error = EBUSY;
-            int fd = open(system_file_name.c_str(), 0);
-            if (fd < 0)
+            int fd = -1;
+
+            void assign_busy_error(std::error_code& ec) { ec.assign(EBUSY, std::generic_category()); }
+
+            bool operator()(std::error_code& ec)
             {
-                ec.assign(errno, std::system_category());
-                return res;
-            }
-            const auto system_try_take_file_lock = [&] {
-                if (flock(fd, LOCK_EX | LOCK_NB) != 0)
+                ec.clear();
+                if (fd == -1)
+                {
+                    fd = ::open(native.c_str(), 0);
+                    if (fd < 0)
+                    {
+                        ec.assign(errno, std::generic_category());
+                        return false;
+                    }
+                }
+
+                if (::flock(fd, LOCK_EX | LOCK_NB) != 0)
                 {
                     if (errno != EWOULDBLOCK)
                     {
-                        ec.assign(errno, std::system_category());
+                        ec.assign(errno, std::generic_category());
                     }
                     return false;
                 }
 
                 res.system_handle = fd;
+                fd = -1;
                 return true;
             };
+
+            ~TakeExclusiveFileLockHelper()
+            {
+                if (fd != -1)
+                {
+                    ::close(fd);
+                }
+            }
 #endif
+        };
 
-            if (system_try_take_file_lock() || ec)
+        virtual fs::SystemHandle take_exclusive_file_lock(const fs::path& path, std::error_code& ec) override
+        {
+            fs::SystemHandle res;
+            TakeExclusiveFileLockHelper helper(res, path.native());
+
+            if (helper(ec) || ec)
+            {
+                return res;
+            }
+
+            System::printf("Waiting to take filesystem lock on %s...\n", path.u8string());
+            const auto wait = std::chrono::milliseconds(1000);
+            for (;;)
+            {
+                std::this_thread::sleep_for(wait);
+                if (helper(ec) || ec)
+                {
+                    return res;
+                }
+            }
+        }
+
+        virtual fs::SystemHandle try_take_exclusive_file_lock(const fs::path& path, std::error_code& ec) override
+        {
+            fs::SystemHandle res;
+            TakeExclusiveFileLockHelper helper(res, path.native());
+
+            if (helper(ec) || ec)
             {
                 return res;
             }
@@ -936,19 +989,17 @@ namespace vcpkg::Files
             while (wait < std::chrono::milliseconds(1000))
             {
                 std::this_thread::sleep_for(wait);
-                if (system_try_take_file_lock() || ec)
+                if (helper(ec) || ec)
                 {
                     return res;
                 }
                 wait *= 2;
             }
 
-#if !defined(WIN32)
-            close(fd);
-#endif
-            ec.assign(busy_error, std::system_category());
+            helper.assign_busy_error(ec);
             return res;
         }
+
         virtual void unlock_file_lock(fs::SystemHandle handle, std::error_code& ec) override
         {
 #if defined(WIN32)
@@ -959,7 +1010,7 @@ namespace vcpkg::Files
 #else
             if (flock(handle.system_handle, LOCK_UN) != 0 || close(handle.system_handle) != 0)
             {
-                ec.assign(errno, std::system_category());
+                ec.assign(errno, std::generic_category());
             }
 #endif
         }
diff --git a/toolsrc/src/vcpkg/commands.list.cpp b/toolsrc/src/vcpkg/commands.list.cpp
index e95b02623..257e4f941 100644
--- a/toolsrc/src/vcpkg/commands.list.cpp
+++ b/toolsrc/src/vcpkg/commands.list.cpp
@@ -4,6 +4,7 @@
 #include <vcpkg/commands.h>
 #include <vcpkg/help.h>
 #include <vcpkg/vcpkglib.h>
+#include <vcpkg/versiont.h>
 
 namespace vcpkg::Commands::List
 {
@@ -12,11 +13,12 @@ namespace vcpkg::Commands::List
 
     static void do_print(const StatusParagraph& pgh, const bool full_desc)
     {
+        auto full_version = VersionT(pgh.package.version, pgh.package.port_version).to_string();
         if (full_desc)
         {
             System::printf("%-50s %-16s %s\n",
                            pgh.package.displayname(),
-                           pgh.package.version,
+                           full_version,
                            Strings::join("\n    ", pgh.package.description));
         }
         else
@@ -28,7 +30,7 @@ namespace vcpkg::Commands::List
             }
             System::printf("%-50s %-16s %s\n",
                            vcpkg::shorten_text(pgh.package.displayname(), 50),
-                           vcpkg::shorten_text(pgh.package.version, 16),
+                           vcpkg::shorten_text(full_version, 16),
                            vcpkg::shorten_text(description, 51));
         }
     }
diff --git a/toolsrc/src/vcpkg/commands.porthistory.cpp b/toolsrc/src/vcpkg/commands.porthistory.cpp
index 0f4d9716f..7e04cbd01 100644
--- a/toolsrc/src/vcpkg/commands.porthistory.cpp
+++ b/toolsrc/src/vcpkg/commands.porthistory.cpp
@@ -35,8 +35,14 @@ namespace vcpkg::Commands::PortHistory
         const std::string cmd = Strings::format(R"(show %s:ports/%s/CONTROL)", commit_id, port_name);
         auto output = run_git_command(paths, cmd);
 
-        const auto version = Strings::find_at_most_one_enclosed(output.output, "Version: ", "\n");
+        const auto version = Strings::find_at_most_one_enclosed(output.output, "\nVersion: ", "\n");
+        const auto port_version = Strings::find_at_most_one_enclosed(output.output, "\nPort-Version: ", "\n");
         Checks::check_exit(VCPKG_LINE_INFO, version.has_value(), "CONTROL file does not have a 'Version' field");
+        if (auto pv = port_version.get())
+        {
+            return Strings::format("%s#%s", version.get()->to_string(), pv->to_string());
+        }
+
         return version.get()->to_string();
     }
 
diff --git a/toolsrc/src/vcpkg/commands.portsdiff.cpp b/toolsrc/src/vcpkg/commands.portsdiff.cpp
index ed5044a60..562917e80 100644
--- a/toolsrc/src/vcpkg/commands.portsdiff.cpp
+++ b/toolsrc/src/vcpkg/commands.portsdiff.cpp
@@ -103,7 +103,10 @@ namespace vcpkg::Commands::PortsDiff
             Paragraphs::load_all_ports(paths.get_filesystem(), temp_checkout_path / ports_dir_name_as_string);
         std::map<std::string, VersionT> names_and_versions;
         for (auto&& port : all_ports)
-            names_and_versions.emplace(port->core_paragraph->name, port->core_paragraph->version);
+        {
+            const auto& core_pgh = *port->core_paragraph;
+            names_and_versions.emplace(port->core_paragraph->name, VersionT(core_pgh.version, core_pgh.port_version));
+        }
         fs.remove_all(temp_checkout_path, VCPKG_LINE_INFO);
         return names_and_versions;
     }
diff --git a/toolsrc/src/vcpkg/commands.search.cpp b/toolsrc/src/vcpkg/commands.search.cpp
index c234c3ab9..126c74dc0 100644
--- a/toolsrc/src/vcpkg/commands.search.cpp
+++ b/toolsrc/src/vcpkg/commands.search.cpp
@@ -8,6 +8,7 @@
 #include <vcpkg/paragraphs.h>
 #include <vcpkg/sourceparagraph.h>
 #include <vcpkg/vcpkglib.h>
+#include <vcpkg/versiont.h>
 
 using vcpkg::PortFileProvider::PathsPortFileProvider;
 
@@ -18,11 +19,12 @@ namespace vcpkg::Commands::Search
 
     static void do_print(const SourceParagraph& source_paragraph, bool full_desc)
     {
+        auto full_version = VersionT(source_paragraph.version, source_paragraph.port_version).to_string();
         if (full_desc)
         {
             System::printf("%-20s %-16s %s\n",
                            source_paragraph.name,
-                           source_paragraph.version,
+                           full_version,
                            Strings::join("\n    ", source_paragraph.description));
         }
         else
@@ -34,7 +36,7 @@ namespace vcpkg::Commands::Search
             }
             System::printf("%-20s %-16s %s\n",
                            vcpkg::shorten_text(source_paragraph.name, 20),
-                           vcpkg::shorten_text(source_paragraph.version, 16),
+                           vcpkg::shorten_text(full_version, 16),
                            vcpkg::shorten_text(description, 81));
         }
     }
diff --git a/toolsrc/src/vcpkg/install.cpp b/toolsrc/src/vcpkg/install.cpp
index 3be938215..f27b2ffae 100644
--- a/toolsrc/src/vcpkg/install.cpp
+++ b/toolsrc/src/vcpkg/install.cpp
@@ -726,8 +726,10 @@ namespace vcpkg::Install
             {
                 for (auto& dep : (*val)->core_paragraph->dependencies)
                 {
-                    specs.push_back(Input::check_and_get_full_package_spec(
-                        std::move(dep.name), default_triplet, COMMAND_STRUCTURE.example_text));
+                    specs.push_back(FullPackageSpec{
+                        {std::move(dep.name), default_triplet},
+                        std::move(dep.features),
+                    });
                 }
             }
             else
diff --git a/toolsrc/src/vcpkg/update.cpp b/toolsrc/src/vcpkg/update.cpp
index aa8d654e3..85241e438 100644
--- a/toolsrc/src/vcpkg/update.cpp
+++ b/toolsrc/src/vcpkg/update.cpp
@@ -26,11 +26,13 @@ namespace vcpkg::Update
             auto maybe_scfl = provider.get_control_file(pgh->package.spec.name());
             if (auto p_scfl = maybe_scfl.get())
             {
-                auto&& port_version = p_scfl->source_control_file->core_paragraph->version;
-                auto&& installed_version = pgh->package.version;
-                if (installed_version != port_version)
+                const auto& latest_pgh = *p_scfl->source_control_file->core_paragraph;
+                auto latest_version = VersionT(latest_pgh.version, latest_pgh.port_version);
+                auto installed_version = VersionT(pgh->package.version, pgh->package.port_version);
+                if (latest_version != installed_version)
                 {
-                    output.push_back({pgh->package.spec, VersionDiff(installed_version, port_version)});
+                    output.push_back(
+                        {pgh->package.spec, VersionDiff(std::move(installed_version), std::move(latest_version))});
                 }
             }
             else
diff --git a/toolsrc/src/vcpkg/vcpkgcmdarguments.cpp b/toolsrc/src/vcpkg/vcpkgcmdarguments.cpp
index 75632541d..6bc14309e 100644
--- a/toolsrc/src/vcpkg/vcpkgcmdarguments.cpp
+++ b/toolsrc/src/vcpkg/vcpkgcmdarguments.cpp
@@ -297,6 +297,7 @@ namespace vcpkg
                 {PRINT_METRICS_SWITCH, &VcpkgCmdArguments::print_metrics},
                 {FEATURE_PACKAGES_SWITCH, &VcpkgCmdArguments::feature_packages},
                 {BINARY_CACHING_SWITCH, &VcpkgCmdArguments::binary_caching},
+                {WAIT_FOR_LOCK_SWITCH, &VcpkgCmdArguments::wait_for_lock},
             };
 
             bool found = false;
@@ -677,13 +678,41 @@ namespace vcpkg
         }
     }
 
+    void VcpkgCmdArguments::debug_print_feature_flags() const
+    {
+        struct
+        {
+            StringView name;
+            Optional<bool> flag;
+        } flags[] = {
+            {BINARY_CACHING_FEATURE, binary_caching},
+            {MANIFEST_MODE_FEATURE, manifest_mode},
+            {COMPILER_TRACKING_FEATURE, compiler_tracking},
+        };
+
+        for (const auto& flag : flags)
+        {
+            if (auto r = flag.flag.get())
+            {
+                Debug::print("Feature flag '", flag.name, "' = ", *r ? "on" : "off", "\n");
+            }
+            else
+            {
+                Debug::print("Feature flag '", flag.name, "' unset\n");
+            }
+        }
+    }
+
     void VcpkgCmdArguments::track_feature_flag_metrics() const
     {
         struct
         {
             StringView flag;
             bool enabled;
-        } flags[] = {{BINARY_CACHING_FEATURE, binary_caching_enabled()}};
+        } flags[] = {
+            {BINARY_CACHING_FEATURE, binary_caching_enabled()},
+            {COMPILER_TRACKING_FEATURE, compiler_tracking_enabled()},
+        };
 
         for (const auto& flag : flags)
         {
diff --git a/toolsrc/src/vcpkg/vcpkgpaths.cpp b/toolsrc/src/vcpkg/vcpkgpaths.cpp
index a957cb27c..0349db1b8 100644
--- a/toolsrc/src/vcpkg/vcpkgpaths.cpp
+++ b/toolsrc/src/vcpkg/vcpkgpaths.cpp
@@ -125,16 +125,7 @@ namespace vcpkg
         Debug::print("Using vcpkg-root: ", root.u8string(), '\n');
 
         std::error_code ec;
-        const auto vcpkg_lock = root / ".vcpkg-root";
-        m_pimpl->file_lock_handle = filesystem.try_take_exclusive_file_lock(vcpkg_lock, ec);
-        if (ec)
-        {
-            System::printf(System::Color::error, "Failed to take the filesystem lock on %s:\n", vcpkg_lock.u8string());
-            System::printf(System::Color::error, "    %s\n", ec.message());
-            System::print2(System::Color::error, "Exiting now.\n");
-            Checks::exit_fail(VCPKG_LINE_INFO);
-        }
-
+        bool manifest_mode_on = args.manifest_mode.value_or(args.manifest_root_dir != nullptr);
         if (args.manifest_root_dir)
         {
             manifest_root_dir = filesystem.canonical(VCPKG_LINE_INFO, fs::u8path(*args.manifest_root_dir));
@@ -145,12 +136,28 @@ namespace vcpkg
         }
         uppercase_win32_drive_letter(manifest_root_dir);
 
-        if (!manifest_root_dir.empty() && args.manifest_mode.value_or(true))
+        if (!manifest_root_dir.empty() && manifest_mode_on)
         {
-            Debug::print("Using manifest-root: ", root.u8string(), '\n');
+            Debug::print("Using manifest-root: ", manifest_root_dir.u8string(), '\n');
 
             installed = process_output_directory(
                 filesystem, manifest_root_dir, args.install_root_dir.get(), "vcpkg_installed", VCPKG_LINE_INFO);
+            const auto vcpkg_lock = root / ".vcpkg-root";
+            if (args.wait_for_lock.value_or(false))
+            {
+                m_pimpl->file_lock_handle = filesystem.take_exclusive_file_lock(vcpkg_lock, ec);
+            }
+            else
+            {
+                m_pimpl->file_lock_handle = filesystem.try_take_exclusive_file_lock(vcpkg_lock, ec);
+            }
+            if (ec)
+            {
+                System::printf(
+                    System::Color::error, "Failed to take the filesystem lock on %s:\n", vcpkg_lock.u8string());
+                System::printf(System::Color::error, "    %s\n", ec.message());
+                Checks::exit_fail(VCPKG_LINE_INFO);
+            }
         }
         else
         {
@@ -404,10 +411,13 @@ If you wish to silence this error and use classic mode, you can:
     VcpkgPaths::~VcpkgPaths()
     {
         std::error_code ec;
-        m_pimpl->fs_ptr->unlock_file_lock(m_pimpl->file_lock_handle, ec);
-        if (ec)
+        if (m_pimpl->file_lock_handle.is_valid())
         {
-            Debug::print("Failed to unlock filesystem lock: ", ec.message(), '\n');
+            m_pimpl->fs_ptr->unlock_file_lock(m_pimpl->file_lock_handle, ec);
+            if (ec)
+            {
+                Debug::print("Failed to unlock filesystem lock: ", ec.message(), '\n');
+            }
         }
     }
 }
diff --git a/toolsrc/src/vcpkg/versiont.cpp b/toolsrc/src/vcpkg/versiont.cpp
index 209a473c6..d6ee6e93b 100644
--- a/toolsrc/src/vcpkg/versiont.cpp
+++ b/toolsrc/src/vcpkg/versiont.cpp
@@ -5,12 +5,20 @@
 
 namespace vcpkg
 {
-    VersionT::VersionT() noexcept : value("0.0.0") { }
-    VersionT::VersionT(std::string&& value) : value(std::move(value)) { }
-    VersionT::VersionT(const std::string& value) : value(value) { }
-    const std::string& VersionT::to_string() const { return value; }
-    bool operator==(const VersionT& left, const VersionT& right) { return left.to_string() == right.to_string(); }
-    bool operator!=(const VersionT& left, const VersionT& right) { return left.to_string() != right.to_string(); }
+    VersionT::VersionT() noexcept : value("0.0.0"), port_version(0) { }
+    VersionT::VersionT(std::string&& value, int port_version) : value(std::move(value)), port_version(port_version) { }
+    VersionT::VersionT(const std::string& value, int port_version) : value(value), port_version(port_version) { }
+
+    std::string VersionT::to_string() const
+    {
+        return port_version == 0 ? value : Strings::format("%s#%d", value, port_version);
+    }
+
+    bool operator==(const VersionT& left, const VersionT& right)
+    {
+        return left.port_version == right.port_version && left.value == right.value;
+    }
+    bool operator!=(const VersionT& left, const VersionT& right) { return !(left == right); }
 
     VersionDiff::VersionDiff() noexcept : left(), right() { }
     VersionDiff::VersionDiff(const VersionT& left, const VersionT& right) : left(left), right(right) { }