Merge branch 'master' into external_file_abi

23 files changed, 636 insertions, 98 deletions

diff --git a/toolsrc/src/vcpkg-tests/arguments.cpp b/toolsrc/src/vcpkg-test/arguments.cpp
index 8c625be0f..3fe5fa420 100644
--- a/toolsrc/src/vcpkg-tests/arguments.cpp
+++ b/toolsrc/src/vcpkg-test/arguments.cpp
@@ -1,4 +1,4 @@
-#include <vcpkg-tests/catch.h>
+#include <vcpkg-test/catch.h>
 
 #include <vcpkg/vcpkgcmdarguments.h>
 
diff --git a/toolsrc/src/vcpkg-tests/catch.cpp b/toolsrc/src/vcpkg-test/catch.cpp
index 701dcb39a..8b5d1aa15 100644
--- a/toolsrc/src/vcpkg-tests/catch.cpp
+++ b/toolsrc/src/vcpkg-test/catch.cpp
@@ -1,5 +1,5 @@
 #define CATCH_CONFIG_RUNNER
-#include <vcpkg-tests/catch.h>
+#include <vcpkg-test/catch.h>
 
 #include <vcpkg/base/system.debug.h>
 
diff --git a/toolsrc/src/vcpkg-tests/chrono.cpp b/toolsrc/src/vcpkg-test/chrono.cpp
index c164753f9..306217ad0 100644
--- a/toolsrc/src/vcpkg-tests/chrono.cpp
+++ b/toolsrc/src/vcpkg-test/chrono.cpp
@@ -1,4 +1,4 @@
-#include <vcpkg-tests/catch.h>
+#include <vcpkg-test/catch.h>
 
 #include <vcpkg/base/chrono.h>
 
diff --git a/toolsrc/src/vcpkg-tests/dependencies.cpp b/toolsrc/src/vcpkg-test/dependencies.cpp
index 0dee6f296..5ed05cc07 100644
--- a/toolsrc/src/vcpkg-tests/dependencies.cpp
+++ b/toolsrc/src/vcpkg-test/dependencies.cpp
@@ -1,4 +1,4 @@
-#include <vcpkg-tests/catch.h>
+#include <vcpkg-test/catch.h>
 
 #include <vcpkg/sourceparagraph.h>
 
diff --git a/toolsrc/src/vcpkg-test/files.cpp b/toolsrc/src/vcpkg-test/files.cpp
new file mode 100644
index 000000000..9e14cec0c
--- /dev/null
+++ b/toolsrc/src/vcpkg-test/files.cpp
@@ -0,0 +1,121 @@
+#include <vcpkg-test/catch.h>
+#include <vcpkg-test/util.h>
+
+#include <vcpkg/base/files.h>
+#include <vcpkg/base/strings.h>
+
+#include <iostream>
+#include <random>
+
+#include <vector>
+
+using vcpkg::Test::SYMLINKS_ALLOWED;
+using vcpkg::Test::TEMPORARY_DIRECTORY;
+
+namespace
+{
+    using uid = std::uniform_int_distribution<std::uint64_t>;
+
+    std::mt19937_64 get_urbg(std::uint64_t index)
+    {
+        // smallest prime > 2**63 - 1
+        return std::mt19937_64{index + 9223372036854775837ULL};
+    }
+
+    std::string get_random_filename(std::mt19937_64& urbg) { return vcpkg::Strings::b32_encode(uid{}(urbg)); }
+
+    void create_directory_tree(std::mt19937_64& urbg,
+                               vcpkg::Files::Filesystem& fs,
+                               std::uint64_t depth,
+                               const fs::path& base)
+    {
+        std::random_device rd;
+        constexpr std::uint64_t max_depth = 5;
+        constexpr std::uint64_t width = 5;
+
+        // we want ~70% of our "files" to be directories, and then a third
+        // each of the remaining ~30% to be regular files, directory symlinks,
+        // and regular symlinks
+        constexpr std::uint64_t directory_min_tag = 0;
+        constexpr std::uint64_t directory_max_tag = 6;
+        constexpr std::uint64_t regular_file_tag = 7;
+        constexpr std::uint64_t regular_symlink_tag = 8;
+        constexpr std::uint64_t directory_symlink_tag = 9;
+
+        // if we're at the max depth, we only want to build non-directories
+        std::uint64_t file_type;
+        if (depth < max_depth)
+        {
+            file_type = uid{directory_min_tag, regular_symlink_tag}(urbg);
+        }
+        else
+        {
+            file_type = uid{regular_file_tag, regular_symlink_tag}(urbg);
+        }
+
+        if (!SYMLINKS_ALLOWED && file_type > regular_file_tag)
+        {
+            file_type = regular_file_tag;
+        }
+
+        std::error_code ec;
+        if (file_type <= directory_max_tag)
+        {
+            fs.create_directory(base, ec);
+            if (ec) {
+                INFO("File that failed: " << base);
+                REQUIRE_FALSE(ec);
+            }
+
+            for (int i = 0; i < width; ++i)
+            {
+                create_directory_tree(urbg, fs, depth + 1, base / get_random_filename(urbg));
+            }
+        }
+        else if (file_type == regular_file_tag)
+        {
+            // regular file
+            fs.write_contents(base, "", ec);
+        }
+        else if (file_type == regular_symlink_tag)
+        {
+            // regular symlink
+            fs.write_contents(base, "", ec);
+            REQUIRE_FALSE(ec);
+            auto base_link = base;
+            base_link.replace_filename(base.filename().u8string() + "-link");
+            vcpkg::Test::create_symlink(base, base_link, ec);
+        }
+        else // type == directory_symlink_tag
+        {
+            // directory symlink
+            vcpkg::Test::create_directory_symlink(base / "..", base, ec);
+        }
+
+        REQUIRE_FALSE(ec);
+    }
+}
+
+TEST_CASE ("remove all", "[files]")
+{
+    auto urbg = get_urbg(0);
+
+    fs::path temp_dir = TEMPORARY_DIRECTORY / get_random_filename(urbg);
+
+    auto& fs = vcpkg::Files::get_real_filesystem();
+
+    std::error_code ec;
+    fs.create_directory(TEMPORARY_DIRECTORY, ec);
+
+    REQUIRE_FALSE(ec);
+
+    INFO("temp dir is: " << temp_dir);
+
+    create_directory_tree(urbg, fs, 0, temp_dir);
+
+    fs::path fp;
+    fs.remove_all(temp_dir, ec, fp);
+    REQUIRE_FALSE(ec);
+
+    REQUIRE_FALSE(fs.exists(temp_dir));
+}
diff --git a/toolsrc/src/vcpkg-tests/paragraph.cpp b/toolsrc/src/vcpkg-test/paragraph.cpp
index 0fb85ec69..a95879cfa 100644
--- a/toolsrc/src/vcpkg-tests/paragraph.cpp
+++ b/toolsrc/src/vcpkg-test/paragraph.cpp
@@ -1,5 +1,5 @@
-#include <vcpkg-tests/catch.h>
-#include <vcpkg-tests/util.h>
+#include <vcpkg-test/catch.h>
+#include <vcpkg-test/util.h>
 
 #include <vcpkg/base/strings.h>
 
diff --git a/toolsrc/src/vcpkg-tests/plan.cpp b/toolsrc/src/vcpkg-test/plan.cpp
index 7ecab460b..049ef2066 100644
--- a/toolsrc/src/vcpkg-tests/plan.cpp
+++ b/toolsrc/src/vcpkg-test/plan.cpp
@@ -1,5 +1,5 @@
-#include <vcpkg-tests/catch.h>
-#include <vcpkg-tests/util.h>
+#include <vcpkg-test/catch.h>
+#include <vcpkg-test/util.h>
 
 #include <vcpkg/dependencies.h>
 #include <vcpkg/sourceparagraph.h>
diff --git a/toolsrc/src/vcpkg-tests/specifier.cpp b/toolsrc/src/vcpkg-test/specifier.cpp
index 52ef044e7..330a96d78 100644
--- a/toolsrc/src/vcpkg-tests/specifier.cpp
+++ b/toolsrc/src/vcpkg-test/specifier.cpp
@@ -1,4 +1,4 @@
-#include <vcpkg-tests/catch.h>
+#include <vcpkg-test/catch.h>
 
 #include <vcpkg/base/util.h>
 #include <vcpkg/packagespec.h>
diff --git a/toolsrc/src/vcpkg-tests/statusparagraphs.cpp b/toolsrc/src/vcpkg-test/statusparagraphs.cpp
index df52ccb87..c0833e8ba 100644
--- a/toolsrc/src/vcpkg-tests/statusparagraphs.cpp
+++ b/toolsrc/src/vcpkg-test/statusparagraphs.cpp
@@ -1,5 +1,5 @@
-#include <vcpkg-tests/catch.h>
-#include <vcpkg-tests/util.h>
+#include <vcpkg-test/catch.h>
+#include <vcpkg-test/util.h>
 
 #include <vcpkg/base/util.h>
 #include <vcpkg/paragraphs.h>
diff --git a/toolsrc/src/vcpkg-test/strings.cpp b/toolsrc/src/vcpkg-test/strings.cpp
new file mode 100644
index 000000000..6b744eee6
--- /dev/null
+++ b/toolsrc/src/vcpkg-test/strings.cpp
@@ -0,0 +1,33 @@
+#include <vcpkg-test/catch.h>
+
+#include <vcpkg/base/strings.h>
+
+#include <cstdint>
+#include <utility>
+#include <vector>
+
+TEST_CASE ("b32 encoding", "[strings]")
+{
+    using u64 = std::uint64_t;
+
+    std::vector<std::pair<std::uint64_t, std::string>> map;
+
+    map.emplace_back(0, "AAAAAAAAAAAAA");
+    map.emplace_back(1, "BAAAAAAAAAAAA");
+
+    map.emplace_back(u64(1) << 32, "AAAAAAEAAAAAA");
+    map.emplace_back((u64(1) << 32) + 1, "BAAAAAEAAAAAA");
+
+    map.emplace_back(0xE4D0'1065'D11E'0229, "JRA4RIXMQAUJO");
+    map.emplace_back(0xA626'FE45'B135'07FF, "77BKTYWI6XJMK");
+    map.emplace_back(0xEE36'D228'0C31'D405, "FAVDDGAFSWN4O");
+    map.emplace_back(0x1405'64E7'FE7E'A88C, "MEK5H774ELBIB");
+    map.emplace_back(0xFFFF'FFFF'FFFF'FFFF, "777777777777P");
+
+    std::string result;
+    for (const auto& pr : map)
+    {
+        result = vcpkg::Strings::b32_encode(pr.first);
+        REQUIRE(vcpkg::Strings::b32_encode(pr.first) == pr.second);
+    }
+}
diff --git a/toolsrc/src/vcpkg-tests/supports.cpp b/toolsrc/src/vcpkg-test/supports.cpp
index c6c88bdbc..8bd386da0 100644
--- a/toolsrc/src/vcpkg-tests/supports.cpp
+++ b/toolsrc/src/vcpkg-test/supports.cpp
@@ -1,4 +1,4 @@
-#include <vcpkg-tests/catch.h>
+#include <vcpkg-test/catch.h>
 
 #include <vcpkg/sourceparagraph.h>
 
diff --git a/toolsrc/src/vcpkg-tests/update.cpp b/toolsrc/src/vcpkg-test/update.cpp
index 93a8f74a9..70b2f04c1 100644
--- a/toolsrc/src/vcpkg-tests/update.cpp
+++ b/toolsrc/src/vcpkg-test/update.cpp
@@ -1,5 +1,5 @@
-#include <vcpkg-tests/catch.h>
-#include <vcpkg-tests/util.h>
+#include <vcpkg-test/catch.h>
+#include <vcpkg-test/util.h>
 
 #include <vcpkg/base/sortedvector.h>
 
diff --git a/toolsrc/src/vcpkg-test/util.cpp b/toolsrc/src/vcpkg-test/util.cpp
new file mode 100644
index 000000000..a80ab36a0
--- /dev/null
+++ b/toolsrc/src/vcpkg-test/util.cpp
@@ -0,0 +1,177 @@
+#include <vcpkg-test/catch.h>
+#include <vcpkg-test/util.h>
+
+#include <vcpkg/base/checks.h>
+#include <vcpkg/base/files.h>
+#include <vcpkg/statusparagraph.h>
+
+// used to get the implementation specific compiler flags (i.e., __cpp_lib_filesystem)
+#include <ciso646>
+
+#include <iostream>
+#include <memory>
+
+#if defined(_WIN32)
+#include <windows.h>
+#endif
+
+#define FILESYSTEM_SYMLINK_STD 0
+#define FILESYSTEM_SYMLINK_UNIX 1
+#define FILESYSTEM_SYMLINK_NONE 2
+
+#if defined(__cpp_lib_filesystem)
+
+#define FILESYSTEM_SYMLINK FILESYSTEM_SYMLINK_STD
+#include <filesystem> // required for filesystem::create_{directory_}symlink
+
+#elif !defined(_MSC_VER)
+
+#define FILESYSTEM_SYMLINK FILESYSTEM_SYMLINK_UNIX
+#include <unistd.h>
+
+#else
+
+#define FILESYSTEM_SYMLINK FILESYSTEM_SYMLINK_NONE
+
+#endif
+
+namespace vcpkg::Test
+{
+    std::unique_ptr<vcpkg::StatusParagraph> make_status_pgh(const char* name,
+                                                            const char* depends,
+                                                            const char* default_features,
+                                                            const char* triplet)
+    {
+        using Pgh = std::unordered_map<std::string, std::string>;
+        return std::make_unique<StatusParagraph>(Pgh{{"Package", name},
+                                                     {"Version", "1"},
+                                                     {"Architecture", triplet},
+                                                     {"Multi-Arch", "same"},
+                                                     {"Depends", depends},
+                                                     {"Default-Features", default_features},
+                                                     {"Status", "install ok installed"}});
+    }
+
+    std::unique_ptr<StatusParagraph> make_status_feature_pgh(const char* name,
+                                                             const char* feature,
+                                                             const char* depends,
+                                                             const char* triplet)
+    {
+        using Pgh = std::unordered_map<std::string, std::string>;
+        return std::make_unique<StatusParagraph>(Pgh{{"Package", name},
+                                                     {"Version", "1"},
+                                                     {"Feature", feature},
+                                                     {"Architecture", triplet},
+                                                     {"Multi-Arch", "same"},
+                                                     {"Depends", depends},
+                                                     {"Status", "install ok installed"}});
+    }
+
+    PackageSpec unsafe_pspec(std::string name, Triplet t)
+    {
+        auto m_ret = PackageSpec::from_name_and_triplet(name, t);
+        REQUIRE(m_ret.has_value());
+        return m_ret.value_or_exit(VCPKG_LINE_INFO);
+    }
+
+    static bool system_allows_symlinks()
+    {
+#if defined(_WIN32)
+        if (!__cpp_lib_filesystem)
+        {
+            return false;
+        }
+
+        HKEY key;
+        bool allow_symlinks = true;
+
+        const auto status = RegOpenKeyExW(
+            HKEY_LOCAL_MACHINE, LR"(SOFTWARE\Microsoft\Windows\CurrentVersion\AppModelUnlock)", 0, 0, &key);
+
+        if (status == ERROR_FILE_NOT_FOUND)
+        {
+            allow_symlinks = false;
+            std::clog << "Symlinks are not allowed on this system\n";
+        }
+
+        if (status == ERROR_SUCCESS) RegCloseKey(key);
+
+        return allow_symlinks;
+#else
+        return true;
+#endif
+    }
+
+    static fs::path internal_temporary_directory()
+    {
+#if defined(_WIN32)
+        wchar_t* tmp = static_cast<wchar_t*>(std::calloc(32'767, 2));
+
+        if (!GetEnvironmentVariableW(L"TEMP", tmp, 32'767))
+        {
+            std::cerr << "No temporary directory found.\n";
+            std::abort();
+        }
+
+        fs::path result = tmp;
+        std::free(tmp);
+
+        return result / L"vcpkg-test";
+#else
+        return "/tmp/vcpkg-test";
+#endif
+    }
+
+    const bool SYMLINKS_ALLOWED = system_allows_symlinks();
+    const fs::path TEMPORARY_DIRECTORY = internal_temporary_directory();
+
+#if FILESYSTEM_SYMLINK == FILSYSTEM_SYMLINK_NONE
+    constexpr inline char no_filesystem_message[] =
+        "<filesystem> doesn't exist; on windows, we don't attempt to use the win32 calls to create symlinks";
+#endif
+
+    void create_symlink(const fs::path& target, const fs::path& file, std::error_code& ec)
+    {
+#if FILESYSTEM_SYMLINK == FILESYSTEM_SYMLINK_STD
+        if (SYMLINKS_ALLOWED)
+        {
+            std::filesystem::path targetp = target.native();
+            std::filesystem::path filep = file.native();
+
+            std::filesystem::create_symlink(targetp, filep);
+        }
+        else
+        {
+            vcpkg::Checks::exit_with_message(VCPKG_LINE_INFO, "Symlinks are not allowed on this system");
+        }
+#elif FILESYSTEM_SYMLINK == FILESYSTEM_SYMLINK_UNIX
+        if (symlink(target.c_str(), file.c_str()) != 0)
+        {
+            ec.assign(errno, std::system_category());
+        }
+#else
+        vcpkg::Checks::exit_with_message(VCPKG_LINE_INFO, no_filesystem_message);
+#endif
+    }
+
+    void create_directory_symlink(const fs::path& target, const fs::path& file, std::error_code& ec)
+    {
+#if FILESYSTEM_SYMLINK == FILESYSTEM_SYMLINK_STD
+        if (SYMLINKS_ALLOWED)
+        {
+            std::filesystem::path targetp = target.native();
+            std::filesystem::path filep = file.native();
+
+            std::filesystem::create_symlink(targetp, filep);
+        }
+        else
+        {
+            vcpkg::Checks::exit_with_message(VCPKG_LINE_INFO, "Symlinks are not allowed on this system");
+        }
+#elif FILESYSTEM_SYMLINK == FILESYSTEM_SYMLINK_UNIX
+        ::vcpkg::Test::create_symlink(target, file, ec);
+#else
+        vcpkg::Checks::exit_with_message(VCPKG_LINE_INFO, no_filesystem_message);
+#endif
+    }
+}
diff --git a/toolsrc/src/vcpkg-tests/util.cpp b/toolsrc/src/vcpkg-tests/util.cpp
deleted file mode 100644
index 54102f1aa..000000000
--- a/toolsrc/src/vcpkg-tests/util.cpp
+++ /dev/null
@@ -1,47 +0,0 @@
-#include <vcpkg-tests/catch.h>
-#include <vcpkg-tests/util.h>
-
-#include <vcpkg/statusparagraph.h>
-
-#include <memory>
-
-namespace vcpkg::Test
-{
-    std::unique_ptr<vcpkg::StatusParagraph> make_status_pgh(const char* name,
-                                                            const char* depends,
-                                                            const char* default_features,
-                                                            const char* triplet)
-    {
-        using Pgh = std::unordered_map<std::string, std::string>;
-        return std::make_unique<StatusParagraph>(Pgh{{"Package", name},
-                                                     {"Version", "1"},
-                                                     {"Architecture", triplet},
-                                                     {"Multi-Arch", "same"},
-                                                     {"Depends", depends},
-                                                     {"Default-Features", default_features},
-                                                     {"Status", "install ok installed"}});
-    }
-
-    std::unique_ptr<StatusParagraph> make_status_feature_pgh(const char* name,
-                                                             const char* feature,
-                                                             const char* depends,
-                                                             const char* triplet)
-    {
-        using Pgh = std::unordered_map<std::string, std::string>;
-        return std::make_unique<StatusParagraph>(Pgh{{"Package", name},
-                                                     {"Version", "1"},
-                                                     {"Feature", feature},
-                                                     {"Architecture", triplet},
-                                                     {"Multi-Arch", "same"},
-                                                     {"Depends", depends},
-                                                     {"Status", "install ok installed"}});
-    }
-
-    PackageSpec unsafe_pspec(std::string name, Triplet t)
-    {
-        auto m_ret = PackageSpec::from_name_and_triplet(name, t);
-        REQUIRE(m_ret.has_value());
-        return m_ret.value_or_exit(VCPKG_LINE_INFO);
-    }
-
-}
diff --git a/toolsrc/src/vcpkg/archives.cpp b/toolsrc/src/vcpkg/archives.cpp
index 69a916828..d22e841de 100644
--- a/toolsrc/src/vcpkg/archives.cpp
+++ b/toolsrc/src/vcpkg/archives.cpp
@@ -15,9 +15,10 @@ namespace vcpkg::Archives
 #endif
             ;
 
+        fs.remove_all(to_path, VCPKG_LINE_INFO);
+        fs.remove_all(to_path_partial, VCPKG_LINE_INFO);
+        // TODO: check this error code
         std::error_code ec;
-        fs.remove_all(to_path, ec);
-        fs.remove_all(to_path_partial, ec);
         fs.create_directories(to_path_partial, ec);
         const auto ext = archive.extension();
 #if defined(_WIN32)
diff --git a/toolsrc/src/vcpkg/base/files.cpp b/toolsrc/src/vcpkg/base/files.cpp
index 5099795e9..6c6945e44 100644
--- a/toolsrc/src/vcpkg/base/files.cpp
+++ b/toolsrc/src/vcpkg/base/files.cpp
@@ -6,6 +6,7 @@
 #include <vcpkg/base/system.print.h>
 #include <vcpkg/base/system.process.h>
 #include <vcpkg/base/util.h>
+#include <vcpkg/base/work_queue.h>
 
 #if defined(__linux__) || defined(__APPLE__)
 #include <fcntl.h>
@@ -20,6 +21,59 @@
 #include <copyfile.h>
 #endif
 
+namespace fs::detail
+{
+    file_status symlink_status_t::operator()(const path& p, std::error_code& ec) const noexcept
+    {
+#if defined(_WIN32)
+        static_cast<void>(ec);
+
+        /*
+            do not find the permissions of the file -- it's unnecessary for the
+            things that vcpkg does.
+            if one were to add support for this in the future, one should look
+            into GetFileSecurityW
+        */
+        perms permissions = perms::unknown;
+
+        WIN32_FILE_ATTRIBUTE_DATA file_attributes;
+        file_type ft = file_type::unknown;
+        if (!GetFileAttributesExW(p.c_str(), GetFileExInfoStandard, &file_attributes))
+        {
+            ft = file_type::not_found;
+        }
+        else if (file_attributes.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT)
+        {
+            // check for reparse point -- if yes, then symlink
+            ft = file_type::symlink;
+        }
+        else if (file_attributes.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)
+        {
+            ft = file_type::directory;
+        }
+        else
+        {
+            // otherwise, the file is a regular file
+            ft = file_type::regular;
+        }
+
+        return file_status(ft, permissions);
+
+#else
+        return stdfs::symlink_status(p, ec);
+#endif
+    }
+
+    file_status symlink_status_t::operator()(const path& p, vcpkg::LineInfo li) const noexcept
+    {
+        std::error_code ec;
+        auto result = symlink_status(p, ec);
+        if (ec) vcpkg::Checks::exit_with_message(li, "error getting status of path %s: %s", p.string(), ec.message());
+
+        return result;
+    }
+}
+
 namespace vcpkg::Files
 {
     static const std::regex FILESYSTEM_INVALID_CHARACTERS_REGEX = std::regex(R"([\/:*?"<>|])");
@@ -63,6 +117,25 @@ namespace vcpkg::Files
         if (ec) Checks::exit_with_message(linfo, "error writing lines: %s: %s", path.u8string(), ec.message());
     }
 
+    std::uintmax_t Filesystem::remove_all(const fs::path& path, LineInfo li)
+    {
+        std::error_code ec;
+        fs::path failure_point;
+
+        const auto result = this->remove_all(path, ec, failure_point);
+
+        if (ec)
+        {
+            Checks::exit_with_message(li,
+                                      "Failure to remove_all(%s) due to file %s: %s",
+                                      path.string(),
+                                      failure_point.string(),
+                                      ec.message());
+        }
+
+        return result;
+    }
+
     struct RealFilesystem final : Filesystem
     {
         virtual Expected<std::string> read_contents(const fs::path& file_path) const override
@@ -87,7 +160,7 @@ namespace vcpkg::Files
             file_stream.read(&output[0], length);
             file_stream.close();
 
-            return std::move(output);
+            return output;
         }
         virtual Expected<std::vector<std::string>> read_lines(const fs::path& file_path) const override
         {
@@ -105,7 +178,7 @@ namespace vcpkg::Files
             }
             file_stream.close();
 
-            return std::move(output);
+            return output;
         }
         virtual fs::path find_file_recursively_up(const fs::path& starting_dir,
                                                   const std::string& filename) const override
@@ -254,28 +327,163 @@ namespace vcpkg::Files
 #endif
         }
         virtual bool remove(const fs::path& path, std::error_code& ec) override { return fs::stdfs::remove(path, ec); }
-        virtual std::uintmax_t remove_all(const fs::path& path, std::error_code& ec) override
+        virtual std::uintmax_t remove_all(const fs::path& path, std::error_code& ec, fs::path& failure_point) override
         {
-            // Working around the currently buggy remove_all()
-            std::uintmax_t out = fs::stdfs::remove_all(path, ec);
+            /*
+                does not use the std::filesystem call since it is buggy, and can
+                have spurious errors before VS 2017 update 6, and on later versions
+                (as well as on macOS and Linux), this is just as fast and will have
+                fewer spurious errors due to locks.
+            */
+
+            /*
+                `remove` doesn't actually remove anything -- it simply moves the
+                files into a parent directory (which ends up being at `path`),
+                and then inserts `actually_remove{current_path}` into the work
+                queue.
+            */
+            struct remove
+            {
+                struct tld
+                {
+                    const fs::path& tmp_directory;
+                    std::uint64_t index;
+
+                    std::atomic<std::uintmax_t>& files_deleted;
 
-            for (int i = 0; i < 5 && this->exists(path); i++)
+                    std::mutex& ec_mutex;
+                    std::error_code& ec;
+                    fs::path& failure_point;
+                };
+
+                struct actually_remove;
+                using queue = WorkQueue<actually_remove, tld>;
+
+                /*
+                    if `current_path` is a directory, first `remove`s all
+                    elements of the directory, then calls remove.
+
+                    else, just calls remove.
+                */
+                struct actually_remove
+                {
+                    fs::path current_path;
+
+                    void operator()(tld& info, const queue& queue) const
+                    {
+                        std::error_code ec;
+                        const auto path_type = fs::symlink_status(current_path, ec).type();
+
+                        if (check_ec(ec, info, queue, current_path)) return;
+
+                        if (path_type == fs::file_type::directory)
+                        {
+                            for (const auto& entry : fs::stdfs::directory_iterator(current_path))
+                            {
+                                remove{}(entry, info, queue);
+                            }
+                        }
+
+                        if (fs::stdfs::remove(current_path, ec))
+                        {
+                            info.files_deleted.fetch_add(1, std::memory_order_relaxed);
+                        }
+                        else
+                        {
+                            check_ec(ec, info, queue, current_path);
+                        }
+                    }
+                };
+
+                static bool check_ec(const std::error_code& ec,
+                                     tld& info,
+                                     const queue& queue,
+                                     const fs::path& failure_point)
+                {
+                    if (ec)
+                    {
+                        queue.terminate();
+
+                        auto lck = std::unique_lock<std::mutex>(info.ec_mutex);
+                        if (!info.ec)
+                        {
+                            info.ec = ec;
+                            info.failure_point = failure_point;
+                        }
+
+                        return true;
+                    }
+                    else
+                    {
+                        return false;
+                    }
+                }
+
+                void operator()(const fs::path& current_path, tld& info, const queue& queue) const
+                {
+                    std::error_code ec;
+
+                    const auto tmp_name = Strings::b32_encode(info.index++);
+                    const auto tmp_path = info.tmp_directory / tmp_name;
+
+                    fs::stdfs::rename(current_path, tmp_path, ec);
+                    if (check_ec(ec, info, queue, current_path)) return;
+
+                    queue.enqueue_action(actually_remove{std::move(tmp_path)});
+                }
+            };
+
+            const auto path_type = fs::symlink_status(path, ec).type();
+
+            std::atomic<std::uintmax_t> files_deleted{0};
+
+            if (path_type == fs::file_type::directory)
             {
-                using namespace std::chrono_literals;
-                std::this_thread::sleep_for(i * 100ms);
-                out += fs::stdfs::remove_all(path, ec);
+                std::uint64_t index = 0;
+                std::mutex ec_mutex;
+
+                auto const tld_gen = [&] {
+                    index += static_cast<std::uint64_t>(1) << 32;
+                    return remove::tld{path, index, files_deleted, ec_mutex, ec, failure_point};
+                };
+
+                remove::queue queue{4, VCPKG_LINE_INFO, tld_gen};
+
+                // note: we don't actually start the queue running until the
+                // `join()`. This allows us to rename all the top-level files in
+                // peace, so that we don't get collisions.
+                auto main_tld = tld_gen();
+                for (const auto& entry : fs::stdfs::directory_iterator(path))
+                {
+                    remove{}(entry, main_tld, queue);
+                }
+
+                queue.join(VCPKG_LINE_INFO);
             }
 
-            if (this->exists(path))
+            /*
+                we need to do backoff on the removal of the top level directory,
+                since we need to place all moved files into that top level
+                directory, and so we can only delete the directory after all the
+                lower levels have been deleted.
+            */
+            for (int backoff = 0; backoff < 5; ++backoff)
             {
-                System::print2(
-                    System::Color::warning,
-                    "Some files in ",
-                    path.u8string(),
-                    " were unable to be removed. Close any editors operating in this directory and retry.\n");
+                if (backoff)
+                {
+                    using namespace std::chrono_literals;
+                    auto backoff_time = 100ms * backoff;
+                    std::this_thread::sleep_for(backoff_time);
+                }
+
+                if (fs::stdfs::remove(path, ec))
+                {
+                    files_deleted.fetch_add(1, std::memory_order_relaxed);
+                    break;
+                }
             }
 
-            return out;
+            return files_deleted;
         }
         virtual bool exists(const fs::path& path) const override { return fs::stdfs::exists(path); }
         virtual bool is_directory(const fs::path& path) const override { return fs::stdfs::is_directory(path); }
@@ -307,11 +515,11 @@ namespace vcpkg::Files
 
         virtual fs::file_status status(const fs::path& path, std::error_code& ec) const override
         {
-            return fs::stdfs::status(path, ec);
+            return fs::status(path, ec);
         }
         virtual fs::file_status symlink_status(const fs::path& path, std::error_code& ec) const override
         {
-            return fs::stdfs::symlink_status(path, ec);
+            return fs::symlink_status(path, ec);
         }
         virtual void write_contents(const fs::path& file_path, const std::string& data, std::error_code& ec) override
         {
diff --git a/toolsrc/src/vcpkg/base/strings.cpp b/toolsrc/src/vcpkg/base/strings.cpp
index 54a74a7a1..46e78a363 100644
--- a/toolsrc/src/vcpkg/base/strings.cpp
+++ b/toolsrc/src/vcpkg/base/strings.cpp
@@ -288,3 +288,43 @@ bool Strings::contains(StringView haystack, StringView needle)
 {
     return Strings::search(haystack, needle) != haystack.end();
 }
+
+namespace vcpkg::Strings
+{
+    namespace
+    {
+        template<class Integral>
+        std::string b32_encode_implementation(Integral x)
+        {
+            static_assert(std::is_integral<Integral>::value, "b64url_encode must take an integer type");
+            using Unsigned = std::make_unsigned_t<Integral>;
+            auto value = static_cast<Unsigned>(x);
+
+            // 32 values, plus the implicit \0
+            constexpr static char map[33] = "ABCDEFGHIJKLMNOP"
+                                            "QRSTUVWXYZ234567";
+
+            // log2(32)
+            constexpr static int shift = 5;
+            // 32 - 1
+            constexpr static auto mask = 31;
+
+            // ceiling(bitsize(Integral) / log2(32))
+            constexpr static auto result_size = (sizeof(value) * 8 + shift - 1) / shift;
+
+            std::string result;
+            result.reserve(result_size);
+
+            for (std::size_t i = 0; i < result_size; ++i)
+            {
+                result.push_back(map[value & mask]);
+                value >>= shift;
+            }
+
+            return result;
+        }
+    }
+
+    std::string b32_encode(std::uint64_t x) noexcept { return b32_encode_implementation(x); }
+
+}
diff --git a/toolsrc/src/vcpkg/build.cpp b/toolsrc/src/vcpkg/build.cpp
index 9d5b490e3..a5383b2a8 100644
--- a/toolsrc/src/vcpkg/build.cpp
+++ b/toolsrc/src/vcpkg/build.cpp
@@ -642,7 +642,8 @@ namespace vcpkg::Build
                 if (fs.is_directory(file)) // Will only keep the logs

                 {

                     std::error_code ec;

-                    fs.remove_all(file, ec);

+                    fs::path failure_point;

+                    fs.remove_all(file, ec, failure_point);

                 }

             }

         }

@@ -787,8 +788,8 @@ namespace vcpkg::Build
         auto& fs = paths.get_filesystem();

 

         auto pkg_path = paths.package_dir(spec);

+        fs.remove_all(pkg_path, VCPKG_LINE_INFO);

         std::error_code ec;

-        fs.remove_all(pkg_path, ec);

         fs.create_directories(pkg_path, ec);

         auto files = fs.get_files_non_recursive(pkg_path);

         Checks::check_exit(VCPKG_LINE_INFO, files.empty(), "unable to clear path: %s", pkg_path.u8string());

@@ -980,7 +981,7 @@ namespace vcpkg::Build
                 fs.rename_or_copy(tmp_failure_zip, archive_tombstone_path, ".tmp", ec);

 

                 // clean up temporary directory

-                fs.remove_all(tmp_log_path, ec);

+                fs.remove_all(tmp_log_path, VCPKG_LINE_INFO);

             }

         }

 

@@ -1143,7 +1144,7 @@ namespace vcpkg::Build
             {

                 switch (maybe_option->second)

                 {

-                case VcpkgTripletVar::TARGET_ARCHITECTURE : 

+                case VcpkgTripletVar::TARGET_ARCHITECTURE :

                     pre_build_info.target_architecture = variable_value;

                     break;

                 case VcpkgTripletVar::CMAKE_SYSTEM_NAME :

diff --git a/toolsrc/src/vcpkg/commands.exportifw.cpp b/toolsrc/src/vcpkg/commands.exportifw.cpp
index f0946110c..3d963a297 100644
--- a/toolsrc/src/vcpkg/commands.exportifw.cpp
+++ b/toolsrc/src/vcpkg/commands.exportifw.cpp
@@ -352,13 +352,15 @@ namespace vcpkg::Export::IFW
         System::print2("Generating repository ", repository_dir.generic_u8string(), "...\n");
 
         std::error_code ec;
+        fs::path failure_point;
         Files::Filesystem& fs = paths.get_filesystem();
 
-        fs.remove_all(repository_dir, ec);
+        fs.remove_all(repository_dir, ec, failure_point);
         Checks::check_exit(VCPKG_LINE_INFO,
                            !ec,
-                           "Could not remove outdated repository directory %s",
-                           repository_dir.generic_u8string());
+                           "Could not remove outdated repository directory %s due to file %s",
+                           repository_dir.generic_u8string(),
+                           failure_point.string());
 
         const auto cmd_line = Strings::format(R"("%s" --packages "%s" "%s" > nul)",
                                               repogen_exe.u8string(),
@@ -414,16 +416,18 @@ namespace vcpkg::Export::IFW
                    const VcpkgPaths& paths)
     {
         std::error_code ec;
+        fs::path failure_point;
         Files::Filesystem& fs = paths.get_filesystem();
 
         // Prepare packages directory
         const fs::path ifw_packages_dir_path = get_packages_dir_path(export_id, ifw_options, paths);
 
-        fs.remove_all(ifw_packages_dir_path, ec);
+        fs.remove_all(ifw_packages_dir_path, ec, failure_point);
         Checks::check_exit(VCPKG_LINE_INFO,
                            !ec,
-                           "Could not remove outdated packages directory %s",
-                           ifw_packages_dir_path.generic_u8string());
+                           "Could not remove outdated packages directory %s due to file %s",
+                           ifw_packages_dir_path.generic_u8string(),
+                           failure_point.string());
 
         fs.create_directory(ifw_packages_dir_path, ec);
         Checks::check_exit(
diff --git a/toolsrc/src/vcpkg/commands.portsdiff.cpp b/toolsrc/src/vcpkg/commands.portsdiff.cpp
index b30c38f43..cddc274b8 100644
--- a/toolsrc/src/vcpkg/commands.portsdiff.cpp
+++ b/toolsrc/src/vcpkg/commands.portsdiff.cpp
@@ -105,7 +105,7 @@ namespace vcpkg::Commands::PortsDiff
         std::map<std::string, VersionT> names_and_versions;
         for (auto&& port : all_ports)
             names_and_versions.emplace(port->core_paragraph->name, port->core_paragraph->version);
-        fs.remove_all(temp_checkout_path, ec);
+        fs.remove_all(temp_checkout_path, VCPKG_LINE_INFO);
         return names_and_versions;
     }
 
diff --git a/toolsrc/src/vcpkg/export.cpp b/toolsrc/src/vcpkg/export.cpp
index 88c1526c5..f306bf4e6 100644
--- a/toolsrc/src/vcpkg/export.cpp
+++ b/toolsrc/src/vcpkg/export.cpp
@@ -400,8 +400,10 @@ namespace vcpkg::Export
         Files::Filesystem& fs = paths.get_filesystem();
         const fs::path export_to_path = paths.root;
         const fs::path raw_exported_dir_path = export_to_path / export_id;
+        fs.remove_all(raw_exported_dir_path, VCPKG_LINE_INFO);
+
+        // TODO: error handling
         std::error_code ec;
-        fs.remove_all(raw_exported_dir_path, ec);
         fs.create_directory(raw_exported_dir_path, ec);
 
         // execute the plan
@@ -476,7 +478,7 @@ With a project open, go to Tools->NuGet Package Manager->Package Manager Console
 
         if (!opts.raw)
         {
-            fs.remove_all(raw_exported_dir_path, ec);
+            fs.remove_all(raw_exported_dir_path, VCPKG_LINE_INFO);
         }
     }
 
diff --git a/toolsrc/src/vcpkg/install.cpp b/toolsrc/src/vcpkg/install.cpp
index 425a4cdbc..32af57b39 100644
--- a/toolsrc/src/vcpkg/install.cpp
+++ b/toolsrc/src/vcpkg/install.cpp
@@ -354,8 +354,7 @@ namespace vcpkg::Install
             {
                 auto& fs = paths.get_filesystem();
                 const fs::path package_dir = paths.package_dir(action.spec);
-                std::error_code ec;
-                fs.remove_all(package_dir, ec);
+                fs.remove_all(package_dir, VCPKG_LINE_INFO);
             }
 
             if (action.build_options.clean_downloads == Build::CleanDownloads::YES)
diff --git a/toolsrc/src/vcpkg/remove.cpp b/toolsrc/src/vcpkg/remove.cpp
index a40b27bd7..84ec6c981 100644
--- a/toolsrc/src/vcpkg/remove.cpp
+++ b/toolsrc/src/vcpkg/remove.cpp
@@ -179,8 +179,7 @@ namespace vcpkg::Remove
         {
             System::printf("Purging package %s...\n", display_name);
             Files::Filesystem& fs = paths.get_filesystem();
-            std::error_code ec;
-            fs.remove_all(paths.packages / action.spec.dir(), ec);
+            fs.remove_all(paths.packages / action.spec.dir(), VCPKG_LINE_INFO);
             System::printf(System::Color::success, "Purging package %s... done\n", display_name);
         }
     }