add external file hashes to the binary paragraph

3 files changed, 159 insertions, 80 deletions

diff --git a/toolsrc/src/vcpkg/binaryparagraph.cpp b/toolsrc/src/vcpkg/binaryparagraph.cpp
index ad7790fe1..2e9415dab 100644
--- a/toolsrc/src/vcpkg/binaryparagraph.cpp
+++ b/toolsrc/src/vcpkg/binaryparagraph.cpp
@@ -2,6 +2,7 @@
 
 #include <vcpkg/base/checks.h>
 #include <vcpkg/base/system.print.h>
+#include <vcpkg/base/util.h>
 #include <vcpkg/binaryparagraph.h>
 #include <vcpkg/parse.h>
 
@@ -23,7 +24,7 @@ namespace vcpkg
         static const std::string MAINTAINER = "Maintainer";
         static const std::string DEPENDS = "Depends";
         static const std::string DEFAULTFEATURES = "Default-Features";
-        static const std::string TYPE = "Type";
+        static const std::string EXTERNALFILES = "External-Files";
     }
 
     BinaryParagraph::BinaryParagraph() = default;
@@ -61,8 +62,23 @@ namespace vcpkg
             this->default_features = parse_comma_list(parser.optional_field(Fields::DEFAULTFEATURES));
         }
 
-        this->type =
-            SourceParagraph::type_from_string(parser.optional_field(Fields::TYPE));
+        std::vector<std::string> external_files_or_hashes =
+            parse_comma_list(parser.optional_field(Fields::EXTERNALFILES));
+
+        if (external_files_or_hashes.size() % 2 != 0)
+        {
+            Checks::exit_with_message(
+                    VCPKG_LINE_INFO,
+                    "The External-Files field is not composed of key-value pairs for ",
+                    this->spec);
+        }
+
+        for (int i = 0; i < external_files_or_hashes.size(); i += 2)
+        {
+            external_files.emplace(
+                    std::move(external_files_or_hashes[i]),
+                    std::move(external_files_or_hashes[i+1]));
+        }
 
         if (const auto err = parser.error_info(this->spec.to_string()))
         {
@@ -124,6 +140,16 @@ namespace vcpkg
         if (!pgh.abi.empty()) out_str.append("Abi: ").append(pgh.abi).push_back('\n');
         if (!pgh.description.empty()) out_str.append("Description: ").append(pgh.description).push_back('\n');
 
-        out_str.append("Type: ").append(SourceParagraph::string_from_type(pgh.type)).push_back('\n');
+        if (!pgh.external_files.empty())
+        {
+            out_str.append("External-Files: ");
+            out_str.append(Strings::join(",",
+                        Util::fmap(
+                            pgh.external_files,
+                            [](const std::pair<std::string, std::string>& kv)
+                            {
+                                return kv.first + "," + kv.second;
+                            }))).push_back('\n');
+        }
     }
 }
diff --git a/toolsrc/src/vcpkg/build.cpp b/toolsrc/src/vcpkg/build.cpp
index 47599bfca..110a571b0 100644
--- a/toolsrc/src/vcpkg/build.cpp
+++ b/toolsrc/src/vcpkg/build.cpp
@@ -9,6 +9,7 @@
 #include <vcpkg/base/system.debug.h>

 #include <vcpkg/base/system.print.h>

 #include <vcpkg/base/system.process.h>

+#include <vcpkg/base/util.h>

 

 #include <vcpkg/build.h>

 #include <vcpkg/commands.h>

@@ -261,10 +262,12 @@ namespace vcpkg::Build
         return BinaryParagraph(source_paragraph, feature_paragraph, triplet);

     }

 

-    static std::unique_ptr<BinaryControlFile> create_binary_control_file(const SourceParagraph& source_paragraph,

-                                                                         const Triplet& triplet,

-                                                                         const BuildInfo& build_info,

-                                                                         const std::string& abi_tag)

+    static std::unique_ptr<BinaryControlFile> create_binary_control_file(

+        const SourceParagraph& source_paragraph,

+        const Triplet& triplet,

+        const BuildInfo& build_info,

+        const PreBuildInfo& pre_build_info,

+        const std::string& abi_tag)

     {

         auto bcf = std::make_unique<BinaryControlFile>();

         BinaryParagraph bpgh(source_paragraph, triplet, abi_tag);

@@ -272,6 +275,14 @@ namespace vcpkg::Build
         {

             bpgh.version = *p_ver;

         }

+

+        for (auto& file_hash : pre_build_info.external_files)

+        {

+            bpgh.external_files.emplace(

+                std::move(file_hash.first),

+                std::move(file_hash.second));

+        }

+

         bcf->core_paragraph = std::move(bpgh);

         return bcf;

     }

@@ -447,6 +458,41 @@ namespace vcpkg::Build
         return command;

     }

 

+    static std::vector<std::pair<std::string, std::string>> get_external_file_hashes(

+        const VcpkgPaths& paths,

+        const std::vector<fs::path>& files)

+    {

+        static std::map<fs::path, std::string> s_hash_cache;

+

+        const auto& fs = paths.get_filesystem();

+

+        std::vector<std::pair<std::string, std::string>> hashes;

+        for (const fs::path& external_file : files)

+        {

+            auto it_hash = s_hash_cache.find(external_file);

+

+            if (it_hash != s_hash_cache.end())

+            {

+                hashes.emplace_back(external_file, it_hash->second);

+            }

+            else if (fs.is_regular_file(external_file))

+            {

+                auto emp = s_hash_cache.emplace(external_file.u8string(),

+                                                Hash::get_file_hash(fs, external_file, "SHA1"));

+                hashes.emplace_back(external_file, emp.first->second);

+            }

+            else

+            {

+                Checks::exit_with_message(

+                        VCPKG_LINE_INFO,

+                        external_file.u8string() +

+                        " was listed as an additional file for calculating the abi, but was not found.");

+            }

+        }

+

+        return hashes;

+    }

+

     static std::string get_triplet_abi(const VcpkgPaths& paths,

                                        const PreBuildInfo& pre_build_info,

                                        const Triplet& triplet)

@@ -496,31 +542,6 @@ namespace vcpkg::Build
             s_hash_cache.emplace(triplet_file_path, hash);

         }

 

-        for (const fs::path& additional_file : pre_build_info.additional_files)

-        {

-            it_hash = s_hash_cache.find(additional_file);

-

-            if (it_hash != s_hash_cache.end())

-            {

-                hash += "-";

-                hash += it_hash->second;

-            }

-            else if (fs.is_regular_file(additional_file))

-            {

-                std::string tmp_hash = Hash::get_file_hash(fs, additional_file, "SHA1");

-                hash += "-";

-                hash += tmp_hash;

-

-                s_hash_cache.emplace(additional_file, tmp_hash);

-            }

-            else

-            {

-                Checks::exit_with_message(

-                        VCPKG_LINE_INFO,

-                        additional_file.u8string() + " was listed as an additional file for calculating the abi, but was not found.");

-            }

-        }

-

         return hash;

     }

 

@@ -572,7 +593,13 @@ namespace vcpkg::Build
         const size_t error_count =

             PostBuildLint::perform_all_checks(spec, paths, pre_build_info, build_info, config.port_dir);

 

-        auto bcf = create_binary_control_file(*config.scf.core_paragraph, triplet, build_info, abi_tag);

+        std::unique_ptr<BinaryControlFile> bcf =

+            create_binary_control_file(

+                *config.scf.core_paragraph,

+                triplet,

+                build_info,

+                pre_build_info,

+                abi_tag);

 

         if (error_count != 0)

         {

@@ -629,7 +656,9 @@ namespace vcpkg::Build
 

         std::vector<AbiEntry> abi_tag_entries(dependency_abis.begin(), dependency_abis.end());

 

-        abi_tag_entries.emplace_back(AbiEntry{"cmake", paths.get_tool_version(Tools::CMAKE)});

+        // Sorted here as the order of dependency_abis is the only

+        // non-deterministicly ordered set of AbiEntries

+        Util::sort(abi_tag_entries);

 

         // If there is an unusually large number of files in the port then

         // something suspicious is going on.  Rather than hash all of them

@@ -637,13 +666,16 @@ namespace vcpkg::Build
         const int max_port_file_count = 100;

 

         // the order of recursive_directory_iterator is undefined so save the names to sort

-        std::vector<fs::path> port_files;

+        std::vector<std::pair<std::string, std::string>> hashes_files;

         for (auto& port_file : fs::stdfs::recursive_directory_iterator(config.port_dir))

         {

             if (fs::is_regular_file(status(port_file)))

             {

-                port_files.push_back(port_file);

-                if (port_files.size() > max_port_file_count)

+                hashes_files.emplace_back(

+                    vcpkg::Hash::get_file_hash(fs, port_file, "SHA1"),

+                    port_file.path().filename());

+

+                if (hashes_files.size() > max_port_file_count)

                 {

                     abi_tag_entries.emplace_back(AbiEntry{"no_hash_max_portfile", ""});

                     break;

@@ -651,25 +683,45 @@ namespace vcpkg::Build
             }

         }

 

-        if (port_files.size() <= max_port_file_count)

+        if (hashes_files.size() <= max_port_file_count)

         {

-            std::sort(port_files.begin(), port_files.end());

+            Util::sort(hashes_files);

 

-            int counter = 0;

-            for (auto& port_file : port_files)

+            for (auto& hash_file : hashes_files)

             {

-                // When vcpkg takes a dependency on C++17 it can use fs::relative,

-                // which will give a stable ordering and better names in the key entry.

-                // this is not available in the filesystem TS so instead number the files for the key.

-                std::string key = Strings::format("file_%03d", counter++);

-                if (Debug::g_debugging)

-                {

-                    System::print2("[DEBUG] mapping ", key, " from ", port_file.u8string(), "\n");

-                }

-                abi_tag_entries.emplace_back(AbiEntry{key, vcpkg::Hash::get_file_hash(fs, port_file, "SHA1")});

+                // We've already sorted by hash so it's safe to write down the

+                // filename, which will be consistent across machines.

+                abi_tag_entries.emplace_back(

+                    AbiEntry{

+                        std::move(hash_file.second),

+                        std::move(hash_file.first)

+                    });

             }

         }

 

+        //Make a copy of the external files and their hashes, and sort by hash

+        auto additional_file_hashes = pre_build_info.external_files;

+        std::sort(

+            additional_file_hashes.begin(),

+            additional_file_hashes.end(),

+            [](const std::pair<fs::path, std::string>& l,

+               const std::pair<fs::path, std::string>& r)

+            {

+                return l.second < r.second ||

+                    (l.second == r.second && l.first < r.first);

+            });

+

+        for (auto& hash_file : additional_file_hashes)

+        {

+            abi_tag_entries.emplace_back(

+                AbiEntry{

+                    std::move(hash_file.first),

+                    std::move(hash_file.second)

+                });

+        }

+

+        abi_tag_entries.emplace_back(AbiEntry{"cmake", paths.get_tool_version(Tools::CMAKE)});

+

         abi_tag_entries.emplace_back(AbiEntry{

             "vcpkg_fixup_cmake_targets",

             vcpkg::Hash::get_file_hash(fs, paths.scripts / "cmake" / "vcpkg_fixup_cmake_targets.cmake", "SHA1")});

@@ -682,8 +734,6 @@ namespace vcpkg::Build
         if (config.build_package_options.use_head_version == UseHeadVersion::YES)

             abi_tag_entries.emplace_back(AbiEntry{"head", ""});

 

-        Util::sort(abi_tag_entries);

-

         const std::string full_abi_info =

             Strings::join("", abi_tag_entries, [](const AbiEntry& p) { return p.key + " " + p.value + "\n"; });

 

@@ -1114,12 +1164,15 @@ namespace vcpkg::Build
                 case VcpkgTripletVar::ENV_PASSTHROUGH :

                     pre_build_info.passthrough_env_vars = Strings::split(variable_value, ";");

                     break;

-                case VcpkgTripletVar::ABI_ADDITIONAL_FILES :

-                    pre_build_info.additional_files = Util::fmap(Strings::split(variable_value, ";"),

+                case VcpkgTripletVar::EXTERNAL_FILES :

+                    pre_build_info.external_files = 

+                        get_external_file_hashes(

+                            paths,

+                            Util::fmap(Strings::split(variable_value, ";"),

                             [](const std::string& path)

                             {

                                 return fs::path{path};

-                            });

+                            }));

                     break;

                 }

             }

diff --git a/toolsrc/src/vcpkg/statusparagraphs.cpp b/toolsrc/src/vcpkg/statusparagraphs.cpp
index f204ed568..4f01b9918 100644
--- a/toolsrc/src/vcpkg/statusparagraphs.cpp
+++ b/toolsrc/src/vcpkg/statusparagraphs.cpp
@@ -118,30 +118,30 @@ namespace vcpkg
         return it != end() && (*it)->is_installed();
     }
 
-    bool vcpkg::StatusParagraphs::needs_rebuild(const PackageSpec& spec)
-    {
-        auto it = find(spec);
-        if (it != end())
-        {
-            for (const std::string& dep : (*it)->package.depends)
-            {
-                PackageSpec dep_spec =
-                    PackageSpec::from_name_and_triplet(
-                            dep,
-                            spec.triplet()).value_or_exit(VCPKG_LINE_INFO);
-
-                if (needs_rebuild(dep_spec))
-                {
-                    (*it)->state = InstallState::NEEDS_REBUILD;
-                    return true;
-                }
-            }
-
-            return (*it)->needs_rebuild();
-        }
-
-        return false;
-    }
+    //bool vcpkg::StatusParagraphs::needs_rebuild(const PackageSpec& spec)
+    //{
+    //    auto it = find(spec);
+    //    if (it != end())
+    //    {
+    //        for (const std::string& dep : (*it)->package.depends)
+    //        {
+    //            PackageSpec dep_spec =
+    //                PackageSpec::from_name_and_triplet(
+    //                        dep,
+    //                        spec.triplet()).value_or_exit(VCPKG_LINE_INFO);
+
+    //            if (needs_rebuild(dep_spec))
+    //            {
+    //                (*it)->state = InstallState::NEEDS_REBUILD;
+    //                return true;
+    //            }
+    //        }
+
+    //        return (*it)->needs_rebuild();
+    //    }
+
+    //    return false;
+    //}
 
     StatusParagraphs::iterator StatusParagraphs::insert(std::unique_ptr<StatusParagraph> pgh)
     {