From f0902b35370fbebcf257de88e43ee47379185a10 Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Thu, 20 Jun 2019 11:46:55 -0700 Subject: VS 2019 16.3 deprecates . (#6968) VS 2019 16.3 will contain a couple of source-breaking changes: * will be deprecated via an impossible-to-miss preprocessor "#error The header providing std::experimental::filesystem is deprecated by Microsoft and will be REMOVED. It is superseded by the C++17 header providing std::filesystem. You can define _SILENCE_EXPERIMENTAL_FILESYSTEM_DEPRECATION_WARNING to acknowledge that you have received this warning." * will no longer include . In the long term, I believe that vcpkg should detect when it's being built with VS 2017 15.7 or newer, compile in C++17 mode, include , and use std::filesystem. (Activating this for VS 2019 16.0 or newer would also be reasonable.) Similarly for other toolsets supporting std::filesystem. In the short term, this commit makes vcpkg compatible with the upcoming deprecation. First, we need to define the silencing macro before including the appropriate header. I've chosen to define it unconditionally (without checking for platform or version), since it has no effect for other platforms or versions. Second, we need to deal with no longer including . I verified that VS 2015 Update 3 contained (back then, it simply included the header, where the experimental implementation was defined; this was later reorganized). Therefore, all of vcpkg's supported MSVC toolsets have , so we can simply always include it. I've verified that this builds with both VS 2015 Update 3 and VS 2019 16.1.3 (the current production version). --- toolsrc/include/pch.h | 5 +---- toolsrc/include/vcpkg/base/files.h | 5 +---- 2 files changed, 2 insertions(+), 8 deletions(-) (limited to 'toolsrc') diff --git a/toolsrc/include/pch.h b/toolsrc/include/pch.h index fa2c2bb72..15ec25e76 100644 --- a/toolsrc/include/pch.h +++ b/toolsrc/include/pch.h @@ -37,11 +37,8 @@ #include #include #include -#if defined(_WIN32) -#include -#else +#define _SILENCE_EXPERIMENTAL_FILESYSTEM_DEPRECATION_WARNING #include -#endif #include #include #include diff --git a/toolsrc/include/vcpkg/base/files.h b/toolsrc/include/vcpkg/base/files.h index 02e2b8db8..3ea0d6036 100644 --- a/toolsrc/include/vcpkg/base/files.h +++ b/toolsrc/include/vcpkg/base/files.h @@ -2,11 +2,8 @@ #include -#if defined(_WIN32) -#include -#else +#define _SILENCE_EXPERIMENTAL_FILESYSTEM_DEPRECATION_WARNING #include -#endif namespace fs { -- cgit v1.2.3 From 31184ac70d6b96e52eff60ee8eace9af2f24e2a2 Mon Sep 17 00:00:00 2001 From: Farwaykorse Date: Sat, 22 Jun 2019 06:26:10 +0200 Subject: Bump version to 2019.06.21 (#6987) --- toolsrc/VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'toolsrc') diff --git a/toolsrc/VERSION.txt b/toolsrc/VERSION.txt index ce62ec959..c2116e0e8 100644 --- a/toolsrc/VERSION.txt +++ b/toolsrc/VERSION.txt @@ -1 +1 @@ -"2018.11.23" \ No newline at end of file +"2019.06.21" \ No newline at end of file -- cgit v1.2.3 From f3db66b403840b24ea2612d09cca30a5285f5ea3 Mon Sep 17 00:00:00 2001 From: Victor Romero Date: Fri, 21 Jun 2019 23:50:05 -0700 Subject: Ports Overlay partial implementation (#6981) * Ports Overlay feature spec * Ports Overlay implementation * [--overlay-ports] Refactor handling of additional paths * Code cleanup * [--overlay-ports] Add help * [depend-info] Support --overlay-ports * Add method to load all ports using PathsPortFileProvider * Make PortFileProvider::load_all_control_files() const * Remove unused code * [vcpkg] Avoid double-load of source control file between Build::perform_and_exit and Build::perform_and_exit_ex * [vcpkg] Clang format * [vcpkg] Fixup build failure introduced in b069ceb2f231 * Report errors from Paragraphs::try_load_port() --- toolsrc/include/vcpkg/build.h | 2 +- toolsrc/include/vcpkg/dependencies.h | 27 ++-- toolsrc/include/vcpkg/postbuildlint.h | 3 +- toolsrc/include/vcpkg/sourceparagraph.h | 9 ++ toolsrc/include/vcpkg/vcpkgcmdarguments.h | 16 +- toolsrc/include/vcpkg/vcpkgpaths.h | 2 - toolsrc/src/tests.plan.cpp | 8 +- toolsrc/src/tests.update.cpp | 16 +- toolsrc/src/vcpkg/build.cpp | 45 +++--- toolsrc/src/vcpkg/commands.autocomplete.cpp | 3 +- toolsrc/src/vcpkg/commands.buildexternal.cpp | 9 +- toolsrc/src/vcpkg/commands.ci.cpp | 21 ++- toolsrc/src/vcpkg/commands.dependinfo.cpp | 19 ++- toolsrc/src/vcpkg/commands.edit.cpp | 1 + toolsrc/src/vcpkg/commands.search.cpp | 8 +- toolsrc/src/vcpkg/commands.upgrade.cpp | 9 +- toolsrc/src/vcpkg/dependencies.cpp | 170 +++++++++++++++++---- toolsrc/src/vcpkg/export.cpp | 5 +- toolsrc/src/vcpkg/help.cpp | 4 +- toolsrc/src/vcpkg/install.cpp | 14 +- toolsrc/src/vcpkg/postbuildlint.cpp | 5 +- toolsrc/src/vcpkg/remove.cpp | 3 +- toolsrc/src/vcpkg/update.cpp | 8 +- toolsrc/src/vcpkg/vcpkgcmdarguments.cpp | 115 +++++++++++++- toolsrc/src/vcpkg/vcpkgpaths.cpp | 3 - toolsrc/vcpkg/vcpkg.vcxproj | 4 +- toolsrc/vcpkglib/vcpkglib.vcxproj | 4 +- .../vcpkgmetricsuploader.vcxproj | 4 +- toolsrc/vcpkgtest/vcpkgtest.vcxproj | 6 +- 29 files changed, 420 insertions(+), 123 deletions(-) (limited to 'toolsrc') diff --git a/toolsrc/include/vcpkg/build.h b/toolsrc/include/vcpkg/build.h index 9044cb556..04cd7cf87 100644 --- a/toolsrc/include/vcpkg/build.h +++ b/toolsrc/include/vcpkg/build.h @@ -20,7 +20,7 @@ namespace vcpkg::Build namespace Command { void perform_and_exit_ex(const FullPackageSpec& full_spec, - const fs::path& port_dir, + const SourceControlFileLocation& scfl, const ParsedArguments& options, const VcpkgPaths& paths); diff --git a/toolsrc/include/vcpkg/dependencies.h b/toolsrc/include/vcpkg/dependencies.h index 16fdb3f73..8c2050b3d 100644 --- a/toolsrc/include/vcpkg/dependencies.h +++ b/toolsrc/include/vcpkg/dependencies.h @@ -48,7 +48,7 @@ namespace vcpkg::Dependencies const RequestType& request_type); InstallPlanAction(const PackageSpec& spec, - const SourceControlFile& scf, + const SourceControlFileLocation& scfl, const std::set& features, const RequestType& request_type, std::vector&& dependencies); @@ -57,7 +57,7 @@ namespace vcpkg::Dependencies PackageSpec spec; - Optional source_control_file; + Optional source_control_file_location; Optional installed_package; InstallPlanType plan_type; @@ -129,26 +129,31 @@ namespace vcpkg::Dependencies struct PortFileProvider { - virtual Optional get_control_file(const std::string& src_name) const = 0; + virtual Optional get_control_file(const std::string& src_name) const = 0; + virtual std::vector load_all_control_files() const = 0; }; struct MapPortFileProvider : Util::ResourceBase, PortFileProvider { - explicit MapPortFileProvider(const std::unordered_map& map); - Optional get_control_file(const std::string& src_name) const override; + explicit MapPortFileProvider(const std::unordered_map& map); + Optional get_control_file(const std::string& src_name) const override; + std::vector load_all_control_files() const override; private: - const std::unordered_map& ports; + const std::unordered_map& ports; }; struct PathsPortFileProvider : Util::ResourceBase, PortFileProvider { - explicit PathsPortFileProvider(const VcpkgPaths& paths); - Optional get_control_file(const std::string& src_name) const override; + explicit PathsPortFileProvider(const vcpkg::VcpkgPaths& paths, + const std::vector* ports_dirs_paths); + Optional get_control_file(const std::string& src_name) const override; + std::vector load_all_control_files() const override; private: - const VcpkgPaths& ports; - mutable std::unordered_map cache; + Files::Filesystem& filesystem; + std::vector ports_dirs; + mutable std::unordered_map cache; }; struct ClusterGraph; @@ -181,7 +186,7 @@ namespace vcpkg::Dependencies std::vector create_export_plan(const std::vector& specs, const StatusParagraphs& status_db); - std::vector create_feature_install_plan(const std::unordered_map& map, + std::vector create_feature_install_plan(const std::unordered_map& map, const std::vector& specs, const StatusParagraphs& status_db); diff --git a/toolsrc/include/vcpkg/postbuildlint.h b/toolsrc/include/vcpkg/postbuildlint.h index 5dcfeb8df..027619eb8 100644 --- a/toolsrc/include/vcpkg/postbuildlint.h +++ b/toolsrc/include/vcpkg/postbuildlint.h @@ -9,5 +9,6 @@ namespace vcpkg::PostBuildLint size_t perform_all_checks(const PackageSpec& spec, const VcpkgPaths& paths, const Build::PreBuildInfo& pre_build_info, - const Build::BuildInfo& build_info); + const Build::BuildInfo& build_info, + const fs::path& port_dir); } diff --git a/toolsrc/include/vcpkg/sourceparagraph.h b/toolsrc/include/vcpkg/sourceparagraph.h index d70fd4337..6232a3fd2 100644 --- a/toolsrc/include/vcpkg/sourceparagraph.h +++ b/toolsrc/include/vcpkg/sourceparagraph.h @@ -68,6 +68,15 @@ namespace vcpkg Optional find_feature(const std::string& featurename) const; }; + /// + /// Full metadata of a package: core and other features. As well as the location the SourceControlFile was loaded from. + /// + struct SourceControlFileLocation + { + std::unique_ptr source_control_file; + fs::path source_location; + }; + void print_error_message(Span> error_info_list); inline void print_error_message(const std::unique_ptr& error_info_list) { diff --git a/toolsrc/include/vcpkg/vcpkgcmdarguments.h b/toolsrc/include/vcpkg/vcpkgcmdarguments.h index de65eec28..cad013eb8 100644 --- a/toolsrc/include/vcpkg/vcpkgcmdarguments.h +++ b/toolsrc/include/vcpkg/vcpkgcmdarguments.h @@ -15,6 +15,7 @@ namespace vcpkg { std::unordered_set switches; std::unordered_map settings; + std::unordered_map> multisettings; }; struct VcpkgPaths; @@ -41,10 +42,22 @@ namespace vcpkg StringLiteral short_help_text; }; + struct CommandMultiSetting + { + constexpr CommandMultiSetting(const StringLiteral& name, const StringLiteral& short_help_text) + : name(name), short_help_text(short_help_text) + { + } + + StringLiteral name; + StringLiteral short_help_text; + }; + struct CommandOptionsStructure { Span switches; Span settings; + Span multisettings; }; struct CommandStructure @@ -74,6 +87,7 @@ namespace vcpkg std::unique_ptr vcpkg_root_dir; std::unique_ptr triplet; + std::unique_ptr> overlay_ports; Optional debug = nullopt; Optional sendmetrics = nullopt; Optional printmetrics = nullopt; @@ -88,6 +102,6 @@ namespace vcpkg ParsedArguments parse_arguments(const CommandStructure& command_structure) const; private: - std::unordered_map> optional_command_arguments; + std::unordered_map>> optional_command_arguments; }; } diff --git a/toolsrc/include/vcpkg/vcpkgpaths.h b/toolsrc/include/vcpkg/vcpkgpaths.h index 42de40d9c..b09169b02 100644 --- a/toolsrc/include/vcpkg/vcpkgpaths.h +++ b/toolsrc/include/vcpkg/vcpkgpaths.h @@ -50,8 +50,6 @@ namespace vcpkg static Expected create(const fs::path& vcpkg_root_dir, const std::string& default_vs_path); fs::path package_dir(const PackageSpec& spec) const; - fs::path port_dir(const PackageSpec& spec) const; - fs::path port_dir(const std::string& name) const; fs::path build_info_file_path(const PackageSpec& spec) const; fs::path listfile_path(const BinaryParagraph& pgh) const; diff --git a/toolsrc/src/tests.plan.cpp b/toolsrc/src/tests.plan.cpp index 238aa7032..ab24266b4 100644 --- a/toolsrc/src/tests.plan.cpp +++ b/toolsrc/src/tests.plan.cpp @@ -47,7 +47,8 @@ namespace UnitTest1 Assert::AreEqual(plan.spec.triplet().to_string().c_str(), triplet.to_string().c_str()); - Assert::AreEqual(pkg_name.c_str(), plan.source_control_file.get()->core_paragraph->name.c_str()); + auto* scfl = plan.source_control_file_location.get(); + Assert::AreEqual(pkg_name.c_str(), scfl->source_control_file->core_paragraph->name.c_str()); Assert::AreEqual(size_t(vec.size()), feature_list.size()); for (auto&& feature_name : vec) @@ -79,7 +80,7 @@ namespace UnitTest1 /// struct PackageSpecMap { - std::unordered_map map; + std::unordered_map map; Triplet triplet; PackageSpecMap(const Triplet& t = Triplet::X86_WINDOWS) noexcept { triplet = t; } @@ -94,7 +95,8 @@ namespace UnitTest1 { auto spec = PackageSpec::from_name_and_triplet(scf.core_paragraph->name, triplet); Assert::IsTrue(spec.has_value()); - map.emplace(scf.core_paragraph->name, std::move(scf)); + map.emplace(scf.core_paragraph->name, + SourceControlFileLocation{std::unique_ptr(std::move(&scf)), ""}); return PackageSpec{*spec.get()}; } }; diff --git a/toolsrc/src/tests.update.cpp b/toolsrc/src/tests.update.cpp index b6e487c17..5e3f9f3e2 100644 --- a/toolsrc/src/tests.update.cpp +++ b/toolsrc/src/tests.update.cpp @@ -21,9 +21,9 @@ namespace UnitTest1 StatusParagraphs status_db(std::move(status_paragraphs)); - std::unordered_map map; + std::unordered_map map; auto scf = unwrap(SourceControlFile::parse_control_file(Pgh{{{"Source", "a"}, {"Version", "0"}}})); - map.emplace("a", std::move(*scf)); + map.emplace("a", SourceControlFileLocation { std::move(scf), "" }); Dependencies::MapPortFileProvider provider(map); auto pkgs = SortedVector(Update::find_outdated_packages(provider, status_db), @@ -45,9 +45,9 @@ namespace UnitTest1 StatusParagraphs status_db(std::move(status_paragraphs)); - std::unordered_map map; + std::unordered_map map; auto scf = unwrap(SourceControlFile::parse_control_file(Pgh{{{"Source", "a"}, {"Version", "0"}}})); - map.emplace("a", std::move(*scf)); + map.emplace("a", SourceControlFileLocation { std::move(scf), "" }); Dependencies::MapPortFileProvider provider(map); auto pkgs = SortedVector(Update::find_outdated_packages(provider, status_db), @@ -71,9 +71,9 @@ namespace UnitTest1 StatusParagraphs status_db(std::move(status_paragraphs)); - std::unordered_map map; + std::unordered_map map; auto scf = unwrap(SourceControlFile::parse_control_file(Pgh{{{"Source", "a"}, {"Version", "0"}}})); - map.emplace("a", std::move(*scf)); + map.emplace("a", SourceControlFileLocation{ std::move(scf), "" }); Dependencies::MapPortFileProvider provider(map); auto pkgs = SortedVector(Update::find_outdated_packages(provider, status_db), @@ -92,9 +92,9 @@ namespace UnitTest1 StatusParagraphs status_db(std::move(status_paragraphs)); - std::unordered_map map; + std::unordered_map map; auto scf = unwrap(SourceControlFile::parse_control_file(Pgh{{{"Source", "a"}, {"Version", "2"}}})); - map.emplace("a", std::move(*scf)); + map.emplace("a", SourceControlFileLocation{ std::move(scf), "" }); Dependencies::MapPortFileProvider provider(map); auto pkgs = SortedVector(Update::find_outdated_packages(provider, status_db), diff --git a/toolsrc/src/vcpkg/build.cpp b/toolsrc/src/vcpkg/build.cpp index daff7e6c8..059a09432 100644 --- a/toolsrc/src/vcpkg/build.cpp +++ b/toolsrc/src/vcpkg/build.cpp @@ -23,6 +23,7 @@ #include using vcpkg::Build::BuildResult; +using vcpkg::Dependencies::PathsPortFileProvider; using vcpkg::Parse::ParseControlErrorInfo; using vcpkg::Parse::ParseExpected; @@ -34,34 +35,26 @@ namespace vcpkg::Build::Command static constexpr StringLiteral OPTION_CHECKS_ONLY = "--checks-only"; void perform_and_exit_ex(const FullPackageSpec& full_spec, - const fs::path& port_dir, + const SourceControlFileLocation& scfl, const ParsedArguments& options, const VcpkgPaths& paths) { const PackageSpec& spec = full_spec.package_spec; + const auto& scf = *scfl.source_control_file; if (Util::Sets::contains(options.switches, OPTION_CHECKS_ONLY)) { const auto pre_build_info = Build::PreBuildInfo::from_triplet_file(paths, spec.triplet()); const auto build_info = Build::read_build_info(paths.get_filesystem(), paths.build_info_file_path(spec)); - const size_t error_count = PostBuildLint::perform_all_checks(spec, paths, pre_build_info, build_info); + const size_t error_count = + PostBuildLint::perform_all_checks(spec, paths, pre_build_info, build_info, scfl.source_location); Checks::check_exit(VCPKG_LINE_INFO, error_count == 0); Checks::exit_success(VCPKG_LINE_INFO); } - const ParseExpected source_control_file = - Paragraphs::try_load_port(paths.get_filesystem(), port_dir); - - if (!source_control_file.has_value()) - { - print_error_message(source_control_file.error()); - Checks::exit_fail(VCPKG_LINE_INFO); - } - - const auto& scf = source_control_file.value_or_exit(VCPKG_LINE_INFO); Checks::check_exit(VCPKG_LINE_INFO, - spec.name() == scf->core_paragraph->name, + spec.name() == scf.core_paragraph->name, "The Source field inside the CONTROL file does not match the port directory: '%s' != '%s'", - scf->core_paragraph->name, + scf.core_paragraph->name, spec.name()); const StatusParagraphs status_db = database_load_check(paths); @@ -80,7 +73,7 @@ namespace vcpkg::Build::Command features_as_set.emplace("core"); const Build::BuildPackageConfig build_config{ - *scf, spec.triplet(), fs::path{port_dir}, build_package_options, features_as_set}; + scf, spec.triplet(), fs::path(scfl.source_location), build_package_options, features_as_set}; const auto build_timer = Chrono::ElapsedTimer::create_started(); const auto result = Build::build_package(paths, build_config, status_db); @@ -128,10 +121,19 @@ namespace vcpkg::Build::Command // Build only takes a single package and all dependencies must already be installed const ParsedArguments options = args.parse_arguments(COMMAND_STRUCTURE); std::string first_arg = args.command_arguments.at(0); + const FullPackageSpec spec = Input::check_and_get_full_package_spec( std::move(first_arg), default_triplet, COMMAND_STRUCTURE.example_text); + Input::check_triplet(spec.package_spec.triplet(), paths); - perform_and_exit_ex(spec, paths.port_dir(spec.package_spec), options, paths); + + PathsPortFileProvider provider(paths, args.overlay_ports.get()); + const auto port_name = spec.package_spec.name(); + const auto* scfl = provider.get_control_file(port_name).get(); + + Checks::check_exit(VCPKG_LINE_INFO, scfl != nullptr, "Error: Couldn't find port '%s'", port_name); + + perform_and_exit_ex(spec, *scfl, options, paths); } } @@ -360,6 +362,12 @@ namespace vcpkg::Build auto& fs = paths.get_filesystem(); const Triplet& triplet = spec.triplet(); + if (!Strings::starts_with(Strings::ascii_to_lowercase(config.port_dir.u8string()), + Strings::ascii_to_lowercase(paths.ports.u8string()))) + { + System::printf("-- Installing port from location: %s\n", config.port_dir.u8string()); + } + #if !defined(_WIN32) // TODO: remove when vcpkg.exe is in charge for acquiring tools. Change introduced in vcpkg v0.0.107. // bootstrap should have already downloaded ninja, but making sure it is present in case it was deleted. @@ -428,7 +436,8 @@ namespace vcpkg::Build } const BuildInfo build_info = read_build_info(fs, paths.build_info_file_path(spec)); - const size_t error_count = PostBuildLint::perform_all_checks(spec, paths, pre_build_info, build_info); + const size_t error_count = + PostBuildLint::perform_all_checks(spec, paths, pre_build_info, build_info, config.port_dir); auto bcf = create_binary_control_file(*config.scf.core_paragraph, triplet, build_info, abi_tag); @@ -705,7 +714,7 @@ namespace vcpkg::Build } } - System::print2("Could not locate cached archive: ", archive_path.u8string(), "\n"); + System::printf("Could not locate cached archive: %s\n", archive_path.u8string()); ExtendedBuildResult result = do_build_package_and_clean_buildtrees( paths, pre_build_info, spec, maybe_abi_tag_and_file.value_or(AbiTagAndFile{}).tag, config); diff --git a/toolsrc/src/vcpkg/commands.autocomplete.cpp b/toolsrc/src/vcpkg/commands.autocomplete.cpp index afef518eb..3cf4dd98f 100644 --- a/toolsrc/src/vcpkg/commands.autocomplete.cpp +++ b/toolsrc/src/vcpkg/commands.autocomplete.cpp @@ -90,7 +90,8 @@ namespace vcpkg::Commands::Autocomplete const auto port_name = match[2].str(); const auto triplet_prefix = match[3].str(); - auto maybe_port = Paragraphs::try_load_port(paths.get_filesystem(), paths.port_dir(port_name)); + // TODO: Support autocomplete for ports in --overlay-ports + auto maybe_port = Paragraphs::try_load_port(paths.get_filesystem(), paths.ports / port_name); if (maybe_port.error()) { Checks::exit_success(VCPKG_LINE_INFO); diff --git a/toolsrc/src/vcpkg/commands.buildexternal.cpp b/toolsrc/src/vcpkg/commands.buildexternal.cpp index 19b89c2f5..1c3c511c2 100644 --- a/toolsrc/src/vcpkg/commands.buildexternal.cpp +++ b/toolsrc/src/vcpkg/commands.buildexternal.cpp @@ -23,7 +23,12 @@ namespace vcpkg::Commands::BuildExternal std::string(args.command_arguments.at(0)), default_triplet, COMMAND_STRUCTURE.example_text); Input::check_triplet(spec.package_spec.triplet(), paths); - const fs::path port_dir = args.command_arguments.at(1); - Build::Command::perform_and_exit_ex(spec, port_dir, options, paths); + auto overlays = args.overlay_ports ? *args.overlay_ports : std::vector(); + overlays.insert(overlays.begin(), args.command_arguments.at(1)); + Dependencies::PathsPortFileProvider provider(paths, &overlays); + auto maybe_scfl = provider.get_control_file(spec.package_spec.name()); + Checks::check_exit( + VCPKG_LINE_INFO, maybe_scfl.has_value(), "could not load control file for %s", spec.package_spec.name()); + Build::Command::perform_and_exit_ex(spec, maybe_scfl.value_or_exit(VCPKG_LINE_INFO), options, paths); } } diff --git a/toolsrc/src/vcpkg/commands.ci.cpp b/toolsrc/src/vcpkg/commands.ci.cpp index de66d917b..8cab79504 100644 --- a/toolsrc/src/vcpkg/commands.ci.cpp +++ b/toolsrc/src/vcpkg/commands.ci.cpp @@ -232,12 +232,17 @@ namespace vcpkg::Commands::CI { // determine abi tag std::string abi; - if (auto scf = p->source_control_file.get()) + if (auto scfl = p->source_control_file_location.get()) { auto triplet = p->spec.triplet(); const Build::BuildPackageConfig build_config{ - *scf, triplet, paths.port_dir(p->spec), build_options, p->feature_list}; + *scfl->source_control_file, + triplet, + static_cast(scfl->source_location), + build_options, + p->feature_list + }; auto dependency_abis = Util::fmap(p->computed_dependencies, [&](const PackageSpec& spec) -> Build::AbiEntry { @@ -351,7 +356,8 @@ namespace vcpkg::Commands::CI } StatusParagraphs status_db = database_load_check(paths); - const auto& paths_port_file = Dependencies::PathsPortFileProvider(paths); + + Dependencies::PathsPortFileProvider provider(paths, args.overlay_ports.get()); const Build::BuildPackageOptions install_plan_options = { Build::UseHeadVersion::NO, @@ -369,7 +375,10 @@ namespace vcpkg::Commands::CI XunitTestResults xunitTestResults; - std::vector all_ports = Install::get_all_port_names(paths); + std::vector all_ports = + Util::fmap(provider.load_all_control_files(), [](auto&& scfl) -> std::string { + return scfl->source_control_file.get()->core_paragraph->name; + }); std::vector results; auto timer = Chrono::ElapsedTimer::create_started(); for (const Triplet& triplet : triplets) @@ -378,13 +387,13 @@ namespace vcpkg::Commands::CI xunitTestResults.push_collection(triplet.canonical_name()); - Dependencies::PackageGraph pgraph(paths_port_file, status_db); + Dependencies::PackageGraph pgraph(provider, status_db); std::vector specs = PackageSpec::to_package_specs(all_ports, triplet); // Install the default features for every package auto all_feature_specs = Util::fmap(specs, [](auto& spec) { return FeatureSpec(spec, ""); }); auto split_specs = - find_unknown_ports_for_ci(paths, exclusions_set, paths_port_file, all_feature_specs, purge_tombstones); + find_unknown_ports_for_ci(paths, exclusions_set, provider, all_feature_specs, purge_tombstones); auto feature_specs = FullPackageSpec::to_feature_specs(split_specs->unknown); for (auto&& fspec : feature_specs) diff --git a/toolsrc/src/vcpkg/commands.dependinfo.cpp b/toolsrc/src/vcpkg/commands.dependinfo.cpp index 48a289fa5..c0800a4b5 100644 --- a/toolsrc/src/vcpkg/commands.dependinfo.cpp +++ b/toolsrc/src/vcpkg/commands.dependinfo.cpp @@ -6,6 +6,9 @@ #include #include #include +#include + +using vcpkg::Dependencies::PathsPortFileProvider; namespace vcpkg::Commands::DependInfo { @@ -35,7 +38,7 @@ namespace vcpkg::Commands::DependInfo return output; } - std::string create_dot_as_string(const std::vector>& source_control_files) + std::string create_dot_as_string(const std::vector& source_control_files) { int empty_node_count = 0; @@ -64,7 +67,7 @@ namespace vcpkg::Commands::DependInfo return s; } - std::string create_dgml_as_string(const std::vector>& source_control_files) + std::string create_dgml_as_string(const std::vector& source_control_files) { std::string s; s.append(""); @@ -109,7 +112,7 @@ namespace vcpkg::Commands::DependInfo } std::string create_graph_as_string(const std::unordered_set& switches, - const std::vector>& source_control_files) + const std::vector& source_control_files) { if (Util::Sets::contains(switches, OPTION_DOT)) { @@ -124,7 +127,7 @@ namespace vcpkg::Commands::DependInfo void build_dependencies_list(std::set& packages_to_keep, const std::string& requested_package, - const std::vector>& source_control_files, + const std::vector& source_control_files, const std::unordered_set& switches) { const auto source_control_file = @@ -154,7 +157,11 @@ namespace vcpkg::Commands::DependInfo { const ParsedArguments options = args.parse_arguments(COMMAND_STRUCTURE); - auto source_control_files = Paragraphs::load_all_ports(paths.get_filesystem(), paths.ports); + // TODO: Optimize implementation, current implementation needs to load all ports from disk which is too slow. + PathsPortFileProvider provider(paths, args.overlay_ports.get()); + auto source_control_files = Util::fmap(provider.load_all_control_files(), [](auto&& scfl) -> const SourceControlFile * { + return scfl->source_control_file.get(); + }); if (args.command_arguments.size() >= 1) { @@ -178,7 +185,7 @@ namespace vcpkg::Commands::DependInfo for (auto&& source_control_file : source_control_files) { - const SourceParagraph& source_paragraph = *source_control_file->core_paragraph; + const SourceParagraph& source_paragraph = *source_control_file->core_paragraph.get(); const auto s = Strings::join(", ", source_paragraph.depends, [](const Dependency& d) { return d.name(); }); System::print2(source_paragraph.name, ": ", s, "\n"); } diff --git a/toolsrc/src/vcpkg/commands.edit.cpp b/toolsrc/src/vcpkg/commands.edit.cpp index 98176b2b0..f2f0b1569 100644 --- a/toolsrc/src/vcpkg/commands.edit.cpp +++ b/toolsrc/src/vcpkg/commands.edit.cpp @@ -79,6 +79,7 @@ namespace vcpkg::Commands::Edit const auto& fs = paths.get_filesystem(); auto packages = fs.get_files_non_recursive(paths.packages); + // TODO: Support edit for --overlay-ports return Util::fmap(ports, [&](const std::string& port_name) -> std::string { const auto portpath = paths.ports / port_name; const auto portfile = portpath / "portfile.cmake"; diff --git a/toolsrc/src/vcpkg/commands.search.cpp b/toolsrc/src/vcpkg/commands.search.cpp index a0afd69e1..3d8387ee1 100644 --- a/toolsrc/src/vcpkg/commands.search.cpp +++ b/toolsrc/src/vcpkg/commands.search.cpp @@ -7,6 +7,9 @@ #include #include #include +#include + +using vcpkg::Dependencies::PathsPortFileProvider; namespace vcpkg::Commands::Search { @@ -63,7 +66,10 @@ namespace vcpkg::Commands::Search const ParsedArguments options = args.parse_arguments(COMMAND_STRUCTURE); const bool full_description = Util::Sets::contains(options.switches, OPTION_FULLDESC); - auto source_paragraphs = Paragraphs::load_all_ports(paths.get_filesystem(), paths.ports); + PathsPortFileProvider provider(paths, args.overlay_ports.get()); + auto source_paragraphs = Util::fmap(provider.load_all_control_files(), [](auto&& port) -> const SourceControlFile * { + return port->source_control_file.get(); + }); if (args.command_arguments.empty()) { diff --git a/toolsrc/src/vcpkg/commands.upgrade.cpp b/toolsrc/src/vcpkg/commands.upgrade.cpp index 29815ca94..77183ceaf 100644 --- a/toolsrc/src/vcpkg/commands.upgrade.cpp +++ b/toolsrc/src/vcpkg/commands.upgrade.cpp @@ -43,7 +43,8 @@ namespace vcpkg::Commands::Upgrade StatusParagraphs status_db = database_load_check(paths); - Dependencies::PathsPortFileProvider provider(paths); + // Load ports from ports dirs + Dependencies::PathsPortFileProvider provider(paths, args.overlay_ports.get()); Dependencies::PackageGraph graph(provider, status_db); // input sanitization @@ -85,12 +86,12 @@ namespace vcpkg::Commands::Upgrade not_installed.push_back(spec); } - auto maybe_scf = provider.get_control_file(spec.name()); - if (auto p_scf = maybe_scf.get()) + auto maybe_scfl = provider.get_control_file(spec.name()); + if (auto p_scfl = maybe_scfl.get()) { if (it != status_db.end()) { - if (p_scf->core_paragraph->version != (*it)->package.version) + if (p_scfl->source_control_file->core_paragraph->version != (*it)->package.version) { to_upgrade.push_back(spec); } diff --git a/toolsrc/src/vcpkg/dependencies.cpp b/toolsrc/src/vcpkg/dependencies.cpp index f4a05e7dc..df472515f 100644 --- a/toolsrc/src/vcpkg/dependencies.cpp +++ b/toolsrc/src/vcpkg/dependencies.cpp @@ -22,7 +22,7 @@ namespace vcpkg::Dependencies struct ClusterSource { - const SourceControlFile* scf = nullptr; + const SourceControlFileLocation* scfl = nullptr; std::unordered_map> build_edges; }; @@ -92,12 +92,12 @@ namespace vcpkg::Dependencies if (it == m_graph.end()) { // Load on-demand from m_provider - auto maybe_scf = m_provider.get_control_file(spec.name()); + auto maybe_scfl = m_provider.get_control_file(spec.name()); auto& clust = m_graph[spec]; clust.spec = spec; - if (auto p_scf = maybe_scf.get()) + if (auto p_scfl = maybe_scfl.get()) { - clust.source = cluster_from_scf(*p_scf, clust.spec.triplet()); + clust.source = cluster_from_scf(*p_scfl, clust.spec.triplet()); } return clust; } @@ -105,15 +105,17 @@ namespace vcpkg::Dependencies } private: - static ClusterSource cluster_from_scf(const SourceControlFile& scf, Triplet t) + static ClusterSource cluster_from_scf(const SourceControlFileLocation& scfl, Triplet t) { ClusterSource ret; - ret.build_edges.emplace("core", filter_dependencies_to_specs(scf.core_paragraph->depends, t)); + ret.build_edges.emplace("core", + filter_dependencies_to_specs(scfl.source_control_file->core_paragraph->depends, + t)); - for (const auto& feature : scf.feature_paragraphs) + for (const auto& feature : scfl.source_control_file->feature_paragraphs) ret.build_edges.emplace(feature->name, filter_dependencies_to_specs(feature->depends, t)); - ret.scf = &scf; + ret.scfl = &scfl; return ret; } @@ -151,12 +153,12 @@ namespace vcpkg::Dependencies } InstallPlanAction::InstallPlanAction(const PackageSpec& spec, - const SourceControlFile& scf, + const SourceControlFileLocation& scfl, const std::set& features, const RequestType& request_type, std::vector&& dependencies) : spec(spec) - , source_control_file(scf) + , source_control_file_location(scfl) , plan_type(InstallPlanType::BUILD_AND_INSTALL) , request_type(request_type) , build_options{} @@ -268,37 +270,145 @@ namespace vcpkg::Dependencies return left->spec.name() < right->spec.name(); } - MapPortFileProvider::MapPortFileProvider(const std::unordered_map& map) : ports(map) - { - } + MapPortFileProvider::MapPortFileProvider(const std::unordered_map& map) + : ports(map) + {} - Optional MapPortFileProvider::get_control_file(const std::string& spec) const + Optional MapPortFileProvider::get_control_file(const std::string& spec) const { auto scf = ports.find(spec); if (scf == ports.end()) return nullopt; return scf->second; } - PathsPortFileProvider::PathsPortFileProvider(const VcpkgPaths& paths) : ports(paths) {} + std::vector MapPortFileProvider::load_all_control_files() const + { + return Util::fmap(ports, [](auto&& kvpair) -> const SourceControlFileLocation * { return &kvpair.second; }); + } - Optional PathsPortFileProvider::get_control_file(const std::string& spec) const + PathsPortFileProvider::PathsPortFileProvider(const vcpkg::VcpkgPaths& paths, + const std::vector* ports_dirs_paths) + : filesystem(paths.get_filesystem()) + { + if (ports_dirs_paths) + { + for (auto&& overlay_path : *ports_dirs_paths) + { + if (!overlay_path.empty()) + { + auto overlay = fs::stdfs::canonical(fs::u8path(overlay_path)); + + Checks::check_exit(VCPKG_LINE_INFO, + filesystem.exists(overlay), + "Error: Path \"%s\" does not exist", + overlay.string()); + + Checks::check_exit(VCPKG_LINE_INFO, + fs::stdfs::is_directory(overlay), + "Error: Path \"%s\" must be a directory", + overlay.string()); + + ports_dirs.emplace_back(overlay); + } + } + } + ports_dirs.emplace_back(paths.ports); + } + + Optional PathsPortFileProvider::get_control_file(const std::string& spec) const { auto cache_it = cache.find(spec); if (cache_it != cache.end()) { return cache_it->second; } - Parse::ParseExpected source_control_file = - Paragraphs::try_load_port(ports.get_filesystem(), ports.port_dir(spec)); - if (auto scf = source_control_file.get()) + for (auto&& ports_dir : ports_dirs) { - auto it = cache.emplace(spec, std::move(*scf->get())); - return it.first->second; + // Try loading individual port + if (filesystem.exists(ports_dir / "CONTROL")) + { + auto maybe_scf = Paragraphs::try_load_port(filesystem, ports_dir); + if (auto scf = maybe_scf.get()) + { + if (scf->get()->core_paragraph->name == spec) + { + SourceControlFileLocation scfl{ std::move(*scf), ports_dir }; + auto it = cache.emplace(spec, std::move(scfl)); + return it.first->second; + } + } + else + { + vcpkg::print_error_message(maybe_scf.error()); + Checks::exit_with_message(VCPKG_LINE_INFO, + "Error: Failed to load port from %s", + spec, ports_dir.u8string()); + } + } + + auto found_scf = Paragraphs::try_load_port(filesystem, ports_dir / spec); + if (auto scf = found_scf.get()) + { + if (scf->get()->core_paragraph->name == spec) + { + SourceControlFileLocation scfl{ std::move(*scf), ports_dir / spec }; + auto it = cache.emplace(spec, std::move(scfl)); + return it.first->second; + } + } } + return nullopt; } + std::vector PathsPortFileProvider::load_all_control_files() const + { + // Reload cache with ports contained in all ports_dirs + cache.clear(); + std::vector ret; + for (auto&& ports_dir : ports_dirs) + { + // Try loading individual port + if (filesystem.exists(ports_dir / "CONTROL")) + { + auto maybe_scf = Paragraphs::try_load_port(filesystem, ports_dir); + if (auto scf = maybe_scf.get()) + { + auto port_name = scf->get()->core_paragraph->name; + if (cache.find(port_name) == cache.end()) + { + SourceControlFileLocation scfl{ std::move(*scf), ports_dir }; + auto it = cache.emplace(port_name, std::move(scfl)); + ret.emplace_back(&it.first->second); + } + } + else + { + vcpkg::print_error_message(maybe_scf.error()); + Checks::exit_with_message(VCPKG_LINE_INFO, + "Error: Failed to load port from %s", + ports_dir.u8string()); + } + continue; + } + + // Try loading all ports inside ports_dir + auto found_scf = Paragraphs::load_all_ports(filesystem, ports_dir); + for (auto&& scf : found_scf) + { + auto port_name = scf->core_paragraph->name; + if (cache.find(port_name) == cache.end()) + { + SourceControlFileLocation scfl{ std::move(scf), ports_dir / port_name }; + auto it = cache.emplace(port_name, std::move(scfl)); + ret.emplace_back(&it.first->second); + } + } + } + return ret; + } + std::vector create_remove_plan(const std::vector& specs, const StatusParagraphs& status_db) { @@ -495,10 +605,11 @@ namespace vcpkg::Dependencies VCPKG_LINE_INFO, "Error: Cannot find definition for package `%s`.", cluster.spec.name()); } + auto&& control_file = *p_source->scfl->source_control_file.get(); if (feature.empty()) { // Add default features for this package. This is an exact reference, so ignore prevent_default_features. - for (auto&& default_feature : p_source->scf->core_paragraph.get()->default_features) + for (auto&& default_feature : control_file.core_paragraph.get()->default_features) { auto res = mark_plus(default_feature, cluster, graph, graph_plan, prevent_default_features); if (res != MarkPlusResult::SUCCESS) @@ -513,7 +624,7 @@ namespace vcpkg::Dependencies if (feature == "*") { - for (auto&& fpgh : p_source->scf->feature_paragraphs) + for (auto&& fpgh : control_file.feature_paragraphs) { auto res = mark_plus(fpgh->name, cluster, graph, graph_plan, prevent_default_features); @@ -590,7 +701,8 @@ namespace vcpkg::Dependencies // Check if any default features have been added auto& previous_df = p_installed->ipv.core->package.default_features; - for (auto&& default_feature : p_source->scf->core_paragraph->default_features) + auto&& control_file = *p_source->scfl->source_control_file.get(); + for (auto&& default_feature : control_file.core_paragraph->default_features) { if (std::find(previous_df.begin(), previous_df.end(), default_feature) == previous_df.end()) { @@ -635,7 +747,7 @@ namespace vcpkg::Dependencies /// Map of all source control files in the current environment. /// Feature specifications to resolve dependencies for. /// Status of installed packages in the current environment. - std::vector create_feature_install_plan(const std::unordered_map& map, + std::vector create_feature_install_plan(const std::unordered_map& map, const std::vector& specs, const StatusParagraphs& status_db) { @@ -698,7 +810,11 @@ namespace vcpkg::Dependencies if (p_cluster->transient_uninstalled) { // If it will be transiently uninstalled, we need to issue a full installation command - auto pscf = p_cluster->source.value_or_exit(VCPKG_LINE_INFO).scf; + auto* pscfl = p_cluster->source.value_or_exit(VCPKG_LINE_INFO).scfl; + Checks::check_exit(VCPKG_LINE_INFO, + pscfl != nullptr, + "Error: Expected a SourceControlFileLocation to exist"); + auto&& scfl = *pscfl; auto dep_specs = Util::fmap(m_graph_plan->install_graph.adjacency_list(p_cluster), [](ClusterPtr const& p) { return p->spec; }); @@ -706,7 +822,7 @@ namespace vcpkg::Dependencies plan.emplace_back(InstallPlanAction{ p_cluster->spec, - *pscf, + scfl, p_cluster->to_install_features, p_cluster->request_type, std::move(dep_specs), diff --git a/toolsrc/src/vcpkg/export.cpp b/toolsrc/src/vcpkg/export.cpp index 23fc7441a..88c1526c5 100644 --- a/toolsrc/src/vcpkg/export.cpp +++ b/toolsrc/src/vcpkg/export.cpp @@ -488,7 +488,10 @@ With a project open, go to Tools->NuGet Package Manager->Package Manager Console // create the plan const StatusParagraphs status_db = database_load_check(paths); - Dependencies::PathsPortFileProvider provider(paths); + + // Load ports from ports dirs + Dependencies::PathsPortFileProvider provider(paths, args.overlay_ports.get()); + std::vector export_plan = Dependencies::create_export_plan(opts.specs, status_db); Checks::check_exit(VCPKG_LINE_INFO, !export_plan.empty(), "Export plan cannot be empty"); diff --git a/toolsrc/src/vcpkg/help.cpp b/toolsrc/src/vcpkg/help.cpp index b92e9ca59..6ee573dfc 100644 --- a/toolsrc/src/vcpkg/help.cpp +++ b/toolsrc/src/vcpkg/help.cpp @@ -111,10 +111,12 @@ namespace vcpkg::Help " vcpkg contact Display contact information to send feedback\n" "\n" "Options:\n" - " --triplet Specify the target architecture triplet.\n" + " --triplet Specify the target architecture triplet\n" " (default: " ENVVAR(VCPKG_DEFAULT_TRIPLET) // ", see 'vcpkg help triplet')\n" "\n" + " --overlay-ports= Specify directories to be used when searching for ports\n" + "\n" " --vcpkg-root Specify the vcpkg root " "directory\n" " (default: " ENVVAR(VCPKG_ROOT) // diff --git a/toolsrc/src/vcpkg/install.cpp b/toolsrc/src/vcpkg/install.cpp index 88b15fe7a..781629236 100644 --- a/toolsrc/src/vcpkg/install.cpp +++ b/toolsrc/src/vcpkg/install.cpp @@ -330,9 +330,10 @@ namespace vcpkg::Install System::printf("Building package %s...\n", display_name_with_features); auto result = [&]() -> Build::ExtendedBuildResult { - const Build::BuildPackageConfig build_config{action.source_control_file.value_or_exit(VCPKG_LINE_INFO), + const auto& scfl = action.source_control_file_location.value_or_exit(VCPKG_LINE_INFO); + const Build::BuildPackageConfig build_config{*scfl.source_control_file, action.spec.triplet(), - paths.port_dir(action.spec), + static_cast(scfl.source_location), action.build_options, action.feature_list}; return Build::build_package(paths, build_config, status_db); @@ -652,13 +653,10 @@ namespace vcpkg::Install Build::FailOnTombstone::NO, }; - auto all_ports = Paragraphs::load_all_ports(fs, paths.ports); - std::unordered_map scf_map; - for (auto&& port : all_ports) - scf_map[port->core_paragraph->name] = std::move(*port); - MapPortFileProvider provider(scf_map); + //// Load ports from ports dirs + PathsPortFileProvider provider(paths, args.overlay_ports.get()); - // Note: action_plan will hold raw pointers to SourceControlFiles from this map + // Note: action_plan will hold raw pointers to SourceControlFileLocations from this map std::vector action_plan = create_feature_install_plan(provider, FullPackageSpec::to_feature_specs(specs), status_db); diff --git a/toolsrc/src/vcpkg/postbuildlint.cpp b/toolsrc/src/vcpkg/postbuildlint.cpp index d6581c2b4..c760a034f 100644 --- a/toolsrc/src/vcpkg/postbuildlint.cpp +++ b/toolsrc/src/vcpkg/postbuildlint.cpp @@ -857,14 +857,15 @@ namespace vcpkg::PostBuildLint size_t perform_all_checks(const PackageSpec& spec, const VcpkgPaths& paths, const PreBuildInfo& pre_build_info, - const BuildInfo& build_info) + const BuildInfo& build_info, + const fs::path& port_dir) { System::print2("-- Performing post-build validation\n"); const size_t error_count = perform_all_checks_and_return_error_count(spec, paths, pre_build_info, build_info); if (error_count != 0) { - const fs::path portfile = paths.ports / spec.name() / "portfile.cmake"; + const fs::path portfile = port_dir / "portfile.cmake"; System::print2(System::Color::error, "Found ", error_count, diff --git a/toolsrc/src/vcpkg/remove.cpp b/toolsrc/src/vcpkg/remove.cpp index f997667ac..a40b27bd7 100644 --- a/toolsrc/src/vcpkg/remove.cpp +++ b/toolsrc/src/vcpkg/remove.cpp @@ -229,7 +229,8 @@ namespace vcpkg::Remove Checks::exit_fail(VCPKG_LINE_INFO); } - Dependencies::PathsPortFileProvider provider(paths); + // Load ports from ports dirs + Dependencies::PathsPortFileProvider provider(paths, args.overlay_ports.get()); specs = Util::fmap(Update::find_outdated_packages(provider, status_db), [](auto&& outdated) { return outdated.spec; }); diff --git a/toolsrc/src/vcpkg/update.cpp b/toolsrc/src/vcpkg/update.cpp index 2c52d5952..6320bae5b 100644 --- a/toolsrc/src/vcpkg/update.cpp +++ b/toolsrc/src/vcpkg/update.cpp @@ -23,10 +23,10 @@ namespace vcpkg::Update for (auto&& ipv : installed_packages) { const auto& pgh = ipv.core; - auto maybe_scf = provider.get_control_file(pgh->package.spec.name()); - if (auto p_scf = maybe_scf.get()) + auto maybe_scfl = provider.get_control_file(pgh->package.spec.name()); + if (auto p_scfl = maybe_scfl.get()) { - auto&& port_version = p_scf->core_paragraph->version; + auto&& port_version = p_scfl->source_control_file->core_paragraph->version; auto&& installed_version = pgh->package.version; if (installed_version != port_version) { @@ -57,7 +57,7 @@ namespace vcpkg::Update const StatusParagraphs status_db = database_load_check(paths); - Dependencies::PathsPortFileProvider provider(paths); + Dependencies::PathsPortFileProvider provider(paths, args.overlay_ports.get()); const auto outdated_packages = SortedVector(find_outdated_packages(provider, status_db), &OutdatedPackage::compare_by_name); diff --git a/toolsrc/src/vcpkg/vcpkgcmdarguments.cpp b/toolsrc/src/vcpkg/vcpkgcmdarguments.cpp index 8565c28f9..21bf4d028 100644 --- a/toolsrc/src/vcpkg/vcpkgcmdarguments.cpp +++ b/toolsrc/src/vcpkg/vcpkgcmdarguments.cpp @@ -45,6 +45,45 @@ namespace vcpkg option_field = new_setting; } + static void parse_multivalue(const std::string* arg_begin, + const std::string* arg_end, + const std::string& option_name, + std::unique_ptr>& option_field) + { + if (arg_begin == arg_end) + { + System::print2(System::Color::error, "Error: expected value after ", option_name, '\n'); + Metrics::g_metrics.lock()->track_property("error", "error option name"); + Help::print_usage(); + Checks::exit_fail(VCPKG_LINE_INFO); + } + + if (!option_field) + { + option_field = std::make_unique>(); + } + option_field->emplace_back(*arg_begin); + } + + static void parse_cojoined_multivalue(std::string new_value, + const std::string& option_name, + std::unique_ptr>& option_field) + { + if (new_value.empty()) + { + System::print2(System::Color::error, "Error: expected value after ", option_name, '\n'); + Metrics::g_metrics.lock()->track_property("error", "error option name"); + Help::print_usage(); + Checks::exit_fail(VCPKG_LINE_INFO); + } + + if (!option_field) + { + option_field = std::make_unique>(); + } + option_field->emplace_back(std::move(new_value)); + } + VcpkgCmdArguments VcpkgCmdArguments::create_from_command_line(const int argc, const CommandLineCharType* const* const argv) { @@ -117,6 +156,13 @@ namespace vcpkg parse_value(arg_begin, arg_end, "--triplet", args.triplet); continue; } + if (Strings::starts_with(arg, "--overlay-ports=")) + { + parse_cojoined_multivalue(arg.substr(sizeof("--overlay-ports=") - 1), + "--overlay-ports", + args.overlay_ports); + continue; + } if (arg == "--debug") { parse_switch(true, "debug", args.debug); @@ -166,7 +212,21 @@ namespace vcpkg const auto eq_pos = arg.find('='); if (eq_pos != std::string::npos) { - args.optional_command_arguments.emplace(arg.substr(0, eq_pos), arg.substr(eq_pos + 1)); + const auto& key = arg.substr(0, eq_pos); + const auto& value = arg.substr(eq_pos + 1); + + auto it = args.optional_command_arguments.find(key); + if (args.optional_command_arguments.end() == it) + { + args.optional_command_arguments.emplace(key, std::vector { value }); + } + else + { + if (auto* maybe_values = it->second.get()) + { + maybe_values->emplace_back(value); + } + } } else { @@ -264,7 +324,51 @@ namespace vcpkg } else { - output.settings.emplace(option.name, it->second.value_or_exit(VCPKG_LINE_INFO)); + const auto& value = it->second.value_or_exit(VCPKG_LINE_INFO); + if (value.front().empty()) + { + // Fail when not given a value, e.g.: "vcpkg install sqlite3 --additional-ports=" + System::printf( + System::Color::error, "Error: The option '%s' must be passed an argument.\n", option.name); + failed = true; + } + else + { + output.settings.emplace(option.name, value.front()); + options_copy.erase(it); + } + } + } + } + + for (auto&& option : command_structure.options.multisettings) + { + const auto it = options_copy.find(option.name); + if (it != options_copy.end()) + { + if (!it->second.has_value()) + { + // Not having a string value indicates it was passed like '--a' + System::printf( + System::Color::error, "Error: The option '%s' must be passed an argument.\n", option.name); + failed = true; + } + else + { + const auto& value = it->second.value_or_exit(VCPKG_LINE_INFO); + for (auto&& v : value) + { + if (v.empty()) + { + System::printf( + System::Color::error, "Error: The option '%s' must be passed an argument.\n", option.name); + failed = true; + } + else + { + output.multisettings[option.name].emplace_back(v); + } + } options_copy.erase(it); } } @@ -306,7 +410,14 @@ namespace vcpkg { System::printf(" %-40s %s\n", (option.name + "=..."), option.short_help_text); } + for (auto&& option : command_structure.options.multisettings) + { + System::printf(" %-40s %s\n", (option.name + "=..."), option.short_help_text); + } System::printf(" %-40s %s\n", "--triplet ", "Set the default triplet for unqualified packages"); + System::printf(" %-40s %s\n", + "--overlay-ports=", + "Specify directories to be used when searching for ports"); System::printf(" %-40s %s\n", "--vcpkg-root ", "Specify the vcpkg directory to use instead of current directory or tool directory"); diff --git a/toolsrc/src/vcpkg/vcpkgpaths.cpp b/toolsrc/src/vcpkg/vcpkgpaths.cpp index 512bcfc35..562a18c9b 100644 --- a/toolsrc/src/vcpkg/vcpkgpaths.cpp +++ b/toolsrc/src/vcpkg/vcpkgpaths.cpp @@ -81,9 +81,6 @@ namespace vcpkg fs::path VcpkgPaths::package_dir(const PackageSpec& spec) const { return this->packages / spec.dir(); } - fs::path VcpkgPaths::port_dir(const PackageSpec& spec) const { return this->ports / spec.name(); } - fs::path VcpkgPaths::port_dir(const std::string& name) const { return this->ports / name; } - fs::path VcpkgPaths::build_info_file_path(const PackageSpec& spec) const { return this->package_dir(spec) / "BUILD_INFO"; diff --git a/toolsrc/vcpkg/vcpkg.vcxproj b/toolsrc/vcpkg/vcpkg.vcxproj index 8edea2244..06d849a74 100644 --- a/toolsrc/vcpkg/vcpkg.vcxproj +++ b/toolsrc/vcpkg/vcpkg.vcxproj @@ -21,8 +21,8 @@ {34671B80-54F9-46F5-8310-AC429C11D4FB} vcpkg - 8.1 - v140 + 10.0 + v142 diff --git a/toolsrc/vcpkglib/vcpkglib.vcxproj b/toolsrc/vcpkglib/vcpkglib.vcxproj index a64eb42fd..2a6853b8a 100644 --- a/toolsrc/vcpkglib/vcpkglib.vcxproj +++ b/toolsrc/vcpkglib/vcpkglib.vcxproj @@ -21,8 +21,8 @@ {B98C92B7-2874-4537-9D46-D14E5C237F04} vcpkglib - 8.1 - v140 + 10.0 + v142 diff --git a/toolsrc/vcpkgmetricsuploader/vcpkgmetricsuploader.vcxproj b/toolsrc/vcpkgmetricsuploader/vcpkgmetricsuploader.vcxproj index e533d0e15..2e689c7fc 100644 --- a/toolsrc/vcpkgmetricsuploader/vcpkgmetricsuploader.vcxproj +++ b/toolsrc/vcpkgmetricsuploader/vcpkgmetricsuploader.vcxproj @@ -21,8 +21,8 @@ {7D6FDEEB-B299-4A23-85EE-F67C4DED47BE} vcpkgmetricsuploader - 8.1 - v140 + 10.0 + v142 diff --git a/toolsrc/vcpkgtest/vcpkgtest.vcxproj b/toolsrc/vcpkgtest/vcpkgtest.vcxproj index 4cda29461..a7517ec54 100644 --- a/toolsrc/vcpkgtest/vcpkgtest.vcxproj +++ b/toolsrc/vcpkgtest/vcpkgtest.vcxproj @@ -48,8 +48,8 @@ {F27B8DB0-1279-4AF8-A2E3-1D49C4F0220D} Win32Proj vcpkgtest - 8.1 - v140 + 10.0 + v142 @@ -84,7 +84,7 @@ - + $(SolutionDir)msbuild.x86.debug\$(ProjectName)\ $(SolutionDir)msbuild.x86.debug\ -- cgit v1.2.3 From 9e565e986789cf273de12fe1b07ce31800d10417 Mon Sep 17 00:00:00 2001 From: Victor Romero Date: Mon, 24 Jun 2019 12:09:48 -0700 Subject: [--overlay-ports] Show location of overriden ports during install plan (#7002) * [--overlay-ports] Show source location of overlayed ports during install plan * Code cleanup * Code cleanup --- toolsrc/include/vcpkg/dependencies.h | 4 +++- toolsrc/src/vcpkg/commands.ci.cpp | 2 +- toolsrc/src/vcpkg/commands.upgrade.cpp | 2 +- toolsrc/src/vcpkg/dependencies.cpp | 40 +++++++++++++++++++++++++++++----- toolsrc/src/vcpkg/install.cpp | 2 +- 5 files changed, 41 insertions(+), 9 deletions(-) (limited to 'toolsrc') diff --git a/toolsrc/include/vcpkg/dependencies.h b/toolsrc/include/vcpkg/dependencies.h index 8c2050b3d..964158026 100644 --- a/toolsrc/include/vcpkg/dependencies.h +++ b/toolsrc/include/vcpkg/dependencies.h @@ -199,5 +199,7 @@ namespace vcpkg::Dependencies const StatusParagraphs& status_db, const CreateInstallPlanOptions& options = {}); - void print_plan(const std::vector& action_plan, const bool is_recursive = true); + void print_plan(const std::vector& action_plan, + const bool is_recursive = true, + const fs::path& default_ports_dir = ""); } diff --git a/toolsrc/src/vcpkg/commands.ci.cpp b/toolsrc/src/vcpkg/commands.ci.cpp index 8cab79504..c12c26ff7 100644 --- a/toolsrc/src/vcpkg/commands.ci.cpp +++ b/toolsrc/src/vcpkg/commands.ci.cpp @@ -451,7 +451,7 @@ namespace vcpkg::Commands::CI if (is_dry_run) { - Dependencies::print_plan(action_plan); + Dependencies::print_plan(action_plan, true, paths.ports); } else { diff --git a/toolsrc/src/vcpkg/commands.upgrade.cpp b/toolsrc/src/vcpkg/commands.upgrade.cpp index 77183ceaf..1e64b2eb6 100644 --- a/toolsrc/src/vcpkg/commands.upgrade.cpp +++ b/toolsrc/src/vcpkg/commands.upgrade.cpp @@ -171,7 +171,7 @@ namespace vcpkg::Commands::Upgrade } } - Dependencies::print_plan(plan, true); + Dependencies::print_plan(plan, true, paths.ports); if (!no_dry_run) { diff --git a/toolsrc/src/vcpkg/dependencies.cpp b/toolsrc/src/vcpkg/dependencies.cpp index df472515f..b604c9acf 100644 --- a/toolsrc/src/vcpkg/dependencies.cpp +++ b/toolsrc/src/vcpkg/dependencies.cpp @@ -123,6 +123,27 @@ namespace vcpkg::Dependencies const PortFileProvider& m_provider; }; + std::string to_output_string(RequestType request_type, + const CStringView s, + const Build::BuildPackageOptions& options, + const fs::path& install_port_path, + const fs::path& default_port_path) + { + if (!default_port_path.empty() + && !Strings::case_insensitive_ascii_starts_with(install_port_path.u8string(), + default_port_path.u8string())) + { + const char* const from_head = options.use_head_version == Build::UseHeadVersion::YES ? " (from HEAD)" : ""; + switch (request_type) + { + case RequestType::AUTO_SELECTED: return Strings::format(" * %s%s -- %s", s, from_head, install_port_path.u8string()); + case RequestType::USER_REQUESTED: return Strings::format(" %s%s -- %s", s, from_head, install_port_path.u8string()); + default: Checks::unreachable(VCPKG_LINE_INFO); + } + } + return to_output_string(request_type, s, options); + } + std::string to_output_string(RequestType request_type, const CStringView s, const Build::BuildPackageOptions& options) @@ -131,7 +152,7 @@ namespace vcpkg::Dependencies switch (request_type) { - case RequestType::AUTO_SELECTED: return Strings::format(" * %s%s", s, from_head); + case RequestType::AUTO_SELECTED: return Strings::format(" * %s%s", s, from_head); case RequestType::USER_REQUESTED: return Strings::format(" %s%s", s, from_head); default: Checks::unreachable(VCPKG_LINE_INFO); } @@ -141,7 +162,7 @@ namespace vcpkg::Dependencies { switch (request_type) { - case RequestType::AUTO_SELECTED: return Strings::format(" * %s", s); + case RequestType::AUTO_SELECTED: return Strings::format(" * %s", s); case RequestType::USER_REQUESTED: return Strings::format(" %s", s); default: Checks::unreachable(VCPKG_LINE_INFO); } @@ -893,7 +914,7 @@ namespace vcpkg::Dependencies PackageGraph::~PackageGraph() = default; - void print_plan(const std::vector& action_plan, const bool is_recursive) + void print_plan(const std::vector& action_plan, const bool is_recursive, const fs::path& default_ports_dir) { std::vector remove_plans; std::vector rebuilt_plans; @@ -948,8 +969,17 @@ namespace vcpkg::Dependencies std::sort(already_installed_plans.begin(), already_installed_plans.end(), &InstallPlanAction::compare_by_name); std::sort(excluded.begin(), excluded.end(), &InstallPlanAction::compare_by_name); - static auto actions_to_output_string = [](const std::vector& v) { - return Strings::join("\n", v, [](const InstallPlanAction* p) { + static auto actions_to_output_string = [&](const std::vector& v) { + return Strings::join("\n", v, [&](const InstallPlanAction* p) { + if (auto * pscfl = p->source_control_file_location.get()) + { + return to_output_string(p->request_type, + p->displayname(), + p->build_options, + pscfl->source_location, + default_ports_dir); + } + return to_output_string(p->request_type, p->displayname(), p->build_options); }); }; diff --git a/toolsrc/src/vcpkg/install.cpp b/toolsrc/src/vcpkg/install.cpp index 781629236..de19c360a 100644 --- a/toolsrc/src/vcpkg/install.cpp +++ b/toolsrc/src/vcpkg/install.cpp @@ -684,7 +684,7 @@ namespace vcpkg::Install Metrics::g_metrics.lock()->track_property("installplan", specs_string); - Dependencies::print_plan(action_plan, is_recursive); + Dependencies::print_plan(action_plan, is_recursive, paths.ports); if (dry_run) { -- cgit v1.2.3 From 35e985d3ccf60235bc4881df4d934610cd507090 Mon Sep 17 00:00:00 2001 From: Victor Romero Date: Thu, 27 Jun 2019 12:20:12 -0700 Subject: Triplets Overlay Implementation (#7053) * Triplets Overlay Implementation * Use cache for get_triplet_file_path() * Code cleanup --- toolsrc/include/vcpkg/vcpkgcmdarguments.h | 1 + toolsrc/include/vcpkg/vcpkgpaths.h | 11 ++++-- toolsrc/src/vcpkg.cpp | 4 +- toolsrc/src/vcpkg/build.cpp | 13 +++++-- toolsrc/src/vcpkg/help.cpp | 2 + toolsrc/src/vcpkg/vcpkgcmdarguments.cpp | 30 +++++---------- toolsrc/src/vcpkg/vcpkgpaths.cpp | 61 +++++++++++++++++++++++++------ 7 files changed, 83 insertions(+), 39 deletions(-) (limited to 'toolsrc') diff --git a/toolsrc/include/vcpkg/vcpkgcmdarguments.h b/toolsrc/include/vcpkg/vcpkgcmdarguments.h index cad013eb8..ff13ae6bf 100644 --- a/toolsrc/include/vcpkg/vcpkgcmdarguments.h +++ b/toolsrc/include/vcpkg/vcpkgcmdarguments.h @@ -88,6 +88,7 @@ namespace vcpkg std::unique_ptr vcpkg_root_dir; std::unique_ptr triplet; std::unique_ptr> overlay_ports; + std::unique_ptr> overlay_triplets; Optional debug = nullopt; Optional sendmetrics = nullopt; Optional printmetrics = nullopt; diff --git a/toolsrc/include/vcpkg/vcpkgpaths.h b/toolsrc/include/vcpkg/vcpkgpaths.h index b09169b02..a30e0c653 100644 --- a/toolsrc/include/vcpkg/vcpkgpaths.h +++ b/toolsrc/include/vcpkg/vcpkgpaths.h @@ -47,14 +47,17 @@ namespace vcpkg struct VcpkgPaths { - static Expected create(const fs::path& vcpkg_root_dir, const std::string& default_vs_path); + static Expected create(const fs::path& vcpkg_root_dir, + const std::string& default_vs_path, + const std::vector* triplets_dirs); fs::path package_dir(const PackageSpec& spec) const; fs::path build_info_file_path(const PackageSpec& spec) const; fs::path listfile_path(const BinaryParagraph& pgh) const; - - const std::vector& get_available_triplets() const; + bool is_valid_triplet(const Triplet& t) const; + const std::vector& get_available_triplets() const; + const fs::path get_triplet_file_path(const Triplet& triplet) const; fs::path root; fs::path packages; @@ -93,7 +96,9 @@ namespace vcpkg Lazy> toolsets_vs2013; fs::path default_vs_path; + std::vector triplets_dirs; mutable std::unique_ptr m_tool_cache; + mutable vcpkg::Cache m_triplets_cache; }; } diff --git a/toolsrc/src/vcpkg.cpp b/toolsrc/src/vcpkg.cpp index e02bdc71f..5da97b136 100644 --- a/toolsrc/src/vcpkg.cpp +++ b/toolsrc/src/vcpkg.cpp @@ -118,7 +118,9 @@ static void inner(const VcpkgCmdArguments& args) auto default_vs_path = System::get_environment_variable("VCPKG_VISUAL_STUDIO_PATH").value_or(""); - const Expected expected_paths = VcpkgPaths::create(vcpkg_root_dir, default_vs_path); + const Expected expected_paths = VcpkgPaths::create(vcpkg_root_dir, + default_vs_path, + args.overlay_triplets.get()); Checks::check_exit(VCPKG_LINE_INFO, !expected_paths.error(), "Error: Invalid vcpkg root directory %s: %s", diff --git a/toolsrc/src/vcpkg/build.cpp b/toolsrc/src/vcpkg/build.cpp index 059a09432..f826a4865 100644 --- a/toolsrc/src/vcpkg/build.cpp +++ b/toolsrc/src/vcpkg/build.cpp @@ -361,9 +361,15 @@ namespace vcpkg::Build { auto& fs = paths.get_filesystem(); const Triplet& triplet = spec.triplet(); + const auto& triplet_file_path = paths.get_triplet_file_path(spec.triplet()).u8string(); - if (!Strings::starts_with(Strings::ascii_to_lowercase(config.port_dir.u8string()), - Strings::ascii_to_lowercase(paths.ports.u8string()))) + if (!Strings::case_insensitive_ascii_starts_with(triplet_file_path, + paths.triplets.u8string())) + { + System::printf("-- Loading triplet configuration from: %s\n", triplet_file_path); + } + if (!Strings::case_insensitive_ascii_starts_with(config.port_dir.u8string(), + paths.ports.u8string())) { System::printf("-- Installing port from location: %s\n", config.port_dir.u8string()); } @@ -390,6 +396,7 @@ namespace vcpkg::Build {"PORT", config.scf.core_paragraph->name}, {"CURRENT_PORT_DIR", config.port_dir}, {"TARGET_TRIPLET", spec.triplet().canonical_name()}, + {"TARGET_TRIPLET_FILE", triplet_file_path}, {"VCPKG_PLATFORM_TOOLSET", toolset.version.c_str()}, {"VCPKG_USE_HEAD_VERSION", Util::Enum::to_bool(config.build_package_options.use_head_version) ? "1" : "0"}, {"DOWNLOADS", paths.downloads}, @@ -890,7 +897,7 @@ namespace vcpkg::Build const fs::path& cmake_exe_path = paths.get_tool_exe(Tools::CMAKE); const fs::path ports_cmake_script_path = paths.scripts / "get_triplet_environment.cmake"; - const fs::path triplet_file_path = paths.triplets / (triplet.canonical_name() + ".cmake"); + const fs::path triplet_file_path = paths.get_triplet_file_path(triplet); const auto cmd_launch_cmake = System::make_cmake_cmd(cmake_exe_path, ports_cmake_script_path, diff --git a/toolsrc/src/vcpkg/help.cpp b/toolsrc/src/vcpkg/help.cpp index 6ee573dfc..896f62661 100644 --- a/toolsrc/src/vcpkg/help.cpp +++ b/toolsrc/src/vcpkg/help.cpp @@ -117,6 +117,8 @@ namespace vcpkg::Help "\n" " --overlay-ports= Specify directories to be used when searching for ports\n" "\n" + " --overlay-triplets= Specify directories containing triplets files\n" + "\n" " --vcpkg-root Specify the vcpkg root " "directory\n" " (default: " ENVVAR(VCPKG_ROOT) // diff --git a/toolsrc/src/vcpkg/vcpkgcmdarguments.cpp b/toolsrc/src/vcpkg/vcpkgcmdarguments.cpp index 21bf4d028..3c1452d47 100644 --- a/toolsrc/src/vcpkg/vcpkgcmdarguments.cpp +++ b/toolsrc/src/vcpkg/vcpkgcmdarguments.cpp @@ -45,26 +45,6 @@ namespace vcpkg option_field = new_setting; } - static void parse_multivalue(const std::string* arg_begin, - const std::string* arg_end, - const std::string& option_name, - std::unique_ptr>& option_field) - { - if (arg_begin == arg_end) - { - System::print2(System::Color::error, "Error: expected value after ", option_name, '\n'); - Metrics::g_metrics.lock()->track_property("error", "error option name"); - Help::print_usage(); - Checks::exit_fail(VCPKG_LINE_INFO); - } - - if (!option_field) - { - option_field = std::make_unique>(); - } - option_field->emplace_back(*arg_begin); - } - static void parse_cojoined_multivalue(std::string new_value, const std::string& option_name, std::unique_ptr>& option_field) @@ -163,6 +143,13 @@ namespace vcpkg args.overlay_ports); continue; } + if (Strings::starts_with(arg, "--overlay-triplets=")) + { + parse_cojoined_multivalue(arg.substr(sizeof("--overlay-triplets=") - 1), + "--overlay-triplets", + args.overlay_triplets); + continue; + } if (arg == "--debug") { parse_switch(true, "debug", args.debug); @@ -418,6 +405,9 @@ namespace vcpkg System::printf(" %-40s %s\n", "--overlay-ports=", "Specify directories to be used when searching for ports"); + System::printf(" %-40s %s\n", + "--overlay-triplets=", + "Specify directories containing triplets files"); System::printf(" %-40s %s\n", "--vcpkg-root ", "Specify the vcpkg directory to use instead of current directory or tool directory"); diff --git a/toolsrc/src/vcpkg/vcpkgpaths.cpp b/toolsrc/src/vcpkg/vcpkgpaths.cpp index 562a18c9b..909fbeb44 100644 --- a/toolsrc/src/vcpkg/vcpkgpaths.cpp +++ b/toolsrc/src/vcpkg/vcpkgpaths.cpp @@ -13,7 +13,9 @@ namespace vcpkg { - Expected VcpkgPaths::create(const fs::path& vcpkg_root_dir, const std::string& default_vs_path) + Expected VcpkgPaths::create(const fs::path& vcpkg_root_dir, + const std::string& default_vs_path, + const std::vector* triplets_dirs) { std::error_code ec; const fs::path canonical_vcpkg_root_dir = fs::stdfs::canonical(vcpkg_root_dir, ec); @@ -76,6 +78,20 @@ namespace vcpkg paths.ports_cmake = paths.scripts / "ports.cmake"; + if (triplets_dirs) + { + for (auto&& triplets_dir : *triplets_dirs) + { + auto path = fs::u8path(triplets_dir); + Checks::check_exit(VCPKG_LINE_INFO, + paths.get_filesystem().exists(path), + "Error: Path does not exist '%s'", + triplets_dir); + paths.triplets_dirs.emplace_back(fs::stdfs::canonical(path)); + } + } + paths.triplets_dirs.emplace_back(fs::stdfs::canonical(paths.root / "triplets")); + return paths; } @@ -91,26 +107,47 @@ namespace vcpkg return this->vcpkg_dir_info / (pgh.fullstem() + ".list"); } + bool VcpkgPaths::is_valid_triplet(const Triplet& t) const + { + const auto it = Util::find_if(this->get_available_triplets(), [&](auto&& available_triplet) { + return t.canonical_name() == available_triplet; + }); + return it != this->get_available_triplets().cend(); + } + const std::vector& VcpkgPaths::get_available_triplets() const { return this->available_triplets.get_lazy([this]() -> std::vector { std::vector output; - for (auto&& path : this->get_filesystem().get_files_non_recursive(this->triplets)) + for (auto&& triplets_dir : triplets_dirs) { - output.push_back(path.stem().filename().string()); + for (auto&& path : this->get_filesystem().get_files_non_recursive(triplets_dir)) + { + output.push_back(path.stem().filename().string()); + } } - Util::sort(output); - + Util::sort_unique_erase(output); return output; - }); + }); } - bool VcpkgPaths::is_valid_triplet(const Triplet& t) const - { - const auto it = Util::find_if(this->get_available_triplets(), [&](auto&& available_triplet) { - return t.canonical_name() == available_triplet; - }); - return it != this->get_available_triplets().cend(); + const fs::path VcpkgPaths::get_triplet_file_path(const Triplet& triplet) const + { + return m_triplets_cache.get_lazy(triplet, [&]()-> auto { + for (auto&& triplet_dir : triplets_dirs) + { + auto&& path = triplet_dir / (triplet.canonical_name() + ".cmake"); + if (this->get_filesystem().exists(path)) + { + return path; + } + } + + Checks::exit_with_message(VCPKG_LINE_INFO, + "Error: Triplet file %s.cmake not found", + triplet.canonical_name()); + }); + } const fs::path& VcpkgPaths::get_tool_exe(const std::string& tool) const -- cgit v1.2.3 From b9b2a38c7bd9fd66a5b324b4f398024ad2569d60 Mon Sep 17 00:00:00 2001 From: Robert Schumacher Date: Sat, 29 Jun 2019 23:05:09 -0700 Subject: [vcpkg-integrate] Improve spelling, help, and autocomplete. (#7095) --- toolsrc/src/vcpkg/commands.integrate.cpp | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) (limited to 'toolsrc') diff --git a/toolsrc/src/vcpkg/commands.integrate.cpp b/toolsrc/src/vcpkg/commands.integrate.cpp index 6921a5390..fda9e2b11 100644 --- a/toolsrc/src/vcpkg/commands.integrate.cpp +++ b/toolsrc/src/vcpkg/commands.integrate.cpp @@ -409,7 +409,7 @@ With a project open, go to Tools->NuGet Package Manager->Package Manager Console Checks::exit_with_code(VCPKG_LINE_INFO, rc); } -#elif defined(__unix__) +#else static void integrate_bash(const VcpkgPaths& paths) { const auto home_path = System::get_environment_variable("HOME").value_or_exit(VCPKG_LINE_INFO); @@ -458,11 +458,12 @@ With a project open, go to Tools->NuGet Package Manager->Package Manager Console "first use\n" " vcpkg integrate remove Remove user-wide integration\n" " vcpkg integrate project Generate a referencing nuget package for individual VS project use\n" - " vcpkg integrate powershell Enable PowerShell Tab-Completion\n"; + " vcpkg integrate powershell Enable PowerShell tab-completion\n"; #else const char* const INTEGRATE_COMMAND_HELPSTRING = " vcpkg integrate install Make installed packages available user-wide.\n" - " vcpkg integrate remove Remove user-wide integration\n"; + " vcpkg integrate remove Remove user-wide integration\n" + " vcpkg integrate bash Enable bash tab-completion\n"; #endif namespace Subcommand @@ -476,7 +477,15 @@ With a project open, go to Tools->NuGet Package Manager->Package Manager Console static std::vector valid_arguments(const VcpkgPaths&) { - return {Subcommand::INSTALL, Subcommand::REMOVE, Subcommand::PROJECT, Subcommand::POWERSHELL, Subcommand::BASH}; + return + { + Subcommand::INSTALL, Subcommand::REMOVE, +#if defined(_WIN32) + Subcommand::PROJECT, Subcommand::POWERSHELL, +#else + Subcommand::BASH, +#endif + }; } const CommandStructure COMMAND_STRUCTURE = { @@ -510,7 +519,7 @@ With a project open, go to Tools->NuGet Package Manager->Package Manager Console { return integrate_powershell(paths); } -#elif defined(__unix__) +#else if (args.command_arguments[0] == Subcommand::BASH) { return integrate_bash(paths); -- cgit v1.2.3 From 8e747d659c40775ce2a5a2e5d0230e4fd659ef57 Mon Sep 17 00:00:00 2001 From: Phil Christensen Date: Sun, 30 Jun 2019 00:15:08 -0700 Subject: [vcpkg] fail archived port install when decompression fails (#7086) * [vcpkg] fail port install when decompression fails * [vcpkg] clang-format --- toolsrc/src/vcpkg/build.cpp | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) (limited to 'toolsrc') diff --git a/toolsrc/src/vcpkg/build.cpp b/toolsrc/src/vcpkg/build.cpp index f826a4865..9694bce4c 100644 --- a/toolsrc/src/vcpkg/build.cpp +++ b/toolsrc/src/vcpkg/build.cpp @@ -594,7 +594,7 @@ namespace vcpkg::Build return nullopt; } - static void decompress_archive(const VcpkgPaths& paths, const PackageSpec& spec, const fs::path& archive_path) + static int decompress_archive(const VcpkgPaths& paths, const PackageSpec& spec, const fs::path& archive_path) { auto& fs = paths.get_filesystem(); @@ -608,12 +608,13 @@ namespace vcpkg::Build #if defined(_WIN32) auto&& seven_zip_exe = paths.get_tool_exe(Tools::SEVEN_ZIP); - System::cmd_execute_clean(Strings::format( + int result = System::cmd_execute_clean(Strings::format( R"("%s" x "%s" -o"%s" -y >nul)", seven_zip_exe.u8string(), archive_path.u8string(), pkg_path.u8string())); #else - System::cmd_execute_clean( + int result = System::cmd_execute_clean( Strings::format(R"(unzip -qq "%s" "-d%s")", archive_path.u8string(), pkg_path.u8string())); #endif + return result; } // Compress the source directory into the destination file. @@ -699,11 +700,16 @@ namespace vcpkg::Build { System::print2("Using cached binary package: ", archive_path.u8string(), "\n"); - decompress_archive(paths, spec, archive_path); + auto archive_result = decompress_archive(paths, spec, archive_path); + + if (archive_result != 0) + { + System::print2("Failed to decompress archive package\n"); + return BuildResult::BUILD_FAILED; + } auto maybe_bcf = Paragraphs::try_load_cached_package(paths, spec); - std::unique_ptr bcf = - std::make_unique(std::move(maybe_bcf).value_or_exit(VCPKG_LINE_INFO)); + auto bcf = std::make_unique(std::move(maybe_bcf).value_or_exit(VCPKG_LINE_INFO)); return {BuildResult::SUCCEEDED, std::move(bcf)}; } -- cgit v1.2.3 From e2049cb9754006b6a2abed781d34030e16702fad Mon Sep 17 00:00:00 2001 From: Victor Romero Date: Sun, 30 Jun 2019 09:31:22 -0700 Subject: [vcpkg_configure_cmake] Add NO_CHARSET_FLAG option (#7074) * [vcpkg_configure_cmake] Add NO_CHARSET_FLAG option * [vcpkg_configure_cmake] Add documentation for new NO_CHARSET_FLAG option * [vcpkg_configure_cmake, windows toolchain] Handle NO_CHARSET_FLAG in toolchain * [build.cpp] Add Windows toolchain to package hash * [duilib,msix,thrift,tidy-html5] Use NO_CHARSET_FLAG to fix regressions --- toolsrc/src/vcpkg/build.cpp | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'toolsrc') diff --git a/toolsrc/src/vcpkg/build.cpp b/toolsrc/src/vcpkg/build.cpp index 9694bce4c..1975d3aaf 100644 --- a/toolsrc/src/vcpkg/build.cpp +++ b/toolsrc/src/vcpkg/build.cpp @@ -1007,6 +1007,12 @@ namespace vcpkg::Build hash += "-"; hash += Hash::get_file_hash(fs, *p, "SHA1"); } + else if (pre_build_info.cmake_system_name.empty() || + pre_build_info.cmake_system_name == "WindowsStore") + { + hash += "-"; + hash += Hash::get_file_hash(fs, paths.scripts / "toolchains" / "windows.cmake", "SHA1"); + } else if (pre_build_info.cmake_system_name == "Linux") { hash += "-"; -- cgit v1.2.3 From 96994f8edeace0064b8d80489643e3cb79514390 Mon Sep 17 00:00:00 2001 From: Victor Romero Date: Mon, 1 Jul 2019 22:49:05 -0700 Subject: Revert Visual Studio projects versions (#7117) --- toolsrc/vcpkg/vcpkg.vcxproj | 4 ++-- toolsrc/vcpkglib/vcpkglib.vcxproj | 4 ++-- toolsrc/vcpkgmetricsuploader/vcpkgmetricsuploader.vcxproj | 4 ++-- toolsrc/vcpkgtest/vcpkgtest.vcxproj | 6 +++--- 4 files changed, 9 insertions(+), 9 deletions(-) (limited to 'toolsrc') diff --git a/toolsrc/vcpkg/vcpkg.vcxproj b/toolsrc/vcpkg/vcpkg.vcxproj index 06d849a74..8edea2244 100644 --- a/toolsrc/vcpkg/vcpkg.vcxproj +++ b/toolsrc/vcpkg/vcpkg.vcxproj @@ -21,8 +21,8 @@ {34671B80-54F9-46F5-8310-AC429C11D4FB} vcpkg - 10.0 - v142 + 8.1 + v140 diff --git a/toolsrc/vcpkglib/vcpkglib.vcxproj b/toolsrc/vcpkglib/vcpkglib.vcxproj index 2a6853b8a..a64eb42fd 100644 --- a/toolsrc/vcpkglib/vcpkglib.vcxproj +++ b/toolsrc/vcpkglib/vcpkglib.vcxproj @@ -21,8 +21,8 @@ {B98C92B7-2874-4537-9D46-D14E5C237F04} vcpkglib - 10.0 - v142 + 8.1 + v140 diff --git a/toolsrc/vcpkgmetricsuploader/vcpkgmetricsuploader.vcxproj b/toolsrc/vcpkgmetricsuploader/vcpkgmetricsuploader.vcxproj index 2e689c7fc..e533d0e15 100644 --- a/toolsrc/vcpkgmetricsuploader/vcpkgmetricsuploader.vcxproj +++ b/toolsrc/vcpkgmetricsuploader/vcpkgmetricsuploader.vcxproj @@ -21,8 +21,8 @@ {7D6FDEEB-B299-4A23-85EE-F67C4DED47BE} vcpkgmetricsuploader - 10.0 - v142 + 8.1 + v140 diff --git a/toolsrc/vcpkgtest/vcpkgtest.vcxproj b/toolsrc/vcpkgtest/vcpkgtest.vcxproj index a7517ec54..4cda29461 100644 --- a/toolsrc/vcpkgtest/vcpkgtest.vcxproj +++ b/toolsrc/vcpkgtest/vcpkgtest.vcxproj @@ -48,8 +48,8 @@ {F27B8DB0-1279-4AF8-A2E3-1D49C4F0220D} Win32Proj vcpkgtest - 10.0 - v142 + 8.1 + v140 @@ -84,7 +84,7 @@ - + $(SolutionDir)msbuild.x86.debug\$(ProjectName)\ $(SolutionDir)msbuild.x86.debug\ -- cgit v1.2.3 From 91da4aab4c74af1d30c68896a058100257910e8d Mon Sep 17 00:00:00 2001 From: martin-s Date: Tue, 2 Jul 2019 05:51:07 +0000 Subject: Allow redirection of the scripts folder. (#6552) * Allow redirection of the scripts folder with an environment variable. * - Updated feature from environment variable to argument. * Fix crash when no scripts override is given and use --scripts-root= format * Update help messages to use --scripts-root= format --- toolsrc/include/vcpkg/vcpkgcmdarguments.h | 1 + toolsrc/include/vcpkg/vcpkgpaths.h | 3 +- toolsrc/src/tests.arguments.cpp | 6 ++-- toolsrc/src/vcpkg.cpp | 10 ++++++ toolsrc/src/vcpkg/help.cpp | 2 ++ toolsrc/src/vcpkg/vcpkgcmdarguments.cpp | 30 +++++++++++++++-- toolsrc/src/vcpkg/vcpkgpaths.cpp | 53 +++++++++++++++++++++---------- 7 files changed, 83 insertions(+), 22 deletions(-) (limited to 'toolsrc') diff --git a/toolsrc/include/vcpkg/vcpkgcmdarguments.h b/toolsrc/include/vcpkg/vcpkgcmdarguments.h index ff13ae6bf..fe7911ae3 100644 --- a/toolsrc/include/vcpkg/vcpkgcmdarguments.h +++ b/toolsrc/include/vcpkg/vcpkgcmdarguments.h @@ -86,6 +86,7 @@ namespace vcpkg static VcpkgCmdArguments create_from_arg_sequence(const std::string* arg_begin, const std::string* arg_end); std::unique_ptr vcpkg_root_dir; + std::unique_ptr scripts_root_dir; std::unique_ptr triplet; std::unique_ptr> overlay_ports; std::unique_ptr> overlay_triplets; diff --git a/toolsrc/include/vcpkg/vcpkgpaths.h b/toolsrc/include/vcpkg/vcpkgpaths.h index a30e0c653..ce442858b 100644 --- a/toolsrc/include/vcpkg/vcpkgpaths.h +++ b/toolsrc/include/vcpkg/vcpkgpaths.h @@ -47,7 +47,8 @@ namespace vcpkg struct VcpkgPaths { - static Expected create(const fs::path& vcpkg_root_dir, + static Expected create(const fs::path& vcpkg_root_dir, + const Optional& vcpkg_scripts_root_dir, const std::string& default_vs_path, const std::vector* triplets_dirs); diff --git a/toolsrc/src/tests.arguments.cpp b/toolsrc/src/tests.arguments.cpp index 51ababd3d..533b3a0d0 100644 --- a/toolsrc/src/tests.arguments.cpp +++ b/toolsrc/src/tests.arguments.cpp @@ -15,9 +15,10 @@ namespace UnitTest1 { TEST_METHOD(create_from_arg_sequence_options_lower) { - std::vector t = {"--vcpkg-root", "C:\\vcpkg", "--debug", "--sendmetrics", "--printmetrics"}; + std::vector t = {"--vcpkg-root", "C:\\vcpkg", "--scripts-root", "C:\\scripts", "--debug", "--sendmetrics", "--printmetrics"}; auto v = VcpkgCmdArguments::create_from_arg_sequence(t.data(), t.data() + t.size()); Assert::AreEqual("C:\\vcpkg", v.vcpkg_root_dir.get()->c_str()); + Assert::AreEqual("C:\\scripts", v.scripts_root_dir.get()->c_str()); Assert::IsTrue(v.debug && *v.debug.get()); Assert::IsTrue(v.sendmetrics && v.sendmetrics.get()); Assert::IsTrue(v.printmetrics && *v.printmetrics.get()); @@ -25,9 +26,10 @@ namespace UnitTest1 TEST_METHOD(create_from_arg_sequence_options_upper) { - std::vector t = {"--VCPKG-ROOT", "C:\\vcpkg", "--DEBUG", "--SENDMETRICS", "--PRINTMETRICS"}; + std::vector t = {"--VCPKG-ROOT", "C:\\vcpkg", "--SCRIPTS-ROOT", "C:\\scripts", "--DEBUG", "--SENDMETRICS", "--PRINTMETRICS"}; auto v = VcpkgCmdArguments::create_from_arg_sequence(t.data(), t.data() + t.size()); Assert::AreEqual("C:\\vcpkg", v.vcpkg_root_dir.get()->c_str()); + Assert::AreEqual("C:\\scripts", v.scripts_root_dir.get()->c_str()); Assert::IsTrue(v.debug && *v.debug.get()); Assert::IsTrue(v.sendmetrics && v.sendmetrics.get()); Assert::IsTrue(v.printmetrics && *v.printmetrics.get()); diff --git a/toolsrc/src/vcpkg.cpp b/toolsrc/src/vcpkg.cpp index 5da97b136..363b39814 100644 --- a/toolsrc/src/vcpkg.cpp +++ b/toolsrc/src/vcpkg.cpp @@ -116,9 +116,19 @@ static void inner(const VcpkgCmdArguments& args) Debug::print("Using vcpkg-root: ", vcpkg_root_dir.u8string(), '\n'); + Optional vcpkg_scripts_root_dir = nullopt; + if (nullptr != args.scripts_root_dir) + { + vcpkg_scripts_root_dir = fs::stdfs::canonical(fs::u8path(*args.scripts_root_dir)); + Debug::print("Using scripts-root: ", vcpkg_scripts_root_dir.value_or_exit(VCPKG_LINE_INFO).u8string(), '\n'); + } + auto default_vs_path = System::get_environment_variable("VCPKG_VISUAL_STUDIO_PATH").value_or(""); + + const Expected expected_paths = VcpkgPaths::create(vcpkg_root_dir, + vcpkg_scripts_root_dir, default_vs_path, args.overlay_triplets.get()); Checks::check_exit(VCPKG_LINE_INFO, diff --git a/toolsrc/src/vcpkg/help.cpp b/toolsrc/src/vcpkg/help.cpp index 896f62661..9b2751739 100644 --- a/toolsrc/src/vcpkg/help.cpp +++ b/toolsrc/src/vcpkg/help.cpp @@ -124,6 +124,8 @@ namespace vcpkg::Help " (default: " ENVVAR(VCPKG_ROOT) // ")\n" "\n" + " --scripts-root= Specify the scripts root directory\n" + "\n" " @response_file Specify a " "response file to provide additional parameters\n" "\n" diff --git a/toolsrc/src/vcpkg/vcpkgcmdarguments.cpp b/toolsrc/src/vcpkg/vcpkgcmdarguments.cpp index 3c1452d47..7a28fb571 100644 --- a/toolsrc/src/vcpkg/vcpkgcmdarguments.cpp +++ b/toolsrc/src/vcpkg/vcpkgcmdarguments.cpp @@ -33,6 +33,21 @@ namespace vcpkg option_field = std::make_unique(*arg_begin); } + static void parse_cojoined_value(std::string new_value, + const std::string& option_name, + std::unique_ptr& option_field) + { + if (nullptr != option_field) + { + System::printf(System::Color::error, "Error: %s specified multiple times\n", option_name); + Metrics::g_metrics.lock()->track_property("error", "error option specified multiple times"); + Help::print_usage(); + Checks::exit_fail(VCPKG_LINE_INFO); + } + + option_field = std::make_unique(std::move(new_value)); + } + static void parse_switch(bool new_setting, const std::string& option_name, Optional& option_field) { if (option_field && option_field != new_setting) @@ -120,9 +135,10 @@ namespace vcpkg if (arg[0] == '-' && arg[1] == '-') { - // make argument case insensitive + // make argument case insensitive before the first = auto& f = std::use_facet>(std::locale()); - f.tolower(&arg[0], &arg[0] + arg.size()); + auto first_eq = std::find(std::begin(arg), std::end(arg), '='); + f.tolower(&arg[0], &arg[0] + (first_eq - std::begin(arg))); // command switch if (arg == "--vcpkg-root") { @@ -130,6 +146,13 @@ namespace vcpkg parse_value(arg_begin, arg_end, "--vcpkg-root", args.vcpkg_root_dir); continue; } + if (Strings::starts_with(arg, "--scripts-root=")) + { + parse_cojoined_value(arg.substr(sizeof("--scripts-root=") - 1), + "--scripts-root", + args.scripts_root_dir); + continue; + } if (arg == "--triplet") { ++arg_begin; @@ -411,5 +434,8 @@ namespace vcpkg System::printf(" %-40s %s\n", "--vcpkg-root ", "Specify the vcpkg directory to use instead of current directory or tool directory"); + System::printf(" %-40s %s\n", + "--scripts-root=", + "Specify the scripts directory to use instead of default vcpkg scripts directory"); } } diff --git a/toolsrc/src/vcpkg/vcpkgpaths.cpp b/toolsrc/src/vcpkg/vcpkgpaths.cpp index 909fbeb44..d16acf2e8 100644 --- a/toolsrc/src/vcpkg/vcpkgpaths.cpp +++ b/toolsrc/src/vcpkg/vcpkgpaths.cpp @@ -13,7 +13,8 @@ namespace vcpkg { - Expected VcpkgPaths::create(const fs::path& vcpkg_root_dir, + Expected VcpkgPaths::create(const fs::path& vcpkg_root_dir, + const Optional& vcpkg_scripts_root_dir, const std::string& default_vs_path, const std::vector* triplets_dirs) { @@ -65,7 +66,25 @@ namespace vcpkg paths.ports = paths.root / "ports"; paths.installed = paths.root / "installed"; paths.triplets = paths.root / "triplets"; - paths.scripts = paths.root / "scripts"; + + if (auto scripts_dir = vcpkg_scripts_root_dir.get()) + { + if (scripts_dir->empty() || !fs::stdfs::is_directory(*scripts_dir)) + { + Metrics::g_metrics.lock()->track_property("error", "Invalid scripts override directory."); + Checks::exit_with_message( + VCPKG_LINE_INFO, + "Invalid scripts override directory: %s; " + "create that directory or unset --scripts-root to use the default scripts location.", + scripts_dir->u8string()); + } + + paths.scripts = *scripts_dir; + } + else + { + paths.scripts = paths.root / "scripts"; + } paths.tools = paths.downloads / "tools"; paths.buildsystems = paths.scripts / "buildsystems"; @@ -132,21 +151,21 @@ namespace vcpkg } const fs::path VcpkgPaths::get_triplet_file_path(const Triplet& triplet) const - { - return m_triplets_cache.get_lazy(triplet, [&]()-> auto { - for (auto&& triplet_dir : triplets_dirs) - { - auto&& path = triplet_dir / (triplet.canonical_name() + ".cmake"); - if (this->get_filesystem().exists(path)) - { - return path; - } - } - - Checks::exit_with_message(VCPKG_LINE_INFO, - "Error: Triplet file %s.cmake not found", - triplet.canonical_name()); - }); + { + return m_triplets_cache.get_lazy(triplet, [&]()-> auto { + for (auto&& triplet_dir : triplets_dirs) + { + auto&& path = triplet_dir / (triplet.canonical_name() + ".cmake"); + if (this->get_filesystem().exists(path)) + { + return path; + } + } + + Checks::exit_with_message(VCPKG_LINE_INFO, + "Error: Triplet file %s.cmake not found", + triplet.canonical_name()); + }); } -- cgit v1.2.3 From 269fa0e6be13ebe64060323579a2af68c0a21e4c Mon Sep 17 00:00:00 2001 From: Victor Romero Date: Tue, 2 Jul 2019 17:19:46 -0700 Subject: Bump version to 2019.06.26 (#7136) --- toolsrc/VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'toolsrc') diff --git a/toolsrc/VERSION.txt b/toolsrc/VERSION.txt index c2116e0e8..6b2a31295 100644 --- a/toolsrc/VERSION.txt +++ b/toolsrc/VERSION.txt @@ -1 +1 @@ -"2019.06.21" \ No newline at end of file +"2019.06.26" \ No newline at end of file -- cgit v1.2.3 From 2b8e225b2ea83a3138d4b7345f104c1bd9a129dd Mon Sep 17 00:00:00 2001 From: Robert Schumacher Date: Sat, 6 Jul 2019 13:29:46 -0700 Subject: [vcpkg] Fix powershell font corruption bug (#7094) * [vcpkg] Fix font corruption bug on Windows by downloading Powershell Core * [vcpkg] Rename subtool to powershell-core * [vcpkg] Add missing includes to project files --- toolsrc/include/vcpkg/base/system.process.h | 3 ++- toolsrc/src/vcpkg/base/system.cpp | 21 ++++++++++++--------- toolsrc/src/vcpkg/build.cpp | 23 +++++++++++++++++------ toolsrc/src/vcpkg/commands.integrate.cpp | 11 ++--------- toolsrc/vcpkglib/vcpkglib.vcxproj | 6 ++++++ toolsrc/vcpkglib/vcpkglib.vcxproj.filters | 18 ++++++++++++++++++ 6 files changed, 57 insertions(+), 25 deletions(-) (limited to 'toolsrc') diff --git a/toolsrc/include/vcpkg/base/system.process.h b/toolsrc/include/vcpkg/base/system.process.h index e409ff950..14e2eb109 100644 --- a/toolsrc/include/vcpkg/base/system.process.h +++ b/toolsrc/include/vcpkg/base/system.process.h @@ -31,7 +31,8 @@ namespace vcpkg::System }; int cmd_execute_clean(const ZStringView cmd_line, - const std::unordered_map& extra_env = {}); + const std::unordered_map& extra_env = {}, + const std::string& prepend_to_path = {}); int cmd_execute(const ZStringView cmd_line); diff --git a/toolsrc/src/vcpkg/base/system.cpp b/toolsrc/src/vcpkg/base/system.cpp index c5ff050f0..3d5c60088 100644 --- a/toolsrc/src/vcpkg/base/system.cpp +++ b/toolsrc/src/vcpkg/base/system.cpp @@ -188,12 +188,17 @@ namespace vcpkg } #if defined(_WIN32) - static std::wstring compute_clean_environment(const std::unordered_map& extra_env) + static std::wstring compute_clean_environment(const std::unordered_map& extra_env, + const std::string& prepend_to_path) { static const std::string SYSTEM_ROOT = get_environment_variable("SystemRoot").value_or_exit(VCPKG_LINE_INFO); static const std::string SYSTEM_32 = SYSTEM_ROOT + R"(\system32)"; - std::string new_path = Strings::format( - R"(Path=%s;%s;%s\Wbem;%s\WindowsPowerShell\v1.0\)", SYSTEM_32, SYSTEM_ROOT, SYSTEM_32, SYSTEM_32); + std::string new_path = Strings::format(R"(Path=%s%s;%s;%s\Wbem;%s\WindowsPowerShell\v1.0\)", + prepend_to_path, + SYSTEM_32, + SYSTEM_ROOT, + SYSTEM_32, + SYSTEM_32); std::vector env_wstrings = { L"ALLUSERSPROFILE", @@ -348,7 +353,8 @@ namespace vcpkg #endif int System::cmd_execute_clean(const ZStringView cmd_line, - const std::unordered_map& extra_env) + const std::unordered_map& extra_env, + const std::string& prepend_to_path) { auto timer = Chrono::ElapsedTimer::create_started(); #if defined(_WIN32) @@ -357,7 +363,7 @@ namespace vcpkg memset(&process_info, 0, sizeof(PROCESS_INFORMATION)); g_ctrl_c_state.transition_to_spawn_process(); - auto clean_env = compute_clean_environment(extra_env); + auto clean_env = compute_clean_environment(extra_env, prepend_to_path); windows_create_process(cmd_line, clean_env.data(), process_info, NULL); CloseHandle(process_info.hThread); @@ -608,10 +614,7 @@ namespace vcpkg void System::register_console_ctrl_handler() {} #endif - int System::get_num_logical_cores() - { - return std::thread::hardware_concurrency(); - } + int System::get_num_logical_cores() { return std::thread::hardware_concurrency(); } } namespace vcpkg::Debug diff --git a/toolsrc/src/vcpkg/build.cpp b/toolsrc/src/vcpkg/build.cpp index 1975d3aaf..68df1f965 100644 --- a/toolsrc/src/vcpkg/build.cpp +++ b/toolsrc/src/vcpkg/build.cpp @@ -363,13 +363,11 @@ namespace vcpkg::Build const Triplet& triplet = spec.triplet(); const auto& triplet_file_path = paths.get_triplet_file_path(spec.triplet()).u8string(); - if (!Strings::case_insensitive_ascii_starts_with(triplet_file_path, - paths.triplets.u8string())) + if (!Strings::case_insensitive_ascii_starts_with(triplet_file_path, paths.triplets.u8string())) { System::printf("-- Loading triplet configuration from: %s\n", triplet_file_path); } - if (!Strings::case_insensitive_ascii_starts_with(config.port_dir.u8string(), - paths.ports.u8string())) + if (!Strings::case_insensitive_ascii_starts_with(config.port_dir.u8string(), paths.ports.u8string())) { System::printf("-- Installing port from location: %s\n", config.port_dir.u8string()); } @@ -382,6 +380,13 @@ namespace vcpkg::Build const fs::path& cmake_exe_path = paths.get_tool_exe(Tools::CMAKE); const fs::path& git_exe_path = paths.get_tool_exe(Tools::GIT); +#if defined(_WIN32) + const fs::path& powershell_exe_path = paths.get_tool_exe("powershell-core"); + if (!fs.exists(powershell_exe_path.parent_path() / "powershell.exe")) + { + fs.copy(powershell_exe_path, powershell_exe_path.parent_path() / "powershell.exe", fs::copy_options::none); + } +#endif std::string all_features; for (auto& feature : config.scf.feature_paragraphs) @@ -425,8 +430,14 @@ namespace vcpkg::Build } command.append(cmd_launch_cmake); const auto timer = Chrono::ElapsedTimer::create_started(); - - const int return_code = System::cmd_execute_clean(command); + const int return_code = System::cmd_execute_clean( + command, + {} +#ifdef _WIN32 + , + powershell_exe_path.parent_path().u8string() + ";" +#endif + ); const auto buildtimeus = timer.microseconds(); const auto spec_string = spec.to_string(); diff --git a/toolsrc/src/vcpkg/commands.integrate.cpp b/toolsrc/src/vcpkg/commands.integrate.cpp index fda9e2b11..8b2b377bf 100644 --- a/toolsrc/src/vcpkg/commands.integrate.cpp +++ b/toolsrc/src/vcpkg/commands.integrate.cpp @@ -380,17 +380,10 @@ With a project open, go to Tools->NuGet Package Manager->Package Manager Console static constexpr StringLiteral TITLE = "PowerShell Tab-Completion"; const fs::path script_path = paths.scripts / "addPoshVcpkgToPowershellProfile.ps1"; - // Console font corruption workaround - SetConsoleCP(437); - SetConsoleOutputCP(437); - + const auto& ps = paths.get_tool_exe("powershell-core"); const std::string cmd = Strings::format( - R"(powershell -NoProfile -ExecutionPolicy Bypass -Command "& {& '%s' %s}")", script_path.u8string(), ""); + R"("%s" -NoProfile -ExecutionPolicy Bypass -Command "& {& '%s' }")", ps.u8string(), script_path.u8string()); const int rc = System::cmd_execute(cmd); - - SetConsoleCP(CP_UTF8); - SetConsoleOutputCP(CP_UTF8); - if (rc) { System::printf(System::Color::error, diff --git a/toolsrc/vcpkglib/vcpkglib.vcxproj b/toolsrc/vcpkglib/vcpkglib.vcxproj index a64eb42fd..2eff1abee 100644 --- a/toolsrc/vcpkglib/vcpkglib.vcxproj +++ b/toolsrc/vcpkglib/vcpkglib.vcxproj @@ -161,8 +161,14 @@ + + + + + + diff --git a/toolsrc/vcpkglib/vcpkglib.vcxproj.filters b/toolsrc/vcpkglib/vcpkglib.vcxproj.filters index 36840c509..aec56b039 100644 --- a/toolsrc/vcpkglib/vcpkglib.vcxproj.filters +++ b/toolsrc/vcpkglib/vcpkglib.vcxproj.filters @@ -383,5 +383,23 @@ Header Files\vcpkg\base + + Header Files\vcpkg\base + + + Header Files\vcpkg\base + + + Header Files\vcpkg\base + + + Header Files\vcpkg\base + + + Header Files\vcpkg\base + + + Header Files\vcpkg\base + \ No newline at end of file -- cgit v1.2.3 From 7f80c0e2d311c7ff2453bdd558e4e7fd91cea872 Mon Sep 17 00:00:00 2001 From: gnaggnoyil Date: Wed, 10 Jul 2019 04:02:48 +0800 Subject: Make handle features (#6797) --- toolsrc/include/vcpkg/base/util.h | 9 +++++ toolsrc/src/vcpkg/commands.dependinfo.cpp | 61 +++++++++++++++++++++++++++++-- 2 files changed, 67 insertions(+), 3 deletions(-) (limited to 'toolsrc') diff --git a/toolsrc/include/vcpkg/base/util.h b/toolsrc/include/vcpkg/base/util.h index 213adb67c..e629ef0b2 100644 --- a/toolsrc/include/vcpkg/base/util.h +++ b/toolsrc/include/vcpkg/base/util.h @@ -6,9 +6,18 @@ #include #include #include +#include namespace vcpkg::Util { + template + constexpr std::add_const_t& as_const(T& t) noexcept + { + return t; + } + template + void as_const(const T&&) = delete; + template using ElementT = std::remove_reference_t::iterator>())>; diff --git a/toolsrc/src/vcpkg/commands.dependinfo.cpp b/toolsrc/src/vcpkg/commands.dependinfo.cpp index c0800a4b5..8394e0166 100644 --- a/toolsrc/src/vcpkg/commands.dependinfo.cpp +++ b/toolsrc/src/vcpkg/commands.dependinfo.cpp @@ -6,6 +6,10 @@ #include #include #include +#include + +#include +#include #include using vcpkg::Dependencies::PathsPortFileProvider; @@ -130,14 +134,29 @@ namespace vcpkg::Commands::DependInfo const std::vector& source_control_files, const std::unordered_set& switches) { + auto maybe_requested_spec = ParsedSpecifier::from_string(requested_package); + // TODO: move this check to the top-level invocation of this function since + // argument `requested_package` shall always be valid in inner-level invocation. + if (!maybe_requested_spec.has_value()) + { + System::print2(System::Color::warning, + "'", + requested_package, + "' is not a valid package specifier: ", + vcpkg::to_string(maybe_requested_spec.error()), + "\n"); + return; + } + auto requested_spec = maybe_requested_spec.get(); + const auto source_control_file = - Util::find_if(source_control_files, [&requested_package](const auto& source_control_file) { - return source_control_file->core_paragraph->name == requested_package; + Util::find_if(source_control_files, [&requested_spec](const auto& source_control_file) { + return source_control_file->core_paragraph->name == requested_spec->name; }); if (source_control_file != source_control_files.end()) { - const auto new_package = packages_to_keep.insert(requested_package).second; + const auto new_package = packages_to_keep.insert(requested_spec->name).second; if (new_package && !Util::Sets::contains(switches, OPTION_NO_RECURSE)) { @@ -145,6 +164,42 @@ namespace vcpkg::Commands::DependInfo { build_dependencies_list(packages_to_keep, dependency.depend.name, source_control_files, switches); } + + // Collect features with `*` considered + std::set collected_features; + for (const auto& requested_feature_name : requested_spec->features) + { + if (requested_feature_name == "*") + { + for (auto &&feature_paragraph : (*source_control_file)->feature_paragraphs) + { + collected_features.insert(std::addressof(Util::as_const(*feature_paragraph))); + } + continue; + } + auto maybe_feature = (*source_control_file)->find_feature(requested_feature_name); + if (auto &&feature_paragraph = maybe_feature.get()) + { + collected_features.insert(std::addressof(Util::as_const(*feature_paragraph))); + } + else + { + System::print2(System::Color::warning, + "dependency '", + requested_feature_name, + "' of package '", + requested_spec->name, + "' does not exist\n"); + continue; + } + } + for (auto feature_paragraph : collected_features) + { + for (const auto& dependency : feature_paragraph->depends) + { + build_dependencies_list(packages_to_keep, dependency.depend.name, source_control_files, switches); + } + } } } else -- cgit v1.2.3 From 60bff8d54996d55ffc81995bcb63510686863c84 Mon Sep 17 00:00:00 2001 From: Phil Christensen Date: Wed, 10 Jul 2019 11:36:37 -0700 Subject: allow spaces in pathname on linux (#7216) --- toolsrc/src/vcpkg/base/hash.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'toolsrc') diff --git a/toolsrc/src/vcpkg/base/hash.cpp b/toolsrc/src/vcpkg/base/hash.cpp index 310b8c35e..e9a7fa2ef 100644 --- a/toolsrc/src/vcpkg/base/hash.cpp +++ b/toolsrc/src/vcpkg/base/hash.cpp @@ -178,9 +178,11 @@ namespace vcpkg::Hash static std::string parse_shasum_output(const std::string& shasum_output) { std::vector split = Strings::split(shasum_output, " "); + // Checking if >= 3 because filenames with spaces will show up as multiple tokens. + // The hash is the first token so we don't need to parse the filename anyway. Checks::check_exit(VCPKG_LINE_INFO, - split.size() == 3, - "Expected output of the form [hash filename\n] (3 tokens), but got\n" + split.size() >= 3, + "Expected output of the form [hash filename\n] (3+ tokens), but got\n" "[%s] (%s tokens)", shasum_output, std::to_string(split.size())); -- cgit v1.2.3 From 7dbe375a2c270e91f9742dd78cf2b579d1f5f2fa Mon Sep 17 00:00:00 2001 From: Victor Romero Date: Thu, 11 Jul 2019 17:00:55 -0700 Subject: Testing for --overlay-ports and --overlay-triplets args (#7243) --- toolsrc/src/tests.arguments.cpp | 40 ++++++++++++++++++++++++++++++++++++++-- toolsrc/src/tests.plan.cpp | 11 ++++++----- 2 files changed, 44 insertions(+), 7 deletions(-) (limited to 'toolsrc') diff --git a/toolsrc/src/tests.arguments.cpp b/toolsrc/src/tests.arguments.cpp index 533b3a0d0..e108b983a 100644 --- a/toolsrc/src/tests.arguments.cpp +++ b/toolsrc/src/tests.arguments.cpp @@ -15,24 +15,60 @@ namespace UnitTest1 { TEST_METHOD(create_from_arg_sequence_options_lower) { - std::vector t = {"--vcpkg-root", "C:\\vcpkg", "--scripts-root", "C:\\scripts", "--debug", "--sendmetrics", "--printmetrics"}; + std::vector t = { + "--vcpkg-root", "C:\\vcpkg", + "--scripts-root=C:\\scripts", + "--debug", + "--sendmetrics", + "--printmetrics", + "--overlay-ports=C:\\ports1", + "--overlay-ports=C:\\ports2", + "--overlay-triplets=C:\\tripletsA", + "--overlay-triplets=C:\\tripletsB" + }; auto v = VcpkgCmdArguments::create_from_arg_sequence(t.data(), t.data() + t.size()); Assert::AreEqual("C:\\vcpkg", v.vcpkg_root_dir.get()->c_str()); Assert::AreEqual("C:\\scripts", v.scripts_root_dir.get()->c_str()); Assert::IsTrue(v.debug && *v.debug.get()); Assert::IsTrue(v.sendmetrics && v.sendmetrics.get()); Assert::IsTrue(v.printmetrics && *v.printmetrics.get()); + + Assert::IsTrue(v.overlay_ports.get()->size() == 2); + Assert::AreEqual("C:\\ports1", v.overlay_ports.get()->at(0).c_str()); + Assert::AreEqual("C:\\ports2", v.overlay_ports.get()->at(1).c_str()); + + Assert::IsTrue(v.overlay_triplets.get()->size() == 2); + Assert::AreEqual("C:\\tripletsA", v.overlay_triplets.get()->at(0).c_str()); + Assert::AreEqual("C:\\tripletsB", v.overlay_triplets.get()->at(1).c_str()); } TEST_METHOD(create_from_arg_sequence_options_upper) { - std::vector t = {"--VCPKG-ROOT", "C:\\vcpkg", "--SCRIPTS-ROOT", "C:\\scripts", "--DEBUG", "--SENDMETRICS", "--PRINTMETRICS"}; + std::vector t = { + "--VCPKG-ROOT", "C:\\vcpkg", + "--SCRIPTS-ROOT=C:\\scripts", + "--DEBUG", + "--SENDMETRICS", + "--PRINTMETRICS", + "--OVERLAY-PORTS=C:\\ports1", + "--OVERLAY-PORTS=C:\\ports2", + "--OVERLAY-TRIPLETS=C:\\tripletsA", + "--OVERLAY-TRIPLETS=C:\\tripletsB" + }; auto v = VcpkgCmdArguments::create_from_arg_sequence(t.data(), t.data() + t.size()); Assert::AreEqual("C:\\vcpkg", v.vcpkg_root_dir.get()->c_str()); Assert::AreEqual("C:\\scripts", v.scripts_root_dir.get()->c_str()); Assert::IsTrue(v.debug && *v.debug.get()); Assert::IsTrue(v.sendmetrics && v.sendmetrics.get()); Assert::IsTrue(v.printmetrics && *v.printmetrics.get()); + + Assert::IsTrue(v.overlay_ports.get()->size() == 2); + Assert::AreEqual("C:\\ports1", v.overlay_ports.get()->at(0).c_str()); + Assert::AreEqual("C:\\ports2", v.overlay_ports.get()->at(1).c_str()); + + Assert::IsTrue(v.overlay_triplets.get()->size() == 2); + Assert::AreEqual("C:\\tripletsA", v.overlay_triplets.get()->at(0).c_str()); + Assert::AreEqual("C:\\tripletsB", v.overlay_triplets.get()->at(1).c_str()); } TEST_METHOD(create_from_arg_sequence_valued_options) diff --git a/toolsrc/src/tests.plan.cpp b/toolsrc/src/tests.plan.cpp index ab24266b4..9d8717878 100644 --- a/toolsrc/src/tests.plan.cpp +++ b/toolsrc/src/tests.plan.cpp @@ -89,14 +89,15 @@ namespace UnitTest1 const std::vector>& features = {}, const std::vector& default_features = {}) { - return emplace(std::move(*make_control_file(name, depends, features, default_features))); + auto scfl = SourceControlFileLocation { make_control_file(name, depends, features, default_features), "" }; + return emplace(std::move(scfl)); } - PackageSpec emplace(vcpkg::SourceControlFile&& scf) + + PackageSpec emplace(vcpkg::SourceControlFileLocation&& scfl) { - auto spec = PackageSpec::from_name_and_triplet(scf.core_paragraph->name, triplet); + auto spec = PackageSpec::from_name_and_triplet(scfl.source_control_file->core_paragraph->name, triplet); Assert::IsTrue(spec.has_value()); - map.emplace(scf.core_paragraph->name, - SourceControlFileLocation{std::unique_ptr(std::move(&scf)), ""}); + map.emplace(scfl.source_control_file->core_paragraph->name, std::move(scfl)); return PackageSpec{*spec.get()}; } }; -- cgit v1.2.3 From 5857e2c680fde9e37abc8f799f8d5509dd47ed62 Mon Sep 17 00:00:00 2001 From: Nicole Mazzuca Date: Mon, 8 Jul 2019 16:45:27 -0700 Subject: initial remove-in-parallel doesn't actually do parallel remove yet --- toolsrc/include/vcpkg/base/rng.h | 96 ++++++++++++++++++++++++++++++++++++ toolsrc/include/vcpkg/base/strings.h | 32 ++++++++++++ toolsrc/src/vcpkg/base/files.cpp | 68 +++++++++++++++++++------ toolsrc/src/vcpkg/base/rng.cpp | 14 ++++++ 4 files changed, 194 insertions(+), 16 deletions(-) create mode 100644 toolsrc/include/vcpkg/base/rng.h create mode 100644 toolsrc/src/vcpkg/base/rng.cpp (limited to 'toolsrc') diff --git a/toolsrc/include/vcpkg/base/rng.h b/toolsrc/include/vcpkg/base/rng.h new file mode 100644 index 000000000..1bcab05b3 --- /dev/null +++ b/toolsrc/include/vcpkg/base/rng.h @@ -0,0 +1,96 @@ +#pragma once + +#include +#include +#include + +namespace vcpkg { + + /* + NOTE(ubsan): taken from the xoshiro paper + initialized from random_device by default + actual code is copied from wikipedia, since I wrote that code + */ + struct splitmix64_engine { + splitmix64_engine() noexcept; + + constexpr splitmix64_engine(std::uint64_t seed) noexcept + : state(seed) {} + + constexpr std::uint64_t operator()() noexcept { + state += 0x9E3779B97F4A7C15; + + std::uint64_t result = state; + result = (result ^ (result >> 30)) * 0xBF58476D1CE4E5B9; + result = (result ^ (result >> 27)) * 0x94D049BB133111EB; + + return result ^ (result >> 31); + } + + constexpr std::uint64_t max() const noexcept { + return std::numeric_limits::max(); + } + + constexpr std::uint64_t min() const noexcept { + return std::numeric_limits::min(); + } + + private: + std::uint64_t state; + }; + + // Sebastian Vigna's xorshift-based xoshiro xoshiro256** engine + // fast and really good + // uses the splitmix64_engine to initialize state + struct xoshiro256ss_engine { + // splitmix64_engine will be initialized with random_device + xoshiro256ss_engine() noexcept { + splitmix64_engine sm64{}; + + for (std::uint64_t& s : this->state) { + s = sm64(); + } + } + + constexpr xoshiro256ss_engine(std::uint64_t seed) noexcept : state() { + splitmix64_engine sm64{seed}; + + for (std::uint64_t& s : this->state) { + s = sm64(); + } + } + + constexpr std::uint64_t operator()() noexcept { + std::uint64_t const result = rol(state[1] * 5, 7) * 9; + + std::uint64_t const t = state[1] << 17; + + // state[i] = state[i] ^ state[i + 4 mod 4] + state[2] ^= state[0]; + state[3] ^= state[1]; + state[1] ^= state[2]; + state[0] ^= state[3]; + + state[2] ^= t; + state[3] ^= rol(state[3], 45); + + return result; + } + + constexpr std::uint64_t max() const noexcept { + return std::numeric_limits::max(); + } + + constexpr std::uint64_t min() const noexcept { + return std::numeric_limits::min(); + } + private: + // rotate left + constexpr std::uint64_t rol(std::uint64_t x, int k) { + return (x << k) | (x >> (64 - k)); + } + + std::uint64_t state[4]; + }; + +} diff --git a/toolsrc/include/vcpkg/base/strings.h b/toolsrc/include/vcpkg/base/strings.h index d553d1d41..423ea2096 100644 --- a/toolsrc/include/vcpkg/base/strings.h +++ b/toolsrc/include/vcpkg/base/strings.h @@ -184,4 +184,36 @@ namespace vcpkg::Strings const char* search(StringView haystack, StringView needle); bool contains(StringView haystack, StringView needle); + + // base 64 encoding with URL and filesafe alphabet (base64url) + // based on IETF RFC 4648 + // ignores padding, since one implicitly knows the length from the size of x + template + std::string b64url_encode(Integral x) { + static_assert(std::is_integral_v); + auto value = static_cast>(x); + + // 64 values, plus the implicit \0 + constexpr static char map[0x41] = + /* 0123456789ABCDEF */ + /*0*/ "ABCDEFGHIJKLMNOP" + /*1*/ "QRSTUVWXYZabcdef" + /*2*/ "ghijklmnopqrstuv" + /*3*/ "wxyz0123456789-_" + ; + + constexpr static std::make_unsigned_t mask = 0x3F; + constexpr static int shift = 5; + + std::string result; + // reserve ceiling(number of bits / 3) + result.reserve((sizeof(value) * 8 + 2) / 3); + + while (value != 0) { + char mapped_value = map[value & mask]; + result.push_back(mapped_value); + } + + return result; + } } diff --git a/toolsrc/src/vcpkg/base/files.cpp b/toolsrc/src/vcpkg/base/files.cpp index 5099795e9..d0926bb4c 100644 --- a/toolsrc/src/vcpkg/base/files.cpp +++ b/toolsrc/src/vcpkg/base/files.cpp @@ -1,6 +1,7 @@ #include "pch.h" #include +#include #include #include #include @@ -256,26 +257,61 @@ namespace vcpkg::Files virtual bool remove(const fs::path& path, std::error_code& ec) override { return fs::stdfs::remove(path, ec); } virtual std::uintmax_t remove_all(const fs::path& path, std::error_code& ec) override { - // Working around the currently buggy remove_all() - std::uintmax_t out = fs::stdfs::remove_all(path, ec); + /* + does not use the std::filesystem call since it is buggy, and can + have spurious errors before VS 2017 update 6, and on later versions + (as well as on macOS and Linux), this is just as fast and will have + fewer spurious errors due to locks. + */ + struct recursive { + const fs::path& tmp_directory; + std::error_code& ec; + xoshiro256ss_engine& rng; + + void operator()(const fs::path& current) const { + const auto type = fs::stdfs::symlink_status(current, ec).type(); + if (ec) return; + + const auto tmp_name = Strings::b64url_encode(rng()); + const auto tmp_path = tmp_directory / tmp_name; + + switch (type) { + case fs::file_type::directory: { + fs::stdfs::rename(current, tmp_path, ec); + if (ec) return; + for (const auto& entry : fs::stdfs::directory_iterator(tmp_path)) { + (*this)(entry); + } + fs::stdfs::remove(tmp_path, ec); + } break; + case fs::file_type::symlink: + case fs::file_type::regular: { + fs::stdfs::rename(current, tmp_path, ec); + fs::stdfs::remove(current, ec); + } break; + case fs::file_type::not_found: return; + case fs::file_type::none: { + Checks::exit_with_message(VCPKG_LINE_INFO, "Error occurred when evaluating file type of file: %s", current); + } + default: { + Checks::exit_with_message(VCPKG_LINE_INFO, "Attempted to delete special file: %s", current); + } + } + } + }; - for (int i = 0; i < 5 && this->exists(path); i++) - { - using namespace std::chrono_literals; - std::this_thread::sleep_for(i * 100ms); - out += fs::stdfs::remove_all(path, ec); - } + auto const real_path = fs::stdfs::absolute(path); - if (this->exists(path)) - { - System::print2( - System::Color::warning, - "Some files in ", - path.u8string(), - " were unable to be removed. Close any editors operating in this directory and retry.\n"); + if (! real_path.has_parent_path()) { + Checks::exit_with_message(VCPKG_LINE_INFO, "Attempted to remove_all the base directory"); } - return out; + // thoughts: is this fine? or should we do something different? + // maybe a temporary directory? + auto const base_path = real_path.parent_path(); + + xoshiro256ss_engine rng{}; + recursive{base_path, ec, rng}(real_path); } virtual bool exists(const fs::path& path) const override { return fs::stdfs::exists(path); } virtual bool is_directory(const fs::path& path) const override { return fs::stdfs::is_directory(path); } diff --git a/toolsrc/src/vcpkg/base/rng.cpp b/toolsrc/src/vcpkg/base/rng.cpp new file mode 100644 index 000000000..9fe2ea3b4 --- /dev/null +++ b/toolsrc/src/vcpkg/base/rng.cpp @@ -0,0 +1,14 @@ +#include + +namespace vcpkg { + namespace { + std::random_device system_entropy{}; + } + + splitmix64_engine::splitmix64_engine() { + std::uint64_t top_half = system_entropy(); + std::uint64_t bottom_half = system_entropy(); + + state = (top_half << 32) | bottom_half; + } +} -- cgit v1.2.3 From 2d6df16849ebcf237d17c919727756d90974daba Mon Sep 17 00:00:00 2001 From: Nicole Mazzuca Date: Wed, 10 Jul 2019 14:35:10 -0700 Subject: remove_all parallelized, and fix the issues with symlink --- toolsrc/include/vcpkg/base/files.h | 38 ++++++- toolsrc/include/vcpkg/base/rng.h | 167 +++++++++++++++++++++------ toolsrc/include/vcpkg/base/work_queue.h | 183 +++++++++++++++++++++++++++++ toolsrc/src/vcpkg/base/files.cpp | 196 +++++++++++++++++++++++++------- toolsrc/src/vcpkg/base/rng.cpp | 4 +- 5 files changed, 512 insertions(+), 76 deletions(-) create mode 100644 toolsrc/include/vcpkg/base/work_queue.h (limited to 'toolsrc') diff --git a/toolsrc/include/vcpkg/base/files.h b/toolsrc/include/vcpkg/base/files.h index 3ea0d6036..178fae541 100644 --- a/toolsrc/include/vcpkg/base/files.h +++ b/toolsrc/include/vcpkg/base/files.h @@ -12,14 +12,50 @@ namespace fs using stdfs::copy_options; using stdfs::file_status; using stdfs::file_type; + using stdfs::perms; using stdfs::path; using stdfs::u8path; + /* + std::experimental::filesystem's file_status and file_type are broken in + the presence of symlinks -- a symlink is treated as the object it points + to for `symlink_status` and `symlink_type` + */ + + using stdfs::status; + + // we want to poison ADL with these niebloids + constexpr struct { + file_status operator()(const path& p, std::error_code& ec) const noexcept; + file_status operator()(const path& p) const noexcept; + } symlink_status{}; + + constexpr struct { + inline bool operator()(file_status s) const { + return stdfs::is_symlink(s); + } + + inline bool operator()(const path& p) const { + return stdfs::is_symlink(symlink_status(p)); + } + inline bool operator()(const path& p, std::error_code& ec) const { + return stdfs::is_symlink(symlink_status(p, ec)); + } + } is_symlink{}; + inline bool is_regular_file(file_status s) { return stdfs::is_regular_file(s); } inline bool is_directory(file_status s) { return stdfs::is_directory(s); } - inline bool is_symlink(file_status s) { return stdfs::is_symlink(s); } } +/* + if someone attempts to use unqualified `symlink_status` or `is_symlink`, + they might get the ADL version, which is broken. + Therefore, put `symlink_status` in the global namespace, so that they get + our symlink_status. +*/ +using fs::symlink_status; +using fs::is_symlink; + namespace vcpkg::Files { struct Filesystem diff --git a/toolsrc/include/vcpkg/base/rng.h b/toolsrc/include/vcpkg/base/rng.h index 1bcab05b3..4a0411f64 100644 --- a/toolsrc/include/vcpkg/base/rng.h +++ b/toolsrc/include/vcpkg/base/rng.h @@ -4,17 +4,56 @@ #include #include -namespace vcpkg { +namespace vcpkg::Rng { + + namespace detail { + template + constexpr std::size_t bitsize = sizeof(T) * CHAR_BITS; + + template + constexpr bool is_valid_shift(int k) { + return 0 <= k && k <= bitsize; + } + + // precondition: 0 <= k < bitsize + template + constexpr T ror(T x, int k) { + if (k == 0) { + return x; + } + return (x >> k) | (x << (bitsize - k)); + } + + // precondition: 0 <= k < bitsize + template + constexpr T rol(T x, int k) { + if (k == 0) { + return x; + } + return (x << k) | (x >> (bitsize - k)); + } + + // there _is_ a way to do this generally, but I don't know how to + template + struct XoshiroJumpTable; + + template <> + struct XoshiroJumpTable { + constexpr static std::uint64_t value[4] = { + 0x180ec6d33cfd0aba, 0xd5a61266f0c9392c, 0xa9582618e03fc9aa, 0x39abdc4529b1661c + }; + }; + } /* NOTE(ubsan): taken from the xoshiro paper initialized from random_device by default actual code is copied from wikipedia, since I wrote that code */ - struct splitmix64_engine { - splitmix64_engine() noexcept; + struct splitmix { + splitmix() noexcept; - constexpr splitmix64_engine(std::uint64_t seed) noexcept + constexpr splitmix(std::uint64_t seed) noexcept : state(seed) {} constexpr std::uint64_t operator()() noexcept { @@ -35,62 +74,126 @@ namespace vcpkg { return std::numeric_limits::min(); } + template + constexpr void fill(T* first, T* last) { + constexpr auto mask = + static_cast(std::numeric_limits::max()); + + const auto remaining = + (last - first) % (sizeof(std::uint64_t) / sizeof(T)); + + for (auto it = first; it != last - remaining;) { + const auto item = (*this)(); + for ( + int shift = 0; + shift < 64; + shift += detail::bitsize, ++it + ) { + *it = static_cast((item >> shift) & mask); + } + } + + if (remaining == 0) return; + + int shift = 0; + const auto item = (*this)(); + for (auto it = last - remaining; + it != last; + shift += detail::bitsize, ++it + ) { + *it = static_cast((item >> shift) & mask); + } + } + private: std::uint64_t state; }; - // Sebastian Vigna's xorshift-based xoshiro xoshiro256** engine + template + struct starstar_scrambler { + constexpr static UIntType scramble(UIntType n) noexcept { + return detail::rol(n * S, R) * T; + } + }; + + // Sebastian Vigna's xorshift-based xoshiro engine // fast and really good - // uses the splitmix64_engine to initialize state - struct xoshiro256ss_engine { - // splitmix64_engine will be initialized with random_device - xoshiro256ss_engine() noexcept { - splitmix64_engine sm64{}; - - for (std::uint64_t& s : this->state) { - s = sm64(); - } + // uses the splitmix to initialize state + template + struct xoshiro_engine { + static_assert(detail::is_valid_shift(A)); + static_assert(detail::is_valid_shift(B)); + static_assert(std::is_unsigned_v); + + // splitmix will be initialized with random_device + xoshiro_engine() noexcept { + splitmix sm{}; + + sm.fill(&state[0], &state[4]); } - constexpr xoshiro256ss_engine(std::uint64_t seed) noexcept : state() { - splitmix64_engine sm64{seed}; + constexpr xoshiro_engine(std::uint64_t seed) noexcept : state() { + splitmix sm{seed}; - for (std::uint64_t& s : this->state) { - s = sm64(); - } + sm.fill(&state[0], &state[4]); } - constexpr std::uint64_t operator()() noexcept { - std::uint64_t const result = rol(state[1] * 5, 7) * 9; + constexpr UIntType operator()() noexcept { + const UIntType result = Scrambler::scramble(state[0]); - std::uint64_t const t = state[1] << 17; + const UIntType t = state[1] << A; - // state[i] = state[i] ^ state[i + 4 mod 4] state[2] ^= state[0]; state[3] ^= state[1]; state[1] ^= state[2]; state[0] ^= state[3]; state[2] ^= t; - state[3] ^= rol(state[3], 45); + state[3] ^= detail::rol(state[3], B); return result; } - constexpr std::uint64_t max() const noexcept { - return std::numeric_limits::max(); + constexpr UIntType max() const noexcept { + return std::numeric_limits::max(); } constexpr std::uint64_t min() const noexcept { - return std::numeric_limits::min(); + return std::numeric_limits::min(); + } + + // quickly jump ahead 2^e steps + // takes 4 * bitsize rng next operations + template + constexpr void discard_e() noexcept { + using JT = detail::XoshiroJumpTable; + + UIntType s[4] = {}; + for (const auto& jump : JT::value) { + for (std::size_t i = 0; i < bitsize; ++i) { + if ((jump >> i) & 1) { + s[0] ^= state[0]; + s[1] ^= state[1]; + s[2] ^= state[2]; + s[3] ^= state[3]; + } + (*this)(); + } + } + + state[0] = s[0]; + state[1] = s[1]; + state[2] = s[2]; + state[3] = s[3]; } private: // rotate left - constexpr std::uint64_t rol(std::uint64_t x, int k) { - return (x << k) | (x >> (64 - k)); - } - - std::uint64_t state[4]; + UIntType state[4]; }; + using xoshiro256ss = xoshiro_engine< + std::uint64_t, + starstar_scrambler, + 17, + 45>; } diff --git a/toolsrc/include/vcpkg/base/work_queue.h b/toolsrc/include/vcpkg/base/work_queue.h new file mode 100644 index 000000000..4db167fa6 --- /dev/null +++ b/toolsrc/include/vcpkg/base/work_queue.h @@ -0,0 +1,183 @@ +#pragma once + +#include +#include + +namespace vcpkg { + namespace detail { + template + auto call_action( + Action& action, + const WorkQueue& work_queue, + ThreadLocalData& tld + ) -> decltype(static_cast(std::move(action)(tld, work_queue))) + { + std::move(action)(tld, work_queue); + } + + template + auto call_action( + Action& action, + const WorkQueue&, + ThreadLocalData& tld + ) -> decltype(static_cast(std::move(action)(tld))) + { + std::move(action)(tld); + } + } + + template + struct WorkQueue { + template + explicit WorkQueue(const F& initializer) noexcept { + state = State::Joining; + + std::size_t num_threads = std::thread::hardware_concurrency(); + if (num_threads == 0) { + num_threads = 4; + } + + m_threads.reserve(num_threads); + for (std::size_t i = 0; i < num_threads; ++i) { + m_threads.emplace_back(this, initializer); + } + } + + WorkQueue(WorkQueue const&) = delete; + WorkQueue(WorkQueue&&) = delete; + + ~WorkQueue() = default; + + // runs all remaining tasks, and blocks on their finishing + // if this is called in an existing task, _will block forever_ + // DO NOT DO THAT + // thread-unsafe + void join() { + { + auto lck = std::unique_lock(m_mutex); + if (m_state == State::Running) { + m_state = State::Joining; + } else if (m_state == State::Joining) { + Checks::exit_with_message(VCPKG_LINE_INFO, "Attempted to join more than once"); + } + } + for (auto& thrd : m_threads) { + thrd.join(); + } + } + + // useful in the case of errors + // doesn't stop any existing running tasks + // returns immediately, so that one can call this in a task + void terminate() const { + { + auto lck = std::unique_lock(m_mutex); + m_state = State::Terminated; + } + m_cv.notify_all(); + } + + void enqueue_action(Action a) const { + { + auto lck = std::unique_lock(m_mutex); + m_actions.push_back(std::move(a)); + } + m_cv.notify_one(); + } + + template + void enqueue_all_actions_by_move(Rng&& rng) const { + { + using std::begin; + using std::end; + + auto lck = std::unique_lock(m_mutex); + + auto first = begin(rng); + auto last = end(rng); + + m_actions.reserve(m_actions.size() + (end - begin)); + + std::move(first, last, std::back_insert_iterator(rng)); + } + + m_cv.notify_all(); + } + + template + void enqueue_all_actions(Rng&& rng) const { + { + using std::begin; + using std::end; + + auto lck = std::unique_lock(m_mutex); + + auto first = begin(rng); + auto last = end(rng); + + m_actions.reserve(m_actions.size() + (end - begin)); + + std::copy(first, last, std::back_insert_iterator(rng)); + } + + m_cv.notify_all(); + } + + private: + friend struct WorkQueueWorker { + const WorkQueue* work_queue; + ThreadLocalData tld; + + template + WorkQueueWorker(const WorkQueue* work_queue, const F& initializer) + : work_queue(work_queue), tld(initializer()) + { } + + void operator()() { + for (;;) { + auto lck = std::unique_lock(work_queue->m_mutex); + ++work_queue->running_workers; + + const auto state = work_queue->m_state; + + if (state == State::Terminated) { + --work_queue->running_workers; + return; + } + + if (work_queue->m_actions.empty()) { + --work_queue->running_workers; + if (state == State::Running || work_queue->running_workers > 0) { + work_queue->m_cv.wait(lck); + continue; + } + + // state == State::Joining and we are the only worker + // no more work! + return; + } + + Action action = work_queue->m_actions.pop_back(); + lck.unlock(); + + detail::call_action(action, *work_queue, tld); + } + } + }; + + enum class State : std::uint16_t { + Running, + Joining, + Terminated, + }; + + mutable std::mutex m_mutex; + // these four are under m_mutex + mutable State m_state; + mutable std::uint16_t running_workers; + mutable std::vector m_actions; + mutable std::condition_variable condition_variable; + + std::vector m_threads; + }; +} diff --git a/toolsrc/src/vcpkg/base/files.cpp b/toolsrc/src/vcpkg/base/files.cpp index d0926bb4c..e89c531be 100644 --- a/toolsrc/src/vcpkg/base/files.cpp +++ b/toolsrc/src/vcpkg/base/files.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #if defined(__linux__) || defined(__APPLE__) @@ -21,6 +22,45 @@ #include #endif +namespace fs { + file_status decltype(symlink_status)::operator()(const path& p, std::error_code& ec) const noexcept { +#if defined(_WIN32) + /* + do not find the permissions of the file -- it's unnecessary for the + things that vcpkg does. + if one were to add support for this in the future, one should look + into GetFileSecurityW + */ + perms permissions = perms::unknown; + + WIN32_FILE_ATTRIBUTE_DATA file_attributes; + file_type ft = file_type::unknown; + if (!GetFileAttributesExW(p.c_str(), GetFileExInfoStandard, &file_attributes)) { + ft = file_type::not_found; + } else if (file_attributes.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) { + // check for reparse point -- if yes, then symlink + ft = file_type::symlink; + } else if (file_attributes.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) { + ft = file_type::directory; + } else { + // otherwise, the file is a regular file + ft = file_type::regular; + } + + return file_status(ft, permissions); + +#else + return stdfs::symlink_status(p, ec); +#endif + } + + file_status decltype(symlink_status)::operator()(const path& p) const noexcept { + std::error_code ec; + auto result = symlink_status(p, ec); + if (ec) vcpkg::Checks::exit_with_message(VCPKG_LINE_INFO, "error getting status of path %s: %s", p, ec.message()); + } +} + namespace vcpkg::Files { static const std::regex FILESYSTEM_INVALID_CHARACTERS_REGEX = std::regex(R"([\/:*?"<>|])"); @@ -263,55 +303,129 @@ namespace vcpkg::Files (as well as on macOS and Linux), this is just as fast and will have fewer spurious errors due to locks. */ - struct recursive { - const fs::path& tmp_directory; - std::error_code& ec; - xoshiro256ss_engine& rng; - - void operator()(const fs::path& current) const { - const auto type = fs::stdfs::symlink_status(current, ec).type(); - if (ec) return; - - const auto tmp_name = Strings::b64url_encode(rng()); - const auto tmp_path = tmp_directory / tmp_name; - - switch (type) { - case fs::file_type::directory: { - fs::stdfs::rename(current, tmp_path, ec); - if (ec) return; - for (const auto& entry : fs::stdfs::directory_iterator(tmp_path)) { - (*this)(entry); + + /* + `remove` doesn't actually remove anything -- it simply moves the + files into a parent directory (which ends up being at `path`), + and then inserts `actually_remove{current_path}` into the work + queue. + */ + struct remove { + struct tld { + const fs::path& tmp_directory; + std::uint64_t index; + + std::atomic& files_deleted; + + std::mutex& ec_mutex; + std::error_code& ec; + }; + + struct actually_remove; + using queue = WorkQueue; + + /* + if `current_path` is a directory, first `remove`s all + elements of the directory, then calls remove. + + else, just calls remove. + */ + struct actually_remove { + fs::path current_path; + + void operator()(tld& info, const queue& queue) const { + std::error_code ec; + const auto path_type = fs::symlink_status(current_path, ec).type(); + + if (check_ec(ec, info, queue)) return; + + if (path_type == fs::file_type::directory) { + for (const auto& entry : fs::stdfs::directory_iterator(current_path)) { + remove{}(entry, info, queue); + } + } + + if (fs::stdfs::remove(current_path, ec)) { + info.files_deleted.fetch_add(1, std::memory_order_relaxed); + } else { + check_ec(ec, info, queue); } - fs::stdfs::remove(tmp_path, ec); - } break; - case fs::file_type::symlink: - case fs::file_type::regular: { - fs::stdfs::rename(current, tmp_path, ec); - fs::stdfs::remove(current, ec); - } break; - case fs::file_type::not_found: return; - case fs::file_type::none: { - Checks::exit_with_message(VCPKG_LINE_INFO, "Error occurred when evaluating file type of file: %s", current); - } - default: { - Checks::exit_with_message(VCPKG_LINE_INFO, "Attempted to delete special file: %s", current); } + }; + + static bool check_ec(const std::error_code& ec, tld& info, const queue& queue) { + if (ec) { + queue.terminate(); + + auto lck = std::unique_lock(info.ec_mutex); + if (!info.ec) { + info.ec = ec; + } + + return true; + } else { + return false; } } + + void operator()(const fs::path& current_path, tld& info, const queue& queue) const { + std::error_code ec; + + const auto type = fs::symlink_status(current_path, ec).type(); + if (check_ec(ec, info, queue)) return; + + const auto tmp_name = Strings::b64url_encode(info.index++); + const auto tmp_path = info.tmp_directory / tmp_name; + + fs::stdfs::rename(current_path, tmp_path, ec); + if (check_ec(ec, info, queue)) return; + + queue.enqueue_action(actually_remove{std::move(tmp_path)}); + } }; - auto const real_path = fs::stdfs::absolute(path); + const auto path_type = fs::symlink_status(path, ec).type(); - if (! real_path.has_parent_path()) { - Checks::exit_with_message(VCPKG_LINE_INFO, "Attempted to remove_all the base directory"); + std::atomic files_deleted = 0; + + if (path_type == fs::file_type::directory) { + std::uint64_t index = 0; + std::mutex ec_mutex; + + auto queue = remove::queue([&] { + index += 1 << 32; + return remove::tld{path, index, files_deleted, ec_mutex, ec}; + }); + + index += 1 << 32; + auto main_tld = remove::tld{path, index, files_deleted, ec_mutex, ec}; + for (const auto& entry : fs::stdfs::directory_iterator(path)) { + remove{}(entry, main_tld, queue); + } + + queue.join(); } - // thoughts: is this fine? or should we do something different? - // maybe a temporary directory? - auto const base_path = real_path.parent_path(); + /* + we need to do backoff on the removal of the top level directory, + since we need to place all moved files into that top level + directory, and so we can only delete the directory after all the + lower levels have been deleted. + */ + for (int backoff = 0; backoff < 5; ++backoff) { + if (backoff) { + using namespace std::chrono_literals; + auto backoff_time = 100ms * backoff; + std::this_thread::sleep_for(backoff_time); + } + + if (fs::stdfs::remove(path, ec)) { + files_deleted.fetch_add(1, std::memory_order_relaxed); + break; + } + } - xoshiro256ss_engine rng{}; - recursive{base_path, ec, rng}(real_path); + return files_deleted; } virtual bool exists(const fs::path& path) const override { return fs::stdfs::exists(path); } virtual bool is_directory(const fs::path& path) const override { return fs::stdfs::is_directory(path); } @@ -343,11 +457,11 @@ namespace vcpkg::Files virtual fs::file_status status(const fs::path& path, std::error_code& ec) const override { - return fs::stdfs::status(path, ec); + return fs::status(path, ec); } virtual fs::file_status symlink_status(const fs::path& path, std::error_code& ec) const override { - return fs::stdfs::symlink_status(path, ec); + return fs::symlink_status(path, ec); } virtual void write_contents(const fs::path& file_path, const std::string& data, std::error_code& ec) override { diff --git a/toolsrc/src/vcpkg/base/rng.cpp b/toolsrc/src/vcpkg/base/rng.cpp index 9fe2ea3b4..40ff646b7 100644 --- a/toolsrc/src/vcpkg/base/rng.cpp +++ b/toolsrc/src/vcpkg/base/rng.cpp @@ -1,11 +1,11 @@ #include -namespace vcpkg { +namespace vcpkg::Rng { namespace { std::random_device system_entropy{}; } - splitmix64_engine::splitmix64_engine() { + splitmix::splitmix() { std::uint64_t top_half = system_entropy(); std::uint64_t bottom_half = system_entropy(); -- cgit v1.2.3 From 43493b56df7c8f7aab02256ab7f65135d4dd1d4c Mon Sep 17 00:00:00 2001 From: Nicole Mazzuca Date: Wed, 10 Jul 2019 15:18:44 -0700 Subject: delete the random number generator --- toolsrc/include/vcpkg/base/rng.h | 199 --------------------------------------- toolsrc/src/vcpkg/base/rng.cpp | 14 --- 2 files changed, 213 deletions(-) delete mode 100644 toolsrc/include/vcpkg/base/rng.h delete mode 100644 toolsrc/src/vcpkg/base/rng.cpp (limited to 'toolsrc') diff --git a/toolsrc/include/vcpkg/base/rng.h b/toolsrc/include/vcpkg/base/rng.h deleted file mode 100644 index 4a0411f64..000000000 --- a/toolsrc/include/vcpkg/base/rng.h +++ /dev/null @@ -1,199 +0,0 @@ -#pragma once - -#include -#include -#include - -namespace vcpkg::Rng { - - namespace detail { - template - constexpr std::size_t bitsize = sizeof(T) * CHAR_BITS; - - template - constexpr bool is_valid_shift(int k) { - return 0 <= k && k <= bitsize; - } - - // precondition: 0 <= k < bitsize - template - constexpr T ror(T x, int k) { - if (k == 0) { - return x; - } - return (x >> k) | (x << (bitsize - k)); - } - - // precondition: 0 <= k < bitsize - template - constexpr T rol(T x, int k) { - if (k == 0) { - return x; - } - return (x << k) | (x >> (bitsize - k)); - } - - // there _is_ a way to do this generally, but I don't know how to - template - struct XoshiroJumpTable; - - template <> - struct XoshiroJumpTable { - constexpr static std::uint64_t value[4] = { - 0x180ec6d33cfd0aba, 0xd5a61266f0c9392c, 0xa9582618e03fc9aa, 0x39abdc4529b1661c - }; - }; - } - - /* - NOTE(ubsan): taken from the xoshiro paper - initialized from random_device by default - actual code is copied from wikipedia, since I wrote that code - */ - struct splitmix { - splitmix() noexcept; - - constexpr splitmix(std::uint64_t seed) noexcept - : state(seed) {} - - constexpr std::uint64_t operator()() noexcept { - state += 0x9E3779B97F4A7C15; - - std::uint64_t result = state; - result = (result ^ (result >> 30)) * 0xBF58476D1CE4E5B9; - result = (result ^ (result >> 27)) * 0x94D049BB133111EB; - - return result ^ (result >> 31); - } - - constexpr std::uint64_t max() const noexcept { - return std::numeric_limits::max(); - } - - constexpr std::uint64_t min() const noexcept { - return std::numeric_limits::min(); - } - - template - constexpr void fill(T* first, T* last) { - constexpr auto mask = - static_cast(std::numeric_limits::max()); - - const auto remaining = - (last - first) % (sizeof(std::uint64_t) / sizeof(T)); - - for (auto it = first; it != last - remaining;) { - const auto item = (*this)(); - for ( - int shift = 0; - shift < 64; - shift += detail::bitsize, ++it - ) { - *it = static_cast((item >> shift) & mask); - } - } - - if (remaining == 0) return; - - int shift = 0; - const auto item = (*this)(); - for (auto it = last - remaining; - it != last; - shift += detail::bitsize, ++it - ) { - *it = static_cast((item >> shift) & mask); - } - } - - private: - std::uint64_t state; - }; - - template - struct starstar_scrambler { - constexpr static UIntType scramble(UIntType n) noexcept { - return detail::rol(n * S, R) * T; - } - }; - - // Sebastian Vigna's xorshift-based xoshiro engine - // fast and really good - // uses the splitmix to initialize state - template - struct xoshiro_engine { - static_assert(detail::is_valid_shift(A)); - static_assert(detail::is_valid_shift(B)); - static_assert(std::is_unsigned_v); - - // splitmix will be initialized with random_device - xoshiro_engine() noexcept { - splitmix sm{}; - - sm.fill(&state[0], &state[4]); - } - - constexpr xoshiro_engine(std::uint64_t seed) noexcept : state() { - splitmix sm{seed}; - - sm.fill(&state[0], &state[4]); - } - - constexpr UIntType operator()() noexcept { - const UIntType result = Scrambler::scramble(state[0]); - - const UIntType t = state[1] << A; - - state[2] ^= state[0]; - state[3] ^= state[1]; - state[1] ^= state[2]; - state[0] ^= state[3]; - - state[2] ^= t; - state[3] ^= detail::rol(state[3], B); - - return result; - } - - constexpr UIntType max() const noexcept { - return std::numeric_limits::max(); - } - - constexpr std::uint64_t min() const noexcept { - return std::numeric_limits::min(); - } - - // quickly jump ahead 2^e steps - // takes 4 * bitsize rng next operations - template - constexpr void discard_e() noexcept { - using JT = detail::XoshiroJumpTable; - - UIntType s[4] = {}; - for (const auto& jump : JT::value) { - for (std::size_t i = 0; i < bitsize; ++i) { - if ((jump >> i) & 1) { - s[0] ^= state[0]; - s[1] ^= state[1]; - s[2] ^= state[2]; - s[3] ^= state[3]; - } - (*this)(); - } - } - - state[0] = s[0]; - state[1] = s[1]; - state[2] = s[2]; - state[3] = s[3]; - } - private: - // rotate left - UIntType state[4]; - }; - - using xoshiro256ss = xoshiro_engine< - std::uint64_t, - starstar_scrambler, - 17, - 45>; -} diff --git a/toolsrc/src/vcpkg/base/rng.cpp b/toolsrc/src/vcpkg/base/rng.cpp deleted file mode 100644 index 40ff646b7..000000000 --- a/toolsrc/src/vcpkg/base/rng.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include - -namespace vcpkg::Rng { - namespace { - std::random_device system_entropy{}; - } - - splitmix::splitmix() { - std::uint64_t top_half = system_entropy(); - std::uint64_t bottom_half = system_entropy(); - - state = (top_half << 32) | bottom_half; - } -} -- cgit v1.2.3 From 3b6d6b3465e0e79999e5995f0104a6e8c021088c Mon Sep 17 00:00:00 2001 From: Nicole Mazzuca Date: Wed, 10 Jul 2019 15:42:13 -0700 Subject: actually get the code compiling --- toolsrc/include/vcpkg/base/work_queue.h | 42 ++++++++++++++++++--------------- toolsrc/src/vcpkg/base/files.cpp | 9 +++---- 2 files changed, 28 insertions(+), 23 deletions(-) (limited to 'toolsrc') diff --git a/toolsrc/include/vcpkg/base/work_queue.h b/toolsrc/include/vcpkg/base/work_queue.h index 4db167fa6..71e00a2ab 100644 --- a/toolsrc/include/vcpkg/base/work_queue.h +++ b/toolsrc/include/vcpkg/base/work_queue.h @@ -1,10 +1,15 @@ #pragma once +#include #include #include namespace vcpkg { + template + struct WorkQueue; + namespace detail { + // for SFINAE purposes, keep out of the class template auto call_action( Action& action, @@ -29,8 +34,8 @@ namespace vcpkg { template struct WorkQueue { template - explicit WorkQueue(const F& initializer) noexcept { - state = State::Joining; + explicit WorkQueue(const F& tld_init) noexcept { + m_state = State::Running; std::size_t num_threads = std::thread::hardware_concurrency(); if (num_threads == 0) { @@ -39,7 +44,7 @@ namespace vcpkg { m_threads.reserve(num_threads); for (std::size_t i = 0; i < num_threads; ++i) { - m_threads.emplace_back(this, initializer); + m_threads.push_back(std::thread(Worker{this, tld_init()})); } } @@ -93,10 +98,10 @@ namespace vcpkg { auto lck = std::unique_lock(m_mutex); - auto first = begin(rng); - auto last = end(rng); + const auto first = begin(rng); + const auto last = end(rng); - m_actions.reserve(m_actions.size() + (end - begin)); + m_actions.reserve(m_actions.size() + (last - first)); std::move(first, last, std::back_insert_iterator(rng)); } @@ -112,10 +117,10 @@ namespace vcpkg { auto lck = std::unique_lock(m_mutex); - auto first = begin(rng); - auto last = end(rng); + const auto first = begin(rng); + const auto last = end(rng); - m_actions.reserve(m_actions.size() + (end - begin)); + m_actions.reserve(m_actions.size() + (last - first)); std::copy(first, last, std::back_insert_iterator(rng)); } @@ -124,18 +129,15 @@ namespace vcpkg { } private: - friend struct WorkQueueWorker { + struct Worker { const WorkQueue* work_queue; ThreadLocalData tld; - template - WorkQueueWorker(const WorkQueue* work_queue, const F& initializer) - : work_queue(work_queue), tld(initializer()) - { } - void operator()() { + // unlocked when waiting, or when in the `call_action` block + // locked otherwise + auto lck = std::unique_lock(work_queue->m_mutex); for (;;) { - auto lck = std::unique_lock(work_queue->m_mutex); ++work_queue->running_workers; const auto state = work_queue->m_state; @@ -157,10 +159,12 @@ namespace vcpkg { return; } - Action action = work_queue->m_actions.pop_back(); - lck.unlock(); + Action action = std::move(work_queue->m_actions.back()); + work_queue->m_actions.pop_back(); + lck.unlock(); detail::call_action(action, *work_queue, tld); + lck.lock(); } } }; @@ -176,7 +180,7 @@ namespace vcpkg { mutable State m_state; mutable std::uint16_t running_workers; mutable std::vector m_actions; - mutable std::condition_variable condition_variable; + mutable std::condition_variable m_cv; std::vector m_threads; }; diff --git a/toolsrc/src/vcpkg/base/files.cpp b/toolsrc/src/vcpkg/base/files.cpp index e89c531be..d8a982164 100644 --- a/toolsrc/src/vcpkg/base/files.cpp +++ b/toolsrc/src/vcpkg/base/files.cpp @@ -1,7 +1,6 @@ #include "pch.h" #include -#include #include #include #include @@ -57,7 +56,9 @@ namespace fs { file_status decltype(symlink_status)::operator()(const path& p) const noexcept { std::error_code ec; auto result = symlink_status(p, ec); - if (ec) vcpkg::Checks::exit_with_message(VCPKG_LINE_INFO, "error getting status of path %s: %s", p, ec.message()); + if (ec) vcpkg::Checks::exit_with_message(VCPKG_LINE_INFO, "error getting status of path %s: %s", p.string(), ec.message()); + + return result; } } @@ -393,11 +394,11 @@ namespace vcpkg::Files std::mutex ec_mutex; auto queue = remove::queue([&] { - index += 1 << 32; + index += static_cast(1) << 32; return remove::tld{path, index, files_deleted, ec_mutex, ec}; }); - index += 1 << 32; + index += static_cast(1) << 32; auto main_tld = remove::tld{path, index, files_deleted, ec_mutex, ec}; for (const auto& entry : fs::stdfs::directory_iterator(path)) { remove{}(entry, main_tld, queue); -- cgit v1.2.3 From 5b76f24f35976739991941d3b6289fb78fd93648 Mon Sep 17 00:00:00 2001 From: Nicole Mazzuca Date: Wed, 10 Jul 2019 16:28:56 -0700 Subject: make this compile on macos --- toolsrc/include/vcpkg/base/files.h | 39 ++++++++++++++++++++---------------- toolsrc/include/vcpkg/base/strings.h | 2 +- toolsrc/src/vcpkg/base/files.cpp | 6 +++--- 3 files changed, 26 insertions(+), 21 deletions(-) (limited to 'toolsrc') diff --git a/toolsrc/include/vcpkg/base/files.h b/toolsrc/include/vcpkg/base/files.h index 178fae541..33f464779 100644 --- a/toolsrc/include/vcpkg/base/files.h +++ b/toolsrc/include/vcpkg/base/files.h @@ -25,23 +25,28 @@ namespace fs using stdfs::status; // we want to poison ADL with these niebloids - constexpr struct { - file_status operator()(const path& p, std::error_code& ec) const noexcept; - file_status operator()(const path& p) const noexcept; - } symlink_status{}; - - constexpr struct { - inline bool operator()(file_status s) const { - return stdfs::is_symlink(s); - } - - inline bool operator()(const path& p) const { - return stdfs::is_symlink(symlink_status(p)); - } - inline bool operator()(const path& p, std::error_code& ec) const { - return stdfs::is_symlink(symlink_status(p, ec)); - } - } is_symlink{}; + + namespace detail { + struct symlink_status_t { + file_status operator()(const path& p, std::error_code& ec) const noexcept; + file_status operator()(const path& p) const noexcept; + }; + struct is_symlink_t { + inline bool operator()(file_status s) const { + return stdfs::is_symlink(s); + } + + inline bool operator()(const path& p) const { + return stdfs::is_symlink(symlink_status(p)); + } + inline bool operator()(const path& p, std::error_code& ec) const { + return stdfs::is_symlink(symlink_status(p, ec)); + } + }; + } + + constexpr detail::symlink_status_t symlink_status{}; + constexpr detail::is_symlink_t is_symlink{}; inline bool is_regular_file(file_status s) { return stdfs::is_regular_file(s); } inline bool is_directory(file_status s) { return stdfs::is_directory(s); } diff --git a/toolsrc/include/vcpkg/base/strings.h b/toolsrc/include/vcpkg/base/strings.h index 423ea2096..82145b826 100644 --- a/toolsrc/include/vcpkg/base/strings.h +++ b/toolsrc/include/vcpkg/base/strings.h @@ -190,7 +190,7 @@ namespace vcpkg::Strings // ignores padding, since one implicitly knows the length from the size of x template std::string b64url_encode(Integral x) { - static_assert(std::is_integral_v); + static_assert(std::is_integral::value, "b64url_encode must take an integer type"); auto value = static_cast>(x); // 64 values, plus the implicit \0 diff --git a/toolsrc/src/vcpkg/base/files.cpp b/toolsrc/src/vcpkg/base/files.cpp index d8a982164..f4c2106d4 100644 --- a/toolsrc/src/vcpkg/base/files.cpp +++ b/toolsrc/src/vcpkg/base/files.cpp @@ -21,8 +21,8 @@ #include #endif -namespace fs { - file_status decltype(symlink_status)::operator()(const path& p, std::error_code& ec) const noexcept { +namespace fs::detail { + file_status symlink_status_t::operator()(const path& p, std::error_code& ec) const noexcept { #if defined(_WIN32) /* do not find the permissions of the file -- it's unnecessary for the @@ -53,7 +53,7 @@ namespace fs { #endif } - file_status decltype(symlink_status)::operator()(const path& p) const noexcept { + file_status symlink_status_t::operator()(const path& p) const noexcept { std::error_code ec; auto result = symlink_status(p, ec); if (ec) vcpkg::Checks::exit_with_message(VCPKG_LINE_INFO, "error getting status of path %s: %s", p.string(), ec.message()); -- cgit v1.2.3 From bb579072077153fabfa74acec852bce222265357 Mon Sep 17 00:00:00 2001 From: Nicole Mazzuca Date: Wed, 10 Jul 2019 17:39:04 -0700 Subject: make it compile on macos under g++6 --- toolsrc/include/vcpkg/base/strings.h | 6 ++++-- toolsrc/include/vcpkg/base/work_queue.h | 4 ++-- toolsrc/src/vcpkg/base/files.cpp | 13 +++++-------- 3 files changed, 11 insertions(+), 12 deletions(-) (limited to 'toolsrc') diff --git a/toolsrc/include/vcpkg/base/strings.h b/toolsrc/include/vcpkg/base/strings.h index 82145b826..9890bedbc 100644 --- a/toolsrc/include/vcpkg/base/strings.h +++ b/toolsrc/include/vcpkg/base/strings.h @@ -191,7 +191,8 @@ namespace vcpkg::Strings template std::string b64url_encode(Integral x) { static_assert(std::is_integral::value, "b64url_encode must take an integer type"); - auto value = static_cast>(x); + using Unsigned = std::make_unsigned_t; + auto value = static_cast(x); // 64 values, plus the implicit \0 constexpr static char map[0x41] = @@ -202,8 +203,8 @@ namespace vcpkg::Strings /*3*/ "wxyz0123456789-_" ; - constexpr static std::make_unsigned_t mask = 0x3F; constexpr static int shift = 5; + constexpr static auto mask = (static_cast(1) << shift) - 1; std::string result; // reserve ceiling(number of bits / 3) @@ -212,6 +213,7 @@ namespace vcpkg::Strings while (value != 0) { char mapped_value = map[value & mask]; result.push_back(mapped_value); + value >>= shift; } return result; diff --git a/toolsrc/include/vcpkg/base/work_queue.h b/toolsrc/include/vcpkg/base/work_queue.h index 71e00a2ab..8a3d27538 100644 --- a/toolsrc/include/vcpkg/base/work_queue.h +++ b/toolsrc/include/vcpkg/base/work_queue.h @@ -103,7 +103,7 @@ namespace vcpkg { m_actions.reserve(m_actions.size() + (last - first)); - std::move(first, last, std::back_insert_iterator(rng)); + std::move(first, last, std::back_inserter(rng)); } m_cv.notify_all(); @@ -122,7 +122,7 @@ namespace vcpkg { m_actions.reserve(m_actions.size() + (last - first)); - std::copy(first, last, std::back_insert_iterator(rng)); + std::copy(first, last, std::back_inserter(rng)); } m_cv.notify_all(); diff --git a/toolsrc/src/vcpkg/base/files.cpp b/toolsrc/src/vcpkg/base/files.cpp index f4c2106d4..8bc37819a 100644 --- a/toolsrc/src/vcpkg/base/files.cpp +++ b/toolsrc/src/vcpkg/base/files.cpp @@ -129,7 +129,7 @@ namespace vcpkg::Files file_stream.read(&output[0], length); file_stream.close(); - return std::move(output); + return output; } virtual Expected> read_lines(const fs::path& file_path) const override { @@ -147,7 +147,7 @@ namespace vcpkg::Files } file_stream.close(); - return std::move(output); + return output; } virtual fs::path find_file_recursively_up(const fs::path& starting_dir, const std::string& filename) const override @@ -372,9 +372,6 @@ namespace vcpkg::Files void operator()(const fs::path& current_path, tld& info, const queue& queue) const { std::error_code ec; - const auto type = fs::symlink_status(current_path, ec).type(); - if (check_ec(ec, info, queue)) return; - const auto tmp_name = Strings::b64url_encode(info.index++); const auto tmp_path = info.tmp_directory / tmp_name; @@ -387,16 +384,16 @@ namespace vcpkg::Files const auto path_type = fs::symlink_status(path, ec).type(); - std::atomic files_deleted = 0; + std::atomic files_deleted{0}; if (path_type == fs::file_type::directory) { std::uint64_t index = 0; std::mutex ec_mutex; - auto queue = remove::queue([&] { + remove::queue queue{[&] { index += static_cast(1) << 32; return remove::tld{path, index, files_deleted, ec_mutex, ec}; - }); + }}; index += static_cast(1) << 32; auto main_tld = remove::tld{path, index, files_deleted, ec_mutex, ec}; -- cgit v1.2.3 From 319023587558a9f8de9d7eabeb7441ef2e7ee277 Mon Sep 17 00:00:00 2001 From: Nicole Mazzuca Date: Thu, 11 Jul 2019 10:53:32 -0700 Subject: fix some comments from code reviewers --- toolsrc/include/vcpkg/base/strings.h | 10 ++++++---- toolsrc/include/vcpkg/base/work_queue.h | 15 +++++++-------- 2 files changed, 13 insertions(+), 12 deletions(-) (limited to 'toolsrc') diff --git a/toolsrc/include/vcpkg/base/strings.h b/toolsrc/include/vcpkg/base/strings.h index 9890bedbc..25cd302b0 100644 --- a/toolsrc/include/vcpkg/base/strings.h +++ b/toolsrc/include/vcpkg/base/strings.h @@ -208,11 +208,13 @@ namespace vcpkg::Strings std::string result; // reserve ceiling(number of bits / 3) - result.reserve((sizeof(value) * 8 + 2) / 3); + result.resize((sizeof(value) * 8 + 2) / 3, map[0]); - while (value != 0) { - char mapped_value = map[value & mask]; - result.push_back(mapped_value); + for (char& c: result) { + if (value == 0) { + break; + } + c = map[value & mask]; value >>= shift; } diff --git a/toolsrc/include/vcpkg/base/work_queue.h b/toolsrc/include/vcpkg/base/work_queue.h index 8a3d27538..b6f070cd8 100644 --- a/toolsrc/include/vcpkg/base/work_queue.h +++ b/toolsrc/include/vcpkg/base/work_queue.h @@ -11,7 +11,7 @@ namespace vcpkg { namespace detail { // for SFINAE purposes, keep out of the class template - auto call_action( + auto call_moved_action( Action& action, const WorkQueue& work_queue, ThreadLocalData& tld @@ -21,7 +21,7 @@ namespace vcpkg { } template - auto call_action( + auto call_moved_action( Action& action, const WorkQueue&, ThreadLocalData& tld @@ -134,21 +134,18 @@ namespace vcpkg { ThreadLocalData tld; void operator()() { - // unlocked when waiting, or when in the `call_action` block + // unlocked when waiting, or when in the `call_moved_action` + // block // locked otherwise auto lck = std::unique_lock(work_queue->m_mutex); for (;;) { - ++work_queue->running_workers; - const auto state = work_queue->m_state; if (state == State::Terminated) { - --work_queue->running_workers; return; } if (work_queue->m_actions.empty()) { - --work_queue->running_workers; if (state == State::Running || work_queue->running_workers > 0) { work_queue->m_cv.wait(lck); continue; @@ -162,9 +159,11 @@ namespace vcpkg { Action action = std::move(work_queue->m_actions.back()); work_queue->m_actions.pop_back(); + ++work_queue->running_workers; lck.unlock(); - detail::call_action(action, *work_queue, tld); + detail::call_moved_action(action, *work_queue, tld); lck.lock(); + --work_queue->running_workers; } } }; -- cgit v1.2.3 From 510b0c5cc0233311b6993b89cd5ce488218ed78d Mon Sep 17 00:00:00 2001 From: Nicole Mazzuca Date: Thu, 11 Jul 2019 15:01:29 -0700 Subject: fix more comments --- toolsrc/include/vcpkg/base/files.h | 29 ++++--- toolsrc/include/vcpkg/base/strings.h | 37 ++------- toolsrc/include/vcpkg/base/work_queue.h | 102 +++++++++++++++++++----- toolsrc/src/vcpkg/archives.cpp | 5 +- toolsrc/src/vcpkg/base/files.cpp | 133 ++++++++++++++++++++++--------- toolsrc/src/vcpkg/base/strings.cpp | 42 ++++++++++ toolsrc/src/vcpkg/build.cpp | 12 +-- toolsrc/src/vcpkg/commands.exportifw.cpp | 16 ++-- toolsrc/src/vcpkg/commands.portsdiff.cpp | 2 +- toolsrc/src/vcpkg/export.cpp | 6 +- toolsrc/src/vcpkg/install.cpp | 3 +- toolsrc/src/vcpkg/remove.cpp | 3 +- 12 files changed, 270 insertions(+), 120 deletions(-) (limited to 'toolsrc') diff --git a/toolsrc/include/vcpkg/base/files.h b/toolsrc/include/vcpkg/base/files.h index 33f464779..6cad3d461 100644 --- a/toolsrc/include/vcpkg/base/files.h +++ b/toolsrc/include/vcpkg/base/files.h @@ -29,27 +29,29 @@ namespace fs namespace detail { struct symlink_status_t { file_status operator()(const path& p, std::error_code& ec) const noexcept; - file_status operator()(const path& p) const noexcept; + file_status operator()(const path& p, vcpkg::LineInfo li) const noexcept; }; struct is_symlink_t { inline bool operator()(file_status s) const { return stdfs::is_symlink(s); } - - inline bool operator()(const path& p) const { - return stdfs::is_symlink(symlink_status(p)); + }; + struct is_regular_file_t { + inline bool operator()(file_status s) const { + return stdfs::is_regular_file(s); } - inline bool operator()(const path& p, std::error_code& ec) const { - return stdfs::is_symlink(symlink_status(p, ec)); + }; + struct is_directory_t { + inline bool operator()(file_status s) const { + return stdfs::is_directory(s); } }; } constexpr detail::symlink_status_t symlink_status{}; constexpr detail::is_symlink_t is_symlink{}; - - inline bool is_regular_file(file_status s) { return stdfs::is_regular_file(s); } - inline bool is_directory(file_status s) { return stdfs::is_directory(s); } + constexpr detail::is_regular_file_t is_regular_file{}; + constexpr detail::is_directory_t is_directory{}; } /* @@ -57,9 +59,14 @@ namespace fs they might get the ADL version, which is broken. Therefore, put `symlink_status` in the global namespace, so that they get our symlink_status. + + We also want to poison the ADL on is_regular_file and is_directory, because + we don't want people calling these functions on paths */ using fs::symlink_status; using fs::is_symlink; +using fs::is_regular_file; +using fs::is_directory; namespace vcpkg::Files { @@ -85,7 +92,9 @@ namespace vcpkg::Files std::error_code& ec) = 0; bool remove(const fs::path& path, LineInfo linfo); virtual bool remove(const fs::path& path, std::error_code& ec) = 0; - virtual std::uintmax_t remove_all(const fs::path& path, std::error_code& ec) = 0; + + virtual std::uintmax_t remove_all(const fs::path& path, std::error_code& ec, fs::path& failure_point) = 0; + std::uintmax_t remove_all(const fs::path& path, LineInfo li); virtual bool exists(const fs::path& path) const = 0; virtual bool is_directory(const fs::path& path) const = 0; virtual bool is_regular_file(const fs::path& path) const = 0; diff --git a/toolsrc/include/vcpkg/base/strings.h b/toolsrc/include/vcpkg/base/strings.h index 25cd302b0..625c0240f 100644 --- a/toolsrc/include/vcpkg/base/strings.h +++ b/toolsrc/include/vcpkg/base/strings.h @@ -188,36 +188,13 @@ namespace vcpkg::Strings // base 64 encoding with URL and filesafe alphabet (base64url) // based on IETF RFC 4648 // ignores padding, since one implicitly knows the length from the size of x - template - std::string b64url_encode(Integral x) { - static_assert(std::is_integral::value, "b64url_encode must take an integer type"); - using Unsigned = std::make_unsigned_t; - auto value = static_cast(x); - - // 64 values, plus the implicit \0 - constexpr static char map[0x41] = - /* 0123456789ABCDEF */ - /*0*/ "ABCDEFGHIJKLMNOP" - /*1*/ "QRSTUVWXYZabcdef" - /*2*/ "ghijklmnopqrstuv" - /*3*/ "wxyz0123456789-_" - ; - - constexpr static int shift = 5; - constexpr static auto mask = (static_cast(1) << shift) - 1; - - std::string result; - // reserve ceiling(number of bits / 3) - result.resize((sizeof(value) * 8 + 2) / 3, map[0]); - - for (char& c: result) { - if (value == 0) { - break; - } - c = map[value & mask]; - value >>= shift; - } + namespace detail { + + struct b64url_encode_t { + std::string operator()(std::uint64_t x) const noexcept; + }; - return result; } + + constexpr detail::b64url_encode_t b64url_encode{}; } diff --git a/toolsrc/include/vcpkg/base/work_queue.h b/toolsrc/include/vcpkg/base/work_queue.h index b6f070cd8..1836404ca 100644 --- a/toolsrc/include/vcpkg/base/work_queue.h +++ b/toolsrc/include/vcpkg/base/work_queue.h @@ -29,18 +29,19 @@ namespace vcpkg { { std::move(action)(tld); } + + struct immediately_run_t {}; } + constexpr detail::immediately_run_t immediately_run{}; + + template struct WorkQueue { template - explicit WorkQueue(const F& tld_init) noexcept { - m_state = State::Running; - - std::size_t num_threads = std::thread::hardware_concurrency(); - if (num_threads == 0) { - num_threads = 4; - } + WorkQueue(std::size_t num_threads, LineInfo li, const F& tld_init) noexcept { + m_line_info = li; + m_state = State::BeforeRun; m_threads.reserve(num_threads); for (std::size_t i = 0; i < num_threads; ++i) { @@ -48,22 +49,52 @@ namespace vcpkg { } } + template + WorkQueue( + detail::immediately_run_t, + std::size_t num_threads, + LineInfo li, + const F& tld_init + ) noexcept : WorkQueue(num_threads, li, tld_init) { + m_state = State::Running; + } + WorkQueue(WorkQueue const&) = delete; WorkQueue(WorkQueue&&) = delete; - ~WorkQueue() = default; + ~WorkQueue() { + auto lck = std::unique_lock(m_mutex); + if (m_state == State::Running) { + Checks::exit_with_message(m_line_info, "Failed to call join() on a WorkQueue that was destroyed"); + } + } + + // should only be called once; anything else is an error + void run(LineInfo li) { + // this should _not_ be locked before `run()` is called; however, we + // want to terminate if someone screws up, rather than cause UB + auto lck = std::unique_lock(m_mutex); + + if (m_state != State::BeforeRun) { + Checks::exit_with_message(li, "Attempted to run() twice"); + } + + m_state = State::Running; + } // runs all remaining tasks, and blocks on their finishing // if this is called in an existing task, _will block forever_ // DO NOT DO THAT // thread-unsafe - void join() { + void join(LineInfo li) { { auto lck = std::unique_lock(m_mutex); - if (m_state == State::Running) { - m_state = State::Joining; - } else if (m_state == State::Joining) { - Checks::exit_with_message(VCPKG_LINE_INFO, "Attempted to join more than once"); + if (is_joined(m_state)) { + Checks::exit_with_message(li, "Attempted to call join() more than once"); + } else if (m_state == State::Terminated) { + m_state = State::TerminatedJoined; + } else { + m_state = State::Joined; } } for (auto& thrd : m_threads) { @@ -77,7 +108,11 @@ namespace vcpkg { void terminate() const { { auto lck = std::unique_lock(m_mutex); - m_state = State::Terminated; + if (is_joined(m_state)) { + m_state = State::TerminatedJoined; + } else { + m_state = State::Terminated; + } } m_cv.notify_all(); } @@ -86,6 +121,8 @@ namespace vcpkg { { auto lck = std::unique_lock(m_mutex); m_actions.push_back(std::move(a)); + + if (m_state == State::BeforeRun) return; } m_cv.notify_one(); } @@ -104,6 +141,8 @@ namespace vcpkg { m_actions.reserve(m_actions.size() + (last - first)); std::move(first, last, std::back_inserter(rng)); + + if (m_state == State::BeforeRun) return; } m_cv.notify_all(); @@ -123,6 +162,8 @@ namespace vcpkg { m_actions.reserve(m_actions.size() + (last - first)); std::copy(first, last, std::back_inserter(rng)); + + if (m_state == State::BeforeRun) return; } m_cv.notify_all(); @@ -134,24 +175,30 @@ namespace vcpkg { ThreadLocalData tld; void operator()() { - // unlocked when waiting, or when in the `call_moved_action` - // block + // unlocked when waiting, or when in the action // locked otherwise auto lck = std::unique_lock(work_queue->m_mutex); + + work_queue->m_cv.wait(lck, [&] { + return work_queue->m_state != State::BeforeRun; + }); + for (;;) { const auto state = work_queue->m_state; - if (state == State::Terminated) { + if (is_terminated(state)) { return; } if (work_queue->m_actions.empty()) { if (state == State::Running || work_queue->running_workers > 0) { + --work_queue->running_workers; work_queue->m_cv.wait(lck); + ++work_queue->running_workers; continue; } - // state == State::Joining and we are the only worker + // the queue isn't running, and we are the only worker // no more work! return; } @@ -159,21 +206,31 @@ namespace vcpkg { Action action = std::move(work_queue->m_actions.back()); work_queue->m_actions.pop_back(); - ++work_queue->running_workers; lck.unlock(); detail::call_moved_action(action, *work_queue, tld); lck.lock(); - --work_queue->running_workers; } } }; - enum class State : std::uint16_t { + enum class State : std::int16_t { + // can only exist upon construction + BeforeRun = -1, + Running, - Joining, + Joined, Terminated, + TerminatedJoined, }; + static bool is_terminated(State st) { + return st == State::Terminated || st == State::TerminatedJoined; + } + + static bool is_joined(State st) { + return st != State::Joined || st == State::TerminatedJoined; + } + mutable std::mutex m_mutex; // these four are under m_mutex mutable State m_state; @@ -182,5 +239,6 @@ namespace vcpkg { mutable std::condition_variable m_cv; std::vector m_threads; + LineInfo m_line_info; }; } diff --git a/toolsrc/src/vcpkg/archives.cpp b/toolsrc/src/vcpkg/archives.cpp index 69a916828..d22e841de 100644 --- a/toolsrc/src/vcpkg/archives.cpp +++ b/toolsrc/src/vcpkg/archives.cpp @@ -15,9 +15,10 @@ namespace vcpkg::Archives #endif ; + fs.remove_all(to_path, VCPKG_LINE_INFO); + fs.remove_all(to_path_partial, VCPKG_LINE_INFO); + // TODO: check this error code std::error_code ec; - fs.remove_all(to_path, ec); - fs.remove_all(to_path_partial, ec); fs.create_directories(to_path_partial, ec); const auto ext = archive.extension(); #if defined(_WIN32) diff --git a/toolsrc/src/vcpkg/base/files.cpp b/toolsrc/src/vcpkg/base/files.cpp index 8bc37819a..a4e67a142 100644 --- a/toolsrc/src/vcpkg/base/files.cpp +++ b/toolsrc/src/vcpkg/base/files.cpp @@ -5,8 +5,8 @@ #include #include #include -#include #include +#include #if defined(__linux__) || defined(__APPLE__) #include @@ -21,8 +21,10 @@ #include #endif -namespace fs::detail { - file_status symlink_status_t::operator()(const path& p, std::error_code& ec) const noexcept { +namespace fs::detail +{ + file_status symlink_status_t::operator()(const path& p, std::error_code& ec) const noexcept + { #if defined(_WIN32) /* do not find the permissions of the file -- it's unnecessary for the @@ -34,14 +36,21 @@ namespace fs::detail { WIN32_FILE_ATTRIBUTE_DATA file_attributes; file_type ft = file_type::unknown; - if (!GetFileAttributesExW(p.c_str(), GetFileExInfoStandard, &file_attributes)) { + if (!GetFileAttributesExW(p.c_str(), GetFileExInfoStandard, &file_attributes)) + { ft = file_type::not_found; - } else if (file_attributes.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) { + } + else if (file_attributes.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) + { // check for reparse point -- if yes, then symlink ft = file_type::symlink; - } else if (file_attributes.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) { + } + else if (file_attributes.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) + { ft = file_type::directory; - } else { + } + else + { // otherwise, the file is a regular file ft = file_type::regular; } @@ -53,12 +62,13 @@ namespace fs::detail { #endif } - file_status symlink_status_t::operator()(const path& p) const noexcept { + file_status symlink_status_t::operator()(const path& p, vcpkg::LineInfo li) const noexcept + { std::error_code ec; auto result = symlink_status(p, ec); - if (ec) vcpkg::Checks::exit_with_message(VCPKG_LINE_INFO, "error getting status of path %s: %s", p.string(), ec.message()); + if (ec) vcpkg::Checks::exit_with_message(li, "error getting status of path %s: %s", p.string(), ec.message()); - return result; + return result; } } @@ -105,6 +115,25 @@ namespace vcpkg::Files if (ec) Checks::exit_with_message(linfo, "error writing lines: %s: %s", path.u8string(), ec.message()); } + std::uintmax_t Filesystem::remove_all(const fs::path& path, LineInfo li) + { + std::error_code ec; + fs::path failure_point; + + const auto result = this->remove_all(path, ec, failure_point); + + if (ec) + { + Checks::exit_with_message(li, + "Failure to remove_all(%s) due to file %s: %s", + path.string(), + failure_point.string(), + ec.message()); + } + + return result; + } + struct RealFilesystem final : Filesystem { virtual Expected read_contents(const fs::path& file_path) const override @@ -296,7 +325,7 @@ namespace vcpkg::Files #endif } virtual bool remove(const fs::path& path, std::error_code& ec) override { return fs::stdfs::remove(path, ec); } - virtual std::uintmax_t remove_all(const fs::path& path, std::error_code& ec) override + virtual std::uintmax_t remove_all(const fs::path& path, std::error_code& ec, fs::path& failure_point) override { /* does not use the std::filesystem call since it is buggy, and can @@ -311,8 +340,10 @@ namespace vcpkg::Files and then inserts `actually_remove{current_path}` into the work queue. */ - struct remove { - struct tld { + struct remove + { + struct tld + { const fs::path& tmp_directory; std::uint64_t index; @@ -320,6 +351,7 @@ namespace vcpkg::Files std::mutex& ec_mutex; std::error_code& ec; + fs::path& failure_point; }; struct actually_remove; @@ -331,52 +363,68 @@ namespace vcpkg::Files else, just calls remove. */ - struct actually_remove { + struct actually_remove + { fs::path current_path; - void operator()(tld& info, const queue& queue) const { + void operator()(tld& info, const queue& queue) const + { std::error_code ec; const auto path_type = fs::symlink_status(current_path, ec).type(); - if (check_ec(ec, info, queue)) return; + if (check_ec(ec, info, queue, current_path)) return; - if (path_type == fs::file_type::directory) { - for (const auto& entry : fs::stdfs::directory_iterator(current_path)) { + if (path_type == fs::file_type::directory) + { + for (const auto& entry : fs::stdfs::directory_iterator(current_path)) + { remove{}(entry, info, queue); } } - if (fs::stdfs::remove(current_path, ec)) { + if (fs::stdfs::remove(current_path, ec)) + { info.files_deleted.fetch_add(1, std::memory_order_relaxed); - } else { - check_ec(ec, info, queue); + } + else + { + check_ec(ec, info, queue, current_path); } } }; - static bool check_ec(const std::error_code& ec, tld& info, const queue& queue) { - if (ec) { + static bool check_ec(const std::error_code& ec, + tld& info, + const queue& queue, + const fs::path& failure_point) + { + if (ec) + { queue.terminate(); auto lck = std::unique_lock(info.ec_mutex); - if (!info.ec) { + if (!info.ec) + { info.ec = ec; } return true; - } else { + } + else + { return false; } } - void operator()(const fs::path& current_path, tld& info, const queue& queue) const { + void operator()(const fs::path& current_path, tld& info, const queue& queue) const + { std::error_code ec; const auto tmp_name = Strings::b64url_encode(info.index++); const auto tmp_path = info.tmp_directory / tmp_name; fs::stdfs::rename(current_path, tmp_path, ec); - if (check_ec(ec, info, queue)) return; + if (check_ec(ec, info, queue, current_path)) return; queue.enqueue_action(actually_remove{std::move(tmp_path)}); } @@ -386,22 +434,28 @@ namespace vcpkg::Files std::atomic files_deleted{0}; - if (path_type == fs::file_type::directory) { + if (path_type == fs::file_type::directory) + { std::uint64_t index = 0; std::mutex ec_mutex; - remove::queue queue{[&] { + auto const tld_gen = [&] { index += static_cast(1) << 32; - return remove::tld{path, index, files_deleted, ec_mutex, ec}; - }}; + return remove::tld{path, index, files_deleted, ec_mutex, ec, failure_point}; + }; - index += static_cast(1) << 32; - auto main_tld = remove::tld{path, index, files_deleted, ec_mutex, ec}; - for (const auto& entry : fs::stdfs::directory_iterator(path)) { + remove::queue queue{4, VCPKG_LINE_INFO, tld_gen}; + + // note: we don't actually start the queue running until the + // `join()`. This allows us to rename all the top-level files in + // peace, so that we don't get collisions. + auto main_tld = tld_gen(); + for (const auto& entry : fs::stdfs::directory_iterator(path)) + { remove{}(entry, main_tld, queue); } - queue.join(); + queue.join(VCPKG_LINE_INFO); } /* @@ -410,14 +464,17 @@ namespace vcpkg::Files directory, and so we can only delete the directory after all the lower levels have been deleted. */ - for (int backoff = 0; backoff < 5; ++backoff) { - if (backoff) { + for (int backoff = 0; backoff < 5; ++backoff) + { + if (backoff) + { using namespace std::chrono_literals; auto backoff_time = 100ms * backoff; std::this_thread::sleep_for(backoff_time); } - if (fs::stdfs::remove(path, ec)) { + if (fs::stdfs::remove(path, ec)) + { files_deleted.fetch_add(1, std::memory_order_relaxed); break; } diff --git a/toolsrc/src/vcpkg/base/strings.cpp b/toolsrc/src/vcpkg/base/strings.cpp index 54a74a7a1..16543046e 100644 --- a/toolsrc/src/vcpkg/base/strings.cpp +++ b/toolsrc/src/vcpkg/base/strings.cpp @@ -288,3 +288,45 @@ bool Strings::contains(StringView haystack, StringView needle) { return Strings::search(haystack, needle) != haystack.end(); } + +namespace vcpkg::Strings::detail { + + template + std::string b64url_encode_implementation(Integral x) { + static_assert(std::is_integral::value, "b64url_encode must take an integer type"); + using Unsigned = std::make_unsigned_t; + auto value = static_cast(x); + + // 64 values, plus the implicit \0 + constexpr static char map[0x41] = + /* 0123456789ABCDEF */ + /*0*/ "ABCDEFGHIJKLMNOP" + /*1*/ "QRSTUVWXYZabcdef" + /*2*/ "ghijklmnopqrstuv" + /*3*/ "wxyz0123456789-_" + ; + + constexpr static int shift = 5; + constexpr static auto mask = (static_cast(1) << shift) - 1; + + std::string result; + // reserve ceiling(number of bits / 3) + result.resize((sizeof(value) * 8 + 2) / 3, map[0]); + + for (char& c: result) { + if (value == 0) { + break; + } + c = map[value & mask]; + value >>= shift; + } + + return result; + } + + std::string b64url_encode_t::operator()(std::uint64_t x) const noexcept{ + return b64url_encode_implementation(x); + } + +} + diff --git a/toolsrc/src/vcpkg/build.cpp b/toolsrc/src/vcpkg/build.cpp index 68df1f965..c50acce7e 100644 --- a/toolsrc/src/vcpkg/build.cpp +++ b/toolsrc/src/vcpkg/build.cpp @@ -363,7 +363,8 @@ namespace vcpkg::Build const Triplet& triplet = spec.triplet(); const auto& triplet_file_path = paths.get_triplet_file_path(spec.triplet()).u8string(); - if (!Strings::case_insensitive_ascii_starts_with(triplet_file_path, paths.triplets.u8string())) + if (!Strings::case_insensitive_ascii_starts_with(triplet_file_path, + paths.triplets.u8string())) { System::printf("-- Loading triplet configuration from: %s\n", triplet_file_path); } @@ -495,7 +496,8 @@ namespace vcpkg::Build if (fs.is_directory(file)) // Will only keep the logs { std::error_code ec; - fs.remove_all(file, ec); + fs::path failure_point; + fs.remove_all(file, ec, failure_point); } } } @@ -610,8 +612,8 @@ namespace vcpkg::Build auto& fs = paths.get_filesystem(); auto pkg_path = paths.package_dir(spec); + fs.remove_all(pkg_path, VCPKG_LINE_INFO); std::error_code ec; - fs.remove_all(pkg_path, ec); fs.create_directories(pkg_path, ec); auto files = fs.get_files_non_recursive(pkg_path); Checks::check_exit(VCPKG_LINE_INFO, files.empty(), "unable to clear path: %s", pkg_path.u8string()); @@ -794,7 +796,7 @@ namespace vcpkg::Build fs.rename_or_copy(tmp_failure_zip, archive_tombstone_path, ".tmp", ec); // clean up temporary directory - fs.remove_all(tmp_log_path, ec); + fs.remove_all(tmp_log_path, VCPKG_LINE_INFO); } } @@ -1018,7 +1020,7 @@ namespace vcpkg::Build hash += "-"; hash += Hash::get_file_hash(fs, *p, "SHA1"); } - else if (pre_build_info.cmake_system_name.empty() || + else if (pre_build_info.cmake_system_name.empty() || pre_build_info.cmake_system_name == "WindowsStore") { hash += "-"; diff --git a/toolsrc/src/vcpkg/commands.exportifw.cpp b/toolsrc/src/vcpkg/commands.exportifw.cpp index f0946110c..3d963a297 100644 --- a/toolsrc/src/vcpkg/commands.exportifw.cpp +++ b/toolsrc/src/vcpkg/commands.exportifw.cpp @@ -352,13 +352,15 @@ namespace vcpkg::Export::IFW System::print2("Generating repository ", repository_dir.generic_u8string(), "...\n"); std::error_code ec; + fs::path failure_point; Files::Filesystem& fs = paths.get_filesystem(); - fs.remove_all(repository_dir, ec); + fs.remove_all(repository_dir, ec, failure_point); Checks::check_exit(VCPKG_LINE_INFO, !ec, - "Could not remove outdated repository directory %s", - repository_dir.generic_u8string()); + "Could not remove outdated repository directory %s due to file %s", + repository_dir.generic_u8string(), + failure_point.string()); const auto cmd_line = Strings::format(R"("%s" --packages "%s" "%s" > nul)", repogen_exe.u8string(), @@ -414,16 +416,18 @@ namespace vcpkg::Export::IFW const VcpkgPaths& paths) { std::error_code ec; + fs::path failure_point; Files::Filesystem& fs = paths.get_filesystem(); // Prepare packages directory const fs::path ifw_packages_dir_path = get_packages_dir_path(export_id, ifw_options, paths); - fs.remove_all(ifw_packages_dir_path, ec); + fs.remove_all(ifw_packages_dir_path, ec, failure_point); Checks::check_exit(VCPKG_LINE_INFO, !ec, - "Could not remove outdated packages directory %s", - ifw_packages_dir_path.generic_u8string()); + "Could not remove outdated packages directory %s due to file %s", + ifw_packages_dir_path.generic_u8string(), + failure_point.string()); fs.create_directory(ifw_packages_dir_path, ec); Checks::check_exit( diff --git a/toolsrc/src/vcpkg/commands.portsdiff.cpp b/toolsrc/src/vcpkg/commands.portsdiff.cpp index b30c38f43..cddc274b8 100644 --- a/toolsrc/src/vcpkg/commands.portsdiff.cpp +++ b/toolsrc/src/vcpkg/commands.portsdiff.cpp @@ -105,7 +105,7 @@ namespace vcpkg::Commands::PortsDiff std::map names_and_versions; for (auto&& port : all_ports) names_and_versions.emplace(port->core_paragraph->name, port->core_paragraph->version); - fs.remove_all(temp_checkout_path, ec); + fs.remove_all(temp_checkout_path, VCPKG_LINE_INFO); return names_and_versions; } diff --git a/toolsrc/src/vcpkg/export.cpp b/toolsrc/src/vcpkg/export.cpp index 88c1526c5..f306bf4e6 100644 --- a/toolsrc/src/vcpkg/export.cpp +++ b/toolsrc/src/vcpkg/export.cpp @@ -400,8 +400,10 @@ namespace vcpkg::Export Files::Filesystem& fs = paths.get_filesystem(); const fs::path export_to_path = paths.root; const fs::path raw_exported_dir_path = export_to_path / export_id; + fs.remove_all(raw_exported_dir_path, VCPKG_LINE_INFO); + + // TODO: error handling std::error_code ec; - fs.remove_all(raw_exported_dir_path, ec); fs.create_directory(raw_exported_dir_path, ec); // execute the plan @@ -476,7 +478,7 @@ With a project open, go to Tools->NuGet Package Manager->Package Manager Console if (!opts.raw) { - fs.remove_all(raw_exported_dir_path, ec); + fs.remove_all(raw_exported_dir_path, VCPKG_LINE_INFO); } } diff --git a/toolsrc/src/vcpkg/install.cpp b/toolsrc/src/vcpkg/install.cpp index de19c360a..981a9233f 100644 --- a/toolsrc/src/vcpkg/install.cpp +++ b/toolsrc/src/vcpkg/install.cpp @@ -355,8 +355,7 @@ namespace vcpkg::Install { auto& fs = paths.get_filesystem(); const fs::path package_dir = paths.package_dir(action.spec); - std::error_code ec; - fs.remove_all(package_dir, ec); + fs.remove_all(package_dir, VCPKG_LINE_INFO); } if (action.build_options.clean_downloads == Build::CleanDownloads::YES) diff --git a/toolsrc/src/vcpkg/remove.cpp b/toolsrc/src/vcpkg/remove.cpp index a40b27bd7..84ec6c981 100644 --- a/toolsrc/src/vcpkg/remove.cpp +++ b/toolsrc/src/vcpkg/remove.cpp @@ -179,8 +179,7 @@ namespace vcpkg::Remove { System::printf("Purging package %s...\n", display_name); Files::Filesystem& fs = paths.get_filesystem(); - std::error_code ec; - fs.remove_all(paths.packages / action.spec.dir(), ec); + fs.remove_all(paths.packages / action.spec.dir(), VCPKG_LINE_INFO); System::printf(System::Color::success, "Purging package %s... done\n", display_name); } } -- cgit v1.2.3 From a0fe40ea5842006c0da901a99ae07d7a25531175 Mon Sep 17 00:00:00 2001 From: Nicole Mazzuca Date: Thu, 11 Jul 2019 18:16:10 -0700 Subject: add tests! Also, fix all the bugs I found when I wrote the tests! --- toolsrc/include/vcpkg/base/files.h | 35 ++-- toolsrc/include/vcpkg/base/strings.h | 10 +- toolsrc/include/vcpkg/base/work_queue.h | 182 ++++++++++++--------- toolsrc/src/tests.files.cpp | 80 +++++++++ toolsrc/src/tests.strings.cpp | 38 +++++ toolsrc/src/vcpkg/base/files.cpp | 3 +- toolsrc/src/vcpkg/base/strings.cpp | 72 ++++---- toolsrc/vcpkg/vcpkg.vcxproj | 4 +- toolsrc/vcpkglib/vcpkglib.vcxproj | 4 +- .../vcpkgmetricsuploader.vcxproj | 4 +- toolsrc/vcpkgtest/vcpkgtest.vcxproj | 10 +- toolsrc/vcpkgtest/vcpkgtest.vcxproj.filters | 6 + 12 files changed, 295 insertions(+), 153 deletions(-) create mode 100644 toolsrc/src/tests.files.cpp create mode 100644 toolsrc/src/tests.strings.cpp (limited to 'toolsrc') diff --git a/toolsrc/include/vcpkg/base/files.h b/toolsrc/include/vcpkg/base/files.h index 6cad3d461..a5e04db25 100644 --- a/toolsrc/include/vcpkg/base/files.h +++ b/toolsrc/include/vcpkg/base/files.h @@ -12,8 +12,8 @@ namespace fs using stdfs::copy_options; using stdfs::file_status; using stdfs::file_type; - using stdfs::perms; using stdfs::path; + using stdfs::perms; using stdfs::u8path; /* @@ -26,25 +26,24 @@ namespace fs // we want to poison ADL with these niebloids - namespace detail { - struct symlink_status_t { + namespace detail + { + struct symlink_status_t + { file_status operator()(const path& p, std::error_code& ec) const noexcept; file_status operator()(const path& p, vcpkg::LineInfo li) const noexcept; }; - struct is_symlink_t { - inline bool operator()(file_status s) const { - return stdfs::is_symlink(s); - } + struct is_symlink_t + { + inline bool operator()(file_status s) const { return stdfs::is_symlink(s); } }; - struct is_regular_file_t { - inline bool operator()(file_status s) const { - return stdfs::is_regular_file(s); - } + struct is_regular_file_t + { + inline bool operator()(file_status s) const { return stdfs::is_regular_file(s); } }; - struct is_directory_t { - inline bool operator()(file_status s) const { - return stdfs::is_directory(s); - } + struct is_directory_t + { + inline bool operator()(file_status s) const { return stdfs::is_directory(s); } }; } @@ -63,10 +62,10 @@ namespace fs We also want to poison the ADL on is_regular_file and is_directory, because we don't want people calling these functions on paths */ -using fs::symlink_status; -using fs::is_symlink; -using fs::is_regular_file; using fs::is_directory; +using fs::is_regular_file; +using fs::is_symlink; +using fs::symlink_status; namespace vcpkg::Files { diff --git a/toolsrc/include/vcpkg/base/strings.h b/toolsrc/include/vcpkg/base/strings.h index 625c0240f..a1906790f 100644 --- a/toolsrc/include/vcpkg/base/strings.h +++ b/toolsrc/include/vcpkg/base/strings.h @@ -188,13 +188,5 @@ namespace vcpkg::Strings // base 64 encoding with URL and filesafe alphabet (base64url) // based on IETF RFC 4648 // ignores padding, since one implicitly knows the length from the size of x - namespace detail { - - struct b64url_encode_t { - std::string operator()(std::uint64_t x) const noexcept; - }; - - } - - constexpr detail::b64url_encode_t b64url_encode{}; + std::string b64url_encode(std::uint64_t x) noexcept; } diff --git a/toolsrc/include/vcpkg/base/work_queue.h b/toolsrc/include/vcpkg/base/work_queue.h index 1836404ca..d6666770b 100644 --- a/toolsrc/include/vcpkg/base/work_queue.h +++ b/toolsrc/include/vcpkg/base/work_queue.h @@ -4,78 +4,67 @@ #include #include -namespace vcpkg { - template +namespace vcpkg +{ + template struct WorkQueue; - namespace detail { + namespace detail + { // for SFINAE purposes, keep out of the class - template - auto call_moved_action( - Action& action, - const WorkQueue& work_queue, - ThreadLocalData& tld - ) -> decltype(static_cast(std::move(action)(tld, work_queue))) + template + auto call_moved_action(Action& action, + const WorkQueue& work_queue, + ThreadLocalData& tld) -> decltype(static_cast(std::move(action)(tld, work_queue))) { std::move(action)(tld, work_queue); } - template - auto call_moved_action( - Action& action, - const WorkQueue&, - ThreadLocalData& tld - ) -> decltype(static_cast(std::move(action)(tld))) + template + auto call_moved_action(Action& action, const WorkQueue&, ThreadLocalData& tld) + -> decltype(static_cast(std::move(action)(tld))) { std::move(action)(tld); } - - struct immediately_run_t {}; } - constexpr detail::immediately_run_t immediately_run{}; - - - template - struct WorkQueue { - template - WorkQueue(std::size_t num_threads, LineInfo li, const F& tld_init) noexcept { + template + struct WorkQueue + { + template + WorkQueue(std::uint16_t num_threads, LineInfo li, const F& tld_init) noexcept + { m_line_info = li; - m_state = State::BeforeRun; + m_unjoined_workers = num_threads; m_threads.reserve(num_threads); - for (std::size_t i = 0; i < num_threads; ++i) { + for (std::size_t i = 0; i < num_threads; ++i) + { m_threads.push_back(std::thread(Worker{this, tld_init()})); } } - template - WorkQueue( - detail::immediately_run_t, - std::size_t num_threads, - LineInfo li, - const F& tld_init - ) noexcept : WorkQueue(num_threads, li, tld_init) { - m_state = State::Running; - } - WorkQueue(WorkQueue const&) = delete; WorkQueue(WorkQueue&&) = delete; - ~WorkQueue() { + ~WorkQueue() + { auto lck = std::unique_lock(m_mutex); - if (m_state == State::Running) { + if (!is_joined(m_state)) + { Checks::exit_with_message(m_line_info, "Failed to call join() on a WorkQueue that was destroyed"); } } // should only be called once; anything else is an error - void run(LineInfo li) { + void run(LineInfo li) + { // this should _not_ be locked before `run()` is called; however, we // want to terminate if someone screws up, rather than cause UB auto lck = std::unique_lock(m_mutex); - if (m_state != State::BeforeRun) { + if (m_state != State::BeforeRun) + { Checks::exit_with_message(li, "Attempted to run() twice"); } @@ -86,18 +75,40 @@ namespace vcpkg { // if this is called in an existing task, _will block forever_ // DO NOT DO THAT // thread-unsafe - void join(LineInfo li) { + void join(LineInfo li) + { { auto lck = std::unique_lock(m_mutex); - if (is_joined(m_state)) { + if (is_joined(m_state)) + { Checks::exit_with_message(li, "Attempted to call join() more than once"); - } else if (m_state == State::Terminated) { + } + else if (m_state == State::Terminated) + { m_state = State::TerminatedJoined; - } else { + } + else + { m_state = State::Joined; } } - for (auto& thrd : m_threads) { + + for (;;) + { + auto lck = std::unique_lock(m_mutex); + if (!m_unjoined_workers) + break; + + else if (!m_running_workers) + { + lck.unlock(); + m_cv.notify_all(); + } + } + + // all threads have returned -- now, it's time to join them + for (auto& thrd : m_threads) + { thrd.join(); } } @@ -105,19 +116,24 @@ namespace vcpkg { // useful in the case of errors // doesn't stop any existing running tasks // returns immediately, so that one can call this in a task - void terminate() const { + void terminate() const + { { auto lck = std::unique_lock(m_mutex); - if (is_joined(m_state)) { + if (is_joined(m_state)) + { m_state = State::TerminatedJoined; - } else { + } + else + { m_state = State::Terminated; } } m_cv.notify_all(); } - void enqueue_action(Action a) const { + void enqueue_action(Action a) const + { { auto lck = std::unique_lock(m_mutex); m_actions.push_back(std::move(a)); @@ -127,8 +143,9 @@ namespace vcpkg { m_cv.notify_one(); } - template - void enqueue_all_actions_by_move(Rng&& rng) const { + template + void enqueue_all_actions_by_move(Rng&& rng) const + { { using std::begin; using std::end; @@ -148,8 +165,9 @@ namespace vcpkg { m_cv.notify_all(); } - template - void enqueue_all_actions(Rng&& rng) const { + template + void enqueue_all_actions(Rng&& rng) const + { { using std::begin; using std::end; @@ -170,37 +188,41 @@ namespace vcpkg { } private: - struct Worker { + struct Worker + { const WorkQueue* work_queue; ThreadLocalData tld; - void operator()() { + void operator()() + { // unlocked when waiting, or when in the action // locked otherwise auto lck = std::unique_lock(work_queue->m_mutex); - work_queue->m_cv.wait(lck, [&] { - return work_queue->m_state != State::BeforeRun; - }); + work_queue->m_cv.wait(lck, [&] { return work_queue->m_state != State::BeforeRun; }); - for (;;) { + for (;;) + { const auto state = work_queue->m_state; - if (is_terminated(state)) { - return; + if (is_terminated(state)) + { + break; } - if (work_queue->m_actions.empty()) { - if (state == State::Running || work_queue->running_workers > 0) { - --work_queue->running_workers; + if (work_queue->m_actions.empty()) + { + if (state == State::Running || work_queue->m_running_workers > 1) + { + --work_queue->m_running_workers; work_queue->m_cv.wait(lck); - ++work_queue->running_workers; + ++work_queue->m_running_workers; continue; } // the queue isn't running, and we are the only worker // no more work! - return; + break; } Action action = std::move(work_queue->m_actions.back()); @@ -210,10 +232,13 @@ namespace vcpkg { detail::call_moved_action(action, *work_queue, tld); lck.lock(); } + + --work_queue->m_unjoined_workers; } }; - enum class State : std::int16_t { + enum class State : std::int16_t + { // can only exist upon construction BeforeRun = -1, @@ -223,22 +248,19 @@ namespace vcpkg { TerminatedJoined, }; - static bool is_terminated(State st) { - return st == State::Terminated || st == State::TerminatedJoined; - } + static bool is_terminated(State st) { return st == State::Terminated || st == State::TerminatedJoined; } - static bool is_joined(State st) { - return st != State::Joined || st == State::TerminatedJoined; - } + static bool is_joined(State st) { return st == State::Joined || st == State::TerminatedJoined; } - mutable std::mutex m_mutex; - // these four are under m_mutex - mutable State m_state; - mutable std::uint16_t running_workers; - mutable std::vector m_actions; - mutable std::condition_variable m_cv; + mutable std::mutex m_mutex{}; + // these are all under m_mutex + mutable State m_state = State::BeforeRun; + mutable std::uint16_t m_running_workers = 0; + mutable std::uint16_t m_unjoined_workers = 0; // num_threads + mutable std::vector m_actions{}; + mutable std::condition_variable m_cv{}; - std::vector m_threads; + std::vector m_threads{}; LineInfo m_line_info; }; } diff --git a/toolsrc/src/tests.files.cpp b/toolsrc/src/tests.files.cpp new file mode 100644 index 000000000..73c7eb5bc --- /dev/null +++ b/toolsrc/src/tests.files.cpp @@ -0,0 +1,80 @@ +#include "tests.pch.h" + +#include +#include + +#include +#include + +#include + +using namespace Microsoft::VisualStudio::CppUnitTestFramework; + +namespace UnitTest1 { + class FilesTest : public TestClass { + using uid = std::uniform_int_distribution; + + std::string get_random_filename() + { + std::random_device rd; + return vcpkg::Strings::b64url_encode(uid{}(rd)); + } + + void create_directory_tree( + vcpkg::Files::Filesystem& fs, + std::uint64_t depth, + const fs::path& base) + { + std::random_device rd; + constexpr auto max_depth = std::uint64_t(3); + const auto width = depth ? uid{0, (max_depth - depth) * 3 / 2}(rd) : 5; + + std::error_code ec; + if (width == 0) { + fs.write_contents(base, "", ec); + Assert::IsFalse(bool(ec)); + + return; + } + + fs.create_directory(base, ec); + Assert::IsFalse(bool(ec)); + + for (int i = 0; i < width; ++i) { + create_directory_tree(fs, depth + 1, base / get_random_filename()); + } + } + + TEST_METHOD(remove_all) { + fs::path temp_dir; + + { + wchar_t* tmp = static_cast(calloc(32'767, 2)); + + if (!GetEnvironmentVariableW(L"TEMP", tmp, 32'767)) { + Assert::Fail(L"GetEnvironmentVariable(\"TEMP\") failed"); + } + + temp_dir = tmp; + + std::string dir_name = "vcpkg-tmp-dir-"; + dir_name += get_random_filename(); + + temp_dir /= dir_name; + } + + auto& fs = vcpkg::Files::get_real_filesystem(); + + std::cout << "temp dir is: " << temp_dir << '\n'; + + std::error_code ec; + create_directory_tree(fs, 0, temp_dir); + + fs::path fp; + fs.remove_all(temp_dir, ec, fp); + Assert::IsFalse(bool(ec)); + + Assert::IsFalse(fs.exists(temp_dir)); + } + }; +} diff --git a/toolsrc/src/tests.strings.cpp b/toolsrc/src/tests.strings.cpp new file mode 100644 index 000000000..14b449e86 --- /dev/null +++ b/toolsrc/src/tests.strings.cpp @@ -0,0 +1,38 @@ +#include "tests.pch.h" + +#include + +#include +#include +#include + +using namespace Microsoft::VisualStudio::CppUnitTestFramework; + +namespace UnitTest1 { + class StringsTest : public TestClass { + TEST_METHOD(b64url_encode) + { + using u64 = std::uint64_t; + + std::vector> map; + + map.emplace_back(0, "AAAAAAAAAAA"); + map.emplace_back(1, "BAAAAAAAAAA"); + + map.emplace_back(u64(1) << 32, "AAAAAEAAAAA"); + map.emplace_back((u64(1) << 32) + 1, "BAAAAEAAAAA"); + + map.emplace_back(0xE4D0'1065'D11E'0229, "pIgHRXGEQTO"); + map.emplace_back(0xA626'FE45'B135'07FF, "_fQNxWk_mYK"); + map.emplace_back(0xEE36'D228'0C31'D405, "FQdMMgi024O"); + map.emplace_back(0x1405'64E7'FE7E'A88C, "Miqf-fOZFQB"); + map.emplace_back(0xFFFF'FFFF'FFFF'FFFF, "__________P"); + + std::string result; + for (const auto& pr : map) { + result = vcpkg::Strings::b64url_encode(pr.first); + Assert::AreEqual(result, pr.second); + } + } + }; +} diff --git a/toolsrc/src/vcpkg/base/files.cpp b/toolsrc/src/vcpkg/base/files.cpp index a4e67a142..4e36fe990 100644 --- a/toolsrc/src/vcpkg/base/files.cpp +++ b/toolsrc/src/vcpkg/base/files.cpp @@ -23,7 +23,7 @@ namespace fs::detail { - file_status symlink_status_t::operator()(const path& p, std::error_code& ec) const noexcept + file_status symlink_status_t::operator()(const path& p, std::error_code&) const noexcept { #if defined(_WIN32) /* @@ -406,6 +406,7 @@ namespace vcpkg::Files if (!info.ec) { info.ec = ec; + info.failure_point = failure_point; } return true; diff --git a/toolsrc/src/vcpkg/base/strings.cpp b/toolsrc/src/vcpkg/base/strings.cpp index 16543046e..ade4384a9 100644 --- a/toolsrc/src/vcpkg/base/strings.cpp +++ b/toolsrc/src/vcpkg/base/strings.cpp @@ -289,44 +289,46 @@ bool Strings::contains(StringView haystack, StringView needle) return Strings::search(haystack, needle) != haystack.end(); } -namespace vcpkg::Strings::detail { - - template - std::string b64url_encode_implementation(Integral x) { - static_assert(std::is_integral::value, "b64url_encode must take an integer type"); - using Unsigned = std::make_unsigned_t; - auto value = static_cast(x); - - // 64 values, plus the implicit \0 - constexpr static char map[0x41] = - /* 0123456789ABCDEF */ - /*0*/ "ABCDEFGHIJKLMNOP" - /*1*/ "QRSTUVWXYZabcdef" - /*2*/ "ghijklmnopqrstuv" - /*3*/ "wxyz0123456789-_" - ; - - constexpr static int shift = 5; - constexpr static auto mask = (static_cast(1) << shift) - 1; - - std::string result; - // reserve ceiling(number of bits / 3) - result.resize((sizeof(value) * 8 + 2) / 3, map[0]); - - for (char& c: result) { - if (value == 0) { - break; +namespace vcpkg::Strings +{ + namespace + { + template + std::string b64url_encode_implementation(Integral x) + { + static_assert(std::is_integral::value, "b64url_encode must take an integer type"); + using Unsigned = std::make_unsigned_t; + auto value = static_cast(x); + + // 64 values, plus the implicit \0 + constexpr static char map[65] = + /* 0123456789ABCDEF */ + /*0*/ "ABCDEFGHIJKLMNOP" + /*1*/ "QRSTUVWXYZabcdef" + /*2*/ "ghijklmnopqrstuv" + /*3*/ "wxyz0123456789-_"; + + // log2(64) + constexpr static int shift = 6; + // 64 - 1 + constexpr static auto mask = 63; + + // ceiling(bitsize(Integral) / log2(64)) + constexpr static auto result_size = (sizeof(value) * 8 + shift - 1) / shift; + + std::string result; + result.reserve(result_size); + + for (std::size_t i = 0; i < result_size; ++i) + { + result.push_back(map[value & mask]); + value >>= shift; } - c = map[value & mask]; - value >>= shift; - } - return result; + return result; + } } - std::string b64url_encode_t::operator()(std::uint64_t x) const noexcept{ - return b64url_encode_implementation(x); - } + std::string b64url_encode(std::uint64_t x) noexcept { return b64url_encode_implementation(x); } } - diff --git a/toolsrc/vcpkg/vcpkg.vcxproj b/toolsrc/vcpkg/vcpkg.vcxproj index 8edea2244..06d849a74 100644 --- a/toolsrc/vcpkg/vcpkg.vcxproj +++ b/toolsrc/vcpkg/vcpkg.vcxproj @@ -21,8 +21,8 @@ {34671B80-54F9-46F5-8310-AC429C11D4FB} vcpkg - 8.1 - v140 + 10.0 + v142 diff --git a/toolsrc/vcpkglib/vcpkglib.vcxproj b/toolsrc/vcpkglib/vcpkglib.vcxproj index 2eff1abee..9312dd627 100644 --- a/toolsrc/vcpkglib/vcpkglib.vcxproj +++ b/toolsrc/vcpkglib/vcpkglib.vcxproj @@ -21,8 +21,8 @@ {B98C92B7-2874-4537-9D46-D14E5C237F04} vcpkglib - 8.1 - v140 + 10.0 + v142 diff --git a/toolsrc/vcpkgmetricsuploader/vcpkgmetricsuploader.vcxproj b/toolsrc/vcpkgmetricsuploader/vcpkgmetricsuploader.vcxproj index e533d0e15..2e689c7fc 100644 --- a/toolsrc/vcpkgmetricsuploader/vcpkgmetricsuploader.vcxproj +++ b/toolsrc/vcpkgmetricsuploader/vcpkgmetricsuploader.vcxproj @@ -21,8 +21,8 @@ {7D6FDEEB-B299-4A23-85EE-F67C4DED47BE} vcpkgmetricsuploader - 8.1 - v140 + 10.0 + v142 diff --git a/toolsrc/vcpkgtest/vcpkgtest.vcxproj b/toolsrc/vcpkgtest/vcpkgtest.vcxproj index 4cda29461..ac7675ba5 100644 --- a/toolsrc/vcpkgtest/vcpkgtest.vcxproj +++ b/toolsrc/vcpkgtest/vcpkgtest.vcxproj @@ -22,6 +22,7 @@ + @@ -32,6 +33,7 @@ + @@ -48,8 +50,8 @@ {F27B8DB0-1279-4AF8-A2E3-1D49C4F0220D} Win32Proj vcpkgtest - 8.1 - v140 + 10.0 + v142 @@ -84,7 +86,7 @@ - + $(SolutionDir)msbuild.x86.debug\$(ProjectName)\ $(SolutionDir)msbuild.x86.debug\ @@ -193,4 +195,4 @@ - \ No newline at end of file + diff --git a/toolsrc/vcpkgtest/vcpkgtest.vcxproj.filters b/toolsrc/vcpkgtest/vcpkgtest.vcxproj.filters index 0f799426e..057c50a99 100644 --- a/toolsrc/vcpkgtest/vcpkgtest.vcxproj.filters +++ b/toolsrc/vcpkgtest/vcpkgtest.vcxproj.filters @@ -45,6 +45,12 @@ Source Files + + Source Files + + + Source Files + -- cgit v1.2.3 From 771e23c665f1960ef4e7fd9816e0d21c943a7903 Mon Sep 17 00:00:00 2001 From: Nicole Mazzuca Date: Thu, 11 Jul 2019 18:26:42 -0700 Subject: forgot to test on macos >.< --- toolsrc/src/vcpkg/base/files.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'toolsrc') diff --git a/toolsrc/src/vcpkg/base/files.cpp b/toolsrc/src/vcpkg/base/files.cpp index 4e36fe990..e40822705 100644 --- a/toolsrc/src/vcpkg/base/files.cpp +++ b/toolsrc/src/vcpkg/base/files.cpp @@ -23,9 +23,11 @@ namespace fs::detail { - file_status symlink_status_t::operator()(const path& p, std::error_code&) const noexcept + file_status symlink_status_t::operator()(const path& p, std::error_code& ec) const noexcept { #if defined(_WIN32) + static_cast(ec); + /* do not find the permissions of the file -- it's unnecessary for the things that vcpkg does. -- cgit v1.2.3 From 02c977186e890e4090d91c171b42d20729e668af Mon Sep 17 00:00:00 2001 From: Nicole Mazzuca Date: Mon, 15 Jul 2019 16:43:55 -0700 Subject: modify files test to include symlinks --- toolsrc/src/tests.files.cpp | 121 +++++++++++++++++++++++++++++++------------- 1 file changed, 87 insertions(+), 34 deletions(-) (limited to 'toolsrc') diff --git a/toolsrc/src/tests.files.cpp b/toolsrc/src/tests.files.cpp index 73c7eb5bc..c99dbb286 100644 --- a/toolsrc/src/tests.files.cpp +++ b/toolsrc/src/tests.files.cpp @@ -10,65 +10,60 @@ using namespace Microsoft::VisualStudio::CppUnitTestFramework; -namespace UnitTest1 { - class FilesTest : public TestClass { +namespace UnitTest1 +{ + class FilesTest : public TestClass + { using uid = std::uniform_int_distribution; - - std::string get_random_filename() - { - std::random_device rd; - return vcpkg::Strings::b64url_encode(uid{}(rd)); - } - - void create_directory_tree( - vcpkg::Files::Filesystem& fs, - std::uint64_t depth, - const fs::path& base) - { - std::random_device rd; - constexpr auto max_depth = std::uint64_t(3); - const auto width = depth ? uid{0, (max_depth - depth) * 3 / 2}(rd) : 5; - std::error_code ec; - if (width == 0) { - fs.write_contents(base, "", ec); - Assert::IsFalse(bool(ec)); + public: + FilesTest() + { + HKEY key; + const auto status = RegOpenKeyExW( + HKEY_LOCAL_MACHINE, LR"(SOFTWARE\Microsoft\Windows\CurrentVersion\AppModelUnlock)", 0, KEY_READ, &key); - return; + if (!status) + { + ALLOW_SYMLINKS = false; + std::clog << "Symlinks are not allowed on this system\n"; } - - fs.create_directory(base, ec); - Assert::IsFalse(bool(ec)); - - for (int i = 0; i < width; ++i) { - create_directory_tree(fs, depth + 1, base / get_random_filename()); + else + { + ALLOW_SYMLINKS = true; + RegCloseKey(key); } } - - TEST_METHOD(remove_all) { + + private: + TEST_METHOD(remove_all) + { + auto urbg = get_urbg(0); + fs::path temp_dir; { wchar_t* tmp = static_cast(calloc(32'767, 2)); - if (!GetEnvironmentVariableW(L"TEMP", tmp, 32'767)) { + if (!GetEnvironmentVariableW(L"TEMP", tmp, 32'767)) + { Assert::Fail(L"GetEnvironmentVariable(\"TEMP\") failed"); } temp_dir = tmp; std::string dir_name = "vcpkg-tmp-dir-"; - dir_name += get_random_filename(); + dir_name += get_random_filename(urbg); temp_dir /= dir_name; } auto& fs = vcpkg::Files::get_real_filesystem(); - std::cout << "temp dir is: " << temp_dir << '\n'; + std::clog << "temp dir is: " << temp_dir << '\n'; std::error_code ec; - create_directory_tree(fs, 0, temp_dir); + create_directory_tree(urbg, fs, 0, temp_dir); fs::path fp; fs.remove_all(temp_dir, ec, fp); @@ -76,5 +71,63 @@ namespace UnitTest1 { Assert::IsFalse(fs.exists(temp_dir)); } + + bool ALLOW_SYMLINKS; + + std::mt19937_64 get_urbg(std::uint64_t index) + { + // smallest prime > 2**63 - 1 + return std::mt19937_64{index + 9223372036854775837}; + } + + std::string get_random_filename(std::mt19937_64& urbg) { return vcpkg::Strings::b64url_encode(uid{}(urbg)); } + + void create_directory_tree(std::mt19937_64& urbg, + vcpkg::Files::Filesystem& fs, + std::uint64_t depth, + const fs::path& base) + { + std::random_device rd; + constexpr auto max_depth = std::uint64_t(3); + const auto width = depth ? uid{0, (max_depth - depth) * 3 / 2}(urbg) : 5; + + std::error_code ec; + if (width == 0) + { + // I don't want to move urbg forward conditionally + const auto type = uid{0, 3}(urbg); + if (type == 0 || !ALLOW_SYMLINKS) + { + // 0 is a regular file + fs.write_contents(base, "", ec); + } + else if (type == 1) + { + // 1 is a regular symlink + fs.write_contents(base, "", ec); + Assert::IsFalse(bool(ec)); + fs::path base_link = base; + base_link.append("-link"); + fs::stdfs::create_symlink(base, base_link, ec); + } + else + { + // 2 is a directory symlink + fs::stdfs::create_directory_symlink(".", base, ec); + } + + Assert::IsFalse(bool(ec)); + + return; + } + + fs.create_directory(base, ec); + Assert::IsFalse(bool(ec)); + + for (int i = 0; i < width; ++i) + { + create_directory_tree(urbg, fs, depth + 1, base / get_random_filename(urbg)); + } + } }; } -- cgit v1.2.3 From 65d34c5e55ef30a6572dfb3b79d212196fbadc0d Mon Sep 17 00:00:00 2001 From: Nicole Mazzuca Date: Mon, 15 Jul 2019 18:51:03 -0700 Subject: wheeeee more fixes --- toolsrc/include/vcpkg/base/strings.h | 8 +-- toolsrc/include/vcpkg/base/work_queue.h | 89 ++++++++++----------------------- toolsrc/src/tests.files.cpp | 64 +++++++++++++++--------- toolsrc/src/tests.strings.cpp | 20 ++++---- toolsrc/src/vcpkg/base/files.cpp | 2 +- toolsrc/src/vcpkg/base/strings.cpp | 23 ++++----- 6 files changed, 90 insertions(+), 116 deletions(-) (limited to 'toolsrc') diff --git a/toolsrc/include/vcpkg/base/strings.h b/toolsrc/include/vcpkg/base/strings.h index a1906790f..aa4c4d690 100644 --- a/toolsrc/include/vcpkg/base/strings.h +++ b/toolsrc/include/vcpkg/base/strings.h @@ -185,8 +185,8 @@ namespace vcpkg::Strings bool contains(StringView haystack, StringView needle); - // base 64 encoding with URL and filesafe alphabet (base64url) - // based on IETF RFC 4648 - // ignores padding, since one implicitly knows the length from the size of x - std::string b64url_encode(std::uint64_t x) noexcept; + // base 32 encoding, since base64 encoding requires lowercase letters, + // which are not distinct from uppercase letters on macOS or Windows filesystems. + // follows RFC 4648 + std::string b32_encode(std::uint64_t x) noexcept; } diff --git a/toolsrc/include/vcpkg/base/work_queue.h b/toolsrc/include/vcpkg/base/work_queue.h index d6666770b..69ca387f3 100644 --- a/toolsrc/include/vcpkg/base/work_queue.h +++ b/toolsrc/include/vcpkg/base/work_queue.h @@ -36,7 +36,7 @@ namespace vcpkg { m_line_info = li; - m_unjoined_workers = num_threads; + set_unjoined_workers(num_threads); m_threads.reserve(num_threads); for (std::size_t i = 0; i < num_threads; ++i) { @@ -93,20 +93,16 @@ namespace vcpkg } } - for (;;) + while (unjoined_workers()) { - auto lck = std::unique_lock(m_mutex); - if (!m_unjoined_workers) - break; - - else if (!m_running_workers) + if (!running_workers()) { - lck.unlock(); - m_cv.notify_all(); - } + m_cv.notify_one(); + + } } - // all threads have returned -- now, it's time to join them + // wait for all threads to join for (auto& thrd : m_threads) { thrd.join(); @@ -143,50 +139,6 @@ namespace vcpkg m_cv.notify_one(); } - template - void enqueue_all_actions_by_move(Rng&& rng) const - { - { - using std::begin; - using std::end; - - auto lck = std::unique_lock(m_mutex); - - const auto first = begin(rng); - const auto last = end(rng); - - m_actions.reserve(m_actions.size() + (last - first)); - - std::move(first, last, std::back_inserter(rng)); - - if (m_state == State::BeforeRun) return; - } - - m_cv.notify_all(); - } - - template - void enqueue_all_actions(Rng&& rng) const - { - { - using std::begin; - using std::end; - - auto lck = std::unique_lock(m_mutex); - - const auto first = begin(rng); - const auto last = end(rng); - - m_actions.reserve(m_actions.size() + (last - first)); - - std::copy(first, last, std::back_inserter(rng)); - - if (m_state == State::BeforeRun) return; - } - - m_cv.notify_all(); - } - private: struct Worker { @@ -201,6 +153,7 @@ namespace vcpkg work_queue->m_cv.wait(lck, [&] { return work_queue->m_state != State::BeforeRun; }); + work_queue->increment_running_workers(); for (;;) { const auto state = work_queue->m_state; @@ -212,15 +165,15 @@ namespace vcpkg if (work_queue->m_actions.empty()) { - if (state == State::Running || work_queue->m_running_workers > 1) + if (state == State::Running || work_queue->running_workers() > 1) { - --work_queue->m_running_workers; + work_queue->decrement_running_workers(); work_queue->m_cv.wait(lck); - ++work_queue->m_running_workers; + work_queue->increment_running_workers(); continue; } - // the queue isn't running, and we are the only worker + // the queue is joining, and we are the only worker running // no more work! break; } @@ -229,11 +182,13 @@ namespace vcpkg work_queue->m_actions.pop_back(); lck.unlock(); + work_queue->m_cv.notify_one(); detail::call_moved_action(action, *work_queue, tld); lck.lock(); } - --work_queue->m_unjoined_workers; + work_queue->decrement_running_workers(); + work_queue->decrement_unjoined_workers(); } }; @@ -255,11 +210,21 @@ namespace vcpkg mutable std::mutex m_mutex{}; // these are all under m_mutex mutable State m_state = State::BeforeRun; - mutable std::uint16_t m_running_workers = 0; - mutable std::uint16_t m_unjoined_workers = 0; // num_threads mutable std::vector m_actions{}; mutable std::condition_variable m_cv{}; + mutable std::atomic m_workers; + // = unjoined_workers << 16 | running_workers + + void set_unjoined_workers(std::uint16_t threads) { m_workers = std::uint32_t(threads) << 16; } + void decrement_unjoined_workers() const { m_workers -= 1 << 16; } + + std::uint16_t unjoined_workers() const { return std::uint16_t(m_workers >> 16); } + + void increment_running_workers() const { ++m_workers; } + void decrement_running_workers() const { --m_workers; } + std::uint16_t running_workers() const { return std::uint16_t(m_workers); } + std::vector m_threads{}; LineInfo m_line_info; }; diff --git a/toolsrc/src/tests.files.cpp b/toolsrc/src/tests.files.cpp index c99dbb286..56b0ceac6 100644 --- a/toolsrc/src/tests.files.cpp +++ b/toolsrc/src/tests.files.cpp @@ -4,6 +4,7 @@ #include #include +#include // required for filesystem::create_{directory_}symlink #include #include @@ -21,18 +22,20 @@ namespace UnitTest1 { HKEY key; const auto status = RegOpenKeyExW( - HKEY_LOCAL_MACHINE, LR"(SOFTWARE\Microsoft\Windows\CurrentVersion\AppModelUnlock)", 0, KEY_READ, &key); + HKEY_LOCAL_MACHINE, LR"(SOFTWARE\Microsoft\Windows\CurrentVersion\AppModelUnlock)", 0, 0, &key); - if (!status) + if (status == ERROR_FILE_NOT_FOUND) { ALLOW_SYMLINKS = false; std::clog << "Symlinks are not allowed on this system\n"; } else { + // if we get a permissions error, we still know that we're in developer mode ALLOW_SYMLINKS = true; - RegCloseKey(key); } + + if (status == ERROR_SUCCESS) RegCloseKey(key); } private: @@ -62,9 +65,9 @@ namespace UnitTest1 std::clog << "temp dir is: " << temp_dir << '\n'; - std::error_code ec; create_directory_tree(urbg, fs, 0, temp_dir); + std::error_code ec; fs::path fp; fs.remove_all(temp_dir, ec, fp); Assert::IsFalse(bool(ec)); @@ -80,7 +83,7 @@ namespace UnitTest1 return std::mt19937_64{index + 9223372036854775837}; } - std::string get_random_filename(std::mt19937_64& urbg) { return vcpkg::Strings::b64url_encode(uid{}(urbg)); } + std::string get_random_filename(std::mt19937_64& urbg) { return vcpkg::Strings::b32_encode(uid{}(urbg)); } void create_directory_tree(std::mt19937_64& urbg, vcpkg::Files::Filesystem& fs, @@ -88,46 +91,57 @@ namespace UnitTest1 const fs::path& base) { std::random_device rd; - constexpr auto max_depth = std::uint64_t(3); - const auto width = depth ? uid{0, (max_depth - depth) * 3 / 2}(urbg) : 5; + constexpr std::uint64_t max_depth = 5; + constexpr std::uint64_t width = 5; + const auto type = depth < max_depth ? uid{0, 9}(urbg) : uid{7, 9}(urbg); + + // 0 <= type < 7 : directory + // 7 = type : regular + // 8 = type : regular symlink (regular file if !ALLOW_SYMLINKS) + // 9 = type : directory symlink (^^) std::error_code ec; - if (width == 0) + if (type >= 7) { // I don't want to move urbg forward conditionally - const auto type = uid{0, 3}(urbg); - if (type == 0 || !ALLOW_SYMLINKS) + if (type == 7 || !ALLOW_SYMLINKS) { - // 0 is a regular file + // regular file fs.write_contents(base, "", ec); } - else if (type == 1) + else if (type == 8) { - // 1 is a regular symlink + // regular symlink fs.write_contents(base, "", ec); Assert::IsFalse(bool(ec)); - fs::path base_link = base; - base_link.append("-link"); - fs::stdfs::create_symlink(base, base_link, ec); + const std::filesystem::path basep = base.native(); + auto basep_link = basep; + basep_link.replace_filename(basep.filename().native() + L"-link"); + std::filesystem::create_symlink(basep, basep_link, ec); } else { - // 2 is a directory symlink - fs::stdfs::create_directory_symlink(".", base, ec); + // directory symlink + std::filesystem::path basep = base.native(); + std::filesystem::create_directory_symlink(basep / "..", basep, ec); } Assert::IsFalse(bool(ec)); - return; } - - fs.create_directory(base, ec); - Assert::IsFalse(bool(ec)); - - for (int i = 0; i < width; ++i) + else { - create_directory_tree(urbg, fs, depth + 1, base / get_random_filename(urbg)); + // directory + fs.create_directory(base, ec); + Assert::IsFalse(bool(ec)); + + for (int i = 0; i < width; ++i) + { + create_directory_tree(urbg, fs, depth + 1, base / get_random_filename(urbg)); + } + } + } }; } diff --git a/toolsrc/src/tests.strings.cpp b/toolsrc/src/tests.strings.cpp index 14b449e86..6301ea05d 100644 --- a/toolsrc/src/tests.strings.cpp +++ b/toolsrc/src/tests.strings.cpp @@ -16,21 +16,21 @@ namespace UnitTest1 { std::vector> map; - map.emplace_back(0, "AAAAAAAAAAA"); - map.emplace_back(1, "BAAAAAAAAAA"); + map.emplace_back(0, "AAAAAAAAAAAAA"); + map.emplace_back(1, "BAAAAAAAAAAAA"); - map.emplace_back(u64(1) << 32, "AAAAAEAAAAA"); - map.emplace_back((u64(1) << 32) + 1, "BAAAAEAAAAA"); + map.emplace_back(u64(1) << 32, "AAAAAAEAAAAAA"); + map.emplace_back((u64(1) << 32) + 1, "BAAAAAEAAAAAA"); - map.emplace_back(0xE4D0'1065'D11E'0229, "pIgHRXGEQTO"); - map.emplace_back(0xA626'FE45'B135'07FF, "_fQNxWk_mYK"); - map.emplace_back(0xEE36'D228'0C31'D405, "FQdMMgi024O"); - map.emplace_back(0x1405'64E7'FE7E'A88C, "Miqf-fOZFQB"); - map.emplace_back(0xFFFF'FFFF'FFFF'FFFF, "__________P"); + map.emplace_back(0xE4D0'1065'D11E'0229, "JRA4RIXMQAUJO"); + map.emplace_back(0xA626'FE45'B135'07FF, "77BKTYWI6XJMK"); + map.emplace_back(0xEE36'D228'0C31'D405, "FAVDDGAFSWN4O"); + map.emplace_back(0x1405'64E7'FE7E'A88C, "MEK5H774ELBIB"); + map.emplace_back(0xFFFF'FFFF'FFFF'FFFF, "777777777777P"); std::string result; for (const auto& pr : map) { - result = vcpkg::Strings::b64url_encode(pr.first); + result = vcpkg::Strings::b32_encode(pr.first); Assert::AreEqual(result, pr.second); } } diff --git a/toolsrc/src/vcpkg/base/files.cpp b/toolsrc/src/vcpkg/base/files.cpp index e40822705..6c6945e44 100644 --- a/toolsrc/src/vcpkg/base/files.cpp +++ b/toolsrc/src/vcpkg/base/files.cpp @@ -423,7 +423,7 @@ namespace vcpkg::Files { std::error_code ec; - const auto tmp_name = Strings::b64url_encode(info.index++); + const auto tmp_name = Strings::b32_encode(info.index++); const auto tmp_path = info.tmp_directory / tmp_name; fs::stdfs::rename(current_path, tmp_path, ec); diff --git a/toolsrc/src/vcpkg/base/strings.cpp b/toolsrc/src/vcpkg/base/strings.cpp index ade4384a9..7970e1b46 100644 --- a/toolsrc/src/vcpkg/base/strings.cpp +++ b/toolsrc/src/vcpkg/base/strings.cpp @@ -294,26 +294,21 @@ namespace vcpkg::Strings namespace { template - std::string b64url_encode_implementation(Integral x) + std::string b32_encode_implementation(Integral x) { static_assert(std::is_integral::value, "b64url_encode must take an integer type"); using Unsigned = std::make_unsigned_t; auto value = static_cast(x); - // 64 values, plus the implicit \0 - constexpr static char map[65] = - /* 0123456789ABCDEF */ - /*0*/ "ABCDEFGHIJKLMNOP" - /*1*/ "QRSTUVWXYZabcdef" - /*2*/ "ghijklmnopqrstuv" - /*3*/ "wxyz0123456789-_"; + // 32 values, plus the implicit \0 + constexpr static char map[33] = "ABCDEFGHIJKLMNOP" "QRSTUVWXYZ234567"; - // log2(64) - constexpr static int shift = 6; - // 64 - 1 - constexpr static auto mask = 63; + // log2(32) + constexpr static int shift = 5; + // 32 - 1 + constexpr static auto mask = 31; - // ceiling(bitsize(Integral) / log2(64)) + // ceiling(bitsize(Integral) / log2(32)) constexpr static auto result_size = (sizeof(value) * 8 + shift - 1) / shift; std::string result; @@ -329,6 +324,6 @@ namespace vcpkg::Strings } } - std::string b64url_encode(std::uint64_t x) noexcept { return b64url_encode_implementation(x); } + std::string b32_encode(std::uint64_t x) noexcept { return b32_encode_implementation(x); } } -- cgit v1.2.3 From 684989a1e48b6b7f0ac1c340a702236974762f05 Mon Sep 17 00:00:00 2001 From: "Curtis.Bezault" Date: Tue, 16 Jul 2019 14:02:13 -0700 Subject: use additional env param --- toolsrc/include/vcpkg/build.h | 24 ++ toolsrc/include/vcpkg/sourceparagraph.h | 2 + toolsrc/src/vcpkg/build.cpp | 398 +++++++++++++++++++------------- toolsrc/src/vcpkg/sourceparagraph.cpp | 22 ++ 4 files changed, 281 insertions(+), 165 deletions(-) (limited to 'toolsrc') diff --git a/toolsrc/include/vcpkg/build.h b/toolsrc/include/vcpkg/build.h index 04cd7cf87..e26597376 100644 --- a/toolsrc/include/vcpkg/build.h +++ b/toolsrc/include/vcpkg/build.h @@ -117,6 +117,29 @@ namespace vcpkg::Build std::string create_error_message(const BuildResult build_result, const PackageSpec& spec); std::string create_user_troubleshooting_message(const PackageSpec& spec); + enum class VcpkgTripletVar + { + TARGET_ARCHITECTURE = 0, + CMAKE_SYSTEM_NAME, + CMAKE_SYSTEM_VERSION, + PLATFORM_TOOLSET, + VISUAL_STUDIO_PATH, + CHAINLOAD_TOOLCHAIN_FILE, + BUILD_TYPE, + ENV_PASSTHROUGH, + }; + + const std::unordered_map VCPKG_OPTIONS = { + {"VCPKG_TARGET_ARCHITECTURE", VcpkgTripletVar::TARGET_ARCHITECTURE}, + {"VCPKG_CMAKE_SYSTEM_NAME", VcpkgTripletVar::CMAKE_SYSTEM_NAME}, + {"VCPKG_CMAKE_SYSTEM_VERSION", VcpkgTripletVar::CMAKE_SYSTEM_VERSION}, + {"VCPKG_PLATFORM_TOOLSET", VcpkgTripletVar::PLATFORM_TOOLSET}, + {"VCPKG_VISUAL_STUDIO_PATH", VcpkgTripletVar::VISUAL_STUDIO_PATH}, + {"VCPKG_CHAINLOAD_TOOLCHAIN_FILE", VcpkgTripletVar::CHAINLOAD_TOOLCHAIN_FILE}, + {"VCPKG_BUILD_TYPE", VcpkgTripletVar::BUILD_TYPE}, + {"VCPKG_ENV_PASSTHROUGH", VcpkgTripletVar::ENV_PASSTHROUGH}, + }; + /// /// Settings from the triplet file which impact the build environment and post-build checks /// @@ -135,6 +158,7 @@ namespace vcpkg::Build Optional visual_studio_path; Optional external_toolchain_file; Optional build_type; + std::vector passthrough_env_vars; }; std::string make_build_env_cmd(const PreBuildInfo& pre_build_info, const Toolset& toolset); diff --git a/toolsrc/include/vcpkg/sourceparagraph.h b/toolsrc/include/vcpkg/sourceparagraph.h index 6232a3fd2..9fbd83475 100644 --- a/toolsrc/include/vcpkg/sourceparagraph.h +++ b/toolsrc/include/vcpkg/sourceparagraph.h @@ -23,6 +23,8 @@ namespace vcpkg std::vector filter_dependencies(const std::vector& deps, const Triplet& t); std::vector filter_dependencies_to_specs(const std::vector& deps, const Triplet& t); + std::vector filter_dependencies_to_features(const std::vector& deps, + const Triplet& t); // zlib[uwp] becomes Dependency{"zlib", "uwp"} std::vector expand_qualified_dependencies(const std::vector& depends); diff --git a/toolsrc/src/vcpkg/build.cpp b/toolsrc/src/vcpkg/build.cpp index 68df1f965..e0c78f2da 100644 --- a/toolsrc/src/vcpkg/build.cpp +++ b/toolsrc/src/vcpkg/build.cpp @@ -32,24 +32,14 @@ namespace vcpkg::Build::Command using Dependencies::InstallPlanAction; using Dependencies::InstallPlanType; - static constexpr StringLiteral OPTION_CHECKS_ONLY = "--checks-only"; - void perform_and_exit_ex(const FullPackageSpec& full_spec, const SourceControlFileLocation& scfl, const ParsedArguments& options, const VcpkgPaths& paths) { + const StatusParagraphs status_db = database_load_check(paths); const PackageSpec& spec = full_spec.package_spec; - const auto& scf = *scfl.source_control_file; - if (Util::Sets::contains(options.switches, OPTION_CHECKS_ONLY)) - { - const auto pre_build_info = Build::PreBuildInfo::from_triplet_file(paths, spec.triplet()); - const auto build_info = Build::read_build_info(paths.get_filesystem(), paths.build_info_file_path(spec)); - const size_t error_count = - PostBuildLint::perform_all_checks(spec, paths, pre_build_info, build_info, scfl.source_location); - Checks::check_exit(VCPKG_LINE_INFO, error_count == 0); - Checks::exit_success(VCPKG_LINE_INFO); - } + const SourceControlFile& scf = *scfl.source_control_file; Checks::check_exit(VCPKG_LINE_INFO, spec.name() == scf.core_paragraph->name, @@ -57,7 +47,6 @@ namespace vcpkg::Build::Command scf.core_paragraph->name, spec.name()); - const StatusParagraphs status_db = database_load_check(paths); const Build::BuildPackageOptions build_package_options{ Build::UseHeadVersion::NO, Build::AllowDownloads::YES, @@ -104,15 +93,11 @@ namespace vcpkg::Build::Command Checks::exit_success(VCPKG_LINE_INFO); } - static constexpr std::array BUILD_SWITCHES = {{ - {OPTION_CHECKS_ONLY, "Only run checks, do not rebuild package"}, - }}; - const CommandStructure COMMAND_STRUCTURE = { Help::create_example_string("build zlib:x64-windows"), 1, 1, - {BUILD_SWITCHES, {}}, + {{}, {}}, nullptr, }; @@ -230,6 +215,23 @@ namespace vcpkg::Build })); } + std::unordered_map make_env_passthrough(const PreBuildInfo& pre_build_info) + { + std::unordered_map env; + + for (auto&& env_var : pre_build_info.passthrough_env_vars) + { + auto env_val = System::get_environment_variable(env_var); + + if (env_val) + { + env[env_var] = env_val.value_or_exit(VCPKG_LINE_INFO); + } + } + + return env; + } + std::string make_build_env_cmd(const PreBuildInfo& pre_build_info, const Toolset& toolset) { if (pre_build_info.external_toolchain_file.has_value()) return ""; @@ -274,7 +276,7 @@ namespace vcpkg::Build return bcf; } - static void write_binary_control_file(const VcpkgPaths& paths, BinaryControlFile bcf) + static void write_binary_control_file(const VcpkgPaths& paths, const BinaryControlFile& bcf) { std::string start = Strings::serialize(bcf.core_paragraph); for (auto&& feature : bcf.features) @@ -285,23 +287,63 @@ namespace vcpkg::Build paths.get_filesystem().write_contents(binary_control_file, start, VCPKG_LINE_INFO); } - static std::vector compute_required_feature_specs(const BuildPackageConfig& config, - const StatusParagraphs& status_db) + static std::vector get_dependencies(const SourceControlFile& scf, + const std::set& feature_list, + const Triplet& triplet) { - const Triplet& triplet = config.triplet; - - const std::vector dep_strings = - Util::fmap_flatten(config.feature_list, [&](std::string const& feature) -> std::vector { - if (feature == "core") + return Util::fmap_flatten(feature_list, + [&](std::string const& feature) -> std::vector { + if (feature == "core") { - return filter_dependencies(config.scf.core_paragraph->depends, triplet); + return filter_dependencies_to_features(scf.core_paragraph->depends, triplet); } - auto maybe_feature = config.scf.find_feature(feature); + auto maybe_feature = scf.find_feature(feature); Checks::check_exit(VCPKG_LINE_INFO, maybe_feature.has_value()); - return filter_dependencies(maybe_feature.get()->depends, triplet); - }); + return filter_dependencies_to_features(maybe_feature.get()->depends, triplet); + } + ); + } + + static std::vector get_dependency_names(const SourceControlFile& scf, + const std::set& feature_list, + const Triplet& triplet) + { + return Util::fmap(get_dependencies(scf, feature_list, triplet), + [&](const Features& feat) { + return feat.name; + } + ); + } + + static std::vector get_partials(const VcpkgPaths& paths, + const std::vector& dependencies, + const Triplet& triplet) + { + std::vector ret; + Files::Filesystem& fs = paths.get_filesystem(); + + for (const std::string& dependency : dependencies) + { + fs::path partial = + paths.buildtrees / dependency / triplet.canonical_name() / "partial_triplet.cmake"; + if (fs.is_regular_file(partial)) + { + ret.emplace_back(std::move(partial)); + } + } + + return ret; + } + + static std::vector compute_required_feature_specs(const BuildPackageConfig& config, + const StatusParagraphs& status_db) + { + const Triplet& triplet = config.triplet; + + const std::vector dep_strings = + get_dependency_names(config.scf, config.feature_list, triplet); auto dep_fspecs = FeatureSpec::from_strings_and_triplet(dep_strings, triplet); Util::sort_unique_erase(dep_fspecs); @@ -353,40 +395,19 @@ namespace vcpkg::Build return concurrency; } - static ExtendedBuildResult do_build_package(const VcpkgPaths& paths, - const PreBuildInfo& pre_build_info, - const PackageSpec& spec, - const std::string& abi_tag, - const BuildPackageConfig& config) + static std::vector get_cmake_vars(const VcpkgPaths& paths, + const PreBuildInfo& pre_build_info, + const BuildPackageConfig& config, + const Triplet& triplet, + const Toolset& toolset) { - auto& fs = paths.get_filesystem(); - const Triplet& triplet = spec.triplet(); - const auto& triplet_file_path = paths.get_triplet_file_path(spec.triplet()).u8string(); - - if (!Strings::case_insensitive_ascii_starts_with(triplet_file_path, paths.triplets.u8string())) - { - System::printf("-- Loading triplet configuration from: %s\n", triplet_file_path); - } - if (!Strings::case_insensitive_ascii_starts_with(config.port_dir.u8string(), paths.ports.u8string())) - { - System::printf("-- Installing port from location: %s\n", config.port_dir.u8string()); - } - #if !defined(_WIN32) // TODO: remove when vcpkg.exe is in charge for acquiring tools. Change introduced in vcpkg v0.0.107. // bootstrap should have already downloaded ninja, but making sure it is present in case it was deleted. vcpkg::Util::unused(paths.get_tool_exe(Tools::NINJA)); #endif - const fs::path& cmake_exe_path = paths.get_tool_exe(Tools::CMAKE); const fs::path& git_exe_path = paths.get_tool_exe(Tools::GIT); -#if defined(_WIN32) - const fs::path& powershell_exe_path = paths.get_tool_exe("powershell-core"); - if (!fs.exists(powershell_exe_path.parent_path() / "powershell.exe")) - { - fs.copy(powershell_exe_path, powershell_exe_path.parent_path() / "powershell.exe", fs::copy_options::none); - } -#endif std::string all_features; for (auto& feature : config.scf.feature_paragraphs) @@ -394,14 +415,21 @@ namespace vcpkg::Build all_features.append(feature->name + ";"); } - const Toolset& toolset = paths.get_toolset(pre_build_info); + std::vector partials = + Util::fmap( + get_partials(paths, get_dependency_names(config.scf, config.feature_list, triplet), triplet), + [&](const fs::path& path) + { + return path.u8string(); + } + ); std::vector variables{ {"CMD", "BUILD"}, {"PORT", config.scf.core_paragraph->name}, {"CURRENT_PORT_DIR", config.port_dir}, - {"TARGET_TRIPLET", spec.triplet().canonical_name()}, - {"TARGET_TRIPLET_FILE", triplet_file_path}, + {"TARGET_TRIPLET", triplet.canonical_name()}, + {"TARGET_TRIPLET_FILE", paths.get_triplet_file_path(triplet).u8string()}, {"VCPKG_PLATFORM_TOOLSET", toolset.version.c_str()}, {"VCPKG_USE_HEAD_VERSION", Util::Enum::to_bool(config.build_package_options.use_head_version) ? "1" : "0"}, {"DOWNLOADS", paths.downloads}, @@ -410,6 +438,7 @@ namespace vcpkg::Build {"FEATURES", Strings::join(";", config.feature_list)}, {"ALL_FEATURES", all_features}, {"VCPKG_CONCURRENCY", std::to_string(get_concurrency())}, + {"VCPKG_PARTIAL_TRIPLETS", Strings::join(";", partials)}, }; if (!System::get_environment_variable("VCPKG_FORCE_SYSTEM_BINARIES").has_value()) @@ -417,9 +446,22 @@ namespace vcpkg::Build variables.push_back({"GIT", git_exe_path}); } + return variables; + } + + static std::string make_build_cmd(const VcpkgPaths& paths, + const PreBuildInfo& pre_build_info, + const BuildPackageConfig& config, + const Triplet& triplet) + { + const Toolset& toolset = paths.get_toolset(pre_build_info); + const fs::path& cmake_exe_path = paths.get_tool_exe(Tools::CMAKE); + std::vector variables = + get_cmake_vars(paths, pre_build_info, config, triplet, toolset); + const std::string cmd_launch_cmake = System::make_cmake_cmd(cmake_exe_path, paths.ports_cmake, variables); - auto command = make_build_env_cmd(pre_build_info, toolset); + std::string command = make_build_env_cmd(pre_build_info, toolset); if (!command.empty()) { #ifdef _WIN32 @@ -428,16 +470,93 @@ namespace vcpkg::Build command.append(" && "); #endif } + command.append(cmd_launch_cmake); + + return command; + } + + static std::string get_triplet_abi(const VcpkgPaths& paths, + const PreBuildInfo& pre_build_info, + const Triplet& triplet) + { + static std::map s_hash_cache; + + const fs::path triplet_file_path = paths.get_triplet_file_path(triplet); + const auto& fs = paths.get_filesystem(); + + std::string hash; + + auto it_hash = s_hash_cache.find(triplet_file_path); + if (it_hash != s_hash_cache.end()) + { + hash = it_hash->second; + } + else + { + hash = Hash::get_file_hash(fs, triplet_file_path, "SHA1"); + + if (auto p = pre_build_info.external_toolchain_file.get()) + { + hash += "-"; + hash += Hash::get_file_hash(fs, *p, "SHA1"); + } + else if (pre_build_info.cmake_system_name == "Linux") + { + hash += "-"; + hash += Hash::get_file_hash(fs, paths.scripts / "toolchains" / "linux.cmake", "SHA1"); + } + else if (pre_build_info.cmake_system_name == "Darwin") + { + hash += "-"; + hash += Hash::get_file_hash(fs, paths.scripts / "toolchains" / "osx.cmake", "SHA1"); + } + else if (pre_build_info.cmake_system_name == "FreeBSD") + { + hash += "-"; + hash += Hash::get_file_hash(fs, paths.scripts / "toolchains" / "freebsd.cmake", "SHA1"); + } + else if (pre_build_info.cmake_system_name == "Android") + { + hash += "-"; + hash += Hash::get_file_hash(fs, paths.scripts / "toolchains" / "android.cmake", "SHA1"); + } + + s_hash_cache.emplace(triplet_file_path, hash); + } + + return hash; + } + + static ExtendedBuildResult do_build_package(const VcpkgPaths& paths, + const PreBuildInfo& pre_build_info, + const PackageSpec& spec, + const std::string& abi_tag, + const BuildPackageConfig& config) + { + auto& fs = paths.get_filesystem(); + const Triplet& triplet = spec.triplet(); + const auto& triplet_file_path = paths.get_triplet_file_path(spec.triplet()).u8string(); + + if (!Strings::case_insensitive_ascii_starts_with(triplet_file_path, paths.triplets.u8string())) + { + System::printf("-- Loading triplet configuration from: %s\n", triplet_file_path); + } + if (!Strings::case_insensitive_ascii_starts_with(config.port_dir.u8string(), paths.ports.u8string())) + { + System::printf("-- Installing port from location: %s\n", config.port_dir.u8string()); + } + const auto timer = Chrono::ElapsedTimer::create_started(); - const int return_code = System::cmd_execute_clean( - command, - {} -#ifdef _WIN32 - , - powershell_exe_path.parent_path().u8string() + ";" -#endif - ); + + std::string command = + make_build_cmd(paths, pre_build_info, config, triplet); + std::unordered_map env = + make_env_passthrough(pre_build_info); + + const int return_code = + System::cmd_execute_clean(command, env); + const auto buildtimeus = timer.microseconds(); const auto spec_string = spec.to_string(); @@ -711,7 +830,7 @@ namespace vcpkg::Build { System::print2("Using cached binary package: ", archive_path.u8string(), "\n"); - auto archive_result = decompress_archive(paths, spec, archive_path); + int archive_result = decompress_archive(paths, spec, archive_path); if (archive_result != 0) { @@ -946,111 +1065,60 @@ namespace vcpkg::Build const std::string variable_name = s.at(0); const std::string variable_value = variable_with_no_value ? "" : s.at(1); - if (variable_name == "VCPKG_TARGET_ARCHITECTURE") - { - pre_build_info.target_architecture = variable_value; - continue; - } - - if (variable_name == "VCPKG_CMAKE_SYSTEM_NAME") - { - pre_build_info.cmake_system_name = variable_value; - continue; - } - - if (variable_name == "VCPKG_CMAKE_SYSTEM_VERSION") - { - pre_build_info.cmake_system_version = variable_value; - continue; - } - - if (variable_name == "VCPKG_PLATFORM_TOOLSET") - { - pre_build_info.platform_toolset = - variable_value.empty() ? nullopt : Optional{variable_value}; - continue; - } - - if (variable_name == "VCPKG_VISUAL_STUDIO_PATH") + auto maybe_option = VCPKG_OPTIONS.find(variable_name); + if (maybe_option != VCPKG_OPTIONS.end()) { - pre_build_info.visual_studio_path = - variable_value.empty() ? nullopt : Optional{variable_value}; - continue; - } - - if (variable_name == "VCPKG_CHAINLOAD_TOOLCHAIN_FILE") - { - pre_build_info.external_toolchain_file = - variable_value.empty() ? nullopt : Optional{variable_value}; - continue; + switch (maybe_option->second) + { + case VcpkgTripletVar::TARGET_ARCHITECTURE : + pre_build_info.target_architecture = variable_value; + break; + case VcpkgTripletVar::CMAKE_SYSTEM_NAME : + pre_build_info.cmake_system_name = variable_value; + break; + case VcpkgTripletVar::CMAKE_SYSTEM_VERSION : + pre_build_info.cmake_system_version = variable_value; + break; + case VcpkgTripletVar::PLATFORM_TOOLSET : + pre_build_info.platform_toolset = + variable_value.empty() ? nullopt : Optional{variable_value}; + break; + case VcpkgTripletVar::VISUAL_STUDIO_PATH : + pre_build_info.visual_studio_path = + variable_value.empty() ? nullopt : Optional{variable_value}; + break; + case VcpkgTripletVar::CHAINLOAD_TOOLCHAIN_FILE : + pre_build_info.external_toolchain_file = + variable_value.empty() ? nullopt : Optional{variable_value}; + break; + case VcpkgTripletVar::BUILD_TYPE : + if (variable_value.empty()) + pre_build_info.build_type = nullopt; + else if (Strings::case_insensitive_ascii_equals(variable_value, "debug")) + pre_build_info.build_type = ConfigurationType::DEBUG; + else if (Strings::case_insensitive_ascii_equals(variable_value, "release")) + pre_build_info.build_type = ConfigurationType::RELEASE; + else + Checks::exit_with_message( + VCPKG_LINE_INFO, "Unknown setting for VCPKG_BUILD_TYPE: %s", variable_value); + break; + case VcpkgTripletVar::ENV_PASSTHROUGH : + pre_build_info.passthrough_env_vars = Strings::split(variable_value, ";"); + break; + } } - - if (variable_name == "VCPKG_BUILD_TYPE") + else { - if (variable_value.empty()) - pre_build_info.build_type = nullopt; - else if (Strings::case_insensitive_ascii_equals(variable_value, "debug")) - pre_build_info.build_type = ConfigurationType::DEBUG; - else if (Strings::case_insensitive_ascii_equals(variable_value, "release")) - pre_build_info.build_type = ConfigurationType::RELEASE; - else - Checks::exit_with_message( - VCPKG_LINE_INFO, "Unknown setting for VCPKG_BUILD_TYPE: %s", variable_value); - continue; + Checks::exit_with_message(VCPKG_LINE_INFO, "Unknown variable name %s", line); } - - Checks::exit_with_message(VCPKG_LINE_INFO, "Unknown variable name %s", line); } - pre_build_info.triplet_abi_tag = [&]() { - const auto& fs = paths.get_filesystem(); - static std::map s_hash_cache; - - auto it_hash = s_hash_cache.find(triplet_file_path); - if (it_hash != s_hash_cache.end()) - { - return it_hash->second; - } - auto hash = Hash::get_file_hash(fs, triplet_file_path, "SHA1"); - - if (auto p = pre_build_info.external_toolchain_file.get()) - { - hash += "-"; - hash += Hash::get_file_hash(fs, *p, "SHA1"); - } - else if (pre_build_info.cmake_system_name.empty() || - pre_build_info.cmake_system_name == "WindowsStore") - { - hash += "-"; - hash += Hash::get_file_hash(fs, paths.scripts / "toolchains" / "windows.cmake", "SHA1"); - } - else if (pre_build_info.cmake_system_name == "Linux") - { - hash += "-"; - hash += Hash::get_file_hash(fs, paths.scripts / "toolchains" / "linux.cmake", "SHA1"); - } - else if (pre_build_info.cmake_system_name == "Darwin") - { - hash += "-"; - hash += Hash::get_file_hash(fs, paths.scripts / "toolchains" / "osx.cmake", "SHA1"); - } - else if (pre_build_info.cmake_system_name == "FreeBSD") - { - hash += "-"; - hash += Hash::get_file_hash(fs, paths.scripts / "toolchains" / "freebsd.cmake", "SHA1"); - } - else if (pre_build_info.cmake_system_name == "Android") - { - hash += "-"; - hash += Hash::get_file_hash(fs, paths.scripts / "toolchains" / "android.cmake", "SHA1"); - } - - s_hash_cache.emplace(triplet_file_path, hash); - return hash; - }(); + pre_build_info.triplet_abi_tag = + get_triplet_abi(paths, pre_build_info, triplet); return pre_build_info; } + ExtendedBuildResult::ExtendedBuildResult(BuildResult code) : code(code) {} ExtendedBuildResult::ExtendedBuildResult(BuildResult code, std::unique_ptr&& bcf) : code(code), binary_control_file(std::move(bcf)) diff --git a/toolsrc/src/vcpkg/sourceparagraph.cpp b/toolsrc/src/vcpkg/sourceparagraph.cpp index 9bc59cbe7..1a52bd05f 100644 --- a/toolsrc/src/vcpkg/sourceparagraph.cpp +++ b/toolsrc/src/vcpkg/sourceparagraph.cpp @@ -238,6 +238,28 @@ namespace vcpkg return ret; } + std::vector filter_dependencies_to_features(const std::vector& deps, + const Triplet& t) + { + std::vector ret; + for (auto&& dep : deps) + { + auto qualifiers = Strings::split(dep.qualifier, "&"); + if (std::all_of(qualifiers.begin(), qualifiers.end(), [&](const std::string& qualifier) { + if (qualifier.empty()) return true; + if (qualifier[0] == '!') + { + return t.canonical_name().find(qualifier.substr(1)) == std::string::npos; + } + return t.canonical_name().find(qualifier) != std::string::npos; + })) + { + ret.emplace_back(dep.depend); + } + } + return ret; + } + std::vector filter_dependencies_to_specs(const std::vector& deps, const Triplet& t) { return FeatureSpec::from_strings_and_triplet(filter_dependencies(deps, t), t); -- cgit v1.2.3 From 6bef95b6f50b2bb602781d6d8898d27639294ff6 Mon Sep 17 00:00:00 2001 From: "Curtis.Bezault" Date: Tue, 16 Jul 2019 14:43:56 -0700 Subject: remove partials --- toolsrc/src/vcpkg/build.cpp | 30 ------------------------------ 1 file changed, 30 deletions(-) (limited to 'toolsrc') diff --git a/toolsrc/src/vcpkg/build.cpp b/toolsrc/src/vcpkg/build.cpp index e0c78f2da..dd2beec9d 100644 --- a/toolsrc/src/vcpkg/build.cpp +++ b/toolsrc/src/vcpkg/build.cpp @@ -317,26 +317,6 @@ namespace vcpkg::Build ); } - static std::vector get_partials(const VcpkgPaths& paths, - const std::vector& dependencies, - const Triplet& triplet) - { - std::vector ret; - Files::Filesystem& fs = paths.get_filesystem(); - - for (const std::string& dependency : dependencies) - { - fs::path partial = - paths.buildtrees / dependency / triplet.canonical_name() / "partial_triplet.cmake"; - if (fs.is_regular_file(partial)) - { - ret.emplace_back(std::move(partial)); - } - } - - return ret; - } - static std::vector compute_required_feature_specs(const BuildPackageConfig& config, const StatusParagraphs& status_db) { @@ -415,15 +395,6 @@ namespace vcpkg::Build all_features.append(feature->name + ";"); } - std::vector partials = - Util::fmap( - get_partials(paths, get_dependency_names(config.scf, config.feature_list, triplet), triplet), - [&](const fs::path& path) - { - return path.u8string(); - } - ); - std::vector variables{ {"CMD", "BUILD"}, {"PORT", config.scf.core_paragraph->name}, @@ -438,7 +409,6 @@ namespace vcpkg::Build {"FEATURES", Strings::join(";", config.feature_list)}, {"ALL_FEATURES", all_features}, {"VCPKG_CONCURRENCY", std::to_string(get_concurrency())}, - {"VCPKG_PARTIAL_TRIPLETS", Strings::join(";", partials)}, }; if (!System::get_environment_variable("VCPKG_FORCE_SYSTEM_BINARIES").has_value()) -- cgit v1.2.3 From 44dcc3d4f3d2bc56cc9f6d0371a141ad0145d537 Mon Sep 17 00:00:00 2001 From: "Curtis.Bezault" Date: Tue, 16 Jul 2019 15:34:13 -0700 Subject: First pass at port settings --- toolsrc/include/vcpkg/build.h | 48 ++++++++++++++++++++------------------- toolsrc/src/vcpkg/build.cpp | 21 +++++++++++++---- toolsrc/src/vcpkg/commands.ci.cpp | 9 +++++++- 3 files changed, 49 insertions(+), 29 deletions(-) (limited to 'toolsrc') diff --git a/toolsrc/include/vcpkg/build.h b/toolsrc/include/vcpkg/build.h index e26597376..0d10d12a4 100644 --- a/toolsrc/include/vcpkg/build.h +++ b/toolsrc/include/vcpkg/build.h @@ -140,29 +140,6 @@ namespace vcpkg::Build {"VCPKG_ENV_PASSTHROUGH", VcpkgTripletVar::ENV_PASSTHROUGH}, }; - /// - /// Settings from the triplet file which impact the build environment and post-build checks - /// - struct PreBuildInfo - { - /// - /// Runs the triplet file in a "capture" mode to create a PreBuildInfo - /// - static PreBuildInfo from_triplet_file(const VcpkgPaths& paths, const Triplet& triplet); - - std::string triplet_abi_tag; - std::string target_architecture; - std::string cmake_system_name; - std::string cmake_system_version; - Optional platform_toolset; - Optional visual_studio_path; - Optional external_toolchain_file; - Optional build_type; - std::vector passthrough_env_vars; - }; - - std::string make_build_env_cmd(const PreBuildInfo& pre_build_info, const Toolset& toolset); - struct ExtendedBuildResult { ExtendedBuildResult(BuildResult code); @@ -200,6 +177,31 @@ namespace vcpkg::Build const BuildPackageConfig& config, const StatusParagraphs& status_db); + /// + /// Settings from the triplet file which impact the build environment and post-build checks + /// + struct PreBuildInfo + { + /// + /// Runs the triplet file in a "capture" mode to create a PreBuildInfo + /// + static PreBuildInfo from_triplet_file(const VcpkgPaths& paths, + const Triplet& triplet, + Optional port = nullopt); + + std::string triplet_abi_tag; + std::string target_architecture; + std::string cmake_system_name; + std::string cmake_system_version; + Optional platform_toolset; + Optional visual_studio_path; + Optional external_toolchain_file; + Optional build_type; + std::vector passthrough_env_vars; + }; + + std::string make_build_env_cmd(const PreBuildInfo& pre_build_info, const Toolset& toolset); + enum class BuildPolicy { EMPTY_PACKAGE, diff --git a/toolsrc/src/vcpkg/build.cpp b/toolsrc/src/vcpkg/build.cpp index dd2beec9d..646da7398 100644 --- a/toolsrc/src/vcpkg/build.cpp +++ b/toolsrc/src/vcpkg/build.cpp @@ -782,7 +782,8 @@ namespace vcpkg::Build AbiEntry{status_it->get()->package.spec.name(), status_it->get()->package.abi}); } - const auto pre_build_info = PreBuildInfo::from_triplet_file(paths, triplet); + const auto pre_build_info = + PreBuildInfo::from_triplet_file(paths, triplet, config.scf.core_paragraph->name); auto maybe_abi_tag_and_file = compute_abi_tag(paths, config, pre_build_info, dependency_abis); @@ -997,7 +998,9 @@ namespace vcpkg::Build return inner_create_buildinfo(*pghs.get()); } - PreBuildInfo PreBuildInfo::from_triplet_file(const VcpkgPaths& paths, const Triplet& triplet) + PreBuildInfo PreBuildInfo::from_triplet_file(const VcpkgPaths& paths, + const Triplet& triplet, + Optional port) { static constexpr CStringView FLAG_GUID = "c35112b6-d1ba-415b-aa5d-81de856ef8eb"; @@ -1005,11 +1008,19 @@ namespace vcpkg::Build const fs::path ports_cmake_script_path = paths.scripts / "get_triplet_environment.cmake"; const fs::path triplet_file_path = paths.get_triplet_file_path(triplet); + std::vector args{{"CMAKE_TRIPLET_FILE", triplet_file_path}}; + + if (port) + { + args.emplace_back( + "CMAKE_PORT_SETTINGS", + paths.ports / port.value_or_exit(VCPKG_LINE_INFO) / "port_settings.cmake"); + } + const auto cmd_launch_cmake = System::make_cmake_cmd(cmake_exe_path, ports_cmake_script_path, - { - {"CMAKE_TRIPLET_FILE", triplet_file_path}, - }); + args); + const auto ec_data = System::cmd_execute_and_capture_output(cmd_launch_cmake); Checks::check_exit(VCPKG_LINE_INFO, ec_data.exit_code == 0, ec_data.output); diff --git a/toolsrc/src/vcpkg/commands.ci.cpp b/toolsrc/src/vcpkg/commands.ci.cpp index c12c26ff7..493c052cb 100644 --- a/toolsrc/src/vcpkg/commands.ci.cpp +++ b/toolsrc/src/vcpkg/commands.ci.cpp @@ -254,7 +254,14 @@ namespace vcpkg::Commands::CI return {spec.name(), it->second}; }); const auto& pre_build_info = pre_build_info_cache.get_lazy( - triplet, [&]() { return Build::PreBuildInfo::from_triplet_file(paths, triplet); }); + triplet, + [&]() { + return Build::PreBuildInfo::from_triplet_file( + paths, + triplet, + scfl->source_control_file->core_paragraph->name); + } + ); auto maybe_tag_and_file = Build::compute_abi_tag(paths, build_config, pre_build_info, dependency_abis); -- cgit v1.2.3 From 64198a8109b9d1cffcc3830f70d0c3f2b066638d Mon Sep 17 00:00:00 2001 From: "Curtis.Bezault" Date: Tue, 16 Jul 2019 15:51:50 -0700 Subject: Add to vcpkg.cmake --- toolsrc/src/vcpkg/build.cpp | 1 + 1 file changed, 1 insertion(+) (limited to 'toolsrc') diff --git a/toolsrc/src/vcpkg/build.cpp b/toolsrc/src/vcpkg/build.cpp index 646da7398..03e0c0b1d 100644 --- a/toolsrc/src/vcpkg/build.cpp +++ b/toolsrc/src/vcpkg/build.cpp @@ -401,6 +401,7 @@ namespace vcpkg::Build {"CURRENT_PORT_DIR", config.port_dir}, {"TARGET_TRIPLET", triplet.canonical_name()}, {"TARGET_TRIPLET_FILE", paths.get_triplet_file_path(triplet).u8string()}, + {"CMAKE_PORT_SETTINGS", config.port_dir / "port_settings.cmake"}, {"VCPKG_PLATFORM_TOOLSET", toolset.version.c_str()}, {"VCPKG_USE_HEAD_VERSION", Util::Enum::to_bool(config.build_package_options.use_head_version) ? "1" : "0"}, {"DOWNLOADS", paths.downloads}, -- cgit v1.2.3 From 7d9d457f589d82d6675c511636808a0fbee7a4e5 Mon Sep 17 00:00:00 2001 From: "Curtis.Bezault" Date: Tue, 16 Jul 2019 16:09:30 -0700 Subject: revert unecessary reordering --- toolsrc/include/vcpkg/build.h | 94 +++++++++++++++++++++---------------------- 1 file changed, 47 insertions(+), 47 deletions(-) (limited to 'toolsrc') diff --git a/toolsrc/include/vcpkg/build.h b/toolsrc/include/vcpkg/build.h index 0d10d12a4..a0ad64ffd 100644 --- a/toolsrc/include/vcpkg/build.h +++ b/toolsrc/include/vcpkg/build.h @@ -117,28 +117,53 @@ namespace vcpkg::Build std::string create_error_message(const BuildResult build_result, const PackageSpec& spec); std::string create_user_troubleshooting_message(const PackageSpec& spec); - enum class VcpkgTripletVar - { - TARGET_ARCHITECTURE = 0, - CMAKE_SYSTEM_NAME, - CMAKE_SYSTEM_VERSION, - PLATFORM_TOOLSET, - VISUAL_STUDIO_PATH, - CHAINLOAD_TOOLCHAIN_FILE, - BUILD_TYPE, - ENV_PASSTHROUGH, - }; - - const std::unordered_map VCPKG_OPTIONS = { - {"VCPKG_TARGET_ARCHITECTURE", VcpkgTripletVar::TARGET_ARCHITECTURE}, - {"VCPKG_CMAKE_SYSTEM_NAME", VcpkgTripletVar::CMAKE_SYSTEM_NAME}, - {"VCPKG_CMAKE_SYSTEM_VERSION", VcpkgTripletVar::CMAKE_SYSTEM_VERSION}, - {"VCPKG_PLATFORM_TOOLSET", VcpkgTripletVar::PLATFORM_TOOLSET}, - {"VCPKG_VISUAL_STUDIO_PATH", VcpkgTripletVar::VISUAL_STUDIO_PATH}, - {"VCPKG_CHAINLOAD_TOOLCHAIN_FILE", VcpkgTripletVar::CHAINLOAD_TOOLCHAIN_FILE}, - {"VCPKG_BUILD_TYPE", VcpkgTripletVar::BUILD_TYPE}, - {"VCPKG_ENV_PASSTHROUGH", VcpkgTripletVar::ENV_PASSTHROUGH}, - }; + /// + /// Settings from the triplet file which impact the build environment and post-build checks + /// + struct PreBuildInfo + { + /// + /// Runs the triplet file in a "capture" mode to create a PreBuildInfo + /// + static PreBuildInfo from_triplet_file(const VcpkgPaths& paths, + const Triplet& triplet, + Optional port = nullopt); + + std::string triplet_abi_tag; + std::string target_architecture; + std::string cmake_system_name; + std::string cmake_system_version; + Optional platform_toolset; + Optional visual_studio_path; + Optional external_toolchain_file; + Optional build_type; + std::vector passthrough_env_vars; + }; + + std::string make_build_env_cmd(const PreBuildInfo& pre_build_info, const Toolset& toolset); + + enum class VcpkgTripletVar + { + TARGET_ARCHITECTURE = 0, + CMAKE_SYSTEM_NAME, + CMAKE_SYSTEM_VERSION, + PLATFORM_TOOLSET, + VISUAL_STUDIO_PATH, + CHAINLOAD_TOOLCHAIN_FILE, + BUILD_TYPE, + ENV_PASSTHROUGH, + }; + + const std::unordered_map VCPKG_OPTIONS = { + {"VCPKG_TARGET_ARCHITECTURE", VcpkgTripletVar::TARGET_ARCHITECTURE}, + {"VCPKG_CMAKE_SYSTEM_NAME", VcpkgTripletVar::CMAKE_SYSTEM_NAME}, + {"VCPKG_CMAKE_SYSTEM_VERSION", VcpkgTripletVar::CMAKE_SYSTEM_VERSION}, + {"VCPKG_PLATFORM_TOOLSET", VcpkgTripletVar::PLATFORM_TOOLSET}, + {"VCPKG_VISUAL_STUDIO_PATH", VcpkgTripletVar::VISUAL_STUDIO_PATH}, + {"VCPKG_CHAINLOAD_TOOLCHAIN_FILE", VcpkgTripletVar::CHAINLOAD_TOOLCHAIN_FILE}, + {"VCPKG_BUILD_TYPE", VcpkgTripletVar::BUILD_TYPE}, + {"VCPKG_ENV_PASSTHROUGH", VcpkgTripletVar::ENV_PASSTHROUGH}, + }; struct ExtendedBuildResult { @@ -177,31 +202,6 @@ namespace vcpkg::Build const BuildPackageConfig& config, const StatusParagraphs& status_db); - /// - /// Settings from the triplet file which impact the build environment and post-build checks - /// - struct PreBuildInfo - { - /// - /// Runs the triplet file in a "capture" mode to create a PreBuildInfo - /// - static PreBuildInfo from_triplet_file(const VcpkgPaths& paths, - const Triplet& triplet, - Optional port = nullopt); - - std::string triplet_abi_tag; - std::string target_architecture; - std::string cmake_system_name; - std::string cmake_system_version; - Optional platform_toolset; - Optional visual_studio_path; - Optional external_toolchain_file; - Optional build_type; - std::vector passthrough_env_vars; - }; - - std::string make_build_env_cmd(const PreBuildInfo& pre_build_info, const Toolset& toolset); - enum class BuildPolicy { EMPTY_PACKAGE, -- cgit v1.2.3 From d4ab567609495a5c239f537d7c1e200f7b8a19c0 Mon Sep 17 00:00:00 2001 From: "Curtis.Bezault" Date: Wed, 17 Jul 2019 10:10:36 -0700 Subject: first pass at abi additional files --- toolsrc/include/vcpkg/build.h | 3 +++ toolsrc/src/vcpkg/build.cpp | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+) (limited to 'toolsrc') diff --git a/toolsrc/include/vcpkg/build.h b/toolsrc/include/vcpkg/build.h index a0ad64ffd..85d6ebed9 100644 --- a/toolsrc/include/vcpkg/build.h +++ b/toolsrc/include/vcpkg/build.h @@ -138,6 +138,7 @@ namespace vcpkg::Build Optional external_toolchain_file; Optional build_type; std::vector passthrough_env_vars; + std::vector additional_files; }; std::string make_build_env_cmd(const PreBuildInfo& pre_build_info, const Toolset& toolset); @@ -152,6 +153,7 @@ namespace vcpkg::Build CHAINLOAD_TOOLCHAIN_FILE, BUILD_TYPE, ENV_PASSTHROUGH, + ABI_ADDITIONAL_FILES, }; const std::unordered_map VCPKG_OPTIONS = { @@ -163,6 +165,7 @@ namespace vcpkg::Build {"VCPKG_CHAINLOAD_TOOLCHAIN_FILE", VcpkgTripletVar::CHAINLOAD_TOOLCHAIN_FILE}, {"VCPKG_BUILD_TYPE", VcpkgTripletVar::BUILD_TYPE}, {"VCPKG_ENV_PASSTHROUGH", VcpkgTripletVar::ENV_PASSTHROUGH}, + {"VCPKG_ABI_ADDITIONAL_FILES", VcpkgTripletVar::ABI_ADDITIONAL_FILES}, }; struct ExtendedBuildResult diff --git a/toolsrc/src/vcpkg/build.cpp b/toolsrc/src/vcpkg/build.cpp index 03e0c0b1d..daed6f695 100644 --- a/toolsrc/src/vcpkg/build.cpp +++ b/toolsrc/src/vcpkg/build.cpp @@ -496,6 +496,31 @@ namespace vcpkg::Build s_hash_cache.emplace(triplet_file_path, hash); } + for (const fs::path& additional_file : pre_build_info.additional_files) + { + it_hash = s_hash_cache.find(additional_file); + + if (it_hash != s_hash_cache.end()) + { + hash += "-"; + hash += it_hash->second; + } + else if (fs.is_regular_file(additional_file)) + { + std::string tmp_hash = Hash::get_file_hash(fs, additional_file, "SHA1"); + hash += "-"; + hash += tmp_hash; + + s_hash_cache.emplace(additional_file, tmp_hash); + } + else + { + Checks::exit_with_message( + VCPKG_LINE_INFO, + additional_file + " was listed as an additional file for calculating the abi, but was not found."); + } + } + return hash; } @@ -1087,6 +1112,13 @@ namespace vcpkg::Build case VcpkgTripletVar::ENV_PASSTHROUGH : pre_build_info.passthrough_env_vars = Strings::split(variable_value, ";"); break; + case VcpkgTripletVar::ABI_ADDITIONAL_FILES : + pre_build_info.additional_files = Util::fmap(Strings::split(variable_value, ";"), + [](const std::string& path) + { + return fs::path{path}; + }); + break; } } else -- cgit v1.2.3 From e81d22ddec6887a497055d4804a004ca662b4526 Mon Sep 17 00:00:00 2001 From: "Curtis.Bezault" Date: Wed, 17 Jul 2019 10:18:20 -0700 Subject: Convert name of file to u8 string, to compile on windows --- toolsrc/src/vcpkg/build.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'toolsrc') diff --git a/toolsrc/src/vcpkg/build.cpp b/toolsrc/src/vcpkg/build.cpp index daed6f695..b88d4fd1a 100644 --- a/toolsrc/src/vcpkg/build.cpp +++ b/toolsrc/src/vcpkg/build.cpp @@ -517,7 +517,7 @@ namespace vcpkg::Build { Checks::exit_with_message( VCPKG_LINE_INFO, - additional_file + " was listed as an additional file for calculating the abi, but was not found."); + additional_file.u8string() + " was listed as an additional file for calculating the abi, but was not found."); } } -- cgit v1.2.3 From f0f615532f9aa03f1518f03009cff5c716e9b1fd Mon Sep 17 00:00:00 2001 From: "Curtis.Bezault" Date: Wed, 17 Jul 2019 11:40:27 -0700 Subject: always calculate abi --- toolsrc/src/vcpkg/build.cpp | 116 ++++++++++++++++++++++---------------------- 1 file changed, 59 insertions(+), 57 deletions(-) (limited to 'toolsrc') diff --git a/toolsrc/src/vcpkg/build.cpp b/toolsrc/src/vcpkg/build.cpp index b88d4fd1a..47599bfca 100644 --- a/toolsrc/src/vcpkg/build.cpp +++ b/toolsrc/src/vcpkg/build.cpp @@ -623,8 +623,6 @@ namespace vcpkg::Build const PreBuildInfo& pre_build_info, Span dependency_abis) { - if (config.build_package_options.binary_caching == BinaryCaching::NO) return nullopt; - auto& fs = paths.get_filesystem(); const Triplet& triplet = config.triplet; const std::string& name = config.scf.core_paragraph->name; @@ -713,7 +711,7 @@ namespace vcpkg::Build } System::print2( - "Warning: binary caching disabled because abi keys are missing values:\n", + "Warning: abi keys are missing values:\n", Strings::join("", abi_tag_entries_missing, [](const AbiEntry& e) { return " " + e.key + "\n"; }), "\n"); @@ -814,15 +812,14 @@ namespace vcpkg::Build auto maybe_abi_tag_and_file = compute_abi_tag(paths, config, pre_build_info, dependency_abis); const auto abi_tag_and_file = maybe_abi_tag_and_file.get(); + const fs::path archives_root_dir = paths.root / "archives"; + const std::string archive_name = abi_tag_and_file->tag + ".zip"; + const fs::path archive_subpath = fs::u8path(abi_tag_and_file->tag.substr(0, 2)) / archive_name; + const fs::path archive_path = archives_root_dir / archive_subpath; + const fs::path archive_tombstone_path = archives_root_dir / "fail" / archive_subpath; if (config.build_package_options.binary_caching == BinaryCaching::YES && abi_tag_and_file) { - const fs::path archives_root_dir = paths.root / "archives"; - const std::string archive_name = abi_tag_and_file->tag + ".zip"; - const fs::path archive_subpath = fs::u8path(abi_tag_and_file->tag.substr(0, 2)) / archive_name; - const fs::path archive_path = archives_root_dir / archive_subpath; - const fs::path archive_tombstone_path = archives_root_dir / "fail" / archive_subpath; - if (fs.exists(archive_path)) { System::print2("Using cached binary package: ", archive_path.u8string(), "\n"); @@ -855,70 +852,75 @@ namespace vcpkg::Build } System::printf("Could not locate cached archive: %s\n", archive_path.u8string()); + } - ExtendedBuildResult result = do_build_package_and_clean_buildtrees( - paths, pre_build_info, spec, maybe_abi_tag_and_file.value_or(AbiTagAndFile{}).tag, config); + ExtendedBuildResult result = + do_build_package_and_clean_buildtrees( + paths, + pre_build_info, + spec, + maybe_abi_tag_and_file.value_or(AbiTagAndFile{}).tag, + config); - std::error_code ec; - fs.create_directories(paths.package_dir(spec) / "share" / spec.name(), ec); - auto abi_file_in_package = paths.package_dir(spec) / "share" / spec.name() / "vcpkg_abi_info.txt"; - fs.copy_file(abi_tag_and_file->tag_file, abi_file_in_package, fs::stdfs::copy_options::none, ec); - Checks::check_exit(VCPKG_LINE_INFO, !ec, "Could not copy into file: %s", abi_file_in_package.u8string()); + std::error_code ec; + fs.create_directories(paths.package_dir(spec) / "share" / spec.name(), ec); + auto abi_file_in_package = paths.package_dir(spec) / "share" / spec.name() / "vcpkg_abi_info.txt"; + fs.copy_file(abi_tag_and_file->tag_file, abi_file_in_package, fs::stdfs::copy_options::none, ec); + Checks::check_exit(VCPKG_LINE_INFO, !ec, "Could not copy into file: %s", abi_file_in_package.u8string()); - if (result.code == BuildResult::SUCCEEDED) - { - const auto tmp_archive_path = paths.buildtrees / spec.name() / (spec.triplet().to_string() + ".zip"); + if (config.build_package_options.binary_caching == BinaryCaching::YES && + result.code == BuildResult::SUCCEEDED) + { + const auto tmp_archive_path = paths.buildtrees / spec.name() / (spec.triplet().to_string() + ".zip"); - compress_archive(paths, spec, tmp_archive_path); + compress_archive(paths, spec, tmp_archive_path); - fs.create_directories(archive_path.parent_path(), ec); - fs.rename_or_copy(tmp_archive_path, archive_path, ".tmp", ec); - if (ec) - { - System::printf(System::Color::warning, - "Failed to store binary cache %s: %s\n", - archive_path.u8string(), - ec.message()); - } - else - System::printf("Stored binary cache: %s\n", archive_path.u8string()); + fs.create_directories(archive_path.parent_path(), ec); + fs.rename_or_copy(tmp_archive_path, archive_path, ".tmp", ec); + if (ec) + { + System::printf(System::Color::warning, + "Failed to store binary cache %s: %s\n", + archive_path.u8string(), + ec.message()); } - else if (result.code == BuildResult::BUILD_FAILED || result.code == BuildResult::POST_BUILD_CHECKS_FAILED) + else + System::printf("Stored binary cache: %s\n", archive_path.u8string()); + } + else if (config.build_package_options.binary_caching == BinaryCaching::YES && + (result.code == BuildResult::BUILD_FAILED || + result.code == BuildResult::POST_BUILD_CHECKS_FAILED)) + { + if (!fs.exists(archive_tombstone_path)) { - if (!fs.exists(archive_tombstone_path)) - { - // Build failed, store all failure logs in the tombstone. - const auto tmp_log_path = paths.buildtrees / spec.name() / "tmp_failure_logs"; - const auto tmp_log_path_destination = tmp_log_path / spec.name(); - const auto tmp_failure_zip = paths.buildtrees / spec.name() / "failure_logs.zip"; - fs.create_directories(tmp_log_path_destination, ec); + // Build failed, store all failure logs in the tombstone. + const auto tmp_log_path = paths.buildtrees / spec.name() / "tmp_failure_logs"; + const auto tmp_log_path_destination = tmp_log_path / spec.name(); + const auto tmp_failure_zip = paths.buildtrees / spec.name() / "failure_logs.zip"; + fs.create_directories(tmp_log_path_destination, ec); - for (auto& log_file : fs::stdfs::directory_iterator(paths.buildtrees / spec.name())) + for (auto& log_file : fs::stdfs::directory_iterator(paths.buildtrees / spec.name())) + { + if (log_file.path().extension() == ".log") { - if (log_file.path().extension() == ".log") - { - fs.copy_file(log_file.path(), - tmp_log_path_destination / log_file.path().filename(), - fs::stdfs::copy_options::none, - ec); - } + fs.copy_file(log_file.path(), + tmp_log_path_destination / log_file.path().filename(), + fs::stdfs::copy_options::none, + ec); } + } - compress_directory(paths, tmp_log_path, paths.buildtrees / spec.name() / "failure_logs.zip"); + compress_directory(paths, tmp_log_path, paths.buildtrees / spec.name() / "failure_logs.zip"); - fs.create_directories(archive_tombstone_path.parent_path(), ec); - fs.rename_or_copy(tmp_failure_zip, archive_tombstone_path, ".tmp", ec); + fs.create_directories(archive_tombstone_path.parent_path(), ec); + fs.rename_or_copy(tmp_failure_zip, archive_tombstone_path, ".tmp", ec); - // clean up temporary directory - fs.remove_all(tmp_log_path, ec); - } + // clean up temporary directory + fs.remove_all(tmp_log_path, ec); } - - return result; } - return do_build_package_and_clean_buildtrees( - paths, pre_build_info, spec, maybe_abi_tag_and_file.value_or(AbiTagAndFile{}).tag, config); + return result; } const std::string& to_string(const BuildResult build_result) -- cgit v1.2.3 From 58958eb0ea47c6c423fe78f2cd6fd1e31cbcb082 Mon Sep 17 00:00:00 2001 From: "Curtis.Bezault" Date: Wed, 17 Jul 2019 14:27:18 -0700 Subject: sourceparagraph changes --- toolsrc/include/vcpkg/sourceparagraph.h | 10 ++++++++++ toolsrc/src/vcpkg/sourceparagraph.cpp | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) (limited to 'toolsrc') diff --git a/toolsrc/include/vcpkg/sourceparagraph.h b/toolsrc/include/vcpkg/sourceparagraph.h index 9fbd83475..9a1c2c843 100644 --- a/toolsrc/include/vcpkg/sourceparagraph.h +++ b/toolsrc/include/vcpkg/sourceparagraph.h @@ -46,6 +46,12 @@ namespace vcpkg /// struct SourceParagraph { + enum TYPE : unsigned + { + PORT = 0, + SYS_TOOL, + }; + std::string name; std::string version; std::string description; @@ -54,6 +60,10 @@ namespace vcpkg std::vector supports; std::vector depends; std::vector default_features; + TYPE type; + + static TYPE type_from_string(const std::string& in); + static std::string string_from_type(const TYPE& in); }; /// diff --git a/toolsrc/src/vcpkg/sourceparagraph.cpp b/toolsrc/src/vcpkg/sourceparagraph.cpp index 1a52bd05f..26a7ab118 100644 --- a/toolsrc/src/vcpkg/sourceparagraph.cpp +++ b/toolsrc/src/vcpkg/sourceparagraph.cpp @@ -25,6 +25,7 @@ namespace vcpkg static const std::string SUPPORTS = "Supports"; static const std::string VERSION = "Version"; static const std::string HOMEPAGE = "Homepage"; + static const std::string TYPE = "Type"; } static Span get_list_of_valid_fields() @@ -36,6 +37,7 @@ namespace vcpkg SourceParagraphFields::MAINTAINER, SourceParagraphFields::BUILD_DEPENDS, SourceParagraphFields::HOMEPAGE, + SourceParagraphFields::TYPE, }; return valid_fields; @@ -98,6 +100,35 @@ namespace vcpkg } } + static SourceParagraph::TYPE type_from_string(const std::string& in) + { + if (Strings::equals(in, "port") || Strings::equals(in, "")) + { + return SourceParagraph::PORT; + } + + if (Strings::equals(in, "sys-tool")) + { + return SourceParagraph::SYS_TOOL; + } + + System::print2( + in, " is not a valid control file type. Valid types are:", + "\n port\n sys-tool"); + + Checks::exit_fail(VCPKG_LINE_INFO); + } + + static std::string string_from_type(const SourceParagraph::TYPE& in) + { + switch (in) + { + case SourceParagraph::PORT : return "port"; + case SourceParagraph::SYS_TOOL : return "sys-tool"; + default : Checks::exit_with_message(VCPKG_LINE_INFO, "Invalid CONTROL_TYPE value."); + } + } + static ParseExpected parse_source_paragraph(RawParagraph&& fields) { ParagraphParser parser(std::move(fields)); @@ -114,6 +145,7 @@ namespace vcpkg parse_comma_list(parser.optional_field(SourceParagraphFields::BUILD_DEPENDS))); spgh->supports = parse_comma_list(parser.optional_field(SourceParagraphFields::SUPPORTS)); spgh->default_features = parse_comma_list(parser.optional_field(SourceParagraphFields::DEFAULTFEATURES)); + spgh->type = type_from_string(parser.optional_field(SourceParagraphFields::TYPE)); auto err = parser.error_info(spgh->name); if (err) -- cgit v1.2.3 From f18ffe996877a058da9e0208f92331c83517f6a0 Mon Sep 17 00:00:00 2001 From: "Curtis.Bezault" Date: Wed, 17 Jul 2019 16:04:05 -0700 Subject: Add type field --- toolsrc/include/vcpkg/binaryparagraph.h | 1 + toolsrc/src/vcpkg/binaryparagraph.cpp | 10 ++++++++-- toolsrc/src/vcpkg/sourceparagraph.cpp | 6 +++--- 3 files changed, 12 insertions(+), 5 deletions(-) (limited to 'toolsrc') diff --git a/toolsrc/include/vcpkg/binaryparagraph.h b/toolsrc/include/vcpkg/binaryparagraph.h index 3315151c6..fa49edaba 100644 --- a/toolsrc/include/vcpkg/binaryparagraph.h +++ b/toolsrc/include/vcpkg/binaryparagraph.h @@ -31,6 +31,7 @@ namespace vcpkg std::vector default_features; std::vector depends; std::string abi; + SourceParagraph::TYPE type; }; struct BinaryControlFile diff --git a/toolsrc/src/vcpkg/binaryparagraph.cpp b/toolsrc/src/vcpkg/binaryparagraph.cpp index 4b80debab..ad7790fe1 100644 --- a/toolsrc/src/vcpkg/binaryparagraph.cpp +++ b/toolsrc/src/vcpkg/binaryparagraph.cpp @@ -23,6 +23,7 @@ namespace vcpkg static const std::string MAINTAINER = "Maintainer"; static const std::string DEPENDS = "Depends"; static const std::string DEFAULTFEATURES = "Default-Features"; + static const std::string TYPE = "Type"; } BinaryParagraph::BinaryParagraph() = default; @@ -60,6 +61,9 @@ namespace vcpkg this->default_features = parse_comma_list(parser.optional_field(Fields::DEFAULTFEATURES)); } + this->type = + SourceParagraph::type_from_string(parser.optional_field(Fields::TYPE)); + if (const auto err = parser.error_info(this->spec.to_string())) { System::print2(System::Color::error, "Error: while parsing the Binary Paragraph for ", this->spec, '\n'); @@ -72,14 +76,14 @@ namespace vcpkg } BinaryParagraph::BinaryParagraph(const SourceParagraph& spgh, const Triplet& triplet, const std::string& abi_tag) - : version(spgh.version), description(spgh.description), maintainer(spgh.maintainer), abi(abi_tag) + : version(spgh.version), description(spgh.description), maintainer(spgh.maintainer), abi(abi_tag), type(spgh.type) { this->spec = PackageSpec::from_name_and_triplet(spgh.name, triplet).value_or_exit(VCPKG_LINE_INFO); this->depends = filter_dependencies(spgh.depends, triplet); } BinaryParagraph::BinaryParagraph(const SourceParagraph& spgh, const FeatureParagraph& fpgh, const Triplet& triplet) - : version(), description(fpgh.description), maintainer(), feature(fpgh.name) + : version(), description(fpgh.description), maintainer(), feature(fpgh.name), type(spgh.type) { this->spec = PackageSpec::from_name_and_triplet(spgh.name, triplet).value_or_exit(VCPKG_LINE_INFO); this->depends = filter_dependencies(fpgh.depends, triplet); @@ -119,5 +123,7 @@ namespace vcpkg if (!pgh.maintainer.empty()) out_str.append("Maintainer: ").append(pgh.maintainer).push_back('\n'); if (!pgh.abi.empty()) out_str.append("Abi: ").append(pgh.abi).push_back('\n'); if (!pgh.description.empty()) out_str.append("Description: ").append(pgh.description).push_back('\n'); + + out_str.append("Type: ").append(SourceParagraph::string_from_type(pgh.type)).push_back('\n'); } } diff --git a/toolsrc/src/vcpkg/sourceparagraph.cpp b/toolsrc/src/vcpkg/sourceparagraph.cpp index 26a7ab118..6af93a99b 100644 --- a/toolsrc/src/vcpkg/sourceparagraph.cpp +++ b/toolsrc/src/vcpkg/sourceparagraph.cpp @@ -100,7 +100,7 @@ namespace vcpkg } } - static SourceParagraph::TYPE type_from_string(const std::string& in) + SourceParagraph::TYPE SourceParagraph::type_from_string(const std::string& in) { if (Strings::equals(in, "port") || Strings::equals(in, "")) { @@ -119,7 +119,7 @@ namespace vcpkg Checks::exit_fail(VCPKG_LINE_INFO); } - static std::string string_from_type(const SourceParagraph::TYPE& in) + std::string SourceParagraph::string_from_type(const SourceParagraph::TYPE& in) { switch (in) { @@ -145,7 +145,7 @@ namespace vcpkg parse_comma_list(parser.optional_field(SourceParagraphFields::BUILD_DEPENDS))); spgh->supports = parse_comma_list(parser.optional_field(SourceParagraphFields::SUPPORTS)); spgh->default_features = parse_comma_list(parser.optional_field(SourceParagraphFields::DEFAULTFEATURES)); - spgh->type = type_from_string(parser.optional_field(SourceParagraphFields::TYPE)); + spgh->type = SourceParagraph::type_from_string(parser.optional_field(SourceParagraphFields::TYPE)); auto err = parser.error_info(spgh->name); if (err) -- cgit v1.2.3 From f599f19bad8fd97b60f41063537be2e4ecef3ca7 Mon Sep 17 00:00:00 2001 From: Nicole Mazzuca Date: Wed, 17 Jul 2019 18:58:23 -0700 Subject: tests.files.cpp:create_directory_tree -- change magic numbers to names --- toolsrc/src/tests.files.cpp | 79 ++++++++++++++++++++++++--------------------- 1 file changed, 43 insertions(+), 36 deletions(-) (limited to 'toolsrc') diff --git a/toolsrc/src/tests.files.cpp b/toolsrc/src/tests.files.cpp index 56b0ceac6..e60662fd9 100644 --- a/toolsrc/src/tests.files.cpp +++ b/toolsrc/src/tests.files.cpp @@ -93,45 +93,31 @@ namespace UnitTest1 std::random_device rd; constexpr std::uint64_t max_depth = 5; constexpr std::uint64_t width = 5; - const auto type = depth < max_depth ? uid{0, 9}(urbg) : uid{7, 9}(urbg); - // 0 <= type < 7 : directory - // 7 = type : regular - // 8 = type : regular symlink (regular file if !ALLOW_SYMLINKS) - // 9 = type : directory symlink (^^) - - std::error_code ec; - if (type >= 7) - { - // I don't want to move urbg forward conditionally - if (type == 7 || !ALLOW_SYMLINKS) - { - // regular file - fs.write_contents(base, "", ec); - } - else if (type == 8) - { - // regular symlink - fs.write_contents(base, "", ec); - Assert::IsFalse(bool(ec)); - const std::filesystem::path basep = base.native(); - auto basep_link = basep; - basep_link.replace_filename(basep.filename().native() + L"-link"); - std::filesystem::create_symlink(basep, basep_link, ec); - } - else - { - // directory symlink - std::filesystem::path basep = base.native(); - std::filesystem::create_directory_symlink(basep / "..", basep, ec); - } - - Assert::IsFalse(bool(ec)); + // we want ~70% of our "files" to be directories, and then a third + // each of the remaining ~30% to be regular files, directory symlinks, + // and regular symlinks + constexpr std::uint64_t directory_min_tag = 0; + constexpr std::uint64_t directory_max_tag = 6; + constexpr std::uint64_t regular_file_tag = 7; + constexpr std::uint64_t regular_symlink_tag = 8; + constexpr std::uint64_t directory_symlink_tag = 9; + + // if we're at the max depth, we only want to build non-directories + std::uint64_t file_type; + if (depth < max_depth) { + file_type = uid{directory_min_tag, regular_symlink_tag}(urbg); + } else { + file_type = uid{regular_file_tag, regular_symlink_tag}(urbg); + } + if (!ALLOW_SYMLINKS && file_type > regular_file_tag) { + file_type = regular_file_tag; } - else + + std::error_code ec; + if (type <= directory_max_tag) { - // directory fs.create_directory(base, ec); Assert::IsFalse(bool(ec)); @@ -139,9 +125,30 @@ namespace UnitTest1 { create_directory_tree(urbg, fs, depth + 1, base / get_random_filename(urbg)); } - + } + else if (type == regular_file_tag) + { + // regular file + fs.write_contents(base, "", ec); + } + else if (type == regular_symlink_tag) + { + // regular symlink + fs.write_contents(base, "", ec); + Assert::IsFalse(bool(ec)); + const std::filesystem::path basep = base.native(); + auto basep_link = basep; + basep_link.replace_filename(basep.filename().native() + L"-link"); + std::filesystem::create_symlink(basep, basep_link, ec); + } + else // type == directory_symlink_tag + { + // directory symlink + std::filesystem::path basep = base.native(); + std::filesystem::create_directory_symlink(basep / "..", basep, ec); } + Assert::IsFalse(bool(ec)); } }; } -- cgit v1.2.3 From bb3a9ddb6ec917f549e991f6bd344ce77054bb67 Mon Sep 17 00:00:00 2001 From: Curtis J Bezault Date: Thu, 18 Jul 2019 09:02:21 -0700 Subject: [vcpkg] Environment Variable Passthrough (#7290) * use additional env param * remove partials * remove change to linux triplet * Fix some issues that vicroms pointed out * whitespace change --- toolsrc/include/vcpkg/build.h | 24 +++ toolsrc/include/vcpkg/sourceparagraph.h | 2 + toolsrc/src/vcpkg/build.cpp | 367 ++++++++++++++++++-------------- toolsrc/src/vcpkg/sourceparagraph.cpp | 22 ++ 4 files changed, 251 insertions(+), 164 deletions(-) (limited to 'toolsrc') diff --git a/toolsrc/include/vcpkg/build.h b/toolsrc/include/vcpkg/build.h index 04cd7cf87..e26597376 100644 --- a/toolsrc/include/vcpkg/build.h +++ b/toolsrc/include/vcpkg/build.h @@ -117,6 +117,29 @@ namespace vcpkg::Build std::string create_error_message(const BuildResult build_result, const PackageSpec& spec); std::string create_user_troubleshooting_message(const PackageSpec& spec); + enum class VcpkgTripletVar + { + TARGET_ARCHITECTURE = 0, + CMAKE_SYSTEM_NAME, + CMAKE_SYSTEM_VERSION, + PLATFORM_TOOLSET, + VISUAL_STUDIO_PATH, + CHAINLOAD_TOOLCHAIN_FILE, + BUILD_TYPE, + ENV_PASSTHROUGH, + }; + + const std::unordered_map VCPKG_OPTIONS = { + {"VCPKG_TARGET_ARCHITECTURE", VcpkgTripletVar::TARGET_ARCHITECTURE}, + {"VCPKG_CMAKE_SYSTEM_NAME", VcpkgTripletVar::CMAKE_SYSTEM_NAME}, + {"VCPKG_CMAKE_SYSTEM_VERSION", VcpkgTripletVar::CMAKE_SYSTEM_VERSION}, + {"VCPKG_PLATFORM_TOOLSET", VcpkgTripletVar::PLATFORM_TOOLSET}, + {"VCPKG_VISUAL_STUDIO_PATH", VcpkgTripletVar::VISUAL_STUDIO_PATH}, + {"VCPKG_CHAINLOAD_TOOLCHAIN_FILE", VcpkgTripletVar::CHAINLOAD_TOOLCHAIN_FILE}, + {"VCPKG_BUILD_TYPE", VcpkgTripletVar::BUILD_TYPE}, + {"VCPKG_ENV_PASSTHROUGH", VcpkgTripletVar::ENV_PASSTHROUGH}, + }; + /// /// Settings from the triplet file which impact the build environment and post-build checks /// @@ -135,6 +158,7 @@ namespace vcpkg::Build Optional visual_studio_path; Optional external_toolchain_file; Optional build_type; + std::vector passthrough_env_vars; }; std::string make_build_env_cmd(const PreBuildInfo& pre_build_info, const Toolset& toolset); diff --git a/toolsrc/include/vcpkg/sourceparagraph.h b/toolsrc/include/vcpkg/sourceparagraph.h index 6232a3fd2..9fbd83475 100644 --- a/toolsrc/include/vcpkg/sourceparagraph.h +++ b/toolsrc/include/vcpkg/sourceparagraph.h @@ -23,6 +23,8 @@ namespace vcpkg std::vector filter_dependencies(const std::vector& deps, const Triplet& t); std::vector filter_dependencies_to_specs(const std::vector& deps, const Triplet& t); + std::vector filter_dependencies_to_features(const std::vector& deps, + const Triplet& t); // zlib[uwp] becomes Dependency{"zlib", "uwp"} std::vector expand_qualified_dependencies(const std::vector& depends); diff --git a/toolsrc/src/vcpkg/build.cpp b/toolsrc/src/vcpkg/build.cpp index 68df1f965..2f58a3341 100644 --- a/toolsrc/src/vcpkg/build.cpp +++ b/toolsrc/src/vcpkg/build.cpp @@ -32,24 +32,16 @@ namespace vcpkg::Build::Command using Dependencies::InstallPlanAction; using Dependencies::InstallPlanType; - static constexpr StringLiteral OPTION_CHECKS_ONLY = "--checks-only"; - void perform_and_exit_ex(const FullPackageSpec& full_spec, const SourceControlFileLocation& scfl, const ParsedArguments& options, const VcpkgPaths& paths) { + vcpkg::Util::unused(options); + + const StatusParagraphs status_db = database_load_check(paths); const PackageSpec& spec = full_spec.package_spec; - const auto& scf = *scfl.source_control_file; - if (Util::Sets::contains(options.switches, OPTION_CHECKS_ONLY)) - { - const auto pre_build_info = Build::PreBuildInfo::from_triplet_file(paths, spec.triplet()); - const auto build_info = Build::read_build_info(paths.get_filesystem(), paths.build_info_file_path(spec)); - const size_t error_count = - PostBuildLint::perform_all_checks(spec, paths, pre_build_info, build_info, scfl.source_location); - Checks::check_exit(VCPKG_LINE_INFO, error_count == 0); - Checks::exit_success(VCPKG_LINE_INFO); - } + const SourceControlFile& scf = *scfl.source_control_file; Checks::check_exit(VCPKG_LINE_INFO, spec.name() == scf.core_paragraph->name, @@ -57,7 +49,6 @@ namespace vcpkg::Build::Command scf.core_paragraph->name, spec.name()); - const StatusParagraphs status_db = database_load_check(paths); const Build::BuildPackageOptions build_package_options{ Build::UseHeadVersion::NO, Build::AllowDownloads::YES, @@ -104,15 +95,11 @@ namespace vcpkg::Build::Command Checks::exit_success(VCPKG_LINE_INFO); } - static constexpr std::array BUILD_SWITCHES = {{ - {OPTION_CHECKS_ONLY, "Only run checks, do not rebuild package"}, - }}; - const CommandStructure COMMAND_STRUCTURE = { Help::create_example_string("build zlib:x64-windows"), 1, 1, - {BUILD_SWITCHES, {}}, + {{}, {}}, nullptr, }; @@ -230,6 +217,23 @@ namespace vcpkg::Build })); } + std::unordered_map make_env_passthrough(const PreBuildInfo& pre_build_info) + { + std::unordered_map env; + + for (auto&& env_var : pre_build_info.passthrough_env_vars) + { + auto env_val = System::get_environment_variable(env_var); + + if (env_val) + { + env[env_var] = env_val.value_or_exit(VCPKG_LINE_INFO); + } + } + + return env; + } + std::string make_build_env_cmd(const PreBuildInfo& pre_build_info, const Toolset& toolset) { if (pre_build_info.external_toolchain_file.has_value()) return ""; @@ -274,7 +278,7 @@ namespace vcpkg::Build return bcf; } - static void write_binary_control_file(const VcpkgPaths& paths, BinaryControlFile bcf) + static void write_binary_control_file(const VcpkgPaths& paths, const BinaryControlFile& bcf) { std::string start = Strings::serialize(bcf.core_paragraph); for (auto&& feature : bcf.features) @@ -285,23 +289,43 @@ namespace vcpkg::Build paths.get_filesystem().write_contents(binary_control_file, start, VCPKG_LINE_INFO); } - static std::vector compute_required_feature_specs(const BuildPackageConfig& config, - const StatusParagraphs& status_db) + static std::vector get_dependencies(const SourceControlFile& scf, + const std::set& feature_list, + const Triplet& triplet) { - const Triplet& triplet = config.triplet; - - const std::vector dep_strings = - Util::fmap_flatten(config.feature_list, [&](std::string const& feature) -> std::vector { + return Util::fmap_flatten( + feature_list, + [&](std::string const& feature) -> std::vector { if (feature == "core") { - return filter_dependencies(config.scf.core_paragraph->depends, triplet); + return filter_dependencies_to_features(scf.core_paragraph->depends, triplet); } - auto maybe_feature = config.scf.find_feature(feature); + auto maybe_feature = scf.find_feature(feature); Checks::check_exit(VCPKG_LINE_INFO, maybe_feature.has_value()); - return filter_dependencies(maybe_feature.get()->depends, triplet); + return filter_dependencies_to_features(maybe_feature.get()->depends, triplet); }); + } + + static std::vector get_dependency_names(const SourceControlFile& scf, + const std::set& feature_list, + const Triplet& triplet) + { + return Util::fmap(get_dependencies(scf, feature_list, triplet), + [&](const Features& feat) { + return feat.name; + } + ); + } + + static std::vector compute_required_feature_specs(const BuildPackageConfig& config, + const StatusParagraphs& status_db) + { + const Triplet& triplet = config.triplet; + + const std::vector dep_strings = + get_dependency_names(config.scf, config.feature_list, triplet); auto dep_fspecs = FeatureSpec::from_strings_and_triplet(dep_strings, triplet); Util::sort_unique_erase(dep_fspecs); @@ -353,40 +377,18 @@ namespace vcpkg::Build return concurrency; } - static ExtendedBuildResult do_build_package(const VcpkgPaths& paths, - const PreBuildInfo& pre_build_info, - const PackageSpec& spec, - const std::string& abi_tag, - const BuildPackageConfig& config) + static std::vector get_cmake_vars(const VcpkgPaths& paths, + const BuildPackageConfig& config, + const Triplet& triplet, + const Toolset& toolset) { - auto& fs = paths.get_filesystem(); - const Triplet& triplet = spec.triplet(); - const auto& triplet_file_path = paths.get_triplet_file_path(spec.triplet()).u8string(); - - if (!Strings::case_insensitive_ascii_starts_with(triplet_file_path, paths.triplets.u8string())) - { - System::printf("-- Loading triplet configuration from: %s\n", triplet_file_path); - } - if (!Strings::case_insensitive_ascii_starts_with(config.port_dir.u8string(), paths.ports.u8string())) - { - System::printf("-- Installing port from location: %s\n", config.port_dir.u8string()); - } - #if !defined(_WIN32) // TODO: remove when vcpkg.exe is in charge for acquiring tools. Change introduced in vcpkg v0.0.107. // bootstrap should have already downloaded ninja, but making sure it is present in case it was deleted. vcpkg::Util::unused(paths.get_tool_exe(Tools::NINJA)); #endif - const fs::path& cmake_exe_path = paths.get_tool_exe(Tools::CMAKE); const fs::path& git_exe_path = paths.get_tool_exe(Tools::GIT); -#if defined(_WIN32) - const fs::path& powershell_exe_path = paths.get_tool_exe("powershell-core"); - if (!fs.exists(powershell_exe_path.parent_path() / "powershell.exe")) - { - fs.copy(powershell_exe_path, powershell_exe_path.parent_path() / "powershell.exe", fs::copy_options::none); - } -#endif std::string all_features; for (auto& feature : config.scf.feature_paragraphs) @@ -394,14 +396,12 @@ namespace vcpkg::Build all_features.append(feature->name + ";"); } - const Toolset& toolset = paths.get_toolset(pre_build_info); - std::vector variables{ {"CMD", "BUILD"}, {"PORT", config.scf.core_paragraph->name}, {"CURRENT_PORT_DIR", config.port_dir}, - {"TARGET_TRIPLET", spec.triplet().canonical_name()}, - {"TARGET_TRIPLET_FILE", triplet_file_path}, + {"TARGET_TRIPLET", triplet.canonical_name()}, + {"TARGET_TRIPLET_FILE", paths.get_triplet_file_path(triplet).u8string()}, {"VCPKG_PLATFORM_TOOLSET", toolset.version.c_str()}, {"VCPKG_USE_HEAD_VERSION", Util::Enum::to_bool(config.build_package_options.use_head_version) ? "1" : "0"}, {"DOWNLOADS", paths.downloads}, @@ -417,9 +417,22 @@ namespace vcpkg::Build variables.push_back({"GIT", git_exe_path}); } + return variables; + } + + static std::string make_build_cmd(const VcpkgPaths& paths, + const PreBuildInfo& pre_build_info, + const BuildPackageConfig& config, + const Triplet& triplet) + { + const Toolset& toolset = paths.get_toolset(pre_build_info); + const fs::path& cmake_exe_path = paths.get_tool_exe(Tools::CMAKE); + std::vector variables = + get_cmake_vars(paths, config, triplet, toolset); + const std::string cmd_launch_cmake = System::make_cmake_cmd(cmake_exe_path, paths.ports_cmake, variables); - auto command = make_build_env_cmd(pre_build_info, toolset); + std::string command = make_build_env_cmd(pre_build_info, toolset); if (!command.empty()) { #ifdef _WIN32 @@ -428,16 +441,93 @@ namespace vcpkg::Build command.append(" && "); #endif } + command.append(cmd_launch_cmake); + + return command; + } + + static std::string get_triplet_abi(const VcpkgPaths& paths, + const PreBuildInfo& pre_build_info, + const Triplet& triplet) + { + static std::map s_hash_cache; + + const fs::path triplet_file_path = paths.get_triplet_file_path(triplet); + const auto& fs = paths.get_filesystem(); + + std::string hash; + + auto it_hash = s_hash_cache.find(triplet_file_path); + if (it_hash != s_hash_cache.end()) + { + hash = it_hash->second; + } + else + { + hash = Hash::get_file_hash(fs, triplet_file_path, "SHA1"); + + if (auto p = pre_build_info.external_toolchain_file.get()) + { + hash += "-"; + hash += Hash::get_file_hash(fs, *p, "SHA1"); + } + else if (pre_build_info.cmake_system_name == "Linux") + { + hash += "-"; + hash += Hash::get_file_hash(fs, paths.scripts / "toolchains" / "linux.cmake", "SHA1"); + } + else if (pre_build_info.cmake_system_name == "Darwin") + { + hash += "-"; + hash += Hash::get_file_hash(fs, paths.scripts / "toolchains" / "osx.cmake", "SHA1"); + } + else if (pre_build_info.cmake_system_name == "FreeBSD") + { + hash += "-"; + hash += Hash::get_file_hash(fs, paths.scripts / "toolchains" / "freebsd.cmake", "SHA1"); + } + else if (pre_build_info.cmake_system_name == "Android") + { + hash += "-"; + hash += Hash::get_file_hash(fs, paths.scripts / "toolchains" / "android.cmake", "SHA1"); + } + + s_hash_cache.emplace(triplet_file_path, hash); + } + + return hash; + } + + static ExtendedBuildResult do_build_package(const VcpkgPaths& paths, + const PreBuildInfo& pre_build_info, + const PackageSpec& spec, + const std::string& abi_tag, + const BuildPackageConfig& config) + { + auto& fs = paths.get_filesystem(); + const Triplet& triplet = spec.triplet(); + const auto& triplet_file_path = paths.get_triplet_file_path(spec.triplet()).u8string(); + + if (!Strings::case_insensitive_ascii_starts_with(triplet_file_path, paths.triplets.u8string())) + { + System::printf("-- Loading triplet configuration from: %s\n", triplet_file_path); + } + if (!Strings::case_insensitive_ascii_starts_with(config.port_dir.u8string(), paths.ports.u8string())) + { + System::printf("-- Installing port from location: %s\n", config.port_dir.u8string()); + } + const auto timer = Chrono::ElapsedTimer::create_started(); - const int return_code = System::cmd_execute_clean( - command, - {} -#ifdef _WIN32 - , - powershell_exe_path.parent_path().u8string() + ";" -#endif - ); + + std::string command = + make_build_cmd(paths, pre_build_info, config, triplet); + std::unordered_map env = + make_env_passthrough(pre_build_info); + + const int return_code = + System::cmd_execute_clean(command, env); + const auto buildtimeus = timer.microseconds(); const auto spec_string = spec.to_string(); @@ -711,7 +801,7 @@ namespace vcpkg::Build { System::print2("Using cached binary package: ", archive_path.u8string(), "\n"); - auto archive_result = decompress_archive(paths, spec, archive_path); + int archive_result = decompress_archive(paths, spec, archive_path); if (archive_result != 0) { @@ -946,111 +1036,60 @@ namespace vcpkg::Build const std::string variable_name = s.at(0); const std::string variable_value = variable_with_no_value ? "" : s.at(1); - if (variable_name == "VCPKG_TARGET_ARCHITECTURE") - { - pre_build_info.target_architecture = variable_value; - continue; - } - - if (variable_name == "VCPKG_CMAKE_SYSTEM_NAME") - { - pre_build_info.cmake_system_name = variable_value; - continue; - } - - if (variable_name == "VCPKG_CMAKE_SYSTEM_VERSION") - { - pre_build_info.cmake_system_version = variable_value; - continue; - } - - if (variable_name == "VCPKG_PLATFORM_TOOLSET") - { - pre_build_info.platform_toolset = - variable_value.empty() ? nullopt : Optional{variable_value}; - continue; - } - - if (variable_name == "VCPKG_VISUAL_STUDIO_PATH") - { - pre_build_info.visual_studio_path = - variable_value.empty() ? nullopt : Optional{variable_value}; - continue; - } - - if (variable_name == "VCPKG_CHAINLOAD_TOOLCHAIN_FILE") + auto maybe_option = VCPKG_OPTIONS.find(variable_name); + if (maybe_option != VCPKG_OPTIONS.end()) { - pre_build_info.external_toolchain_file = - variable_value.empty() ? nullopt : Optional{variable_value}; - continue; + switch (maybe_option->second) + { + case VcpkgTripletVar::TARGET_ARCHITECTURE : + pre_build_info.target_architecture = variable_value; + break; + case VcpkgTripletVar::CMAKE_SYSTEM_NAME : + pre_build_info.cmake_system_name = variable_value; + break; + case VcpkgTripletVar::CMAKE_SYSTEM_VERSION : + pre_build_info.cmake_system_version = variable_value; + break; + case VcpkgTripletVar::PLATFORM_TOOLSET : + pre_build_info.platform_toolset = + variable_value.empty() ? nullopt : Optional{variable_value}; + break; + case VcpkgTripletVar::VISUAL_STUDIO_PATH : + pre_build_info.visual_studio_path = + variable_value.empty() ? nullopt : Optional{variable_value}; + break; + case VcpkgTripletVar::CHAINLOAD_TOOLCHAIN_FILE : + pre_build_info.external_toolchain_file = + variable_value.empty() ? nullopt : Optional{variable_value}; + break; + case VcpkgTripletVar::BUILD_TYPE : + if (variable_value.empty()) + pre_build_info.build_type = nullopt; + else if (Strings::case_insensitive_ascii_equals(variable_value, "debug")) + pre_build_info.build_type = ConfigurationType::DEBUG; + else if (Strings::case_insensitive_ascii_equals(variable_value, "release")) + pre_build_info.build_type = ConfigurationType::RELEASE; + else + Checks::exit_with_message( + VCPKG_LINE_INFO, "Unknown setting for VCPKG_BUILD_TYPE: %s", variable_value); + break; + case VcpkgTripletVar::ENV_PASSTHROUGH : + pre_build_info.passthrough_env_vars = Strings::split(variable_value, ";"); + break; + } } - - if (variable_name == "VCPKG_BUILD_TYPE") + else { - if (variable_value.empty()) - pre_build_info.build_type = nullopt; - else if (Strings::case_insensitive_ascii_equals(variable_value, "debug")) - pre_build_info.build_type = ConfigurationType::DEBUG; - else if (Strings::case_insensitive_ascii_equals(variable_value, "release")) - pre_build_info.build_type = ConfigurationType::RELEASE; - else - Checks::exit_with_message( - VCPKG_LINE_INFO, "Unknown setting for VCPKG_BUILD_TYPE: %s", variable_value); - continue; + Checks::exit_with_message(VCPKG_LINE_INFO, "Unknown variable name %s", line); } - - Checks::exit_with_message(VCPKG_LINE_INFO, "Unknown variable name %s", line); } - pre_build_info.triplet_abi_tag = [&]() { - const auto& fs = paths.get_filesystem(); - static std::map s_hash_cache; - - auto it_hash = s_hash_cache.find(triplet_file_path); - if (it_hash != s_hash_cache.end()) - { - return it_hash->second; - } - auto hash = Hash::get_file_hash(fs, triplet_file_path, "SHA1"); - - if (auto p = pre_build_info.external_toolchain_file.get()) - { - hash += "-"; - hash += Hash::get_file_hash(fs, *p, "SHA1"); - } - else if (pre_build_info.cmake_system_name.empty() || - pre_build_info.cmake_system_name == "WindowsStore") - { - hash += "-"; - hash += Hash::get_file_hash(fs, paths.scripts / "toolchains" / "windows.cmake", "SHA1"); - } - else if (pre_build_info.cmake_system_name == "Linux") - { - hash += "-"; - hash += Hash::get_file_hash(fs, paths.scripts / "toolchains" / "linux.cmake", "SHA1"); - } - else if (pre_build_info.cmake_system_name == "Darwin") - { - hash += "-"; - hash += Hash::get_file_hash(fs, paths.scripts / "toolchains" / "osx.cmake", "SHA1"); - } - else if (pre_build_info.cmake_system_name == "FreeBSD") - { - hash += "-"; - hash += Hash::get_file_hash(fs, paths.scripts / "toolchains" / "freebsd.cmake", "SHA1"); - } - else if (pre_build_info.cmake_system_name == "Android") - { - hash += "-"; - hash += Hash::get_file_hash(fs, paths.scripts / "toolchains" / "android.cmake", "SHA1"); - } - - s_hash_cache.emplace(triplet_file_path, hash); - return hash; - }(); + pre_build_info.triplet_abi_tag = + get_triplet_abi(paths, pre_build_info, triplet); return pre_build_info; } + ExtendedBuildResult::ExtendedBuildResult(BuildResult code) : code(code) {} ExtendedBuildResult::ExtendedBuildResult(BuildResult code, std::unique_ptr&& bcf) : code(code), binary_control_file(std::move(bcf)) diff --git a/toolsrc/src/vcpkg/sourceparagraph.cpp b/toolsrc/src/vcpkg/sourceparagraph.cpp index 9bc59cbe7..1a52bd05f 100644 --- a/toolsrc/src/vcpkg/sourceparagraph.cpp +++ b/toolsrc/src/vcpkg/sourceparagraph.cpp @@ -238,6 +238,28 @@ namespace vcpkg return ret; } + std::vector filter_dependencies_to_features(const std::vector& deps, + const Triplet& t) + { + std::vector ret; + for (auto&& dep : deps) + { + auto qualifiers = Strings::split(dep.qualifier, "&"); + if (std::all_of(qualifiers.begin(), qualifiers.end(), [&](const std::string& qualifier) { + if (qualifier.empty()) return true; + if (qualifier[0] == '!') + { + return t.canonical_name().find(qualifier.substr(1)) == std::string::npos; + } + return t.canonical_name().find(qualifier) != std::string::npos; + })) + { + ret.emplace_back(dep.depend); + } + } + return ret; + } + std::vector filter_dependencies_to_specs(const std::vector& deps, const Triplet& t) { return FeatureSpec::from_strings_and_triplet(filter_dependencies(deps, t), t); -- cgit v1.2.3 From d39bd70d533c64e929d4399cb9a1bdbfe0efaecd Mon Sep 17 00:00:00 2001 From: "Curtis.Bezault" Date: Thu, 18 Jul 2019 13:24:31 -0700 Subject: add needs_rebuild, should probably be moved to somewhere else --- toolsrc/include/vcpkg/statusparagraphs.h | 2 ++ toolsrc/src/vcpkg/statusparagraphs.cpp | 25 +++++++++++++++++++++++++ 2 files changed, 27 insertions(+) (limited to 'toolsrc') diff --git a/toolsrc/include/vcpkg/statusparagraphs.h b/toolsrc/include/vcpkg/statusparagraphs.h index fa064de7e..3ebad67de 100644 --- a/toolsrc/include/vcpkg/statusparagraphs.h +++ b/toolsrc/include/vcpkg/statusparagraphs.h @@ -62,6 +62,8 @@ namespace vcpkg /// `true` if installed, `false` if not or not found. bool is_installed(const FeatureSpec& spec) const; + bool needs_rebuild(const PackageSpec& spec); + iterator insert(std::unique_ptr); friend void serialize(const StatusParagraphs& pgh, std::string& out_str); diff --git a/toolsrc/src/vcpkg/statusparagraphs.cpp b/toolsrc/src/vcpkg/statusparagraphs.cpp index c642af59b..f204ed568 100644 --- a/toolsrc/src/vcpkg/statusparagraphs.cpp +++ b/toolsrc/src/vcpkg/statusparagraphs.cpp @@ -118,6 +118,31 @@ namespace vcpkg return it != end() && (*it)->is_installed(); } + bool vcpkg::StatusParagraphs::needs_rebuild(const PackageSpec& spec) + { + auto it = find(spec); + if (it != end()) + { + for (const std::string& dep : (*it)->package.depends) + { + PackageSpec dep_spec = + PackageSpec::from_name_and_triplet( + dep, + spec.triplet()).value_or_exit(VCPKG_LINE_INFO); + + if (needs_rebuild(dep_spec)) + { + (*it)->state = InstallState::NEEDS_REBUILD; + return true; + } + } + + return (*it)->needs_rebuild(); + } + + return false; + } + StatusParagraphs::iterator StatusParagraphs::insert(std::unique_ptr pgh) { Checks::check_exit(VCPKG_LINE_INFO, pgh != nullptr, "Inserted null paragraph"); -- cgit v1.2.3 From ef48500ac6ec3f9e0f494329365452d3ad9e1271 Mon Sep 17 00:00:00 2001 From: Dan Nissenbaum Date: Thu, 18 Jul 2019 16:53:24 -0400 Subject: Better error message when VCPKG_ROOT is independently defined (#7229) --- toolsrc/src/vcpkg.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'toolsrc') diff --git a/toolsrc/src/vcpkg.cpp b/toolsrc/src/vcpkg.cpp index 363b39814..46ec8c013 100644 --- a/toolsrc/src/vcpkg.cpp +++ b/toolsrc/src/vcpkg.cpp @@ -143,7 +143,7 @@ static void inner(const VcpkgCmdArguments& args) #else const int exit_code = chdir(paths.root.c_str()); #endif - Checks::check_exit(VCPKG_LINE_INFO, exit_code == 0, "Changing the working dir failed"); + Checks::check_exit(VCPKG_LINE_INFO, exit_code == 0, "Changing the working directory to the vcpkg root directory failed. Did you incorrectly define the VCPKG_ROOT environment variable, or did you mistakenly create a file named .vcpkg-root somewhere?"); if (args.command == "install" || args.command == "remove" || args.command == "export" || args.command == "update") { -- cgit v1.2.3 From 9b5ee9941232dc3dd07085de747b1e95bb059525 Mon Sep 17 00:00:00 2001 From: Curtis J Bezault Date: Thu, 18 Jul 2019 16:20:00 -0700 Subject: Update VERSION.txt --- toolsrc/VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'toolsrc') diff --git a/toolsrc/VERSION.txt b/toolsrc/VERSION.txt index 6b2a31295..d7d695c69 100644 --- a/toolsrc/VERSION.txt +++ b/toolsrc/VERSION.txt @@ -1 +1 @@ -"2019.06.26" \ No newline at end of file +"2019.07.18" -- cgit v1.2.3 From fddebb75da034752fb267ba121497ba58157bb79 Mon Sep 17 00:00:00 2001 From: Nicole Mazzuca Date: Thu, 18 Jul 2019 16:40:52 -0700 Subject: clang-format all the things --- toolsrc/include/vcpkg/base/strings.h | 4 ++-- toolsrc/include/vcpkg/base/work_queue.h | 3 +-- toolsrc/src/tests.files.cpp | 16 ++++++++++------ toolsrc/src/tests.strings.cpp | 9 ++++++--- toolsrc/src/vcpkg/base/strings.cpp | 3 ++- toolsrc/src/vcpkg/build.cpp | 15 ++++++--------- 6 files changed, 27 insertions(+), 23 deletions(-) (limited to 'toolsrc') diff --git a/toolsrc/include/vcpkg/base/strings.h b/toolsrc/include/vcpkg/base/strings.h index aa4c4d690..d7de9b0b2 100644 --- a/toolsrc/include/vcpkg/base/strings.h +++ b/toolsrc/include/vcpkg/base/strings.h @@ -186,7 +186,7 @@ namespace vcpkg::Strings bool contains(StringView haystack, StringView needle); // base 32 encoding, since base64 encoding requires lowercase letters, - // which are not distinct from uppercase letters on macOS or Windows filesystems. - // follows RFC 4648 + // which are not distinct from uppercase letters on macOS or Windows filesystems. + // follows RFC 4648 std::string b32_encode(std::uint64_t x) noexcept; } diff --git a/toolsrc/include/vcpkg/base/work_queue.h b/toolsrc/include/vcpkg/base/work_queue.h index 69ca387f3..70142e110 100644 --- a/toolsrc/include/vcpkg/base/work_queue.h +++ b/toolsrc/include/vcpkg/base/work_queue.h @@ -98,8 +98,7 @@ namespace vcpkg if (!running_workers()) { m_cv.notify_one(); - - } + } } // wait for all threads to join diff --git a/toolsrc/src/tests.files.cpp b/toolsrc/src/tests.files.cpp index e60662fd9..482675c34 100644 --- a/toolsrc/src/tests.files.cpp +++ b/toolsrc/src/tests.files.cpp @@ -3,8 +3,8 @@ #include #include -#include #include // required for filesystem::create_{directory_}symlink +#include #include #include @@ -31,11 +31,11 @@ namespace UnitTest1 } else { - // if we get a permissions error, we still know that we're in developer mode + // if we get a permissions error, we still know that we're in developer mode ALLOW_SYMLINKS = true; } - if (status == ERROR_SUCCESS) RegCloseKey(key); + if (status == ERROR_SUCCESS) RegCloseKey(key); } private: @@ -105,13 +105,17 @@ namespace UnitTest1 // if we're at the max depth, we only want to build non-directories std::uint64_t file_type; - if (depth < max_depth) { + if (depth < max_depth) + { file_type = uid{directory_min_tag, regular_symlink_tag}(urbg); - } else { + } + else + { file_type = uid{regular_file_tag, regular_symlink_tag}(urbg); } - if (!ALLOW_SYMLINKS && file_type > regular_file_tag) { + if (!ALLOW_SYMLINKS && file_type > regular_file_tag) + { file_type = regular_file_tag; } diff --git a/toolsrc/src/tests.strings.cpp b/toolsrc/src/tests.strings.cpp index 6301ea05d..f541a203d 100644 --- a/toolsrc/src/tests.strings.cpp +++ b/toolsrc/src/tests.strings.cpp @@ -8,8 +8,10 @@ using namespace Microsoft::VisualStudio::CppUnitTestFramework; -namespace UnitTest1 { - class StringsTest : public TestClass { +namespace UnitTest1 +{ + class StringsTest : public TestClass + { TEST_METHOD(b64url_encode) { using u64 = std::uint64_t; @@ -29,7 +31,8 @@ namespace UnitTest1 { map.emplace_back(0xFFFF'FFFF'FFFF'FFFF, "777777777777P"); std::string result; - for (const auto& pr : map) { + for (const auto& pr : map) + { result = vcpkg::Strings::b32_encode(pr.first); Assert::AreEqual(result, pr.second); } diff --git a/toolsrc/src/vcpkg/base/strings.cpp b/toolsrc/src/vcpkg/base/strings.cpp index 7970e1b46..46e78a363 100644 --- a/toolsrc/src/vcpkg/base/strings.cpp +++ b/toolsrc/src/vcpkg/base/strings.cpp @@ -301,7 +301,8 @@ namespace vcpkg::Strings auto value = static_cast(x); // 32 values, plus the implicit \0 - constexpr static char map[33] = "ABCDEFGHIJKLMNOP" "QRSTUVWXYZ234567"; + constexpr static char map[33] = "ABCDEFGHIJKLMNOP" + "QRSTUVWXYZ234567"; // log2(32) constexpr static int shift = 5; diff --git a/toolsrc/src/vcpkg/build.cpp b/toolsrc/src/vcpkg/build.cpp index c50acce7e..66d8dae2c 100644 --- a/toolsrc/src/vcpkg/build.cpp +++ b/toolsrc/src/vcpkg/build.cpp @@ -363,8 +363,7 @@ namespace vcpkg::Build const Triplet& triplet = spec.triplet(); const auto& triplet_file_path = paths.get_triplet_file_path(spec.triplet()).u8string(); - if (!Strings::case_insensitive_ascii_starts_with(triplet_file_path, - paths.triplets.u8string())) + if (!Strings::case_insensitive_ascii_starts_with(triplet_file_path, paths.triplets.u8string())) { System::printf("-- Loading triplet configuration from: %s\n", triplet_file_path); } @@ -431,12 +430,11 @@ namespace vcpkg::Build } command.append(cmd_launch_cmake); const auto timer = Chrono::ElapsedTimer::create_started(); - const int return_code = System::cmd_execute_clean( - command, - {} + const int return_code = System::cmd_execute_clean(command, + {} #ifdef _WIN32 - , - powershell_exe_path.parent_path().u8string() + ";" + , + powershell_exe_path.parent_path().u8string() + ";" #endif ); const auto buildtimeus = timer.microseconds(); @@ -1020,8 +1018,7 @@ namespace vcpkg::Build hash += "-"; hash += Hash::get_file_hash(fs, *p, "SHA1"); } - else if (pre_build_info.cmake_system_name.empty() || - pre_build_info.cmake_system_name == "WindowsStore") + else if (pre_build_info.cmake_system_name.empty() || pre_build_info.cmake_system_name == "WindowsStore") { hash += "-"; hash += Hash::get_file_hash(fs, paths.scripts / "toolchains" / "windows.cmake", "SHA1"); -- cgit v1.2.3 From 825055378998ae6bc24e8cb0bce2e1fbf9a425da Mon Sep 17 00:00:00 2001 From: nicole mazzuca Date: Thu, 18 Jul 2019 19:07:00 -0700 Subject: Rewrite the tests! now they're cross-platform! (#7315) * begin exploratory rewriting of tests * continue working on tests * more test work! holy butts vcpkg-tests/plan.cpp was a bunch of work * finish writing new tests - [x] write catch2 tests - [ ] rewrite/at least delete the VS project files - [ ] document running tests * Fix tests to work on WSL, rewrite test vcxproj still need to test on macOS also, delete tests.pch.h * Condense add_test calls --- toolsrc/.clang-format | 1 + toolsrc/CMakeLists.txt | 14 + toolsrc/include/tests.pch.h | 19 - toolsrc/include/tests.utils.h | 76 - toolsrc/include/vcpkg-tests/catch.h | 16865 +++++++++++++++++++++++++ toolsrc/include/vcpkg-tests/util.h | 33 + toolsrc/src/tests.arguments.cpp | 104 - toolsrc/src/tests.chrono.cpp | 41 - toolsrc/src/tests.dependencies.cpp | 110 - toolsrc/src/tests.packagespec.cpp | 136 - toolsrc/src/tests.paragraph.cpp | 441 - toolsrc/src/tests.pch.cpp | 1 - toolsrc/src/tests.plan.cpp | 1261 -- toolsrc/src/tests.statusparagraphs.cpp | 115 - toolsrc/src/tests.update.cpp | 106 - toolsrc/src/tests.utils.cpp | 42 - toolsrc/src/vcpkg-tests/arguments.cpp | 109 + toolsrc/src/vcpkg-tests/catch.cpp | 11 + toolsrc/src/vcpkg-tests/chrono.cpp | 34 + toolsrc/src/vcpkg-tests/dependencies.cpp | 28 + toolsrc/src/vcpkg-tests/paragraph.cpp | 445 + toolsrc/src/vcpkg-tests/plan.cpp | 1241 ++ toolsrc/src/vcpkg-tests/specifier.cpp | 134 + toolsrc/src/vcpkg-tests/statusparagraphs.cpp | 110 + toolsrc/src/vcpkg-tests/supports.cpp | 79 + toolsrc/src/vcpkg-tests/update.cpp | 102 + toolsrc/src/vcpkg-tests/util.cpp | 47 + toolsrc/vcpkgtest/vcpkgtest.vcxproj | 52 +- toolsrc/vcpkgtest/vcpkgtest.vcxproj.filters | 27 +- 29 files changed, 19288 insertions(+), 2496 deletions(-) delete mode 100644 toolsrc/include/tests.pch.h delete mode 100644 toolsrc/include/tests.utils.h create mode 100644 toolsrc/include/vcpkg-tests/catch.h create mode 100644 toolsrc/include/vcpkg-tests/util.h delete mode 100644 toolsrc/src/tests.arguments.cpp delete mode 100644 toolsrc/src/tests.chrono.cpp delete mode 100644 toolsrc/src/tests.dependencies.cpp delete mode 100644 toolsrc/src/tests.packagespec.cpp delete mode 100644 toolsrc/src/tests.paragraph.cpp delete mode 100644 toolsrc/src/tests.pch.cpp delete mode 100644 toolsrc/src/tests.plan.cpp delete mode 100644 toolsrc/src/tests.statusparagraphs.cpp delete mode 100644 toolsrc/src/tests.update.cpp delete mode 100644 toolsrc/src/tests.utils.cpp create mode 100644 toolsrc/src/vcpkg-tests/arguments.cpp create mode 100644 toolsrc/src/vcpkg-tests/catch.cpp create mode 100644 toolsrc/src/vcpkg-tests/chrono.cpp create mode 100644 toolsrc/src/vcpkg-tests/dependencies.cpp create mode 100644 toolsrc/src/vcpkg-tests/paragraph.cpp create mode 100644 toolsrc/src/vcpkg-tests/plan.cpp create mode 100644 toolsrc/src/vcpkg-tests/specifier.cpp create mode 100644 toolsrc/src/vcpkg-tests/statusparagraphs.cpp create mode 100644 toolsrc/src/vcpkg-tests/supports.cpp create mode 100644 toolsrc/src/vcpkg-tests/update.cpp create mode 100644 toolsrc/src/vcpkg-tests/util.cpp (limited to 'toolsrc') diff --git a/toolsrc/.clang-format b/toolsrc/.clang-format index c14765672..4d2c34fc4 100644 --- a/toolsrc/.clang-format +++ b/toolsrc/.clang-format @@ -28,6 +28,7 @@ Cpp11BracedListStyle: true IndentCaseLabels: true KeepEmptyLinesAtTheStartOfBlocks: false NamespaceIndentation: All +ForEachMacros: [TEST_CASE, SECTION] PenaltyReturnTypeOnItsOwnLine: 1000 SpaceAfterTemplateKeyword: false SpaceBeforeCpp11BracedList: false diff --git a/toolsrc/CMakeLists.txt b/toolsrc/CMakeLists.txt index fccf2b7ad..ca23db413 100644 --- a/toolsrc/CMakeLists.txt +++ b/toolsrc/CMakeLists.txt @@ -2,6 +2,8 @@ cmake_minimum_required(VERSION 3.3) project(vcpkg C CXX) +enable_testing() + OPTION(DEFINE_DISABLE_METRICS "Option for disabling metrics" OFF) OPTION(VCPKG_ALLOW_APPLE_CLANG "Option for allowing apple clang" OFF) @@ -33,6 +35,7 @@ if(GCC OR CLANG) endif() file(GLOB_RECURSE VCPKGLIB_SOURCES src/vcpkg/*.cpp) +file(GLOB_RECURSE VCPKGTEST_SOURCES src/vcpkg-tests/*.cpp) if (DEFINE_DISABLE_METRICS) set(DISABLE_METRICS_VALUE "1") @@ -44,6 +47,15 @@ add_executable(vcpkg src/vcpkg.cpp ${VCPKGLIB_SOURCES}) target_compile_definitions(vcpkg PRIVATE -DDISABLE_METRICS=${DISABLE_METRICS_VALUE}) target_include_directories(vcpkg PRIVATE include) + +add_executable(vcpkg-test EXCLUDE_FROM_ALL ${VCPKGTEST_SOURCES} ${VCPKGLIB_SOURCES}) +target_compile_definitions(vcpkg-test PRIVATE -DDISABLE_METRICS=${DISABLE_METRICS_VALUE}) +target_include_directories(vcpkg-test PRIVATE include) + +foreach(TEST_NAME arguments chrono dependencies paragraph plan specifier supports) + add_test(${TEST_NAME} vcpkg-test [${TEST_NAME}]) +endforeach() + if(CLANG) include(CheckCXXSourceCompiles) check_cxx_source_compiles("#include @@ -57,8 +69,10 @@ endif() if(GCC OR (CLANG AND USES_LIBSTDCXX)) target_link_libraries(vcpkg PRIVATE stdc++fs) + target_link_libraries(vcpkg-test PRIVATE stdc++fs) elseif(CLANG) target_link_libraries(vcpkg PRIVATE c++fs) + target_link_libraries(vcpkg-test PRIVATE c++fs) endif() if(MSVC) diff --git a/toolsrc/include/tests.pch.h b/toolsrc/include/tests.pch.h deleted file mode 100644 index 5c00fca4a..000000000 --- a/toolsrc/include/tests.pch.h +++ /dev/null @@ -1,19 +0,0 @@ -#pragma once - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include diff --git a/toolsrc/include/tests.utils.h b/toolsrc/include/tests.utils.h deleted file mode 100644 index 3e8514e67..000000000 --- a/toolsrc/include/tests.utils.h +++ /dev/null @@ -1,76 +0,0 @@ -#pragma once - -#include - -#include -#include -#include -#include -#include - -#include - -namespace Microsoft::VisualStudio::CppUnitTestFramework -{ - template<> - inline std::wstring ToString(const vcpkg::Dependencies::InstallPlanType& t) - { - switch (t) - { - case vcpkg::Dependencies::InstallPlanType::ALREADY_INSTALLED: return L"ALREADY_INSTALLED"; - case vcpkg::Dependencies::InstallPlanType::BUILD_AND_INSTALL: return L"BUILD_AND_INSTALL"; - case vcpkg::Dependencies::InstallPlanType::EXCLUDED: return L"EXCLUDED"; - case vcpkg::Dependencies::InstallPlanType::UNKNOWN: return L"UNKNOWN"; - default: return ToString(static_cast(t)); - } - } - - template<> - inline std::wstring ToString(const vcpkg::Dependencies::RequestType& t) - { - switch (t) - { - case vcpkg::Dependencies::RequestType::AUTO_SELECTED: return L"AUTO_SELECTED"; - case vcpkg::Dependencies::RequestType::USER_REQUESTED: return L"USER_REQUESTED"; - case vcpkg::Dependencies::RequestType::UNKNOWN: return L"UNKNOWN"; - default: return ToString(static_cast(t)); - } - } - - template<> - inline std::wstring ToString(const vcpkg::PackageSpecParseResult& t) - { - return ToString(static_cast(t)); - } - - template<> - inline std::wstring ToString(const vcpkg::PackageSpec& t) - { - return ToString(t.to_string()); - } -} - -std::unique_ptr make_status_pgh(const char* name, - const char* depends = "", - const char* default_features = "", - const char* triplet = "x86-windows"); -std::unique_ptr make_status_feature_pgh(const char* name, - const char* feature, - const char* depends = "", - const char* triplet = "x86-windows"); - -template -T&& unwrap(vcpkg::ExpectedT&& p) -{ - Assert::IsTrue(p.has_value()); - return std::move(*p.get()); -} - -template -T&& unwrap(vcpkg::Optional&& opt) -{ - Assert::IsTrue(opt.has_value()); - return std::move(*opt.get()); -} - -vcpkg::PackageSpec unsafe_pspec(std::string name, vcpkg::Triplet t = vcpkg::Triplet::X86_WINDOWS); diff --git a/toolsrc/include/vcpkg-tests/catch.h b/toolsrc/include/vcpkg-tests/catch.h new file mode 100644 index 000000000..303f664ff --- /dev/null +++ b/toolsrc/include/vcpkg-tests/catch.h @@ -0,0 +1,16865 @@ +/* + * Catch v2.9.1 + * Generated: 2019-06-17 11:59:24.363643 + * ---------------------------------------------------------- + * This file has been merged from multiple headers. Please don't edit it directly + * Copyright (c) 2019 Two Blue Cubes Ltd. All rights reserved. + * + * Distributed under the Boost Software License, Version 1.0. (See accompanying + * file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + */ +#ifndef TWOBLUECUBES_SINGLE_INCLUDE_CATCH_HPP_INCLUDED +#define TWOBLUECUBES_SINGLE_INCLUDE_CATCH_HPP_INCLUDED +// start catch.hpp + + +#define CATCH_VERSION_MAJOR 2 +#define CATCH_VERSION_MINOR 9 +#define CATCH_VERSION_PATCH 1 + +#ifdef __clang__ +# pragma clang system_header +#elif defined __GNUC__ +# pragma GCC system_header +#endif + +// start catch_suppress_warnings.h + +#ifdef __clang__ +# ifdef __ICC // icpc defines the __clang__ macro +# pragma warning(push) +# pragma warning(disable: 161 1682) +# else // __ICC +# pragma clang diagnostic push +# pragma clang diagnostic ignored "-Wpadded" +# pragma clang diagnostic ignored "-Wswitch-enum" +# pragma clang diagnostic ignored "-Wcovered-switch-default" +# endif +#elif defined __GNUC__ + // Because REQUIREs trigger GCC's -Wparentheses, and because still + // supported version of g++ have only buggy support for _Pragmas, + // Wparentheses have to be suppressed globally. +# pragma GCC diagnostic ignored "-Wparentheses" // See #674 for details + +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wunused-variable" +# pragma GCC diagnostic ignored "-Wpadded" +#endif +// end catch_suppress_warnings.h +#if defined(CATCH_CONFIG_MAIN) || defined(CATCH_CONFIG_RUNNER) +# define CATCH_IMPL +# define CATCH_CONFIG_ALL_PARTS +#endif + +// In the impl file, we want to have access to all parts of the headers +// Can also be used to sanely support PCHs +#if defined(CATCH_CONFIG_ALL_PARTS) +# define CATCH_CONFIG_EXTERNAL_INTERFACES +# if defined(CATCH_CONFIG_DISABLE_MATCHERS) +# undef CATCH_CONFIG_DISABLE_MATCHERS +# endif +# if !defined(CATCH_CONFIG_ENABLE_CHRONO_STRINGMAKER) +# define CATCH_CONFIG_ENABLE_CHRONO_STRINGMAKER +# endif +#endif + +#if !defined(CATCH_CONFIG_IMPL_ONLY) +// start catch_platform.h + +#ifdef __APPLE__ +# include +# if TARGET_OS_OSX == 1 +# define CATCH_PLATFORM_MAC +# elif TARGET_OS_IPHONE == 1 +# define CATCH_PLATFORM_IPHONE +# endif + +#elif defined(linux) || defined(__linux) || defined(__linux__) +# define CATCH_PLATFORM_LINUX + +#elif defined(WIN32) || defined(__WIN32__) || defined(_WIN32) || defined(_MSC_VER) || defined(__MINGW32__) +# define CATCH_PLATFORM_WINDOWS +#endif + +// end catch_platform.h + +#ifdef CATCH_IMPL +# ifndef CLARA_CONFIG_MAIN +# define CLARA_CONFIG_MAIN_NOT_DEFINED +# define CLARA_CONFIG_MAIN +# endif +#endif + +// start catch_user_interfaces.h + +namespace Catch { + unsigned int rngSeed(); +} + +// end catch_user_interfaces.h +// start catch_tag_alias_autoregistrar.h + +// start catch_common.h + +// start catch_compiler_capabilities.h + +// Detect a number of compiler features - by compiler +// The following features are defined: +// +// CATCH_CONFIG_COUNTER : is the __COUNTER__ macro supported? +// CATCH_CONFIG_WINDOWS_SEH : is Windows SEH supported? +// CATCH_CONFIG_POSIX_SIGNALS : are POSIX signals supported? +// CATCH_CONFIG_DISABLE_EXCEPTIONS : Are exceptions enabled? +// **************** +// Note to maintainers: if new toggles are added please document them +// in configuration.md, too +// **************** + +// In general each macro has a _NO_ form +// (e.g. CATCH_CONFIG_NO_POSIX_SIGNALS) which disables the feature. +// Many features, at point of detection, define an _INTERNAL_ macro, so they +// can be combined, en-mass, with the _NO_ forms later. + +#ifdef __cplusplus + +# if (__cplusplus >= 201402L) || (defined(_MSVC_LANG) && _MSVC_LANG >= 201402L) +# define CATCH_CPP14_OR_GREATER +# endif + +# if (__cplusplus >= 201703L) || (defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) +# define CATCH_CPP17_OR_GREATER +# endif + +#endif + +#if defined(CATCH_CPP17_OR_GREATER) +# define CATCH_INTERNAL_CONFIG_CPP17_UNCAUGHT_EXCEPTIONS +#endif + +#ifdef __clang__ + +# define CATCH_INTERNAL_SUPPRESS_GLOBALS_WARNINGS \ + _Pragma( "clang diagnostic push" ) \ + _Pragma( "clang diagnostic ignored \"-Wexit-time-destructors\"" ) \ + _Pragma( "clang diagnostic ignored \"-Wglobal-constructors\"") +# define CATCH_INTERNAL_UNSUPPRESS_GLOBALS_WARNINGS \ + _Pragma( "clang diagnostic pop" ) + +# define CATCH_INTERNAL_SUPPRESS_PARENTHESES_WARNINGS \ + _Pragma( "clang diagnostic push" ) \ + _Pragma( "clang diagnostic ignored \"-Wparentheses\"" ) +# define CATCH_INTERNAL_UNSUPPRESS_PARENTHESES_WARNINGS \ + _Pragma( "clang diagnostic pop" ) + +# define CATCH_INTERNAL_SUPPRESS_UNUSED_WARNINGS \ + _Pragma( "clang diagnostic push" ) \ + _Pragma( "clang diagnostic ignored \"-Wunused-variable\"" ) +# define CATCH_INTERNAL_UNSUPPRESS_UNUSED_WARNINGS \ + _Pragma( "clang diagnostic pop" ) + +# define CATCH_INTERNAL_SUPPRESS_ZERO_VARIADIC_WARNINGS \ + _Pragma( "clang diagnostic push" ) \ + _Pragma( "clang diagnostic ignored \"-Wgnu-zero-variadic-macro-arguments\"" ) +# define CATCH_INTERNAL_UNSUPPRESS_ZERO_VARIADIC_WARNINGS \ + _Pragma( "clang diagnostic pop" ) + +#endif // __clang__ + +//////////////////////////////////////////////////////////////////////////////// +// Assume that non-Windows platforms support posix signals by default +#if !defined(CATCH_PLATFORM_WINDOWS) + #define CATCH_INTERNAL_CONFIG_POSIX_SIGNALS +#endif + +//////////////////////////////////////////////////////////////////////////////// +// We know some environments not to support full POSIX signals +#if defined(__CYGWIN__) || defined(__QNX__) || defined(__EMSCRIPTEN__) || defined(__DJGPP__) + #define CATCH_INTERNAL_CONFIG_NO_POSIX_SIGNALS +#endif + +#ifdef __OS400__ +# define CATCH_INTERNAL_CONFIG_NO_POSIX_SIGNALS +# define CATCH_CONFIG_COLOUR_NONE +#endif + +//////////////////////////////////////////////////////////////////////////////// +// Android somehow still does not support std::to_string +#if defined(__ANDROID__) +# define CATCH_INTERNAL_CONFIG_NO_CPP11_TO_STRING +#endif + +//////////////////////////////////////////////////////////////////////////////// +// Not all Windows environments support SEH properly +#if defined(__MINGW32__) +# define CATCH_INTERNAL_CONFIG_NO_WINDOWS_SEH +#endif + +//////////////////////////////////////////////////////////////////////////////// +// PS4 +#if defined(__ORBIS__) +# define CATCH_INTERNAL_CONFIG_NO_NEW_CAPTURE +#endif + +//////////////////////////////////////////////////////////////////////////////// +// Cygwin +#ifdef __CYGWIN__ + +// Required for some versions of Cygwin to declare gettimeofday +// see: http://stackoverflow.com/questions/36901803/gettimeofday-not-declared-in-this-scope-cygwin +# define _BSD_SOURCE +// some versions of cygwin (most) do not support std::to_string. Use the libstd check. +// https://gcc.gnu.org/onlinedocs/gcc-4.8.2/libstdc++/api/a01053_source.html line 2812-2813 +# if !((__cplusplus >= 201103L) && defined(_GLIBCXX_USE_C99) \ + && !defined(_GLIBCXX_HAVE_BROKEN_VSWPRINTF)) + +# define CATCH_INTERNAL_CONFIG_NO_CPP11_TO_STRING + +# endif +#endif // __CYGWIN__ + +//////////////////////////////////////////////////////////////////////////////// +// Visual C++ +#ifdef _MSC_VER + +# if _MSC_VER >= 1900 // Visual Studio 2015 or newer +# define CATCH_INTERNAL_CONFIG_CPP17_UNCAUGHT_EXCEPTIONS +# endif + +// Universal Windows platform does not support SEH +// Or console colours (or console at all...) +# if defined(WINAPI_FAMILY) && (WINAPI_FAMILY == WINAPI_FAMILY_APP) +# define CATCH_CONFIG_COLOUR_NONE +# else +# define CATCH_INTERNAL_CONFIG_WINDOWS_SEH +# endif + +// MSVC traditional preprocessor needs some workaround for __VA_ARGS__ +// _MSVC_TRADITIONAL == 0 means new conformant preprocessor +// _MSVC_TRADITIONAL == 1 means old traditional non-conformant preprocessor +# if !defined(_MSVC_TRADITIONAL) || (defined(_MSVC_TRADITIONAL) && _MSVC_TRADITIONAL) +# define CATCH_INTERNAL_CONFIG_TRADITIONAL_MSVC_PREPROCESSOR +# endif +#endif // _MSC_VER + +#if defined(_REENTRANT) || defined(_MSC_VER) +// Enable async processing, as -pthread is specified or no additional linking is required +# define CATCH_INTERNAL_CONFIG_USE_ASYNC +#endif // _MSC_VER + +//////////////////////////////////////////////////////////////////////////////// +// Check if we are compiled with -fno-exceptions or equivalent +#if defined(__EXCEPTIONS) || defined(__cpp_exceptions) || defined(_CPPUNWIND) +# define CATCH_INTERNAL_CONFIG_EXCEPTIONS_ENABLED +#endif + +//////////////////////////////////////////////////////////////////////////////// +// DJGPP +#ifdef __DJGPP__ +# define CATCH_INTERNAL_CONFIG_NO_WCHAR +#endif // __DJGPP__ + +//////////////////////////////////////////////////////////////////////////////// +// Embarcadero C++Build +#if defined(__BORLANDC__) + #define CATCH_INTERNAL_CONFIG_POLYFILL_ISNAN +#endif + +//////////////////////////////////////////////////////////////////////////////// + +// Use of __COUNTER__ is suppressed during code analysis in +// CLion/AppCode 2017.2.x and former, because __COUNTER__ is not properly +// handled by it. +// Otherwise all supported compilers support COUNTER macro, +// but user still might want to turn it off +#if ( !defined(__JETBRAINS_IDE__) || __JETBRAINS_IDE__ >= 20170300L ) + #define CATCH_INTERNAL_CONFIG_COUNTER +#endif + +//////////////////////////////////////////////////////////////////////////////// +// Check if string_view is available and usable +// The check is split apart to work around v140 (VS2015) preprocessor issue... +#if defined(__has_include) +#if __has_include() && defined(CATCH_CPP17_OR_GREATER) +# define CATCH_INTERNAL_CONFIG_CPP17_STRING_VIEW +#endif +#endif + +//////////////////////////////////////////////////////////////////////////////// +// Check if optional is available and usable +#if defined(__has_include) +# if __has_include() && defined(CATCH_CPP17_OR_GREATER) +# define CATCH_INTERNAL_CONFIG_CPP17_OPTIONAL +# endif // __has_include() && defined(CATCH_CPP17_OR_GREATER) +#endif // __has_include + +//////////////////////////////////////////////////////////////////////////////// +// Check if variant is available and usable +#if defined(__has_include) +# if __has_include() && defined(CATCH_CPP17_OR_GREATER) +# if defined(__clang__) && (__clang_major__ < 8) + // work around clang bug with libstdc++ https://bugs.llvm.org/show_bug.cgi?id=31852 + // fix should be in clang 8, workaround in libstdc++ 8.2 +# include +# if defined(__GLIBCXX__) && defined(_GLIBCXX_RELEASE) && (_GLIBCXX_RELEASE < 9) +# define CATCH_CONFIG_NO_CPP17_VARIANT +# else +# define CATCH_INTERNAL_CONFIG_CPP17_VARIANT +# endif // defined(__GLIBCXX__) && defined(_GLIBCXX_RELEASE) && (_GLIBCXX_RELEASE < 9) +# else +# define CATCH_INTERNAL_CONFIG_CPP17_VARIANT +# endif // defined(__clang__) && (__clang_major__ < 8) +# endif // __has_include() && defined(CATCH_CPP17_OR_GREATER) +#endif // __has_include + +#if defined(CATCH_INTERNAL_CONFIG_COUNTER) && !defined(CATCH_CONFIG_NO_COUNTER) && !defined(CATCH_CONFIG_COUNTER) +# define CATCH_CONFIG_COUNTER +#endif +#if defined(CATCH_INTERNAL_CONFIG_WINDOWS_SEH) && !defined(CATCH_CONFIG_NO_WINDOWS_SEH) && !defined(CATCH_CONFIG_WINDOWS_SEH) && !defined(CATCH_INTERNAL_CONFIG_NO_WINDOWS_SEH) +# define CATCH_CONFIG_WINDOWS_SEH +#endif +// This is set by default, because we assume that unix compilers are posix-signal-compatible by default. +#if defined(CATCH_INTERNAL_CONFIG_POSIX_SIGNALS) && !defined(CATCH_INTERNAL_CONFIG_NO_POSIX_SIGNALS) && !defined(CATCH_CONFIG_NO_POSIX_SIGNALS) && !defined(CATCH_CONFIG_POSIX_SIGNALS) +# define CATCH_CONFIG_POSIX_SIGNALS +#endif +// This is set by default, because we assume that compilers with no wchar_t support are just rare exceptions. +#if !defined(CATCH_INTERNAL_CONFIG_NO_WCHAR) && !defined(CATCH_CONFIG_NO_WCHAR) && !defined(CATCH_CONFIG_WCHAR) +# define CATCH_CONFIG_WCHAR +#endif + +#if !defined(CATCH_INTERNAL_CONFIG_NO_CPP11_TO_STRING) && !defined(CATCH_CONFIG_NO_CPP11_TO_STRING) && !defined(CATCH_CONFIG_CPP11_TO_STRING) +# define CATCH_CONFIG_CPP11_TO_STRING +#endif + +#if defined(CATCH_INTERNAL_CONFIG_CPP17_OPTIONAL) && !defined(CATCH_CONFIG_NO_CPP17_OPTIONAL) && !defined(CATCH_CONFIG_CPP17_OPTIONAL) +# define CATCH_CONFIG_CPP17_OPTIONAL +#endif + +#if defined(CATCH_INTERNAL_CONFIG_CPP17_UNCAUGHT_EXCEPTIONS) && !defined(CATCH_CONFIG_NO_CPP17_UNCAUGHT_EXCEPTIONS) && !defined(CATCH_CONFIG_CPP17_UNCAUGHT_EXCEPTIONS) +# define CATCH_CONFIG_CPP17_UNCAUGHT_EXCEPTIONS +#endif + +#if defined(CATCH_INTERNAL_CONFIG_CPP17_STRING_VIEW) && !defined(CATCH_CONFIG_NO_CPP17_STRING_VIEW) && !defined(CATCH_CONFIG_CPP17_STRING_VIEW) +# define CATCH_CONFIG_CPP17_STRING_VIEW +#endif + +#if defined(CATCH_INTERNAL_CONFIG_CPP17_VARIANT) && !defined(CATCH_CONFIG_NO_CPP17_VARIANT) && !defined(CATCH_CONFIG_CPP17_VARIANT) +# define CATCH_CONFIG_CPP17_VARIANT +#endif + +#if defined(CATCH_CONFIG_EXPERIMENTAL_REDIRECT) +# define CATCH_INTERNAL_CONFIG_NEW_CAPTURE +#endif + +#if defined(CATCH_INTERNAL_CONFIG_NEW_CAPTURE) && !defined(CATCH_INTERNAL_CONFIG_NO_NEW_CAPTURE) && !defined(CATCH_CONFIG_NO_NEW_CAPTURE) && !defined(CATCH_CONFIG_NEW_CAPTURE) +# define CATCH_CONFIG_NEW_CAPTURE +#endif + +#if !defined(CATCH_INTERNAL_CONFIG_EXCEPTIONS_ENABLED) && !defined(CATCH_CONFIG_DISABLE_EXCEPTIONS) +# define CATCH_CONFIG_DISABLE_EXCEPTIONS +#endif + +#if defined(CATCH_INTERNAL_CONFIG_POLYFILL_ISNAN) && !defined(CATCH_CONFIG_NO_POLYFILL_ISNAN) && !defined(CATCH_CONFIG_POLYFILL_ISNAN) +# define CATCH_CONFIG_POLYFILL_ISNAN +#endif + +#if defined(CATCH_INTERNAL_CONFIG_USE_ASYNC) && !defined(CATCH_CONFIG_NO_USE_ASYNC) && !defined(CATCH_CONFIG_USE_ASYNC) +# define CATCH_CONFIG_USE_ASYNC +#endif + +#if !defined(CATCH_INTERNAL_SUPPRESS_PARENTHESES_WARNINGS) +# define CATCH_INTERNAL_SUPPRESS_PARENTHESES_WARNINGS +# define CATCH_INTERNAL_UNSUPPRESS_PARENTHESES_WARNINGS +#endif +#if !defined(CATCH_INTERNAL_SUPPRESS_GLOBALS_WARNINGS) +# define CATCH_INTERNAL_SUPPRESS_GLOBALS_WARNINGS +# define CATCH_INTERNAL_UNSUPPRESS_GLOBALS_WARNINGS +#endif +#if !defined(CATCH_INTERNAL_SUPPRESS_UNUSED_WARNINGS) +# define CATCH_INTERNAL_SUPPRESS_UNUSED_WARNINGS +# define CATCH_INTERNAL_UNSUPPRESS_UNUSED_WARNINGS +#endif +#if !defined(CATCH_INTERNAL_SUPPRESS_ZERO_VARIADIC_WARNINGS) +# define CATCH_INTERNAL_SUPPRESS_ZERO_VARIADIC_WARNINGS +# define CATCH_INTERNAL_UNSUPPRESS_ZERO_VARIADIC_WARNINGS +#endif + +#if defined(CATCH_CONFIG_DISABLE_EXCEPTIONS) +#define CATCH_TRY if ((true)) +#define CATCH_CATCH_ALL if ((false)) +#define CATCH_CATCH_ANON(type) if ((false)) +#else +#define CATCH_TRY try +#define CATCH_CATCH_ALL catch (...) +#define CATCH_CATCH_ANON(type) catch (type) +#endif + +#if defined(CATCH_INTERNAL_CONFIG_TRADITIONAL_MSVC_PREPROCESSOR) && !defined(CATCH_CONFIG_NO_TRADITIONAL_MSVC_PREPROCESSOR) && !defined(CATCH_CONFIG_TRADITIONAL_MSVC_PREPROCESSOR) +#define CATCH_CONFIG_TRADITIONAL_MSVC_PREPROCESSOR +#endif + +// end catch_compiler_capabilities.h +#define INTERNAL_CATCH_UNIQUE_NAME_LINE2( name, line ) name##line +#define INTERNAL_CATCH_UNIQUE_NAME_LINE( name, line ) INTERNAL_CATCH_UNIQUE_NAME_LINE2( name, line ) +#ifdef CATCH_CONFIG_COUNTER +# define INTERNAL_CATCH_UNIQUE_NAME( name ) INTERNAL_CATCH_UNIQUE_NAME_LINE( name, __COUNTER__ ) +#else +# define INTERNAL_CATCH_UNIQUE_NAME( name ) INTERNAL_CATCH_UNIQUE_NAME_LINE( name, __LINE__ ) +#endif + +#include +#include +#include + +// We need a dummy global operator<< so we can bring it into Catch namespace later +struct Catch_global_namespace_dummy {}; +std::ostream& operator<<(std::ostream&, Catch_global_namespace_dummy); + +namespace Catch { + + struct CaseSensitive { enum Choice { + Yes, + No + }; }; + + class NonCopyable { + NonCopyable( NonCopyable const& ) = delete; + NonCopyable( NonCopyable && ) = delete; + NonCopyable& operator = ( NonCopyable const& ) = delete; + NonCopyable& operator = ( NonCopyable && ) = delete; + + protected: + NonCopyable(); + virtual ~NonCopyable(); + }; + + struct SourceLineInfo { + + SourceLineInfo() = delete; + SourceLineInfo( char const* _file, std::size_t _line ) noexcept + : file( _file ), + line( _line ) + {} + + SourceLineInfo( SourceLineInfo const& other ) = default; + SourceLineInfo& operator = ( SourceLineInfo const& ) = default; + SourceLineInfo( SourceLineInfo&& ) noexcept = default; + SourceLineInfo& operator = ( SourceLineInfo&& ) noexcept = default; + + bool empty() const noexcept; + bool operator == ( SourceLineInfo const& other ) const noexcept; + bool operator < ( SourceLineInfo const& other ) const noexcept; + + char const* file; + std::size_t line; + }; + + std::ostream& operator << ( std::ostream& os, SourceLineInfo const& info ); + + // Bring in operator<< from global namespace into Catch namespace + // This is necessary because the overload of operator<< above makes + // lookup stop at namespace Catch + using ::operator<<; + + // Use this in variadic streaming macros to allow + // >> +StreamEndStop + // as well as + // >> stuff +StreamEndStop + struct StreamEndStop { + std::string operator+() const; + }; + template + T const& operator + ( T const& value, StreamEndStop ) { + return value; + } +} + +#define CATCH_INTERNAL_LINEINFO \ + ::Catch::SourceLineInfo( __FILE__, static_cast( __LINE__ ) ) + +// end catch_common.h +namespace Catch { + + struct RegistrarForTagAliases { + RegistrarForTagAliases( char const* alias, char const* tag, SourceLineInfo const& lineInfo ); + }; + +} // end namespace Catch + +#define CATCH_REGISTER_TAG_ALIAS( alias, spec ) \ + CATCH_INTERNAL_SUPPRESS_GLOBALS_WARNINGS \ + namespace{ Catch::RegistrarForTagAliases INTERNAL_CATCH_UNIQUE_NAME( AutoRegisterTagAlias )( alias, spec, CATCH_INTERNAL_LINEINFO ); } \ + CATCH_INTERNAL_UNSUPPRESS_GLOBALS_WARNINGS + +// end catch_tag_alias_autoregistrar.h +// start catch_test_registry.h + +// start catch_interfaces_testcase.h + +#include + +namespace Catch { + + class TestSpec; + + struct ITestInvoker { + virtual void invoke () const = 0; + virtual ~ITestInvoker(); + }; + + class TestCase; + struct IConfig; + + struct ITestCaseRegistry { + virtual ~ITestCaseRegistry(); + virtual std::vector const& getAllTests() const = 0; + virtual std::vector const& getAllTestsSorted( IConfig const& config ) const = 0; + }; + + bool matchTest( TestCase const& testCase, TestSpec const& testSpec, IConfig const& config ); + std::vector filterTests( std::vector const& testCases, TestSpec const& testSpec, IConfig const& config ); + std::vector const& getAllTestCasesSorted( IConfig const& config ); + +} + +// end catch_interfaces_testcase.h +// start catch_stringref.h + +#include +#include +#include + +namespace Catch { + + /// A non-owning string class (similar to the forthcoming std::string_view) + /// Note that, because a StringRef may be a substring of another string, + /// it may not be null terminated. c_str() must return a null terminated + /// string, however, and so the StringRef will internally take ownership + /// (taking a copy), if necessary. In theory this ownership is not externally + /// visible - but it does mean (substring) StringRefs should not be shared between + /// threads. + class StringRef { + public: + using size_type = std::size_t; + + private: + friend struct StringRefTestAccess; + + char const* m_start; + size_type m_size; + + char* m_data = nullptr; + + void takeOwnership(); + + static constexpr char const* const s_empty = ""; + + public: // construction/ assignment + StringRef() noexcept + : StringRef( s_empty, 0 ) + {} + + StringRef( StringRef const& other ) noexcept + : m_start( other.m_start ), + m_size( other.m_size ) + {} + + StringRef( StringRef&& other ) noexcept + : m_start( other.m_start ), + m_size( other.m_size ), + m_data( other.m_data ) + { + other.m_data = nullptr; + } + + StringRef( char const* rawChars ) noexcept; + + StringRef( char const* rawChars, size_type size ) noexcept + : m_start( rawChars ), + m_size( size ) + {} + + StringRef( std::string const& stdString ) noexcept + : m_start( stdString.c_str() ), + m_size( stdString.size() ) + {} + + ~StringRef() noexcept { + delete[] m_data; + } + + auto operator = ( StringRef const &other ) noexcept -> StringRef& { + delete[] m_data; + m_data = nullptr; + m_start = other.m_start; + m_size = other.m_size; + return *this; + } + + operator std::string() const; + + void swap( StringRef& other ) noexcept; + + public: // operators + auto operator == ( StringRef const& other ) const noexcept -> bool; + auto operator != ( StringRef const& other ) const noexcept -> bool; + + auto operator[] ( size_type index ) const noexcept -> char; + + public: // named queries + auto empty() const noexcept -> bool { + return m_size == 0; + } + auto size() const noexcept -> size_type { + return m_size; + } + + auto numberOfCharacters() const noexcept -> size_type; + auto c_str() const -> char const*; + + public: // substrings and searches + auto substr( size_type start, size_type size ) const noexcept -> StringRef; + + // Returns the current start pointer. + // Note that the pointer can change when if the StringRef is a substring + auto currentData() const noexcept -> char const*; + + private: // ownership queries - may not be consistent between calls + auto isOwned() const noexcept -> bool; + auto isSubstring() const noexcept -> bool; + }; + + auto operator + ( StringRef const& lhs, StringRef const& rhs ) -> std::string; + auto operator + ( StringRef const& lhs, char const* rhs ) -> std::string; + auto operator + ( char const* lhs, StringRef const& rhs ) -> std::string; + + auto operator += ( std::string& lhs, StringRef const& sr ) -> std::string&; + auto operator << ( std::ostream& os, StringRef const& sr ) -> std::ostream&; + + inline auto operator "" _sr( char const* rawChars, std::size_t size ) noexcept -> StringRef { + return StringRef( rawChars, size ); + } + +} // namespace Catch + +inline auto operator "" _catch_sr( char const* rawChars, std::size_t size ) noexcept -> Catch::StringRef { + return Catch::StringRef( rawChars, size ); +} + +// end catch_stringref.h +// start catch_type_traits.hpp + + +#include + +namespace Catch{ + +#ifdef CATCH_CPP17_OR_GREATER + template + inline constexpr auto is_unique = std::true_type{}; + + template + inline constexpr auto is_unique = std::bool_constant< + (!std::is_same_v && ...) && is_unique + >{}; +#else + +template +struct is_unique : std::true_type{}; + +template +struct is_unique : std::integral_constant +::value + && is_unique::value + && is_unique::value +>{}; + +#endif +} + +// end catch_type_traits.hpp +// start catch_preprocessor.hpp + + +#define CATCH_RECURSION_LEVEL0(...) __VA_ARGS__ +#define CATCH_RECURSION_LEVEL1(...) CATCH_RECURSION_LEVEL0(CATCH_RECURSION_LEVEL0(CATCH_RECURSION_LEVEL0(__VA_ARGS__))) +#define CATCH_RECURSION_LEVEL2(...) CATCH_RECURSION_LEVEL1(CATCH_RECURSION_LEVEL1(CATCH_RECURSION_LEVEL1(__VA_ARGS__))) +#define CATCH_RECURSION_LEVEL3(...) CATCH_RECURSION_LEVEL2(CATCH_RECURSION_LEVEL2(CATCH_RECURSION_LEVEL2(__VA_ARGS__))) +#define CATCH_RECURSION_LEVEL4(...) CATCH_RECURSION_LEVEL3(CATCH_RECURSION_LEVEL3(CATCH_RECURSION_LEVEL3(__VA_ARGS__))) +#define CATCH_RECURSION_LEVEL5(...) CATCH_RECURSION_LEVEL4(CATCH_RECURSION_LEVEL4(CATCH_RECURSION_LEVEL4(__VA_ARGS__))) + +#ifdef CATCH_CONFIG_TRADITIONAL_MSVC_PREPROCESSOR +#define INTERNAL_CATCH_EXPAND_VARGS(...) __VA_ARGS__ +// MSVC needs more evaluations +#define CATCH_RECURSION_LEVEL6(...) CATCH_RECURSION_LEVEL5(CATCH_RECURSION_LEVEL5(CATCH_RECURSION_LEVEL5(__VA_ARGS__))) +#define CATCH_RECURSE(...) CATCH_RECURSION_LEVEL6(CATCH_RECURSION_LEVEL6(__VA_ARGS__)) +#else +#define CATCH_RECURSE(...) CATCH_RECURSION_LEVEL5(__VA_ARGS__) +#endif + +#define CATCH_REC_END(...) +#define CATCH_REC_OUT + +#define CATCH_EMPTY() +#define CATCH_DEFER(id) id CATCH_EMPTY() + +#define CATCH_REC_GET_END2() 0, CATCH_REC_END +#define CATCH_REC_GET_END1(...) CATCH_REC_GET_END2 +#define CATCH_REC_GET_END(...) CATCH_REC_GET_END1 +#define CATCH_REC_NEXT0(test, next, ...) next CATCH_REC_OUT +#define CATCH_REC_NEXT1(test, next) CATCH_DEFER ( CATCH_REC_NEXT0 ) ( test, next, 0) +#define CATCH_REC_NEXT(test, next) CATCH_REC_NEXT1(CATCH_REC_GET_END test, next) + +#define CATCH_REC_LIST0(f, x, peek, ...) , f(x) CATCH_DEFER ( CATCH_REC_NEXT(peek, CATCH_REC_LIST1) ) ( f, peek, __VA_ARGS__ ) +#define CATCH_REC_LIST1(f, x, peek, ...) , f(x) CATCH_DEFER ( CATCH_REC_NEXT(peek, CATCH_REC_LIST0) ) ( f, peek, __VA_ARGS__ ) +#define CATCH_REC_LIST2(f, x, peek, ...) f(x) CATCH_DEFER ( CATCH_REC_NEXT(peek, CATCH_REC_LIST1) ) ( f, peek, __VA_ARGS__ ) + +#define CATCH_REC_LIST0_UD(f, userdata, x, peek, ...) , f(userdata, x) CATCH_DEFER ( CATCH_REC_NEXT(peek, CATCH_REC_LIST1_UD) ) ( f, userdata, peek, __VA_ARGS__ ) +#define CATCH_REC_LIST1_UD(f, userdata, x, peek, ...) , f(userdata, x) CATCH_DEFER ( CATCH_REC_NEXT(peek, CATCH_REC_LIST0_UD) ) ( f, userdata, peek, __VA_ARGS__ ) +#define CATCH_REC_LIST2_UD(f, userdata, x, peek, ...) f(userdata, x) CATCH_DEFER ( CATCH_REC_NEXT(peek, CATCH_REC_LIST1_UD) ) ( f, userdata, peek, __VA_ARGS__ ) + +// Applies the function macro `f` to each of the remaining parameters, inserts commas between the results, +// and passes userdata as the first parameter to each invocation, +// e.g. CATCH_REC_LIST_UD(f, x, a, b, c) evaluates to f(x, a), f(x, b), f(x, c) +#define CATCH_REC_LIST_UD(f, userdata, ...) CATCH_RECURSE(CATCH_REC_LIST2_UD(f, userdata, __VA_ARGS__, ()()(), ()()(), ()()(), 0)) + +#define CATCH_REC_LIST(f, ...) CATCH_RECURSE(CATCH_REC_LIST2(f, __VA_ARGS__, ()()(), ()()(), ()()(), 0)) + +#define INTERNAL_CATCH_EXPAND1(param) INTERNAL_CATCH_EXPAND2(param) +#define INTERNAL_CATCH_EXPAND2(...) INTERNAL_CATCH_NO## __VA_ARGS__ +#define INTERNAL_CATCH_DEF(...) INTERNAL_CATCH_DEF __VA_ARGS__ +#define INTERNAL_CATCH_NOINTERNAL_CATCH_DEF +#define INTERNAL_CATCH_STRINGIZE(...) INTERNAL_CATCH_STRINGIZE2(__VA_ARGS__) +#ifndef CATCH_CONFIG_TRADITIONAL_MSVC_PREPROCESSOR +#define INTERNAL_CATCH_STRINGIZE2(...) #__VA_ARGS__ +#define INTERNAL_CATCH_STRINGIZE_WITHOUT_PARENS(param) INTERNAL_CATCH_STRINGIZE(INTERNAL_CATCH_REMOVE_PARENS(param)) +#else +// MSVC is adding extra space and needs another indirection to expand INTERNAL_CATCH_NOINTERNAL_CATCH_DEF +#define INTERNAL_CATCH_STRINGIZE2(...) INTERNAL_CATCH_STRINGIZE3(__VA_ARGS__) +#define INTERNAL_CATCH_STRINGIZE3(...) #__VA_ARGS__ +#define INTERNAL_CATCH_STRINGIZE_WITHOUT_PARENS(param) (INTERNAL_CATCH_STRINGIZE(INTERNAL_CATCH_REMOVE_PARENS(param)) + 1) +#endif + +#define INTERNAL_CATCH_MAKE_NAMESPACE2(...) ns_##__VA_ARGS__ +#define INTERNAL_CATCH_MAKE_NAMESPACE(name) INTERNAL_CATCH_MAKE_NAMESPACE2(name) + +#define INTERNAL_CATCH_REMOVE_PARENS(...) INTERNAL_CATCH_EXPAND1(INTERNAL_CATCH_DEF __VA_ARGS__) + +#ifndef CATCH_CONFIG_TRADITIONAL_MSVC_PREPROCESSOR +#define INTERNAL_CATCH_MAKE_TYPE_LIST2(...) decltype(get_wrapper()) +#define INTERNAL_CATCH_MAKE_TYPE_LIST(...) INTERNAL_CATCH_MAKE_TYPE_LIST2(INTERNAL_CATCH_REMOVE_PARENS(__VA_ARGS__)) +#else +#define INTERNAL_CATCH_MAKE_TYPE_LIST2(...) INTERNAL_CATCH_EXPAND_VARGS(decltype(get_wrapper())) +#define INTERNAL_CATCH_MAKE_TYPE_LIST(...) INTERNAL_CATCH_EXPAND_VARGS(INTERNAL_CATCH_MAKE_TYPE_LIST2(INTERNAL_CATCH_REMOVE_PARENS(__VA_ARGS__))) +#endif + +#define INTERNAL_CATCH_MAKE_TYPE_LISTS_FROM_TYPES(...)\ + CATCH_REC_LIST(INTERNAL_CATCH_MAKE_TYPE_LIST,__VA_ARGS__) + +#define INTERNAL_CATCH_REMOVE_PARENS_1_ARG(_0) INTERNAL_CATCH_REMOVE_PARENS(_0) +#define INTERNAL_CATCH_REMOVE_PARENS_2_ARG(_0, _1) INTERNAL_CATCH_REMOVE_PARENS(_0), INTERNAL_CATCH_REMOVE_PARENS_1_ARG(_1) +#define INTERNAL_CATCH_REMOVE_PARENS_3_ARG(_0, _1, _2) INTERNAL_CATCH_REMOVE_PARENS(_0), INTERNAL_CATCH_REMOVE_PARENS_2_ARG(_1, _2) +#define INTERNAL_CATCH_REMOVE_PARENS_4_ARG(_0, _1, _2, _3) INTERNAL_CATCH_REMOVE_PARENS(_0), INTERNAL_CATCH_REMOVE_PARENS_3_ARG(_1, _2, _3) +#define INTERNAL_CATCH_REMOVE_PARENS_5_ARG(_0, _1, _2, _3, _4) INTERNAL_CATCH_REMOVE_PARENS(_0), INTERNAL_CATCH_REMOVE_PARENS_4_ARG(_1, _2, _3, _4) +#define INTERNAL_CATCH_REMOVE_PARENS_6_ARG(_0, _1, _2, _3, _4, _5) INTERNAL_CATCH_REMOVE_PARENS(_0), INTERNAL_CATCH_REMOVE_PARENS_5_ARG(_1, _2, _3, _4, _5) +#define INTERNAL_CATCH_REMOVE_PARENS_7_ARG(_0, _1, _2, _3, _4, _5, _6) INTERNAL_CATCH_REMOVE_PARENS(_0), INTERNAL_CATCH_REMOVE_PARENS_6_ARG(_1, _2, _4, _5, _6) +#define INTERNAL_CATCH_REMOVE_PARENS_8_ARG(_0, _1, _2, _3, _4, _5, _6, _7) INTERNAL_CATCH_REMOVE_PARENS(_0), INTERNAL_CATCH_REMOVE_PARENS_7_ARG(_1, _2, _3, _4, _5, _6, _7) +#define INTERNAL_CATCH_REMOVE_PARENS_9_ARG(_0, _1, _2, _3, _4, _5, _6, _7, _8) INTERNAL_CATCH_REMOVE_PARENS(_0), INTERNAL_CATCH_REMOVE_PARENS_8_ARG(_1, _2, _3, _4, _5, _6, _7, _8) +#define INTERNAL_CATCH_REMOVE_PARENS_10_ARG(_0, _1, _2, _3, _4, _5, _6, _7, _8, _9) INTERNAL_CATCH_REMOVE_PARENS(_0), INTERNAL_CATCH_REMOVE_PARENS_9_ARG(_1, _2, _3, _4, _5, _6, _7, _8, _9) +#define INTERNAL_CATCH_REMOVE_PARENS_11_ARG(_0, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10) INTERNAL_CATCH_REMOVE_PARENS(_0), INTERNAL_CATCH_REMOVE_PARENS_10_ARG(_1, _2, _3, _4, _5, _6, _7, _8, _9, _10) + +#define INTERNAL_CATCH_VA_NARGS_IMPL(_0, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, N, ...) N + +#define INTERNAL_CATCH_TYPE_GEN\ + template struct TypeList {};\ + template\ + constexpr auto get_wrapper() noexcept -> TypeList { return {}; }\ + \ + template class L1, typename...E1, template class L2, typename...E2> \ + constexpr auto append(L1, L2) noexcept -> L1 { return {}; }\ + template< template class L1, typename...E1, template class L2, typename...E2, typename...Rest>\ + constexpr auto append(L1, L2, Rest...) noexcept -> decltype(append(L1{}, Rest{}...)) { return {}; }\ + template< template class L1, typename...E1, typename...Rest>\ + constexpr auto append(L1, TypeList, Rest...) noexcept -> L1 { return {}; }\ + \ + template< template class Container, template class List, typename...elems>\ + constexpr auto rewrap(List) noexcept -> TypeList> { return {}; }\ + template< template class Container, template class List, class...Elems, typename...Elements>\ + constexpr auto rewrap(List,Elements...) noexcept -> decltype(append(TypeList>{}, rewrap(Elements{}...))) { return {}; }\ + \ + template