diff options
Diffstat (limited to 'toolsrc/src')
| -rw-r--r-- | toolsrc/src/tests.plan.cpp | 84 | ||||
| -rw-r--r-- | toolsrc/src/vcpkg/base/system.cpp | 10 | ||||
| -rw-r--r-- | toolsrc/src/vcpkg/build.cpp | 44 | ||||
| -rw-r--r-- | toolsrc/src/vcpkg/commands.ci.cpp | 138 | ||||
| -rw-r--r-- | toolsrc/src/vcpkg/dependencies.cpp | 11 | ||||
| -rw-r--r-- | toolsrc/src/vcpkg/install.cpp | 59 | ||||
| -rw-r--r-- | toolsrc/src/vcpkg/sourceparagraph.cpp | 10 |
7 files changed, 289 insertions, 67 deletions
diff --git a/toolsrc/src/tests.plan.cpp b/toolsrc/src/tests.plan.cpp index 95056810c..a99f1abb9 100644 --- a/toolsrc/src/tests.plan.cpp +++ b/toolsrc/src/tests.plan.cpp @@ -593,6 +593,87 @@ namespace UnitTest1 features_check(&install_plan[0], "a", {"core"}, Triplet::X64_WINDOWS); } + TEST_METHOD(install_plan_action_dependencies) + { + std::vector<std::unique_ptr<StatusParagraph>> status_paragraphs; + + // Add a port "a" which depends on the core of "b", which was already + // installed explicitly + PackageSpecMap spec_map(Triplet::X64_WINDOWS); + auto spec_c = spec_map.emplace("c"); + auto spec_b = spec_map.emplace("b", "c"); + spec_map.emplace("a", "b"); + + // Install "a" (without explicit feature specification) + auto install_specs = FullPackageSpec::from_string("a", Triplet::X64_WINDOWS); + auto install_plan = Dependencies::create_feature_install_plan( + spec_map.map, + FullPackageSpec::to_feature_specs({install_specs.value_or_exit(VCPKG_LINE_INFO)}), + StatusParagraphs(std::move(status_paragraphs))); + + Assert::IsTrue(install_plan.size() == 3); + features_check(&install_plan[0], "c", {"core"}, Triplet::X64_WINDOWS); + + features_check(&install_plan[1], "b", {"core"}, Triplet::X64_WINDOWS); + Assert::IsTrue(install_plan[1].install_action.get()->computed_dependencies == + std::vector<PackageSpec>{spec_c}); + + features_check(&install_plan[2], "a", {"core"}, Triplet::X64_WINDOWS); + Assert::IsTrue(install_plan[2].install_action.get()->computed_dependencies == + std::vector<PackageSpec>{spec_b}); + } + + TEST_METHOD(install_plan_action_dependencies_2) + { + std::vector<std::unique_ptr<StatusParagraph>> status_paragraphs; + + // Add a port "a" which depends on the core of "b", which was already + // installed explicitly + PackageSpecMap spec_map(Triplet::X64_WINDOWS); + auto spec_c = spec_map.emplace("c"); + auto spec_b = spec_map.emplace("b", "c"); + spec_map.emplace("a", "c, b"); + + // Install "a" (without explicit feature specification) + auto install_specs = FullPackageSpec::from_string("a", Triplet::X64_WINDOWS); + auto install_plan = Dependencies::create_feature_install_plan( + spec_map.map, + FullPackageSpec::to_feature_specs({install_specs.value_or_exit(VCPKG_LINE_INFO)}), + StatusParagraphs(std::move(status_paragraphs))); + + Assert::IsTrue(install_plan.size() == 3); + features_check(&install_plan[0], "c", {"core"}, Triplet::X64_WINDOWS); + + features_check(&install_plan[1], "b", {"core"}, Triplet::X64_WINDOWS); + Assert::IsTrue(install_plan[1].install_action.get()->computed_dependencies == + std::vector<PackageSpec>{spec_c}); + + features_check(&install_plan[2], "a", {"core"}, Triplet::X64_WINDOWS); + Assert::IsTrue(install_plan[2].install_action.get()->computed_dependencies == + std::vector<PackageSpec>{spec_b, spec_c}); + } + + TEST_METHOD(install_plan_action_dependencies_3) + { + std::vector<std::unique_ptr<StatusParagraph>> status_paragraphs; + + // Add a port "a" which depends on the core of "b", which was already + // installed explicitly + PackageSpecMap spec_map(Triplet::X64_WINDOWS); + spec_map.emplace("a", "", {{"0", ""}, {"1", "a[0]"}}, {"1"}); + + // Install "a" (without explicit feature specification) + auto install_specs = FullPackageSpec::from_string("a", Triplet::X64_WINDOWS); + auto install_plan = Dependencies::create_feature_install_plan( + spec_map.map, + FullPackageSpec::to_feature_specs({install_specs.value_or_exit(VCPKG_LINE_INFO)}), + StatusParagraphs(std::move(status_paragraphs))); + + Assert::IsTrue(install_plan.size() == 1); + features_check(&install_plan[0], "a", {"1", "0", "core"}, Triplet::X64_WINDOWS); + Assert::IsTrue(install_plan[0].install_action.get()->computed_dependencies == std::vector<PackageSpec>{}); + } + TEST_METHOD(upgrade_with_default_features_1) { std::vector<std::unique_ptr<StatusParagraph>> pghs; @@ -619,7 +700,6 @@ namespace UnitTest1 Assert::IsTrue(plan[0].remove_action.has_value()); Assert::AreEqual("a", plan[1].spec().name().c_str()); - Assert::IsTrue(plan[1].install_action.has_value()); features_check(&plan[1], "a", {"core", "0"}, Triplet::X86_WINDOWS); } @@ -651,11 +731,9 @@ namespace UnitTest1 Assert::IsTrue(plan[0].remove_action.has_value()); Assert::AreEqual("b", plan[1].spec().name().c_str()); - Assert::IsTrue(plan[1].install_action.has_value()); features_check(&plan[1], "b", {"b0", "core"}, Triplet::X64_WINDOWS); Assert::AreEqual("a", plan[2].spec().name().c_str()); - Assert::IsTrue(plan[2].install_action.has_value()); features_check(&plan[2], "a", {"core"}, Triplet::X64_WINDOWS); } diff --git a/toolsrc/src/vcpkg/base/system.cpp b/toolsrc/src/vcpkg/base/system.cpp index f339afa51..a315c7880 100644 --- a/toolsrc/src/vcpkg/base/system.cpp +++ b/toolsrc/src/vcpkg/base/system.cpp @@ -314,6 +314,8 @@ namespace vcpkg::System // Flush stdout before launching external process fflush(stdout); + auto timer = Chrono::ElapsedTimer::create_started(); + #if defined(_WIN32) const auto actual_cmd_line = Strings::format(R"###("%s 2>&1")###", cmd_line); @@ -335,8 +337,10 @@ namespace vcpkg::System } const auto ec = _pclose(pipe); - Debug::println("_pclose() returned %d", ec); remove_byte_order_marks(&output); + + Debug::println("_pclose() returned %d after %8d us", ec, (int)timer.microseconds()); + return {ec, Strings::to_utf8(output)}; #else const auto actual_cmd_line = Strings::format(R"###(%s 2>&1)###", cmd_line); @@ -359,7 +363,9 @@ namespace vcpkg::System } const auto ec = pclose(pipe); - Debug::println("pclose() returned %d", ec); + + Debug::println("_pclose() returned %d after %8d us", ec, (int)timer.microseconds()); + return {ec, output}; #endif } diff --git a/toolsrc/src/vcpkg/build.cpp b/toolsrc/src/vcpkg/build.cpp index b71bdbf25..bc9eca3ba 100644 --- a/toolsrc/src/vcpkg/build.cpp +++ b/toolsrc/src/vcpkg/build.cpp @@ -273,12 +273,10 @@ namespace vcpkg::Build return filter_dependencies(config.scf.core_paragraph->depends, triplet); } - auto it = - Util::find_if(config.scf.feature_paragraphs, - [&](std::unique_ptr<FeatureParagraph> const& fpgh) { return fpgh->name == feature; }); - Checks::check_exit(VCPKG_LINE_INFO, it != config.scf.feature_paragraphs.end()); + auto maybe_feature = config.scf.find_feature(feature); + Checks::check_exit(VCPKG_LINE_INFO, maybe_feature.has_value()); - return filter_dependencies(it->get()->depends, triplet); + return filter_dependencies(maybe_feature.get()->depends, triplet); }); auto dep_fspecs = FeatureSpec::from_strings_and_triplet(dep_strings, triplet); @@ -423,22 +421,10 @@ namespace vcpkg::Build return result; } - struct AbiEntry - { - std::string key; - std::string value; - }; - - struct AbiTagAndFile - { - std::string tag; - fs::path tag_file; - }; - - static Optional<AbiTagAndFile> compute_abi_tag(const VcpkgPaths& paths, - const BuildPackageConfig& config, - const PreBuildInfo& pre_build_info, - Span<const AbiEntry> dependency_abis) + Optional<AbiTagAndFile> compute_abi_tag(const VcpkgPaths& paths, + const BuildPackageConfig& config, + const PreBuildInfo& pre_build_info, + Span<const AbiEntry> dependency_abis) { if (!GlobalState::g_binary_caching) return nullopt; @@ -584,17 +570,17 @@ namespace vcpkg::Build const auto pre_build_info = PreBuildInfo::from_triplet_file(paths, triplet); - auto abi_tag_and_file = compute_abi_tag(paths, config, pre_build_info, dependency_abis); + auto maybe_abi_tag_and_file = compute_abi_tag(paths, config, pre_build_info, dependency_abis); std::unique_ptr<BinaryControlFile> bcf; - auto maybe_abi_tag_and_file = abi_tag_and_file.get(); + auto abi_tag_and_file = maybe_abi_tag_and_file.get(); - if (GlobalState::g_binary_caching && maybe_abi_tag_and_file) + if (GlobalState::g_binary_caching && abi_tag_and_file) { auto archives_root_dir = paths.root / "archives"; - auto archive_name = maybe_abi_tag_and_file->tag + ".zip"; - auto archive_subpath = fs::u8path(maybe_abi_tag_and_file->tag.substr(0, 2)) / archive_name; + auto archive_name = abi_tag_and_file->tag + ".zip"; + auto archive_subpath = fs::u8path(abi_tag_and_file->tag.substr(0, 2)) / archive_name; auto archive_path = archives_root_dir / archive_subpath; auto archive_tombstone_path = archives_root_dir / "fail" / archive_subpath; @@ -617,12 +603,12 @@ namespace vcpkg::Build System::println("Could not locate cached archive: %s", archive_path.u8string()); ExtendedBuildResult result = do_build_package_and_clean_buildtrees( - paths, pre_build_info, spec, abi_tag_and_file.value_or(AbiTagAndFile{}).tag, config, status_db); + paths, pre_build_info, spec, maybe_abi_tag_and_file.value_or(AbiTagAndFile{}).tag, config, status_db); std::error_code ec; fs.create_directories(paths.package_dir(spec) / "share" / spec.name(), ec); auto abi_file_in_package = paths.package_dir(spec) / "share" / spec.name() / "vcpkg_abi_info.txt"; - fs.copy_file(maybe_abi_tag_and_file->tag_file, abi_file_in_package, fs::stdfs::copy_options::none, ec); + fs.copy_file(abi_tag_and_file->tag_file, abi_file_in_package, fs::stdfs::copy_options::none, ec); Checks::check_exit(VCPKG_LINE_INFO, !ec, "Could not copy into file: %s", abi_file_in_package.u8string()); if (result.code == BuildResult::SUCCEEDED) @@ -648,7 +634,7 @@ namespace vcpkg::Build else { return do_build_package_and_clean_buildtrees( - paths, pre_build_info, spec, abi_tag_and_file.value_or(AbiTagAndFile{}).tag, config, status_db); + paths, pre_build_info, spec, maybe_abi_tag_and_file.value_or(AbiTagAndFile{}).tag, config, status_db); } } diff --git a/toolsrc/src/vcpkg/commands.ci.cpp b/toolsrc/src/vcpkg/commands.ci.cpp index e01072a0a..45eb1c83e 100644 --- a/toolsrc/src/vcpkg/commands.ci.cpp +++ b/toolsrc/src/vcpkg/commands.ci.cpp @@ -1,5 +1,6 @@ #include "pch.h" +#include <vcpkg/base/cache.h> #include <vcpkg/base/files.h> #include <vcpkg/base/stringliteral.h> #include <vcpkg/base/system.h> @@ -7,6 +8,7 @@ #include <vcpkg/build.h> #include <vcpkg/commands.h> #include <vcpkg/dependencies.h> +#include <vcpkg/globalstate.h> #include <vcpkg/help.h> #include <vcpkg/input.h> #include <vcpkg/install.h> @@ -44,8 +46,120 @@ namespace vcpkg::Commands::CI nullptr, }; + UnknownCIPortsResults find_unknown_ports_for_ci(const VcpkgPaths& paths, + const std::set<std::string>& exclusions, + const Dependencies::PortFileProvider& provider, + const std::vector<FeatureSpec>& fspecs) + { + UnknownCIPortsResults ret; + + auto& fs = paths.get_filesystem(); + + std::map<PackageSpec, std::string> abi_tag_map; + std::set<PackageSpec> will_fail; + + const Build::BuildPackageOptions install_plan_options = {Build::UseHeadVersion::NO, + Build::AllowDownloads::YES, + Build::CleanBuildtrees::YES, + Build::CleanPackages::YES}; + + vcpkg::Cache<Triplet, Build::PreBuildInfo> pre_build_info_cache; + + auto action_plan = Dependencies::create_feature_install_plan(provider, fspecs, StatusParagraphs{}); + + for (auto&& action : action_plan) + { + if (auto p = action.install_action.get()) + { + // determine abi tag + std::string abi; + if (auto scf = p->source_control_file.get()) + { + auto triplet = p->spec.triplet(); + + const Build::BuildPackageConfig build_config{p->source_control_file.value_or_exit(VCPKG_LINE_INFO), + triplet, + paths.port_dir(p->spec), + install_plan_options, + p->feature_list}; + + auto dependency_abis = + Util::fmap(p->computed_dependencies, [&](const PackageSpec& spec) -> Build::AbiEntry { + auto it = abi_tag_map.find(spec); + + if (it == abi_tag_map.end()) + return {spec.name(), ""}; + else + return {spec.name(), it->second}; + }); + const auto& pre_build_info = pre_build_info_cache.get_lazy( + triplet, [&]() { return Build::PreBuildInfo::from_triplet_file(paths, triplet); }); + + auto maybe_tag_and_file = + Build::compute_abi_tag(paths, build_config, pre_build_info, dependency_abis); + if (auto tag_and_file = maybe_tag_and_file.get()) + { + abi = tag_and_file->tag; + abi_tag_map.emplace(p->spec, abi); + } + } + else if (auto ipv = p->installed_package.get()) + { + abi = ipv->core->package.abi; + if (!abi.empty()) abi_tag_map.emplace(p->spec, abi); + } + + std::string state; + + auto archives_root_dir = paths.root / "archives"; + auto archive_name = abi + ".zip"; + auto archive_subpath = fs::u8path(abi.substr(0, 2)) / archive_name; + auto archive_path = archives_root_dir / archive_subpath; + auto archive_tombstone_path = archives_root_dir / "fail" / archive_subpath; + + bool b_will_build = false; + + if (Util::Sets::contains(exclusions, p->spec.name())) + { + ret.known.emplace(p->spec, BuildResult::EXCLUDED); + will_fail.emplace(p->spec); + } + else if (std::any_of(p->computed_dependencies.begin(), + p->computed_dependencies.end(), + [&](const PackageSpec& spec) { return Util::Sets::contains(will_fail, spec); })) + { + ret.known.emplace(p->spec, BuildResult::CASCADED_DUE_TO_MISSING_DEPENDENCIES); + will_fail.emplace(p->spec); + } + else if (fs.exists(archive_path)) + { + state += "pass"; + ret.known.emplace(p->spec, BuildResult::SUCCEEDED); + } + else if (fs.exists(archive_tombstone_path)) + { + state += "fail"; + ret.known.emplace(p->spec, BuildResult::BUILD_FAILED); + will_fail.emplace(p->spec); + } + else + { + ret.unknown.push_back(p->spec); + b_will_build = true; + } + + System::println("%40s: %1s %8s: %s", p->spec, (b_will_build ? "*" : " "), state, abi); + } + } + + return ret; + } + void perform_and_exit(const VcpkgCmdArguments& args, const VcpkgPaths& paths, const Triplet& default_triplet) { + Checks::check_exit( + VCPKG_LINE_INFO, GlobalState::g_binary_caching, "The ci command requires binary caching to be enabled."); + const ParsedArguments options = args.parse_arguments(COMMAND_STRUCTURE); std::set<std::string> exclusions_set; @@ -77,16 +191,21 @@ namespace vcpkg::Commands::CI Build::CleanBuildtrees::YES, Build::CleanPackages::YES}; - std::vector<std::string> ports = Install::get_all_port_names(paths); + std::vector<std::map<PackageSpec, BuildResult>> all_known_results; + + std::vector<std::string> all_ports = Install::get_all_port_names(paths); std::vector<TripletAndSummary> results; for (const Triplet& triplet : triplets) { Input::check_triplet(triplet, paths); - std::vector<PackageSpec> specs = PackageSpec::to_package_specs(ports, triplet); + + std::vector<PackageSpec> specs = PackageSpec::to_package_specs(all_ports, triplet); // Install the default features for every package - const auto featurespecs = Util::fmap(specs, [](auto& spec) { return FeatureSpec(spec, ""); }); + auto all_fspecs = Util::fmap(specs, [](auto& spec) { return FeatureSpec(spec, ""); }); + auto split_specs = find_unknown_ports_for_ci(paths, exclusions_set, paths_port_file, all_fspecs); + auto fspecs = Util::fmap(split_specs.unknown, [](auto& spec) { return FeatureSpec(spec, ""); }); - auto action_plan = Dependencies::create_feature_install_plan(paths_port_file, featurespecs, status_db); + auto action_plan = Dependencies::create_feature_install_plan(paths_port_file, fspecs, status_db); for (auto&& action : action_plan) { @@ -107,7 +226,10 @@ namespace vcpkg::Commands::CI else { auto summary = Install::perform(action_plan, Install::KeepGoing::YES, paths, status_db); + for (auto&& result : summary.results) + split_specs.known.erase(result.spec); results.push_back({triplet, std::move(summary)}); + all_known_results.emplace_back(std::move(split_specs.known)); } } @@ -125,6 +247,14 @@ namespace vcpkg::Commands::CI for (auto&& result : results) xunit_doc += result.summary.xunit_results(); + for (auto&& known_result : all_known_results) + { + for (auto&& result : known_result) + { + xunit_doc += + Install::InstallSummary::xunit_result(result.first, Chrono::ElapsedTime{}, result.second); + } + } xunit_doc += "</collection></assembly></assemblies>\n"; paths.get_filesystem().write_contents(fs::u8path(it_xunit->second), xunit_doc); diff --git a/toolsrc/src/vcpkg/dependencies.cpp b/toolsrc/src/vcpkg/dependencies.cpp index 1d017a8d3..f6d81c973 100644 --- a/toolsrc/src/vcpkg/dependencies.cpp +++ b/toolsrc/src/vcpkg/dependencies.cpp @@ -144,12 +144,14 @@ namespace vcpkg::Dependencies InstallPlanAction::InstallPlanAction(const PackageSpec& spec, const SourceControlFile& scf, const std::set<std::string>& features, - const RequestType& request_type) + const RequestType& request_type, + std::vector<PackageSpec>&& dependencies) : spec(spec) , source_control_file(scf) , plan_type(InstallPlanType::BUILD_AND_INSTALL) , request_type(request_type) , feature_list(features) + , computed_dependencies(std::move(dependencies)) { } @@ -162,6 +164,7 @@ namespace vcpkg::Dependencies , plan_type(InstallPlanType::ALREADY_INSTALLED) , request_type(request_type) , feature_list(features) + , computed_dependencies(installed_package.get()->dependencies()) { } @@ -683,11 +686,17 @@ namespace vcpkg::Dependencies // If it will be transiently uninstalled, we need to issue a full installation command auto pscf = p_cluster->source_control_file.value_or_exit(VCPKG_LINE_INFO); Checks::check_exit(VCPKG_LINE_INFO, pscf != nullptr); + + auto dep_specs = Util::fmap(m_graph_plan->install_graph.adjacency_list(p_cluster), + [](ClusterPtr const& p) { return p->spec; }); + Util::sort_unique_erase(dep_specs); + plan.emplace_back(InstallPlanAction{ p_cluster->spec, *pscf, p_cluster->to_install_features, p_cluster->request_type, + std::move(dep_specs), }); } else diff --git a/toolsrc/src/vcpkg/install.cpp b/toolsrc/src/vcpkg/install.cpp index 7d36fba82..e30a34c18 100644 --- a/toolsrc/src/vcpkg/install.cpp +++ b/toolsrc/src/vcpkg/install.cpp @@ -668,39 +668,42 @@ namespace vcpkg::Install return nullptr; } + std::string InstallSummary::xunit_result(const PackageSpec& spec, Chrono::ElapsedTime time, BuildResult code) + { + std::string inner_block; + const char* result_string = ""; + switch (code) + { + case BuildResult::POST_BUILD_CHECKS_FAILED: + case BuildResult::FILE_CONFLICTS: + case BuildResult::BUILD_FAILED: + result_string = "Fail"; + inner_block = Strings::format("<failure><message><![CDATA[%s]]></message></failure>", to_string(code)); + break; + case BuildResult::EXCLUDED: + case BuildResult::CASCADED_DUE_TO_MISSING_DEPENDENCIES: + result_string = "Skip"; + inner_block = Strings::format("<reason><![CDATA[%s]]></reason>", to_string(code)); + break; + case BuildResult::SUCCEEDED: result_string = "Pass"; break; + default: Checks::exit_fail(VCPKG_LINE_INFO); + } + + return Strings::format(R"(<test name="%s" method="%s" time="%lld" result="%s">%s</test>)" + "\n", + spec, + spec, + time.as<std::chrono::seconds>().count(), + result_string, + inner_block); + } + std::string InstallSummary::xunit_results() const { std::string xunit_doc; for (auto&& result : results) { - std::string inner_block; - const char* result_string = ""; - switch (result.build_result.code) - { - case BuildResult::POST_BUILD_CHECKS_FAILED: - case BuildResult::FILE_CONFLICTS: - case BuildResult::BUILD_FAILED: - result_string = "Fail"; - inner_block = Strings::format("<failure><message><![CDATA[%s]]></message></failure>", - to_string(result.build_result.code)); - break; - case BuildResult::EXCLUDED: - case BuildResult::CASCADED_DUE_TO_MISSING_DEPENDENCIES: - result_string = "Skip"; - inner_block = - Strings::format("<reason><![CDATA[%s]]></reason>", to_string(result.build_result.code)); - break; - case BuildResult::SUCCEEDED: result_string = "Pass"; break; - default: Checks::exit_fail(VCPKG_LINE_INFO); - } - - xunit_doc += Strings::format(R"(<test name="%s" method="%s" time="%lld" result="%s">%s</test>)" - "\n", - result.spec, - result.spec, - result.timing.as<std::chrono::seconds>().count(), - result_string, - inner_block); + xunit_doc += xunit_result(result.spec, result.timing, result.build_result.code); } return xunit_doc; } diff --git a/toolsrc/src/vcpkg/sourceparagraph.cpp b/toolsrc/src/vcpkg/sourceparagraph.cpp index 0b4baf189..ed61cb42a 100644 --- a/toolsrc/src/vcpkg/sourceparagraph.cpp +++ b/toolsrc/src/vcpkg/sourceparagraph.cpp @@ -158,6 +158,16 @@ namespace vcpkg return std::move(control_file); } + Optional<const FeatureParagraph&> SourceControlFile::find_feature(const std::string& featurename) const + { + auto it = Util::find_if(feature_paragraphs, + [&](const std::unique_ptr<FeatureParagraph>& p) { return p->name == featurename; }); + if (it != feature_paragraphs.end()) + return **it; + else + return nullopt; + } + Dependency Dependency::parse_dependency(std::string name, std::string qualifier) { Dependency dep; |