diff options
Diffstat (limited to 'toolsrc/src/vcpkg/build.cpp')
| -rw-r--r-- | toolsrc/src/vcpkg/build.cpp | 462 |
1 files changed, 263 insertions, 199 deletions
diff --git a/toolsrc/src/vcpkg/build.cpp b/toolsrc/src/vcpkg/build.cpp index 5870bd187..953e77460 100644 --- a/toolsrc/src/vcpkg/build.cpp +++ b/toolsrc/src/vcpkg/build.cpp @@ -209,6 +209,7 @@ namespace vcpkg::Build std::string make_build_env_cmd(const PreBuildInfo& pre_build_info, const Toolset& toolset) { if (pre_build_info.external_toolchain_file.has_value()) return ""; + if (!pre_build_info.cmake_system_name.empty() && pre_build_info.cmake_system_name != "WindowsStore") return ""; const char* tonull = " >nul"; if (GlobalState::debugging) @@ -272,12 +273,10 @@ namespace vcpkg::Build return filter_dependencies(config.scf.core_paragraph->depends, triplet); } - auto it = - Util::find_if(config.scf.feature_paragraphs, - [&](std::unique_ptr<FeatureParagraph> const& fpgh) { return fpgh->name == feature; }); - Checks::check_exit(VCPKG_LINE_INFO, it != config.scf.feature_paragraphs.end()); + auto maybe_feature = config.scf.find_feature(feature); + Checks::check_exit(VCPKG_LINE_INFO, maybe_feature.has_value()); - return filter_dependencies(it->get()->depends, triplet); + return filter_dependencies(maybe_feature.get()->depends, triplet); }); auto dep_fspecs = FeatureSpec::from_strings_and_triplet(dep_strings, triplet); @@ -314,63 +313,134 @@ namespace vcpkg::Build } static ExtendedBuildResult do_build_package(const VcpkgPaths& paths, + const PreBuildInfo& pre_build_info, + const PackageSpec& spec, + const std::string& abi_tag, const BuildPackageConfig& config, const StatusParagraphs& status_db) { auto& fs = paths.get_filesystem(); - const Triplet& triplet = config.triplet; + const Triplet& triplet = spec.triplet(); + + const fs::path& cmake_exe_path = paths.get_cmake_exe(); + const fs::path& git_exe_path = paths.get_git_exe(); - struct AbiEntry + std::string all_features; + for (auto& feature : config.scf.feature_paragraphs) { - std::string key; - std::string value; - }; + all_features.append(feature->name + ";"); + } - std::vector<AbiEntry> abi_tag_entries; + const Toolset& toolset = paths.get_toolset(pre_build_info); + const std::string cmd_launch_cmake = System::make_cmake_cmd( + cmake_exe_path, + paths.ports_cmake, + { + {"CMD", "BUILD"}, + {"PORT", config.scf.core_paragraph->name}, + {"CURRENT_PORT_DIR", config.port_dir / "/."}, + {"TARGET_TRIPLET", spec.triplet().canonical_name()}, + {"VCPKG_PLATFORM_TOOLSET", toolset.version.c_str()}, + {"VCPKG_USE_HEAD_VERSION", + Util::Enum::to_bool(config.build_package_options.use_head_version) ? "1" : "0"}, + {"_VCPKG_NO_DOWNLOADS", !Util::Enum::to_bool(config.build_package_options.allow_downloads) ? "1" : "0"}, + {"GIT", git_exe_path}, + {"FEATURES", Strings::join(";", config.feature_list)}, + {"ALL_FEATURES", all_features}, + }); - const PackageSpec spec = - PackageSpec::from_name_and_triplet(config.scf.core_paragraph->name, triplet).value_or_exit(VCPKG_LINE_INFO); + auto command = make_build_env_cmd(pre_build_info, toolset); + if (!command.empty()) command.append(" && "); + command.append(cmd_launch_cmake); - std::vector<FeatureSpec> required_fspecs = compute_required_feature_specs(config, status_db); + const auto timer = Chrono::ElapsedTimer::create_started(); - // extract out the actual package ids - auto dep_pspecs = Util::fmap(required_fspecs, [](FeatureSpec const& fspec) { return fspec.spec(); }); - Util::sort_unique_erase(dep_pspecs); - - // Find all features that aren't installed. This destroys required_fspecs. - Util::unstable_keep_if(required_fspecs, [&](FeatureSpec const& fspec) { - return !status_db.is_installed(fspec) && fspec.name() != spec.name(); - }); + const int return_code = System::cmd_execute_clean(command); + const auto buildtimeus = timer.microseconds(); + const auto spec_string = spec.to_string(); - if (!required_fspecs.empty()) { - return {BuildResult::CASCADED_DUE_TO_MISSING_DEPENDENCIES, std::move(required_fspecs)}; + auto locked_metrics = Metrics::g_metrics.lock(); + locked_metrics->track_buildtime(spec.to_string() + ":[" + Strings::join(",", config.feature_list) + "]", + buildtimeus); + if (return_code != 0) + { + locked_metrics->track_property("error", "build failed"); + locked_metrics->track_property("build_error", spec_string); + return BuildResult::BUILD_FAILED; + } } - // dep_pspecs was not destroyed - for (auto&& pspec : dep_pspecs) + const BuildInfo build_info = read_build_info(fs, paths.build_info_file_path(spec)); + const size_t error_count = PostBuildLint::perform_all_checks(spec, paths, pre_build_info, build_info); + + auto bcf = create_binary_control_file(*config.scf.core_paragraph, triplet, build_info, abi_tag); + + if (error_count != 0) { - if (pspec == spec) continue; - auto status_it = status_db.find_installed(pspec); - Checks::check_exit(VCPKG_LINE_INFO, status_it != status_db.end()); - abi_tag_entries.emplace_back( - AbiEntry{status_it->get()->package.spec.name(), status_it->get()->package.abi}); + return BuildResult::POST_BUILD_CHECKS_FAILED; + } + for (auto&& feature : config.feature_list) + { + for (auto&& f_pgh : config.scf.feature_paragraphs) + { + if (f_pgh->name == feature) + bcf->features.push_back( + create_binary_feature_control_file(*config.scf.core_paragraph, *f_pgh, triplet)); + } } - const fs::path& cmake_exe_path = paths.get_cmake_exe(); - const fs::path& git_exe_path = paths.get_git_exe(); + write_binary_control_file(paths, *bcf); + return {BuildResult::SUCCEEDED, std::move(bcf)}; + } - const fs::path ports_cmake_script_path = paths.ports_cmake; + static ExtendedBuildResult do_build_package_and_clean_buildtrees(const VcpkgPaths& paths, + const PreBuildInfo& pre_build_info, + const PackageSpec& spec, + const std::string& abi_tag, + const BuildPackageConfig& config, + const StatusParagraphs& status_db) + { + auto result = do_build_package(paths, pre_build_info, spec, abi_tag, config, status_db); - if (GlobalState::g_binary_caching) + if (config.build_package_options.clean_buildtrees == CleanBuildtrees::YES) { - abi_tag_entries.emplace_back(AbiEntry{ - "portfile", Commands::Hash::get_file_hash(cmake_exe_path, config.port_dir / "portfile.cmake", "SHA1")}); - abi_tag_entries.emplace_back(AbiEntry{ - "control", Commands::Hash::get_file_hash(cmake_exe_path, config.port_dir / "CONTROL", "SHA1")}); + auto& fs = paths.get_filesystem(); + const fs::path buildtrees_dir = paths.buildtrees / config.scf.core_paragraph->name; + auto buildtree_files = fs.get_files_non_recursive(buildtrees_dir); + for (auto&& file : buildtree_files) + { + if (fs.is_directory(file) && file.filename() != "src") + { + std::error_code ec; + fs.remove_all(file, ec); + } + } } - const auto pre_build_info = PreBuildInfo::from_triplet_file(paths, triplet); + return result; + } + + Optional<AbiTagAndFile> compute_abi_tag(const VcpkgPaths& paths, + const BuildPackageConfig& config, + const PreBuildInfo& pre_build_info, + Span<const AbiEntry> dependency_abis) + { + if (!GlobalState::g_binary_caching) return nullopt; + + auto& fs = paths.get_filesystem(); + const Triplet& triplet = config.triplet; + const std::string& name = config.scf.core_paragraph->name; + + std::vector<AbiEntry> abi_tag_entries; + + abi_tag_entries.insert(abi_tag_entries.end(), dependency_abis.begin(), dependency_abis.end()); + + abi_tag_entries.emplace_back( + AbiEntry{"portfile", Commands::Hash::get_file_hash(paths, config.port_dir / "portfile.cmake", "SHA1")}); + abi_tag_entries.emplace_back( + AbiEntry{"control", Commands::Hash::get_file_hash(paths, config.port_dir / "CONTROL", "SHA1")}); + abi_tag_entries.emplace_back(AbiEntry{"triplet", pre_build_info.triplet_abi_tag}); std::string features = Strings::join(";", config.feature_list); @@ -379,204 +449,198 @@ namespace vcpkg::Build if (config.build_package_options.use_head_version == UseHeadVersion::YES) abi_tag_entries.emplace_back(AbiEntry{"head", ""}); + Util::sort(abi_tag_entries); + std::string full_abi_info = Strings::join("", abi_tag_entries, [](const AbiEntry& p) { return p.key + " " + p.value + "\n"; }); - std::string abi_tag; - - if (GlobalState::g_binary_caching) + if (GlobalState::debugging) { - if (GlobalState::debugging) + System::println("[DEBUG] <abientries>"); + for (auto&& entry : abi_tag_entries) { - System::println("[DEBUG] <abientries>"); - for (auto&& entry : abi_tag_entries) - { - System::println("[DEBUG] %s|%s", entry.key, entry.value); - } - System::println("[DEBUG] </abientries>"); + System::println("[DEBUG] %s|%s", entry.key, entry.value); } + System::println("[DEBUG] </abientries>"); + } - auto abi_tag_entries_missing = abi_tag_entries; - Util::stable_keep_if(abi_tag_entries_missing, [](const AbiEntry& p) { return p.value.empty(); }); + auto abi_tag_entries_missing = abi_tag_entries; + Util::stable_keep_if(abi_tag_entries_missing, [](const AbiEntry& p) { return p.value.empty(); }); - if (abi_tag_entries_missing.empty()) - { - std::error_code ec; - fs.create_directories(paths.buildtrees / spec.name(), ec); - auto abi_file_path = paths.buildtrees / spec.name() / "vcpkg_abi_info"; - fs.write_contents(abi_file_path, full_abi_info); + if (abi_tag_entries_missing.empty()) + { + std::error_code ec; + fs.create_directories(paths.buildtrees / name, ec); + auto abi_file_path = paths.buildtrees / name / (triplet.canonical_name() + ".vcpkg_abi_info.txt"); + fs.write_contents(abi_file_path, full_abi_info); - abi_tag = Commands::Hash::get_file_hash(paths.get_cmake_exe(), abi_file_path, "SHA1"); - } - else - { - System::println("Warning: binary caching disabled because abi keys are missing values:\n%s", - Strings::join("", abi_tag_entries_missing, [](const AbiEntry& e) { - return " " + e.key + "\n"; - })); - } + return AbiTagAndFile{Commands::Hash::get_file_hash(paths, abi_file_path, "SHA1"), abi_file_path}; } - - std::unique_ptr<BinaryControlFile> bcf; - - auto archives_dir = paths.root / "archives"; - if (!abi_tag.empty()) + else { - archives_dir /= abi_tag.substr(0, 2); + System::println( + "Warning: binary caching disabled because abi keys are missing values:\n%s", + Strings::join("", abi_tag_entries_missing, [](const AbiEntry& e) { return " " + e.key + "\n"; })); + + return nullopt; } - auto archive_path = archives_dir / (abi_tag + ".zip"); + } - if (GlobalState::g_binary_caching && !abi_tag.empty() && fs.exists(archive_path)) - { - System::println("Using cached binary package: %s", archive_path.u8string()); + static void decompress_archive(const VcpkgPaths& paths, const PackageSpec& spec, const fs::path& archive_path) + { + auto& fs = paths.get_filesystem(); - auto pkg_path = paths.package_dir(spec); - std::error_code ec; - fs.remove_all(pkg_path, ec); - fs.create_directories(pkg_path, ec); - auto files = fs.get_files_non_recursive(pkg_path); - Checks::check_exit(VCPKG_LINE_INFO, files.empty(), "unable to clear path: %s", pkg_path.u8string()); + auto pkg_path = paths.package_dir(spec); + std::error_code ec; + fs.remove_all(pkg_path, ec); + fs.create_directories(pkg_path, ec); + auto files = fs.get_files_non_recursive(pkg_path); + Checks::check_exit(VCPKG_LINE_INFO, files.empty(), "unable to clear path: %s", pkg_path.u8string()); + +#if defined(_WIN32) + auto&& _7za = paths.get_7za_exe(); + + System::cmd_execute_clean(Strings::format( + R"("%s" x "%s" -o"%s" -y >nul)", _7za.u8string(), archive_path.u8string(), pkg_path.u8string())); +#else + System::cmd_execute_clean(Strings::format( + R"(unzip -qq "%s" "-d%s")", archive_path.u8string(), pkg_path.u8string())); +#endif + } - auto&& _7za = paths.get_7za_exe(); + static void compress_archive(const VcpkgPaths& paths, const PackageSpec& spec, const fs::path& tmp_archive_path) + { + auto& fs = paths.get_filesystem(); + + std::error_code ec; + + fs.remove(tmp_archive_path, ec); + Checks::check_exit( + VCPKG_LINE_INFO, !fs.exists(tmp_archive_path), "Could not remove file: %s", tmp_archive_path.u8string()); +#if defined(_WIN32) + auto&& _7za = paths.get_7za_exe(); + + System::cmd_execute_clean(Strings::format( + R"("%s" a "%s" "%s\*" >nul)", + _7za.u8string(), + tmp_archive_path.u8string(), + paths.package_dir(spec).u8string())); +#else + System::cmd_execute_clean(Strings::format( + R"(cd '%s' && zip --quiet -r '%s' *)", paths.package_dir(spec).u8string(), tmp_archive_path.u8string())); +#endif + } + + ExtendedBuildResult build_package(const VcpkgPaths& paths, + const BuildPackageConfig& config, + const StatusParagraphs& status_db) + { + auto& fs = paths.get_filesystem(); + const Triplet& triplet = config.triplet; + const std::string& name = config.scf.core_paragraph->name; + + std::vector<FeatureSpec> required_fspecs = compute_required_feature_specs(config, status_db); - System::cmd_execute_clean(Strings::format( - R"("%s" x "%s" -o"%s" -y >nul)", _7za.u8string(), archive_path.u8string(), pkg_path.u8string())); + // extract out the actual package ids + auto dep_pspecs = Util::fmap(required_fspecs, [](FeatureSpec const& fspec) { return fspec.spec(); }); + Util::sort_unique_erase(dep_pspecs); - auto maybe_bcf = Paragraphs::try_load_cached_control_package(paths, spec); - bcf = std::make_unique<BinaryControlFile>(std::move(maybe_bcf).value_or_exit(VCPKG_LINE_INFO)); + // Find all features that aren't installed. This destroys required_fspecs. + Util::unstable_keep_if(required_fspecs, [&](FeatureSpec const& fspec) { + return !status_db.is_installed(fspec) && fspec.name() != name; + }); + + if (!required_fspecs.empty()) + { + return {BuildResult::CASCADED_DUE_TO_MISSING_DEPENDENCIES, std::move(required_fspecs)}; } - else + + const PackageSpec spec = + PackageSpec::from_name_and_triplet(config.scf.core_paragraph->name, triplet).value_or_exit(VCPKG_LINE_INFO); + + std::vector<AbiEntry> dependency_abis; + + // dep_pspecs was not destroyed + for (auto&& pspec : dep_pspecs) { - if (GlobalState::g_binary_caching && !abi_tag.empty()) - { - System::println("Could not locate cached archive: %s", archive_path.u8string()); - } + if (pspec == spec) continue; + auto status_it = status_db.find_installed(pspec); + Checks::check_exit(VCPKG_LINE_INFO, status_it != status_db.end()); + dependency_abis.emplace_back( + AbiEntry{status_it->get()->package.spec.name(), status_it->get()->package.abi}); + } - std::string all_features; - for (auto& feature : config.scf.feature_paragraphs) - { - all_features.append(feature->name + ";"); - } + const auto pre_build_info = PreBuildInfo::from_triplet_file(paths, triplet); - const Toolset& toolset = paths.get_toolset(pre_build_info); - const std::string cmd_launch_cmake = System::make_cmake_cmd( - cmake_exe_path, - ports_cmake_script_path, - { - {"CMD", "BUILD"}, - {"PORT", config.scf.core_paragraph->name}, - {"CURRENT_PORT_DIR", config.port_dir / "/."}, - {"TARGET_TRIPLET", triplet.canonical_name()}, - {"VCPKG_PLATFORM_TOOLSET", toolset.version.c_str()}, - {"VCPKG_USE_HEAD_VERSION", - Util::Enum::to_bool(config.build_package_options.use_head_version) ? "1" : "0"}, - {"_VCPKG_NO_DOWNLOADS", - !Util::Enum::to_bool(config.build_package_options.allow_downloads) ? "1" : "0"}, - {"GIT", git_exe_path}, - {"FEATURES", features}, - {"ALL_FEATURES", all_features}, - }); - - auto command = make_build_env_cmd(pre_build_info, toolset); - if (!command.empty()) command.append(" && "); - command.append(cmd_launch_cmake); - - const auto timer = Chrono::ElapsedTimer::create_started(); - - const int return_code = System::cmd_execute_clean(command); - const auto buildtimeus = timer.microseconds(); - const auto spec_string = spec.to_string(); + auto maybe_abi_tag_and_file = compute_abi_tag(paths, config, pre_build_info, dependency_abis); - { - auto locked_metrics = Metrics::g_metrics.lock(); - locked_metrics->track_buildtime(spec.to_string() + ":[" + Strings::join(",", config.feature_list) + "]", - buildtimeus); - if (return_code != 0) - { - locked_metrics->track_property("error", "build failed"); - locked_metrics->track_property("build_error", spec_string); - return BuildResult::BUILD_FAILED; - } - } + std::unique_ptr<BinaryControlFile> bcf; - const BuildInfo build_info = read_build_info(fs, paths.build_info_file_path(spec)); - const size_t error_count = PostBuildLint::perform_all_checks(spec, paths, pre_build_info, build_info); + auto abi_tag_and_file = maybe_abi_tag_and_file.get(); - bcf = create_binary_control_file(*config.scf.core_paragraph, triplet, build_info, abi_tag); + if (GlobalState::g_binary_caching && abi_tag_and_file) + { + auto archives_root_dir = paths.root / "archives"; + auto archive_name = abi_tag_and_file->tag + ".zip"; + auto archive_subpath = fs::u8path(abi_tag_and_file->tag.substr(0, 2)) / archive_name; + auto archive_path = archives_root_dir / archive_subpath; + auto archive_tombstone_path = archives_root_dir / "fail" / archive_subpath; - if (error_count != 0) + if (fs.exists(archive_path)) { - return BuildResult::POST_BUILD_CHECKS_FAILED; + System::println("Using cached binary package: %s", archive_path.u8string()); + + decompress_archive(paths, spec, archive_path); + + auto maybe_bcf = Paragraphs::try_load_cached_package(paths, spec); + bcf = std::make_unique<BinaryControlFile>(std::move(maybe_bcf).value_or_exit(VCPKG_LINE_INFO)); + return {BuildResult::SUCCEEDED, std::move(bcf)}; } - for (auto&& feature : config.feature_list) + else if (fs.exists(archive_tombstone_path)) { - for (auto&& f_pgh : config.scf.feature_paragraphs) - { - if (f_pgh->name == feature) - bcf->features.push_back( - create_binary_feature_control_file(*config.scf.core_paragraph, *f_pgh, triplet)); - } + System::println("Found failure tombstone: %s", archive_tombstone_path.u8string()); + return BuildResult::BUILD_FAILED; } - if (GlobalState::g_binary_caching && !abi_tag.empty()) - { - std::error_code ec; - fs.write_contents( - paths.package_dir(spec) / "share" / spec.name() / "vcpkg_abi_info.txt", full_abi_info, ec); - } + System::println("Could not locate cached archive: %s", archive_path.u8string()); + + ExtendedBuildResult result = do_build_package_and_clean_buildtrees( + paths, pre_build_info, spec, maybe_abi_tag_and_file.value_or(AbiTagAndFile{}).tag, config, status_db); - write_binary_control_file(paths, *bcf); + std::error_code ec; + fs.create_directories(paths.package_dir(spec) / "share" / spec.name(), ec); + auto abi_file_in_package = paths.package_dir(spec) / "share" / spec.name() / "vcpkg_abi_info.txt"; + fs.copy_file(abi_tag_and_file->tag_file, abi_file_in_package, fs::stdfs::copy_options::none, ec); + Checks::check_exit(VCPKG_LINE_INFO, !ec, "Could not copy into file: %s", abi_file_in_package.u8string()); - if (GlobalState::g_binary_caching && !abi_tag.empty()) + if (result.code == BuildResult::SUCCEEDED) { auto tmp_archive_path = paths.buildtrees / spec.name() / (spec.triplet().to_string() + ".zip"); - std::error_code ec; - fs.remove(tmp_archive_path, ec); - Checks::check_exit(VCPKG_LINE_INFO, - !fs.exists(tmp_archive_path), - "Could not remove file: %s", - tmp_archive_path.u8string()); - auto&& _7za = paths.get_7za_exe(); - - System::cmd_execute_clean(Strings::format( - R"("%s" a "%s" "%s\*" >nul)", - _7za.u8string(), - tmp_archive_path.u8string(), - paths.package_dir(spec).u8string())); - - fs.create_directories(archives_dir, ec); - - fs.rename(tmp_archive_path, archive_path); + compress_archive(paths, spec, tmp_archive_path); - System::println("Stored binary cache: %s", archive_path.u8string()); + fs.create_directories(archive_path.parent_path(), ec); + fs.rename(tmp_archive_path, archive_path, ec); + if (ec) + System::println( + System::Color::warning, "Failed to store binary cache: %s", archive_path.u8string()); + else + System::println("Stored binary cache: %s", archive_path.u8string()); } - } - return {BuildResult::SUCCEEDED, std::move(bcf)}; - } - - ExtendedBuildResult build_package(const VcpkgPaths& paths, - const BuildPackageConfig& config, - const StatusParagraphs& status_db) - { - ExtendedBuildResult result = do_build_package(paths, config, status_db); - - if (config.build_package_options.clean_buildtrees == CleanBuildtrees::YES) - { - auto& fs = paths.get_filesystem(); - const fs::path buildtrees_dir = paths.buildtrees / config.scf.core_paragraph->name; - auto buildtree_files = fs.get_files_non_recursive(buildtrees_dir); - for (auto&& file : buildtree_files) + else if (result.code == BuildResult::BUILD_FAILED || result.code == BuildResult::POST_BUILD_CHECKS_FAILED) { - if (fs.is_directory(file) && file.filename() != "src") - { - std::error_code ec; - fs.remove_all(file, ec); - } + // Build failed, so store tombstone archive + fs.create_directories(archive_tombstone_path.parent_path(), ec); + fs.write_contents(archive_tombstone_path, "", ec); } - } - return result; + return result; + } + else + { + return do_build_package_and_clean_buildtrees( + paths, pre_build_info, spec, maybe_abi_tag_and_file.value_or(AbiTagAndFile{}).tag, config, status_db); + } } const std::string& to_string(const BuildResult build_result) @@ -700,7 +764,7 @@ namespace vcpkg::Build { return it_hash->second; } - auto hash = Commands::Hash::get_file_hash(paths.get_cmake_exe(), triplet_file_path, "SHA1"); + auto hash = Commands::Hash::get_file_hash(paths, triplet_file_path, "SHA1"); s_hash_cache.emplace(triplet_file_path, hash); return hash; } |