From 6fb904ca4ed1398eb1b987706bf10ec30a752f48 Mon Sep 17 00:00:00 2001 From: Milan Crha Date: Fri, 8 Nov 2024 12:31:30 +0100 Subject: [PATCH] Support repo's Appstream data download and install Repositories can provide Appstream data for the packages they contain. This Appstream data is consumed by applications like the GNOME Software or KDE Discover, thus the users can see the packages (apps) in them. This is to be in pair with PackageKit, which does download and install the repo's Appstream data. As this adds a dependency on the `appstream` library, there is also a CMake option WITH_APPSTREAM to be able to turn the support off. The support is built by default, but the Appstream data is not downloaded unless "optional_metadata_types" config option contains "appstream". Closes https://github.com/rpm-software-management/dnf5/issues/1564 --- CMakeLists.txt | 5 ++++ dnf5.spec | 5 ++++ include/libdnf5/conf/const.hpp | 8 +++++- include/libdnf5/repo/repo.hpp | 4 +++ libdnf5/CMakeLists.txt | 7 +++++ libdnf5/repo/repo.cpp | 35 +++++++++++++++++++++++ libdnf5/repo/repo_downloader.cpp | 48 +++++++++++++++++++++++++++++++- libdnf5/repo/repo_downloader.hpp | 3 ++ libdnf5/repo/repo_sack.cpp | 1 + libdnf5/repo/solv_repo.cpp | 23 +++++++++++---- libdnf5/repo/solv_repo.hpp | 5 ++-- 11 files changed, 134 insertions(+), 10 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b031aea84..835a59c6e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -34,6 +34,7 @@ option(WITH_COMPS "Build with comps groups and environments support" ON) option(WITH_MODULEMD "Build with modulemd modules support" ON) option(WITH_ZCHUNK "Build with zchunk delta compression support" ON) option(WITH_SYSTEMD "Build with systemd and D-Bus features" ON) +option(WITH_APPSTREAM "Build with appstream support" ON) option(ENABLE_SOLV_URPMREORDER "Build with support for URPM-like solution reordering?" OFF) option(ENABLE_SOLV_FOCUSNEW "Build with SOLVER_FLAG_FOCUS_NEW libsolv flag enabled to ensure new dependencies are installed in latests versions?" ON) @@ -129,6 +130,10 @@ if (WITH_MODULEMD) add_definitions(-DWITH_MODULEMD) endif() +if (WITH_APPSTREAM) + add_definitions(-DWITH_APPSTREAM) +endif() + include_directories("${PROJECT_SOURCE_DIR}/include") include_directories("${PROJECT_SOURCE_DIR}/common") diff --git a/dnf5.spec b/dnf5.spec index 9b302c074..d7dda0cfc 100644 --- a/dnf5.spec +++ b/dnf5.spec @@ -71,6 +71,7 @@ Provides: dnf5-command(versionlock) # ========== build options ========== +%bcond_without appstream %bcond_without dnf5daemon_client %bcond_without dnf5daemon_server %bcond_without libdnf_cli @@ -143,6 +144,9 @@ BuildRequires: bash-completion BuildRequires: cmake BuildRequires: doxygen BuildRequires: gettext +%if %{with appstream} +BuildRequires: pkgconfig(appstream) >= 0.16 +%endif BuildRequires: pkgconfig(check) BuildRequires: pkgconfig(fmt) BuildRequires: pkgconfig(json-c) @@ -797,6 +801,7 @@ automatically and regularly from systemd timers, cron jobs or similar. \ -DENABLE_SOLV_FOCUSNEW=%{?with_focus_new:ON}%{!?with_focus_new:OFF} \ \ + -DWITH_APPSTREAM=%{?with_appstream:ON}%{!?with_appstream:OFF} \ -DWITH_DNF5DAEMON_CLIENT=%{?with_dnf5daemon_client:ON}%{!?with_dnf5daemon_client:OFF} \ -DWITH_DNF5DAEMON_SERVER=%{?with_dnf5daemon_server:ON}%{!?with_dnf5daemon_server:OFF} \ -DWITH_LIBDNF5_CLI=%{?with_libdnf_cli:ON}%{!?with_libdnf_cli:OFF} \ diff --git a/include/libdnf5/conf/const.hpp b/include/libdnf5/conf/const.hpp index db349c204..2163d54d7 100644 --- a/include/libdnf5/conf/const.hpp +++ b/include/libdnf5/conf/const.hpp @@ -64,9 +64,15 @@ constexpr const char * METADATA_TYPE_OTHER = "other"; constexpr const char * METADATA_TYPE_PRESTO = "presto"; constexpr const char * METADATA_TYPE_UPDATEINFO = "updateinfo"; constexpr const char * METADATA_TYPE_ALL = "all"; +constexpr const char * METADATA_TYPE_APPSTREAM = "appstream"; const std::set OPTIONAL_METADATA_TYPES{ - METADATA_TYPE_COMPS, METADATA_TYPE_FILELISTS, METADATA_TYPE_OTHER, METADATA_TYPE_PRESTO, METADATA_TYPE_UPDATEINFO}; + METADATA_TYPE_COMPS, + METADATA_TYPE_FILELISTS, + METADATA_TYPE_OTHER, + METADATA_TYPE_PRESTO, + METADATA_TYPE_UPDATEINFO, + METADATA_TYPE_APPSTREAM}; } // namespace libdnf5 diff --git a/include/libdnf5/repo/repo.hpp b/include/libdnf5/repo/repo.hpp index 53c8d678e..292bd2a88 100644 --- a/include/libdnf5/repo/repo.hpp +++ b/include/libdnf5/repo/repo.hpp @@ -144,6 +144,10 @@ class LIBDNF_API Repo { // @replaces libdnf:repo/Repo.hpp:method:Repo.loadCache(bool throwExcept) void read_metadata_cache(); + /// Installs downloaded Appstream data for the repo, if available and + /// if built with the Appstream support. + void install_appstream(); + /// Checks whether the locally downloaded metadata are in sync with the origin. /// @return `true` if metadata are in sync with the origin, `false` otherwise. bool is_in_sync(); diff --git a/libdnf5/CMakeLists.txt b/libdnf5/CMakeLists.txt index d360901ee..b2a0798ce 100644 --- a/libdnf5/CMakeLists.txt +++ b/libdnf5/CMakeLists.txt @@ -81,6 +81,13 @@ if (WITH_MODULEMD) target_link_libraries(libdnf5_static PRIVATE ${LIBMODULEMD_LIBRARIES}) endif() +if (WITH_APPSTREAM) + pkg_check_modules(APPSTREAM REQUIRED appstream>=0.16) + include_directories(${APPSTREAM_INCLUDE_DIRS}) + target_link_libraries(libdnf5 PRIVATE ${APPSTREAM_LIBRARIES}) + target_link_libraries(libdnf5_static PRIVATE ${APPSTREAM_LIBRARIES}) +endif() + if (ENABLE_SOLV_FOCUSNEW) pkg_check_modules(LIBSOLV REQUIRED libsolv>=0.7.30) else() diff --git a/libdnf5/repo/repo.cpp b/libdnf5/repo/repo.cpp index 83392efd4..c031438f2 100644 --- a/libdnf5/repo/repo.cpp +++ b/libdnf5/repo/repo.cpp @@ -52,6 +52,9 @@ extern "C" { #include #include +#ifdef WITH_APPSTREAM +#include +#endif namespace libdnf5::repo { @@ -203,6 +206,31 @@ void Repo::read_metadata_cache() { p_impl->downloader->load_local(); } +void Repo::install_appstream() { +#ifdef WITH_APPSTREAM + if (!p_impl->config.get_main_config().get_optional_metadata_types_option().get_value().contains( + libdnf5::METADATA_TYPE_APPSTREAM)) + return; + + std::string repo_id = p_impl->config.get_id(); + auto appstream_metadata = p_impl->downloader->get_appstream_metadata(); + for (auto & item : appstream_metadata) { + const std::string path = item.second; + GError * local_error = NULL; + + if (!as_utils_install_metadata_file( + AS_METADATA_LOCATION_CACHE, path.c_str(), repo_id.c_str(), NULL, &local_error)) { + p_impl->base->get_logger()->debug( + "Failed to install Appstream metadata file '{}' for repo '{}': {}", + path, + repo_id, + local_error ? local_error->message : "Unknown error"); + } + + g_clear_error(&local_error); + } +#endif +} bool Repo::is_in_sync() { if (!p_impl->config.get_metalink_option().empty() && !p_impl->config.get_metalink_option().get_value().empty()) { @@ -413,6 +441,13 @@ void Repo::load_available_repo() { auto optional_metadata = p_impl->config.get_main_config().get_optional_metadata_types_option().get_value(); const bool all_metadata = optional_metadata.contains(libdnf5::METADATA_TYPE_ALL); + if (all_metadata || optional_metadata.contains(libdnf5::METADATA_TYPE_APPSTREAM)) { + auto appstream_metadata = p_impl->downloader->get_appstream_metadata(); + for (auto & item : appstream_metadata) { + p_impl->solv_repo->load_repo_ext(RepodataType::APPSTREAM, item.first, *p_impl->downloader.get()); + } + } + if (all_metadata || optional_metadata.contains(libdnf5::METADATA_TYPE_FILELISTS)) { p_impl->solv_repo->load_repo_ext(RepodataType::FILELISTS, *p_impl->downloader.get()); } diff --git a/libdnf5/repo/repo_downloader.cpp b/libdnf5/repo/repo_downloader.cpp index eecbd75e5..c969e25e7 100644 --- a/libdnf5/repo/repo_downloader.cpp +++ b/libdnf5/repo/repo_downloader.cpp @@ -389,6 +389,25 @@ const std::string & RepoDownloader::get_metadata_path(const std::string & metada return it != metadata_paths.end() ? it->second : empty; } +bool RepoDownloader::is_appstream_metadata_type(const std::string & type) const { + /* TODO: make the list configurable with this default */ + return utils::string::starts_with(type, "appstream") || utils::string::starts_with(type, "appdata"); +} + +std::vector> RepoDownloader::get_appstream_metadata() const { + std::vector> appstream_metadata; + /* The RepoDownloader::common_handle_setup() sets the expected names, + check for the starts_with() only here, to avoid copying the list here. */ + + for (auto it = metadata_paths.begin(); it != metadata_paths.end(); it++) { + const std::string type = it->first; + const std::string path = it->second; + + if (is_appstream_metadata_type(type)) + appstream_metadata.push_back(std::pair(type, path)); + } + return appstream_metadata; +} LibrepoHandle RepoDownloader::init_local_handle() { LibrepoHandle h; @@ -498,8 +517,35 @@ void RepoDownloader::common_handle_setup(LibrepoHandle & h) { // download the rest metadata added by 3rd parties for (auto & item : optional_metadata) { - dlist.push_back(item.c_str()); + // the appstream metadata is a "virtual" type, the list + // of the types is read from the repomd file + if (item.compare(libdnf5::METADATA_TYPE_APPSTREAM) != 0) + dlist.push_back(item.c_str()); } +#ifdef WITH_APPSTREAM + if (optional_metadata.extract(libdnf5::METADATA_TYPE_APPSTREAM)) { + // ideally, the repomd.xml file should be read and every type matching is_appstream_metadata_type() + // would be added from it, but the content is not known at this point and when it is known, then + // it's too late, thus declare some "expected" types to be downloaded here + dlist.push_back("appstream"); + dlist.push_back("appstream-icons"); + dlist.push_back("appstream-icons-48x48"); + dlist.push_back("appstream-icons-48x48@2"); + dlist.push_back("appstream-icons-64x64"); + dlist.push_back("appstream-icons-64x64@2"); + dlist.push_back("appstream-icons-128x128"); + dlist.push_back("appstream-icons-128x128@2"); + // consult the prefixes with the is_appstream_metadata_type() + dlist.push_back("appdata"); + dlist.push_back("appdata-icons"); + dlist.push_back("appdata-icons-48x48"); + dlist.push_back("appdata-icons-48x48@2"); + dlist.push_back("appdata-icons-64x64"); + dlist.push_back("appdata-icons-64x64@2"); + dlist.push_back("appdata-icons-128x128"); + dlist.push_back("appdata-icons-128x128@2"); + } +#endif dlist.push_back(nullptr); h.set_opt(LRO_YUMDLIST, dlist.data()); } diff --git a/libdnf5/repo/repo_downloader.hpp b/libdnf5/repo/repo_downloader.hpp index 1c14b272b..3f2fba090 100644 --- a/libdnf5/repo/repo_downloader.hpp +++ b/libdnf5/repo/repo_downloader.hpp @@ -56,6 +56,7 @@ class RepoDownloader { static constexpr const char * MD_FILENAME_GROUP_GZ = "group_gz"; static constexpr const char * MD_FILENAME_GROUP = "group"; static constexpr const char * MD_FILENAME_MODULES = "modules"; + static constexpr const char * MD_FILENAME_APPSTREAM = "appstream"; RepoDownloader(const libdnf5::BaseWeakPtr & base, const ConfigRepo & config, Repo::Type repo_type); @@ -74,6 +75,7 @@ class RepoDownloader { void * get_user_data() const noexcept; const std::string & get_metadata_path(const std::string & metadata_type) const; + std::vector> get_appstream_metadata() const; private: @@ -99,6 +101,7 @@ class RepoDownloader { time_t get_system_epoch() const; std::set get_optional_metadata() const; + bool is_appstream_metadata_type(const std::string & type) const; libdnf5::BaseWeakPtr base; const ConfigRepo & config; diff --git a/libdnf5/repo/repo_sack.cpp b/libdnf5/repo/repo_sack.cpp index 8e7e9366c..295c09b87 100644 --- a/libdnf5/repo/repo_sack.cpp +++ b/libdnf5/repo/repo_sack.cpp @@ -575,6 +575,7 @@ void RepoSack::Impl::update_and_load_repos(libdnf5::repo::RepoQuery & repos, boo RepoCache(base, cache_dir).remove_attribute(RepoCache::ATTRIBUTE_EXPIRED); repo->mark_fresh(); repo->read_metadata_cache(); + repo->install_appstream(); repos_for_processing.erase(repos_for_processing.begin() + static_cast(idx)); send_to_sack_loader(repo); diff --git a/libdnf5/repo/solv_repo.cpp b/libdnf5/repo/solv_repo.cpp index 78657e1d4..7497de9ee 100644 --- a/libdnf5/repo/solv_repo.cpp +++ b/libdnf5/repo/solv_repo.cpp @@ -179,6 +179,9 @@ static const char * repodata_type_to_name(RepodataType type) { return RepoDownloader::MD_FILENAME_GROUP; case RepodataType::OTHER: return RepoDownloader::MD_FILENAME_OTHER; + case RepodataType::APPSTREAM: + libdnf_throw_assertion("No static filename for RepodataType::APPSTREAM"); + break; } libdnf_throw_assertion("Unknown RepodataType: {}", utils::to_underlying(type)); @@ -197,6 +200,8 @@ static int repodata_type_to_flags(RepodataType type) { return 0; case RepodataType::OTHER: return REPO_EXTEND_SOLVABLES | REPO_LOCALPOOL; + case RepodataType::APPSTREAM: + return 0; } libdnf_throw_assertion("Unknown RepodataType: {}", utils::to_underlying(type)); @@ -313,17 +318,22 @@ void SolvRepo::load_system_repo_ext(RepodataType type) { case RepodataType::OTHER: case RepodataType::PRESTO: case RepodataType::UPDATEINFO: + case RepodataType::APPSTREAM: throw SolvError(M_("Unsupported extended repodata type for the system repo: \"{}\"."), type_name); } } - void SolvRepo::load_repo_ext(RepodataType type, const RepoDownloader & downloader) { + std::string type_name = ""; + load_repo_ext(type, type_name, downloader); +} + +void SolvRepo::load_repo_ext(RepodataType type, const std::string & in_type_name, const RepoDownloader & downloader) { auto & logger = *base->get_logger(); solv::Pool & pool = type == RepodataType::COMPS ? static_cast(get_comps_pool(base)) : static_cast(get_rpm_pool(base)); - std::string type_name = repodata_type_to_name(type); + std::string type_name = in_type_name.empty() ? repodata_type_to_name(type) : in_type_name; std::string ext_fn; @@ -375,6 +385,8 @@ void SolvRepo::load_repo_ext(RepodataType type, const RepoDownloader & downloade case RepodataType::OTHER: res = repo_add_rpmmd(repo, ext_file.get(), 0, REPO_EXTEND_SOLVABLES); break; + case RepodataType::APPSTREAM: + break; } if (res != 0) { @@ -388,9 +400,9 @@ void SolvRepo::load_repo_ext(RepodataType type, const RepoDownloader & downloade if (config.get_build_cache_option().get_value()) { if (type == RepodataType::COMPS) { - write_ext(comps_repo->nrepodata - 1, type); + write_ext(comps_repo->nrepodata - 1, type, type_name); } else { - write_ext(repo->nrepodata - 1, type); + write_ext(repo->nrepodata - 1, type, type_name); } } } @@ -610,14 +622,13 @@ void SolvRepo::write_main(bool load_after_write) { } -void SolvRepo::write_ext(Id repodata_id, RepodataType type) { +void SolvRepo::write_ext(Id repodata_id, RepodataType type, const std::string & type_name) { libdnf_assert(repodata_id != 0, "0 is not a valid repodata id"); auto & logger = *base->get_logger(); solv::Pool & pool = type == RepodataType::COMPS ? static_cast(get_comps_pool(base)) : static_cast(get_rpm_pool(base)); - const std::string type_name = repodata_type_to_name(type); const auto solvfile_path = solv_file_path(type_name.c_str()); const auto solvfile_parent_dir = solvfile_path.parent_path(); diff --git a/libdnf5/repo/solv_repo.hpp b/libdnf5/repo/solv_repo.hpp index ae6e62e47..9f64efee6 100644 --- a/libdnf5/repo/solv_repo.hpp +++ b/libdnf5/repo/solv_repo.hpp @@ -52,7 +52,7 @@ struct SolvUserdata { namespace libdnf5::repo { using LibsolvRepo = ::Repo; -enum class RepodataType { FILELISTS, PRESTO, UPDATEINFO, COMPS, OTHER }; +enum class RepodataType { FILELISTS, PRESTO, UPDATEINFO, COMPS, OTHER, APPSTREAM }; class SolvError : public Error { @@ -73,6 +73,7 @@ class SolvRepo { /// Loads additional metadata (filelist, others, ...) from available repo. void load_repo_ext(RepodataType type, const RepoDownloader & downloader); + void load_repo_ext(RepodataType type, const std::string & in_type_name, const RepoDownloader & downloader); /// Loads system repository into the pool. /// @@ -128,7 +129,7 @@ class SolvRepo { void write_main(bool load_after_write); /// Writes libsolv's .solvx cache file with extended libsolv repodata. - void write_ext(Id repodata_id, RepodataType type); + void write_ext(Id repodata_id, RepodataType type, const std::string & type_name); std::string solv_file_name(const char * type = nullptr); std::filesystem::path solv_file_path(const char * type = nullptr);