From 729496a07cb2dd65de7896660e553eb1cc8017a6 Mon Sep 17 00:00:00 2001 From: Rafael Fonseca <r4f4rfs@gmail.com> Date: Thu, 23 May 2019 15:18:53 +0200 Subject: [PATCH 1/4] Refactor: move repo functions to their own file This way we can focus the solver related functions to fus.c Signed-off-by: Rafael Fonseca <r4f4rfs@gmail.com> --- fus.c | 695 +--------------------------------------------------- fus.h | 26 ++ main.c | 4 +- meson.build | 6 +- repo.c | 683 +++++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 718 insertions(+), 696 deletions(-) create mode 100644 fus.h create mode 100644 repo.c diff --git a/fus.c b/fus.c index d155099..930c8e1 100644 --- a/fus.c +++ b/fus.c @@ -1,676 +1,8 @@ -#include <errno.h> -#include <stdint.h> -#include <glib.h> +#include "fus.h" + #include <gio/gio.h> -#include <modulemd.h> -#include <solv/chksum.h> #include <solv/policy.h> -#include <solv/pool.h> #include <solv/poolarch.h> -#include <solv/selection.h> -#include <solv/testcase.h> -#include <solv/transaction.h> -#include <solv/repo_comps.h> -#include <solv/repo_repomdxml.h> -#include <solv/repo_rpmmd.h> -#include <solv/solv_xfopen.h> -#include <libsoup/soup.h> - -G_DEFINE_AUTOPTR_CLEANUP_FUNC(Pool, pool_free); -G_DEFINE_AUTOPTR_CLEANUP_FUNC(Solver, solver_free); -G_DEFINE_AUTOPTR_CLEANUP_FUNC(Transaction, transaction_free); -G_DEFINE_AUTO_CLEANUP_CLEAR_FUNC(Queue, queue_free); -G_DEFINE_AUTO_CLEANUP_CLEAR_FUNC(Map, map_free); - -#define TMPL_NPROV "module(%s)" -#define TMPL_NSPROV "module(%s:%s)" -#define MODPKG_PROV "modular-package()" - -static inline Id -dep_or_rel (Pool *pool, Id dep, Id rel, Id op) -{ - return dep ? pool_rel2id (pool, dep, rel, op, 1) : rel; -} - -static Id -parse_module_stream_requires (Pool *pool, - const char *module, - GStrv streams) -{ - Id req_neg = 0, req_pos = 0; - for (GStrv ss = streams; *ss; ss++) - { - const char *s = *ss; - - Id *r; - if (s[0] == '-') - { - r = &req_neg; - s++; - } - else - r = &req_pos; - - g_autofree char *nsprov = g_strdup_printf (TMPL_NSPROV, module, s); - *r = dep_or_rel (pool, *r, pool_str2id (pool, nsprov, 1), REL_OR); - } - - g_autofree char *nprov = g_strdup_printf (TMPL_NPROV, module); - Id req = pool_str2id (pool, nprov, 1); - if (req_pos) - req = dep_or_rel (pool, req, req_pos, REL_WITH); - else if (req_neg) - req = dep_or_rel (pool, req, req_neg, REL_WITHOUT); - - return req; -} - -typedef GStrv (*module_func_t)(ModulemdDependencies *); -typedef GStrv (*stream_func_t)(ModulemdDependencies *, const char *); - -static Id -parse_module_requires (Pool *pool, - ModulemdDependencies *deps, - module_func_t modules_get, - stream_func_t streams_get) -{ - Id require = 0; - g_auto(GStrv) namesv = modules_get (deps); - for (GStrv n = namesv; n && *n; n++) - { - g_auto(GStrv) reqv = streams_get (deps, *n); - Id r = parse_module_stream_requires (pool, *n, reqv); - require = dep_or_rel (pool, require, r, REL_AND); - } - - return require; -} - -const module_func_t buildtime_modules_get = - modulemd_dependencies_get_buildtime_modules_as_strv; -const stream_func_t buildtime_streams_get = - modulemd_dependencies_get_buildtime_streams_as_strv; - -static void -add_source_package (Repo *repo, - ModulemdDependencies *deps, - const char *name) -{ - Pool *pool = repo->pool; - Solvable *solvable = pool_id2solvable (pool, repo_add_solvable(repo)); - solvable->name = pool_str2id (pool, name, 1); - solvable->evr = ID_EMPTY; - solvable->arch = ARCH_SRC; - - Id requires = parse_module_requires (pool, - deps, - buildtime_modules_get, - buildtime_streams_get); - solvable_add_deparray (solvable, SOLVABLE_REQUIRES, requires, 0); -} - -const module_func_t runtime_modules_get = - modulemd_dependencies_get_runtime_modules_as_strv; -const stream_func_t runtime_streams_get = - modulemd_dependencies_get_runtime_streams_as_strv; - -static void -add_module_dependencies (Pool *pool, - Solvable *solvable, - GPtrArray *deps) -{ - Id requires = 0; - for (unsigned int i = 0; i < deps->len; i++) - { - ModulemdDependencies *dep = g_ptr_array_index (deps, i); - Id require = parse_module_requires (pool, - dep, - runtime_modules_get, - runtime_streams_get); - requires = dep_or_rel (pool, requires, require, REL_OR); - } - solvable_add_deparray (solvable, SOLVABLE_REQUIRES, requires, 0); -} - -static void -add_artifacts_dependencies (Pool *pool, - Queue *sel, - Id sdep) -{ - g_auto(Queue) rpms; - queue_init (&rpms); - selection_solvables (pool, sel, &rpms); - for (int i = 0; i < rpms.count; i++) - { - Solvable *s = pool_id2solvable (pool, rpms.elements[i]); - - /* Req: module:$n:$s:$v:$c . $a */ - solvable_add_deparray (s, SOLVABLE_REQUIRES, sdep, 0); - - /* Prv: modular-package() */ - Id modpkg = pool_str2id (pool, MODPKG_PROV, 1); - solvable_add_deparray (s, SOLVABLE_PROVIDES, modpkg, 0); - } -} - -static void -add_module_rpm_artifacts (Pool *pool, - ModulemdModuleStreamV2 *module, - Id sdep) -{ - g_auto(GStrv) rpm_artifacts = modulemd_module_stream_v2_get_rpm_artifacts_as_strv (module); - g_auto(Queue) sel; - queue_init (&sel); - for (GStrv artifact = rpm_artifacts; *artifact; artifact++) - { - const char *nevra = *artifact; - - const char *evr_delimiter = NULL; - const char *rel_delimiter = NULL; - const char *arch_delimiter = NULL; - const char *end; - - for (end = nevra; *end; ++end) - { - if (*end == '-') - { - evr_delimiter = rel_delimiter; - rel_delimiter = end; - } - else if (*end == '.') - arch_delimiter = end; - } - - if (!evr_delimiter || evr_delimiter == nevra) - continue; - - size_t name_len = evr_delimiter - nevra; - - /* Strip "0:" epoch if present */ - if (evr_delimiter[1] == '0' && evr_delimiter[2] == ':') - evr_delimiter += 2; - - if (rel_delimiter - evr_delimiter <= 1 || - !arch_delimiter || arch_delimiter <= rel_delimiter + 1 || arch_delimiter == end - 1) - continue; - - Id nid, evrid, aid; - if (!(nid = pool_strn2id (pool, nevra, name_len, 0))) - continue; - evr_delimiter++; - if (!(evrid = pool_strn2id (pool, evr_delimiter, arch_delimiter - evr_delimiter, 0))) - continue; - arch_delimiter++; - if (!(aid = pool_strn2id (pool, arch_delimiter, end - arch_delimiter, 0))) - continue; - - /* $n.$a = $evr */ - Id rid = pool_rel2id (pool, nid, aid, REL_ARCH, 1); - rid = pool_rel2id (pool, rid, evrid, REL_EQ, 1); - - queue_push2 (&sel, SOLVER_SOLVABLE_NAME | SOLVER_SETEVR | SOLVER_SETARCH, rid); - } - - add_artifacts_dependencies (pool, &sel, sdep); -} - -/* TODO: implement usage of REPO_EXTEND_SOLVABLES, but modulemd doesn't store chksums */ -static void -add_module_solvables (Repo *repo, - ModulemdModuleStream *module) -{ - Pool *pool = repo->pool; - - const char *n = modulemd_module_stream_get_module_name (module); - g_autofree char *nprov = g_strdup_printf (TMPL_NPROV, n); - const char *s = modulemd_module_stream_get_stream_name (module); - g_autofree char *nsprov = g_strdup_printf (TMPL_NSPROV, n, s); - const uint64_t v = modulemd_module_stream_get_version (module); - g_autofree char *vs = g_strdup_printf ("%" G_GUINT64_FORMAT, v); - const char *c = modulemd_module_stream_get_context (module); - const char *a = modulemd_module_stream_v2_get_arch ((ModulemdModuleStreamV2 *) module); - if (!a) - a = "noarch"; - - GPtrArray *deps = modulemd_module_stream_v2_get_dependencies ((ModulemdModuleStreamV2 *) module); - - /* If context is defined, then it's built artefact */ - if (c) - { - Solvable *solvable = pool_id2solvable (pool, repo_add_solvable (repo)); - g_autofree char *name = g_strdup_printf ("module:%s:%s:%s:%s", n, s, vs, c); - solvable->name = pool_str2id (pool, name, 1); - solvable->evr = ID_EMPTY; - solvable->arch = pool_str2id (pool, a, 1); - - /* Prv: module:$n:$s:$v:$c . $a */ - Id sdep = pool_rel2id (pool, solvable->name, solvable->arch, REL_ARCH, 1); - solvable_add_deparray (solvable, SOLVABLE_PROVIDES, sdep, 0); - - /* Prv: module() */ - solvable_add_deparray (solvable, SOLVABLE_PROVIDES, - pool_str2id (pool, "module()", 1), - 0); - - /* Prv: module($n) */ - solvable_add_deparray (solvable, SOLVABLE_PROVIDES, - pool_str2id (pool, nprov, 1), - 0); - - /* Prv: module($n:$s) = $v */ - solvable_add_deparray (solvable, SOLVABLE_PROVIDES, - pool_rel2id (pool, - pool_str2id (pool, nsprov, 1), - pool_str2id (pool, vs, 1), - REL_EQ, - 1), - 0); - - /* Con: module($n) */ - solvable_add_deparray (solvable, SOLVABLE_CONFLICTS, - pool_str2id (pool, nprov, 1), - 0); - - add_module_dependencies (pool, solvable, deps); -#ifdef FUS_TESTING - /* This is needed when running tests because add_module_rpm_artifacts - * relies on provides being created - */ - pool_createwhatprovides (pool); -#endif - add_module_rpm_artifacts (pool, (ModulemdModuleStreamV2 *) module, sdep); - } - - /* Add source packages */ - for (unsigned int i = 0; i < deps->len; i++) - { - g_autofree char *name = g_strdup_printf ("module:%s:%s:%s:%u", n, s, vs, i); - ModulemdDependencies *req = g_ptr_array_index (deps, i); - - add_source_package (repo, req, name); - } -} - -static void -_repo_add_modulemd_streams (Repo *repo, - GPtrArray *streams, - const char *language, - int flags) -{ - for (unsigned int i = 0; i < streams->len; i++) - add_module_solvables (repo, g_ptr_array_index (streams, i)); - - pool_createwhatprovides (repo->pool); -} - -static void -_repo_add_modulemd_defaults (Repo *repo, - ModulemdDefaults *defaults) -{ - Pool *pool = repo->pool; - const char *n = modulemd_defaults_get_module_name (defaults); - const char *s = modulemd_defaults_v1_get_default_stream ((ModulemdDefaultsV1 *)defaults, NULL); - g_autofree char *mprov = g_strdup_printf ("module(%s:%s)", n, s); - - Id dep = pool_str2id (pool, mprov, 0); - if (!dep) - return; - - Id *pp = pool_whatprovides_ptr (pool, dep); - for (; *pp; pp++) - { - Solvable *s = pool_id2solvable (pool, *pp); - solvable_add_deparray (s, SOLVABLE_PROVIDES, - pool_str2id (pool, "module-default()", 1), - 0); - } -} - -static gboolean -repo_add_modulemd (Repo *repo, - FILE *fp, - const char *language, - int flags, - GError **error) -{ - g_autoptr(GPtrArray) failures = NULL; - g_autoptr(ModulemdModuleIndex) index = modulemd_module_index_new (); - if (!modulemd_module_index_update_from_stream (index, fp, TRUE, &failures, error)) - { - if (*error) - return FALSE; - - for (unsigned int i = 0; i < failures->len; i++) - { - ModulemdSubdocumentInfo *info = g_ptr_array_index (failures, i); - const GError *e = modulemd_subdocument_info_get_gerror (info); - g_warning ("Failed reading from stream: %s", e->message); - } - - return FALSE; - } - - /* Make sure we are working with the expected version of modulemd documents */ - if (!modulemd_module_index_upgrade_streams (index, MD_MODULESTREAM_VERSION_TWO, error) || - !modulemd_module_index_upgrade_defaults (index, MD_DEFAULTS_VERSION_ONE, error)) - return FALSE; - - g_auto(GStrv) modnames = modulemd_module_index_get_module_names_as_strv (index); - for (GStrv names = modnames; names && *names; names++) - { - ModulemdModule *mod = modulemd_module_index_get_module (index, *names); - - GPtrArray *streams = modulemd_module_get_all_streams (mod); - _repo_add_modulemd_streams (repo, streams, language, flags); - - ModulemdDefaults *defaults = modulemd_module_get_defaults (mod); - if (defaults) - _repo_add_modulemd_defaults (repo, defaults); - } - - return TRUE; -} - -#ifdef FUS_TESTING -static Repo * -create_test_repo (Pool *pool, - const char *name, - const char *type, - const char *path, - GError **error) -{ - FILE *fp = fopen (path, "r"); - if (!fp) - { - g_set_error (error, - G_OPTION_ERROR, G_OPTION_ERROR_FAILED, - "Could not open %s: %s", - path, g_strerror (errno)); - return NULL; - } - - Repo *repo = repo_create (pool, name); - - /* Open a file with module metadata and load the content to the repo */ - if (g_strcmp0 (type, "modular") == 0) - { - if (!repo_add_modulemd (repo, fp, NULL, 0, error)) - { - fclose (fp); - return NULL; - } - } - else - testcase_add_testtags (repo, fp, REPO_LOCALPOOL | REPO_EXTEND_SOLVABLES); - - fclose (fp); - - return repo; -} - -#else - -static const char * -repomd_find (Repo *repo, - const char *what, - const unsigned char **chksump, - Id *chksumtypep) -{ - Pool *pool = repo->pool; - Dataiterator di; - - dataiterator_init (&di, pool, repo, SOLVID_META, REPOSITORY_REPOMD_TYPE, what, SEARCH_STRING); - dataiterator_prepend_keyname (&di, REPOSITORY_REPOMD); - - const char *filename = NULL; - *chksump = NULL; - *chksumtypep = 0; - - if (dataiterator_step (&di)) - { - dataiterator_setpos_parent (&di); - filename = pool_lookup_str (pool, SOLVID_POS, REPOSITORY_REPOMD_LOCATION); - *chksump = pool_lookup_bin_checksum (pool, SOLVID_POS, REPOSITORY_REPOMD_CHECKSUM, chksumtypep); - } - - dataiterator_free (&di); - - if (filename && !*chksumtypep) - { - g_warning ("No %s file checksum", what); - filename = NULL; - } - - return filename; -} - -static gboolean -checksum_matches (const char *filepath, - const unsigned char *chksum, - Id chksum_type) -{ - int ret = 0; - gsize len = 0; - Chksum *file_sum, *md_sum = NULL; - g_autoptr(GFile) file = g_file_new_for_path (filepath); - g_autoptr(GBytes) bytes = g_file_load_bytes (file, NULL, NULL, NULL); - gconstpointer bp = g_bytes_get_data (bytes, &len); - - file_sum = solv_chksum_create (chksum_type); - solv_chksum_add (file_sum, bp, len); - - md_sum = solv_chksum_create_from_bin (chksum_type, chksum); - - ret = solv_chksum_cmp (file_sum, md_sum); - - solv_chksum_free (file_sum, NULL); - solv_chksum_free (md_sum, NULL); - - return ret == 1; -} - -static gboolean -download_to_path (SoupSession *session, - const char *url, - const char *path, - GError **error) -{ - g_autoptr(SoupURI) parsed = soup_uri_new (url); - g_autoptr(GFile) file = g_file_new_for_path (path); - - if (!SOUP_URI_VALID_FOR_HTTP (parsed)) - { - /* Local file, so just copy to the cache */ - g_autoptr(GFile) local = g_file_new_for_path (url); - return g_file_copy (local, file, G_FILE_COPY_OVERWRITE, NULL, NULL, NULL, error); - } - - g_debug ("Downloading %s to %s", url, path); - - g_autoptr(SoupMessage) msg = soup_message_new_from_uri ("GET", parsed); - g_autoptr(GInputStream) istream = soup_session_send (session, msg, NULL, error); - if (!SOUP_STATUS_IS_SUCCESSFUL (msg->status_code)) - return FALSE; - - g_autoptr(GFileOutputStream) ostream = g_file_replace (file, - NULL, - FALSE, - G_FILE_CREATE_REPLACE_DESTINATION, - NULL, - error); - if (!ostream) - return FALSE; - - if (g_output_stream_splice (G_OUTPUT_STREAM (ostream), istream, 0, NULL, error) == -1) - return FALSE; - - return TRUE; -} - -static const char * -download_repo_metadata (SoupSession *session, - Repo *repo, - const char *type, - const char *repo_url, - const char *cachedir) -{ - Id chksumtype; - const char *fpath, *fname; - const unsigned char *chksum; - - fname = repomd_find (repo, type, &chksum, &chksumtype); - if (!fname) - return NULL; - - fpath = pool_tmpjoin (repo->pool, cachedir, "/", fname); - if (!g_file_test (fpath, G_FILE_TEST_IS_REGULAR) || - !checksum_matches (fpath, chksum, chksumtype)) - { - g_autoptr(GError) error = NULL; - const char *furl = pool_tmpjoin (repo->pool, repo_url, "/", fname); - if (!download_to_path (session, furl, fpath, &error)) - { - g_warning ("Could not download %s: %s", furl, error->message); - return NULL; - } - } - - return fpath; -} - -static int -filelist_loadcb (Pool *pool, - Repodata *data, - void *cdata) -{ - FILE *fp; - const char *path, *type, *fname; - Repo *repo = data->repo; - SoupSession *session = (SoupSession *) cdata; - - type = repodata_lookup_str (data, SOLVID_META, REPOSITORY_REPOMD_TYPE); - if (g_strcmp0 (type, "filelists") != 0) - return 0; - - path = repodata_lookup_str (data, SOLVID_META, REPOSITORY_REPOMD_LOCATION); - if (!path) - return 0; - - g_autofree gchar *cachedir = g_build_filename (g_get_user_cache_dir (), - "fus", repo->name, NULL); - - fname = download_repo_metadata (session, repo, type, path, cachedir); - fp = solv_xfopen (fname, 0); - if (!fp) - { - g_warning ("Could not open filelists %s: %s", fname, g_strerror (errno)); - return 0; - } - repo_add_rpmmd (repo, fp, NULL, REPO_USE_LOADING | REPO_LOCALPOOL | REPO_EXTEND_SOLVABLES); - fclose (fp); - - return 1; -} - -static Repo * -create_repo (Pool *pool, - SoupSession *session, - const char *name, - const char *path, - GError **error) -{ - FILE *fp; - Id chksumtype; - const unsigned char *chksum; - const char *fname, *url, *destdir; - - g_autofree gchar *cachedir = g_build_filename (g_get_user_cache_dir (), - "fus", name, NULL); - - destdir = pool_tmpjoin (pool, cachedir, "/", "repodata"); - if (g_mkdir_with_parents (destdir, 0700) == -1) - { - g_set_error (error, - G_OPTION_ERROR, G_OPTION_ERROR_FAILED, - "Could not create cache dir %s: %s", - destdir, g_strerror (errno)); - return NULL; - } - - /* We can't use `download_repo_metadata` for repomd.xml because it's a - * special case: it's always downloaded if the path provided is a repo URL. - */ - url = pool_tmpjoin (pool, path, "/", "repodata/repomd.xml"); - fname = pool_tmpjoin (pool, destdir, "/", "repomd.xml"); - if (!download_to_path (session, url, fname, error)) - return NULL; - - fp = solv_xfopen (fname, "r"); - if (!fp) - { - g_set_error (error, - G_OPTION_ERROR, G_OPTION_ERROR_FAILED, - "Could not open repomd.xml for %s: %s", - path, g_strerror (errno)); - return NULL; - } - - Repo *repo = repo_create (pool, name); - repo_add_repomdxml (repo, fp, 0); - fclose (fp); - - fname = download_repo_metadata (session, repo, "primary", path, cachedir); - fp = solv_xfopen (fname, "r"); - if (fp != NULL) - { - repo_add_rpmmd (repo, fp, NULL, 0); - fclose (fp); - } - - fname = download_repo_metadata (session, repo, "group_gz", path, cachedir); - if (!fname) - fname = download_repo_metadata (session, repo, "group", path, cachedir); - fp = solv_xfopen (fname, "r"); - if (fp != NULL) - { - repo_add_comps (repo, fp, 0); - fclose (fp); - } - - /* filelists metadata will only be downloaded if/when needed */ - fname = repomd_find (repo, "filelists", &chksum, &chksumtype); - if (fname) - { - Repodata *data = repo_add_repodata (repo, 0); - repodata_extend_block (data, repo->start, repo->end - repo->start); - Id handle = repodata_new_handle (data); - repodata_set_poolstr (data, handle, REPOSITORY_REPOMD_TYPE, "filelists"); - repodata_set_str (data, handle, REPOSITORY_REPOMD_LOCATION, path); - repodata_set_bin_checksum (data, handle, REPOSITORY_REPOMD_CHECKSUM, chksumtype, chksum); - repodata_add_idarray (data, handle, REPOSITORY_KEYS, SOLVABLE_FILELIST); - repodata_add_idarray (data, handle, REPOSITORY_KEYS, REPOKEY_TYPE_DIRSTRARRAY); - repodata_add_flexarray (data, SOLVID_META, REPOSITORY_EXTERNAL, handle); - repodata_internalize (data); - repodata_create_stubs (repo_last_repodata (repo)); - } - - pool_createwhatprovides (pool); - - fname = download_repo_metadata (session, repo, "modules", path, cachedir); - fp = solv_xfopen (fname, "r"); - if (fp != NULL) - { - g_autoptr(GError) e = NULL; - if (!repo_add_modulemd (repo, fp, NULL, REPO_LOCALPOOL | REPO_EXTEND_SOLVABLES, &e)) - g_warning ("Could not add modules from repo %s: %s", name, e->message); - fclose (fp); - } - - pool_createwhatprovides (pool); - - return repo; -} -#endif static Solver * solve (Pool *pool, Queue *jobs) @@ -895,23 +227,6 @@ apply_excludes (Pool *pool, return excludes; } -static void -add_platform_module (const char *platform, - const char *arch, - Repo *system) -{ - g_autoptr(ModulemdModuleStream) module = - modulemd_module_stream_new (MD_MODULESTREAM_VERSION_TWO, "platform", platform); - modulemd_module_stream_set_version (module, 0); - modulemd_module_stream_set_context (module, "00000000"); - modulemd_module_stream_v2_set_arch ((ModulemdModuleStreamV2 *) module, arch); - add_module_solvables (system, module); - - g_autoptr(ModulemdDefaultsV1) defaults = modulemd_defaults_v1_new ("platform"); - modulemd_defaults_v1_set_default_stream (defaults, platform, NULL); - _repo_add_modulemd_defaults (system, (ModulemdDefaults *)defaults); -} - static Map precompute_modular_packages (Pool *pool) { @@ -1381,12 +696,8 @@ fus_depsolve (const char *arch, pool_setarch (pool, arch); - Repo *system = repo_create (pool, "@system"); + Repo *system = create_system_repo (pool, platform, arch); pool_set_installed (pool, system); - if (platform) - { - add_platform_module (platform, arch, system); - } g_autoptr(GHashTable) lookaside_repos = g_hash_table_new (g_direct_hash, NULL); g_hash_table_add (lookaside_repos, system); diff --git a/fus.h b/fus.h new file mode 100644 index 0000000..a6177cb --- /dev/null +++ b/fus.h @@ -0,0 +1,26 @@ +#pragma once + +#include <glib.h> +#include <libsoup/soup.h> +#include <solv/pool.h> +#include <solv/repo.h> +#include <solv/selection.h> +#include <solv/solver.h> +#include <solv/transaction.h> + +#define TMPL_NPROV "module(%s)" +#define TMPL_NSPROV "module(%s:%s)" +#define MODPKG_PROV "modular-package()" + +G_DEFINE_AUTOPTR_CLEANUP_FUNC(Pool, pool_free); +G_DEFINE_AUTOPTR_CLEANUP_FUNC(Solver, solver_free); +G_DEFINE_AUTOPTR_CLEANUP_FUNC(Transaction, transaction_free); +G_DEFINE_AUTO_CLEANUP_CLEAR_FUNC(Queue, queue_free); +G_DEFINE_AUTO_CLEANUP_CLEAR_FUNC(Map, map_free); + +Repo *create_repo (Pool *pool, SoupSession *session, const char *name, const char *path, GError **error); +Repo *create_test_repo (Pool *pool, const char *name, const char *type, const char *path, GError **error); +Repo *create_system_repo (Pool *pool, const char *platform, const char *arch); +int filelist_loadcb (Pool *pool, Repodata *data, void *cdata); + +GPtrArray *fus_depsolve (const char *arch, const char *platform, const GStrv exclude_packages, const GStrv repos, const GStrv solvables, GError **error); diff --git a/main.c b/main.c index 5dde8b2..119110c 100644 --- a/main.c +++ b/main.c @@ -1,10 +1,10 @@ +#include "fus.h" + #include <locale.h> #include <stdlib.h> #include <sys/utsname.h> #include <glib.h> -extern GPtrArray *fus_depsolve(const char *, const char *, const GStrv, const GStrv, const GStrv, GError **); - static inline void print_package (void *package, gpointer user_data) { diff --git a/meson.build b/meson.build index 60a2fbb..5b07bfc 100644 --- a/meson.build +++ b/meson.build @@ -8,8 +8,10 @@ dep_modulemd = dependency('modulemd-2.0', version : '>= 2.0') dep_libsoup = dependency('libsoup-2.4', version: '>= 2.4') add_project_arguments('-DG_LOG_DOMAIN="fus"', language : 'c') -exe_main = executable('fus', 'fus.c', 'main.c', dependencies : [dep_glib, dep_gio, dep_libsolv, dep_libsolvext, dep_modulemd, dep_libsoup], install : true) -exe_test = executable('tests', 'fus.c', 'tests.c', +exe_main = executable('fus', 'repo.c', 'fus.c', 'main.c', + dependencies : [dep_glib, dep_gio, dep_libsolv, dep_libsolvext, dep_modulemd, dep_libsoup], + install : true) +exe_test = executable('tests', 'repo.c', 'fus.c', 'tests.c', dependencies : [dep_glib, dep_gio, dep_libsolv, dep_libsolvext, dep_modulemd, dep_libsoup], install : false, c_args : '-DFUS_TESTING') diff --git a/repo.c b/repo.c new file mode 100644 index 0000000..e4775ab --- /dev/null +++ b/repo.c @@ -0,0 +1,683 @@ +#include "fus.h" + +#include <errno.h> +#include <modulemd.h> +#include <solv/chksum.h> +#include <solv/repo_comps.h> +#include <solv/repo_repomdxml.h> +#include <solv/repo_rpmmd.h> +#include <solv/repo_write.h> +#include <solv/solv_xfopen.h> +#include <solv/testcase.h> +#include <stdint.h> + +static inline Id +dep_or_rel (Pool *pool, Id dep, Id rel, Id op) +{ + return dep ? pool_rel2id (pool, dep, rel, op, 1) : rel; +} + +static Id +parse_module_stream_requires (Pool *pool, + const char *module, + GStrv streams) +{ + Id req_neg = 0, req_pos = 0; + for (GStrv ss = streams; *ss; ss++) + { + const char *s = *ss; + + Id *r; + if (s[0] == '-') + { + r = &req_neg; + s++; + } + else + r = &req_pos; + + g_autofree char *nsprov = g_strdup_printf (TMPL_NSPROV, module, s); + *r = dep_or_rel (pool, *r, pool_str2id (pool, nsprov, 1), REL_OR); + } + + g_autofree char *nprov = g_strdup_printf (TMPL_NPROV, module); + Id req = pool_str2id (pool, nprov, 1); + if (req_pos) + req = dep_or_rel (pool, req, req_pos, REL_WITH); + else if (req_neg) + req = dep_or_rel (pool, req, req_neg, REL_WITHOUT); + + return req; +} + +typedef GStrv (*module_func_t)(ModulemdDependencies *); +typedef GStrv (*stream_func_t)(ModulemdDependencies *, const char *); + +static Id +parse_module_requires (Pool *pool, + ModulemdDependencies *deps, + module_func_t modules_get, + stream_func_t streams_get) +{ + Id require = 0; + g_auto(GStrv) namesv = modules_get (deps); + for (GStrv n = namesv; n && *n; n++) + { + g_auto(GStrv) reqv = streams_get (deps, *n); + Id r = parse_module_stream_requires (pool, *n, reqv); + require = dep_or_rel (pool, require, r, REL_AND); + } + + return require; +} + +const module_func_t buildtime_modules_get = + modulemd_dependencies_get_buildtime_modules_as_strv; +const stream_func_t buildtime_streams_get = + modulemd_dependencies_get_buildtime_streams_as_strv; + +static void +add_source_package (Repo *repo, + ModulemdDependencies *deps, + const char *name) +{ + Pool *pool = repo->pool; + Solvable *solvable = pool_id2solvable (pool, repo_add_solvable(repo)); + solvable->name = pool_str2id (pool, name, 1); + solvable->evr = ID_EMPTY; + solvable->arch = ARCH_SRC; + + Id requires = parse_module_requires (pool, + deps, + buildtime_modules_get, + buildtime_streams_get); + solvable_add_deparray (solvable, SOLVABLE_REQUIRES, requires, 0); +} + +const module_func_t runtime_modules_get = + modulemd_dependencies_get_runtime_modules_as_strv; +const stream_func_t runtime_streams_get = + modulemd_dependencies_get_runtime_streams_as_strv; + +static void +add_module_dependencies (Pool *pool, + Solvable *solvable, + GPtrArray *deps) +{ + Id requires = 0; + for (unsigned int i = 0; i < deps->len; i++) + { + ModulemdDependencies *dep = g_ptr_array_index (deps, i); + Id require = parse_module_requires (pool, + dep, + runtime_modules_get, + runtime_streams_get); + requires = dep_or_rel (pool, requires, require, REL_OR); + } + solvable_add_deparray (solvable, SOLVABLE_REQUIRES, requires, 0); +} + +static void +add_artifacts_dependencies (Pool *pool, + Queue *sel, + Id sdep) +{ + g_auto(Queue) rpms; + queue_init (&rpms); + selection_solvables (pool, sel, &rpms); + for (int i = 0; i < rpms.count; i++) + { + Solvable *s = pool_id2solvable (pool, rpms.elements[i]); + + /* Req: module:$n:$s:$v:$c . $a */ + solvable_add_deparray (s, SOLVABLE_REQUIRES, sdep, 0); + + /* Prv: modular-package() */ + Id modpkg = pool_str2id (pool, MODPKG_PROV, 1); + solvable_add_deparray (s, SOLVABLE_PROVIDES, modpkg, 0); + } +} + +static void +add_module_rpm_artifacts (Pool *pool, + ModulemdModuleStreamV2 *module, + Id sdep) +{ + g_auto(GStrv) rpm_artifacts = modulemd_module_stream_v2_get_rpm_artifacts_as_strv (module); + g_auto(Queue) sel; + queue_init (&sel); + for (GStrv artifact = rpm_artifacts; *artifact; artifact++) + { + const char *nevra = *artifact; + + const char *evr_delimiter = NULL; + const char *rel_delimiter = NULL; + const char *arch_delimiter = NULL; + const char *end; + + for (end = nevra; *end; ++end) + { + if (*end == '-') + { + evr_delimiter = rel_delimiter; + rel_delimiter = end; + } + else if (*end == '.') + arch_delimiter = end; + } + + if (!evr_delimiter || evr_delimiter == nevra) + continue; + + size_t name_len = evr_delimiter - nevra; + + /* Strip "0:" epoch if present */ + if (evr_delimiter[1] == '0' && evr_delimiter[2] == ':') + evr_delimiter += 2; + + if (rel_delimiter - evr_delimiter <= 1 || + !arch_delimiter || arch_delimiter <= rel_delimiter + 1 || arch_delimiter == end - 1) + continue; + + Id nid, evrid, aid; + if (!(nid = pool_strn2id (pool, nevra, name_len, 0))) + continue; + evr_delimiter++; + if (!(evrid = pool_strn2id (pool, evr_delimiter, arch_delimiter - evr_delimiter, 0))) + continue; + arch_delimiter++; + if (!(aid = pool_strn2id (pool, arch_delimiter, end - arch_delimiter, 0))) + continue; + + /* $n.$a = $evr */ + Id rid = pool_rel2id (pool, nid, aid, REL_ARCH, 1); + rid = pool_rel2id (pool, rid, evrid, REL_EQ, 1); + + queue_push2 (&sel, SOLVER_SOLVABLE_NAME | SOLVER_SETEVR | SOLVER_SETARCH, rid); + } + + add_artifacts_dependencies (pool, &sel, sdep); +} + +/* TODO: implement usage of REPO_EXTEND_SOLVABLES, but modulemd doesn't store chksums */ +static void +add_module_solvables (Repo *repo, + ModulemdModuleStream *module) +{ + Pool *pool = repo->pool; + + const char *n = modulemd_module_stream_get_module_name (module); + g_autofree char *nprov = g_strdup_printf (TMPL_NPROV, n); + const char *s = modulemd_module_stream_get_stream_name (module); + g_autofree char *nsprov = g_strdup_printf (TMPL_NSPROV, n, s); + const uint64_t v = modulemd_module_stream_get_version (module); + g_autofree char *vs = g_strdup_printf ("%" G_GUINT64_FORMAT, v); + const char *c = modulemd_module_stream_get_context (module); + const char *a = modulemd_module_stream_v2_get_arch ((ModulemdModuleStreamV2 *) module); + if (!a) + a = "noarch"; + + GPtrArray *deps = modulemd_module_stream_v2_get_dependencies ((ModulemdModuleStreamV2 *) module); + + /* If context is defined, then it's built artefact */ + if (c) + { + Solvable *solvable = pool_id2solvable (pool, repo_add_solvable (repo)); + g_autofree char *name = g_strdup_printf ("module:%s:%s:%s:%s", n, s, vs, c); + solvable->name = pool_str2id (pool, name, 1); + solvable->evr = ID_EMPTY; + solvable->arch = pool_str2id (pool, a, 1); + + /* Prv: module:$n:$s:$v:$c . $a */ + Id sdep = pool_rel2id (pool, solvable->name, solvable->arch, REL_ARCH, 1); + solvable_add_deparray (solvable, SOLVABLE_PROVIDES, sdep, 0); + + /* Prv: module() */ + solvable_add_deparray (solvable, SOLVABLE_PROVIDES, + pool_str2id (pool, "module()", 1), + 0); + + /* Prv: module($n) */ + solvable_add_deparray (solvable, SOLVABLE_PROVIDES, + pool_str2id (pool, nprov, 1), + 0); + + /* Prv: module($n:$s) = $v */ + solvable_add_deparray (solvable, SOLVABLE_PROVIDES, + pool_rel2id (pool, + pool_str2id (pool, nsprov, 1), + pool_str2id (pool, vs, 1), + REL_EQ, + 1), + 0); + + /* Con: module($n) */ + solvable_add_deparray (solvable, SOLVABLE_CONFLICTS, + pool_str2id (pool, nprov, 1), + 0); + + add_module_dependencies (pool, solvable, deps); +#ifdef FUS_TESTING + /* This is needed when running tests because add_module_rpm_artifacts + * relies on provides being created + */ + pool_createwhatprovides (pool); +#endif + add_module_rpm_artifacts (pool, (ModulemdModuleStreamV2 *) module, sdep); + } + + /* Add source packages */ + for (unsigned int i = 0; i < deps->len; i++) + { + g_autofree char *name = g_strdup_printf ("module:%s:%s:%s:%u", n, s, vs, i); + ModulemdDependencies *req = g_ptr_array_index (deps, i); + + add_source_package (repo, req, name); + } +} + +static void +_repo_add_modulemd_streams (Repo *repo, + GPtrArray *streams, + const char *language, + int flags) +{ + for (unsigned int i = 0; i < streams->len; i++) + add_module_solvables (repo, g_ptr_array_index (streams, i)); + + pool_createwhatprovides (repo->pool); +} + +static void +_repo_add_modulemd_defaults (Repo *repo, + ModulemdDefaults *defaults) +{ + Pool *pool = repo->pool; + const char *n = modulemd_defaults_get_module_name (defaults); + const char *s = modulemd_defaults_v1_get_default_stream ((ModulemdDefaultsV1 *)defaults, NULL); + g_autofree char *mprov = g_strdup_printf ("module(%s:%s)", n, s); + + Id dep = pool_str2id (pool, mprov, 0); + if (!dep) + return; + + Id *pp = pool_whatprovides_ptr (pool, dep); + for (; *pp; pp++) + { + Solvable *s = pool_id2solvable (pool, *pp); + solvable_add_deparray (s, SOLVABLE_PROVIDES, + pool_str2id (pool, "module-default()", 1), + 0); + } +} + +static gboolean +repo_add_modulemd (Repo *repo, + FILE *fp, + const char *language, + int flags, + GError **error) +{ + g_autoptr(GPtrArray) failures = NULL; + g_autoptr(ModulemdModuleIndex) index = modulemd_module_index_new (); + if (!modulemd_module_index_update_from_stream (index, fp, TRUE, &failures, error)) + { + if (*error) + return FALSE; + + for (unsigned int i = 0; i < failures->len; i++) + { + ModulemdSubdocumentInfo *info = g_ptr_array_index (failures, i); + const GError *e = modulemd_subdocument_info_get_gerror (info); + g_warning ("Failed reading from stream: %s", e->message); + } + + return FALSE; + } + + /* Make sure we are working with the expected version of modulemd documents */ + if (!modulemd_module_index_upgrade_streams (index, MD_MODULESTREAM_VERSION_TWO, error) || + !modulemd_module_index_upgrade_defaults (index, MD_DEFAULTS_VERSION_ONE, error)) + return FALSE; + + g_auto(GStrv) modnames = modulemd_module_index_get_module_names_as_strv (index); + for (GStrv names = modnames; names && *names; names++) + { + ModulemdModule *mod = modulemd_module_index_get_module (index, *names); + + GPtrArray *streams = modulemd_module_get_all_streams (mod); + _repo_add_modulemd_streams (repo, streams, language, flags); + + ModulemdDefaults *defaults = modulemd_module_get_defaults (mod); + if (defaults) + _repo_add_modulemd_defaults (repo, defaults); + } + + return TRUE; +} + +#ifdef FUS_TESTING +Repo * +create_test_repo (Pool *pool, + const char *name, + const char *type, + const char *path, + GError **error) +{ + FILE *fp = fopen (path, "r"); + if (!fp) + { + g_set_error (error, + G_OPTION_ERROR, G_OPTION_ERROR_FAILED, + "Could not open %s: %s", + path, g_strerror (errno)); + return NULL; + } + + Repo *repo = repo_create (pool, name); + + /* Open a file with module metadata and load the content to the repo */ + if (g_strcmp0 (type, "modular") == 0) + { + if (!repo_add_modulemd (repo, fp, NULL, 0, error)) + { + fclose (fp); + return NULL; + } + } + else + testcase_add_testtags (repo, fp, REPO_LOCALPOOL | REPO_EXTEND_SOLVABLES); + + fclose (fp); + + return repo; +} + +#else + +static const char * +repomd_find (Repo *repo, + const char *what, + const unsigned char **chksump, + Id *chksumtypep) +{ + Pool *pool = repo->pool; + Dataiterator di; + + dataiterator_init (&di, pool, repo, SOLVID_META, REPOSITORY_REPOMD_TYPE, what, SEARCH_STRING); + dataiterator_prepend_keyname (&di, REPOSITORY_REPOMD); + + const char *filename = NULL; + *chksump = NULL; + *chksumtypep = 0; + + if (dataiterator_step (&di)) + { + dataiterator_setpos_parent (&di); + filename = pool_lookup_str (pool, SOLVID_POS, REPOSITORY_REPOMD_LOCATION); + *chksump = pool_lookup_bin_checksum (pool, SOLVID_POS, REPOSITORY_REPOMD_CHECKSUM, chksumtypep); + } + + dataiterator_free (&di); + + if (filename && !*chksumtypep) + { + g_warning ("No %s file checksum", what); + filename = NULL; + } + + return filename; +} + +static gboolean +checksum_matches (const char *filepath, + const unsigned char *chksum, + Id chksum_type) +{ + int ret = 0; + gsize len = 0; + Chksum *file_sum, *md_sum = NULL; + g_autoptr(GFile) file = g_file_new_for_path (filepath); + g_autoptr(GBytes) bytes = g_file_load_bytes (file, NULL, NULL, NULL); + gconstpointer bp = g_bytes_get_data (bytes, &len); + + file_sum = solv_chksum_create (chksum_type); + solv_chksum_add (file_sum, bp, len); + + md_sum = solv_chksum_create_from_bin (chksum_type, chksum); + + ret = solv_chksum_cmp (file_sum, md_sum); + + solv_chksum_free (file_sum, NULL); + solv_chksum_free (md_sum, NULL); + + return ret == 1; +} + +static gboolean +download_to_path (SoupSession *session, + const char *url, + const char *path, + GError **error) +{ + g_autoptr(SoupURI) parsed = soup_uri_new (url); + g_autoptr(GFile) file = g_file_new_for_path (path); + + if (!SOUP_URI_VALID_FOR_HTTP (parsed)) + { + g_autoptr(GFile) local = g_file_new_for_path (url); + return g_file_copy (local, file, G_FILE_COPY_OVERWRITE, NULL, NULL, NULL, error); + } + + g_debug ("Downloading %s to %s", url, path); + + g_autoptr(SoupMessage) msg = soup_message_new_from_uri ("GET", parsed); + g_autoptr(GInputStream) istream = soup_session_send (session, msg, NULL, error); + if (!SOUP_STATUS_IS_SUCCESSFUL (msg->status_code)) + return FALSE; + + g_autoptr(GFileOutputStream) ostream = g_file_replace (file, + NULL, + FALSE, + G_FILE_CREATE_REPLACE_DESTINATION, + NULL, + error); + if (!ostream) + return FALSE; + + if (g_output_stream_splice (G_OUTPUT_STREAM (ostream), istream, 0, NULL, error) == -1) + return FALSE; + + return TRUE; +} + +static const char * +download_repo_metadata (SoupSession *session, + Repo *repo, + const char *type, + const char *repo_url, + const char *cachedir) +{ + Id chksumtype; + const char *fpath, *fname; + const unsigned char *chksum; + + fname = repomd_find (repo, type, &chksum, &chksumtype); + if (!fname) + return NULL; + + fpath = pool_tmpjoin (repo->pool, cachedir, "/", fname); + if (!g_file_test (fpath, G_FILE_TEST_IS_REGULAR) || + !checksum_matches (fpath, chksum, chksumtype)) + { + g_autoptr(GError) error = NULL; + const char *furl = pool_tmpjoin (repo->pool, repo_url, "/", fname); + if (!download_to_path (session, furl, fpath, &error)) + { + g_warning ("Could not download %s: %s", furl, error->message); + return NULL; + } + } + + return fpath; +} + +int +filelist_loadcb (Pool *pool, + Repodata *data, + void *cdata) +{ + FILE *fp; + const char *path, *type, *fname; + Repo *repo = data->repo; + SoupSession *session = (SoupSession *) cdata; + + type = repodata_lookup_str (data, SOLVID_META, REPOSITORY_REPOMD_TYPE); + if (g_strcmp0 (type, "filelists") != 0) + return 0; + + path = repodata_lookup_str (data, SOLVID_META, REPOSITORY_REPOMD_LOCATION); + if (!path) + return 0; + + g_autofree gchar *cachedir = g_build_filename (g_get_user_cache_dir (), + "fus", repo->name, NULL); + + fname = download_repo_metadata (session, repo, type, path, cachedir); + fp = solv_xfopen (fname, 0); + if (!fp) + { + g_warning ("Could not open filelists %s: %s", fname, g_strerror (errno)); + return 0; + } + repo_add_rpmmd (repo, fp, NULL, REPO_USE_LOADING | REPO_LOCALPOOL | REPO_EXTEND_SOLVABLES); + fclose (fp); + + return 1; +} + +Repo * +create_repo (Pool *pool, + SoupSession *session, + const char *name, + const char *path, + GError **error) +{ + FILE *fp; + Id chksumtype; + const unsigned char *chksum; + const char *fname, *url, *destdir; + + g_autofree gchar *cachedir = g_build_filename (g_get_user_cache_dir (), + "fus", name, NULL); + + destdir = pool_tmpjoin (pool, cachedir, "/", "repodata"); + if (g_mkdir_with_parents (destdir, 0700) == -1) + { + g_set_error (error, + G_OPTION_ERROR, G_OPTION_ERROR_FAILED, + "Could not create cache dir %s: %s", + destdir, g_strerror (errno)); + return NULL; + } + + /* We can't use `download_repo_metadata` for repomd.xml because it's a + * special case: it's always downloaded if the path provided is a repo URL. + */ + url = pool_tmpjoin (pool, path, "/", "repodata/repomd.xml"); + fname = pool_tmpjoin (pool, destdir, "/", "repomd.xml"); + if (!download_to_path (session, url, fname, error)) + return NULL; + + fp = solv_xfopen (fname, "r"); + if (!fp) + { + g_set_error (error, + G_OPTION_ERROR, G_OPTION_ERROR_FAILED, + "Could not open repomd.xml for %s: %s", + path, g_strerror (errno)); + return NULL; + } + + Repo *repo = repo_create (pool, name); + repo_add_repomdxml (repo, fp, 0); + fclose (fp); + + fname = download_repo_metadata (session, repo, "primary", path, cachedir); + fp = solv_xfopen (fname, "r"); + if (fp != NULL) + { + repo_add_rpmmd (repo, fp, NULL, 0); + fclose (fp); + } + + fname = download_repo_metadata (session, repo, "group_gz", path, cachedir); + if (!fname) + fname = download_repo_metadata (session, repo, "group", path, cachedir); + fp = solv_xfopen (fname, "r"); + if (fp != NULL) + { + repo_add_comps (repo, fp, 0); + fclose (fp); + } + + /* filelists metadata will only be downloaded if/when needed */ + fname = repomd_find (repo, "filelists", &chksum, &chksumtype); + if (fname) + { + Repodata *data = repo_add_repodata (repo, 0); + repodata_extend_block (data, repo->start, repo->end - repo->start); + Id handle = repodata_new_handle (data); + repodata_set_poolstr (data, handle, REPOSITORY_REPOMD_TYPE, "filelists"); + repodata_set_str (data, handle, REPOSITORY_REPOMD_LOCATION, path); + repodata_set_bin_checksum (data, handle, REPOSITORY_REPOMD_CHECKSUM, chksumtype, chksum); + repodata_add_idarray (data, handle, REPOSITORY_KEYS, SOLVABLE_FILELIST); + repodata_add_idarray (data, handle, REPOSITORY_KEYS, REPOKEY_TYPE_DIRSTRARRAY); + repodata_add_flexarray (data, SOLVID_META, REPOSITORY_EXTERNAL, handle); + repodata_internalize (data); + repodata_create_stubs (repo_last_repodata (repo)); + } + + pool_createwhatprovides (pool); + + fname = download_repo_metadata (session, repo, "modules", path, cachedir); + fp = solv_xfopen (fname, "r"); + if (fp != NULL) + { + g_autoptr(GError) e = NULL; + if (!repo_add_modulemd (repo, fp, NULL, REPO_LOCALPOOL | REPO_EXTEND_SOLVABLES, &e)) + g_warning ("Could not add modules from repo %s: %s", name, e->message); + fclose (fp); + } + + pool_createwhatprovides (pool); + + return repo; +} +#endif /* FUS_TESTING */ + +static void +add_platform_module (const char *platform, + const char *arch, + Repo *system) +{ + g_autoptr(ModulemdModuleStream) module = + modulemd_module_stream_new (MD_MODULESTREAM_VERSION_TWO, "platform", platform); + modulemd_module_stream_set_version (module, 0); + modulemd_module_stream_set_context (module, "00000000"); + modulemd_module_stream_v2_set_arch ((ModulemdModuleStreamV2 *) module, arch); + add_module_solvables (system, module); + + g_autoptr(ModulemdDefaultsV1) defaults = modulemd_defaults_v1_new ("platform"); + modulemd_defaults_v1_set_default_stream (defaults, platform, NULL); + _repo_add_modulemd_defaults (system, (ModulemdDefaults *)defaults); +} + +Repo * +create_system_repo (Pool *pool, const char *platform, const char *arch) +{ + Repo *system = repo_create (pool, "@system"); + if (platform) + add_platform_module (platform, arch, system); + return system; +} From 234eff16748d8dee6d275828f6752a60854c5bc9 Mon Sep 17 00:00:00 2001 From: Rafael Fonseca <r4f4rfs@gmail.com> Date: Fri, 24 May 2019 00:05:41 +0200 Subject: [PATCH 2/4] Refactor: move cachedir definition to a function Signed-off-by: Rafael Fonseca <r4f4rfs@gmail.com> --- repo.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/repo.c b/repo.c index e4775ab..f9892ae 100644 --- a/repo.c +++ b/repo.c @@ -522,6 +522,12 @@ download_repo_metadata (SoupSession *session, return fpath; } +static inline gchar * +get_repo_cachedir (const char *name) +{ + return g_build_filename (g_get_user_cache_dir (), "fus", name, NULL); +} + int filelist_loadcb (Pool *pool, Repodata *data, @@ -540,8 +546,7 @@ filelist_loadcb (Pool *pool, if (!path) return 0; - g_autofree gchar *cachedir = g_build_filename (g_get_user_cache_dir (), - "fus", repo->name, NULL); + g_autofree gchar *cachedir = get_repo_cachedir (repo->name); fname = download_repo_metadata (session, repo, type, path, cachedir); fp = solv_xfopen (fname, 0); @@ -568,8 +573,7 @@ create_repo (Pool *pool, const unsigned char *chksum; const char *fname, *url, *destdir; - g_autofree gchar *cachedir = g_build_filename (g_get_user_cache_dir (), - "fus", name, NULL); + g_autofree gchar *cachedir = get_repo_cachedir (name); destdir = pool_tmpjoin (pool, cachedir, "/", "repodata"); if (g_mkdir_with_parents (destdir, 0700) == -1) From a638ad4b0ffe79665987261d2580ccf0c50eac70 Mon Sep 17 00:00:00 2001 From: Rafael Fonseca <r4f4rfs@gmail.com> Date: Fri, 24 May 2019 21:57:17 +0200 Subject: [PATCH 3/4] Refactor: simplify checksum-related functions Avoid doing unnecessary allocations and use memcmp directly to compare the checksums Signed-off-by: Rafael Fonseca <r4f4rfs@gmail.com> --- repo.c | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/repo.c b/repo.c index f9892ae..b5f24ad 100644 --- a/repo.c +++ b/repo.c @@ -432,26 +432,24 @@ repomd_find (Repo *repo, static gboolean checksum_matches (const char *filepath, const unsigned char *chksum, - Id chksum_type) + Id chksumtype) { - int ret = 0; gsize len = 0; - Chksum *file_sum, *md_sum = NULL; - g_autoptr(GFile) file = g_file_new_for_path (filepath); - g_autoptr(GBytes) bytes = g_file_load_bytes (file, NULL, NULL, NULL); - gconstpointer bp = g_bytes_get_data (bytes, &len); - - file_sum = solv_chksum_create (chksum_type); - solv_chksum_add (file_sum, bp, len); - - md_sum = solv_chksum_create_from_bin (chksum_type, chksum); + g_autofree gchar *contents = NULL; + if (!g_file_get_contents (filepath, &contents, &len, NULL)) + return FALSE; - ret = solv_chksum_cmp (file_sum, md_sum); + Chksum *filechksum = solv_chksum_create (chksumtype); + if (!filechksum) + return FALSE; + solv_chksum_add (filechksum, contents, len); - solv_chksum_free (file_sum, NULL); - solv_chksum_free (md_sum, NULL); + int clen; + const unsigned char *binsum = solv_chksum_get (filechksum, &clen); + int ret = memcmp (chksum, binsum, clen); + solv_chksum_free (filechksum, NULL); - return ret == 1; + return ret == 0; } static gboolean From 3552c670bf6a864c98055d5f83e0422d0c45ac60 Mon Sep 17 00:00:00 2001 From: Rafael Fonseca <r4f4rfs@gmail.com> Date: Fri, 24 May 2019 23:05:08 +0200 Subject: [PATCH 4/4] Use libsolv caches to speed things up This can be specially helpful when doing multiple queries on the same repo metadata. Solves #65 Signed-off-by: Rafael Fonseca <r4f4rfs@gmail.com> --- fus.c | 7 +++ repo.c | 155 +++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 159 insertions(+), 3 deletions(-) diff --git a/fus.c b/fus.c index 930c8e1..cfaa56c 100644 --- a/fus.c +++ b/fus.c @@ -770,5 +770,12 @@ fus_depsolve (const char *arch, g_ptr_array_add (output, name); } + /* We need to free the repomd checksum we saved in repo->appdata */ + int id; + Repo *r; + FOR_REPOS(id, r) + if (r->appdata) + g_free (r->appdata); + return output; } diff --git a/repo.c b/repo.c index b5f24ad..b3f3ec4 100644 --- a/repo.c +++ b/repo.c @@ -6,6 +6,7 @@ #include <solv/repo_comps.h> #include <solv/repo_repomdxml.h> #include <solv/repo_rpmmd.h> +#include <solv/repo_solv.h> #include <solv/repo_write.h> #include <solv/solv_xfopen.h> #include <solv/testcase.h> @@ -429,6 +430,18 @@ repomd_find (Repo *repo, return filename; } +/* Returns checksum as a hexadecimal string which we can use as filename */ +static gchar * +chksum_string_for_filepath (GChecksumType type, const char *filepath) +{ + gsize len = 0; + g_autofree gchar *data = NULL; + if (!g_file_get_contents (filepath, &data, &len, NULL)) + return NULL; + + return g_compute_checksum_for_string (type, data, len); +} + static gboolean checksum_matches (const char *filepath, const unsigned char *chksum, @@ -520,6 +533,110 @@ download_repo_metadata (SoupSession *session, return fpath; } +static void +switch_to_cached_repo (Repo *repo, + Repodata *repodata, + const char *repoext, + const char *cachepath) +{ + int i; + FILE *fp; + + if (!repoext && repodata) + return; + + for (i = repo->start; i < repo->end; i++) + if (repo->pool->solvables[i].repo != repo) + break; + + if (i < repo->end) + return; /* not a simple block */ + + fp = g_fopen (cachepath, "rb"); + if (!fp) + return; + + /* main repo */ + if (!repoext) + { + repo_empty (repo, 1); + if (repo_add_solv (repo, fp, SOLV_ADD_NO_STUBS)) + { + g_warning ("Could not add solvables from cache file"); + return; + } + } + else + { + int flags = REPO_USE_LOADING | REPO_EXTEND_SOLVABLES | REPO_LOCALPOOL; + /* make sure repodata contains complete repo */ + /* (this is how repodata_write saves it) */ + repodata_extend_block (repodata, repo->start, repo->end - repo->start); + repodata->state = REPODATA_LOADING; + repo_add_solv (repo, fp, flags); + repodata->state = REPODATA_AVAILABLE; /* in case the load failed */ + } + + fclose (fp); +} + +static gboolean +write_repo_cache (Repo *repo, + Repodata *repodata, + const char *repoext, + const gchar *cachename) +{ + FILE *fp = g_fopen (cachename, "wb"); + if (!fp) + { + g_warning ("Could not open cache file %s: %s", cachename, g_strerror (errno)); + return FALSE; + } + + if (!repodata) + repo_write (repo, fp); /* main repo */ + else + repodata_write (repodata, fp); + + if (fclose (fp)) + { + g_warning ("Error when closing %s: %s", cachename, g_strerror (errno)); + g_unlink (cachename); + return FALSE; + } + + /* Switch to just saved repo to activate paging and save memory */ + switch_to_cached_repo (repo, repodata, repoext, cachename); + + return TRUE; +} + +static gboolean +load_cached_repo (Repo *repo, + const char *cachefn, + const char *repoext) +{ + int ret, flags = 0; + + if (!g_file_test (cachefn, G_FILE_TEST_IS_REGULAR)) + { + g_debug ("Cache %s for repo %s not found", cachefn, repo->name); + return FALSE; + } + + FILE *fp = g_fopen (cachefn, "rb"); + if (!fp) + return FALSE; + + if (repoext) + flags = REPO_USE_LOADING | REPO_EXTEND_SOLVABLES | REPO_LOCALPOOL; + + ret = repo_add_solv (repo, fp, flags); + fclose (fp); + + return ret == 0; +} + static inline gchar * get_repo_cachedir (const char *name) { @@ -540,12 +657,21 @@ filelist_loadcb (Pool *pool, if (g_strcmp0 (type, "filelists") != 0) return 0; + g_autofree gchar *cachedir = get_repo_cachedir (repo->name); + + /* repo ext cache name is $(CHECKSUM(REPOMD)).solvx */ + const char *cachefn = pool_tmpjoin (pool, cachedir, "/", repo->appdata); + cachefn = pool_tmpappend (pool, cachefn, ".solvx", 0); + if (load_cached_repo (repo, cachefn, type)) + { + g_debug ("Using cached repo for \"%s\" filelists", repo->name); + return 1; + } + path = repodata_lookup_str (data, SOLVID_META, REPOSITORY_REPOMD_LOCATION); if (!path) return 0; - g_autofree gchar *cachedir = get_repo_cachedir (repo->name); - fname = download_repo_metadata (session, repo, type, path, cachedir); fp = solv_xfopen (fname, 0); if (!fp) @@ -556,6 +682,9 @@ filelist_loadcb (Pool *pool, repo_add_rpmmd (repo, fp, NULL, REPO_USE_LOADING | REPO_LOCALPOOL | REPO_EXTEND_SOLVABLES); fclose (fp); + if (write_repo_cache (repo, data, type, cachefn)) + g_debug ("Wrote cache file %s for repo \"%s\"", cachefn, repo->name); + return 1; } @@ -591,6 +720,10 @@ create_repo (Pool *pool, if (!download_to_path (session, url, fname, error)) return NULL; + gchar *mdchksum = chksum_string_for_filepath (G_CHECKSUM_SHA256, fname); + if (!mdchksum) + return NULL; + fp = solv_xfopen (fname, "r"); if (!fp) { @@ -602,6 +735,18 @@ create_repo (Pool *pool, } Repo *repo = repo_create (pool, name); + /* Save repomd checksum to the repo's appdata so we just calculate it once */ + repo->appdata = mdchksum; + + /* repo main cache name is $(CHECKSUM(REPOMD)).solv */ + const char *cachefn = pool_tmpjoin (pool, cachedir, "/", mdchksum); + cachefn = pool_tmpappend (pool, cachefn, ".solv", 0); + if (load_cached_repo (repo, cachefn, NULL)) + { + g_debug ("Using cached repo for \"%s\"", name); + return repo; + } + repo_add_repomdxml (repo, fp, 0); fclose (fp); @@ -637,7 +782,6 @@ create_repo (Pool *pool, repodata_add_idarray (data, handle, REPOSITORY_KEYS, REPOKEY_TYPE_DIRSTRARRAY); repodata_add_flexarray (data, SOLVID_META, REPOSITORY_EXTERNAL, handle); repodata_internalize (data); - repodata_create_stubs (repo_last_repodata (repo)); } pool_createwhatprovides (pool); @@ -654,6 +798,11 @@ create_repo (Pool *pool, pool_createwhatprovides (pool); + if (write_repo_cache (repo, NULL, NULL, cachefn)) + g_debug ("Wrote cache file %s for repo \"%s\" filelists", cachefn, repo->name); + + repodata_create_stubs (repo_last_repodata (repo)); + return repo; } #endif /* FUS_TESTING */