// SPDX-License-Identifier: GPL-2.0-only /* Copyright (C) 2019 Daniel Borkmann */ #include #include #include #include "l2md.h" static int progress_sideband(const char *msg, int len, void *private_data) { verbose("Remote: %.*s", len, msg); return 0; } static void progress_checkout(const char *path, size_t curr, size_t total, void *private_data) { verbose("Checkout: %zu/%zu\r", curr, total); } static void setup_fetch_opts(git_fetch_options *opts) { opts->callbacks.sideband_progress = progress_sideband; opts->proxy_opts.type = GIT_PROXY_AUTO; } static void setup_checkout_opts(git_checkout_options *opts) { opts->checkout_strategy = GIT_CHECKOUT_SAFE; opts->progress_cb = progress_checkout; } static void setup_clone_opts(git_clone_options *opts) { setup_checkout_opts(&opts->checkout_opts); setup_fetch_opts(&opts->fetch_opts); } static void panic_git(const char *subject) { const git_error *err = git_error_last(); const char *err_str = err ? err->message : "unknown error"; panic("%s: %s\n", subject, err_str); } static void warn_git(const char *subject) { const git_error *err = git_error_last(); const char *err_str = err ? err->message : "unknown error"; warn("%s: %s\n", subject, err_str); } void repo_clone(struct config_repo *repo, struct config_url *url, const char *target) { git_clone_options clone_opts = GIT_CLONE_OPTIONS_INIT; int ret; setup_clone_opts(&clone_opts); verbose("Cloning: %s\n", url->path); ret = git_clone(&repo->git, url->path, target, &clone_opts); if (ret) panic_git("Cannot clone git repo"); git_repository_free(repo->git); } static int fetch_head(const char *ref_name, const char *remote_url, const git_oid *oid, unsigned int merge, void *private_data) { char out[GIT_OID_HEXSZ + 1]; if (merge) { verbose("Merging: %s commit %s\n", ref_name, git_oid_tostr(out, sizeof(out), oid)); git_oid_cpy(private_data, oid); } return 0; } void repo_pull(struct config_repo *repo, struct config_url *url, const char *target) { git_checkout_options checkout_opts = GIT_CHECKOUT_OPTIONS_INIT; git_fetch_options fetch_opts = GIT_FETCH_OPTIONS_INIT; git_reference *head_cur, *head_new; git_annotated_commit *head; git_oid oid_to_merge; git_object *head_obj; git_remote *remote; int ret; ret = git_repository_open(&repo->git, target); if (ret) panic_git("Cannot open git repo"); ret = git_remote_lookup(&remote, repo->git, "origin"); if (ret) panic_git("Cannot look up origin"); setup_fetch_opts(&fetch_opts); verbose("Fetching: %s\n", url->path); ret = git_remote_fetch(remote, NULL, &fetch_opts, NULL); if (ret) { warn_git("Cannot fetch remote"); goto out; } git_repository_fetchhead_foreach(repo->git, fetch_head, &oid_to_merge); ret = git_annotated_commit_lookup(&head, repo->git, &oid_to_merge); if (ret) panic_git("Cannot lookup head"); ret = git_repository_head(&head_cur, repo->git); if (ret) panic_git("Cannot lookup current head"); ret = git_object_lookup(&head_obj, repo->git, &oid_to_merge, GIT_OBJECT_COMMIT); if (ret) panic_git("Cannot lookup HEAD object"); setup_checkout_opts(&checkout_opts); ret = git_checkout_tree(repo->git, head_obj, &checkout_opts); if (ret) panic_git("Cannot checkout tree"); ret = git_reference_set_target(&head_new, head_cur, &oid_to_merge, NULL); if (ret) panic_git("Cannot repoint HEAD"); git_repository_state_cleanup(repo->git); git_annotated_commit_free(head); git_reference_free(head_cur); git_reference_free(head_new); git_object_free(head_obj); out: git_remote_free(remote); git_repository_free(repo->git); } static void __repo_local_get(struct config *cfg, struct config_repo *repo, struct config_url *url, const char *which, char *out, size_t len) { char tmp[PATH_MAX]; strcpy(tmp, url->path); slprintf(out, len, "%s/%s/%s/%s", cfg->general.base, which, repo->name, basename(tmp)); } void repo_local_path(struct config *cfg, struct config_repo *repo, struct config_url *url, char *out, size_t len) { __repo_local_get(cfg, repo, url, REPOS, out, len); } void repo_local_oid(struct config *cfg, struct config_repo *repo, struct config_url *url, char *out, size_t len) { __repo_local_get(cfg, repo, url, OIDS, out, len); } void repo_walk_files(struct config *cfg, struct config_repo *repo, uint32_t url, const char *path, const char *oid_last, char *oid_done, void (*repo_walker)(struct config *cfg, struct config_repo *repo, uint32_t url, const char *oid, const void *raw, size_t len)) { char oid_curr[GIT_OID_HEXSZ + 1]; struct timeval start, end, res; git_oid coid, loid, foid; bool have_first = false; const git_blob *blob; char spec[PATH_MAX]; git_revwalk *walker; git_commit *commit; git_object *object; uint32_t count = 0; int ret, it; ret = git_repository_open(&repo->git, path); if (ret) panic_git("Cannot open git repo"); ret = git_revwalk_new(&walker, repo->git); if (ret) panic_git("Cannot create ref walker"); git_revwalk_sorting(walker, GIT_SORT_TIME); ret = git_revwalk_push_head(walker); if (ret) panic_git("Cannot push head onto ref walker"); if (oid_last) { ret = git_oid_fromstrn(&loid, oid_last, GIT_OID_HEXSZ); if (ret) panic_git("Cannot convert to git oid"); } gettimeofday(&start, NULL); while (!(it = git_revwalk_next(&coid, walker))) { ret = git_commit_lookup(&commit, repo->git, &coid); if (ret) panic_git("Cannot look up commit"); if (!have_first) { git_oid_cpy(&foid, &coid); have_first = true; } if (oid_last && !git_oid_cmp(&loid, &coid)) { git_commit_free(commit); break; } if (!oid_last && repo->limit) { if (repo->initial_import == 0) { git_commit_free(commit); break; } repo->initial_import--; } git_oid_tostr(oid_curr, sizeof(oid_curr), &coid); slprintf(spec, sizeof(spec), "%s:%s", oid_curr, MAIL); ret = git_revparse_single(&object, repo->git, spec); if (ret == GIT_ENOTFOUND) goto skip; if (ret || git_object_type(object) != GIT_OBJECT_BLOB) panic_git("Cannot revparse object"); blob = (const git_blob *)object; repo_walker(cfg, repo, url, oid_curr, git_blob_rawcontent(blob), (size_t)git_blob_rawsize(blob)); git_object_free(object); count++; skip: git_commit_free(commit); } if (it && it != GIT_ITEROVER && !count) warn_git("Performing revwalk did not succeed! Repo might need repack"); gettimeofday(&end, NULL); timeval_sub(&res, &end, &start); if (have_first) git_oid_tostr(oid_done, GIT_OID_HEXSZ + 1, &foid); if (have_first && count) { git_oid_tostr(oid_curr, sizeof(oid_curr), &loid); printf("Processed %u new mail(s) for %s. %s: %s -> %s in %ld.%02lds.\n", count, repo->name, repo->urls[url].path, oid_last ? oid_curr : "[none]", oid_done, res.tv_sec, (long)res.tv_usec); } git_revwalk_free(walker); git_repository_free(repo->git); }