From 595bef7180b57889a4dec4b675a7fc6084c863ac Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Mon, 11 Aug 2025 15:46:41 +0200 Subject: [PATCH 01/10] odb: store locality in object database sources Object database sources are classified either as: - Local, which means that the source is the repository's primary source. This is typically ".git/objects". - Non-local, which is everything else. Most importantly this includes alternates and quarantine directories. This locality is often computed ad-hoc by checking whether a given object source is the first one. This works, but it is quite roundabout. Refactor the code so that we store locality when creating the sources in the first place. This makes it both more accessible and robust. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- midx.c | 5 +++-- midx.h | 2 +- odb.c | 1 + odb.h | 8 ++++++++ packfile.c | 9 ++++----- repository.c | 1 + 6 files changed, 18 insertions(+), 8 deletions(-) diff --git a/midx.c b/midx.c index 7d407682e6..b9ca0915a6 100644 --- a/midx.c +++ b/midx.c @@ -723,7 +723,7 @@ int midx_preferred_pack(struct multi_pack_index *m, uint32_t *pack_int_id) return 0; } -int prepare_multi_pack_index_one(struct odb_source *source, int local) +int prepare_multi_pack_index_one(struct odb_source *source) { struct repository *r = source->odb->repo; @@ -734,7 +734,8 @@ int prepare_multi_pack_index_one(struct odb_source *source, int local) if (source->midx) return 1; - source->midx = load_multi_pack_index(r, source->path, local); + source->midx = load_multi_pack_index(r, source->path, + source->local); return !!source->midx; } diff --git a/midx.h b/midx.h index 076382de8a..28c426a823 100644 --- a/midx.h +++ b/midx.h @@ -122,7 +122,7 @@ int fill_midx_entry(struct repository *r, const struct object_id *oid, struct pa int midx_contains_pack(struct multi_pack_index *m, const char *idx_or_pack_name); int midx_preferred_pack(struct multi_pack_index *m, uint32_t *pack_int_id); -int prepare_multi_pack_index_one(struct odb_source *source, int local); +int prepare_multi_pack_index_one(struct odb_source *source); /* * Variant of write_midx_file which writes a MIDX containing only the packs diff --git a/odb.c b/odb.c index 1f48a0448e..1761a50840 100644 --- a/odb.c +++ b/odb.c @@ -176,6 +176,7 @@ static int link_alt_odb_entry(struct object_database *odb, CALLOC_ARRAY(alternate, 1); alternate->odb = odb; + alternate->local = false; /* pathbuf.buf is already in r->objects->source_by_path */ alternate->path = strbuf_detach(&pathbuf, NULL); diff --git a/odb.h b/odb.h index 09177bf430..f9300439ba 100644 --- a/odb.h +++ b/odb.h @@ -63,6 +63,14 @@ struct odb_source { */ struct multi_pack_index *midx; + /* + * Figure out whether this is the local source of the owning + * repository, which would typically be its ".git/objects" directory. + * This local object directory is usually where objects would be + * written to. + */ + bool local; + /* * This is a temporary object store created by the tmp_objdir * facility. Disable ref updates since the objects in the store diff --git a/packfile.c b/packfile.c index 5d73932f50..a38544b87b 100644 --- a/packfile.c +++ b/packfile.c @@ -935,14 +935,14 @@ static void prepare_pack(const char *full_name, size_t full_name_len, report_garbage(PACKDIR_FILE_GARBAGE, full_name); } -static void prepare_packed_git_one(struct odb_source *source, int local) +static void prepare_packed_git_one(struct odb_source *source) { struct string_list garbage = STRING_LIST_INIT_DUP; struct prepare_pack_data data = { .m = source->midx, .r = source->odb->repo, .garbage = &garbage, - .local = local, + .local = source->local, }; for_each_file_in_pack_dir(source->path, prepare_pack, &data); @@ -1037,9 +1037,8 @@ static void prepare_packed_git(struct repository *r) odb_prepare_alternates(r->objects); for (source = r->objects->sources; source; source = source->next) { - int local = (source == r->objects->sources); - prepare_multi_pack_index_one(source, local); - prepare_packed_git_one(source, local); + prepare_multi_pack_index_one(source); + prepare_packed_git_one(source); } rearrange_packed_git(r); diff --git a/repository.c b/repository.c index ecd691181f..97f0578381 100644 --- a/repository.c +++ b/repository.c @@ -168,6 +168,7 @@ void repo_set_gitdir(struct repository *repo, if (!repo->objects->sources) { CALLOC_ARRAY(repo->objects->sources, 1); repo->objects->sources->odb = repo->objects; + repo->objects->sources->local = true; repo->objects->sources_tail = &repo->objects->sources->next; } expand_base_dir(&repo->objects->sources->path, o->object_dir, From 0d61933b8f9a0392310196578e1374283496843c Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Mon, 11 Aug 2025 15:46:42 +0200 Subject: [PATCH 02/10] odb: allow `odb_find_source()` to fail When trying to locate a source for an unknown object directory we will die right away. In subsequent patches we will add new callsites though that want to handle this situation gracefully instead. Refactor the function to return a `NULL` pointer if the source could not be found and adapt the callsites to die instead. Introduce a new wrapper `odb_find_source_or_die()` that continues to die in case the source could not be found. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- builtin/commit-graph.c | 4 ++-- midx-write.c | 2 +- odb.c | 6 ++++++ odb.h | 7 +++++-- 4 files changed, 14 insertions(+), 5 deletions(-) diff --git a/builtin/commit-graph.c b/builtin/commit-graph.c index 25018a0b9d..33fb7a5145 100644 --- a/builtin/commit-graph.c +++ b/builtin/commit-graph.c @@ -101,7 +101,7 @@ static int graph_verify(int argc, const char **argv, const char *prefix, if (opts.progress) flags |= COMMIT_GRAPH_WRITE_PROGRESS; - source = odb_find_source(the_repository->objects, opts.obj_dir); + source = odb_find_source_or_die(the_repository->objects, opts.obj_dir); graph_name = get_commit_graph_filename(source); chain_name = get_commit_graph_chain_filename(source); if (open_commit_graph(graph_name, &fd, &st)) @@ -289,7 +289,7 @@ static int graph_write(int argc, const char **argv, const char *prefix, git_env_bool(GIT_TEST_COMMIT_GRAPH_CHANGED_PATHS, 0)) flags |= COMMIT_GRAPH_WRITE_BLOOM_FILTERS; - source = odb_find_source(the_repository->objects, opts.obj_dir); + source = odb_find_source_or_die(the_repository->objects, opts.obj_dir); if (opts.reachable) { if (write_commit_graph_reachable(source, flags, &write_opts)) diff --git a/midx-write.c b/midx-write.c index c1ae62d354..d38caceadb 100644 --- a/midx-write.c +++ b/midx-write.c @@ -916,7 +916,7 @@ cleanup: static struct multi_pack_index *lookup_multi_pack_index(struct repository *r, const char *object_dir) { - struct odb_source *source = odb_find_source(r->objects, object_dir); + struct odb_source *source = odb_find_source_or_die(r->objects, object_dir); return get_multi_pack_index(source); } diff --git a/odb.c b/odb.c index 1761a50840..4e7f14be4a 100644 --- a/odb.c +++ b/odb.c @@ -464,6 +464,12 @@ struct odb_source *odb_find_source(struct object_database *odb, const char *obj_ free(obj_dir_real); strbuf_release(&odb_path_real); + return source; +} + +struct odb_source *odb_find_source_or_die(struct object_database *odb, const char *obj_dir) +{ + struct odb_source *source = odb_find_source(odb, obj_dir); if (!source) die(_("could not find object directory matching %s"), obj_dir); return source; diff --git a/odb.h b/odb.h index f9300439ba..312921077b 100644 --- a/odb.h +++ b/odb.h @@ -186,11 +186,14 @@ struct object_database *odb_new(struct repository *repo); void odb_clear(struct object_database *o); /* - * Find source by its object directory path. Dies in case the source couldn't - * be found. + * Find source by its object directory path. Returns a `NULL` pointer in case + * the source could not be found. */ struct odb_source *odb_find_source(struct object_database *odb, const char *obj_dir); +/* Same as `odb_find_source()`, but dies in case the source doesn't exist. */ +struct odb_source *odb_find_source_or_die(struct object_database *odb, const char *obj_dir); + /* * Replace the current writable object directory with the specified temporary * object directory; returns the former primary source. From 25c532f6e0797ef501ce43835fb4af4bd9c33de5 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Mon, 11 Aug 2025 15:46:43 +0200 Subject: [PATCH 03/10] odb: consistently use "dir" to refer to alternate's directory The functions that add an alternate object directory to the object database are somewhat inconsistent in how they call the paramater that refers to the directory path: in our headers we refer to it as "dir", whereas in the implementation we often call it "reference" or "entry". Unify this and consistently call the parameter "dir". While at it, refactor `link_alt_odb_entry()` to accept a C string instead of a `struct strbuf` as parameter to clarify that we really only need the path and nothing else. Suggested-by: Taylor Blau Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- odb.c | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/odb.c b/odb.c index 4e7f14be4a..e41e3952ea 100644 --- a/odb.c +++ b/odb.c @@ -140,7 +140,7 @@ static void read_info_alternates(struct object_database *odb, int depth); static int link_alt_odb_entry(struct object_database *odb, - const struct strbuf *entry, + const char *dir, const char *relative_base, int depth, const char *normalized_objdir) @@ -151,11 +151,11 @@ static int link_alt_odb_entry(struct object_database *odb, khiter_t pos; int ret = -1; - if (!is_absolute_path(entry->buf) && relative_base) { + if (!is_absolute_path(dir) && relative_base) { strbuf_realpath(&pathbuf, relative_base, 1); strbuf_addch(&pathbuf, '/'); } - strbuf_addbuf(&pathbuf, entry); + strbuf_addstr(&pathbuf, dir); if (!strbuf_realpath(&tmp, pathbuf.buf, 0)) { error(_("unable to normalize alternate object path: %s"), @@ -229,7 +229,7 @@ static void link_alt_odb_entries(struct object_database *odb, const char *alt, int sep, const char *relative_base, int depth) { struct strbuf objdirbuf = STRBUF_INIT; - struct strbuf entry = STRBUF_INIT; + struct strbuf dir = STRBUF_INIT; if (!alt || !*alt) return; @@ -243,13 +243,13 @@ static void link_alt_odb_entries(struct object_database *odb, const char *alt, strbuf_realpath(&objdirbuf, odb->sources->path, 1); while (*alt) { - alt = parse_alt_odb_entry(alt, sep, &entry); - if (!entry.len) + alt = parse_alt_odb_entry(alt, sep, &dir); + if (!dir.len) continue; - link_alt_odb_entry(odb, &entry, + link_alt_odb_entry(odb, dir.buf, relative_base, depth, objdirbuf.buf); } - strbuf_release(&entry); + strbuf_release(&dir); strbuf_release(&objdirbuf); } @@ -273,7 +273,7 @@ static void read_info_alternates(struct object_database *odb, } void odb_add_to_alternates_file(struct object_database *odb, - const char *reference) + const char *dir) { struct lock_file lock = LOCK_INIT; char *alts = repo_git_path(odb->repo, "objects/info/alternates"); @@ -290,7 +290,7 @@ void odb_add_to_alternates_file(struct object_database *odb, struct strbuf line = STRBUF_INIT; while (strbuf_getline(&line, in) != EOF) { - if (!strcmp(reference, line.buf)) { + if (!strcmp(dir, line.buf)) { found = 1; break; } @@ -306,18 +306,17 @@ void odb_add_to_alternates_file(struct object_database *odb, if (found) { rollback_lock_file(&lock); } else { - fprintf_or_die(out, "%s\n", reference); + fprintf_or_die(out, "%s\n", dir); if (commit_lock_file(&lock)) die_errno(_("unable to move new alternates file into place")); if (odb->loaded_alternates) - link_alt_odb_entries(odb, reference, - '\n', NULL, 0); + link_alt_odb_entries(odb, dir, '\n', NULL, 0); } free(alts); } void odb_add_to_alternates_memory(struct object_database *odb, - const char *reference) + const char *dir) { /* * Make sure alternates are initialized, or else our entry may be @@ -325,8 +324,7 @@ void odb_add_to_alternates_memory(struct object_database *odb, */ odb_prepare_alternates(odb); - link_alt_odb_entries(odb, reference, - '\n', NULL, 0); + link_alt_odb_entries(odb, dir, '\n', NULL, 0); } struct odb_source *odb_set_temporary_primary_source(struct object_database *odb, From a59d44ff3f0f308f9577b05c858c063d2466b061 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Mon, 11 Aug 2025 15:46:44 +0200 Subject: [PATCH 04/10] odb: return newly created in-memory sources Callers have no trivial way to obtain the newly created object database source when adding it to the in-memory list of alternates. While not yet needed anywhere, a subsequent commit will want to obtain that pointer. Refactor the function to return the source to make it easily accessible. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- odb.c | 30 ++++++++++++++++++------------ odb.h | 4 ++-- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/odb.c b/odb.c index e41e3952ea..0c808bb288 100644 --- a/odb.c +++ b/odb.c @@ -139,17 +139,16 @@ static void read_info_alternates(struct object_database *odb, const char *relative_base, int depth); -static int link_alt_odb_entry(struct object_database *odb, - const char *dir, - const char *relative_base, - int depth, - const char *normalized_objdir) +static struct odb_source *link_alt_odb_entry(struct object_database *odb, + const char *dir, + const char *relative_base, + int depth, + const char *normalized_objdir) { - struct odb_source *alternate; + struct odb_source *alternate = NULL; struct strbuf pathbuf = STRBUF_INIT; struct strbuf tmp = STRBUF_INIT; khiter_t pos; - int ret = -1; if (!is_absolute_path(dir) && relative_base) { strbuf_realpath(&pathbuf, relative_base, 1); @@ -189,11 +188,11 @@ static int link_alt_odb_entry(struct object_database *odb, /* recursively add alternates */ read_info_alternates(odb, alternate->path, depth + 1); - ret = 0; + error: strbuf_release(&tmp); strbuf_release(&pathbuf); - return ret; + return alternate; } static const char *parse_alt_odb_entry(const char *string, @@ -315,16 +314,23 @@ void odb_add_to_alternates_file(struct object_database *odb, free(alts); } -void odb_add_to_alternates_memory(struct object_database *odb, - const char *dir) +struct odb_source *odb_add_to_alternates_memory(struct object_database *odb, + const char *dir) { + struct odb_source *alternate; + char *objdir; + /* * Make sure alternates are initialized, or else our entry may be * overwritten when they are. */ odb_prepare_alternates(odb); - link_alt_odb_entries(odb, dir, '\n', NULL, 0); + objdir = real_pathdup(odb->sources->path, 1); + alternate = link_alt_odb_entry(odb, dir, NULL, 0, objdir); + + free(objdir); + return alternate; } struct odb_source *odb_set_temporary_primary_source(struct object_database *odb, diff --git a/odb.h b/odb.h index 312921077b..d769132699 100644 --- a/odb.h +++ b/odb.h @@ -268,8 +268,8 @@ void odb_add_to_alternates_file(struct object_database *odb, * recursive alternates it points to), but do not modify the on-disk alternates * file. */ -void odb_add_to_alternates_memory(struct object_database *odb, - const char *dir); +struct odb_source *odb_add_to_alternates_memory(struct object_database *odb, + const char *dir); /* * Read an object from the database. Returns the object data and assigns object From 57363dfa0dce05aac735d5cfd626e6aac8cb706c Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Mon, 11 Aug 2025 15:46:45 +0200 Subject: [PATCH 05/10] odb: simplify calling `link_alt_odb_entry()` Callers of `link_alt_odb_entry()` are expected to pass in three different paths: - The (potentially relative) path of the object directory that we're about to add. - The base that should be used to resolve a relative object directory path. - The resolved path to the object database's objects directory. Juggling those three paths makes the calling convention somewhat hard to grok at first. As it turns out, the third parameter is redundant: we always pass in the resolved path of the object database's primary source, and we already pass in the database itself. So instead, we can resolve that path in the function itself. One downside of this is that one caller of `link_alt_odb_entry()` calls this function in a loop, so we were able to resolve the directory a single time, only. But ultimately, we only ever end up with a rather limited number of alternates anyway, so the extra couple of cycles we save feels more like a micro optimization. Refactor the code accordingly. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- odb.c | 25 +++++++------------------ 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/odb.c b/odb.c index 0c808bb288..4f884e3b50 100644 --- a/odb.c +++ b/odb.c @@ -142,8 +142,7 @@ static void read_info_alternates(struct object_database *odb, static struct odb_source *link_alt_odb_entry(struct object_database *odb, const char *dir, const char *relative_base, - int depth, - const char *normalized_objdir) + int depth) { struct odb_source *alternate = NULL; struct strbuf pathbuf = STRBUF_INIT; @@ -170,7 +169,10 @@ static struct odb_source *link_alt_odb_entry(struct object_database *odb, while (pathbuf.len && pathbuf.buf[pathbuf.len - 1] == '/') strbuf_setlen(&pathbuf, pathbuf.len - 1); - if (!alt_odb_usable(odb, &pathbuf, normalized_objdir, &pos)) + strbuf_reset(&tmp); + strbuf_realpath(&tmp, odb->sources->path, 1); + + if (!alt_odb_usable(odb, &pathbuf, tmp.buf, &pos)) goto error; CALLOC_ARRAY(alternate, 1); @@ -227,7 +229,6 @@ static const char *parse_alt_odb_entry(const char *string, static void link_alt_odb_entries(struct object_database *odb, const char *alt, int sep, const char *relative_base, int depth) { - struct strbuf objdirbuf = STRBUF_INIT; struct strbuf dir = STRBUF_INIT; if (!alt || !*alt) @@ -239,17 +240,13 @@ static void link_alt_odb_entries(struct object_database *odb, const char *alt, return; } - strbuf_realpath(&objdirbuf, odb->sources->path, 1); - while (*alt) { alt = parse_alt_odb_entry(alt, sep, &dir); if (!dir.len) continue; - link_alt_odb_entry(odb, dir.buf, - relative_base, depth, objdirbuf.buf); + link_alt_odb_entry(odb, dir.buf, relative_base, depth); } strbuf_release(&dir); - strbuf_release(&objdirbuf); } static void read_info_alternates(struct object_database *odb, @@ -317,20 +314,12 @@ void odb_add_to_alternates_file(struct object_database *odb, struct odb_source *odb_add_to_alternates_memory(struct object_database *odb, const char *dir) { - struct odb_source *alternate; - char *objdir; - /* * Make sure alternates are initialized, or else our entry may be * overwritten when they are. */ odb_prepare_alternates(odb); - - objdir = real_pathdup(odb->sources->path, 1); - alternate = link_alt_odb_entry(odb, dir, NULL, 0, objdir); - - free(objdir); - return alternate; + return link_alt_odb_entry(odb, dir, NULL, 0); } struct odb_source *odb_set_temporary_primary_source(struct object_database *odb, From 9ff212961506679c1e2c1541b17ab2bd8563ff15 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Mon, 11 Aug 2025 15:46:46 +0200 Subject: [PATCH 06/10] midx: drop redundant `struct repository` parameter There are a couple of functions that take both a `struct repository` and a `struct multi_pack_index`. This provides redundant information though without much benefit given that the multi-pack index already has a pointer to its owning repository. Drop the `struct repository` parameter from such functions. While at it, reorder the list of parameters of `fill_midx_entry()` so that the MIDX comes first to better align with our coding guidelines. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- builtin/pack-objects.c | 2 +- midx-write.c | 16 +++++++--------- midx.c | 18 +++++++++--------- midx.h | 6 +++--- pack-bitmap.c | 4 ++-- packfile.c | 4 ++-- t/helper/test-read-midx.c | 4 ++-- 7 files changed, 26 insertions(+), 28 deletions(-) diff --git a/builtin/pack-objects.c b/builtin/pack-objects.c index 3dd84495b8..b9fd685b8f 100644 --- a/builtin/pack-objects.c +++ b/builtin/pack-objects.c @@ -1733,7 +1733,7 @@ static int want_object_in_pack_mtime(const struct object_id *oid, struct multi_pack_index *m = get_multi_pack_index(source); struct pack_entry e; - if (m && fill_midx_entry(the_repository, oid, &e, m)) { + if (m && fill_midx_entry(m, oid, &e)) { want = want_object_in_pack_one(e.p, oid, exclude, found_pack, found_offset, found_mtime); if (want != -1) return want; diff --git a/midx-write.c b/midx-write.c index d38caceadb..b858be475f 100644 --- a/midx-write.c +++ b/midx-write.c @@ -942,8 +942,7 @@ static int fill_packs_from_midx(struct write_midx_context *ctx, */ if (flags & MIDX_WRITE_REV_INDEX || preferred_pack_name) { - if (prepare_midx_pack(ctx->repo, m, - m->num_packs_in_base + i)) { + if (prepare_midx_pack(m, m->num_packs_in_base + i)) { error(_("could not load pack")); return 1; } @@ -1566,7 +1565,7 @@ int expire_midx_packs(struct repository *r, const char *object_dir, unsigned fla if (count[i]) continue; - if (prepare_midx_pack(r, m, i)) + if (prepare_midx_pack(m, i)) continue; if (m->packs[i]->pack_keep || m->packs[i]->is_cruft) @@ -1612,13 +1611,12 @@ static int compare_by_mtime(const void *a_, const void *b_) return 0; } -static int want_included_pack(struct repository *r, - struct multi_pack_index *m, +static int want_included_pack(struct multi_pack_index *m, int pack_kept_objects, uint32_t pack_int_id) { struct packed_git *p; - if (prepare_midx_pack(r, m, pack_int_id)) + if (prepare_midx_pack(m, pack_int_id)) return 0; p = m->packs[pack_int_id]; if (!pack_kept_objects && p->pack_keep) @@ -1640,7 +1638,7 @@ static void fill_included_packs_all(struct repository *r, repo_config_get_bool(r, "repack.packkeptobjects", &pack_kept_objects); for (i = 0; i < m->num_packs; i++) { - if (!want_included_pack(r, m, pack_kept_objects, i)) + if (!want_included_pack(m, pack_kept_objects, i)) continue; include_pack[i] = 1; @@ -1664,7 +1662,7 @@ static void fill_included_packs_batch(struct repository *r, for (i = 0; i < m->num_packs; i++) { pack_info[i].pack_int_id = i; - if (prepare_midx_pack(r, m, i)) + if (prepare_midx_pack(m, i)) continue; pack_info[i].mtime = m->packs[i]->mtime; @@ -1683,7 +1681,7 @@ static void fill_included_packs_batch(struct repository *r, struct packed_git *p = m->packs[pack_int_id]; uint64_t expected_size; - if (!want_included_pack(r, m, pack_kept_objects, pack_int_id)) + if (!want_included_pack(m, pack_kept_objects, pack_int_id)) continue; /* diff --git a/midx.c b/midx.c index b9ca0915a6..8459dda8c9 100644 --- a/midx.c +++ b/midx.c @@ -450,9 +450,10 @@ static uint32_t midx_for_pack(struct multi_pack_index **_m, return pack_int_id - m->num_packs_in_base; } -int prepare_midx_pack(struct repository *r, struct multi_pack_index *m, +int prepare_midx_pack(struct multi_pack_index *m, uint32_t pack_int_id) { + struct repository *r = m->repo; struct strbuf pack_name = STRBUF_INIT; struct strbuf key = STRBUF_INIT; struct packed_git *p; @@ -507,7 +508,7 @@ struct packed_git *nth_midxed_pack(struct multi_pack_index *m, #define MIDX_CHUNK_BITMAPPED_PACKS_WIDTH (2 * sizeof(uint32_t)) -int nth_bitmapped_pack(struct repository *r, struct multi_pack_index *m, +int nth_bitmapped_pack(struct multi_pack_index *m, struct bitmapped_pack *bp, uint32_t pack_int_id) { uint32_t local_pack_int_id = midx_for_pack(&m, pack_int_id); @@ -515,7 +516,7 @@ int nth_bitmapped_pack(struct repository *r, struct multi_pack_index *m, if (!m->chunk_bitmapped_packs) return error(_("MIDX does not contain the BTMP chunk")); - if (prepare_midx_pack(r, m, pack_int_id)) + if (prepare_midx_pack(m, pack_int_id)) return error(_("could not load bitmapped pack %"PRIu32), pack_int_id); bp->p = m->packs[local_pack_int_id]; @@ -600,10 +601,9 @@ uint32_t nth_midxed_pack_int_id(struct multi_pack_index *m, uint32_t pos) (off_t)pos * MIDX_CHUNK_OFFSET_WIDTH); } -int fill_midx_entry(struct repository *r, +int fill_midx_entry(struct multi_pack_index *m, const struct object_id *oid, - struct pack_entry *e, - struct multi_pack_index *m) + struct pack_entry *e) { uint32_t pos; uint32_t pack_int_id; @@ -615,7 +615,7 @@ int fill_midx_entry(struct repository *r, midx_for_object(&m, pos); pack_int_id = nth_midxed_pack_int_id(m, pos); - if (prepare_midx_pack(r, m, pack_int_id)) + if (prepare_midx_pack(m, pack_int_id)) return 0; p = m->packs[pack_int_id - m->num_packs_in_base]; @@ -912,7 +912,7 @@ int verify_midx_file(struct repository *r, const char *object_dir, unsigned flag _("Looking for referenced packfiles"), m->num_packs + m->num_packs_in_base); for (i = 0; i < m->num_packs + m->num_packs_in_base; i++) { - if (prepare_midx_pack(r, m, i)) + if (prepare_midx_pack(m, i)) midx_report("failed to load pack in position %d", i); display_progress(progress, i + 1); @@ -989,7 +989,7 @@ int verify_midx_file(struct repository *r, const char *object_dir, unsigned flag nth_midxed_object_oid(&oid, m, pairs[i].pos); - if (!fill_midx_entry(r, &oid, &e, m)) { + if (!fill_midx_entry(m, &oid, &e)) { midx_report(_("failed to load pack entry for oid[%d] = %s"), pairs[i].pos, oid_to_hex(&oid)); continue; diff --git a/midx.h b/midx.h index 28c426a823..f7e07083e1 100644 --- a/midx.h +++ b/midx.h @@ -103,10 +103,10 @@ void get_split_midx_filename_ext(const struct git_hash_algo *hash_algo, struct multi_pack_index *load_multi_pack_index(struct repository *r, const char *object_dir, int local); -int prepare_midx_pack(struct repository *r, struct multi_pack_index *m, uint32_t pack_int_id); +int prepare_midx_pack(struct multi_pack_index *m, uint32_t pack_int_id); struct packed_git *nth_midxed_pack(struct multi_pack_index *m, uint32_t pack_int_id); -int nth_bitmapped_pack(struct repository *r, struct multi_pack_index *m, +int nth_bitmapped_pack(struct multi_pack_index *m, struct bitmapped_pack *bp, uint32_t pack_int_id); int bsearch_one_midx(const struct object_id *oid, struct multi_pack_index *m, uint32_t *result); @@ -118,7 +118,7 @@ uint32_t nth_midxed_pack_int_id(struct multi_pack_index *m, uint32_t pos); struct object_id *nth_midxed_object_oid(struct object_id *oid, struct multi_pack_index *m, uint32_t n); -int fill_midx_entry(struct repository *r, const struct object_id *oid, struct pack_entry *e, struct multi_pack_index *m); +int fill_midx_entry(struct multi_pack_index *m, const struct object_id *oid, struct pack_entry *e); int midx_contains_pack(struct multi_pack_index *m, const char *idx_or_pack_name); int midx_preferred_pack(struct multi_pack_index *m, uint32_t *pack_int_id); diff --git a/pack-bitmap.c b/pack-bitmap.c index d14421ee20..fb0b11ca07 100644 --- a/pack-bitmap.c +++ b/pack-bitmap.c @@ -493,7 +493,7 @@ static int open_midx_bitmap_1(struct bitmap_index *bitmap_git, } for (i = 0; i < bitmap_git->midx->num_packs + bitmap_git->midx->num_packs_in_base; i++) { - if (prepare_midx_pack(bitmap_repo(bitmap_git), bitmap_git->midx, i)) { + if (prepare_midx_pack(bitmap_git->midx, i)) { warning(_("could not open pack %s"), bitmap_git->midx->pack_names[i]); goto cleanup; @@ -2466,7 +2466,7 @@ void reuse_partial_packfile_from_bitmap(struct bitmap_index *bitmap_git, struct multi_pack_index *m = bitmap_git->midx; for (i = 0; i < m->num_packs + m->num_packs_in_base; i++) { struct bitmapped_pack pack; - if (nth_bitmapped_pack(r, bitmap_git->midx, &pack, i) < 0) { + if (nth_bitmapped_pack(bitmap_git->midx, &pack, i) < 0) { warning(_("unable to load pack: '%s', disabling pack-reuse"), bitmap_git->midx->pack_names[i]); free(packs); diff --git a/packfile.c b/packfile.c index a38544b87b..acb680966d 100644 --- a/packfile.c +++ b/packfile.c @@ -1091,7 +1091,7 @@ struct packed_git *get_all_packs(struct repository *r) if (!m) continue; for (uint32_t i = 0; i < m->num_packs + m->num_packs_in_base; i++) - prepare_midx_pack(r, m, i); + prepare_midx_pack(m, i); } return r->objects->packed_git; @@ -2077,7 +2077,7 @@ int find_pack_entry(struct repository *r, const struct object_id *oid, struct pa prepare_packed_git(r); for (struct odb_source *source = r->objects->sources; source; source = source->next) - if (source->midx && fill_midx_entry(r, oid, e, source->midx)) + if (source->midx && fill_midx_entry(source->midx, oid, e)) return 1; if (!r->objects->packed_git) diff --git a/t/helper/test-read-midx.c b/t/helper/test-read-midx.c index da2aa036b5..e430aa247c 100644 --- a/t/helper/test-read-midx.c +++ b/t/helper/test-read-midx.c @@ -65,7 +65,7 @@ static int read_midx_file(const char *object_dir, const char *checksum, for (i = 0; i < m->num_objects; i++) { nth_midxed_object_oid(&oid, m, i + m->num_objects_in_base); - fill_midx_entry(the_repository, &oid, &e, m); + fill_midx_entry(m, &oid, &e); printf("%s %"PRIu64"\t%s\n", oid_to_hex(&oid), e.offset, e.p->pack_name); @@ -126,7 +126,7 @@ static int read_midx_bitmapped_packs(const char *object_dir) return 1; for (i = 0; i < midx->num_packs + midx->num_packs_in_base; i++) { - if (nth_bitmapped_pack(the_repository, midx, &pack, i) < 0) { + if (nth_bitmapped_pack(midx, &pack, i) < 0) { close_midx(midx); return 1; } From 017db7bb14246dea55b678fc20e34ce91c28968a Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Mon, 11 Aug 2025 15:46:47 +0200 Subject: [PATCH 07/10] midx: load multi-pack indices via their source To load a multi-pack index the caller is expected to pass both the repository and the object directory where the multi-pack index is located. While this works, this layout has a couple of downsides: - We need to pass in information reduntant with the owning source, namely its object directory and whether the source is local or not. - We don't have access to the source when loading the multi-pack index. If we had that access, we could store a pointer to the owning source in the MIDX and thus deduplicate some information. - Multi-pack indices are inherently specific to the object source and its format. With the goal of pluggable object backends in mind we will eventually want the backends to own the logic of reading and writing multi-pack indices. Making the logic work on top of object sources is a step into that direction. Refactor loading of multi-pack indices accordingly. This surfaces one small problem though: git-multi-pack-index(1) and our MIDX test helper both know to read and write multi-pack-indices located in a different object directory. This issue is addressed by adding the user-provided object directory as an in-memory alternate. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- builtin/multi-pack-index.c | 18 ++++++++++-- midx.c | 57 ++++++++++++++++--------------------- midx.h | 6 ++-- t/helper/test-read-midx.c | 25 +++++++++------- t/t5319-multi-pack-index.sh | 8 +++--- 5 files changed, 62 insertions(+), 52 deletions(-) diff --git a/builtin/multi-pack-index.c b/builtin/multi-pack-index.c index aa25b06f9d..e4a9305af3 100644 --- a/builtin/multi-pack-index.c +++ b/builtin/multi-pack-index.c @@ -64,12 +64,20 @@ static int parse_object_dir(const struct option *opt, const char *arg, char **value = opt->value; free(*value); if (unset) - *value = xstrdup(repo_get_object_directory(the_repository)); + *value = xstrdup(the_repository->objects->sources->path); else *value = real_pathdup(arg, 1); return 0; } +static struct odb_source *handle_object_dir_option(struct repository *repo) +{ + struct odb_source *source = odb_find_source(repo->objects, opts.object_dir); + if (!source) + source = odb_add_to_alternates_memory(repo->objects, opts.object_dir); + return source; +} + static struct option common_opts[] = { OPT_CALLBACK(0, "object-dir", &opts.object_dir, N_("directory"), @@ -157,6 +165,7 @@ static int cmd_multi_pack_index_write(int argc, const char **argv, if (argc) usage_with_options(builtin_multi_pack_index_write_usage, options); + handle_object_dir_option(repo); FREE_AND_NULL(options); @@ -193,6 +202,8 @@ static int cmd_multi_pack_index_verify(int argc, const char **argv, N_("force progress reporting"), MIDX_PROGRESS), OPT_END(), }; + struct odb_source *source; + options = add_common_options(builtin_multi_pack_index_verify_options); trace2_cmd_mode(argv[0]); @@ -205,10 +216,11 @@ static int cmd_multi_pack_index_verify(int argc, const char **argv, if (argc) usage_with_options(builtin_multi_pack_index_verify_usage, options); + source = handle_object_dir_option(the_repository); FREE_AND_NULL(options); - return verify_midx_file(the_repository, opts.object_dir, opts.flags); + return verify_midx_file(source, opts.flags); } static int cmd_multi_pack_index_expire(int argc, const char **argv, @@ -233,6 +245,7 @@ static int cmd_multi_pack_index_expire(int argc, const char **argv, if (argc) usage_with_options(builtin_multi_pack_index_expire_usage, options); + handle_object_dir_option(the_repository); FREE_AND_NULL(options); @@ -265,6 +278,7 @@ static int cmd_multi_pack_index_repack(int argc, const char **argv, if (argc) usage_with_options(builtin_multi_pack_index_repack_usage, options); + handle_object_dir_option(the_repository); FREE_AND_NULL(options); diff --git a/midx.c b/midx.c index 8459dda8c9..831a7e9b5f 100644 --- a/midx.c +++ b/midx.c @@ -95,11 +95,10 @@ static int midx_read_object_offsets(const unsigned char *chunk_start, return 0; } -static struct multi_pack_index *load_multi_pack_index_one(struct repository *r, - const char *object_dir, - const char *midx_name, - int local) +static struct multi_pack_index *load_multi_pack_index_one(struct odb_source *source, + const char *midx_name) { + struct repository *r = source->odb->repo; struct multi_pack_index *m = NULL; int fd; struct stat st; @@ -129,10 +128,10 @@ static struct multi_pack_index *load_multi_pack_index_one(struct repository *r, midx_map = xmmap(NULL, midx_size, PROT_READ, MAP_PRIVATE, fd, 0); close(fd); - FLEX_ALLOC_STR(m, object_dir, object_dir); + FLEX_ALLOC_STR(m, object_dir, source->path); m->data = midx_map; m->data_len = midx_size; - m->local = local; + m->local = source->local; m->repo = r; m->signature = get_be32(m->data); @@ -297,19 +296,18 @@ static int add_midx_to_chain(struct multi_pack_index *midx, return 1; } -static struct multi_pack_index *load_midx_chain_fd_st(struct repository *r, - const char *object_dir, - int local, +static struct multi_pack_index *load_midx_chain_fd_st(struct odb_source *source, int fd, struct stat *st, int *incomplete_chain) { + const struct git_hash_algo *hash_algo = source->odb->repo->hash_algo; struct multi_pack_index *midx_chain = NULL; struct strbuf buf = STRBUF_INIT; int valid = 1; uint32_t i, count; FILE *fp = xfdopen(fd, "r"); - count = st->st_size / (r->hash_algo->hexsz + 1); + count = st->st_size / (hash_algo->hexsz + 1); for (i = 0; i < count; i++) { struct multi_pack_index *m; @@ -318,7 +316,7 @@ static struct multi_pack_index *load_midx_chain_fd_st(struct repository *r, if (strbuf_getline_lf(&buf, fp) == EOF) break; - if (get_oid_hex_algop(buf.buf, &layer, r->hash_algo)) { + if (get_oid_hex_algop(buf.buf, &layer, hash_algo)) { warning(_("invalid multi-pack-index chain: line '%s' " "not a hash"), buf.buf); @@ -329,9 +327,9 @@ static struct multi_pack_index *load_midx_chain_fd_st(struct repository *r, valid = 0; strbuf_reset(&buf); - get_split_midx_filename_ext(r->hash_algo, &buf, object_dir, + get_split_midx_filename_ext(hash_algo, &buf, source->path, layer.hash, MIDX_EXT_MIDX); - m = load_multi_pack_index_one(r, object_dir, buf.buf, local); + m = load_multi_pack_index_one(source, buf.buf); if (m) { if (add_midx_to_chain(m, midx_chain)) { @@ -354,40 +352,35 @@ static struct multi_pack_index *load_midx_chain_fd_st(struct repository *r, return midx_chain; } -static struct multi_pack_index *load_multi_pack_index_chain(struct repository *r, - const char *object_dir, - int local) +static struct multi_pack_index *load_multi_pack_index_chain(struct odb_source *source) { struct strbuf chain_file = STRBUF_INIT; struct stat st; int fd; struct multi_pack_index *m = NULL; - get_midx_chain_filename(&chain_file, object_dir); - if (open_multi_pack_index_chain(r->hash_algo, chain_file.buf, &fd, &st)) { + get_midx_chain_filename(&chain_file, source->path); + if (open_multi_pack_index_chain(source->odb->repo->hash_algo, chain_file.buf, &fd, &st)) { int incomplete; /* ownership of fd is taken over by load function */ - m = load_midx_chain_fd_st(r, object_dir, local, fd, &st, - &incomplete); + m = load_midx_chain_fd_st(source, fd, &st, &incomplete); } strbuf_release(&chain_file); return m; } -struct multi_pack_index *load_multi_pack_index(struct repository *r, - const char *object_dir, - int local) +struct multi_pack_index *load_multi_pack_index(struct odb_source *source) { struct strbuf midx_name = STRBUF_INIT; struct multi_pack_index *m; - get_midx_filename(r->hash_algo, &midx_name, object_dir); + get_midx_filename(source->odb->repo->hash_algo, &midx_name, + source->path); - m = load_multi_pack_index_one(r, object_dir, - midx_name.buf, local); + m = load_multi_pack_index_one(source, midx_name.buf); if (!m) - m = load_multi_pack_index_chain(r, object_dir, local); + m = load_multi_pack_index_chain(source); strbuf_release(&midx_name); @@ -734,8 +727,7 @@ int prepare_multi_pack_index_one(struct odb_source *source) if (source->midx) return 1; - source->midx = load_multi_pack_index(r, source->path, - source->local); + source->midx = load_multi_pack_index(source); return !!source->midx; } @@ -880,12 +872,13 @@ static int compare_pair_pos_vs_id(const void *_a, const void *_b) display_progress(progress, _n); \ } while (0) -int verify_midx_file(struct repository *r, const char *object_dir, unsigned flags) +int verify_midx_file(struct odb_source *source, unsigned flags) { + struct repository *r = source->odb->repo; struct pair_pos_vs_id *pairs = NULL; uint32_t i; struct progress *progress = NULL; - struct multi_pack_index *m = load_multi_pack_index(r, object_dir, 1); + struct multi_pack_index *m = load_multi_pack_index(source); struct multi_pack_index *curr; verify_midx_error = 0; @@ -894,7 +887,7 @@ int verify_midx_file(struct repository *r, const char *object_dir, unsigned flag struct stat sb; struct strbuf filename = STRBUF_INIT; - get_midx_filename(r->hash_algo, &filename, object_dir); + get_midx_filename(r->hash_algo, &filename, source->path); if (!stat(filename.buf, &sb)) { error(_("multi-pack-index file exists, but failed to parse")); diff --git a/midx.h b/midx.h index f7e07083e1..970d043989 100644 --- a/midx.h +++ b/midx.h @@ -100,9 +100,7 @@ void get_split_midx_filename_ext(const struct git_hash_algo *hash_algo, struct strbuf *buf, const char *object_dir, const unsigned char *hash, const char *ext); -struct multi_pack_index *load_multi_pack_index(struct repository *r, - const char *object_dir, - int local); +struct multi_pack_index *load_multi_pack_index(struct odb_source *source); int prepare_midx_pack(struct multi_pack_index *m, uint32_t pack_int_id); struct packed_git *nth_midxed_pack(struct multi_pack_index *m, uint32_t pack_int_id); @@ -136,7 +134,7 @@ int write_midx_file_only(struct repository *r, const char *object_dir, const char *preferred_pack_name, const char *refs_snapshot, unsigned flags); void clear_midx_file(struct repository *r); -int verify_midx_file(struct repository *r, const char *object_dir, unsigned flags); +int verify_midx_file(struct odb_source *source, unsigned flags); int expire_midx_packs(struct repository *r, const char *object_dir, unsigned flags); int midx_repack(struct repository *r, const char *object_dir, size_t batch_size, unsigned flags); diff --git a/t/helper/test-read-midx.c b/t/helper/test-read-midx.c index e430aa247c..bcb8ea7671 100644 --- a/t/helper/test-read-midx.c +++ b/t/helper/test-read-midx.c @@ -11,14 +11,24 @@ #include "gettext.h" #include "pack-revindex.h" +static struct multi_pack_index *setup_midx(const char *object_dir) +{ + struct odb_source *source; + setup_git_directory(); + source = odb_find_source(the_repository->objects, object_dir); + if (!source) + source = odb_add_to_alternates_memory(the_repository->objects, + object_dir); + return load_multi_pack_index(source); +} + static int read_midx_file(const char *object_dir, const char *checksum, int show_objects) { uint32_t i; struct multi_pack_index *m; - setup_git_directory(); - m = load_multi_pack_index(the_repository, object_dir, 1); + m = setup_midx(object_dir); if (!m) return 1; @@ -81,8 +91,7 @@ static int read_midx_checksum(const char *object_dir) { struct multi_pack_index *m; - setup_git_directory(); - m = load_multi_pack_index(the_repository, object_dir, 1); + m = setup_midx(object_dir); if (!m) return 1; printf("%s\n", hash_to_hex(get_midx_checksum(m))); @@ -96,9 +105,7 @@ static int read_midx_preferred_pack(const char *object_dir) struct multi_pack_index *midx = NULL; uint32_t preferred_pack; - setup_git_directory(); - - midx = load_multi_pack_index(the_repository, object_dir, 1); + midx = setup_midx(object_dir); if (!midx) return 1; @@ -119,9 +126,7 @@ static int read_midx_bitmapped_packs(const char *object_dir) struct bitmapped_pack pack; uint32_t i; - setup_git_directory(); - - midx = load_multi_pack_index(the_repository, object_dir, 1); + midx = setup_midx(object_dir); if (!midx) return 1; diff --git a/t/t5319-multi-pack-index.sh b/t/t5319-multi-pack-index.sh index bd75dea950..4e5e882989 100755 --- a/t/t5319-multi-pack-index.sh +++ b/t/t5319-multi-pack-index.sh @@ -28,11 +28,11 @@ midx_read_expect () { EOF if test $NUM_PACKS -ge 1 then - ls $OBJECT_DIR/pack/ | grep idx | sort + ls "$OBJECT_DIR"/pack/ | grep idx | sort fi && printf "object-dir: $OBJECT_DIR\n" } >expect && - test-tool read-midx $OBJECT_DIR >actual && + test-tool read-midx "$OBJECT_DIR" >actual && test_cmp expect actual } @@ -305,7 +305,7 @@ test_expect_success 'midx picks objects from preferred pack' ' ofs=$(git show-index expect && grep ^$b out >actual && @@ -639,7 +639,7 @@ test_expect_success 'force some 64-bit offsets with pack-objects' ' ( cd ../objects64 && pwd ) >.git/objects/info/alternates && midx64=$(git multi-pack-index --object-dir=../objects64 write) ) && - midx_read_expect 1 63 5 objects64 " large-offsets" + midx_read_expect 1 63 5 "$(pwd)/objects64" " large-offsets" ' test_expect_success 'verify multi-pack-index with 64-bit offsets' ' From c3f5d251469525a52074b0373671a588f0e5b972 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Mon, 11 Aug 2025 15:46:48 +0200 Subject: [PATCH 08/10] midx: write multi-pack indices via their source Similar to the preceding commit, refactor the writing side of multi-pack indices so that we pass in the object database source where the index should be written to. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- builtin/multi-pack-index.c | 19 ++++++----- builtin/repack.c | 2 +- midx-write.c | 67 ++++++++++++++++++-------------------- midx.h | 8 ++--- 4 files changed, 47 insertions(+), 49 deletions(-) diff --git a/builtin/multi-pack-index.c b/builtin/multi-pack-index.c index e4a9305af3..b1e971e535 100644 --- a/builtin/multi-pack-index.c +++ b/builtin/multi-pack-index.c @@ -147,6 +147,7 @@ static int cmd_multi_pack_index_write(int argc, const char **argv, N_("refs snapshot for selecting bitmap commits")), OPT_END(), }; + struct odb_source *source; int ret; opts.flags |= MIDX_WRITE_BITMAP_HASH_CACHE; @@ -165,7 +166,7 @@ static int cmd_multi_pack_index_write(int argc, const char **argv, if (argc) usage_with_options(builtin_multi_pack_index_write_usage, options); - handle_object_dir_option(repo); + source = handle_object_dir_option(repo); FREE_AND_NULL(options); @@ -174,7 +175,7 @@ static int cmd_multi_pack_index_write(int argc, const char **argv, read_packs_from_stdin(&packs); - ret = write_midx_file_only(repo, opts.object_dir, &packs, + ret = write_midx_file_only(source, &packs, opts.preferred_pack, opts.refs_snapshot, opts.flags); @@ -185,7 +186,7 @@ static int cmd_multi_pack_index_write(int argc, const char **argv, } - ret = write_midx_file(repo, opts.object_dir, opts.preferred_pack, + ret = write_midx_file(source, opts.preferred_pack, opts.refs_snapshot, opts.flags); free(opts.refs_snapshot); @@ -233,6 +234,8 @@ static int cmd_multi_pack_index_expire(int argc, const char **argv, N_("force progress reporting"), MIDX_PROGRESS), OPT_END(), }; + struct odb_source *source; + options = add_common_options(builtin_multi_pack_index_expire_options); trace2_cmd_mode(argv[0]); @@ -245,11 +248,11 @@ static int cmd_multi_pack_index_expire(int argc, const char **argv, if (argc) usage_with_options(builtin_multi_pack_index_expire_usage, options); - handle_object_dir_option(the_repository); + source = handle_object_dir_option(the_repository); FREE_AND_NULL(options); - return expire_midx_packs(the_repository, opts.object_dir, opts.flags); + return expire_midx_packs(source, opts.flags); } static int cmd_multi_pack_index_repack(int argc, const char **argv, @@ -264,6 +267,7 @@ static int cmd_multi_pack_index_repack(int argc, const char **argv, N_("force progress reporting"), MIDX_PROGRESS), OPT_END(), }; + struct odb_source *source; options = add_common_options(builtin_multi_pack_index_repack_options); @@ -278,12 +282,11 @@ static int cmd_multi_pack_index_repack(int argc, const char **argv, if (argc) usage_with_options(builtin_multi_pack_index_repack_usage, options); - handle_object_dir_option(the_repository); + source = handle_object_dir_option(the_repository); FREE_AND_NULL(options); - return midx_repack(the_repository, opts.object_dir, - (size_t)opts.batch_size, opts.flags); + return midx_repack(source, (size_t)opts.batch_size, opts.flags); } int cmd_multi_pack_index(int argc, diff --git a/builtin/repack.c b/builtin/repack.c index 21723866b9..94dec26f18 100644 --- a/builtin/repack.c +++ b/builtin/repack.c @@ -1711,7 +1711,7 @@ int cmd_repack(int argc, unsigned flags = 0; if (git_env_bool(GIT_TEST_MULTI_PACK_INDEX_WRITE_INCREMENTAL, 0)) flags |= MIDX_WRITE_INCREMENTAL; - write_midx_file(the_repository, repo_get_object_directory(the_repository), + write_midx_file(the_repository->objects->sources, NULL, NULL, flags); } diff --git a/midx-write.c b/midx-write.c index b858be475f..bf7c01d4b1 100644 --- a/midx-write.c +++ b/midx-write.c @@ -913,13 +913,6 @@ cleanup: return ret; } -static struct multi_pack_index *lookup_multi_pack_index(struct repository *r, - const char *object_dir) -{ - struct odb_source *source = odb_find_source_or_die(r->objects, object_dir); - return get_multi_pack_index(source); -} - static int fill_packs_from_midx(struct write_midx_context *ctx, const char *preferred_pack_name, uint32_t flags) { @@ -1010,7 +1003,7 @@ done: return ret; } -static void clear_midx_files(struct repository *r, const char *object_dir, +static void clear_midx_files(struct odb_source *source, const char **hashes, uint32_t hashes_nr, unsigned incremental) { @@ -1029,16 +1022,16 @@ static void clear_midx_files(struct repository *r, const char *object_dir, uint32_t i, j; for (i = 0; i < ARRAY_SIZE(exts); i++) { - clear_incremental_midx_files_ext(object_dir, exts[i], + clear_incremental_midx_files_ext(source->path, exts[i], hashes, hashes_nr); for (j = 0; j < hashes_nr; j++) - clear_midx_files_ext(object_dir, exts[i], hashes[j]); + clear_midx_files_ext(source->path, exts[i], hashes[j]); } if (incremental) - get_midx_filename(r->hash_algo, &buf, object_dir); + get_midx_filename(source->odb->repo->hash_algo, &buf, source->path); else - get_midx_chain_filename(&buf, object_dir); + get_midx_chain_filename(&buf, source->path); if (unlink(buf.buf) && errno != ENOENT) die_errno(_("failed to clear multi-pack-index at %s"), buf.buf); @@ -1046,13 +1039,14 @@ static void clear_midx_files(struct repository *r, const char *object_dir, strbuf_release(&buf); } -static int write_midx_internal(struct repository *r, const char *object_dir, +static int write_midx_internal(struct odb_source *source, struct string_list *packs_to_include, struct string_list *packs_to_drop, const char *preferred_pack_name, const char *refs_snapshot, unsigned flags) { + struct repository *r = source->odb->repo; struct strbuf midx_name = STRBUF_INIT; unsigned char midx_hash[GIT_MAX_RAWSZ]; uint32_t i, start_pack; @@ -1076,15 +1070,15 @@ static int write_midx_internal(struct repository *r, const char *object_dir, if (ctx.incremental) strbuf_addf(&midx_name, "%s/pack/multi-pack-index.d/tmp_midx_XXXXXX", - object_dir); + source->path); else - get_midx_filename(r->hash_algo, &midx_name, object_dir); + get_midx_filename(r->hash_algo, &midx_name, source->path); if (safe_create_leading_directories(r, midx_name.buf)) die_errno(_("unable to create leading directories of %s"), midx_name.buf); if (!packs_to_include || ctx.incremental) { - struct multi_pack_index *m = lookup_multi_pack_index(r, object_dir); + struct multi_pack_index *m = get_multi_pack_index(source); if (m && !midx_checksum_valid(m)) { warning(_("ignoring existing multi-pack-index; checksum mismatch")); m = NULL; @@ -1138,7 +1132,7 @@ static int write_midx_internal(struct repository *r, const char *object_dir, ctx.to_include = packs_to_include; - for_each_file_in_pack_dir(object_dir, add_pack_to_midx, &ctx); + for_each_file_in_pack_dir(source->path, add_pack_to_midx, &ctx); stop_progress(&ctx.progress); if ((ctx.m && ctx.nr == ctx.m->num_packs + ctx.m->num_packs_in_base) && @@ -1158,7 +1152,7 @@ static int write_midx_internal(struct repository *r, const char *object_dir, * corresponding bitmap (or one wasn't requested). */ if (!want_bitmap) - clear_midx_files_ext(object_dir, "bitmap", NULL); + clear_midx_files_ext(source->path, "bitmap", NULL); goto cleanup; } } @@ -1326,7 +1320,7 @@ static int write_midx_internal(struct repository *r, const char *object_dir, if (ctx.incremental) { struct strbuf lock_name = STRBUF_INIT; - get_midx_chain_filename(&lock_name, object_dir); + get_midx_chain_filename(&lock_name, source->path); hold_lock_file_for_update(&lk, lock_name.buf, LOCK_DIE_ON_ERROR); strbuf_release(&lock_name); @@ -1389,7 +1383,7 @@ static int write_midx_internal(struct repository *r, const char *object_dir, if (flags & MIDX_WRITE_REV_INDEX && git_env_bool("GIT_TEST_MIDX_WRITE_REV", 0)) - write_midx_reverse_index(&ctx, object_dir, midx_hash); + write_midx_reverse_index(&ctx, source->path, midx_hash); if (flags & MIDX_WRITE_BITMAP) { struct packing_data pdata; @@ -1412,7 +1406,7 @@ static int write_midx_internal(struct repository *r, const char *object_dir, FREE_AND_NULL(ctx.entries); ctx.entries_nr = 0; - if (write_midx_bitmap(&ctx, object_dir, + if (write_midx_bitmap(&ctx, source->path, midx_hash, &pdata, commits, commits_nr, flags) < 0) { error(_("could not write multi-pack bitmap")); @@ -1446,7 +1440,7 @@ static int write_midx_internal(struct repository *r, const char *object_dir, return -1; get_split_midx_filename_ext(r->hash_algo, &final_midx_name, - object_dir, midx_hash, MIDX_EXT_MIDX); + source->path, midx_hash, MIDX_EXT_MIDX); if (rename_tempfile(&incr, final_midx_name.buf) < 0) { error_errno(_("unable to rename new multi-pack-index layer")); @@ -1479,7 +1473,7 @@ static int write_midx_internal(struct repository *r, const char *object_dir, if (commit_lock_file(&lk) < 0) die_errno(_("could not write multi-pack-index")); - clear_midx_files(r, object_dir, keep_hashes, + clear_midx_files(source, keep_hashes, ctx.num_multi_pack_indexes_before + 1, ctx.incremental); @@ -1508,29 +1502,29 @@ cleanup: return result; } -int write_midx_file(struct repository *r, const char *object_dir, +int write_midx_file(struct odb_source *source, const char *preferred_pack_name, const char *refs_snapshot, unsigned flags) { - return write_midx_internal(r, object_dir, NULL, NULL, + return write_midx_internal(source, NULL, NULL, preferred_pack_name, refs_snapshot, flags); } -int write_midx_file_only(struct repository *r, const char *object_dir, +int write_midx_file_only(struct odb_source *source, struct string_list *packs_to_include, const char *preferred_pack_name, const char *refs_snapshot, unsigned flags) { - return write_midx_internal(r, object_dir, packs_to_include, NULL, + return write_midx_internal(source, packs_to_include, NULL, preferred_pack_name, refs_snapshot, flags); } -int expire_midx_packs(struct repository *r, const char *object_dir, unsigned flags) +int expire_midx_packs(struct odb_source *source, unsigned flags) { uint32_t i, *count, result = 0; struct string_list packs_to_drop = STRING_LIST_INIT_DUP; - struct multi_pack_index *m = lookup_multi_pack_index(r, object_dir); + struct multi_pack_index *m = get_multi_pack_index(source); struct progress *progress = NULL; if (!m) @@ -1543,7 +1537,7 @@ int expire_midx_packs(struct repository *r, const char *object_dir, unsigned fla if (flags & MIDX_PROGRESS) progress = start_delayed_progress( - r, + source->odb->repo, _("Counting referenced objects"), m->num_objects); for (i = 0; i < m->num_objects; i++) { @@ -1555,7 +1549,7 @@ int expire_midx_packs(struct repository *r, const char *object_dir, unsigned fla if (flags & MIDX_PROGRESS) progress = start_delayed_progress( - r, + source->odb->repo, _("Finding and deleting unreferenced packfiles"), m->num_packs); for (i = 0; i < m->num_packs; i++) { @@ -1583,7 +1577,7 @@ int expire_midx_packs(struct repository *r, const char *object_dir, unsigned fla free(count); if (packs_to_drop.nr) - result = write_midx_internal(r, object_dir, NULL, + result = write_midx_internal(source, NULL, &packs_to_drop, NULL, NULL, flags); string_list_clear(&packs_to_drop, 0); @@ -1708,14 +1702,15 @@ static void fill_included_packs_batch(struct repository *r, free(pack_info); } -int midx_repack(struct repository *r, const char *object_dir, size_t batch_size, unsigned flags) +int midx_repack(struct odb_source *source, size_t batch_size, unsigned flags) { + struct repository *r = source->odb->repo; int result = 0; uint32_t i, packs_to_repack = 0; unsigned char *include_pack; struct child_process cmd = CHILD_PROCESS_INIT; FILE *cmd_in; - struct multi_pack_index *m = lookup_multi_pack_index(r, object_dir); + struct multi_pack_index *m = get_multi_pack_index(source); /* * When updating the default for these configuration @@ -1749,7 +1744,7 @@ int midx_repack(struct repository *r, const char *object_dir, size_t batch_size, strvec_push(&cmd.args, "pack-objects"); - strvec_pushf(&cmd.args, "%s/pack/pack", object_dir); + strvec_pushf(&cmd.args, "%s/pack/pack", source->path); if (delta_base_offset) strvec_push(&cmd.args, "--delta-base-offset"); @@ -1790,7 +1785,7 @@ int midx_repack(struct repository *r, const char *object_dir, size_t batch_size, goto cleanup; } - result = write_midx_internal(r, object_dir, NULL, NULL, NULL, NULL, + result = write_midx_internal(source, NULL, NULL, NULL, NULL, flags); cleanup: diff --git a/midx.h b/midx.h index 970d043989..d162001fbb 100644 --- a/midx.h +++ b/midx.h @@ -126,17 +126,17 @@ int prepare_multi_pack_index_one(struct odb_source *source); * Variant of write_midx_file which writes a MIDX containing only the packs * specified in packs_to_include. */ -int write_midx_file(struct repository *r, const char *object_dir, +int write_midx_file(struct odb_source *source, const char *preferred_pack_name, const char *refs_snapshot, unsigned flags); -int write_midx_file_only(struct repository *r, const char *object_dir, +int write_midx_file_only(struct odb_source *source, struct string_list *packs_to_include, const char *preferred_pack_name, const char *refs_snapshot, unsigned flags); void clear_midx_file(struct repository *r); int verify_midx_file(struct odb_source *source, unsigned flags); -int expire_midx_packs(struct repository *r, const char *object_dir, unsigned flags); -int midx_repack(struct repository *r, const char *object_dir, size_t batch_size, unsigned flags); +int expire_midx_packs(struct odb_source *source, unsigned flags); +int midx_repack(struct odb_source *source, size_t batch_size, unsigned flags); void close_midx(struct multi_pack_index *m); From 7744936f374308d6fa3c6e317fb8fe0b685d0ef2 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Mon, 11 Aug 2025 15:46:49 +0200 Subject: [PATCH 09/10] midx: stop duplicating info redundant with its owning source Multi-pack indices store some information that is redundant with their owning source: - The locality bit that tracks whether the source is the primary object source or an alternate. - The object directory path the multi-pack index is located in. - The pointer to the owning parent directory. All of this information is already contained in `struct odb_source`. So now that we always have that struct available when loading a multi-pack index we have it readily accessible. Drop the redundant information and instead store a pointer to the object source. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- builtin/repack.c | 5 +++-- midx-write.c | 9 +++++---- midx.c | 21 +++++++++++---------- midx.h | 7 ++----- pack-bitmap.c | 13 +++++++------ pack-revindex.c | 14 +++++++------- t/helper/test-read-midx.c | 2 +- 7 files changed, 36 insertions(+), 35 deletions(-) diff --git a/builtin/repack.c b/builtin/repack.c index 94dec26f18..5af3e27357 100644 --- a/builtin/repack.c +++ b/builtin/repack.c @@ -223,9 +223,10 @@ static void mark_packs_for_deletion(struct existing_packs *existing, static void remove_redundant_pack(const char *dir_name, const char *base_name) { struct strbuf buf = STRBUF_INIT; - struct multi_pack_index *m = get_multi_pack_index(the_repository->objects->sources); + struct odb_source *source = the_repository->objects->sources; + struct multi_pack_index *m = get_multi_pack_index(source); strbuf_addf(&buf, "%s.pack", base_name); - if (m && m->local && midx_contains_pack(m, buf.buf)) + if (m && source->local && midx_contains_pack(m, buf.buf)) clear_midx_file(the_repository); strbuf_insertf(&buf, 0, "%s/", dir_name); unlink_pack_path(buf.buf, 1); diff --git a/midx-write.c b/midx-write.c index bf7c01d4b1..84f76856d6 100644 --- a/midx-write.c +++ b/midx-write.c @@ -981,10 +981,11 @@ static int link_midx_to_chain(struct multi_pack_index *m) for (i = 0; i < ARRAY_SIZE(midx_exts); i++) { const unsigned char *hash = get_midx_checksum(m); - get_midx_filename_ext(m->repo->hash_algo, &from, m->object_dir, + get_midx_filename_ext(m->source->odb->repo->hash_algo, &from, + m->source->path, hash, midx_exts[i].non_split); - get_split_midx_filename_ext(m->repo->hash_algo, &to, - m->object_dir, hash, + get_split_midx_filename_ext(m->source->odb->repo->hash_algo, &to, + m->source->path, hash, midx_exts[i].split); if (link(from.buf, to.buf) < 0 && errno != ENOENT) { @@ -1109,7 +1110,7 @@ static int write_midx_internal(struct odb_source *source, if (flags & MIDX_WRITE_BITMAP && load_midx_revindex(m)) { error(_("could not load reverse index for MIDX %s"), hash_to_hex_algop(get_midx_checksum(m), - m->repo->hash_algo)); + m->source->odb->repo->hash_algo)); result = 1; goto cleanup; } diff --git a/midx.c b/midx.c index 831a7e9b5f..81bf3c4d5f 100644 --- a/midx.c +++ b/midx.c @@ -26,7 +26,7 @@ int cmp_idx_or_pack_name(const char *idx_or_pack_name, const unsigned char *get_midx_checksum(struct multi_pack_index *m) { - return m->data + m->data_len - m->repo->hash_algo->rawsz; + return m->data + m->data_len - m->source->odb->repo->hash_algo->rawsz; } void get_midx_filename(const struct git_hash_algo *hash_algo, @@ -128,11 +128,10 @@ static struct multi_pack_index *load_multi_pack_index_one(struct odb_source *sou midx_map = xmmap(NULL, midx_size, PROT_READ, MAP_PRIVATE, fd, 0); close(fd); - FLEX_ALLOC_STR(m, object_dir, source->path); + CALLOC_ARRAY(m, 1); m->data = midx_map; m->data_len = midx_size; - m->local = source->local; - m->repo = r; + m->source = source; m->signature = get_be32(m->data); if (m->signature != MIDX_SIGNATURE) @@ -446,7 +445,7 @@ static uint32_t midx_for_pack(struct multi_pack_index **_m, int prepare_midx_pack(struct multi_pack_index *m, uint32_t pack_int_id) { - struct repository *r = m->repo; + struct repository *r = m->source->odb->repo; struct strbuf pack_name = STRBUF_INIT; struct strbuf key = STRBUF_INIT; struct packed_git *p; @@ -458,7 +457,7 @@ int prepare_midx_pack(struct multi_pack_index *m, if (m->packs[pack_int_id]) return 0; - strbuf_addf(&pack_name, "%s/pack/%s", m->object_dir, + strbuf_addf(&pack_name, "%s/pack/%s", m->source->path, m->pack_names[pack_int_id]); /* pack_map holds the ".pack" name, but we have the .idx */ @@ -469,7 +468,8 @@ int prepare_midx_pack(struct multi_pack_index *m, strhash(key.buf), key.buf, struct packed_git, packmap_ent); if (!p) { - p = add_packed_git(r, pack_name.buf, pack_name.len, m->local); + p = add_packed_git(r, pack_name.buf, pack_name.len, + m->source->local); if (p) { install_packed_git(r, p); list_add_tail(&p->mru, &r->objects->packed_git_mru); @@ -528,7 +528,8 @@ int bsearch_one_midx(const struct object_id *oid, struct multi_pack_index *m, uint32_t *result) { int ret = bsearch_hash(oid->hash, m->chunk_oid_fanout, - m->chunk_oid_lookup, m->repo->hash_algo->rawsz, + m->chunk_oid_lookup, + m->source->odb->repo->hash_algo->rawsz, result); if (result) *result += m->num_objects_in_base; @@ -559,7 +560,7 @@ struct object_id *nth_midxed_object_oid(struct object_id *oid, n = midx_for_object(&m, n); oidread(oid, m->chunk_oid_lookup + st_mult(m->hash_len, n), - m->repo->hash_algo); + m->source->odb->repo->hash_algo); return oid; } @@ -734,7 +735,7 @@ int prepare_multi_pack_index_one(struct odb_source *source) int midx_checksum_valid(struct multi_pack_index *m) { - return hashfile_checksum_valid(m->repo->hash_algo, + return hashfile_checksum_valid(m->source->odb->repo->hash_algo, m->data, m->data_len); } diff --git a/midx.h b/midx.h index d162001fbb..71dbdec66e 100644 --- a/midx.h +++ b/midx.h @@ -35,6 +35,8 @@ struct odb_source; "GIT_TEST_MULTI_PACK_INDEX_WRITE_INCREMENTAL" struct multi_pack_index { + struct odb_source *source; + const unsigned char *data; size_t data_len; @@ -50,7 +52,6 @@ struct multi_pack_index { uint32_t num_objects; int preferred_pack_idx; - int local; int has_chain; const unsigned char *chunk_pack_names; @@ -71,10 +72,6 @@ struct multi_pack_index { const char **pack_names; struct packed_git **packs; - - struct repository *repo; - - char object_dir[FLEX_ARRAY]; }; #define MIDX_PROGRESS (1 << 0) diff --git a/pack-bitmap.c b/pack-bitmap.c index fb0b11ca07..01e14c34bd 100644 --- a/pack-bitmap.c +++ b/pack-bitmap.c @@ -216,7 +216,7 @@ static uint32_t bitmap_num_objects(struct bitmap_index *index) static struct repository *bitmap_repo(struct bitmap_index *bitmap_git) { if (bitmap_is_midx(bitmap_git)) - return bitmap_git->midx->repo; + return bitmap_git->midx->source->odb->repo; return bitmap_git->pack->repo; } @@ -418,13 +418,13 @@ char *midx_bitmap_filename(struct multi_pack_index *midx) { struct strbuf buf = STRBUF_INIT; if (midx->has_chain) - get_split_midx_filename_ext(midx->repo->hash_algo, &buf, - midx->object_dir, + get_split_midx_filename_ext(midx->source->odb->repo->hash_algo, &buf, + midx->source->path, get_midx_checksum(midx), MIDX_EXT_BITMAP); else - get_midx_filename_ext(midx->repo->hash_algo, &buf, - midx->object_dir, get_midx_checksum(midx), + get_midx_filename_ext(midx->source->odb->repo->hash_algo, &buf, + midx->source->path, get_midx_checksum(midx), MIDX_EXT_BITMAP); return strbuf_detach(&buf, NULL); @@ -463,7 +463,8 @@ static int open_midx_bitmap_1(struct bitmap_index *bitmap_git, if (bitmap_git->pack || bitmap_git->midx) { struct strbuf buf = STRBUF_INIT; - get_midx_filename(midx->repo->hash_algo, &buf, midx->object_dir); + get_midx_filename(midx->source->odb->repo->hash_algo, &buf, + midx->source->path); trace2_data_string("bitmap", bitmap_repo(bitmap_git), "ignoring extra midx bitmap file", buf.buf); close(fd); diff --git a/pack-revindex.c b/pack-revindex.c index 0cc422a1e6..b206518dcb 100644 --- a/pack-revindex.c +++ b/pack-revindex.c @@ -379,25 +379,25 @@ int load_midx_revindex(struct multi_pack_index *m) * not want to accidentally call munmap() in the middle of the * MIDX. */ - trace2_data_string("load_midx_revindex", m->repo, + trace2_data_string("load_midx_revindex", m->source->odb->repo, "source", "midx"); m->revindex_data = (const uint32_t *)m->chunk_revindex; return 0; } - trace2_data_string("load_midx_revindex", m->repo, + trace2_data_string("load_midx_revindex", m->source->odb->repo, "source", "rev"); if (m->has_chain) - get_split_midx_filename_ext(m->repo->hash_algo, &revindex_name, - m->object_dir, get_midx_checksum(m), + get_split_midx_filename_ext(m->source->odb->repo->hash_algo, &revindex_name, + m->source->path, get_midx_checksum(m), MIDX_EXT_REV); else - get_midx_filename_ext(m->repo->hash_algo, &revindex_name, - m->object_dir, get_midx_checksum(m), + get_midx_filename_ext(m->source->odb->repo->hash_algo, &revindex_name, + m->source->path, get_midx_checksum(m), MIDX_EXT_REV); - ret = load_revindex_from_disk(m->repo->hash_algo, + ret = load_revindex_from_disk(m->source->odb->repo->hash_algo, revindex_name.buf, m->num_objects, &m->revindex_map, diff --git a/t/helper/test-read-midx.c b/t/helper/test-read-midx.c index bcb8ea7671..6de5d1665a 100644 --- a/t/helper/test-read-midx.c +++ b/t/helper/test-read-midx.c @@ -66,7 +66,7 @@ static int read_midx_file(const char *object_dir, const char *checksum, for (i = 0; i < m->num_packs; i++) printf("%s\n", m->pack_names[i]); - printf("object-dir: %s\n", m->object_dir); + printf("object-dir: %s\n", m->source->path); if (show_objects) { struct object_id oid; From 13296ac909d53e14712f89a7f4fda94dd0465479 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Mon, 11 Aug 2025 15:46:50 +0200 Subject: [PATCH 10/10] midx: compute paths via their source With the preceding commits we started to always have the object database source available when we load, write or access multi-pack indices. With this in place we can change how MIDX paths are computed so that we don't have to pass in the combination of a hash algorithm and object directory anymore, but only the object database source. Refactor the code accordingly. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- midx-write.c | 52 ++++++++++++++++++++++------------------------- midx.c | 54 +++++++++++++++++++++++-------------------------- midx.h | 13 +++++------- pack-bitmap.c | 10 ++++----- pack-revindex.c | 8 ++++---- 5 files changed, 62 insertions(+), 75 deletions(-) diff --git a/midx-write.c b/midx-write.c index 84f76856d6..1dcdf3dc0f 100644 --- a/midx-write.c +++ b/midx-write.c @@ -26,9 +26,9 @@ #define MIDX_CHUNK_LARGE_OFFSET_WIDTH (sizeof(uint64_t)) extern int midx_checksum_valid(struct multi_pack_index *m); -extern void clear_midx_files_ext(const char *object_dir, const char *ext, +extern void clear_midx_files_ext(struct odb_source *source, const char *ext, const char *keep_hash); -extern void clear_incremental_midx_files_ext(const char *object_dir, +extern void clear_incremental_midx_files_ext(struct odb_source *source, const char *ext, const char **keep_hashes, uint32_t hashes_nr); @@ -112,6 +112,7 @@ struct write_midx_context { struct string_list *to_include; struct repository *repo; + struct odb_source *source; }; static int should_include_pack(const struct write_midx_context *ctx, @@ -648,7 +649,6 @@ static uint32_t *midx_pack_order(struct write_midx_context *ctx) } static void write_midx_reverse_index(struct write_midx_context *ctx, - const char *object_dir, unsigned char *midx_hash) { struct strbuf buf = STRBUF_INIT; @@ -657,11 +657,10 @@ static void write_midx_reverse_index(struct write_midx_context *ctx, trace2_region_enter("midx", "write_midx_reverse_index", ctx->repo); if (ctx->incremental) - get_split_midx_filename_ext(ctx->repo->hash_algo, &buf, - object_dir, midx_hash, - MIDX_EXT_REV); + get_split_midx_filename_ext(ctx->source, &buf, + midx_hash, MIDX_EXT_REV); else - get_midx_filename_ext(ctx->repo->hash_algo, &buf, object_dir, + get_midx_filename_ext(ctx->source, &buf, midx_hash, MIDX_EXT_REV); tmp_file = write_rev_file_order(ctx->repo, NULL, ctx->pack_order, @@ -836,7 +835,6 @@ static struct commit **find_commits_for_midx_bitmap(uint32_t *indexed_commits_nr } static int write_midx_bitmap(struct write_midx_context *ctx, - const char *object_dir, const unsigned char *midx_hash, struct packing_data *pdata, struct commit **commits, @@ -852,12 +850,11 @@ static int write_midx_bitmap(struct write_midx_context *ctx, trace2_region_enter("midx", "write_midx_bitmap", ctx->repo); if (ctx->incremental) - get_split_midx_filename_ext(ctx->repo->hash_algo, &bitmap_name, - object_dir, midx_hash, - MIDX_EXT_BITMAP); + get_split_midx_filename_ext(ctx->source, &bitmap_name, + midx_hash, MIDX_EXT_BITMAP); else - get_midx_filename_ext(ctx->repo->hash_algo, &bitmap_name, - object_dir, midx_hash, MIDX_EXT_BITMAP); + get_midx_filename_ext(ctx->source, &bitmap_name, + midx_hash, MIDX_EXT_BITMAP); if (flags & MIDX_WRITE_BITMAP_HASH_CACHE) options |= BITMAP_OPT_HASH_CACHE; @@ -981,11 +978,9 @@ static int link_midx_to_chain(struct multi_pack_index *m) for (i = 0; i < ARRAY_SIZE(midx_exts); i++) { const unsigned char *hash = get_midx_checksum(m); - get_midx_filename_ext(m->source->odb->repo->hash_algo, &from, - m->source->path, + get_midx_filename_ext(m->source, &from, hash, midx_exts[i].non_split); - get_split_midx_filename_ext(m->source->odb->repo->hash_algo, &to, - m->source->path, hash, + get_split_midx_filename_ext(m->source, &to, hash, midx_exts[i].split); if (link(from.buf, to.buf) < 0 && errno != ENOENT) { @@ -1023,16 +1018,16 @@ static void clear_midx_files(struct odb_source *source, uint32_t i, j; for (i = 0; i < ARRAY_SIZE(exts); i++) { - clear_incremental_midx_files_ext(source->path, exts[i], + clear_incremental_midx_files_ext(source, exts[i], hashes, hashes_nr); for (j = 0; j < hashes_nr; j++) - clear_midx_files_ext(source->path, exts[i], hashes[j]); + clear_midx_files_ext(source, exts[i], hashes[j]); } if (incremental) - get_midx_filename(source->odb->repo->hash_algo, &buf, source->path); + get_midx_filename(source, &buf); else - get_midx_chain_filename(&buf, source->path); + get_midx_chain_filename(source, &buf); if (unlink(buf.buf) && errno != ENOENT) die_errno(_("failed to clear multi-pack-index at %s"), buf.buf); @@ -1065,6 +1060,7 @@ static int write_midx_internal(struct odb_source *source, trace2_region_enter("midx", "write_midx_internal", r); ctx.repo = r; + ctx.source = source; ctx.incremental = !!(flags & MIDX_WRITE_INCREMENTAL); @@ -1073,7 +1069,7 @@ static int write_midx_internal(struct odb_source *source, "%s/pack/multi-pack-index.d/tmp_midx_XXXXXX", source->path); else - get_midx_filename(r->hash_algo, &midx_name, source->path); + get_midx_filename(source, &midx_name); if (safe_create_leading_directories(r, midx_name.buf)) die_errno(_("unable to create leading directories of %s"), midx_name.buf); @@ -1153,7 +1149,7 @@ static int write_midx_internal(struct odb_source *source, * corresponding bitmap (or one wasn't requested). */ if (!want_bitmap) - clear_midx_files_ext(source->path, "bitmap", NULL); + clear_midx_files_ext(source, "bitmap", NULL); goto cleanup; } } @@ -1321,7 +1317,7 @@ static int write_midx_internal(struct odb_source *source, if (ctx.incremental) { struct strbuf lock_name = STRBUF_INIT; - get_midx_chain_filename(&lock_name, source->path); + get_midx_chain_filename(source, &lock_name); hold_lock_file_for_update(&lk, lock_name.buf, LOCK_DIE_ON_ERROR); strbuf_release(&lock_name); @@ -1384,7 +1380,7 @@ static int write_midx_internal(struct odb_source *source, if (flags & MIDX_WRITE_REV_INDEX && git_env_bool("GIT_TEST_MIDX_WRITE_REV", 0)) - write_midx_reverse_index(&ctx, source->path, midx_hash); + write_midx_reverse_index(&ctx, midx_hash); if (flags & MIDX_WRITE_BITMAP) { struct packing_data pdata; @@ -1407,7 +1403,7 @@ static int write_midx_internal(struct odb_source *source, FREE_AND_NULL(ctx.entries); ctx.entries_nr = 0; - if (write_midx_bitmap(&ctx, source->path, + if (write_midx_bitmap(&ctx, midx_hash, &pdata, commits, commits_nr, flags) < 0) { error(_("could not write multi-pack bitmap")); @@ -1440,8 +1436,8 @@ static int write_midx_internal(struct odb_source *source, if (link_midx_to_chain(ctx.base_midx) < 0) return -1; - get_split_midx_filename_ext(r->hash_algo, &final_midx_name, - source->path, midx_hash, MIDX_EXT_MIDX); + get_split_midx_filename_ext(source, &final_midx_name, + midx_hash, MIDX_EXT_MIDX); if (rename_tempfile(&incr, final_midx_name.buf) < 0) { error_errno(_("unable to rename new multi-pack-index layer")); diff --git a/midx.c b/midx.c index 81bf3c4d5f..7726c13d7e 100644 --- a/midx.c +++ b/midx.c @@ -16,9 +16,9 @@ #define MIDX_PACK_ERROR ((void *)(intptr_t)-1) int midx_checksum_valid(struct multi_pack_index *m); -void clear_midx_files_ext(const char *object_dir, const char *ext, +void clear_midx_files_ext(struct odb_source *source, const char *ext, const char *keep_hash); -void clear_incremental_midx_files_ext(const char *object_dir, const char *ext, +void clear_incremental_midx_files_ext(struct odb_source *source, const char *ext, char **keep_hashes, uint32_t hashes_nr); int cmp_idx_or_pack_name(const char *idx_or_pack_name, @@ -29,19 +29,17 @@ const unsigned char *get_midx_checksum(struct multi_pack_index *m) return m->data + m->data_len - m->source->odb->repo->hash_algo->rawsz; } -void get_midx_filename(const struct git_hash_algo *hash_algo, - struct strbuf *out, const char *object_dir) +void get_midx_filename(struct odb_source *source, struct strbuf *out) { - get_midx_filename_ext(hash_algo, out, object_dir, NULL, NULL); + get_midx_filename_ext(source, out, NULL, NULL); } -void get_midx_filename_ext(const struct git_hash_algo *hash_algo, - struct strbuf *out, const char *object_dir, +void get_midx_filename_ext(struct odb_source *source, struct strbuf *out, const unsigned char *hash, const char *ext) { - strbuf_addf(out, "%s/pack/multi-pack-index", object_dir); + strbuf_addf(out, "%s/pack/multi-pack-index", source->path); if (ext) - strbuf_addf(out, "-%s.%s", hash_to_hex_algop(hash, hash_algo), ext); + strbuf_addf(out, "-%s.%s", hash_to_hex_algop(hash, source->odb->repo->hash_algo), ext); } static int midx_read_oid_fanout(const unsigned char *chunk_start, @@ -222,24 +220,23 @@ cleanup_fail: return NULL; } -void get_midx_chain_dirname(struct strbuf *buf, const char *object_dir) +void get_midx_chain_dirname(struct odb_source *source, struct strbuf *buf) { - strbuf_addf(buf, "%s/pack/multi-pack-index.d", object_dir); + strbuf_addf(buf, "%s/pack/multi-pack-index.d", source->path); } -void get_midx_chain_filename(struct strbuf *buf, const char *object_dir) +void get_midx_chain_filename(struct odb_source *source, struct strbuf *buf) { - get_midx_chain_dirname(buf, object_dir); + get_midx_chain_dirname(source, buf); strbuf_addstr(buf, "/multi-pack-index-chain"); } -void get_split_midx_filename_ext(const struct git_hash_algo *hash_algo, - struct strbuf *buf, const char *object_dir, +void get_split_midx_filename_ext(struct odb_source *source, struct strbuf *buf, const unsigned char *hash, const char *ext) { - get_midx_chain_dirname(buf, object_dir); + get_midx_chain_dirname(source, buf); strbuf_addf(buf, "/multi-pack-index-%s.%s", - hash_to_hex_algop(hash, hash_algo), ext); + hash_to_hex_algop(hash, source->odb->repo->hash_algo), ext); } static int open_multi_pack_index_chain(const struct git_hash_algo *hash_algo, @@ -326,7 +323,7 @@ static struct multi_pack_index *load_midx_chain_fd_st(struct odb_source *source, valid = 0; strbuf_reset(&buf); - get_split_midx_filename_ext(hash_algo, &buf, source->path, + get_split_midx_filename_ext(source, &buf, layer.hash, MIDX_EXT_MIDX); m = load_multi_pack_index_one(source, buf.buf); @@ -358,7 +355,7 @@ static struct multi_pack_index *load_multi_pack_index_chain(struct odb_source *s int fd; struct multi_pack_index *m = NULL; - get_midx_chain_filename(&chain_file, source->path); + get_midx_chain_filename(source, &chain_file); if (open_multi_pack_index_chain(source->odb->repo->hash_algo, chain_file.buf, &fd, &st)) { int incomplete; /* ownership of fd is taken over by load function */ @@ -374,8 +371,7 @@ struct multi_pack_index *load_multi_pack_index(struct odb_source *source) struct strbuf midx_name = STRBUF_INIT; struct multi_pack_index *m; - get_midx_filename(source->odb->repo->hash_algo, &midx_name, - source->path); + get_midx_filename(source, &midx_name); m = load_multi_pack_index_one(source, midx_name.buf); if (!m) @@ -762,7 +758,7 @@ static void clear_midx_file_ext(const char *full_path, size_t full_path_len UNUS die_errno(_("failed to remove %s"), full_path); } -void clear_midx_files_ext(const char *object_dir, const char *ext, +void clear_midx_files_ext(struct odb_source *source, const char *ext, const char *keep_hash) { struct clear_midx_data data; @@ -776,7 +772,7 @@ void clear_midx_files_ext(const char *object_dir, const char *ext, } data.ext = ext; - for_each_file_in_pack_dir(object_dir, + for_each_file_in_pack_dir(source->path, clear_midx_file_ext, &data); @@ -785,7 +781,7 @@ void clear_midx_files_ext(const char *object_dir, const char *ext, free(data.keep); } -void clear_incremental_midx_files_ext(const char *object_dir, const char *ext, +void clear_incremental_midx_files_ext(struct odb_source *source, const char *ext, char **keep_hashes, uint32_t hashes_nr) { @@ -801,7 +797,7 @@ void clear_incremental_midx_files_ext(const char *object_dir, const char *ext, data.keep_nr = hashes_nr; data.ext = ext; - for_each_file_in_pack_subdir(object_dir, "multi-pack-index.d", + for_each_file_in_pack_subdir(source->path, "multi-pack-index.d", clear_midx_file_ext, &data); for (i = 0; i < hashes_nr; i++) @@ -813,7 +809,7 @@ void clear_midx_file(struct repository *r) { struct strbuf midx = STRBUF_INIT; - get_midx_filename(r->hash_algo, &midx, r->objects->sources->path); + get_midx_filename(r->objects->sources, &midx); if (r->objects) { struct odb_source *source; @@ -828,8 +824,8 @@ void clear_midx_file(struct repository *r) if (remove_path(midx.buf)) die(_("failed to clear multi-pack-index at %s"), midx.buf); - clear_midx_files_ext(r->objects->sources->path, MIDX_EXT_BITMAP, NULL); - clear_midx_files_ext(r->objects->sources->path, MIDX_EXT_REV, NULL); + clear_midx_files_ext(r->objects->sources, MIDX_EXT_BITMAP, NULL); + clear_midx_files_ext(r->objects->sources, MIDX_EXT_REV, NULL); strbuf_release(&midx); } @@ -888,7 +884,7 @@ int verify_midx_file(struct odb_source *source, unsigned flags) struct stat sb; struct strbuf filename = STRBUF_INIT; - get_midx_filename(r->hash_algo, &filename, source->path); + get_midx_filename(source, &filename); if (!stat(filename.buf, &sb)) { error(_("multi-pack-index file exists, but failed to parse")); diff --git a/midx.h b/midx.h index 71dbdec66e..e241d2d690 100644 --- a/midx.h +++ b/midx.h @@ -86,15 +86,12 @@ struct multi_pack_index { #define MIDX_EXT_MIDX "midx" const unsigned char *get_midx_checksum(struct multi_pack_index *m); -void get_midx_filename(const struct git_hash_algo *hash_algo, - struct strbuf *out, const char *object_dir); -void get_midx_filename_ext(const struct git_hash_algo *hash_algo, - struct strbuf *out, const char *object_dir, +void get_midx_filename(struct odb_source *source, struct strbuf *out); +void get_midx_filename_ext(struct odb_source *source, struct strbuf *out, const unsigned char *hash, const char *ext); -void get_midx_chain_dirname(struct strbuf *buf, const char *object_dir); -void get_midx_chain_filename(struct strbuf *buf, const char *object_dir); -void get_split_midx_filename_ext(const struct git_hash_algo *hash_algo, - struct strbuf *buf, const char *object_dir, +void get_midx_chain_dirname(struct odb_source *source, struct strbuf *out); +void get_midx_chain_filename(struct odb_source *source, struct strbuf *out); +void get_split_midx_filename_ext(struct odb_source *source, struct strbuf *buf, const unsigned char *hash, const char *ext); struct multi_pack_index *load_multi_pack_index(struct odb_source *source); diff --git a/pack-bitmap.c b/pack-bitmap.c index 01e14c34bd..058bdb5d7d 100644 --- a/pack-bitmap.c +++ b/pack-bitmap.c @@ -418,13 +418,12 @@ char *midx_bitmap_filename(struct multi_pack_index *midx) { struct strbuf buf = STRBUF_INIT; if (midx->has_chain) - get_split_midx_filename_ext(midx->source->odb->repo->hash_algo, &buf, - midx->source->path, + get_split_midx_filename_ext(midx->source, &buf, get_midx_checksum(midx), MIDX_EXT_BITMAP); else - get_midx_filename_ext(midx->source->odb->repo->hash_algo, &buf, - midx->source->path, get_midx_checksum(midx), + get_midx_filename_ext(midx->source, &buf, + get_midx_checksum(midx), MIDX_EXT_BITMAP); return strbuf_detach(&buf, NULL); @@ -463,8 +462,7 @@ static int open_midx_bitmap_1(struct bitmap_index *bitmap_git, if (bitmap_git->pack || bitmap_git->midx) { struct strbuf buf = STRBUF_INIT; - get_midx_filename(midx->source->odb->repo->hash_algo, &buf, - midx->source->path); + get_midx_filename(midx->source, &buf); trace2_data_string("bitmap", bitmap_repo(bitmap_git), "ignoring extra midx bitmap file", buf.buf); close(fd); diff --git a/pack-revindex.c b/pack-revindex.c index b206518dcb..d0791cc493 100644 --- a/pack-revindex.c +++ b/pack-revindex.c @@ -389,12 +389,12 @@ int load_midx_revindex(struct multi_pack_index *m) "source", "rev"); if (m->has_chain) - get_split_midx_filename_ext(m->source->odb->repo->hash_algo, &revindex_name, - m->source->path, get_midx_checksum(m), + get_split_midx_filename_ext(m->source, &revindex_name, + get_midx_checksum(m), MIDX_EXT_REV); else - get_midx_filename_ext(m->source->odb->repo->hash_algo, &revindex_name, - m->source->path, get_midx_checksum(m), + get_midx_filename_ext(m->source, &revindex_name, + get_midx_checksum(m), MIDX_EXT_REV); ret = load_revindex_from_disk(m->source->odb->repo->hash_algo,