global/git
1
0
Fork 0
mirror of https://github.com/git/git.git synced 2026-01-10 10:13:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

midx: write multi-pack indices via their source

Browse Source 
Similar to the preceding commit, refactor the writing side of multi-pack
indices so that we pass in the object database source where the index
should be written to.

Signed-off-by: Patrick Steinhardt <ps@pks.im>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
This commit is contained in:
Patrick Steinhardt
2025-08-11 15:46:48 +02:00
committed by Junio C Hamano
parent 017db7bb14
commit c3f5d25146
4 changed files with 47 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
builtin/multi-pack-index.c
View File

@@ -147,6 +147,7 @@ static int cmd_multi_pack_index_write(int argc, const char **argv,
N_("refs snapshot for selecting bitmap commits")), N_("refs snapshot for selecting bitmap commits")),
OPT_END(), OPT_END(),
}; };
struct odb_source *source;
int ret; int ret;
opts.flags |= MIDX_WRITE_BITMAP_HASH_CACHE; opts.flags |= MIDX_WRITE_BITMAP_HASH_CACHE;
@@ -165,7 +166,7 @@ static int cmd_multi_pack_index_write(int argc, const char **argv,
if (argc) if (argc)
usage_with_options(builtin_multi_pack_index_write_usage, usage_with_options(builtin_multi_pack_index_write_usage,
options); options);
handle_object_dir_option(repo); source = handle_object_dir_option(repo);
FREE_AND_NULL(options); FREE_AND_NULL(options);
@@ -174,7 +175,7 @@ static int cmd_multi_pack_index_write(int argc, const char **argv,
read_packs_from_stdin(&packs); read_packs_from_stdin(&packs);
ret = write_midx_file_only(repo, opts.object_dir, &packs, ret = write_midx_file_only(source, &packs,
opts.preferred_pack, opts.preferred_pack,
opts.refs_snapshot, opts.flags); opts.refs_snapshot, opts.flags);
@@ -185,7 +186,7 @@ static int cmd_multi_pack_index_write(int argc, const char **argv,
} }
ret = write_midx_file(repo, opts.object_dir, opts.preferred_pack, ret = write_midx_file(source, opts.preferred_pack,
opts.refs_snapshot, opts.flags); opts.refs_snapshot, opts.flags);
free(opts.refs_snapshot); free(opts.refs_snapshot);
@@ -233,6 +234,8 @@ static int cmd_multi_pack_index_expire(int argc, const char **argv,
N_("force progress reporting"), MIDX_PROGRESS), N_("force progress reporting"), MIDX_PROGRESS),
OPT_END(), OPT_END(),
}; };
struct odb_source *source;
options = add_common_options(builtin_multi_pack_index_expire_options); options = add_common_options(builtin_multi_pack_index_expire_options);
trace2_cmd_mode(argv[0]); trace2_cmd_mode(argv[0]);
@@ -245,11 +248,11 @@ static int cmd_multi_pack_index_expire(int argc, const char **argv,
if (argc) if (argc)
usage_with_options(builtin_multi_pack_index_expire_usage, usage_with_options(builtin_multi_pack_index_expire_usage,
options); options);
handle_object_dir_option(the_repository); source = handle_object_dir_option(the_repository);
FREE_AND_NULL(options); FREE_AND_NULL(options);
return expire_midx_packs(the_repository, opts.object_dir, opts.flags); return expire_midx_packs(source, opts.flags);
} }
static int cmd_multi_pack_index_repack(int argc, const char **argv, static int cmd_multi_pack_index_repack(int argc, const char **argv,
@@ -264,6 +267,7 @@ static int cmd_multi_pack_index_repack(int argc, const char **argv,
N_("force progress reporting"), MIDX_PROGRESS), N_("force progress reporting"), MIDX_PROGRESS),
OPT_END(), OPT_END(),
}; };
struct odb_source *source;
options = add_common_options(builtin_multi_pack_index_repack_options); options = add_common_options(builtin_multi_pack_index_repack_options);
@@ -278,12 +282,11 @@ static int cmd_multi_pack_index_repack(int argc, const char **argv,
if (argc) if (argc)
usage_with_options(builtin_multi_pack_index_repack_usage, usage_with_options(builtin_multi_pack_index_repack_usage,
options); options);
handle_object_dir_option(the_repository); source = handle_object_dir_option(the_repository);
FREE_AND_NULL(options); FREE_AND_NULL(options);
return midx_repack(the_repository, opts.object_dir, return midx_repack(source, (size_t)opts.batch_size, opts.flags);
(size_t)opts.batch_size, opts.flags);
} }
int cmd_multi_pack_index(int argc, int cmd_multi_pack_index(int argc,

2
builtin/repack.c
View File

@@ -1711,7 +1711,7 @@ int cmd_repack(int argc,
unsigned flags = 0; unsigned flags = 0;
if (git_env_bool(GIT_TEST_MULTI_PACK_INDEX_WRITE_INCREMENTAL, 0)) if (git_env_bool(GIT_TEST_MULTI_PACK_INDEX_WRITE_INCREMENTAL, 0))
flags |= MIDX_WRITE_INCREMENTAL; flags |= MIDX_WRITE_INCREMENTAL;
write_midx_file(the_repository, repo_get_object_directory(the_repository), write_midx_file(the_repository->objects->sources,
NULL, NULL, flags); NULL, NULL, flags);
} }

67
midx-write.c
View File

@@ -913,13 +913,6 @@ cleanup:
return ret; return ret;
} }
static struct multi_pack_index *lookup_multi_pack_index(struct repository *r,
const char *object_dir)
{
struct odb_source *source = odb_find_source_or_die(r->objects, object_dir);
return get_multi_pack_index(source);
}
static int fill_packs_from_midx(struct write_midx_context *ctx, static int fill_packs_from_midx(struct write_midx_context *ctx,
const char *preferred_pack_name, uint32_t flags) const char *preferred_pack_name, uint32_t flags)
{ {
@@ -1010,7 +1003,7 @@ done:
return ret; return ret;
} }
static void clear_midx_files(struct repository *r, const char *object_dir, static void clear_midx_files(struct odb_source *source,
const char **hashes, uint32_t hashes_nr, const char **hashes, uint32_t hashes_nr,
unsigned incremental) unsigned incremental)
{ {
@@ -1029,16 +1022,16 @@ static void clear_midx_files(struct repository *r, const char *object_dir,
uint32_t i, j; uint32_t i, j;
for (i = 0; i < ARRAY_SIZE(exts); i++) { for (i = 0; i < ARRAY_SIZE(exts); i++) {
clear_incremental_midx_files_ext(object_dir, exts[i], clear_incremental_midx_files_ext(source->path, exts[i],
hashes, hashes_nr); hashes, hashes_nr);
for (j = 0; j < hashes_nr; j++) for (j = 0; j < hashes_nr; j++)
clear_midx_files_ext(object_dir, exts[i], hashes[j]); clear_midx_files_ext(source->path, exts[i], hashes[j]);
} }
if (incremental) if (incremental)
get_midx_filename(r->hash_algo, &buf, object_dir); get_midx_filename(source->odb->repo->hash_algo, &buf, source->path);
else else
get_midx_chain_filename(&buf, object_dir); get_midx_chain_filename(&buf, source->path);
if (unlink(buf.buf) && errno != ENOENT) if (unlink(buf.buf) && errno != ENOENT)
die_errno(_("failed to clear multi-pack-index at %s"), buf.buf); die_errno(_("failed to clear multi-pack-index at %s"), buf.buf);
@@ -1046,13 +1039,14 @@ static void clear_midx_files(struct repository *r, const char *object_dir,
strbuf_release(&buf); strbuf_release(&buf);
} }
static int write_midx_internal(struct repository *r, const char *object_dir, static int write_midx_internal(struct odb_source *source,
struct string_list *packs_to_include, struct string_list *packs_to_include,
struct string_list *packs_to_drop, struct string_list *packs_to_drop,
const char *preferred_pack_name, const char *preferred_pack_name,
const char *refs_snapshot, const char *refs_snapshot,
unsigned flags) unsigned flags)
{ {
struct repository *r = source->odb->repo;
struct strbuf midx_name = STRBUF_INIT; struct strbuf midx_name = STRBUF_INIT;
unsigned char midx_hash[GIT_MAX_RAWSZ]; unsigned char midx_hash[GIT_MAX_RAWSZ];
uint32_t i, start_pack; uint32_t i, start_pack;
@@ -1076,15 +1070,15 @@ static int write_midx_internal(struct repository *r, const char *object_dir,
if (ctx.incremental) if (ctx.incremental)
strbuf_addf(&midx_name, strbuf_addf(&midx_name,
"%s/pack/multi-pack-index.d/tmp_midx_XXXXXX", "%s/pack/multi-pack-index.d/tmp_midx_XXXXXX",
object_dir); source->path);
else else
get_midx_filename(r->hash_algo, &midx_name, object_dir); get_midx_filename(r->hash_algo, &midx_name, source->path);
if (safe_create_leading_directories(r, midx_name.buf)) if (safe_create_leading_directories(r, midx_name.buf))
die_errno(_("unable to create leading directories of %s"), die_errno(_("unable to create leading directories of %s"),
midx_name.buf); midx_name.buf);
if (!packs_to_include || ctx.incremental) { if (!packs_to_include || ctx.incremental) {
struct multi_pack_index *m = lookup_multi_pack_index(r, object_dir); struct multi_pack_index *m = get_multi_pack_index(source);
if (m && !midx_checksum_valid(m)) { if (m && !midx_checksum_valid(m)) {
warning(_("ignoring existing multi-pack-index; checksum mismatch")); warning(_("ignoring existing multi-pack-index; checksum mismatch"));
m = NULL; m = NULL;
@@ -1138,7 +1132,7 @@ static int write_midx_internal(struct repository *r, const char *object_dir,
ctx.to_include = packs_to_include; ctx.to_include = packs_to_include;
for_each_file_in_pack_dir(object_dir, add_pack_to_midx, &ctx); for_each_file_in_pack_dir(source->path, add_pack_to_midx, &ctx);
stop_progress(&ctx.progress); stop_progress(&ctx.progress);
if ((ctx.m && ctx.nr == ctx.m->num_packs + ctx.m->num_packs_in_base) && if ((ctx.m && ctx.nr == ctx.m->num_packs + ctx.m->num_packs_in_base) &&
@@ -1158,7 +1152,7 @@ static int write_midx_internal(struct repository *r, const char *object_dir,
* corresponding bitmap (or one wasn't requested). * corresponding bitmap (or one wasn't requested).
*/ */
if (!want_bitmap) if (!want_bitmap)
clear_midx_files_ext(object_dir, "bitmap", NULL); clear_midx_files_ext(source->path, "bitmap", NULL);
goto cleanup; goto cleanup;
} }
} }
@@ -1326,7 +1320,7 @@ static int write_midx_internal(struct repository *r, const char *object_dir,
if (ctx.incremental) { if (ctx.incremental) {
struct strbuf lock_name = STRBUF_INIT; struct strbuf lock_name = STRBUF_INIT;
get_midx_chain_filename(&lock_name, object_dir); get_midx_chain_filename(&lock_name, source->path);
hold_lock_file_for_update(&lk, lock_name.buf, LOCK_DIE_ON_ERROR); hold_lock_file_for_update(&lk, lock_name.buf, LOCK_DIE_ON_ERROR);
strbuf_release(&lock_name); strbuf_release(&lock_name);
@@ -1389,7 +1383,7 @@ static int write_midx_internal(struct repository *r, const char *object_dir,
if (flags & MIDX_WRITE_REV_INDEX && if (flags & MIDX_WRITE_REV_INDEX &&
git_env_bool("GIT_TEST_MIDX_WRITE_REV", 0)) git_env_bool("GIT_TEST_MIDX_WRITE_REV", 0))
write_midx_reverse_index(&ctx, object_dir, midx_hash); write_midx_reverse_index(&ctx, source->path, midx_hash);
if (flags & MIDX_WRITE_BITMAP) { if (flags & MIDX_WRITE_BITMAP) {
struct packing_data pdata; struct packing_data pdata;
@@ -1412,7 +1406,7 @@ static int write_midx_internal(struct repository *r, const char *object_dir,
FREE_AND_NULL(ctx.entries); FREE_AND_NULL(ctx.entries);
ctx.entries_nr = 0; ctx.entries_nr = 0;
if (write_midx_bitmap(&ctx, object_dir, if (write_midx_bitmap(&ctx, source->path,
midx_hash, &pdata, commits, commits_nr, midx_hash, &pdata, commits, commits_nr,
flags) < 0) { flags) < 0) {
error(_("could not write multi-pack bitmap")); error(_("could not write multi-pack bitmap"));
@@ -1446,7 +1440,7 @@ static int write_midx_internal(struct repository *r, const char *object_dir,
return -1; return -1;
get_split_midx_filename_ext(r->hash_algo, &final_midx_name, get_split_midx_filename_ext(r->hash_algo, &final_midx_name,
object_dir, midx_hash, MIDX_EXT_MIDX); source->path, midx_hash, MIDX_EXT_MIDX);
if (rename_tempfile(&incr, final_midx_name.buf) < 0) { if (rename_tempfile(&incr, final_midx_name.buf) < 0) {
error_errno(_("unable to rename new multi-pack-index layer")); error_errno(_("unable to rename new multi-pack-index layer"));
@@ -1479,7 +1473,7 @@ static int write_midx_internal(struct repository *r, const char *object_dir,
if (commit_lock_file(&lk) < 0) if (commit_lock_file(&lk) < 0)
die_errno(_("could not write multi-pack-index")); die_errno(_("could not write multi-pack-index"));
clear_midx_files(r, object_dir, keep_hashes, clear_midx_files(source, keep_hashes,
ctx.num_multi_pack_indexes_before + 1, ctx.num_multi_pack_indexes_before + 1,
ctx.incremental); ctx.incremental);
@@ -1508,29 +1502,29 @@ cleanup:
return result; return result;
} }
int write_midx_file(struct repository *r, const char *object_dir, int write_midx_file(struct odb_source *source,
const char *preferred_pack_name, const char *preferred_pack_name,
const char *refs_snapshot, unsigned flags) const char *refs_snapshot, unsigned flags)
{ {
return write_midx_internal(r, object_dir, NULL, NULL, return write_midx_internal(source, NULL, NULL,
preferred_pack_name, refs_snapshot, preferred_pack_name, refs_snapshot,
flags); flags);
} }
int write_midx_file_only(struct repository *r, const char *object_dir, int write_midx_file_only(struct odb_source *source,
struct string_list *packs_to_include, struct string_list *packs_to_include,
const char *preferred_pack_name, const char *preferred_pack_name,
const char *refs_snapshot, unsigned flags) const char *refs_snapshot, unsigned flags)
{ {
return write_midx_internal(r, object_dir, packs_to_include, NULL, return write_midx_internal(source, packs_to_include, NULL,
preferred_pack_name, refs_snapshot, flags); preferred_pack_name, refs_snapshot, flags);
} }
int expire_midx_packs(struct repository *r, const char *object_dir, unsigned flags) int expire_midx_packs(struct odb_source *source, unsigned flags)
{ {
uint32_t i, *count, result = 0; uint32_t i, *count, result = 0;
struct string_list packs_to_drop = STRING_LIST_INIT_DUP; struct string_list packs_to_drop = STRING_LIST_INIT_DUP;
struct multi_pack_index *m = lookup_multi_pack_index(r, object_dir); struct multi_pack_index *m = get_multi_pack_index(source);
struct progress *progress = NULL; struct progress *progress = NULL;
if (!m) if (!m)
@@ -1543,7 +1537,7 @@ int expire_midx_packs(struct repository *r, const char *object_dir, unsigned fla
if (flags & MIDX_PROGRESS) if (flags & MIDX_PROGRESS)
progress = start_delayed_progress( progress = start_delayed_progress(
r, source->odb->repo,
_("Counting referenced objects"), _("Counting referenced objects"),
m->num_objects); m->num_objects);
for (i = 0; i < m->num_objects; i++) { for (i = 0; i < m->num_objects; i++) {
@@ -1555,7 +1549,7 @@ int expire_midx_packs(struct repository *r, const char *object_dir, unsigned fla
if (flags & MIDX_PROGRESS) if (flags & MIDX_PROGRESS)
progress = start_delayed_progress( progress = start_delayed_progress(
r, source->odb->repo,
_("Finding and deleting unreferenced packfiles"), _("Finding and deleting unreferenced packfiles"),
m->num_packs); m->num_packs);
for (i = 0; i < m->num_packs; i++) { for (i = 0; i < m->num_packs; i++) {
@@ -1583,7 +1577,7 @@ int expire_midx_packs(struct repository *r, const char *object_dir, unsigned fla
free(count); free(count);
if (packs_to_drop.nr) if (packs_to_drop.nr)
result = write_midx_internal(r, object_dir, NULL, result = write_midx_internal(source, NULL,
&packs_to_drop, NULL, NULL, flags); &packs_to_drop, NULL, NULL, flags);
string_list_clear(&packs_to_drop, 0); string_list_clear(&packs_to_drop, 0);
@@ -1708,14 +1702,15 @@ static void fill_included_packs_batch(struct repository *r,
free(pack_info); free(pack_info);
} }
int midx_repack(struct repository *r, const char *object_dir, size_t batch_size, unsigned flags) int midx_repack(struct odb_source *source, size_t batch_size, unsigned flags)
{ {
struct repository *r = source->odb->repo;
int result = 0; int result = 0;
uint32_t i, packs_to_repack = 0; uint32_t i, packs_to_repack = 0;
unsigned char *include_pack; unsigned char *include_pack;
struct child_process cmd = CHILD_PROCESS_INIT; struct child_process cmd = CHILD_PROCESS_INIT;
FILE *cmd_in; FILE *cmd_in;
struct multi_pack_index *m = lookup_multi_pack_index(r, object_dir); struct multi_pack_index *m = get_multi_pack_index(source);
/* /*
* When updating the default for these configuration * When updating the default for these configuration
@@ -1749,7 +1744,7 @@ int midx_repack(struct repository *r, const char *object_dir, size_t batch_size,
strvec_push(&cmd.args, "pack-objects"); strvec_push(&cmd.args, "pack-objects");
strvec_pushf(&cmd.args, "%s/pack/pack", object_dir); strvec_pushf(&cmd.args, "%s/pack/pack", source->path);
if (delta_base_offset) if (delta_base_offset)
strvec_push(&cmd.args, "--delta-base-offset"); strvec_push(&cmd.args, "--delta-base-offset");
@@ -1790,7 +1785,7 @@ int midx_repack(struct repository *r, const char *object_dir, size_t batch_size,
goto cleanup; goto cleanup;
} }
result = write_midx_internal(r, object_dir, NULL, NULL, NULL, NULL, result = write_midx_internal(source, NULL, NULL, NULL, NULL,
flags); flags);
cleanup: cleanup:

8
midx.h
View File

@@ -126,17 +126,17 @@ int prepare_multi_pack_index_one(struct odb_source *source);
* Variant of write_midx_file which writes a MIDX containing only the packs * Variant of write_midx_file which writes a MIDX containing only the packs
* specified in packs_to_include. * specified in packs_to_include.
*/ */
int write_midx_file(struct repository *r, const char *object_dir, int write_midx_file(struct odb_source *source,
const char *preferred_pack_name, const char *refs_snapshot, const char *preferred_pack_name, const char *refs_snapshot,
unsigned flags); unsigned flags);
int write_midx_file_only(struct repository *r, const char *object_dir, int write_midx_file_only(struct odb_source *source,
struct string_list *packs_to_include, struct string_list *packs_to_include,
const char *preferred_pack_name, const char *preferred_pack_name,
const char *refs_snapshot, unsigned flags); const char *refs_snapshot, unsigned flags);
void clear_midx_file(struct repository *r); void clear_midx_file(struct repository *r);
int verify_midx_file(struct odb_source *source, unsigned flags); int verify_midx_file(struct odb_source *source, unsigned flags);
int expire_midx_packs(struct repository *r, const char *object_dir, unsigned flags); int expire_midx_packs(struct odb_source *source, unsigned flags);
int midx_repack(struct repository *r, const char *object_dir, size_t batch_size, unsigned flags); int midx_repack(struct odb_source *source, size_t batch_size, unsigned flags);
void close_midx(struct multi_pack_index *m); void close_midx(struct multi_pack_index *m);