diff --git a/Makefile b/Makefile index 15b1ded1a0..58fb895f4e 100644 --- a/Makefile +++ b/Makefile @@ -1214,6 +1214,8 @@ LIB_OBJS += object-file.o LIB_OBJS += object-name.o LIB_OBJS += object.o LIB_OBJS += odb.o +LIB_OBJS += odb/source.o +LIB_OBJS += odb/source-files.o LIB_OBJS += odb/streaming.o LIB_OBJS += oid-array.o LIB_OBJS += oidmap.o diff --git a/builtin/cat-file.c b/builtin/cat-file.c index da059d0e26..b6f12f41d6 100644 --- a/builtin/cat-file.c +++ b/builtin/cat-file.c @@ -882,7 +882,8 @@ static void batch_each_object(struct batch_options *opt, struct object_info oi = { 0 }; for (source = the_repository->objects->sources; source; source = source->next) { - int ret = packfile_store_for_each_object(source->packfiles, &oi, + struct odb_source_files *files = odb_source_files_downcast(source); + int ret = packfile_store_for_each_object(files->packed, &oi, batch_one_object_oi, &payload, flags); if (ret) break; diff --git a/builtin/fast-import.c b/builtin/fast-import.c index b8a7757cfd..a41f95191e 100644 --- a/builtin/fast-import.c +++ b/builtin/fast-import.c @@ -875,6 +875,7 @@ static void end_packfile(void) running = 1; clear_delta_base_cache(); if (object_count) { + struct odb_source_files *files = odb_source_files_downcast(pack_data->repo->objects->sources); struct packed_git *new_p; struct object_id cur_pack_oid; char *idx_name; @@ -900,8 +901,7 @@ static void end_packfile(void) idx_name = keep_pack(create_index()); /* Register the packfile with core git's machinery. */ - new_p = packfile_store_load_pack(pack_data->repo->objects->sources->packfiles, - idx_name, 1); + new_p = packfile_store_load_pack(files->packed, idx_name, 1); if (!new_p) die(_("core Git rejected index %s"), idx_name); all_packs[pack_id] = new_p; @@ -982,7 +982,9 @@ static int store_object( } for (source = the_repository->objects->sources; source; source = source->next) { - if (!packfile_list_find_oid(packfile_store_get_packs(source->packfiles), &oid)) + struct odb_source_files *files = odb_source_files_downcast(source); + + if (!packfile_list_find_oid(packfile_store_get_packs(files->packed), &oid)) continue; e->type = type; e->pack_id = MAX_PACK_ID; @@ -1187,7 +1189,9 @@ static void stream_blob(uintmax_t len, struct object_id *oidout, uintmax_t mark) } for (source = the_repository->objects->sources; source; source = source->next) { - if (!packfile_list_find_oid(packfile_store_get_packs(source->packfiles), &oid)) + struct odb_source_files *files = odb_source_files_downcast(source); + + if (!packfile_list_find_oid(packfile_store_get_packs(files->packed), &oid)) continue; e->type = OBJ_BLOB; e->pack_id = MAX_PACK_ID; diff --git a/builtin/grep.c b/builtin/grep.c index b6cecfd9b6..e33285e5e6 100644 --- a/builtin/grep.c +++ b/builtin/grep.c @@ -1218,8 +1218,10 @@ int cmd_grep(int argc, struct odb_source *source; odb_prepare_alternates(the_repository->objects); - for (source = the_repository->objects->sources; source; source = source->next) - packfile_store_prepare(source->packfiles); + for (source = the_repository->objects->sources; source; source = source->next) { + struct odb_source_files *files = odb_source_files_downcast(source); + packfile_store_prepare(files->packed); + } } start_threads(&opt); diff --git a/builtin/index-pack.c b/builtin/index-pack.c index b67fb0256c..d1e47279a8 100644 --- a/builtin/index-pack.c +++ b/builtin/index-pack.c @@ -1637,9 +1637,11 @@ static void final(const char *final_pack_name, const char *curr_pack_name, rename_tmp_packfile(&final_index_name, curr_index_name, &index_name, hash, "idx", 1); - if (do_fsck_object && startup_info->have_repository) - packfile_store_load_pack(the_repository->objects->sources->packfiles, - final_index_name, 0); + if (do_fsck_object && startup_info->have_repository) { + struct odb_source_files *files = + odb_source_files_downcast(the_repository->objects->sources); + packfile_store_load_pack(files->packed, final_index_name, 0); + } if (!from_stdin) { printf("%s\n", hash_to_hex(hash)); diff --git a/builtin/pack-objects.c b/builtin/pack-objects.c index c1ee4d5ed7..29d930e4b1 100644 --- a/builtin/pack-objects.c +++ b/builtin/pack-objects.c @@ -1531,7 +1531,8 @@ static int want_cruft_object_mtime(struct repository *r, struct odb_source *source; for (source = r->objects->sources; source; source = source->next) { - struct packed_git **cache = packfile_store_get_kept_pack_cache(source->packfiles, flags); + struct odb_source_files *files = odb_source_files_downcast(source); + struct packed_git **cache = packfile_store_get_kept_pack_cache(files->packed, flags); for (; *cache; cache++) { struct packed_git *p = *cache; @@ -1753,11 +1754,13 @@ static int want_object_in_pack_mtime(const struct object_id *oid, } for (source = the_repository->objects->sources; source; source = source->next) { - for (e = source->packfiles->packs.head; e; e = e->next) { + struct odb_source_files *files = odb_source_files_downcast(source); + + for (e = files->packed->packs.head; e; e = e->next) { struct packed_git *p = e->pack; want = want_object_in_pack_one(p, oid, exclude, found_pack, found_offset, found_mtime); if (!exclude && want > 0) - packfile_list_prepend(&source->packfiles->packs, p); + packfile_list_prepend(&files->packed->packs, p); if (want != -1) return want; } @@ -4347,10 +4350,12 @@ static void add_objects_in_unpacked_packs(void) odb_prepare_alternates(to_pack.repo->objects); for (source = to_pack.repo->objects->sources; source; source = source->next) { + struct odb_source_files *files = odb_source_files_downcast(source); + if (!source->local) continue; - if (packfile_store_for_each_object(source->packfiles, &oi, + if (packfile_store_for_each_object(files->packed, &oi, add_object_in_unpacked_pack, NULL, ODB_FOR_EACH_OBJECT_PACK_ORDER | ODB_FOR_EACH_OBJECT_LOCAL_ONLY | diff --git a/commit-graph.c b/commit-graph.c index d250a729b1..f8e24145a5 100644 --- a/commit-graph.c +++ b/commit-graph.c @@ -1980,9 +1980,11 @@ static void fill_oids_from_all_packs(struct write_commit_graph_context *ctx) ctx->approx_nr_objects); odb_prepare_alternates(ctx->r->objects); - for (source = ctx->r->objects->sources; source; source = source->next) - packfile_store_for_each_object(source->packfiles, &oi, add_packed_commits_oi, + for (source = ctx->r->objects->sources; source; source = source->next) { + struct odb_source_files *files = odb_source_files_downcast(source); + packfile_store_for_each_object(files->packed, &oi, add_packed_commits_oi, ctx, ODB_FOR_EACH_OBJECT_PACK_ORDER); + } if (ctx->progress_done < ctx->approx_nr_objects) display_progress(ctx->progress, ctx->approx_nr_objects); diff --git a/http.c b/http.c index 7815f144de..8ea1b9d1f6 100644 --- a/http.c +++ b/http.c @@ -2543,8 +2543,9 @@ cleanup: void http_install_packfile(struct packed_git *p, struct packfile_list *list_to_remove_from) { + struct odb_source_files *files = odb_source_files_downcast(the_repository->objects->sources); packfile_list_remove(list_to_remove_from, p); - packfile_store_add_pack(the_repository->objects->sources->packfiles, p); + packfile_store_add_pack(files->packed, p); } struct http_pack_request *new_http_pack_request( diff --git a/loose.c b/loose.c index 56cf64b648..07333be696 100644 --- a/loose.c +++ b/loose.c @@ -3,6 +3,7 @@ #include "path.h" #include "object-file.h" #include "odb.h" +#include "odb/source-files.h" #include "hex.h" #include "repository.h" #include "wrapper.h" @@ -49,27 +50,29 @@ static int insert_loose_map(struct odb_source *source, const struct object_id *oid, const struct object_id *compat_oid) { - struct loose_object_map *map = source->loose->map; + struct odb_source_files *files = odb_source_files_downcast(source); + struct loose_object_map *map = files->loose->map; int inserted = 0; inserted |= insert_oid_pair(map->to_compat, oid, compat_oid); inserted |= insert_oid_pair(map->to_storage, compat_oid, oid); if (inserted) - oidtree_insert(source->loose->cache, compat_oid); + oidtree_insert(files->loose->cache, compat_oid); return inserted; } static int load_one_loose_object_map(struct repository *repo, struct odb_source *source) { + struct odb_source_files *files = odb_source_files_downcast(source); struct strbuf buf = STRBUF_INIT, path = STRBUF_INIT; FILE *fp; - if (!source->loose->map) - loose_object_map_init(&source->loose->map); - if (!source->loose->cache) { - ALLOC_ARRAY(source->loose->cache, 1); - oidtree_init(source->loose->cache); + if (!files->loose->map) + loose_object_map_init(&files->loose->map); + if (!files->loose->cache) { + ALLOC_ARRAY(files->loose->cache, 1); + oidtree_init(files->loose->cache); } insert_loose_map(source, repo->hash_algo->empty_tree, repo->compat_hash_algo->empty_tree); @@ -125,7 +128,8 @@ int repo_read_loose_object_map(struct repository *repo) int repo_write_loose_object_map(struct repository *repo) { - kh_oid_map_t *map = repo->objects->sources->loose->map->to_compat; + struct odb_source_files *files = odb_source_files_downcast(repo->objects->sources); + kh_oid_map_t *map = files->loose->map->to_compat; struct lock_file lock; int fd; khiter_t iter; @@ -231,7 +235,8 @@ int repo_loose_object_map_oid(struct repository *repo, khiter_t pos; for (source = repo->objects->sources; source; source = source->next) { - struct loose_object_map *loose_map = source->loose->map; + struct odb_source_files *files = odb_source_files_downcast(source); + struct loose_object_map *loose_map = files->loose->map; if (!loose_map) continue; map = (to == repo->compat_hash_algo) ? diff --git a/meson.build b/meson.build index 4b536e0124..b62d49b9d8 100644 --- a/meson.build +++ b/meson.build @@ -399,6 +399,8 @@ libgit_sources = [ 'object-name.c', 'object.c', 'odb.c', + 'odb/source.c', + 'odb/source-files.c', 'odb/streaming.c', 'oid-array.c', 'oidmap.c', diff --git a/midx.c b/midx.c index a75ea99a0d..ab8e2611d1 100644 --- a/midx.c +++ b/midx.c @@ -95,8 +95,9 @@ static int midx_read_object_offsets(const unsigned char *chunk_start, struct multi_pack_index *get_multi_pack_index(struct odb_source *source) { - packfile_store_prepare(source->packfiles); - return source->packfiles->midx; + struct odb_source_files *files = odb_source_files_downcast(source); + packfile_store_prepare(files->packed); + return files->packed->midx; } static struct multi_pack_index *load_multi_pack_index_one(struct odb_source *source, @@ -447,6 +448,7 @@ static uint32_t midx_for_pack(struct multi_pack_index **_m, int prepare_midx_pack(struct multi_pack_index *m, uint32_t pack_int_id) { + struct odb_source_files *files = odb_source_files_downcast(m->source); struct strbuf pack_name = STRBUF_INIT; struct packed_git *p; @@ -457,10 +459,10 @@ int prepare_midx_pack(struct multi_pack_index *m, if (m->packs[pack_int_id]) return 0; - strbuf_addf(&pack_name, "%s/pack/%s", m->source->path, + strbuf_addf(&pack_name, "%s/pack/%s", files->base.path, m->pack_names[pack_int_id]); - p = packfile_store_load_pack(m->source->packfiles, - pack_name.buf, m->source->local); + p = packfile_store_load_pack(files->packed, + pack_name.buf, files->base.local); strbuf_release(&pack_name); if (!p) { @@ -703,18 +705,19 @@ int midx_preferred_pack(struct multi_pack_index *m, uint32_t *pack_int_id) int prepare_multi_pack_index_one(struct odb_source *source) { + struct odb_source_files *files = odb_source_files_downcast(source); struct repository *r = source->odb->repo; prepare_repo_settings(r); if (!r->settings.core_multi_pack_index) return 0; - if (source->packfiles->midx) + if (files->packed->midx) return 1; - source->packfiles->midx = load_multi_pack_index(source); + files->packed->midx = load_multi_pack_index(source); - return !!source->packfiles->midx; + return !!files->packed->midx; } int midx_checksum_valid(struct multi_pack_index *m) @@ -803,9 +806,10 @@ void clear_midx_file(struct repository *r) struct odb_source *source; for (source = r->objects->sources; source; source = source->next) { - if (source->packfiles->midx) - close_midx(source->packfiles->midx); - source->packfiles->midx = NULL; + struct odb_source_files *files = odb_source_files_downcast(source); + if (files->packed->midx) + close_midx(files->packed->midx); + files->packed->midx = NULL; } } diff --git a/object-file.c b/object-file.c index 3094140055..a3ff7f586c 100644 --- a/object-file.c +++ b/object-file.c @@ -215,8 +215,9 @@ static void *odb_source_loose_map_object(struct odb_source *source, const struct object_id *oid, unsigned long *size) { + struct odb_source_files *files = odb_source_files_downcast(source); const char *p; - int fd = open_loose_object(source->loose, oid, &p); + int fd = open_loose_object(files->loose, oid, &p); if (fd < 0) return NULL; @@ -397,6 +398,7 @@ static int read_object_info_from_path(struct odb_source *source, struct object_info *oi, enum object_info_flags flags) { + struct odb_source_files *files = odb_source_files_downcast(source); int ret; int fd; unsigned long mapsize; @@ -419,7 +421,7 @@ static int read_object_info_from_path(struct odb_source *source, struct stat st; if ((!oi || (!oi->disk_sizep && !oi->mtimep)) && (flags & OBJECT_INFO_QUICK)) { - ret = quick_has_loose(source->loose, oid) ? 0 : -1; + ret = quick_has_loose(files->loose, oid) ? 0 : -1; goto out; } @@ -540,6 +542,16 @@ int odb_source_loose_read_object_info(struct odb_source *source, enum object_info_flags flags) { static struct strbuf buf = STRBUF_INIT; + + /* + * The second read shouldn't cause new loose objects to show up, unless + * there was a race condition with a secondary process. We don't care + * about this case though, so we simply skip reading loose objects a + * second time. + */ + if (flags & OBJECT_INFO_SECOND_READ) + return -1; + odb_loose_path(source, &buf, oid); return read_object_info_from_path(source, buf.buf, oid, oi, flags); } @@ -1867,33 +1879,34 @@ static int append_loose_object(const struct object_id *oid, struct oidtree *odb_source_loose_cache(struct odb_source *source, const struct object_id *oid) { + struct odb_source_files *files = odb_source_files_downcast(source); int subdir_nr = oid->hash[0]; struct strbuf buf = STRBUF_INIT; - size_t word_bits = bitsizeof(source->loose->subdir_seen[0]); + size_t word_bits = bitsizeof(files->loose->subdir_seen[0]); size_t word_index = subdir_nr / word_bits; size_t mask = (size_t)1u << (subdir_nr % word_bits); uint32_t *bitmap; if (subdir_nr < 0 || - (size_t) subdir_nr >= bitsizeof(source->loose->subdir_seen)) + (size_t) subdir_nr >= bitsizeof(files->loose->subdir_seen)) BUG("subdir_nr out of range"); - bitmap = &source->loose->subdir_seen[word_index]; + bitmap = &files->loose->subdir_seen[word_index]; if (*bitmap & mask) - return source->loose->cache; - if (!source->loose->cache) { - ALLOC_ARRAY(source->loose->cache, 1); - oidtree_init(source->loose->cache); + return files->loose->cache; + if (!files->loose->cache) { + ALLOC_ARRAY(files->loose->cache, 1); + oidtree_init(files->loose->cache); } strbuf_addstr(&buf, source->path); for_each_file_in_obj_subdir(subdir_nr, &buf, source->odb->repo->hash_algo, append_loose_object, NULL, NULL, - source->loose->cache); + files->loose->cache); *bitmap |= mask; strbuf_release(&buf); - return source->loose->cache; + return files->loose->cache; } static void odb_source_loose_clear_cache(struct odb_source_loose *loose) @@ -1906,7 +1919,8 @@ static void odb_source_loose_clear_cache(struct odb_source_loose *loose) void odb_source_loose_reprepare(struct odb_source *source) { - odb_source_loose_clear_cache(source->loose); + struct odb_source_files *files = odb_source_files_downcast(source); + odb_source_loose_clear_cache(files->loose); } static int check_stream_oid(git_zstream *stream, diff --git a/odb.c b/odb.c index 776de5356c..84a31084d3 100644 --- a/odb.c +++ b/odb.c @@ -1,6 +1,5 @@ #include "git-compat-util.h" #include "abspath.h" -#include "chdir-notify.h" #include "commit-graph.h" #include "config.h" #include "dir.h" @@ -132,10 +131,10 @@ out: return usable; } -static void parse_alternates(const char *string, - int sep, - const char *relative_base, - struct strvec *out) +void parse_alternates(const char *string, + int sep, + const char *relative_base, + struct strvec *out) { struct strbuf pathbuf = STRBUF_INIT; struct strbuf buf = STRBUF_INIT; @@ -199,41 +198,6 @@ static void parse_alternates(const char *string, strbuf_release(&buf); } -static void odb_source_read_alternates(struct odb_source *source, - struct strvec *out) -{ - struct strbuf buf = STRBUF_INIT; - char *path; - - path = xstrfmt("%s/info/alternates", source->path); - if (strbuf_read_file(&buf, path, 1024) < 0) { - warn_on_fopen_errors(path); - free(path); - return; - } - parse_alternates(buf.buf, '\n', source->path, out); - - strbuf_release(&buf); - free(path); -} - - -static struct odb_source *odb_source_new(struct object_database *odb, - const char *path, - bool local) -{ - struct odb_source *source; - - CALLOC_ARRAY(source, 1); - source->odb = odb; - source->local = local; - source->path = xstrdup(path); - source->loose = odb_source_loose_new(source); - source->packfiles = packfile_store_new(source); - - return source; -} - static struct odb_source *odb_add_alternate_recursively(struct object_database *odb, const char *source, int depth) @@ -272,58 +236,6 @@ static struct odb_source *odb_add_alternate_recursively(struct object_database * return alternate; } -static int odb_source_write_alternate(struct odb_source *source, - const char *alternate) -{ - struct lock_file lock = LOCK_INIT; - char *path = xstrfmt("%s/%s", source->path, "info/alternates"); - FILE *in, *out; - int found = 0; - int ret; - - hold_lock_file_for_update(&lock, path, LOCK_DIE_ON_ERROR); - out = fdopen_lock_file(&lock, "w"); - if (!out) { - ret = error_errno(_("unable to fdopen alternates lockfile")); - goto out; - } - - in = fopen(path, "r"); - if (in) { - struct strbuf line = STRBUF_INIT; - - while (strbuf_getline(&line, in) != EOF) { - if (!strcmp(alternate, line.buf)) { - found = 1; - break; - } - fprintf_or_die(out, "%s\n", line.buf); - } - - strbuf_release(&line); - fclose(in); - } else if (errno != ENOENT) { - ret = error_errno(_("unable to read alternates file")); - goto out; - } - - if (found) { - rollback_lock_file(&lock); - } else { - fprintf_or_die(out, "%s\n", alternate); - if (commit_lock_file(&lock)) { - ret = error_errno(_("unable to move new alternates file into place")); - goto out; - } - } - - ret = 0; - -out: - free(path); - return ret; -} - void odb_add_to_alternates_file(struct object_database *odb, const char *dir) { @@ -373,14 +285,6 @@ struct odb_source *odb_set_temporary_primary_source(struct object_database *odb, return source->next; } -static void odb_source_free(struct odb_source *source) -{ - free(source->path); - odb_source_loose_free(source->loose); - packfile_store_free(source->packfiles); - free(source); -} - void odb_restore_primary_source(struct object_database *odb, struct odb_source *restore_source, const char *old_path) @@ -714,18 +618,19 @@ static int do_oid_object_info_extended(struct object_database *odb, while (1) { struct odb_source *source; - /* Most likely it's a loose object. */ - for (source = odb->sources; source; source = source->next) { - if (!packfile_store_read_object_info(source->packfiles, real, oi, flags) || - !odb_source_loose_read_object_info(source, real, oi, flags)) + for (source = odb->sources; source; source = source->next) + if (!odb_source_read_object_info(source, real, oi, flags)) return 0; - } - /* Not a loose object; someone else may have just packed it. */ + /* + * When the object hasn't been found we try a second read and + * tell the sources so. This may cause them to invalidate + * caches or reload on-disk state. + */ if (!(flags & OBJECT_INFO_QUICK)) { - odb_reprepare(odb->repo->objects); for (source = odb->sources; source; source = source->next) - if (!packfile_store_read_object_info(source->packfiles, real, oi, flags)) + if (!odb_source_read_object_info(source, real, oi, + flags | OBJECT_INFO_SECOND_READ)) return 0; } @@ -984,16 +889,10 @@ int odb_freshen_object(struct object_database *odb, const struct object_id *oid) { struct odb_source *source; - odb_prepare_alternates(odb); - for (source = odb->sources; source; source = source->next) { - if (packfile_store_freshen_object(source->packfiles, oid)) + for (source = odb->sources; source; source = source->next) + if (odb_source_freshen_object(source, oid)) return 1; - - if (odb_source_loose_freshen_object(source, oid)) - return 1; - } - return 0; } @@ -1010,15 +909,7 @@ int odb_for_each_object(struct object_database *odb, if (flags & ODB_FOR_EACH_OBJECT_LOCAL_ONLY && !source->local) continue; - if (!(flags & ODB_FOR_EACH_OBJECT_PROMISOR_ONLY)) { - ret = odb_source_loose_for_each_object(source, request, - cb, cb_data, flags); - if (ret) - return ret; - } - - ret = packfile_store_for_each_object(source->packfiles, request, - cb, cb_data, flags); + ret = odb_source_for_each_object(source, request, cb, cb_data, flags); if (ret) return ret; } @@ -1044,47 +935,15 @@ int odb_write_object_ext(struct object_database *odb, struct object_id *compat_oid, unsigned flags) { - return odb_source_loose_write_object(odb->sources, buf, len, type, - oid, compat_oid, flags); + return odb_source_write_object(odb->sources, buf, len, type, + oid, compat_oid, flags); } int odb_write_object_stream(struct object_database *odb, struct odb_write_stream *stream, size_t len, struct object_id *oid) { - return odb_source_loose_write_stream(odb->sources, stream, len, oid); -} - -static void odb_update_commondir(const char *name UNUSED, - const char *old_cwd, - const char *new_cwd, - void *cb_data) -{ - struct object_database *odb = cb_data; - struct tmp_objdir *tmp_objdir; - struct odb_source *source; - - tmp_objdir = tmp_objdir_unapply_primary_odb(); - - /* - * In theory, we only have to do this for the primary object source, as - * alternates' paths are always resolved to an absolute path. - */ - for (source = odb->sources; source; source = source->next) { - char *path; - - if (is_absolute_path(source->path)) - continue; - - path = reparent_relative_path(old_cwd, new_cwd, - source->path); - - free(source->path); - source->path = path; - } - - if (tmp_objdir) - tmp_objdir_reapply_primary_odb(tmp_objdir, old_cwd, new_cwd); + return odb_source_write_object_stream(odb->sources, stream, len, oid); } struct object_database *odb_new(struct repository *repo, @@ -1107,8 +966,6 @@ struct object_database *odb_new(struct repository *repo, free(to_free); - chdir_notify_register(NULL, odb_update_commondir, o); - return o; } @@ -1116,7 +973,7 @@ void odb_close(struct object_database *o) { struct odb_source *source; for (source = o->sources; source; source = source->next) - packfile_store_close(source->packfiles); + odb_source_close(source); close_commit_graph(o); } @@ -1152,8 +1009,6 @@ void odb_free(struct object_database *o) string_list_clear(&o->submodule_source_paths, 0); - chdir_notify_unregister(NULL, odb_update_commondir, o); - free(o); } @@ -1172,10 +1027,8 @@ void odb_reprepare(struct object_database *o) o->loaded_alternates = 0; odb_prepare_alternates(o); - for (source = o->sources; source; source = source->next) { - odb_source_loose_reprepare(source); - packfile_store_reprepare(source->packfiles); - } + for (source = o->sources; source; source = source->next) + odb_source_reprepare(source); o->approximate_object_count_valid = 0; diff --git a/odb.h b/odb.h index 68b8ec2289..86e0365c24 100644 --- a/odb.h +++ b/odb.h @@ -3,6 +3,7 @@ #include "hashmap.h" #include "object.h" +#include "odb/source.h" #include "oidset.h" #include "oidmap.h" #include "string-list.h" @@ -30,50 +31,6 @@ extern int fetch_if_missing; */ char *compute_alternate_path(const char *path, struct strbuf *err); -/* - * The source is the part of the object database that stores the actual - * objects. It thus encapsulates the logic to read and write the specific - * on-disk format. An object database can have multiple sources: - * - * - The primary source, which is typically located in "$GIT_DIR/objects". - * This is where new objects are usually written to. - * - * - Alternate sources, which are configured via "objects/info/alternates" or - * via the GIT_ALTERNATE_OBJECT_DIRECTORIES environment variable. These - * alternate sources are only used to read objects. - */ -struct odb_source { - struct odb_source *next; - - /* Object database that owns this object source. */ - struct object_database *odb; - - /* Private state for loose objects. */ - struct odb_source_loose *loose; - - /* Should only be accessed directly by packfile.c and midx.c. */ - struct packfile_store *packfiles; - - /* - * Figure out whether this is the local source of the owning - * repository, which would typically be its ".git/objects" directory. - * This local object directory is usually where objects would be - * written to. - */ - bool local; - - /* - * This object store is ephemeral, so there is no need to fsync. - */ - int will_destroy; - - /* - * Path to the source. If this is a relative path, it is relative to - * the current working directory. - */ - char *path; -}; - struct packed_git; struct packfile_store; struct cached_object_entry; @@ -382,30 +339,6 @@ struct object_info { */ #define OBJECT_INFO_INIT { 0 } -/* Flags that can be passed to `odb_read_object_info_extended()`. */ -enum object_info_flags { - /* Invoke lookup_replace_object() on the given hash. */ - OBJECT_INFO_LOOKUP_REPLACE = (1 << 0), - - /* Do not reprepare object sources when the first lookup has failed. */ - OBJECT_INFO_QUICK = (1 << 1), - - /* - * Do not attempt to fetch the object if missing (even if fetch_is_missing is - * nonzero). - */ - OBJECT_INFO_SKIP_FETCH_OBJECT = (1 << 2), - - /* Die if object corruption (not just an object being missing) was detected. */ - OBJECT_INFO_DIE_IF_CORRUPT = (1 << 3), - - /* - * This is meant for bulk prefetching of missing blobs in a partial - * clone. Implies OBJECT_INFO_SKIP_FETCH_OBJECT and OBJECT_INFO_QUICK. - */ - OBJECT_INFO_FOR_PREFETCH = (OBJECT_INFO_SKIP_FETCH_OBJECT | OBJECT_INFO_QUICK), -}; - /* * Read object info from the object database and populate the `object_info` * structure. Returns 0 on success, a negative error code otherwise. @@ -499,18 +432,6 @@ enum odb_for_each_object_flags { ODB_FOR_EACH_OBJECT_SKIP_ON_DISK_KEPT_PACKS = (1<<4), }; -/* - * A callback function that can be used to iterate through objects. If given, - * the optional `oi` parameter will be populated the same as if you would call - * `odb_read_object_info()`. - * - * Returning a non-zero error code will cause iteration to abort. The error - * code will be propagated. - */ -typedef int (*odb_for_each_object_cb)(const struct object_id *oid, - struct object_info *oi, - void *cb_data); - /* * Iterate through all objects contained in the object database. Note that * objects may be iterated over multiple times in case they are either stored @@ -579,4 +500,9 @@ int odb_write_object_stream(struct object_database *odb, struct odb_write_stream *stream, size_t len, struct object_id *oid); +void parse_alternates(const char *string, + int sep, + const char *relative_base, + struct strvec *out); + #endif /* ODB_H */ diff --git a/odb/source-files.c b/odb/source-files.c new file mode 100644 index 0000000000..14cb9adeca --- /dev/null +++ b/odb/source-files.c @@ -0,0 +1,239 @@ +#include "git-compat-util.h" +#include "abspath.h" +#include "chdir-notify.h" +#include "gettext.h" +#include "lockfile.h" +#include "object-file.h" +#include "odb.h" +#include "odb/source.h" +#include "odb/source-files.h" +#include "packfile.h" +#include "strbuf.h" +#include "write-or-die.h" + +static void odb_source_files_reparent(const char *name UNUSED, + const char *old_cwd, + const char *new_cwd, + void *cb_data) +{ + struct odb_source_files *files = cb_data; + char *path = reparent_relative_path(old_cwd, new_cwd, + files->base.path); + free(files->base.path); + files->base.path = path; +} + +static void odb_source_files_free(struct odb_source *source) +{ + struct odb_source_files *files = odb_source_files_downcast(source); + chdir_notify_unregister(NULL, odb_source_files_reparent, files); + odb_source_loose_free(files->loose); + packfile_store_free(files->packed); + odb_source_release(&files->base); + free(files); +} + +static void odb_source_files_close(struct odb_source *source) +{ + struct odb_source_files *files = odb_source_files_downcast(source); + packfile_store_close(files->packed); +} + +static void odb_source_files_reprepare(struct odb_source *source) +{ + struct odb_source_files *files = odb_source_files_downcast(source); + odb_source_loose_reprepare(&files->base); + packfile_store_reprepare(files->packed); +} + +static int odb_source_files_read_object_info(struct odb_source *source, + const struct object_id *oid, + struct object_info *oi, + enum object_info_flags flags) +{ + struct odb_source_files *files = odb_source_files_downcast(source); + + if (!packfile_store_read_object_info(files->packed, oid, oi, flags) || + !odb_source_loose_read_object_info(source, oid, oi, flags)) + return 0; + + return -1; +} + +static int odb_source_files_read_object_stream(struct odb_read_stream **out, + struct odb_source *source, + const struct object_id *oid) +{ + struct odb_source_files *files = odb_source_files_downcast(source); + if (!packfile_store_read_object_stream(out, files->packed, oid) || + !odb_source_loose_read_object_stream(out, source, oid)) + return 0; + return -1; +} + +static int odb_source_files_for_each_object(struct odb_source *source, + const struct object_info *request, + odb_for_each_object_cb cb, + void *cb_data, + unsigned flags) +{ + struct odb_source_files *files = odb_source_files_downcast(source); + int ret; + + if (!(flags & ODB_FOR_EACH_OBJECT_PROMISOR_ONLY)) { + ret = odb_source_loose_for_each_object(source, request, cb, cb_data, flags); + if (ret) + return ret; + } + + ret = packfile_store_for_each_object(files->packed, request, cb, cb_data, flags); + if (ret) + return ret; + + return 0; +} + +static int odb_source_files_freshen_object(struct odb_source *source, + const struct object_id *oid) +{ + struct odb_source_files *files = odb_source_files_downcast(source); + if (packfile_store_freshen_object(files->packed, oid) || + odb_source_loose_freshen_object(source, oid)) + return 1; + return 0; +} + +static int odb_source_files_write_object(struct odb_source *source, + const void *buf, unsigned long len, + enum object_type type, + struct object_id *oid, + struct object_id *compat_oid, + unsigned flags) +{ + return odb_source_loose_write_object(source, buf, len, type, + oid, compat_oid, flags); +} + +static int odb_source_files_write_object_stream(struct odb_source *source, + struct odb_write_stream *stream, + size_t len, + struct object_id *oid) +{ + return odb_source_loose_write_stream(source, stream, len, oid); +} + +static int odb_source_files_begin_transaction(struct odb_source *source, + struct odb_transaction **out) +{ + struct odb_transaction *tx = odb_transaction_files_begin(source); + if (!tx) + return -1; + *out = tx; + return 0; +} + +static int odb_source_files_read_alternates(struct odb_source *source, + struct strvec *out) +{ + struct strbuf buf = STRBUF_INIT; + char *path; + + path = xstrfmt("%s/info/alternates", source->path); + if (strbuf_read_file(&buf, path, 1024) < 0) { + warn_on_fopen_errors(path); + free(path); + return 0; + } + parse_alternates(buf.buf, '\n', source->path, out); + + strbuf_release(&buf); + free(path); + return 0; +} + +static int odb_source_files_write_alternate(struct odb_source *source, + const char *alternate) +{ + struct lock_file lock = LOCK_INIT; + char *path = xstrfmt("%s/%s", source->path, "info/alternates"); + FILE *in, *out; + int found = 0; + int ret; + + hold_lock_file_for_update(&lock, path, LOCK_DIE_ON_ERROR); + out = fdopen_lock_file(&lock, "w"); + if (!out) { + ret = error_errno(_("unable to fdopen alternates lockfile")); + goto out; + } + + in = fopen(path, "r"); + if (in) { + struct strbuf line = STRBUF_INIT; + + while (strbuf_getline(&line, in) != EOF) { + if (!strcmp(alternate, line.buf)) { + found = 1; + break; + } + fprintf_or_die(out, "%s\n", line.buf); + } + + strbuf_release(&line); + fclose(in); + } else if (errno != ENOENT) { + ret = error_errno(_("unable to read alternates file")); + goto out; + } + + if (found) { + rollback_lock_file(&lock); + } else { + fprintf_or_die(out, "%s\n", alternate); + if (commit_lock_file(&lock)) { + ret = error_errno(_("unable to move new alternates file into place")); + goto out; + } + } + + ret = 0; + +out: + free(path); + return ret; +} + +struct odb_source_files *odb_source_files_new(struct object_database *odb, + const char *path, + bool local) +{ + struct odb_source_files *files; + + CALLOC_ARRAY(files, 1); + odb_source_init(&files->base, odb, ODB_SOURCE_FILES, path, local); + files->loose = odb_source_loose_new(&files->base); + files->packed = packfile_store_new(&files->base); + + files->base.free = odb_source_files_free; + files->base.close = odb_source_files_close; + files->base.reprepare = odb_source_files_reprepare; + files->base.read_object_info = odb_source_files_read_object_info; + files->base.read_object_stream = odb_source_files_read_object_stream; + files->base.for_each_object = odb_source_files_for_each_object; + files->base.freshen_object = odb_source_files_freshen_object; + files->base.write_object = odb_source_files_write_object; + files->base.write_object_stream = odb_source_files_write_object_stream; + files->base.begin_transaction = odb_source_files_begin_transaction; + files->base.read_alternates = odb_source_files_read_alternates; + files->base.write_alternate = odb_source_files_write_alternate; + + /* + * Ideally, we would only ever store absolute paths in the source. This + * is not (yet) possible though because we access and assume relative + * paths in the primary ODB source in some user-facing functionality. + */ + if (!is_absolute_path(path)) + chdir_notify_register(NULL, odb_source_files_reparent, files); + + return files; +} diff --git a/odb/source-files.h b/odb/source-files.h new file mode 100644 index 0000000000..23a3b4e04b --- /dev/null +++ b/odb/source-files.h @@ -0,0 +1,35 @@ +#ifndef ODB_SOURCE_FILES_H +#define ODB_SOURCE_FILES_H + +#include "odb/source.h" + +struct odb_source_loose; +struct packfile_store; + +/* + * The files object database source uses a combination of loose objects and + * packfiles. It is the default backend used by Git to store objects. + */ +struct odb_source_files { + struct odb_source base; + struct odb_source_loose *loose; + struct packfile_store *packed; +}; + +/* Allocate and initialize a new object source. */ +struct odb_source_files *odb_source_files_new(struct object_database *odb, + const char *path, + bool local); + +/* + * Cast the given object database source to the files backend. This will cause + * a BUG in case the source doesn't use this backend. + */ +static inline struct odb_source_files *odb_source_files_downcast(struct odb_source *source) +{ + if (source->type != ODB_SOURCE_FILES) + BUG("trying to downcast source of type '%d' to files", source->type); + return container_of(source, struct odb_source_files, base); +} + +#endif diff --git a/odb/source.c b/odb/source.c new file mode 100644 index 0000000000..7993dcbd65 --- /dev/null +++ b/odb/source.c @@ -0,0 +1,38 @@ +#include "git-compat-util.h" +#include "object-file.h" +#include "odb/source-files.h" +#include "odb/source.h" +#include "packfile.h" + +struct odb_source *odb_source_new(struct object_database *odb, + const char *path, + bool local) +{ + return &odb_source_files_new(odb, path, local)->base; +} + +void odb_source_init(struct odb_source *source, + struct object_database *odb, + enum odb_source_type type, + const char *path, + bool local) +{ + source->odb = odb; + source->type = type; + source->local = local; + source->path = xstrdup(path); +} + +void odb_source_free(struct odb_source *source) +{ + if (!source) + return; + source->free(source); +} + +void odb_source_release(struct odb_source *source) +{ + if (!source) + return; + free(source->path); +} diff --git a/odb/source.h b/odb/source.h new file mode 100644 index 0000000000..0e99052e08 --- /dev/null +++ b/odb/source.h @@ -0,0 +1,464 @@ +#ifndef ODB_SOURCE_H +#define ODB_SOURCE_H + +#include "object.h" + +enum odb_source_type { + /* + * The "unknown" type, which should never be in use. This is type + * mostly exists to catch cases where the type field remains zeroed + * out. + */ + ODB_SOURCE_UNKNOWN, + + /* The "files" backend that uses loose objects and packfiles. */ + ODB_SOURCE_FILES, +}; + +/* Flags that can be passed to `odb_read_object_info_extended()`. */ +enum object_info_flags { + /* Invoke lookup_replace_object() on the given hash. */ + OBJECT_INFO_LOOKUP_REPLACE = (1 << 0), + + /* Do not reprepare object sources when the first lookup has failed. */ + OBJECT_INFO_QUICK = (1 << 1), + + /* + * Do not attempt to fetch the object if missing (even if fetch_is_missing is + * nonzero). + */ + OBJECT_INFO_SKIP_FETCH_OBJECT = (1 << 2), + + /* Die if object corruption (not just an object being missing) was detected. */ + OBJECT_INFO_DIE_IF_CORRUPT = (1 << 3), + + /* + * We have already tried reading the object, but it couldn't be found + * via any of the attached sources, and are now doing a second read. + * This second read asks the individual sources to also evaluate + * whether any on-disk state may have changed that may have caused the + * object to appear. + * + * This flag is for internal use, only. The second read only occurs + * when `OBJECT_INFO_QUICK` was not passed. + */ + OBJECT_INFO_SECOND_READ = (1 << 4), + + /* + * This is meant for bulk prefetching of missing blobs in a partial + * clone. Implies OBJECT_INFO_SKIP_FETCH_OBJECT and OBJECT_INFO_QUICK. + */ + OBJECT_INFO_FOR_PREFETCH = (OBJECT_INFO_SKIP_FETCH_OBJECT | OBJECT_INFO_QUICK), +}; + +struct object_id; +struct object_info; +struct odb_read_stream; +struct odb_transaction; +struct odb_write_stream; +struct strvec; + +/* + * A callback function that can be used to iterate through objects. If given, + * the optional `oi` parameter will be populated the same as if you would call + * `odb_read_object_info()`. + * + * Returning a non-zero error code will cause iteration to abort. The error + * code will be propagated. + */ +typedef int (*odb_for_each_object_cb)(const struct object_id *oid, + struct object_info *oi, + void *cb_data); + +/* + * The source is the part of the object database that stores the actual + * objects. It thus encapsulates the logic to read and write the specific + * on-disk format. An object database can have multiple sources: + * + * - The primary source, which is typically located in "$GIT_DIR/objects". + * This is where new objects are usually written to. + * + * - Alternate sources, which are configured via "objects/info/alternates" or + * via the GIT_ALTERNATE_OBJECT_DIRECTORIES environment variable. These + * alternate sources are only used to read objects. + */ +struct odb_source { + struct odb_source *next; + + /* Object database that owns this object source. */ + struct object_database *odb; + + /* The type used by this source. */ + enum odb_source_type type; + + /* + * Figure out whether this is the local source of the owning + * repository, which would typically be its ".git/objects" directory. + * This local object directory is usually where objects would be + * written to. + */ + bool local; + + /* + * This object store is ephemeral, so there is no need to fsync. + */ + int will_destroy; + + /* + * Path to the source. If this is a relative path, it is relative to + * the current working directory. + */ + char *path; + + /* + * This callback is expected to free the underlying object database source and + * all associated resources. The function will never be called with a NULL pointer. + */ + void (*free)(struct odb_source *source); + + /* + * This callback is expected to close any open resources, like for + * example file descriptors or connections. The source is expected to + * still be usable after it has been closed. Closed resources may need + * to be reopened in that case. + */ + void (*close)(struct odb_source *source); + + /* + * This callback is expected to clear underlying caches of the object + * database source. The function is called when the repository has for + * example just been repacked so that new objects will become visible. + */ + void (*reprepare)(struct odb_source *source); + + /* + * This callback is expected to read object information from the object + * database source. The object info will be partially populated with + * pointers for each bit of information that was requested by the + * caller. + * + * The flags field is a combination of `OBJECT_INFO` flags. Only the + * following fields need to be handled by the backend: + * + * - `OBJECT_INFO_QUICK` indicates it is fine to use caches without + * re-verifying the data. + * + * - `OBJECT_INFO_SECOND_READ` indicates that the initial object + * lookup has failed and that the object sources should check + * whether any of its on-disk state has changed that may have + * caused the object to appear. Sources are free to ignore the + * second read in case they know that the first read would have + * already surfaced the object without reloading any on-disk state. + * + * The callback is expected to return a negative error code in case + * reading the object has failed, 0 otherwise. + */ + int (*read_object_info)(struct odb_source *source, + const struct object_id *oid, + struct object_info *oi, + enum object_info_flags flags); + + /* + * This callback is expected to create a new read stream that can be + * used to stream the object identified by the given ID. + * + * The callback is expected to return a negative error code in case + * creating the object stream has failed, 0 otherwise. + */ + int (*read_object_stream)(struct odb_read_stream **out, + struct odb_source *source, + const struct object_id *oid); + + /* + * This callback is expected to iterate over all objects stored in this + * source and invoke the callback function for each of them. It is + * valid to yield the same object multiple time. A non-zero exit code + * from the object callback shall abort iteration. + * + * The optional `oi` structure shall be populated similar to how an individual + * call to `odb_source_read_object_info()` would have behaved. If the caller + * passes a `NULL` pointer then the object itself shall not be read. + * + * The callback is expected to return a negative error code in case the + * iteration has failed to read all objects, 0 otherwise. When the + * callback function returns a non-zero error code then that error code + * should be returned. + */ + int (*for_each_object)(struct odb_source *source, + const struct object_info *request, + odb_for_each_object_cb cb, + void *cb_data, + unsigned flags); + + /* + * This callback is expected to freshen the given object so that its + * last access time is set to the current time. This is used to ensure + * that objects that are recent will not get garbage collected even if + * they were unreachable. + * + * Returns 0 in case the object does not exist, 1 in case the object + * has been freshened. + */ + int (*freshen_object)(struct odb_source *source, + const struct object_id *oid); + + /* + * This callback is expected to persist the given object into the + * object source. In case the object already exists it shall be + * freshened. + * + * The flags field is a combination of `WRITE_OBJECT` flags. + * + * The resulting object ID (and optionally the compatibility object ID) + * shall be written into the out pointers. The callback is expected to + * return 0 on success, a negative error code otherwise. + */ + int (*write_object)(struct odb_source *source, + const void *buf, unsigned long len, + enum object_type type, + struct object_id *oid, + struct object_id *compat_oid, + unsigned flags); + + /* + * This callback is expected to persist the given object stream into + * the object source. + * + * The resulting object ID shall be written into the out pointer. The + * callback is expected to return 0 on success, a negative error code + * otherwise. + */ + int (*write_object_stream)(struct odb_source *source, + struct odb_write_stream *stream, size_t len, + struct object_id *oid); + + /* + * This callback is expected to create a new transaction that can be + * used to write objects to. The objects shall only be persisted into + * the object database when the transcation's commit function is + * called. Otherwise, the objects shall be discarded. + * + * Returns 0 on success, in which case the `*out` pointer will have + * been populated with the object database transaction. Returns a + * negative error code otherwise. + */ + int (*begin_transaction)(struct odb_source *source, + struct odb_transaction **out); + + /* + * This callback is expected to read the list of alternate object + * database sources connected to it and write them into the `strvec`. + * + * The format is expected to follow the "objectStorage" extension + * format with `(backend://)?payload` syntax. If the payload contains + * paths, these paths must be resolved to absolute paths. + * + * The callback is expected to return 0 on success, a negative error + * code otherwise. + */ + int (*read_alternates)(struct odb_source *source, + struct strvec *out); + + /* + * This callback is expected to persist the singular alternate passed + * to it into its list of alternates. Any pre-existing alternates are + * expected to remain active. Subsequent calls to `read_alternates` are + * thus expected to yield the pre-existing list of alternates plus the + * newly added alternate appended to its end. + * + * The callback is expected to return 0 on success, a negative error + * code otherwise. + */ + int (*write_alternate)(struct odb_source *source, + const char *alternate); +}; + +/* + * Allocate and initialize a new source for the given object database located + * at `path`. `local` indicates whether or not the source is the local and thus + * primary object source of the object database. + */ +struct odb_source *odb_source_new(struct object_database *odb, + const char *path, + bool local); + +/* + * Initialize the source for the given object database located at `path`. + * `local` indicates whether or not the source is the local and thus primary + * object source of the object database. + * + * This function is only supposed to be called by specific object source + * implementations. + */ +void odb_source_init(struct odb_source *source, + struct object_database *odb, + enum odb_source_type type, + const char *path, + bool local); + +/* + * Free the object database source, releasing all associated resources and + * freeing the structure itself. + */ +void odb_source_free(struct odb_source *source); + +/* + * Release the object database source, releasing all associated resources. + * + * This function is only supposed to be called by specific object source + * implementations. + */ +void odb_source_release(struct odb_source *source); + +/* + * Close the object database source without releasing he underlying data. The + * source can still be used going forward, but it first needs to be reopened. + * This can be useful to reduce resource usage. + */ +static inline void odb_source_close(struct odb_source *source) +{ + source->close(source); +} + +/* + * Reprepare the object database source and clear any caches. Depending on the + * backend used this may have the effect that concurrently-written objects + * become visible. + */ +static inline void odb_source_reprepare(struct odb_source *source) +{ + source->reprepare(source); +} + +/* + * Read an object from the object database source identified by its object ID. + * Returns 0 on success, a negative error code otherwise. + */ +static inline int odb_source_read_object_info(struct odb_source *source, + const struct object_id *oid, + struct object_info *oi, + enum object_info_flags flags) +{ + return source->read_object_info(source, oid, oi, flags); +} + +/* + * Create a new read stream for the given object ID. Returns 0 on success, a + * negative error code otherwise. + */ +static inline int odb_source_read_object_stream(struct odb_read_stream **out, + struct odb_source *source, + const struct object_id *oid) +{ + return source->read_object_stream(out, source, oid); +} + +/* + * Iterate through all objects contained in the given source and invoke the + * callback function for each of them. Returning a non-zero code from the + * callback function aborts iteration. There is no guarantee that objects + * are only iterated over once. + * + * The optional `oi` structure shall be populated similar to how an individual + * call to `odb_source_read_object_info()` would have behaved. If the caller + * passes a `NULL` pointer then the object itself shall not be read. + * + * The flags is a bitfield of `ODB_FOR_EACH_OBJECT_*` flags. Not all flags may + * apply to a specific backend, so whether or not they are honored is defined + * by the implementation. + * + * Returns 0 when all objects have been iterated over, a negative error code in + * case iteration has failed, or a non-zero value returned from the callback. + */ +static inline int odb_source_for_each_object(struct odb_source *source, + const struct object_info *request, + odb_for_each_object_cb cb, + void *cb_data, + unsigned flags) +{ + return source->for_each_object(source, request, cb, cb_data, flags); +} + +/* + * Freshen an object in the object database by updating its timestamp. + * Returns 1 in case the object has been freshened, 0 in case the object does + * not exist. + */ +static inline int odb_source_freshen_object(struct odb_source *source, + const struct object_id *oid) +{ + return source->freshen_object(source, oid); +} + +/* + * Write an object into the object database source. Returns 0 on success, a + * negative error code otherwise. Populates the given out pointers for the + * object ID and the compatibility object ID, if non-NULL. + */ +static inline int odb_source_write_object(struct odb_source *source, + const void *buf, unsigned long len, + enum object_type type, + struct object_id *oid, + struct object_id *compat_oid, + unsigned flags) +{ + return source->write_object(source, buf, len, type, oid, + compat_oid, flags); +} + +/* + * Write an object into the object database source via a stream. The overall + * length of the object must be known in advance. + * + * Return 0 on success, a negative error code otherwise. Populates the given + * out pointer for the object ID. + */ +static inline int odb_source_write_object_stream(struct odb_source *source, + struct odb_write_stream *stream, + size_t len, + struct object_id *oid) +{ + return source->write_object_stream(source, stream, len, oid); +} + +/* + * Read the list of alternative object database sources from the given backend + * and populate the `strvec` with them. The listing is not recursive -- that + * is, if any of the yielded alternate sources has alternates itself, those + * will not be yielded as part of this function call. + * + * Return 0 on success, a negative error code otherwise. + */ +static inline int odb_source_read_alternates(struct odb_source *source, + struct strvec *out) +{ + return source->read_alternates(source, out); +} + +/* + * Write and persist a new alternate object database source for the given + * source. Any preexisting alternates are expected to stay valid, and the new + * alternate shall be appended to the end of the list. + * + * Returns 0 on success, a negative error code otherwise. + */ +static inline int odb_source_write_alternate(struct odb_source *source, + const char *alternate) +{ + return source->write_alternate(source, alternate); +} + +/* + * Create a new transaction that can be used to write objects into a temporary + * staging area. The objects will only be persisted when the transaction is + * committed. + * + * Returns 0 on success, a negative error code otherwise. + */ +static inline int odb_source_begin_transaction(struct odb_source *source, + struct odb_transaction **out) +{ + return source->begin_transaction(source, out); +} + +#endif diff --git a/odb/streaming.c b/odb/streaming.c index 4a4474f891..a4355cd245 100644 --- a/odb/streaming.c +++ b/odb/streaming.c @@ -6,11 +6,9 @@ #include "convert.h" #include "environment.h" #include "repository.h" -#include "object-file.h" #include "odb.h" #include "odb/streaming.h" #include "replace-object.h" -#include "packfile.h" #define FILTER_BUFFER (1024*16) @@ -186,11 +184,9 @@ static int istream_source(struct odb_read_stream **out, struct odb_source *source; odb_prepare_alternates(odb); - for (source = odb->sources; source; source = source->next) { - if (!packfile_store_read_object_stream(out, source->packfiles, oid) || - !odb_source_loose_read_object_stream(out, source, oid)) + for (source = odb->sources; source; source = source->next) + if (!odb_source_read_object_stream(out, source, oid)) return 0; - } return open_istream_incore(out, odb, oid); } diff --git a/packfile.c b/packfile.c index 4a6d4a80ea..215a23e42b 100644 --- a/packfile.c +++ b/packfile.c @@ -362,9 +362,11 @@ static int unuse_one_window(struct object_database *odb) struct packed_git *lru_p = NULL; struct pack_window *lru_w = NULL, *lru_l = NULL; - for (source = odb->sources; source; source = source->next) - for (e = source->packfiles->packs.head; e; e = e->next) + for (source = odb->sources; source; source = source->next) { + struct odb_source_files *files = odb_source_files_downcast(source); + for (e = files->packed->packs.head; e; e = e->next) scan_windows(e->pack, &lru_p, &lru_w, &lru_l); + } if (lru_p) { munmap(lru_w->base, lru_w->len); @@ -537,7 +539,8 @@ static int close_one_pack(struct repository *r) int accept_windows_inuse = 1; for (source = r->objects->sources; source; source = source->next) { - for (e = source->packfiles->packs.head; e; e = e->next) { + struct odb_source_files *files = odb_source_files_downcast(source); + for (e = files->packed->packs.head; e; e = e->next) { if (e->pack->pack_fd == -1) continue; find_lru_pack(e->pack, &lru_p, &mru_w, &accept_windows_inuse); @@ -987,13 +990,14 @@ static void prepare_pack(const char *full_name, size_t full_name_len, const char *file_name, void *_data) { struct prepare_pack_data *data = (struct prepare_pack_data *)_data; + struct odb_source_files *files = odb_source_files_downcast(data->source); size_t base_len = full_name_len; if (strip_suffix_mem(full_name, &base_len, ".idx") && - !(data->source->packfiles->midx && - midx_contains_pack(data->source->packfiles->midx, file_name))) { + !(files->packed->midx && + midx_contains_pack(files->packed->midx, file_name))) { char *trimmed_path = xstrndup(full_name, full_name_len); - packfile_store_load_pack(data->source->packfiles, + packfile_store_load_pack(files->packed, trimmed_path, data->source->local); free(trimmed_path); } @@ -1247,8 +1251,10 @@ const struct packed_git *has_packed_and_bad(struct repository *r, struct odb_source *source; for (source = r->objects->sources; source; source = source->next) { + struct odb_source_files *files = odb_source_files_downcast(source); struct packfile_list_entry *e; - for (e = source->packfiles->packs.head; e; e = e->next) + + for (e = files->packed->packs.head; e; e = e->next) if (oidset_contains(&e->pack->bad_objects, oid)) return e->pack; } @@ -2175,11 +2181,19 @@ int packfile_store_freshen_object(struct packfile_store *store, int packfile_store_read_object_info(struct packfile_store *store, const struct object_id *oid, struct object_info *oi, - enum object_info_flags flags UNUSED) + enum object_info_flags flags) { struct pack_entry e; int ret; + /* + * In case the first read didn't surface the object, we have to reload + * packfiles. This may cause us to discover new packfiles that have + * been added since the last time we have prepared the packfile store. + */ + if (flags & OBJECT_INFO_SECOND_READ) + packfile_store_reprepare(store); + if (!find_pack_entry(store, oid, &e)) return 1; @@ -2254,7 +2268,8 @@ int has_object_pack(struct repository *r, const struct object_id *oid) odb_prepare_alternates(r->objects); for (source = r->objects->sources; source; source = source->next) { - int ret = find_pack_entry(source->packfiles, oid, &e); + struct odb_source_files *files = odb_source_files_downcast(source); + int ret = find_pack_entry(files->packed, oid, &e); if (ret) return ret; } @@ -2269,9 +2284,10 @@ int has_object_kept_pack(struct repository *r, const struct object_id *oid, struct pack_entry e; for (source = r->objects->sources; source; source = source->next) { + struct odb_source_files *files = odb_source_files_downcast(source); struct packed_git **cache; - cache = packfile_store_get_kept_pack_cache(source->packfiles, flags); + cache = packfile_store_get_kept_pack_cache(files->packed, flags); for (; *cache; cache++) { struct packed_git *p = *cache; diff --git a/packfile.h b/packfile.h index 85ce44f8ee..8b04a258a7 100644 --- a/packfile.h +++ b/packfile.h @@ -4,6 +4,7 @@ #include "list.h" #include "object.h" #include "odb.h" +#include "odb/source-files.h" #include "oidset.h" #include "repository.h" #include "strmap.h" @@ -192,7 +193,8 @@ static inline struct repo_for_each_pack_data repo_for_eack_pack_data_init(struct odb_prepare_alternates(repo->objects); for (struct odb_source *source = repo->objects->sources; source; source = source->next) { - struct packfile_list_entry *entry = packfile_store_get_packs(source->packfiles); + struct odb_source_files *files = odb_source_files_downcast(source); + struct packfile_list_entry *entry = packfile_store_get_packs(files->packed); if (!entry) continue; data.source = source; @@ -212,7 +214,8 @@ static inline void repo_for_each_pack_data_next(struct repo_for_each_pack_data * return; for (source = data->source->next; source; source = source->next) { - struct packfile_list_entry *entry = packfile_store_get_packs(source->packfiles); + struct odb_source_files *files = odb_source_files_downcast(source); + struct packfile_list_entry *entry = packfile_store_get_packs(files->packed); if (!entry) continue; data->source = source; diff --git a/tmp-objdir.c b/tmp-objdir.c index 9f5a1788cd..e436eed07e 100644 --- a/tmp-objdir.c +++ b/tmp-objdir.c @@ -36,6 +36,21 @@ static void tmp_objdir_free(struct tmp_objdir *t) free(t); } +static void tmp_objdir_reparent(const char *name UNUSED, + const char *old_cwd, + const char *new_cwd, + void *cb_data) +{ + struct tmp_objdir *t = cb_data; + char *path; + + path = reparent_relative_path(old_cwd, new_cwd, + t->path.buf); + strbuf_reset(&t->path); + strbuf_addstr(&t->path, path); + free(path); +} + int tmp_objdir_destroy(struct tmp_objdir *t) { int err; @@ -51,6 +66,7 @@ int tmp_objdir_destroy(struct tmp_objdir *t) err = remove_dir_recursively(&t->path, 0); + chdir_notify_unregister(NULL, tmp_objdir_reparent, t); tmp_objdir_free(t); return err; @@ -137,6 +153,9 @@ struct tmp_objdir *tmp_objdir_create(struct repository *r, strbuf_addf(&t->path, "%s/tmp_objdir-%s-XXXXXX", repo_get_object_directory(r), prefix); + if (!is_absolute_path(t->path.buf)) + chdir_notify_register(NULL, tmp_objdir_reparent, t); + if (!mkdtemp(t->path.buf)) { /* free, not destroy, as we never touched the filesystem */ tmp_objdir_free(t); @@ -315,26 +334,3 @@ void tmp_objdir_replace_primary_odb(struct tmp_objdir *t, int will_destroy) t->path.buf, will_destroy); t->will_destroy = will_destroy; } - -struct tmp_objdir *tmp_objdir_unapply_primary_odb(void) -{ - if (!the_tmp_objdir || !the_tmp_objdir->prev_source) - return NULL; - - odb_restore_primary_source(the_tmp_objdir->repo->objects, - the_tmp_objdir->prev_source, the_tmp_objdir->path.buf); - the_tmp_objdir->prev_source = NULL; - return the_tmp_objdir; -} - -void tmp_objdir_reapply_primary_odb(struct tmp_objdir *t, const char *old_cwd, - const char *new_cwd) -{ - char *path; - - path = reparent_relative_path(old_cwd, new_cwd, t->path.buf); - strbuf_reset(&t->path); - strbuf_addstr(&t->path, path); - free(path); - tmp_objdir_replace_primary_odb(t, t->will_destroy); -} diff --git a/tmp-objdir.h b/tmp-objdir.h index fceda14979..ccf800faa7 100644 --- a/tmp-objdir.h +++ b/tmp-objdir.h @@ -68,19 +68,4 @@ void tmp_objdir_add_as_alternate(const struct tmp_objdir *); */ void tmp_objdir_replace_primary_odb(struct tmp_objdir *, int will_destroy); -/* - * If the primary object database was replaced by a temporary object directory, - * restore it to its original value while keeping the directory contents around. - * Returns NULL if the primary object database was not replaced. - */ -struct tmp_objdir *tmp_objdir_unapply_primary_odb(void); - -/* - * Reapplies the former primary temporary object database, after potentially - * changing its relative path. - */ -void tmp_objdir_reapply_primary_odb(struct tmp_objdir *, const char *old_cwd, - const char *new_cwd); - - #endif /* TMP_OBJDIR_H */