diff --git a/builtin/cat-file.c b/builtin/cat-file.c index 0e3f10a946..3914a2a3f6 100644 --- a/builtin/cat-file.c +++ b/builtin/cat-file.c @@ -169,7 +169,8 @@ static int cat_one_file(int opt, const char *exp_type, const char *obj_name, goto cleanup; case 'e': - ret = !repo_has_object_file(the_repository, &oid); + ret = !has_object(the_repository, &oid, + HAS_OBJECT_RECHECK_PACKED | HAS_OBJECT_FETCH_PROMISOR); goto cleanup; case 'w': diff --git a/builtin/clone.c b/builtin/clone.c index b6f378c438..91b9cd0d16 100644 --- a/builtin/clone.c +++ b/builtin/clone.c @@ -504,9 +504,7 @@ static void write_followtags(const struct ref *refs, const char *msg) continue; if (ends_with(ref->name, "^{}")) continue; - if (!repo_has_object_file_with_flags(the_repository, &ref->old_oid, - OBJECT_INFO_QUICK | - OBJECT_INFO_SKIP_FETCH_OBJECT)) + if (!has_object(the_repository, &ref->old_oid, 0)) continue; refs_update_ref(get_main_ref_store(the_repository), msg, ref->name, &ref->old_oid, NULL, 0, diff --git a/builtin/count-objects.c b/builtin/count-objects.c index 0bb5360b2f..a88c0c9c09 100644 --- a/builtin/count-objects.c +++ b/builtin/count-objects.c @@ -12,7 +12,7 @@ #include "parse-options.h" #include "quote.h" #include "packfile.h" -#include "object-store.h" +#include "object-file.h" static unsigned long garbage; static off_t size_garbage; diff --git a/builtin/fast-import.c b/builtin/fast-import.c index c1e198f4e3..b2839c5f43 100644 --- a/builtin/fast-import.c +++ b/builtin/fast-import.c @@ -811,7 +811,8 @@ static char *keep_pack(const char *curr_index_name) int keep_fd; odb_pack_name(pack_data->repo, &name, pack_data->hash, "keep"); - keep_fd = odb_pack_keep(name.buf); + keep_fd = safe_create_file_with_leading_directories(pack_data->repo, + name.buf); if (keep_fd < 0) die_errno("cannot create keep file"); write_or_die(keep_fd, keep_msg, strlen(keep_msg)); diff --git a/builtin/fetch.c b/builtin/fetch.c index 5279997c96..cda6eaf1fd 100644 --- a/builtin/fetch.c +++ b/builtin/fetch.c @@ -337,7 +337,6 @@ static void find_non_local_tags(const struct ref *refs, struct string_list_item *remote_ref_item; const struct ref *ref; struct refname_hash_entry *item = NULL; - const int quick_flags = OBJECT_INFO_QUICK | OBJECT_INFO_SKIP_FETCH_OBJECT; refname_hash_init(&existing_refs); refname_hash_init(&remote_refs); @@ -367,9 +366,9 @@ static void find_non_local_tags(const struct ref *refs, */ if (ends_with(ref->name, "^{}")) { if (item && - !repo_has_object_file_with_flags(the_repository, &ref->old_oid, quick_flags) && + !has_object(the_repository, &ref->old_oid, 0) && !oidset_contains(&fetch_oids, &ref->old_oid) && - !repo_has_object_file_with_flags(the_repository, &item->oid, quick_flags) && + !has_object(the_repository, &item->oid, 0) && !oidset_contains(&fetch_oids, &item->oid)) clear_item(item); item = NULL; @@ -383,7 +382,7 @@ static void find_non_local_tags(const struct ref *refs, * fetch. */ if (item && - !repo_has_object_file_with_flags(the_repository, &item->oid, quick_flags) && + !has_object(the_repository, &item->oid, 0) && !oidset_contains(&fetch_oids, &item->oid)) clear_item(item); @@ -404,7 +403,7 @@ static void find_non_local_tags(const struct ref *refs, * checked to see if it needs fetching. */ if (item && - !repo_has_object_file_with_flags(the_repository, &item->oid, quick_flags) && + !has_object(the_repository, &item->oid, 0) && !oidset_contains(&fetch_oids, &item->oid)) clear_item(item); @@ -911,7 +910,8 @@ static int update_local_ref(struct ref *ref, struct commit *current = NULL, *updated; int fast_forward = 0; - if (!repo_has_object_file(the_repository, &ref->new_oid)) + if (!has_object(the_repository, &ref->new_oid, + HAS_OBJECT_RECHECK_PACKED | HAS_OBJECT_FETCH_PROMISOR)) die(_("object %s not found"), oid_to_hex(&ref->new_oid)); if (oideq(&ref->old_oid, &ref->new_oid)) { @@ -1330,8 +1330,7 @@ static int check_exist_and_connected(struct ref *ref_map) * we need all direct targets to exist. */ for (r = rm; r; r = r->next) { - if (!repo_has_object_file_with_flags(the_repository, &r->old_oid, - OBJECT_INFO_SKIP_FETCH_OBJECT)) + if (!has_object(the_repository, &r->old_oid, HAS_OBJECT_RECHECK_PACKED)) return -1; } diff --git a/builtin/gc.c b/builtin/gc.c index e690453d4f..78a2751aa8 100644 --- a/builtin/gc.c +++ b/builtin/gc.c @@ -28,7 +28,7 @@ #include "commit.h" #include "commit-graph.h" #include "packfile.h" -#include "object-store.h" +#include "object-file.h" #include "pack.h" #include "pack-objects.h" #include "path.h" diff --git a/builtin/index-pack.c b/builtin/index-pack.c index 60a8ee05db..147e9b8b47 100644 --- a/builtin/index-pack.c +++ b/builtin/index-pack.c @@ -892,9 +892,8 @@ static void sha1_object(const void *data, struct object_entry *obj_entry, if (startup_info->have_repository) { read_lock(); - collision_test_needed = - repo_has_object_file_with_flags(the_repository, oid, - OBJECT_INFO_QUICK); + collision_test_needed = has_object(the_repository, oid, + HAS_OBJECT_FETCH_PROMISOR); read_unlock(); } @@ -1565,7 +1564,7 @@ static void write_special_file(const char *suffix, const char *msg, else filename = odb_pack_name(the_repository, &name_buf, hash, suffix); - fd = odb_pack_keep(filename); + fd = safe_create_file_with_leading_directories(the_repository, filename); if (fd < 0) { if (errno != EEXIST) die_errno(_("cannot write %s file '%s'"), diff --git a/builtin/receive-pack.c b/builtin/receive-pack.c index be314879e8..c92e57ba18 100644 --- a/builtin/receive-pack.c +++ b/builtin/receive-pack.c @@ -1506,7 +1506,9 @@ static const char *update(struct command *cmd, struct shallow_info *si) } } - if (!is_null_oid(new_oid) && !repo_has_object_file(the_repository, new_oid)) { + if (!is_null_oid(new_oid) && + !has_object(the_repository, new_oid, + HAS_OBJECT_RECHECK_PACKED | HAS_OBJECT_FETCH_PROMISOR)) { error("unpack should have generated %s, " "but I can't find it!", oid_to_hex(new_oid)); ret = "bad pack"; diff --git a/builtin/remote.c b/builtin/remote.c index b4baa34e66..0d6755bcb7 100644 --- a/builtin/remote.c +++ b/builtin/remote.c @@ -454,7 +454,8 @@ static int get_push_ref_states(const struct ref *remote_refs, info->status = PUSH_STATUS_UPTODATE; else if (is_null_oid(&ref->old_oid)) info->status = PUSH_STATUS_CREATE; - else if (repo_has_object_file(the_repository, &ref->old_oid) && + else if (has_object(the_repository, &ref->old_oid, + HAS_OBJECT_RECHECK_PACKED | HAS_OBJECT_FETCH_PROMISOR) && ref_newer(&ref->new_oid, &ref->old_oid)) info->status = PUSH_STATUS_FASTFORWARD; else diff --git a/builtin/show-ref.c b/builtin/show-ref.c index f81209f23c..623a52a45f 100644 --- a/builtin/show-ref.c +++ b/builtin/show-ref.c @@ -35,7 +35,8 @@ static void show_one(const struct show_one_options *opts, const char *hex; struct object_id peeled; - if (!repo_has_object_file(the_repository, oid)) + if (!has_object(the_repository, oid, + HAS_OBJECT_RECHECK_PACKED | HAS_OBJECT_FETCH_PROMISOR)) die("git show-ref: bad ref %s (%s)", refname, oid_to_hex(oid)); diff --git a/builtin/unpack-objects.c b/builtin/unpack-objects.c index 661be789f1..e905d5f4e1 100644 --- a/builtin/unpack-objects.c +++ b/builtin/unpack-objects.c @@ -449,7 +449,8 @@ static void unpack_delta_entry(enum object_type type, unsigned long delta_size, delta_data = get_data(delta_size); if (!delta_data) return; - if (repo_has_object_file(the_repository, &base_oid)) + if (has_object(the_repository, &base_oid, + HAS_OBJECT_RECHECK_PACKED | HAS_OBJECT_FETCH_PROMISOR)) ; /* Ok we have this one */ else if (resolve_against_held(nr, &base_oid, delta_data, delta_size)) diff --git a/bulk-checkin.c b/bulk-checkin.c index c31c31b18d..678e2ecc2c 100644 --- a/bulk-checkin.c +++ b/bulk-checkin.c @@ -130,7 +130,8 @@ static void flush_batch_fsync(void) static int already_written(struct bulk_checkin_packfile *state, struct object_id *oid) { /* The object may already exist in the repository */ - if (repo_has_object_file(the_repository, oid)) + if (has_object(the_repository, oid, + HAS_OBJECT_RECHECK_PACKED | HAS_OBJECT_FETCH_PROMISOR)) return 1; /* Might want to keep the list sorted */ diff --git a/cache-tree.c b/cache-tree.c index c0e1e9ee1d..fa3858e282 100644 --- a/cache-tree.c +++ b/cache-tree.c @@ -238,7 +238,9 @@ int cache_tree_fully_valid(struct cache_tree *it) int i; if (!it) return 0; - if (it->entry_count < 0 || !repo_has_object_file(the_repository, &it->oid)) + if (it->entry_count < 0 || + has_object(the_repository, &it->oid, + HAS_OBJECT_RECHECK_PACKED | HAS_OBJECT_FETCH_PROMISOR)) return 0; for (i = 0; i < it->subtree_nr; i++) { if (!cache_tree_fully_valid(it->down[i]->cache_tree)) @@ -289,7 +291,9 @@ static int update_one(struct cache_tree *it, } } - if (0 <= it->entry_count && repo_has_object_file(the_repository, &it->oid)) + if (0 <= it->entry_count && + has_object(the_repository, &it->oid, + HAS_OBJECT_RECHECK_PACKED | HAS_OBJECT_FETCH_PROMISOR)) return it->entry_count; /* @@ -395,7 +399,8 @@ static int update_one(struct cache_tree *it, ce_missing_ok = mode == S_IFGITLINK || missing_ok || !must_check_existence(ce); if (is_null_oid(oid) || - (!ce_missing_ok && !repo_has_object_file(the_repository, oid))) { + (!ce_missing_ok && !has_object(the_repository, oid, + HAS_OBJECT_RECHECK_PACKED | HAS_OBJECT_FETCH_PROMISOR))) { strbuf_release(&buffer); if (expected_missing) return -1; @@ -443,7 +448,7 @@ static int update_one(struct cache_tree *it, struct object_id oid; hash_object_file(the_hash_algo, buffer.buf, buffer.len, OBJ_TREE, &oid); - if (repo_has_object_file_with_flags(the_repository, &oid, OBJECT_INFO_SKIP_FETCH_OBJECT)) + if (has_object(the_repository, &oid, HAS_OBJECT_RECHECK_PACKED)) oidcpy(&it->oid, &oid); else to_invalidate = 1; diff --git a/convert.c b/convert.c index 8783e17941..b5f7cf6306 100644 --- a/convert.c +++ b/convert.c @@ -8,7 +8,7 @@ #include "copy.h" #include "gettext.h" #include "hex.h" -#include "object-store.h" +#include "object-file.h" #include "attr.h" #include "run-command.h" #include "quote.h" diff --git a/diffcore-rename.c b/diffcore-rename.c index 179731462b..7723bc3334 100644 --- a/diffcore-rename.c +++ b/diffcore-rename.c @@ -8,7 +8,7 @@ #include "git-compat-util.h" #include "diff.h" #include "diffcore.h" -#include "object-store.h" +#include "object-file.h" #include "hashmap.h" #include "mem-pool.h" #include "oid-array.h" diff --git a/dir.c b/dir.c index 5c4675b4ac..e11342e136 100644 --- a/dir.c +++ b/dir.c @@ -17,7 +17,7 @@ #include "environment.h" #include "gettext.h" #include "name-hash.h" -#include "object-store.h" +#include "object-file.h" #include "path.h" #include "refs.h" #include "repository.h" diff --git a/fetch-pack.c b/fetch-pack.c index 210dc30d50..fa4231fee7 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -769,9 +769,7 @@ static void mark_complete_and_common_ref(struct fetch_negotiator *negotiator, if (!commit) { struct object *o; - if (!repo_has_object_file_with_flags(the_repository, &ref->old_oid, - OBJECT_INFO_QUICK | - OBJECT_INFO_SKIP_FETCH_OBJECT)) + if (!has_object(the_repository, &ref->old_oid, 0)) continue; o = parse_object(the_repository, &ref->old_oid); if (!o || o->type != OBJ_COMMIT) @@ -1985,7 +1983,8 @@ static void update_shallow(struct fetch_pack_args *args, struct oid_array extra = OID_ARRAY_INIT; struct object_id *oid = si->shallow->oid; for (i = 0; i < si->shallow->nr; i++) - if (repo_has_object_file(the_repository, &oid[i])) + if (has_object(the_repository, &oid[i], + HAS_OBJECT_RECHECK_PACKED | HAS_OBJECT_FETCH_PROMISOR)) oid_array_append(&extra, &oid[i]); if (extra.nr) { setup_alternate_shallow(&shallow_lock, diff --git a/http-push.c b/http-push.c index 32e37565f4..f9e67cabd4 100644 --- a/http-push.c +++ b/http-push.c @@ -1446,7 +1446,9 @@ static void one_remote_ref(const char *refname) * Fetch a copy of the object if it doesn't exist locally - it * may be required for updating server info later. */ - if (repo->can_update_info_refs && !repo_has_object_file(the_repository, &ref->old_oid)) { + if (repo->can_update_info_refs && + !has_object(the_repository, &ref->old_oid, + HAS_OBJECT_RECHECK_PACKED | HAS_OBJECT_FETCH_PROMISOR)) { obj = lookup_unknown_object(the_repository, &ref->old_oid); fprintf(stderr, " fetch %s for %s\n", oid_to_hex(&ref->old_oid), refname); @@ -1651,14 +1653,14 @@ static int delete_remote_branch(const char *pattern, int force) return error("Remote HEAD symrefs too deep"); if (is_null_oid(&head_oid)) return error("Unable to resolve remote HEAD"); - if (!repo_has_object_file(the_repository, &head_oid)) + if (!has_object(the_repository, &head_oid, HAS_OBJECT_RECHECK_PACKED | HAS_OBJECT_FETCH_PROMISOR)) return error("Remote HEAD resolves to object %s\nwhich does not exist locally, perhaps you need to fetch?", oid_to_hex(&head_oid)); /* Remote branch must resolve to a known object */ if (is_null_oid(&remote_ref->old_oid)) return error("Unable to resolve remote branch %s", remote_ref->name); - if (!repo_has_object_file(the_repository, &remote_ref->old_oid)) + if (!has_object(the_repository, &remote_ref->old_oid, HAS_OBJECT_RECHECK_PACKED | HAS_OBJECT_FETCH_PROMISOR)) return error("Remote branch %s resolves to object %s\nwhich does not exist locally, perhaps you need to fetch?", remote_ref->name, oid_to_hex(&remote_ref->old_oid)); /* Remote branch must be an ancestor of remote HEAD */ @@ -1879,7 +1881,8 @@ int cmd_main(int argc, const char **argv) if (!force_all && !is_null_oid(&ref->old_oid) && !ref->force) { - if (!repo_has_object_file(the_repository, &ref->old_oid) || + if (!has_object(the_repository, &ref->old_oid, + HAS_OBJECT_RECHECK_PACKED | HAS_OBJECT_FETCH_PROMISOR) || !ref_newer(&ref->peer_ref->new_oid, &ref->old_oid)) { /* diff --git a/http-walker.c b/http-walker.c index 882cae19c2..463f7b119a 100644 --- a/http-walker.c +++ b/http-walker.c @@ -9,6 +9,7 @@ #include "list.h" #include "transport.h" #include "packfile.h" +#include "object-file.h" #include "object-store.h" struct alt_base { @@ -137,7 +138,8 @@ static int fill_active_slot(void *data UNUSED) list_for_each_safe(pos, tmp, head) { obj_req = list_entry(pos, struct object_request, node); if (obj_req->state == WAITING) { - if (repo_has_object_file(the_repository, &obj_req->oid)) + if (has_object(the_repository, &obj_req->oid, + HAS_OBJECT_RECHECK_PACKED | HAS_OBJECT_FETCH_PROMISOR)) obj_req->state = COMPLETE; else { start_object_request(obj_req); @@ -495,7 +497,8 @@ static int fetch_object(struct walker *walker, const struct object_id *oid) if (!obj_req) return error("Couldn't find request for %s in the queue", hex); - if (repo_has_object_file(the_repository, &obj_req->oid)) { + if (has_object(the_repository, &obj_req->oid, + HAS_OBJECT_RECHECK_PACKED | HAS_OBJECT_FETCH_PROMISOR)) { if (obj_req->req) abort_http_object_request(&obj_req->req); abort_object_request(obj_req); @@ -540,7 +543,7 @@ static int fetch_object(struct walker *walker, const struct object_id *oid) ret = error("File %s has bad hash", hex); } else if (req->rename < 0) { struct strbuf buf = STRBUF_INIT; - loose_object_path(the_repository, &buf, &req->oid); + odb_loose_path(the_repository->objects->odb, &buf, &req->oid); ret = error("unable to write sha1 filename %s", buf.buf); strbuf_release(&buf); } diff --git a/http.c b/http.c index 0c41138042..3c029cf894 100644 --- a/http.c +++ b/http.c @@ -2662,7 +2662,7 @@ struct http_object_request *new_http_object_request(const char *base_url, oidcpy(&freq->oid, oid); freq->localfile = -1; - loose_object_path(the_repository, &filename, oid); + odb_loose_path(the_repository->objects->odb, &filename, oid); strbuf_addf(&freq->tmpfile, "%s.temp", filename.buf); strbuf_addf(&prevfile, "%s.prev", filename.buf); @@ -2814,7 +2814,7 @@ int finish_http_object_request(struct http_object_request *freq) unlink_or_warn(freq->tmpfile.buf); return -1; } - loose_object_path(the_repository, &filename, &freq->oid); + odb_loose_path(the_repository->objects->odb, &filename, &freq->oid); freq->rename = finalize_object_file(freq->tmpfile.buf, filename.buf); strbuf_release(&filename); diff --git a/list-objects.c b/list-objects.c index 1e5512e131..597114281f 100644 --- a/list-objects.c +++ b/list-objects.c @@ -74,7 +74,8 @@ static void process_blob(struct traversal_context *ctx, * of missing objects. */ if (ctx->revs->exclude_promisor_objects && - !repo_has_object_file(the_repository, &obj->oid) && + !has_object(the_repository, &obj->oid, + HAS_OBJECT_RECHECK_PACKED | HAS_OBJECT_FETCH_PROMISOR) && is_promisor_object(ctx->revs->repo, &obj->oid)) return; diff --git a/log-tree.c b/log-tree.c index a4d4ab59ca..1d05dc1c70 100644 --- a/log-tree.c +++ b/log-tree.c @@ -9,7 +9,7 @@ #include "environment.h" #include "hex.h" #include "object-name.h" -#include "object-store.h" +#include "object-file.h" #include "repository.h" #include "tmp-objdir.h" #include "commit.h" diff --git a/notes.c b/notes.c index d9645c4b5d..0a128f1de9 100644 --- a/notes.c +++ b/notes.c @@ -794,7 +794,8 @@ static int prune_notes_helper(const struct object_id *object_oid, struct note_delete_list **l = (struct note_delete_list **) cb_data; struct note_delete_list *n; - if (repo_has_object_file(the_repository, object_oid)) + if (has_object(the_repository, object_oid, + HAS_OBJECT_RECHECK_PACKED | HAS_OBJECT_FETCH_PROMISOR)) return 0; /* nothing to do for this note */ /* failed to find object => prune this note */ diff --git a/object-file.c b/object-file.c index 9cc3a24a40..dc56a4766d 100644 --- a/object-file.c +++ b/object-file.c @@ -932,7 +932,7 @@ static int write_loose_object(const struct object_id *oid, char *hdr, if (batch_fsync_enabled(FSYNC_COMPONENT_LOOSE_OBJECT)) prepare_loose_object_bulk_checkin(); - loose_object_path(the_repository, &filename, oid); + odb_loose_path(the_repository->objects->odb, &filename, oid); fd = start_loose_object_common(&tmp_file, filename.buf, flags, &stream, compressed, sizeof(compressed), @@ -1079,7 +1079,7 @@ int stream_loose_object(struct input_stream *in_stream, size_t len, goto cleanup; } - loose_object_path(the_repository, &filename, oid); + odb_loose_path(the_repository->objects->odb, &filename, oid); /* We finally know the object path, and create the missing dir. */ dirlen = directory_size(filename.buf); diff --git a/object-file.h b/object-file.h index c002fbe234..a85b2e5b49 100644 --- a/object-file.h +++ b/object-file.h @@ -3,6 +3,7 @@ #include "git-zlib.h" #include "object.h" +#include "object-store.h" struct index_state; @@ -25,6 +26,20 @@ int index_path(struct index_state *istate, struct object_id *oid, const char *pa struct object_directory; +/* + * Populate and return the loose object cache array corresponding to the + * given object ID. + */ +struct oidtree *odb_loose_cache(struct object_directory *odb, + const struct object_id *oid); + +/* Empty the loose object cache for the specified object directory. */ +void odb_clear_loose_cache(struct object_directory *odb); + +/* + * Put in `buf` the name of the file in the local object database that + * would be used to store a loose object with the specified oid. + */ const char *odb_loose_path(struct object_directory *odb, struct strbuf *buf, const struct object_id *oid); @@ -38,6 +53,68 @@ int has_loose_object_nonlocal(const struct object_id *); int has_loose_object(const struct object_id *); +void *map_loose_object(struct repository *r, const struct object_id *oid, + unsigned long *size); + +/* + * Iterate over the files in the loose-object parts of the object + * directory "path", triggering the following callbacks: + * + * - loose_object is called for each loose object we find. + * + * - loose_cruft is called for any files that do not appear to be + * loose objects. Note that we only look in the loose object + * directories "objects/[0-9a-f]{2}/", so we will not report + * "objects/foobar" as cruft. + * + * - loose_subdir is called for each top-level hashed subdirectory + * of the object directory (e.g., "$OBJDIR/f0"). It is called + * after the objects in the directory are processed. + * + * Any callback that is NULL will be ignored. Callbacks returning non-zero + * will end the iteration. + * + * In the "buf" variant, "path" is a strbuf which will also be used as a + * scratch buffer, but restored to its original contents before + * the function returns. + */ +typedef int each_loose_object_fn(const struct object_id *oid, + const char *path, + void *data); +typedef int each_loose_cruft_fn(const char *basename, + const char *path, + void *data); +typedef int each_loose_subdir_fn(unsigned int nr, + const char *path, + void *data); +int for_each_file_in_obj_subdir(unsigned int subdir_nr, + struct strbuf *path, + each_loose_object_fn obj_cb, + each_loose_cruft_fn cruft_cb, + each_loose_subdir_fn subdir_cb, + void *data); +int for_each_loose_file_in_objdir(const char *path, + each_loose_object_fn obj_cb, + each_loose_cruft_fn cruft_cb, + each_loose_subdir_fn subdir_cb, + void *data); +int for_each_loose_file_in_objdir_buf(struct strbuf *path, + each_loose_object_fn obj_cb, + each_loose_cruft_fn cruft_cb, + each_loose_subdir_fn subdir_cb, + void *data); + +/* + * Iterate over all accessible loose objects without respect to + * reachability. By default, this includes both local and alternate objects. + * The order in which objects are visited is unspecified. + * + * Any flags specific to packs are ignored. + */ +int for_each_loose_object(each_loose_object_fn, void *, + enum for_each_object_flags flags); + + /** * format_object_header() is a thin wrapper around s xsnprintf() that * writes the initial " " part of the loose object @@ -154,6 +231,10 @@ int finalize_object_file(const char *tmpfile, const char *filename); int finalize_object_file_flags(const char *tmpfile, const char *filename, enum finalize_object_file_flags flags); +void hash_object_file(const struct git_hash_algo *algo, const void *buf, + unsigned long len, enum object_type type, + struct object_id *oid); + /* Helper to check and "touch" a file */ int check_and_freshen_file(const char *fn, int freshen); diff --git a/object-name.c b/object-name.c index 2c751a5352..9288b2dd24 100644 --- a/object-name.c +++ b/object-name.c @@ -19,7 +19,7 @@ #include "oidtree.h" #include "packfile.h" #include "pretty.h" -#include "object-store.h" +#include "object-file.h" #include "read-cache-ll.h" #include "repo-settings.h" #include "repository.h" diff --git a/object-store.c b/object-store.c index 6ab50d25d3..2f51d0e3b0 100644 --- a/object-store.c +++ b/object-store.c @@ -83,25 +83,6 @@ int odb_mkstemp(struct strbuf *temp_filename, const char *pattern) return xmkstemp_mode(temp_filename->buf, mode); } -int odb_pack_keep(const char *name) -{ - int fd; - - fd = open(name, O_RDWR|O_CREAT|O_EXCL, 0600); - if (0 <= fd) - return fd; - - /* slow path */ - safe_create_leading_directories_const(the_repository, name); - return open(name, O_RDWR|O_CREAT|O_EXCL, 0600); -} - -const char *loose_object_path(struct repository *r, struct strbuf *buf, - const struct object_id *oid) -{ - return odb_loose_path(r->objects->odb, buf, oid); -} - /* * Return non-zero iff the path is usable as an alternate object database. */ @@ -866,7 +847,7 @@ int pretend_object_file(struct repository *repo, char *co_buf; hash_object_file(repo->hash_algo, buf, len, type, oid); - if (repo_has_object_file_with_flags(repo, oid, OBJECT_INFO_QUICK | OBJECT_INFO_SKIP_FETCH_OBJECT) || + if (has_object(repo, oid, 0) || find_cached_object(repo->objects, oid)) return 0; @@ -956,29 +937,18 @@ void *read_object_with_reference(struct repository *r, int has_object(struct repository *r, const struct object_id *oid, unsigned flags) { - int quick = !(flags & HAS_OBJECT_RECHECK_PACKED); - unsigned object_info_flags = OBJECT_INFO_SKIP_FETCH_OBJECT | - (quick ? OBJECT_INFO_QUICK : 0); + unsigned object_info_flags = 0; if (!startup_info->have_repository) return 0; + if (!(flags & HAS_OBJECT_RECHECK_PACKED)) + object_info_flags |= OBJECT_INFO_QUICK; + if (!(flags & HAS_OBJECT_FETCH_PROMISOR)) + object_info_flags |= OBJECT_INFO_SKIP_FETCH_OBJECT; + return oid_object_info_extended(r, oid, NULL, object_info_flags) >= 0; } -int repo_has_object_file_with_flags(struct repository *r, - const struct object_id *oid, int flags) -{ - if (!startup_info->have_repository) - return 0; - return oid_object_info_extended(r, oid, NULL, flags) >= 0; -} - -int repo_has_object_file(struct repository *r, - const struct object_id *oid) -{ - return repo_has_object_file_with_flags(r, oid, 0); -} - void assert_oid_type(const struct object_id *oid, enum object_type expect) { enum object_type type = oid_object_info(the_repository, oid, NULL); diff --git a/object-store.h b/object-store.h index 46961dc954..c2fe5a1960 100644 --- a/object-store.h +++ b/object-store.h @@ -82,75 +82,8 @@ struct object_directory *set_temporary_primary_odb(const char *dir, int will_des */ void restore_primary_odb(struct object_directory *restore_odb, const char *old_path); -/* - * Populate and return the loose object cache array corresponding to the - * given object ID. - */ -struct oidtree *odb_loose_cache(struct object_directory *odb, - const struct object_id *oid); - -/* Empty the loose object cache for the specified object directory. */ -void odb_clear_loose_cache(struct object_directory *odb); - -struct packed_git { - struct hashmap_entry packmap_ent; - struct packed_git *next; - struct list_head mru; - struct pack_window *windows; - off_t pack_size; - const void *index_data; - size_t index_size; - uint32_t num_objects; - size_t crc_offset; - struct oidset bad_objects; - int index_version; - time_t mtime; - int pack_fd; - int index; /* for builtin/pack-objects.c */ - unsigned pack_local:1, - pack_keep:1, - pack_keep_in_core:1, - freshened:1, - do_not_close:1, - pack_promisor:1, - multi_pack_index:1, - is_cruft:1; - unsigned char hash[GIT_MAX_RAWSZ]; - struct revindex_entry *revindex; - const uint32_t *revindex_data; - const uint32_t *revindex_map; - size_t revindex_size; - /* - * mtimes_map points at the beginning of the memory mapped region of - * this pack's corresponding .mtimes file, and mtimes_size is the size - * of that .mtimes file - */ - const uint32_t *mtimes_map; - size_t mtimes_size; - - /* repo denotes the repository this packfile belongs to */ - struct repository *repo; - - /* something like ".git/objects/pack/xxxxx.pack" */ - char pack_name[FLEX_ARRAY]; /* more */ -}; - +struct packed_git; struct multi_pack_index; - -static inline int pack_map_entry_cmp(const void *cmp_data UNUSED, - const struct hashmap_entry *entry, - const struct hashmap_entry *entry2, - const void *keydata) -{ - const char *key = keydata; - const struct packed_git *pg1, *pg2; - - pg1 = container_of(entry, const struct packed_git, packmap_ent); - pg2 = container_of(entry2, const struct packed_git, packmap_ent); - - return strcmp(pg1->pack_name, key ? key : pg2->pack_name); -} - struct cached_object_entry; struct raw_object_store { @@ -246,23 +179,6 @@ void raw_object_store_clear(struct raw_object_store *o); */ int odb_mkstemp(struct strbuf *temp_filename, const char *pattern); -/* - * Create a pack .keep file named "name" (which should generally be the output - * of odb_pack_name). Returns a file descriptor opened for writing, or -1 on - * error. - */ -int odb_pack_keep(const char *name); - -/* - * Put in `buf` the name of the file in the local object database that - * would be used to store a loose object with the specified oid. - */ -const char *loose_object_path(struct repository *r, struct strbuf *buf, - const struct object_id *oid); - -void *map_loose_object(struct repository *r, const struct object_id *oid, - unsigned long *size); - void *repo_read_object_file(struct repository *r, const struct object_id *oid, enum object_type *type, @@ -271,10 +187,6 @@ void *repo_read_object_file(struct repository *r, /* Read and unpack an object file into memory, write memory to an object file */ int oid_object_info(struct repository *r, const struct object_id *, unsigned long *); -void hash_object_file(const struct git_hash_algo *algo, const void *buf, - unsigned long len, enum object_type type, - struct object_id *oid); - /* * Add an object file to the in-memory object store, without writing it * to disk. @@ -350,33 +262,20 @@ int oid_object_info_extended(struct repository *r, const struct object_id *, struct object_info *, unsigned flags); -/* Retry packed storage after checking packed and loose storage */ -#define HAS_OBJECT_RECHECK_PACKED 1 +enum { + /* Retry packed storage after checking packed and loose storage */ + HAS_OBJECT_RECHECK_PACKED = (1 << 0), + /* Allow fetching the object in case the repository has a promisor remote. */ + HAS_OBJECT_FETCH_PROMISOR = (1 << 1), +}; /* * Returns 1 if the object exists. This function will not lazily fetch objects - * in a partial clone. + * in a partial clone by default. */ int has_object(struct repository *r, const struct object_id *oid, unsigned flags); -/* - * These macros and functions are deprecated. If checking existence for an - * object that is likely to be missing and/or whose absence is relatively - * inconsequential (or is consequential but the caller is prepared to handle - * it), use has_object(), which has better defaults (no lazy fetch in a partial - * clone and no rechecking of packed storage). In the unlikely event that a - * caller needs to assert existence of an object that it fully expects to - * exist, and wants to trigger a lazy fetch in a partial clone, use - * oid_object_info_extended() with a NULL struct object_info. - * - * These functions can be removed once all callers have migrated to - * has_object() and/or oid_object_info_extended(). - */ -int repo_has_object_file(struct repository *r, const struct object_id *oid); -int repo_has_object_file_with_flags(struct repository *r, - const struct object_id *oid, int flags); - void assert_oid_type(const struct object_id *oid, enum object_type expect); /* @@ -411,56 +310,7 @@ static inline void obj_read_unlock(void) if(obj_read_use_lock) pthread_mutex_unlock(&obj_read_mutex); } - -/* - * Iterate over the files in the loose-object parts of the object - * directory "path", triggering the following callbacks: - * - * - loose_object is called for each loose object we find. - * - * - loose_cruft is called for any files that do not appear to be - * loose objects. Note that we only look in the loose object - * directories "objects/[0-9a-f]{2}/", so we will not report - * "objects/foobar" as cruft. - * - * - loose_subdir is called for each top-level hashed subdirectory - * of the object directory (e.g., "$OBJDIR/f0"). It is called - * after the objects in the directory are processed. - * - * Any callback that is NULL will be ignored. Callbacks returning non-zero - * will end the iteration. - * - * In the "buf" variant, "path" is a strbuf which will also be used as a - * scratch buffer, but restored to its original contents before - * the function returns. - */ -typedef int each_loose_object_fn(const struct object_id *oid, - const char *path, - void *data); -typedef int each_loose_cruft_fn(const char *basename, - const char *path, - void *data); -typedef int each_loose_subdir_fn(unsigned int nr, - const char *path, - void *data); -int for_each_file_in_obj_subdir(unsigned int subdir_nr, - struct strbuf *path, - each_loose_object_fn obj_cb, - each_loose_cruft_fn cruft_cb, - each_loose_subdir_fn subdir_cb, - void *data); -int for_each_loose_file_in_objdir(const char *path, - each_loose_object_fn obj_cb, - each_loose_cruft_fn cruft_cb, - each_loose_subdir_fn subdir_cb, - void *data); -int for_each_loose_file_in_objdir_buf(struct strbuf *path, - each_loose_object_fn obj_cb, - each_loose_cruft_fn cruft_cb, - each_loose_subdir_fn subdir_cb, - void *data); - -/* Flags for for_each_*_object() below. */ +/* Flags for for_each_*_object(). */ enum for_each_object_flags { /* Iterate only over local objects, not alternates. */ FOR_EACH_OBJECT_LOCAL_ONLY = (1<<0), @@ -480,33 +330,6 @@ enum for_each_object_flags { FOR_EACH_OBJECT_SKIP_ON_DISK_KEPT_PACKS = (1<<4), }; -/* - * Iterate over all accessible loose objects without respect to - * reachability. By default, this includes both local and alternate objects. - * The order in which objects are visited is unspecified. - * - * Any flags specific to packs are ignored. - */ -int for_each_loose_object(each_loose_object_fn, void *, - enum for_each_object_flags flags); - -/* - * Iterate over all accessible packed objects without respect to reachability. - * By default, this includes both local and alternate packs. - * - * Note that some objects may appear twice if they are found in multiple packs. - * Each pack is visited in an unspecified order. By default, objects within a - * pack are visited in pack-idx order (i.e., sorted by oid). - */ -typedef int each_packed_object_fn(const struct object_id *oid, - struct packed_git *pack, - uint32_t pos, - void *data); -int for_each_object_in_pack(struct packed_git *p, - each_packed_object_fn, void *data, - enum for_each_object_flags flags); -int for_each_packed_object(struct repository *repo, each_packed_object_fn cb, - void *data, enum for_each_object_flags flags); void *read_object_with_reference(struct repository *r, const struct object_id *oid, diff --git a/pack-objects.h b/pack-objects.h index d1c4ae7f9b..475a2d67ce 100644 --- a/pack-objects.h +++ b/pack-objects.h @@ -4,6 +4,7 @@ #include "object-store.h" #include "thread-utils.h" #include "pack.h" +#include "packfile.h" struct repository; diff --git a/packfile.h b/packfile.h index 25097213d0..3a3c77cf05 100644 --- a/packfile.h +++ b/packfile.h @@ -1,13 +1,71 @@ #ifndef PACKFILE_H #define PACKFILE_H +#include "list.h" #include "object.h" +#include "object-store.h" #include "oidset.h" /* in object-store.h */ -struct packed_git; struct object_info; +struct packed_git { + struct hashmap_entry packmap_ent; + struct packed_git *next; + struct list_head mru; + struct pack_window *windows; + off_t pack_size; + const void *index_data; + size_t index_size; + uint32_t num_objects; + size_t crc_offset; + struct oidset bad_objects; + int index_version; + time_t mtime; + int pack_fd; + int index; /* for builtin/pack-objects.c */ + unsigned pack_local:1, + pack_keep:1, + pack_keep_in_core:1, + freshened:1, + do_not_close:1, + pack_promisor:1, + multi_pack_index:1, + is_cruft:1; + unsigned char hash[GIT_MAX_RAWSZ]; + struct revindex_entry *revindex; + const uint32_t *revindex_data; + const uint32_t *revindex_map; + size_t revindex_size; + /* + * mtimes_map points at the beginning of the memory mapped region of + * this pack's corresponding .mtimes file, and mtimes_size is the size + * of that .mtimes file + */ + const uint32_t *mtimes_map; + size_t mtimes_size; + + /* repo denotes the repository this packfile belongs to */ + struct repository *repo; + + /* something like ".git/objects/pack/xxxxx.pack" */ + char pack_name[FLEX_ARRAY]; /* more */ +}; + +static inline int pack_map_entry_cmp(const void *cmp_data UNUSED, + const struct hashmap_entry *entry, + const struct hashmap_entry *entry2, + const void *keydata) +{ + const char *key = keydata; + const struct packed_git *pg1, *pg2; + + pg1 = container_of(entry, const struct packed_git, packmap_ent); + pg2 = container_of(entry2, const struct packed_git, packmap_ent); + + return strcmp(pg1->pack_name, key ? key : pg2->pack_name); +} + struct pack_window { struct pack_window *next; unsigned char *base; @@ -60,6 +118,24 @@ void for_each_file_in_pack_dir(const char *objdir, each_file_in_pack_dir_fn fn, void *data); +/* + * Iterate over all accessible packed objects without respect to reachability. + * By default, this includes both local and alternate packs. + * + * Note that some objects may appear twice if they are found in multiple packs. + * Each pack is visited in an unspecified order. By default, objects within a + * pack are visited in pack-idx order (i.e., sorted by oid). + */ +typedef int each_packed_object_fn(const struct object_id *oid, + struct packed_git *pack, + uint32_t pos, + void *data); +int for_each_object_in_pack(struct packed_git *p, + each_packed_object_fn, void *data, + enum for_each_object_flags flags); +int for_each_packed_object(struct repository *repo, each_packed_object_fn cb, + void *data, enum for_each_object_flags flags); + /* A hook to report invalid files in pack directory */ #define PACKDIR_FILE_PACK 1 #define PACKDIR_FILE_IDX 2 diff --git a/path.c b/path.c index 4505bb78e8..3b598b2847 100644 --- a/path.c +++ b/path.c @@ -1011,6 +1011,20 @@ enum scld_error safe_create_leading_directories_const(struct repository *repo, return result; } +int safe_create_file_with_leading_directories(struct repository *repo, + const char *path) +{ + int fd; + + fd = open(path, O_RDWR|O_CREAT|O_EXCL, 0600); + if (0 <= fd) + return fd; + + /* slow path */ + safe_create_leading_directories_const(repo, path); + return open(path, O_RDWR|O_CREAT|O_EXCL, 0600); +} + static int have_same_root(const char *path1, const char *path2) { int is_abs1, is_abs2; diff --git a/path.h b/path.h index fd1a194b06..e67348f253 100644 --- a/path.h +++ b/path.h @@ -266,6 +266,13 @@ enum scld_error safe_create_leading_directories_const(struct repository *repo, const char *path); enum scld_error safe_create_leading_directories_no_share(char *path); +/* + * Create a file, potentially creating its leading directories in case they + * don't exist. Returns the return value of the open(3p) call. + */ +int safe_create_file_with_leading_directories(struct repository *repo, + const char *path); + # ifdef USE_THE_REPOSITORY_VARIABLE # include "strbuf.h" # include "repository.h" diff --git a/prune-packed.c b/prune-packed.c index c1d95a519d..92fb4fbb0e 100644 --- a/prune-packed.c +++ b/prune-packed.c @@ -2,7 +2,7 @@ #include "git-compat-util.h" #include "gettext.h" -#include "object-store.h" +#include "object-file.h" #include "packfile.h" #include "progress.h" #include "prune-packed.h" diff --git a/reachable.c b/reachable.c index e5f56f4018..9dc748f0b9 100644 --- a/reachable.c +++ b/reachable.c @@ -14,7 +14,7 @@ #include "list-objects.h" #include "packfile.h" #include "worktree.h" -#include "object-store.h" +#include "object-file.h" #include "pack-bitmap.h" #include "pack-mtimes.h" #include "config.h" diff --git a/reflog.c b/reflog.c index 12f7a02e34..15d81ebea9 100644 --- a/reflog.c +++ b/reflog.c @@ -152,7 +152,8 @@ static int tree_is_complete(const struct object_id *oid) init_tree_desc(&desc, &tree->object.oid, tree->buffer, tree->size); complete = 1; while (tree_entry(&desc, &entry)) { - if (!repo_has_object_file(the_repository, &entry.oid) || + if (!has_object(the_repository, &entry.oid, + HAS_OBJECT_RECHECK_PACKED | HAS_OBJECT_FETCH_PROMISOR) || (S_ISDIR(entry.mode) && !tree_is_complete(&entry.oid))) { tree->object.flags |= INCOMPLETE; complete = 0; diff --git a/refs.c b/refs.c index 6559db3789..dce5c49ca2 100644 --- a/refs.c +++ b/refs.c @@ -376,7 +376,7 @@ int ref_resolves_to_object(const char *refname, { if (flags & REF_ISBROKEN) return 0; - if (!repo_has_object_file(repo, oid)) { + if (!has_object(repo, oid, HAS_OBJECT_RECHECK_PACKED | HAS_OBJECT_FETCH_PROMISOR)) { error(_("%s does not point to a valid object!"), refname); return 0; } diff --git a/remote.c b/remote.c index 9fa3614e7a..4099183cac 100644 --- a/remote.c +++ b/remote.c @@ -1702,7 +1702,7 @@ void set_ref_status_for_push(struct ref *remote_refs, int send_mirror, if (!reject_reason && !ref->deletion && !is_null_oid(&ref->old_oid)) { if (starts_with(ref->name, "refs/tags/")) reject_reason = REF_STATUS_REJECT_ALREADY_EXISTS; - else if (!repo_has_object_file_with_flags(the_repository, &ref->old_oid, OBJECT_INFO_SKIP_FETCH_OBJECT)) + else if (!has_object(the_repository, &ref->old_oid, HAS_OBJECT_RECHECK_PACKED)) reject_reason = REF_STATUS_REJECT_FETCH_FIRST; else if (!lookup_commit_reference_gently(the_repository, &ref->old_oid, 1) || !lookup_commit_reference_gently(the_repository, &ref->new_oid, 1)) diff --git a/send-pack.c b/send-pack.c index 5005689cb5..86592ce526 100644 --- a/send-pack.c +++ b/send-pack.c @@ -45,10 +45,7 @@ int option_parse_push_signed(const struct option *opt, static void feed_object(struct repository *r, const struct object_id *oid, FILE *fh, int negative) { - if (negative && - !repo_has_object_file_with_flags(r, oid, - OBJECT_INFO_SKIP_FETCH_OBJECT | - OBJECT_INFO_QUICK)) + if (negative && !has_object(r, oid, 0)) return; if (negative) diff --git a/shallow.c b/shallow.c index 2f82ebd6e3..faeeeb45f9 100644 --- a/shallow.c +++ b/shallow.c @@ -310,7 +310,8 @@ static int write_one_shallow(const struct commit_graft *graft, void *cb_data) if (graft->nr_parent != -1) return 0; if (data->flags & QUICK) { - if (!repo_has_object_file(the_repository, &graft->oid)) + if (!has_object(the_repository, &graft->oid, + HAS_OBJECT_RECHECK_PACKED | HAS_OBJECT_FETCH_PROMISOR)) return 0; } else if (data->flags & SEEN_ONLY) { struct commit *c = lookup_commit(the_repository, &graft->oid); @@ -476,7 +477,8 @@ void prepare_shallow_info(struct shallow_info *info, struct oid_array *sa) ALLOC_ARRAY(info->ours, sa->nr); ALLOC_ARRAY(info->theirs, sa->nr); for (size_t i = 0; i < sa->nr; i++) { - if (repo_has_object_file(the_repository, sa->oid + i)) { + if (has_object(the_repository, sa->oid + i, + HAS_OBJECT_RECHECK_PACKED | HAS_OBJECT_FETCH_PROMISOR)) { struct commit_graft *graft; graft = lookup_commit_graft(the_repository, &sa->oid[i]); @@ -513,7 +515,8 @@ void remove_nonexistent_theirs_shallow(struct shallow_info *info) for (i = dst = 0; i < info->nr_theirs; i++) { if (i != dst) info->theirs[dst] = info->theirs[i]; - if (repo_has_object_file(the_repository, oid + info->theirs[i])) + if (has_object(the_repository, oid + info->theirs[i], + HAS_OBJECT_RECHECK_PACKED | HAS_OBJECT_FETCH_PROMISOR)) dst++; } info->nr_theirs = dst; diff --git a/upload-pack.c b/upload-pack.c index 30e4630f3a..956da5b061 100644 --- a/upload-pack.c +++ b/upload-pack.c @@ -509,8 +509,7 @@ static int got_oid(struct upload_pack_data *data, { if (get_oid_hex(hex, oid)) die("git upload-pack: expected SHA1 object, got '%s'", hex); - if (!repo_has_object_file_with_flags(the_repository, oid, - OBJECT_INFO_QUICK | OBJECT_INFO_SKIP_FETCH_OBJECT)) + if (!has_object(the_repository, oid, 0)) return -1; return do_got_oid(data, oid); } diff --git a/walker.c b/walker.c index 4fedc19f34..b470d43e54 100644 --- a/walker.c +++ b/walker.c @@ -150,7 +150,8 @@ static int process(struct walker *walker, struct object *obj) return 0; obj->flags |= SEEN; - if (repo_has_object_file(the_repository, &obj->oid)) { + if (has_object(the_repository, &obj->oid, + HAS_OBJECT_RECHECK_PACKED | HAS_OBJECT_FETCH_PROMISOR)) { /* We already have it, so we should scan it now. */ obj->flags |= TO_SCAN; }