global/git
1
0
Fork 0
mirror of https://github.com/git/git.git synced 2026-01-10 10:13:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

pack-redundant: new algorithm to find min packs

Browse Source 
When calling `git pack-redundant --all`, if there are too many local
packs and too many redundant objects within them, the too deep iteration
of `get_permutations` will exhaust all the resources, and the process of
`git pack-redundant` will be killed.

The following script could create a repository with too many redundant
packs, and running `git pack-redundant --all` in the `test.git` repo
will die soon.

    #!/bin/sh

    repo="$(pwd)/test.git"
    work="$(pwd)/test"
    i=1
    max=199

    if test -d "$repo" || test -d "$work"; then
    	echo >&2 "ERROR: '$repo' or '$work' already exist"
    	exit 1
    fi

    git init -q --bare "$repo"
    git --git-dir="$repo" config gc.auto 0
    git --git-dir="$repo" config transfer.unpackLimit 0
    git clone -q "$repo" "$work" 2>/dev/null

    while :; do
        cd "$work"
        echo "loop $i: $(date +%s)" >$i
        git add $i
        git commit -q -sm "loop $i"
        git push -q origin HEAD:master
        printf "\rCreate pack %4d/%d\t" $i $max
        if test $i -ge $max; then break; fi

        cd "$repo"
        git repack -q
        if test $(($i % 2)) -eq 0; then
            git repack -aq
            pack=$(ls -t $repo/objects/pack/*.pack | head -1)
            touch "${pack%.pack}.keep"
        fi
        i=$((i+1))
    done
    printf "\ndone\n"

To get the `min` unique pack list, we can replace the iteration in
`minimize` function with a new algorithm, and this could solve this
issue:

1. Get the unique and non_uniqe packs, add the unique packs to the
   `min` list.

2. Remove the objects of unique packs from non_unique packs, then each
   object left in the non_unique packs will have at least two copies.

3. Sort the non_unique packs by the objects' size, more objects first,
   and add the first non_unique pack to `min` list.

4. Drop the duplicated objects from other packs in the ordered
   non_unique pack list, and repeat step 3.

Some test cases will fail on Mac OS X. Mark them and will resolve in
later commit.

Original PR and discussions: https://github.com/jiangxin/git/pull/25

Signed-off-by: Sun Chao <sunchao9@huawei.com>
Signed-off-by: Jiang Xin <zhiyou.jx@alibaba-inc.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
This commit is contained in:
Sun Chao
2019-02-02 21:30:15 +08:00
committed by Junio C Hamano
parent 8822859363
commit 3084a01e5e
2 changed files with 72 additions and 132 deletions
Download Patch File Download Diff File Expand all files Collapse all files

192
builtin/pack-redundant.c
View File

@@ -35,11 +35,6 @@ static struct pack_list {
struct llist *all_objects;
} *local_packs = NULL, *altodb_packs = NULL;
struct pll {
struct pll *next;
struct pack_list *pl;
};
static struct llist_item *free_nodes;
static inline void llist_item_put(struct llist_item *item)
@@ -63,15 +58,6 @@ static inline struct llist_item *llist_item_get(void)
return new_item;
}
static void llist_free(struct llist *list)
{
while ((list->back = list->front)) {
list->front = list->front->next;
llist_item_put(list->back);
}
free(list);
}
static inline void llist_init(struct llist **list)
{
*list = xmalloc(sizeof(struct llist));
@@ -290,78 +276,6 @@ static void cmp_two_packs(struct pack_list *p1, struct pack_list *p2)
}
}
static void pll_free(struct pll *l)
{
struct pll *old;
struct pack_list *opl;
while (l) {
old = l;
while (l->pl) {
opl = l->pl;
l->pl = opl->next;
free(opl);
}
l = l->next;
free(old);
}
}
/* all the permutations have to be free()d at the same time,
* since they refer to each other
*/
static struct pll * get_permutations(struct pack_list *list, int n)
{
struct pll *subset, *ret = NULL, *new_pll = NULL;
if (list == NULL || pack_list_size(list) < n || n == 0)
return NULL;
if (n == 1) {
while (list) {
new_pll = xmalloc(sizeof(*new_pll));
new_pll->pl = NULL;
pack_list_insert(&new_pll->pl, list);
new_pll->next = ret;
ret = new_pll;
list = list->next;
}
return ret;
}
while (list->next) {
subset = get_permutations(list->next, n - 1);
while (subset) {
new_pll = xmalloc(sizeof(*new_pll));
new_pll->pl = subset->pl;
pack_list_insert(&new_pll->pl, list);
new_pll->next = ret;
ret = new_pll;
subset = subset->next;
}
list = list->next;
}
return ret;
}
static int is_superset(struct pack_list *pl, struct llist *list)
{
struct llist *diff;
diff = llist_copy(list);
while (pl) {
llist_sorted_difference_inplace(diff, pl->all_objects);
if (diff->size == 0) { /* we're done */
llist_free(diff);
return 1;
}
pl = pl->next;
}
llist_free(diff);
return 0;
}
static size_t sizeof_union(struct packed_git *p1, struct packed_git *p2)
{
size_t ret = 0;
@@ -426,14 +340,52 @@ static inline off_t pack_set_bytecount(struct pack_list *pl)
return ret;
}
static int cmp_pack_list_reverse(const void *a, const void *b)
{
struct pack_list *pl_a = *((struct pack_list **)a);
struct pack_list *pl_b = *((struct pack_list **)b);
size_t sz_a = pl_a->all_objects->size;
size_t sz_b = pl_b->all_objects->size;
if (sz_a == sz_b)
return 0;
else if (sz_a < sz_b)
return 1;
else
return -1;
}
/* Sort pack_list, greater size of all_objects first */
static void sort_pack_list(struct pack_list **pl)
{
struct pack_list **ary, *p;
int i;
size_t n = pack_list_size(*pl);
if (n < 2)
return;
/* prepare an array of packed_list for easier sorting */
ary = xcalloc(n, sizeof(struct pack_list *));
for (n = 0, p = *pl; p; p = p->next)
ary[n++] = p;
QSORT(ary, n, cmp_pack_list_reverse);
/* link them back again */
for (i = 0; i < n - 1; i++)
ary[i]->next = ary[i + 1];
ary[n - 1]->next = NULL;
*pl = ary[0];
free(ary);
}
static void minimize(struct pack_list **min)
{
struct pack_list *pl, *unique = NULL,
*non_unique = NULL, *min_perm = NULL;
struct pll *perm, *perm_all, *perm_ok = NULL, *new_perm;
struct llist *missing;
off_t min_perm_size = 0, perm_size;
int n;
struct pack_list *pl, *unique = NULL, *non_unique = NULL;
struct llist *missing, *unique_pack_objects;
pl = local_packs;
while (pl) {
@@ -451,49 +403,37 @@ static void minimize(struct pack_list **min)
pl = pl->next;
}
*min = unique;
/* return if there are no objects missing from the unique set */
if (missing->size == 0) {
*min = unique;
free(missing);
return;
}
/* find the permutations which contain all missing objects */
for (n = 1; n <= pack_list_size(non_unique) && !perm_ok; n++) {
perm_all = perm = get_permutations(non_unique, n);
while (perm) {
if (is_superset(perm->pl, missing)) {
new_perm = xmalloc(sizeof(struct pll));
memcpy(new_perm, perm, sizeof(struct pll));
new_perm->next = perm_ok;
perm_ok = new_perm;
}
perm = perm->next;
}
if (perm_ok)
break;
pll_free(perm_all);
}
if (perm_ok == NULL)
die("Internal error: No complete sets found!");
unique_pack_objects = llist_copy(all_objects);
llist_sorted_difference_inplace(unique_pack_objects, missing);
/* find the permutation with the smallest size */
perm = perm_ok;
while (perm) {
perm_size = pack_set_bytecount(perm->pl);
if (!min_perm_size || min_perm_size > perm_size) {
min_perm_size = perm_size;
min_perm = perm->pl;
}
perm = perm->next;
}
*min = min_perm;
/* add the unique packs to the list */
pl = unique;
/* remove unique pack objects from the non_unique packs */
pl = non_unique;
while (pl) {
pack_list_insert(min, pl);
llist_sorted_difference_inplace(pl->all_objects, unique_pack_objects);
pl = pl->next;
}
while (non_unique) {
/* sort the non_unique packs, greater size of all_objects first */
sort_pack_list(&non_unique);
if (non_unique->all_objects->size == 0)
break;
pack_list_insert(min, non_unique);
for (pl = non_unique->next; pl && pl->all_objects->size > 0; pl = pl->next)
llist_sorted_difference_inplace(pl->all_objects, non_unique->all_objects);
non_unique = non_unique->next;
}
}
static void load_all_objects(void)
@@ -606,7 +546,7 @@ static void load_all(void)
int cmd_pack_redundant(int argc, const char **argv, const char *prefix)
{
int i;
struct pack_list *min, *red, *pl;
struct pack_list *min = NULL, *red, *pl;
struct llist *ignore;
struct object_id *oid;
char buf[GIT_MAX_HEXSZ + 2]; /* hex hash + \n + \0 */