From a41fae9c46a4cb5e59cc1a17d19f6a3a6cbfbb2f Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Sat, 9 Sep 2006 22:21:27 -0700 Subject: [PATCH 01/10] get_sha1_hex() micro-optimization The function appeared high on a gprof output for a rev-list run of a non-trivial size, and it was an obvious low-hanging fruit. The code is from Linus. Signed-off-by: Junio C Hamano --- sha1_file.c | 44 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 36 insertions(+), 8 deletions(-) diff --git a/sha1_file.c b/sha1_file.c index 428d791ba8..b64b92de4e 100644 --- a/sha1_file.c +++ b/sha1_file.c @@ -26,15 +26,43 @@ const unsigned char null_sha1[20]; static unsigned int sha1_file_open_flag = O_NOATIME; -static unsigned hexval(char c) +static inline unsigned int hexval(unsigned int c) { - if (c >= '0' && c <= '9') - return c - '0'; - if (c >= 'a' && c <= 'f') - return c - 'a' + 10; - if (c >= 'A' && c <= 'F') - return c - 'A' + 10; - return ~0; + static signed char val[256] = { + -1, -1, -1, -1, -1, -1, -1, -1, /* 00-07 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 08-0f */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 10-17 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 18-1f */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 20-27 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 28-2f */ + 0, 1, 2, 3, 4, 5, 6, 7, /* 30-37 */ + 8, 9, -1, -1, -1, -1, -1, -1, /* 38-3f */ + -1, 10, 11, 12, 13, 14, 15, -1, /* 40-47 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 48-4f */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 50-57 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 58-5f */ + -1, 10, 11, 12, 13, 14, 15, -1, /* 60-67 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 68-67 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 70-77 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 78-7f */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 80-87 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 88-8f */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 90-97 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 98-9f */ + -1, -1, -1, -1, -1, -1, -1, -1, /* a0-a7 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* a8-af */ + -1, -1, -1, -1, -1, -1, -1, -1, /* b0-b7 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* b8-bf */ + -1, -1, -1, -1, -1, -1, -1, -1, /* c0-c7 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* c8-cf */ + -1, -1, -1, -1, -1, -1, -1, -1, /* d0-d7 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* d8-df */ + -1, -1, -1, -1, -1, -1, -1, -1, /* e0-e7 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* e8-ef */ + -1, -1, -1, -1, -1, -1, -1, -1, /* f0-f7 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* f8-ff */ + }; + return val[c]; } int get_sha1_hex(const char *hex, unsigned char *sha1) From 37f944363d5b5fb5bcbf2d184865534739713c01 Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Sat, 9 Sep 2006 23:48:03 -0700 Subject: [PATCH 02/10] archive: allow remote to have more formats than we understand. This fixes git-archive --remote not to parse archiver arguments; otherwise if the remote end implements formats other than the one known locally we will not be able to access that format. Signed-off-by: Junio C Hamano --- archive.h | 1 - builtin-archive.c | 79 ++++++++++++++++++++++++++++------------------- 2 files changed, 47 insertions(+), 33 deletions(-) diff --git a/archive.h b/archive.h index d8cca735d2..e0782b9dcf 100644 --- a/archive.h +++ b/archive.h @@ -19,7 +19,6 @@ typedef void *(*parse_extra_args_fn_t)(int argc, const char **argv); struct archiver { const char *name; - const char *remote; struct archiver_args args; write_archive_fn_t write_archive; parse_extra_args_fn_t parse_extra; diff --git a/builtin-archive.c b/builtin-archive.c index b944737550..c70488c537 100644 --- a/builtin-archive.c +++ b/builtin-archive.c @@ -26,7 +26,7 @@ struct archiver archivers[] = { }, }; -static int run_remote_archiver(struct archiver *ar, int argc, +static int run_remote_archiver(const char *remote, int argc, const char **argv) { char *url, buf[1024]; @@ -35,16 +35,13 @@ static int run_remote_archiver(struct archiver *ar, int argc, sprintf(buf, "git-upload-archive"); - url = xstrdup(ar->remote); + url = xstrdup(remote); pid = git_connect(fd, url, buf); if (pid < 0) return pid; - for (i = 1; i < argc; i++) { - if (!strncmp(argv[i], "--remote=", 9)) - continue; + for (i = 1; i < argc; i++) packet_write(fd[1], "argument %s\n", argv[i]); - } packet_flush(fd[1]); len = packet_read_line(fd[0], buf, sizeof(buf)); @@ -150,17 +147,16 @@ int parse_archive_args(int argc, const char **argv, struct archiver *ar) const char *extra_argv[MAX_EXTRA_ARGS]; int extra_argc = 0; const char *format = NULL; /* might want to default to "tar" */ - const char *remote = NULL; const char *base = ""; - int list = 0; int i; for (i = 1; i < argc; i++) { const char *arg = argv[i]; if (!strcmp(arg, "--list") || !strcmp(arg, "-l")) { - list = 1; - continue; + for (i = 0; i < ARRAY_SIZE(archivers); i++) + printf("%s\n", archivers[i].name); + exit(0); } if (!strncmp(arg, "--format=", 9)) { format = arg + 9; @@ -170,10 +166,6 @@ int parse_archive_args(int argc, const char **argv, struct archiver *ar) base = arg + 9; continue; } - if (!strncmp(arg, "--remote=", 9)) { - remote = arg + 9; - continue; - } if (!strcmp(arg, "--")) { i++; break; @@ -187,44 +179,67 @@ int parse_archive_args(int argc, const char **argv, struct archiver *ar) break; } - if (list) { - if (!remote) { - for (i = 0; i < ARRAY_SIZE(archivers); i++) - printf("%s\n", archivers[i].name); - exit(0); - } - die("--list and --remote are mutually exclusive"); - } - - if (argc - i < 1) + /* We need at least one parameter -- tree-ish */ + if (argc - 1 < i) usage(archive_usage); if (!format) die("You must specify an archive format"); if (init_archiver(format, ar) < 0) die("Unknown archive format '%s'", format); - if (extra_argc && !remote) { - if (!ar->parse_extra) { + if (extra_argc) { + if (!ar->parse_extra) die("%s", default_parse_extra(ar, extra_argv)); - } ar->args.extra = ar->parse_extra(extra_argc, extra_argv); } - ar->remote = remote; ar->args.base = base; return i; } +static const char *remote_request(int *ac, const char **av) +{ + int ix, iy, cnt = *ac; + int no_more_options = 0; + const char *remote = NULL; + + for (ix = iy = 1; ix < cnt; ix++) { + const char *arg = av[ix]; + if (!strcmp(arg, "--")) + no_more_options = 1; + if (!no_more_options) { + if (!strncmp(arg, "--remote=", 9)) { + if (remote) + die("Multiple --remote specified"); + remote = arg + 9; + continue; + } + if (arg[0] != '-') + no_more_options = 1; + } + if (ix != iy) + av[iy] = arg; + iy++; + } + if (remote) { + av[--cnt] = NULL; + *ac = cnt; + } + return remote; +} + int cmd_archive(int argc, const char **argv, const char *prefix) { struct archiver ar; int tree_idx; + const char *remote = NULL; + remote = remote_request(&argc, argv); + if (remote) + return run_remote_archiver(remote, argc, argv); + + memset(&ar, 0, sizeof(ar)); tree_idx = parse_archive_args(argc, argv, &ar); - - if (ar.remote) - return run_remote_archiver(&ar, argc, argv); - if (prefix == NULL) prefix = setup_git_directory(); From 49a52b1d1fb120f52de3a67f1e4e5ae81512ab81 Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Sun, 10 Sep 2006 01:06:33 -0700 Subject: [PATCH 03/10] Move sideband client side support into reusable form. This moves the receiver side of the sideband support from fetch-clone.c to sideband.c and its header file, so that archiver protocol can use it. Signed-off-by: Junio C Hamano --- Makefile | 4 ++-- fetch-clone.c | 32 +++++--------------------------- sideband.c | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ sideband.h | 11 +++++++++++ 4 files changed, 66 insertions(+), 29 deletions(-) create mode 100644 sideband.c create mode 100644 sideband.h diff --git a/Makefile b/Makefile index 7b3114f3aa..a46cd52713 100644 --- a/Makefile +++ b/Makefile @@ -233,7 +233,7 @@ XDIFF_LIB=xdiff/lib.a LIB_H = \ blob.h cache.h commit.h csum-file.h delta.h \ - diff.h object.h pack.h pkt-line.h quote.h refs.h \ + diff.h object.h pack.h pkt-line.h quote.h refs.h sideband.h \ run-command.h strbuf.h tag.h tree.h git-compat-util.h revision.h \ tree-walk.h log-tree.h dir.h path-list.h unpack-trees.h builtin.h @@ -245,7 +245,7 @@ DIFF_OBJS = \ LIB_OBJS = \ blob.o commit.o connect.o csum-file.o cache-tree.o base85.o \ date.o diff-delta.o entry.o exec_cmd.o ident.o lockfile.o \ - object.o pack-check.o patch-delta.o path.o pkt-line.o \ + object.o pack-check.o patch-delta.o path.o pkt-line.o sideband.o \ quote.o read-cache.o refs.o run-command.o dir.o object-refs.o \ server-info.o setup.o sha1_file.o sha1_name.o strbuf.o \ tag.o tree.o usage.o config.o environment.o ctype.o copy.o \ diff --git a/fetch-clone.c b/fetch-clone.c index c5cf4776fa..b62feac17d 100644 --- a/fetch-clone.c +++ b/fetch-clone.c @@ -1,6 +1,7 @@ #include "cache.h" #include "exec_cmd.h" #include "pkt-line.h" +#include "sideband.h" #include #include @@ -114,36 +115,13 @@ static pid_t setup_sideband(int sideband, const char *me, int fd[2], int xd[2]) die("%s: unable to fork off sideband demultiplexer", me); if (!side_pid) { /* subprocess */ + char buf[DEFAULT_PACKET_MAX]; + close(fd[0]); if (xd[0] != xd[1]) close(xd[1]); - while (1) { - char buf[1024]; - int len = packet_read_line(xd[0], buf, sizeof(buf)); - if (len == 0) - break; - if (len < 1) - die("%s: protocol error: no band designator", - me); - len--; - switch (buf[0] & 0xFF) { - case 3: - safe_write(2, "remote: ", 8); - safe_write(2, buf+1, len); - safe_write(2, "\n", 1); - exit(1); - case 2: - safe_write(2, "remote: ", 8); - safe_write(2, buf+1, len); - continue; - case 1: - safe_write(fd[1], buf+1, len); - continue; - default: - die("%s: protocol error: bad band #%d", - me, (buf[0] & 0xFF)); - } - } + if (recv_sideband(me, xd[0], fd[1], 2, buf, sizeof(buf))) + exit(1); exit(0); } close(xd[0]); diff --git a/sideband.c b/sideband.c new file mode 100644 index 0000000000..861f6219db --- /dev/null +++ b/sideband.c @@ -0,0 +1,48 @@ +#include "pkt-line.h" +#include "sideband.h" + +/* + * Receive multiplexed output stream over git native protocol. + * in_stream is the input stream from the remote, which carries data + * in pkt_line format with band designator. Demultiplex it into out + * and err and return error appropriately. Band #1 carries the + * primary payload. Things coming over band #2 is not necessarily + * error; they are usually informative message on the standard error + * stream, aka "verbose"). A message over band #3 is a signal that + * the remote died unexpectedly. A flush() concludes the stream. + */ +int recv_sideband(const char *me, int in_stream, int out, int err, char *buf, int bufsz) +{ + while (1) { + int len = packet_read_line(in_stream, buf, bufsz); + if (len == 0) + break; + if (len < 1) { + len = sprintf(buf, "%s: protocol error: no band designator\n", me); + safe_write(err, buf, len); + return SIDEBAND_PROTOCOL_ERROR; + } + len--; + switch (buf[0] & 0xFF) { + case 3: + safe_write(err, "remote: ", 8); + safe_write(err, buf+1, len); + safe_write(err, "\n", 1); + return SIDEBAND_REMOTE_ERROR; + case 2: + safe_write(err, "remote: ", 8); + safe_write(err, buf+1, len); + continue; + case 1: + safe_write(out, buf+1, len); + continue; + default: + len = sprintf(buf + 1, + "%s: protocol error: bad band #%d\n", + me, buf[0] & 0xFF); + safe_write(err, buf+1, len); + return SIDEBAND_PROTOCOL_ERROR; + } + } + return 0; +} diff --git a/sideband.h b/sideband.h new file mode 100644 index 0000000000..90b385580e --- /dev/null +++ b/sideband.h @@ -0,0 +1,11 @@ +#ifndef SIDEBAND_H +#define SIDEBAND_H + +#define SIDEBAND_PROTOCOL_ERROR -2 +#define SIDEBAND_REMOTE_ERROR -1 + +#define DEFAULT_PACKET_MAX 1000 + +int recv_sideband(const char *me, int in_stream, int out, int err, char *, int); + +#endif From 958c24b1b8f463bca857f45c41a2f8198e345c2f Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Sun, 10 Sep 2006 03:20:24 -0700 Subject: [PATCH 04/10] Move sideband server side support into reusable form. The server side support; this is just the very low level, and the caller needs to know which band it wants to send things out. Signed-off-by: Junio C Hamano (cherry picked from b786552b67878c7780c50def4c069d46dc54efbe commit) --- sideband.c | 26 ++++++++++++++++++++++++++ sideband.h | 1 + upload-pack.c | 48 ++++++++++++------------------------------------ 3 files changed, 39 insertions(+), 36 deletions(-) diff --git a/sideband.c b/sideband.c index 861f6219db..1b14ff8892 100644 --- a/sideband.c +++ b/sideband.c @@ -46,3 +46,29 @@ int recv_sideband(const char *me, int in_stream, int out, int err, char *buf, in } return 0; } + +/* + * fd is connected to the remote side; send the sideband data + * over multiplexed packet stream. + */ +ssize_t send_sideband(int fd, int band, const char *data, ssize_t sz, int packet_max) +{ + ssize_t ssz = sz; + const char *p = data; + + while (sz) { + unsigned n; + char hdr[5]; + + n = sz; + if (packet_max - 5 < n) + n = packet_max - 5; + sprintf(hdr, "%04x", n + 5); + hdr[4] = band; + safe_write(fd, hdr, 5); + safe_write(fd, p, n); + p += n; + sz -= n; + } + return ssz; +} diff --git a/sideband.h b/sideband.h index 90b385580e..c645cf2c52 100644 --- a/sideband.h +++ b/sideband.h @@ -7,5 +7,6 @@ #define DEFAULT_PACKET_MAX 1000 int recv_sideband(const char *me, int in_stream, int out, int err, char *, int); +ssize_t send_sideband(int fd, int band, const char *data, ssize_t sz, int packet_max); #endif diff --git a/upload-pack.c b/upload-pack.c index 51ce936b06..1f2f7f75e5 100644 --- a/upload-pack.c +++ b/upload-pack.c @@ -4,6 +4,7 @@ #include "cache.h" #include "refs.h" #include "pkt-line.h" +#include "sideband.h" #include "tag.h" #include "object.h" #include "commit.h" @@ -33,45 +34,19 @@ static int strip(char *line, int len) return len; } -#define PACKET_MAX 1000 static ssize_t send_client_data(int fd, const char *data, ssize_t sz) { - ssize_t ssz; - const char *p; + if (use_sideband) + return send_sideband(1, fd, data, sz, DEFAULT_PACKET_MAX); - if (!data) { - if (!use_sideband) - return 0; - packet_flush(1); + if (fd == 3) + /* emergency quit */ + fd = 2; + if (fd == 2) { + xwrite(fd, data, sz); + return sz; } - - if (!use_sideband) { - if (fd == 3) - /* emergency quit */ - fd = 2; - if (fd == 2) { - xwrite(fd, data, sz); - return sz; - } - return safe_write(fd, data, sz); - } - p = data; - ssz = sz; - while (sz) { - unsigned n; - char hdr[5]; - - n = sz; - if (PACKET_MAX - 5 < n) - n = PACKET_MAX - 5; - sprintf(hdr, "%04x", n + 5); - hdr[4] = fd; - safe_write(1, hdr, 5); - safe_write(1, p, n); - p += n; - sz -= n; - } - return ssz; + return safe_write(fd, data, sz); } static void create_pack_file(void) @@ -308,7 +283,8 @@ static void create_pack_file(void) goto fail; fprintf(stderr, "flushed.\n"); } - send_client_data(1, NULL, 0); + if (use_sideband) + packet_flush(1); return; } fail: From 326711c16879792da8d7159bf29d080569c3a1e0 Mon Sep 17 00:00:00 2001 From: Rene Scharfe Date: Sun, 10 Sep 2006 18:10:01 +0200 Subject: [PATCH 05/10] Use xstrdup instead of strdup in builtin-{tar,zip}-tree.c Signed-off-by: Rene Scharfe Signed-off-by: Junio C Hamano (cherry picked from 5d2aea4cb383a43e40d47ab69d8ad7a495df6ea2 commit) --- builtin-tar-tree.c | 2 +- builtin-zip-tree.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/builtin-tar-tree.c b/builtin-tar-tree.c index c20eb0e364..e8e492fa0f 100644 --- a/builtin-tar-tree.c +++ b/builtin-tar-tree.c @@ -390,7 +390,7 @@ int write_tar_archive(struct archiver_args *args) write_global_extended_header(args->commit_sha1); if (args->base && plen > 0 && args->base[plen - 1] == '/') { - char *base = strdup(args->base); + char *base = xstrdup(args->base); int baselen = strlen(base); while (baselen > 0 && base[baselen - 1] == '/') diff --git a/builtin-zip-tree.c b/builtin-zip-tree.c index 4e796338af..fdac2bdd70 100644 --- a/builtin-zip-tree.c +++ b/builtin-zip-tree.c @@ -363,7 +363,7 @@ int write_zip_archive(struct archiver_args *args) zip_dir_size = ZIP_DIRECTORY_MIN_SIZE; if (args->base && plen > 0 && args->base[plen - 1] == '/') { - char *base = strdup(args->base); + char *base = xstrdup(args->base); int baselen = strlen(base); while (baselen > 0 && base[baselen - 1] == '/') From 8142f603b9955648228549d2e83ace7fbe834114 Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Sun, 10 Sep 2006 04:16:39 -0700 Subject: [PATCH 06/10] archive: force line buffered output to stderr Otherwise the remote notification that comes with -v option can get clumped together. Signed-off-by: Junio C Hamano (cherry picked from a675cda60ead41f439b04bc69e0f19ace04e59d3 commit) --- builtin-archive.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/builtin-archive.c b/builtin-archive.c index c70488c537..3a8be57e15 100644 --- a/builtin-archive.c +++ b/builtin-archive.c @@ -238,6 +238,8 @@ int cmd_archive(int argc, const char **argv, const char *prefix) if (remote) return run_remote_archiver(remote, argc, argv); + setlinebuf(stderr); + memset(&ar, 0, sizeof(ar)); tree_idx = parse_archive_args(argc, argv, &ar); if (prefix == NULL) From e0ffb24877d4530208905512f7c91dd8d71e2c95 Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Sat, 9 Sep 2006 22:42:02 -0700 Subject: [PATCH 07/10] Add --verbose to git-archive And teach backends about it. Signed-off-by: Junio C Hamano (cherry picked from 9e2c44a2893ae90944a0b7c9f40a9d22b759b5c0 commit) --- archive.h | 1 + builtin-archive.c | 8 +++++++- builtin-tar-tree.c | 4 ++++ builtin-zip-tree.c | 4 ++++ 4 files changed, 16 insertions(+), 1 deletion(-) diff --git a/archive.h b/archive.h index e0782b9dcf..16dcdb875c 100644 --- a/archive.h +++ b/archive.h @@ -10,6 +10,7 @@ struct archiver_args { const unsigned char *commit_sha1; time_t time; const char **pathspec; + unsigned int verbose : 1; void *extra; }; diff --git a/builtin-archive.c b/builtin-archive.c index 3a8be57e15..7544ad3ca1 100644 --- a/builtin-archive.c +++ b/builtin-archive.c @@ -12,7 +12,7 @@ #include "pkt-line.h" static const char archive_usage[] = \ -"git-archive --format= [--prefix=/] [] [path...]"; +"git-archive --format= [--prefix=/] [--verbose] [] [path...]"; struct archiver archivers[] = { { @@ -148,6 +148,7 @@ int parse_archive_args(int argc, const char **argv, struct archiver *ar) int extra_argc = 0; const char *format = NULL; /* might want to default to "tar" */ const char *base = ""; + int verbose = 0; int i; for (i = 1; i < argc; i++) { @@ -158,6 +159,10 @@ int parse_archive_args(int argc, const char **argv, struct archiver *ar) printf("%s\n", archivers[i].name); exit(0); } + if (!strcmp(arg, "--verbose") || !strcmp(arg, "-v")) { + verbose = 1; + continue; + } if (!strncmp(arg, "--format=", 9)) { format = arg + 9; continue; @@ -192,6 +197,7 @@ int parse_archive_args(int argc, const char **argv, struct archiver *ar) die("%s", default_parse_extra(ar, extra_argv)); ar->args.extra = ar->parse_extra(extra_argc, extra_argv); } + ar->args.verbose = verbose; ar->args.base = base; return i; diff --git a/builtin-tar-tree.c b/builtin-tar-tree.c index e8e492fa0f..f2679a8637 100644 --- a/builtin-tar-tree.c +++ b/builtin-tar-tree.c @@ -22,6 +22,7 @@ static unsigned long offset; static time_t archive_time; static int tar_umask; +static int verbose; /* writes out the whole block, but only if it is full */ static void write_if_needed(void) @@ -169,6 +170,8 @@ static void write_entry(const unsigned char *sha1, struct strbuf *path, mode = 0100666; sprintf(header.name, "%s.paxheader", sha1_to_hex(sha1)); } else { + if (verbose) + fprintf(stderr, "%.*s\n", path->len, path->buf); if (S_ISDIR(mode)) { *header.typeflag = TYPEFLAG_DIR; mode = (mode | 0777) & ~tar_umask; @@ -385,6 +388,7 @@ int write_tar_archive(struct archiver_args *args) git_config(git_tar_config); archive_time = args->time; + verbose = args->verbose; if (args->commit_sha1) write_global_extended_header(args->commit_sha1); diff --git a/builtin-zip-tree.c b/builtin-zip-tree.c index fdac2bdd70..52d4b7a17e 100644 --- a/builtin-zip-tree.c +++ b/builtin-zip-tree.c @@ -13,6 +13,7 @@ static const char zip_tree_usage[] = "git-zip-tree [-0|...|-9] [ ]"; +static int verbose; static int zip_date; static int zip_time; @@ -164,6 +165,8 @@ static int write_zip_entry(const unsigned char *sha1, crc = crc32(0, Z_NULL, 0); path = construct_path(base, baselen, filename, S_ISDIR(mode), &pathlen); + if (verbose) + fprintf(stderr, "%s\n", path); if (pathlen > 0xffff) { error("path too long (%d chars, SHA1: %s): %s", pathlen, sha1_to_hex(sha1), path); @@ -361,6 +364,7 @@ int write_zip_archive(struct archiver_args *args) zip_dir = xmalloc(ZIP_DIRECTORY_MIN_SIZE); zip_dir_size = ZIP_DIRECTORY_MIN_SIZE; + verbose = args->verbose; if (args->base && plen > 0 && args->base[plen - 1] == '/') { char *base = xstrdup(args->base); From fe5ab763f848cfcda22001f9280625f06c4c3760 Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Sun, 10 Sep 2006 04:02:57 -0700 Subject: [PATCH 08/10] Teach --exec to git-archive --remote Some people needed --exec to specify the location of the upload-pack executable, because their default SSH log-in does not include the directory they have their own private copy of git on the $PATH. These people need to be able to say --exec to git-archive --remote for the same reason. Signed-off-by: Junio C Hamano --- builtin-archive.c | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/builtin-archive.c b/builtin-archive.c index 7544ad3ca1..dd7ffc043d 100644 --- a/builtin-archive.c +++ b/builtin-archive.c @@ -32,16 +32,30 @@ static int run_remote_archiver(const char *remote, int argc, char *url, buf[1024]; int fd[2], i, len, rv; pid_t pid; + const char *exec = "git-upload-archive"; + int exec_at = 0; - sprintf(buf, "git-upload-archive"); + for (i = 1; i < argc; i++) { + const char *arg = argv[i]; + if (!strncmp("--exec=", arg, 7)) { + if (exec_at) + die("multiple --exec specified"); + exec = arg + 7; + exec_at = i; + break; + } + } url = xstrdup(remote); - pid = git_connect(fd, url, buf); + pid = git_connect(fd, url, exec); if (pid < 0) return pid; - for (i = 1; i < argc; i++) + for (i = 1; i < argc; i++) { + if (i == exec_at) + continue; packet_write(fd[1], "argument %s\n", argv[i]); + } packet_flush(fd[1]); len = packet_read_line(fd[0], buf, sizeof(buf)); From d47f3db75c58139cdcbca5cc63b17bf5db293b6a Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Sun, 10 Sep 2006 16:27:08 -0700 Subject: [PATCH 09/10] Prepare larger packet buffer for upload-pack protocol. The original side-band support added to the upload-pack protocol used the default 1000-byte packet length. The pkt-line format allows up to 64k, so prepare the receiver for the maximum size, and have the uploader and downloader negotiate if larger packet length is allowed. Signed-off-by: Junio C Hamano --- fetch-clone.c | 2 +- fetch-pack.c | 12 +++++++++--- sideband.h | 1 + upload-pack.c | 14 +++++++++----- 4 files changed, 20 insertions(+), 9 deletions(-) diff --git a/fetch-clone.c b/fetch-clone.c index b62feac17d..b632ca0438 100644 --- a/fetch-clone.c +++ b/fetch-clone.c @@ -115,7 +115,7 @@ static pid_t setup_sideband(int sideband, const char *me, int fd[2], int xd[2]) die("%s: unable to fork off sideband demultiplexer", me); if (!side_pid) { /* subprocess */ - char buf[DEFAULT_PACKET_MAX]; + char buf[LARGE_PACKET_MAX]; close(fd[0]); if (xd[0] != xd[1]) diff --git a/fetch-pack.c b/fetch-pack.c index 377feded1c..1b2d6ee20d 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -166,10 +166,11 @@ static int find_common(int fd[2], unsigned char *result_sha1, } if (!fetching) - packet_write(fd[1], "want %s%s%s%s\n", + packet_write(fd[1], "want %s%s%s%s%s\n", sha1_to_hex(remote), (multi_ack ? " multi_ack" : ""), - (use_sideband ? " side-band" : ""), + (use_sideband == 2 ? " side-band-64k" : ""), + (use_sideband == 1 ? " side-band" : ""), (use_thin_pack ? " thin-pack" : "")); else packet_write(fd[1], "want %s\n", sha1_to_hex(remote)); @@ -426,7 +427,12 @@ static int fetch_pack(int fd[2], int nr_match, char **match) fprintf(stderr, "Server supports multi_ack\n"); multi_ack = 1; } - if (server_supports("side-band")) { + if (server_supports("side-band-64k")) { + if (verbose) + fprintf(stderr, "Server supports side-band-64k\n"); + use_sideband = 2; + } + else if (server_supports("side-band")) { if (verbose) fprintf(stderr, "Server supports side-band\n"); use_sideband = 1; diff --git a/sideband.h b/sideband.h index c645cf2c52..4872106fa0 100644 --- a/sideband.h +++ b/sideband.h @@ -5,6 +5,7 @@ #define SIDEBAND_REMOTE_ERROR -1 #define DEFAULT_PACKET_MAX 1000 +#define LARGE_PACKET_MAX 65520 int recv_sideband(const char *me, int in_stream, int out, int err, char *, int); ssize_t send_sideband(int fd, int band, const char *data, ssize_t sz, int packet_max); diff --git a/upload-pack.c b/upload-pack.c index 1f2f7f75e5..b673d8cb97 100644 --- a/upload-pack.c +++ b/upload-pack.c @@ -20,6 +20,9 @@ static int use_thin_pack; static struct object_array have_obj; static struct object_array want_obj; static unsigned int timeout; +/* 0 for no sideband, + * otherwise maximum packet size (up to 65520 bytes). + */ static int use_sideband; static void reset_timeout(void) @@ -37,8 +40,7 @@ static int strip(char *line, int len) static ssize_t send_client_data(int fd, const char *data, ssize_t sz) { if (use_sideband) - return send_sideband(1, fd, data, sz, DEFAULT_PACKET_MAX); - + return send_sideband(1, fd, data, sz, use_sideband); if (fd == 3) /* emergency quit */ fd = 2; @@ -389,8 +391,10 @@ static void receive_needs(void) multi_ack = 1; if (strstr(line+45, "thin-pack")) use_thin_pack = 1; - if (strstr(line+45, "side-band")) - use_sideband = 1; + if (strstr(line+45, "side-band-64k")) + use_sideband = LARGE_PACKET_MAX; + else if (strstr(line+45, "side-band")) + use_sideband = DEFAULT_PACKET_MAX; /* We have sent all our refs already, and the other end * should have chosen out of them; otherwise they are @@ -412,7 +416,7 @@ static void receive_needs(void) static int send_ref(const char *refname, const unsigned char *sha1) { - static const char *capabilities = "multi_ack thin-pack side-band"; + static const char *capabilities = "multi_ack thin-pack side-band side-band-64k"; struct object *o = parse_object(sha1); if (!o) From 23d6d112c004d4242f9dbd8161f79ccdeb47bde8 Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Sun, 10 Sep 2006 03:33:34 -0700 Subject: [PATCH 10/10] Add sideband status report to git-archive protocol Using the refactored sideband code from existing upload-pack protocol, this lets the error condition and status output sent from the remote process to be shown locally. Signed-off-by: Junio C Hamano --- builtin-archive.c | 6 +-- builtin-upload-archive.c | 94 ++++++++++++++++++++++++++++++++++++++-- 2 files changed, 93 insertions(+), 7 deletions(-) diff --git a/builtin-archive.c b/builtin-archive.c index dd7ffc043d..cb883dfe5f 100644 --- a/builtin-archive.c +++ b/builtin-archive.c @@ -10,6 +10,7 @@ #include "tree-walk.h" #include "exec_cmd.h" #include "pkt-line.h" +#include "sideband.h" static const char archive_usage[] = \ "git-archive --format= [--prefix=/] [--verbose] [] [path...]"; @@ -29,7 +30,7 @@ struct archiver archivers[] = { static int run_remote_archiver(const char *remote, int argc, const char **argv) { - char *url, buf[1024]; + char *url, buf[LARGE_PACKET_MAX]; int fd[2], i, len, rv; pid_t pid; const char *exec = "git-upload-archive"; @@ -74,8 +75,7 @@ static int run_remote_archiver(const char *remote, int argc, die("git-archive: expected a flush"); /* Now, start reading from fd[0] and spit it out to stdout */ - rv = copy_fd(fd[0], 1); - + rv = recv_sideband("archive", fd[0], 1, 2, buf, sizeof(buf)); close(fd[0]); rv |= finish_connect(pid); diff --git a/builtin-upload-archive.c b/builtin-upload-archive.c index 3bdb607e37..42cb9f8876 100644 --- a/builtin-upload-archive.c +++ b/builtin-upload-archive.c @@ -6,12 +6,18 @@ #include "builtin.h" #include "archive.h" #include "pkt-line.h" +#include "sideband.h" +#include +#include static const char upload_archive_usage[] = "git-upload-archive "; +static const char deadchild[] = +"git-upload-archive: archiver died with error"; -int cmd_upload_archive(int argc, const char **argv, const char *prefix) + +static int run_upload_archive(int argc, const char **argv, const char *prefix) { struct archiver ar; const char *sent_argv[MAX_ARGS]; @@ -64,9 +70,89 @@ int cmd_upload_archive(int argc, const char **argv, const char *prefix) parse_treeish_arg(sent_argv + treeish_idx, &ar.args, prefix); parse_pathspec_arg(sent_argv + treeish_idx + 1, &ar.args); - packet_write(1, "ACK\n"); - packet_flush(1); - return ar.write_archive(&ar.args); } +int cmd_upload_archive(int argc, const char **argv, const char *prefix) +{ + pid_t writer; + int fd1[2], fd2[2]; + /* + * Set up sideband subprocess. + * + * We (parent) monitor and read from child, sending its fd#1 and fd#2 + * multiplexed out to our fd#1. If the child dies, we tell the other + * end over channel #3. + */ + if (pipe(fd1) < 0 || pipe(fd2) < 0) { + int err = errno; + packet_write(1, "NACK pipe failed on the remote side\n"); + die("upload-archive: %s", strerror(err)); + } + writer = fork(); + if (writer < 0) { + int err = errno; + packet_write(1, "NACK fork failed on the remote side\n"); + die("upload-archive: %s", strerror(err)); + } + if (!writer) { + /* child - connect fd#1 and fd#2 to the pipe */ + dup2(fd1[1], 1); + dup2(fd2[1], 2); + close(fd1[1]); close(fd2[1]); + close(fd1[0]); close(fd2[0]); /* we do not read from pipe */ + + exit(run_upload_archive(argc, argv, prefix)); + } + + /* parent - read from child, multiplex and send out to fd#1 */ + close(fd1[1]); close(fd2[1]); /* we do not write to pipe */ + packet_write(1, "ACK\n"); + packet_flush(1); + + while (1) { + struct pollfd pfd[2]; + char buf[16384]; + ssize_t sz; + pid_t pid; + int status; + + pfd[0].fd = fd1[0]; + pfd[0].events = POLLIN; + pfd[1].fd = fd2[0]; + pfd[1].events = POLLIN; + if (poll(pfd, 2, -1) < 0) { + if (errno != EINTR) { + error("poll failed resuming: %s", + strerror(errno)); + sleep(1); + } + continue; + } + if (pfd[0].revents & (POLLIN|POLLHUP)) { + /* Data stream ready */ + sz = read(pfd[0].fd, buf, sizeof(buf)); + send_sideband(1, 1, buf, sz, LARGE_PACKET_MAX); + } + if (pfd[1].revents & (POLLIN|POLLHUP)) { + /* Status stream ready */ + sz = read(pfd[1].fd, buf, sizeof(buf)); + send_sideband(1, 2, buf, sz, LARGE_PACKET_MAX); + } + + if (((pfd[0].revents | pfd[1].revents) & POLLHUP) == 0) + continue; + /* did it die? */ + pid = waitpid(writer, &status, WNOHANG); + if (!pid) { + fprintf(stderr, "Hmph, HUP?\n"); + continue; + } + if (!WIFEXITED(status) || WEXITSTATUS(status) > 0) + send_sideband(1, 3, deadchild, strlen(deadchild), + LARGE_PACKET_MAX); + packet_flush(1); + break; + } + return 0; +}