Merge branch 'jc/archive' into next

* jc/archive:
  Add sideband status report to git-archive protocol
  Prepare larger packet buffer for upload-pack protocol.
  Teach --exec to git-archive --remote
  Add --verbose to git-archive
  archive: force line buffered output to stderr
  Use xstrdup instead of strdup in builtin-{tar,zip}-tree.c
  Move sideband server side support into reusable form.
  Move sideband client side support into reusable form.
  archive: allow remote to have more formats than we understand.
  get_sha1_hex() micro-optimization
This commit is contained in:
Junio C Hamano
2006-09-10 18:20:28 -07:00
12 changed files with 331 additions and 121 deletions

View File

@@ -244,7 +244,7 @@ XDIFF_LIB=xdiff/lib.a
LIB_H = \
archive.h blob.h cache.h commit.h csum-file.h delta.h \
diff.h object.h pack.h pkt-line.h quote.h refs.h list-objects.h \
diff.h object.h pack.h pkt-line.h quote.h refs.h list-objects.h sideband.h \
run-command.h strbuf.h tag.h tree.h git-compat-util.h revision.h \
tree-walk.h log-tree.h dir.h path-list.h unpack-trees.h builtin.h
@@ -256,7 +256,7 @@ DIFF_OBJS = \
LIB_OBJS = \
blob.o commit.o connect.o csum-file.o cache-tree.o base85.o \
date.o diff-delta.o entry.o exec_cmd.o ident.o lockfile.o \
object.o pack-check.o patch-delta.o path.o pkt-line.o \
object.o pack-check.o patch-delta.o path.o pkt-line.o sideband.o \
quote.o read-cache.o refs.o run-command.o dir.o object-refs.o \
server-info.o setup.o sha1_file.o sha1_name.o strbuf.o \
tag.o tree.o usage.o config.o environment.o ctype.o copy.o \

View File

@@ -10,6 +10,7 @@ struct archiver_args {
const unsigned char *commit_sha1;
time_t time;
const char **pathspec;
unsigned int verbose : 1;
void *extra;
};
@@ -19,7 +20,6 @@ typedef void *(*parse_extra_args_fn_t)(int argc, const char **argv);
struct archiver {
const char *name;
const char *remote;
struct archiver_args args;
write_archive_fn_t write_archive;
parse_extra_args_fn_t parse_extra;

View File

@@ -10,9 +10,10 @@
#include "tree-walk.h"
#include "exec_cmd.h"
#include "pkt-line.h"
#include "sideband.h"
static const char archive_usage[] = \
"git-archive --format=<fmt> [--prefix=<prefix>/] [<extra>] <tree-ish> [path...]";
"git-archive --format=<fmt> [--prefix=<prefix>/] [--verbose] [<extra>] <tree-ish> [path...]";
struct archiver archivers[] = {
{
@@ -26,22 +27,33 @@ struct archiver archivers[] = {
},
};
static int run_remote_archiver(struct archiver *ar, int argc,
static int run_remote_archiver(const char *remote, int argc,
const char **argv)
{
char *url, buf[1024];
char *url, buf[LARGE_PACKET_MAX];
int fd[2], i, len, rv;
pid_t pid;
const char *exec = "git-upload-archive";
int exec_at = 0;
sprintf(buf, "git-upload-archive");
for (i = 1; i < argc; i++) {
const char *arg = argv[i];
if (!strncmp("--exec=", arg, 7)) {
if (exec_at)
die("multiple --exec specified");
exec = arg + 7;
exec_at = i;
break;
}
}
url = xstrdup(ar->remote);
pid = git_connect(fd, url, buf);
url = xstrdup(remote);
pid = git_connect(fd, url, exec);
if (pid < 0)
return pid;
for (i = 1; i < argc; i++) {
if (!strncmp(argv[i], "--remote=", 9))
if (i == exec_at)
continue;
packet_write(fd[1], "argument %s\n", argv[i]);
}
@@ -63,8 +75,7 @@ static int run_remote_archiver(struct archiver *ar, int argc,
die("git-archive: expected a flush");
/* Now, start reading from fd[0] and spit it out to stdout */
rv = copy_fd(fd[0], 1);
rv = recv_sideband("archive", fd[0], 1, 2, buf, sizeof(buf));
close(fd[0]);
rv |= finish_connect(pid);
@@ -150,16 +161,20 @@ int parse_archive_args(int argc, const char **argv, struct archiver *ar)
const char *extra_argv[MAX_EXTRA_ARGS];
int extra_argc = 0;
const char *format = NULL; /* might want to default to "tar" */
const char *remote = NULL;
const char *base = "";
int list = 0;
int verbose = 0;
int i;
for (i = 1; i < argc; i++) {
const char *arg = argv[i];
if (!strcmp(arg, "--list") || !strcmp(arg, "-l")) {
list = 1;
for (i = 0; i < ARRAY_SIZE(archivers); i++)
printf("%s\n", archivers[i].name);
exit(0);
}
if (!strcmp(arg, "--verbose") || !strcmp(arg, "-v")) {
verbose = 1;
continue;
}
if (!strncmp(arg, "--format=", 9)) {
@@ -170,10 +185,6 @@ int parse_archive_args(int argc, const char **argv, struct archiver *ar)
base = arg + 9;
continue;
}
if (!strncmp(arg, "--remote=", 9)) {
remote = arg + 9;
continue;
}
if (!strcmp(arg, "--")) {
i++;
break;
@@ -187,44 +198,70 @@ int parse_archive_args(int argc, const char **argv, struct archiver *ar)
break;
}
if (list) {
if (!remote) {
for (i = 0; i < ARRAY_SIZE(archivers); i++)
printf("%s\n", archivers[i].name);
exit(0);
}
die("--list and --remote are mutually exclusive");
}
if (argc - i < 1)
/* We need at least one parameter -- tree-ish */
if (argc - 1 < i)
usage(archive_usage);
if (!format)
die("You must specify an archive format");
if (init_archiver(format, ar) < 0)
die("Unknown archive format '%s'", format);
if (extra_argc && !remote) {
if (!ar->parse_extra) {
if (extra_argc) {
if (!ar->parse_extra)
die("%s", default_parse_extra(ar, extra_argv));
}
ar->args.extra = ar->parse_extra(extra_argc, extra_argv);
}
ar->remote = remote;
ar->args.verbose = verbose;
ar->args.base = base;
return i;
}
static const char *remote_request(int *ac, const char **av)
{
int ix, iy, cnt = *ac;
int no_more_options = 0;
const char *remote = NULL;
for (ix = iy = 1; ix < cnt; ix++) {
const char *arg = av[ix];
if (!strcmp(arg, "--"))
no_more_options = 1;
if (!no_more_options) {
if (!strncmp(arg, "--remote=", 9)) {
if (remote)
die("Multiple --remote specified");
remote = arg + 9;
continue;
}
if (arg[0] != '-')
no_more_options = 1;
}
if (ix != iy)
av[iy] = arg;
iy++;
}
if (remote) {
av[--cnt] = NULL;
*ac = cnt;
}
return remote;
}
int cmd_archive(int argc, const char **argv, const char *prefix)
{
struct archiver ar;
int tree_idx;
const char *remote = NULL;
remote = remote_request(&argc, argv);
if (remote)
return run_remote_archiver(remote, argc, argv);
setlinebuf(stderr);
memset(&ar, 0, sizeof(ar));
tree_idx = parse_archive_args(argc, argv, &ar);
if (ar.remote)
return run_remote_archiver(&ar, argc, argv);
if (prefix == NULL)
prefix = setup_git_directory();

View File

@@ -22,6 +22,7 @@ static unsigned long offset;
static time_t archive_time;
static int tar_umask;
static int verbose;
/* writes out the whole block, but only if it is full */
static void write_if_needed(void)
@@ -169,6 +170,8 @@ static void write_entry(const unsigned char *sha1, struct strbuf *path,
mode = 0100666;
sprintf(header.name, "%s.paxheader", sha1_to_hex(sha1));
} else {
if (verbose)
fprintf(stderr, "%.*s\n", path->len, path->buf);
if (S_ISDIR(mode)) {
*header.typeflag = TYPEFLAG_DIR;
mode = (mode | 0777) & ~tar_umask;
@@ -385,12 +388,13 @@ int write_tar_archive(struct archiver_args *args)
git_config(git_tar_config);
archive_time = args->time;
verbose = args->verbose;
if (args->commit_sha1)
write_global_extended_header(args->commit_sha1);
if (args->base && plen > 0 && args->base[plen - 1] == '/') {
char *base = strdup(args->base);
char *base = xstrdup(args->base);
int baselen = strlen(base);
while (baselen > 0 && base[baselen - 1] == '/')

View File

@@ -6,12 +6,18 @@
#include "builtin.h"
#include "archive.h"
#include "pkt-line.h"
#include "sideband.h"
#include <sys/wait.h>
#include <sys/poll.h>
static const char upload_archive_usage[] =
"git-upload-archive <repo>";
static const char deadchild[] =
"git-upload-archive: archiver died with error";
int cmd_upload_archive(int argc, const char **argv, const char *prefix)
static int run_upload_archive(int argc, const char **argv, const char *prefix)
{
struct archiver ar;
const char *sent_argv[MAX_ARGS];
@@ -64,9 +70,89 @@ int cmd_upload_archive(int argc, const char **argv, const char *prefix)
parse_treeish_arg(sent_argv + treeish_idx, &ar.args, prefix);
parse_pathspec_arg(sent_argv + treeish_idx + 1, &ar.args);
packet_write(1, "ACK\n");
packet_flush(1);
return ar.write_archive(&ar.args);
}
int cmd_upload_archive(int argc, const char **argv, const char *prefix)
{
pid_t writer;
int fd1[2], fd2[2];
/*
* Set up sideband subprocess.
*
* We (parent) monitor and read from child, sending its fd#1 and fd#2
* multiplexed out to our fd#1. If the child dies, we tell the other
* end over channel #3.
*/
if (pipe(fd1) < 0 || pipe(fd2) < 0) {
int err = errno;
packet_write(1, "NACK pipe failed on the remote side\n");
die("upload-archive: %s", strerror(err));
}
writer = fork();
if (writer < 0) {
int err = errno;
packet_write(1, "NACK fork failed on the remote side\n");
die("upload-archive: %s", strerror(err));
}
if (!writer) {
/* child - connect fd#1 and fd#2 to the pipe */
dup2(fd1[1], 1);
dup2(fd2[1], 2);
close(fd1[1]); close(fd2[1]);
close(fd1[0]); close(fd2[0]); /* we do not read from pipe */
exit(run_upload_archive(argc, argv, prefix));
}
/* parent - read from child, multiplex and send out to fd#1 */
close(fd1[1]); close(fd2[1]); /* we do not write to pipe */
packet_write(1, "ACK\n");
packet_flush(1);
while (1) {
struct pollfd pfd[2];
char buf[16384];
ssize_t sz;
pid_t pid;
int status;
pfd[0].fd = fd1[0];
pfd[0].events = POLLIN;
pfd[1].fd = fd2[0];
pfd[1].events = POLLIN;
if (poll(pfd, 2, -1) < 0) {
if (errno != EINTR) {
error("poll failed resuming: %s",
strerror(errno));
sleep(1);
}
continue;
}
if (pfd[0].revents & (POLLIN|POLLHUP)) {
/* Data stream ready */
sz = read(pfd[0].fd, buf, sizeof(buf));
send_sideband(1, 1, buf, sz, LARGE_PACKET_MAX);
}
if (pfd[1].revents & (POLLIN|POLLHUP)) {
/* Status stream ready */
sz = read(pfd[1].fd, buf, sizeof(buf));
send_sideband(1, 2, buf, sz, LARGE_PACKET_MAX);
}
if (((pfd[0].revents | pfd[1].revents) & POLLHUP) == 0)
continue;
/* did it die? */
pid = waitpid(writer, &status, WNOHANG);
if (!pid) {
fprintf(stderr, "Hmph, HUP?\n");
continue;
}
if (!WIFEXITED(status) || WEXITSTATUS(status) > 0)
send_sideband(1, 3, deadchild, strlen(deadchild),
LARGE_PACKET_MAX);
packet_flush(1);
break;
}
return 0;
}

View File

@@ -13,6 +13,7 @@
static const char zip_tree_usage[] =
"git-zip-tree [-0|...|-9] <tree-ish> [ <base> ]";
static int verbose;
static int zip_date;
static int zip_time;
@@ -164,6 +165,8 @@ static int write_zip_entry(const unsigned char *sha1,
crc = crc32(0, Z_NULL, 0);
path = construct_path(base, baselen, filename, S_ISDIR(mode), &pathlen);
if (verbose)
fprintf(stderr, "%s\n", path);
if (pathlen > 0xffff) {
error("path too long (%d chars, SHA1: %s): %s", pathlen,
sha1_to_hex(sha1), path);
@@ -361,9 +364,10 @@ int write_zip_archive(struct archiver_args *args)
zip_dir = xmalloc(ZIP_DIRECTORY_MIN_SIZE);
zip_dir_size = ZIP_DIRECTORY_MIN_SIZE;
verbose = args->verbose;
if (args->base && plen > 0 && args->base[plen - 1] == '/') {
char *base = strdup(args->base);
char *base = xstrdup(args->base);
int baselen = strlen(base);
while (baselen > 0 && base[baselen - 1] == '/')

View File

@@ -1,6 +1,7 @@
#include "cache.h"
#include "exec_cmd.h"
#include "pkt-line.h"
#include "sideband.h"
#include <sys/wait.h>
#include <sys/time.h>
@@ -114,36 +115,13 @@ static pid_t setup_sideband(int sideband, const char *me, int fd[2], int xd[2])
die("%s: unable to fork off sideband demultiplexer", me);
if (!side_pid) {
/* subprocess */
char buf[LARGE_PACKET_MAX];
close(fd[0]);
if (xd[0] != xd[1])
close(xd[1]);
while (1) {
char buf[1024];
int len = packet_read_line(xd[0], buf, sizeof(buf));
if (len == 0)
break;
if (len < 1)
die("%s: protocol error: no band designator",
me);
len--;
switch (buf[0] & 0xFF) {
case 3:
safe_write(2, "remote: ", 8);
safe_write(2, buf+1, len);
safe_write(2, "\n", 1);
exit(1);
case 2:
safe_write(2, "remote: ", 8);
safe_write(2, buf+1, len);
continue;
case 1:
safe_write(fd[1], buf+1, len);
continue;
default:
die("%s: protocol error: bad band #%d",
me, (buf[0] & 0xFF));
}
}
if (recv_sideband(me, xd[0], fd[1], 2, buf, sizeof(buf)))
exit(1);
exit(0);
}
close(xd[0]);

View File

@@ -166,10 +166,11 @@ static int find_common(int fd[2], unsigned char *result_sha1,
}
if (!fetching)
packet_write(fd[1], "want %s%s%s%s\n",
packet_write(fd[1], "want %s%s%s%s%s\n",
sha1_to_hex(remote),
(multi_ack ? " multi_ack" : ""),
(use_sideband ? " side-band" : ""),
(use_sideband == 2 ? " side-band-64k" : ""),
(use_sideband == 1 ? " side-band" : ""),
(use_thin_pack ? " thin-pack" : ""));
else
packet_write(fd[1], "want %s\n", sha1_to_hex(remote));
@@ -426,7 +427,12 @@ static int fetch_pack(int fd[2], int nr_match, char **match)
fprintf(stderr, "Server supports multi_ack\n");
multi_ack = 1;
}
if (server_supports("side-band")) {
if (server_supports("side-band-64k")) {
if (verbose)
fprintf(stderr, "Server supports side-band-64k\n");
use_sideband = 2;
}
else if (server_supports("side-band")) {
if (verbose)
fprintf(stderr, "Server supports side-band\n");
use_sideband = 1;

View File

@@ -26,15 +26,43 @@ const unsigned char null_sha1[20];
static unsigned int sha1_file_open_flag = O_NOATIME;
static unsigned hexval(char c)
static inline unsigned int hexval(unsigned int c)
{
if (c >= '0' && c <= '9')
return c - '0';
if (c >= 'a' && c <= 'f')
return c - 'a' + 10;
if (c >= 'A' && c <= 'F')
return c - 'A' + 10;
return ~0;
static signed char val[256] = {
-1, -1, -1, -1, -1, -1, -1, -1, /* 00-07 */
-1, -1, -1, -1, -1, -1, -1, -1, /* 08-0f */
-1, -1, -1, -1, -1, -1, -1, -1, /* 10-17 */
-1, -1, -1, -1, -1, -1, -1, -1, /* 18-1f */
-1, -1, -1, -1, -1, -1, -1, -1, /* 20-27 */
-1, -1, -1, -1, -1, -1, -1, -1, /* 28-2f */
0, 1, 2, 3, 4, 5, 6, 7, /* 30-37 */
8, 9, -1, -1, -1, -1, -1, -1, /* 38-3f */
-1, 10, 11, 12, 13, 14, 15, -1, /* 40-47 */
-1, -1, -1, -1, -1, -1, -1, -1, /* 48-4f */
-1, -1, -1, -1, -1, -1, -1, -1, /* 50-57 */
-1, -1, -1, -1, -1, -1, -1, -1, /* 58-5f */
-1, 10, 11, 12, 13, 14, 15, -1, /* 60-67 */
-1, -1, -1, -1, -1, -1, -1, -1, /* 68-67 */
-1, -1, -1, -1, -1, -1, -1, -1, /* 70-77 */
-1, -1, -1, -1, -1, -1, -1, -1, /* 78-7f */
-1, -1, -1, -1, -1, -1, -1, -1, /* 80-87 */
-1, -1, -1, -1, -1, -1, -1, -1, /* 88-8f */
-1, -1, -1, -1, -1, -1, -1, -1, /* 90-97 */
-1, -1, -1, -1, -1, -1, -1, -1, /* 98-9f */
-1, -1, -1, -1, -1, -1, -1, -1, /* a0-a7 */
-1, -1, -1, -1, -1, -1, -1, -1, /* a8-af */
-1, -1, -1, -1, -1, -1, -1, -1, /* b0-b7 */
-1, -1, -1, -1, -1, -1, -1, -1, /* b8-bf */
-1, -1, -1, -1, -1, -1, -1, -1, /* c0-c7 */
-1, -1, -1, -1, -1, -1, -1, -1, /* c8-cf */
-1, -1, -1, -1, -1, -1, -1, -1, /* d0-d7 */
-1, -1, -1, -1, -1, -1, -1, -1, /* d8-df */
-1, -1, -1, -1, -1, -1, -1, -1, /* e0-e7 */
-1, -1, -1, -1, -1, -1, -1, -1, /* e8-ef */
-1, -1, -1, -1, -1, -1, -1, -1, /* f0-f7 */
-1, -1, -1, -1, -1, -1, -1, -1, /* f8-ff */
};
return val[c];
}
int get_sha1_hex(const char *hex, unsigned char *sha1)

74
sideband.c Normal file
View File

@@ -0,0 +1,74 @@
#include "pkt-line.h"
#include "sideband.h"
/*
* Receive multiplexed output stream over git native protocol.
* in_stream is the input stream from the remote, which carries data
* in pkt_line format with band designator. Demultiplex it into out
* and err and return error appropriately. Band #1 carries the
* primary payload. Things coming over band #2 is not necessarily
* error; they are usually informative message on the standard error
* stream, aka "verbose"). A message over band #3 is a signal that
* the remote died unexpectedly. A flush() concludes the stream.
*/
int recv_sideband(const char *me, int in_stream, int out, int err, char *buf, int bufsz)
{
while (1) {
int len = packet_read_line(in_stream, buf, bufsz);
if (len == 0)
break;
if (len < 1) {
len = sprintf(buf, "%s: protocol error: no band designator\n", me);
safe_write(err, buf, len);
return SIDEBAND_PROTOCOL_ERROR;
}
len--;
switch (buf[0] & 0xFF) {
case 3:
safe_write(err, "remote: ", 8);
safe_write(err, buf+1, len);
safe_write(err, "\n", 1);
return SIDEBAND_REMOTE_ERROR;
case 2:
safe_write(err, "remote: ", 8);
safe_write(err, buf+1, len);
continue;
case 1:
safe_write(out, buf+1, len);
continue;
default:
len = sprintf(buf + 1,
"%s: protocol error: bad band #%d\n",
me, buf[0] & 0xFF);
safe_write(err, buf+1, len);
return SIDEBAND_PROTOCOL_ERROR;
}
}
return 0;
}
/*
* fd is connected to the remote side; send the sideband data
* over multiplexed packet stream.
*/
ssize_t send_sideband(int fd, int band, const char *data, ssize_t sz, int packet_max)
{
ssize_t ssz = sz;
const char *p = data;
while (sz) {
unsigned n;
char hdr[5];
n = sz;
if (packet_max - 5 < n)
n = packet_max - 5;
sprintf(hdr, "%04x", n + 5);
hdr[4] = band;
safe_write(fd, hdr, 5);
safe_write(fd, p, n);
p += n;
sz -= n;
}
return ssz;
}

13
sideband.h Normal file
View File

@@ -0,0 +1,13 @@
#ifndef SIDEBAND_H
#define SIDEBAND_H
#define SIDEBAND_PROTOCOL_ERROR -2
#define SIDEBAND_REMOTE_ERROR -1
#define DEFAULT_PACKET_MAX 1000
#define LARGE_PACKET_MAX 65520
int recv_sideband(const char *me, int in_stream, int out, int err, char *, int);
ssize_t send_sideband(int fd, int band, const char *data, ssize_t sz, int packet_max);
#endif

View File

@@ -4,6 +4,7 @@
#include "cache.h"
#include "refs.h"
#include "pkt-line.h"
#include "sideband.h"
#include "tag.h"
#include "object.h"
#include "commit.h"
@@ -25,6 +26,9 @@ static int use_thin_pack;
static struct object_array have_obj;
static struct object_array want_obj;
static unsigned int timeout;
/* 0 for no sideband,
* otherwise maximum packet size (up to 65520 bytes).
*/
static int use_sideband;
static void reset_timeout(void)
@@ -39,45 +43,18 @@ static int strip(char *line, int len)
return len;
}
#define PACKET_MAX 1000
static ssize_t send_client_data(int fd, const char *data, ssize_t sz)
{
ssize_t ssz;
const char *p;
if (!data) {
if (!use_sideband)
return 0;
packet_flush(1);
if (use_sideband)
return send_sideband(1, fd, data, sz, use_sideband);
if (fd == 3)
/* emergency quit */
fd = 2;
if (fd == 2) {
xwrite(fd, data, sz);
return sz;
}
if (!use_sideband) {
if (fd == 3)
/* emergency quit */
fd = 2;
if (fd == 2) {
xwrite(fd, data, sz);
return sz;
}
return safe_write(fd, data, sz);
}
p = data;
ssz = sz;
while (sz) {
unsigned n;
char hdr[5];
n = sz;
if (PACKET_MAX - 5 < n)
n = PACKET_MAX - 5;
sprintf(hdr, "%04x", n + 5);
hdr[4] = fd;
safe_write(1, hdr, 5);
safe_write(1, p, n);
p += n;
sz -= n;
}
return ssz;
return safe_write(fd, data, sz);
}
static void create_pack_file(void)
@@ -314,7 +291,8 @@ static void create_pack_file(void)
goto fail;
fprintf(stderr, "flushed.\n");
}
send_client_data(1, NULL, 0);
if (use_sideband)
packet_flush(1);
return;
}
fail:
@@ -498,8 +476,10 @@ static void receive_needs(void)
multi_ack = 1;
if (strstr(line+45, "thin-pack"))
use_thin_pack = 1;
if (strstr(line+45, "side-band"))
use_sideband = 1;
if (strstr(line+45, "side-band-64k"))
use_sideband = LARGE_PACKET_MAX;
else if (strstr(line+45, "side-band"))
use_sideband = DEFAULT_PACKET_MAX;
/* We have sent all our refs already, and the other end
* should have chosen out of them; otherwise they are
@@ -521,7 +501,7 @@ static void receive_needs(void)
static int send_ref(const char *refname, const unsigned char *sha1)
{
static const char *capabilities = "multi_ack thin-pack side-band";
static const char *capabilities = "multi_ack thin-pack side-band side-band-64k";
struct object *o = parse_object(sha1);
if (!o)