Merge branch 'js/async-thread' (early part) into next

* 'js/async-thread' (early part):
  Dying in an async procedure should only exit the thread, not the process.
  Reimplement async procedures using pthreads
  Windows: more pthreads functions
  Fix signature of fcntl() compatibility dummy
  Make report() from usage.c public as vreportf() and use it.
  Modernize t5530-upload-pack-error.

Conflicts:
	http-backend.c

Makefile

@@ -989,6 +989,7 @@ ifeq ($(uname_S),Windows)
 	NO_CURL = YesPlease
 	NO_PYTHON = YesPlease
 	BLK_SHA1 = YesPlease
+	ASYNC_AS_THREAD = YesPlease
 
 	CC = compat/vcbuild/scripts/clink.pl
 	AR = compat/vcbuild/scripts/lib.pl
@@ -1040,6 +1041,7 @@ ifneq (,$(findstring MINGW,$(uname_S)))
 	NO_REGEX = YesPlease
 	NO_PYTHON = YesPlease
 	BLK_SHA1 = YesPlease
+	ASYNC_AS_THREAD = YesPlease
 	COMPAT_CFLAGS += -D__USE_MINGW_ACCESS -DNOGDI -Icompat -Icompat/fnmatch -Icompat/win32
 	COMPAT_CFLAGS += -DSTRIP_EXTENSION=\".exe\"
 	COMPAT_OBJS += compat/mingw.o compat/fnmatch/fnmatch.o compat/winansi.o \
@@ -1351,6 +1353,9 @@ ifdef NO_PTHREADS
 else
 	EXTLIBS += $(PTHREAD_LIBS)
 	LIB_OBJS += thread-utils.o
+ifdef ASYNC_AS_THREAD
+	BASIC_CFLAGS += -DASYNC_AS_THREAD
+endif
 endif
 
 ifdef DIR_HAS_BSD_GROUP_SEMANTICS

compat/mingw.h

@@ -89,7 +89,7 @@ static inline int getuid()
 { return 1; }
 static inline struct passwd *getpwnam(const char *name)
 { return NULL; }
-static inline int fcntl(int fd, int cmd, long arg)
+static inline int fcntl(int fd, int cmd, ...)
 {
 	if (cmd == F_GETFD || cmd == F_SETFD)
 		return 0;

compat/win32/pthread.c

@@ -16,6 +16,7 @@
 static unsigned __stdcall win32_start_routine(void *arg)
 {
 	pthread_t *thread = arg;
+	thread->tid = GetCurrentThreadId();
 	thread->arg = thread->start_routine(thread->arg);
 	return 0;
 }
@@ -49,6 +50,13 @@ int win32_pthread_join(pthread_t *thread, void **value_ptr)
 	}
 }
 
+pthread_t pthread_self(void)
+{
+	pthread_t t = { 0 };
+	t.tid = GetCurrentThreadId();
+	return t;
+}
+
 int pthread_cond_init(pthread_cond_t *cond, const void *unused)
 {
 	cond->waiters = 0;

compat/win32/pthread.h

@@ -52,6 +52,7 @@ typedef struct {
 	HANDLE handle;
 	void *(*start_routine)(void*);
 	void *arg;
+	DWORD tid;
 } pthread_t;
 
 extern int pthread_create(pthread_t *thread, const void *unused,
@@ -65,4 +66,28 @@ extern int pthread_create(pthread_t *thread, const void *unused,
 
 extern int win32_pthread_join(pthread_t *thread, void **value_ptr);
 
+#define pthread_equal(t1, t2) ((t1).tid == (t2).tid)
+extern pthread_t pthread_self(void);
+
+static inline int pthread_exit(void *ret)
+{
+	ExitThread((DWORD)ret);
+}
+
+typedef DWORD pthread_key_t;
+static inline int pthread_key_create(pthread_key_t *keyp, void (*destructor)(void *value))
+{
+	return (*keyp = TlsAlloc()) == TLS_OUT_OF_INDEXES ? EAGAIN : 0;
+}
+
+static inline int pthread_setspecific(pthread_key_t key, const void *value)
+{
+	return TlsSetValue(key, (void *)value) ? 0 : EINVAL;
+}
+
+static inline void *pthread_getspecific(pthread_key_t key)
+{
+	return TlsGetValue(key);
+}
+
 #endif /* PTHREAD_H */

fast-import.c

@@ -483,14 +483,12 @@ static void dump_marks(void);
 static NORETURN void die_nicely(const char *err, va_list params)
 {
 	static int zombie;
-	char message[2 * PATH_MAX];
 
-	vsnprintf(message, sizeof(message), err, params);
-	fputs("fatal: ", stderr);
-	fputs(message, stderr);
-	fputc('\n', stderr);
+	vreportf("fatal: ", err, params);
 
 	if (!zombie) {
+		char message[2 * PATH_MAX];
+
 		zombie = 1;
 		write_crash_report(message);
 		end_packfile();

git-compat-util.h

@@ -193,6 +193,7 @@ extern char *gitbasename(char *);
 #include "compat/bswap.h"
 
 /* General helper functions */
+extern void vreportf(const char *prefix, const char *err, va_list params);
 extern NORETURN void usage(const char *err);
 extern NORETURN void usagef(const char *err, ...) __attribute__((format (printf, 1, 2)));
 extern NORETURN void die(const char *err, ...) __attribute__((format (printf, 1, 2)));

http-backend.c

@@ -541,14 +541,12 @@ static NORETURN void die_webcgi(const char *err, va_list params)
 	static int dead;
 
 	if (!dead) {
-		char buffer[1000];
 		dead = 1;
-
-		vsnprintf(buffer, sizeof(buffer), err, params);
-		fprintf(stderr, "fatal: %s\n", buffer);
 		http_status(500, "Internal Server Error");
 		hdr_nocache();
 		end_headers();
+
+		vreportf("fatal: ", err, params);
 	}
 	exit(0); /* we successfully reported a failure ;-) */
 }

run-command.c

@@ -84,6 +84,7 @@ static NORETURN void die_child(const char *err, va_list params)
 	unused = write(child_err, "\n", 1);
 	exit(128);
 }
+#endif
 
 static inline void set_cloexec(int fd)
 {
@@ -91,7 +92,6 @@ static inline void set_cloexec(int fd)
 	if (flags >= 0)
 		fcntl(fd, F_SETFD, flags | FD_CLOEXEC);
 }
-#endif
 
 static int wait_or_whine(pid_t pid, const char *argv0, int silent_exec_failure)
 {
@@ -447,11 +447,35 @@ int run_command_v_opt_cd_env(const char **argv, int opt, const char *dir, const
 	return run_command(&cmd);
 }
 
-#ifdef WIN32
-static unsigned __stdcall run_thread(void *data)
+#ifdef ASYNC_AS_THREAD
+static pthread_t main_thread;
+static int main_thread_set;
+static pthread_key_t async_key;
+
+static void *run_thread(void *data)
 {
 	struct async *async = data;
-	return async->proc(async->proc_in, async->proc_out, async->data);
+
+	pthread_setspecific(async_key, async);
+
+	intptr_t ret = async->proc(async->proc_in, async->proc_out, async->data);
+	return (void *)ret;
+}
+
+static NORETURN void die_async(const char *err, va_list params)
+{
+	vreportf("fatal: ", err, params);
+
+	if (!pthread_equal(main_thread, pthread_self())) {
+		struct async *async = pthread_getspecific(async_key);
+		if (async->proc_in >= 0)
+			close(async->proc_in);
+		if (async->proc_out >= 0)
+			close(async->proc_out);
+		pthread_exit((void *)128);
+	}
+
+	exit(128);
 }
 #endif
 
@@ -497,7 +521,7 @@ int start_async(struct async *async)
 	else
 		proc_out = -1;
 
-#ifndef WIN32
+#ifndef ASYNC_AS_THREAD
 	/* Flush stdio before fork() to avoid cloning buffers */
 	fflush(NULL);
 
@@ -524,12 +548,29 @@ int start_async(struct async *async)
 	else if (async->out)
 		close(async->out);
 #else
+	if (!main_thread_set) {
+		/*
+		 * We assume that the first time that start_async is called
+		 * it is from the main thread.
+		 */
+		main_thread_set = 1;
+		main_thread = pthread_self();
+		pthread_key_create(&async_key, NULL);
+		set_die_routine(die_async);
+	}
+
+	if (proc_in >= 0)
+		set_cloexec(proc_in);
+	if (proc_out >= 0)
+		set_cloexec(proc_out);
 	async->proc_in = proc_in;
 	async->proc_out = proc_out;
-	async->tid = (HANDLE) _beginthreadex(NULL, 0, run_thread, async, 0, NULL);
-	if (!async->tid) {
-		error("cannot create thread: %s", strerror(errno));
-		goto error;
+	{
+		int err = pthread_create(&async->tid, NULL, run_thread, async);
+		if (err) {
+			error("cannot create thread: %s", strerror(err));
+			goto error;
+		}
 	}
 #endif
 	return 0;
@@ -549,17 +590,15 @@ error:
 
 int finish_async(struct async *async)
 {
-#ifndef WIN32
-	int ret = wait_or_whine(async->pid, "child process", 0);
+#ifndef ASYNC_AS_THREAD
+	return wait_or_whine(async->pid, "child process", 0);
 #else
-	DWORD ret = 0;
-	if (WaitForSingleObject(async->tid, INFINITE) != WAIT_OBJECT_0)
-		ret = error("waiting for thread failed: %lu", GetLastError());
-	else if (!GetExitCodeThread(async->tid, &ret))
-		ret = error("cannot get thread exit code: %lu", GetLastError());
-	CloseHandle(async->tid);
+	void *ret = (void *)(intptr_t)(-1);
+
+	if (pthread_join(async->tid, &ret))
+		error("pthread_join failed");
+	return (int)(intptr_t)ret;
 #endif
-	return ret;
 }
 
 int run_hook(const char *index_file, const char *name, ...)

run-command.h

@@ -1,6 +1,10 @@
 #ifndef RUN_COMMAND_H
 #define RUN_COMMAND_H
 
+#ifdef ASYNC_AS_THREAD
+#include <pthread.h>
+#endif
+
 struct child_process {
 	const char **argv;
 	pid_t pid;
@@ -74,10 +78,10 @@ struct async {
 	void *data;
 	int in;		/* caller writes here and closes it */
 	int out;	/* caller reads from here and closes it */
-#ifndef WIN32
+#ifndef ASYNC_AS_THREAD
 	pid_t pid;
 #else
-	HANDLE tid;
+	pthread_t tid;
 	int proc_in;
 	int proc_out;
 #endif

t/t5530-upload-pack-error.sh

@@ -32,9 +32,9 @@ test_expect_success 'fsck fails' '
 
 test_expect_success 'upload-pack fails due to error in pack-objects packing' '
 
-	! echo "0032want $(git rev-parse HEAD)
-00000009done
-0000" | git upload-pack . > /dev/null 2> output.err &&
+	printf "0032want %s\n00000009done\n0000" \
+		$(git rev-parse HEAD) >input &&
+	test_must_fail git upload-pack . <input >/dev/null 2>output.err &&
 	grep "unable to read" output.err &&
 	grep "pack-objects died" output.err
 '
@@ -51,9 +51,9 @@ test_expect_success 'fsck fails' '
 '
 test_expect_success 'upload-pack fails due to error in rev-list' '
 
-	! echo "0032want $(git rev-parse HEAD)
-0034shallow $(git rev-parse HEAD^)00000009done
-0000" | git upload-pack . > /dev/null 2> output.err &&
+	printf "0032want %s\n0034shallow %s00000009done\n0000" \
+		$(git rev-parse HEAD) $(git rev-parse HEAD^) >input &&
+	test_must_fail git upload-pack . <input >/dev/null 2>output.err &&
 	# pack-objects survived
 	grep "Total.*, reused" output.err &&
 	# but there was an error, which must have been in rev-list
@@ -62,9 +62,9 @@ test_expect_success 'upload-pack fails due to error in rev-list' '
 
 test_expect_success 'upload-pack fails due to error in pack-objects enumeration' '
 
-	! echo "0032want $(git rev-parse HEAD)
-00000009done
-0000" | git upload-pack . > /dev/null 2> output.err &&
+	printf "0032want %s\n00000009done\n0000" \
+		$(git rev-parse HEAD) >input &&
+	test_must_fail git upload-pack . <input >/dev/null 2>output.err &&
 	grep "bad tree object" output.err &&
 	grep "pack-objects died" output.err
 '

usage.c

@@ -5,7 +5,7 @@
  */
 #include "git-compat-util.h"
 
-static void report(const char *prefix, const char *err, va_list params)
+void vreportf(const char *prefix, const char *err, va_list params)
 {
 	char msg[4096];
 	vsnprintf(msg, sizeof(msg), err, params);
@@ -14,24 +14,24 @@ static void report(const char *prefix, const char *err, va_list params)
 
 static NORETURN void usage_builtin(const char *err, va_list params)
 {
-	report("usage: ", err, params);
+	vreportf("usage: ", err, params);
 	exit(129);
 }
 
 static NORETURN void die_builtin(const char *err, va_list params)
 {
-	report("fatal: ", err, params);
+	vreportf("fatal: ", err, params);
 	exit(128);
 }
 
 static void error_builtin(const char *err, va_list params)
 {
-	report("error: ", err, params);
+	vreportf("error: ", err, params);
 }
 
 static void warn_builtin(const char *warn, va_list params)
 {
-	report("warning: ", warn, params);
+	vreportf("warning: ", warn, params);
 }
 
 /* If we are in a dlopen()ed .so write to a global variable would segfault