From b0e7870546a142869ebc626cceb0ca251839f09f Mon Sep 17 00:00:00 2001 From: Sebastian Schuberth Date: Wed, 4 Sep 2013 18:18:49 +0200 Subject: [PATCH 01/41] Makefile: Set htmldir to match the default HTML docs location under MSYS Signed-off-by: Sebastian Schuberth --- config.mak.uname | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.mak.uname b/config.mak.uname index 08193cd740..9c7ead65fc 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -615,7 +615,7 @@ ifneq (,$(findstring MINGW,$(uname_S))) NATIVE_CRLF = YesPlease X = .exe ifneq (,$(wildcard ../THIS_IS_MSYSGIT)) - htmldir = doc/git/html/ + htmldir = share/doc/git/$(firstword $(subst -, ,$(GIT_VERSION)))/html prefix = INSTALL = /bin/install EXTLIBS += /mingw/lib/libz.a From 5a6e755d790278a076ff184232ea623bb3345519 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 20 Feb 2015 09:52:07 +0000 Subject: [PATCH 02/41] Help debugging with MSys2 by optionally executing bash with strace MSys2's strace facility is very useful for debugging... With this patch, the bash will be executed through strace if the environment variable GIT_STRACE_COMMANDS is set, which comes in real handy when investigating issues in the test suite. Also support passing a path to a log file via GIT_STRACE_COMMANDS to force Git to call strace.exe with the `-o ` argument, i.e. to log into a file rather than print the log directly. That comes in handy when the output would otherwise misinterpreted by a calling process as part of Git's output. Note: the values "1", "yes" or "true" are *not* specifying paths, but tell Git to let strace.exe log directly to the console. Signed-off-by: Johannes Schindelin --- compat/mingw.c | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/compat/mingw.c b/compat/mingw.c index 9b01a4f868..e041be6db8 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -1437,6 +1437,7 @@ static pid_t mingw_spawnve_fd(const char *cmd, const char **argv, char **deltaen HANDLE cons; const char *(*quote_arg)(const char *arg) = is_msys2_sh(*argv) ? quote_arg_msys2 : quote_arg_msvc; + const char *strace_env; do_unset_environment_variables(); @@ -1494,6 +1495,31 @@ static pid_t mingw_spawnve_fd(const char *cmd, const char **argv, char **deltaen free(quoted); } + strace_env = getenv("GIT_STRACE_COMMANDS"); + if (strace_env) { + char *p = path_lookup("strace.exe", 1); + if (!p) + return error("strace not found!"); + if (xutftowcs_path(wcmd, p) < 0) { + free(p); + return -1; + } + free(p); + if (!strcmp("1", strace_env) || + !strcasecmp("yes", strace_env) || + !strcasecmp("true", strace_env)) + strbuf_insert(&args, 0, "strace ", 7); + else { + const char *quoted = quote_arg(strace_env); + struct strbuf buf = STRBUF_INIT; + strbuf_addf(&buf, "strace -o %s ", quoted); + if (quoted != strace_env) + free((char *)quoted); + strbuf_insert(&args, 0, buf.buf, buf.len); + strbuf_release(&buf); + } + } + ALLOC_ARRAY(wargs, st_add(st_mult(2, args.len), 1)); xutftowcs(wargs, args.buf, 2 * args.len + 1); strbuf_release(&args); From 56e85d91b1b86da320eabee84970f6c049c3c628 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 16 Feb 2015 14:06:59 +0100 Subject: [PATCH 03/41] Build Python stuff with MSys2 Signed-off-by: Johannes Schindelin --- config.mak.uname | 1 + 1 file changed, 1 insertion(+) diff --git a/config.mak.uname b/config.mak.uname index 9c7ead65fc..c112af1282 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -658,6 +658,7 @@ else NO_LIBPCRE1_JIT = UnfortunatelyYes NO_CURL = USE_NED_ALLOCATOR = YesPlease + NO_PYTHON = else COMPAT_CFLAGS += -D__USE_MINGW_ANSI_STDIO NO_CURL = YesPlease From bcd9be027547802fa535ee5d03c7e21219227d07 Mon Sep 17 00:00:00 2001 From: Cesar Eduardo Barros Date: Mon, 9 Mar 2015 08:51:38 +0100 Subject: [PATCH 04/41] mingw: Embed a manifest to trick UAC into Doing The Right Thing On Windows >= Vista, not having an application manifest with a requestedExecutionLevel can cause several kinds of confusing behavior. The first and more obvious behavior is "Installer Detection", where Windows sometimes decides (by looking at things like the file name and even sequences of bytes within the executable) that an executable is an installer and should run elevated (causing the well-known popup dialog to appear). In Git's context, subcommands such as "git patch-id" or "git update-index" fall prey to this behavior. The second and more confusing behavior is "File Virtualization". It means that when files are written without having write permission, it does not fail (as expected), but they are instead redirected to somewhere else. When the files are read, the original contents are returned, though, not the ones that were just written somewhere else. Even more confusing, not all write accesses are redirected; Trying to write to write-protected .exe files, for example, will fail instead of redirecting. In addition to being unwanted behavior, File Virtualization causes dramatic slowdowns in Git (see for instance http://code.google.com/p/msysgit/issues/detail?id=320). There are two ways to prevent those two behaviors: Either you embed an application manifest within all your executables, or you add an external manifest (a file with the same name followed by .manifest) to all your executables. Since Git's builtins are hardlinked (or copied), it is simpler and more robust to embed a manifest. A recent enough MSVC compiler should already embed a working internal manifest, but for MinGW you have to do so by hand. Very lightly tested on Wine, where like on Windows XP it should not make any difference. References: - New UAC Technologies for Windows Vista http://msdn.microsoft.com/en-us/library/bb756960.aspx - Create and Embed an Application Manifest (UAC) http://msdn.microsoft.com/en-us/library/bb756929.aspx [js: simplified the embedding dramatically by reusing Git for Windows' existing Windows resource file, removed the optional (and dubious) processorArchitecture attribute of the manifest's assemblyIdentity section.] Signed-off-by: Cesar Eduardo Barros Signed-off-by: Johannes Schindelin --- compat/win32/git.manifest | 25 +++++++++++++++++++++++++ git.rc | 2 ++ 2 files changed, 27 insertions(+) create mode 100644 compat/win32/git.manifest diff --git a/compat/win32/git.manifest b/compat/win32/git.manifest new file mode 100644 index 0000000000..771e3cce43 --- /dev/null +++ b/compat/win32/git.manifest @@ -0,0 +1,25 @@ + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/git.rc b/git.rc index 49002e0d54..cc3fdc6cc6 100644 --- a/git.rc +++ b/git.rc @@ -20,3 +20,5 @@ BEGIN VALUE "Translation", 0x409, 1200 END END + +1 RT_MANIFEST "compat/win32/git.manifest" From 1d5bef2874eb89dc9c3ad9f1af0610bd5461459c Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 2 Nov 2015 08:41:10 +0100 Subject: [PATCH 05/41] mingw: enable stack smashing protector As suggested privately to Brendan Forster by some unnamed person (suggestion for the future: use the public mailing list, or even the public GitHub issue tracker, that is a much better place to offer such suggestions), we should make use of gcc's stack smashing protector that helps detect stack buffer overruns early. Rather than using -fstack-protector, we use -fstack-protector-strong because it strikes a better balance between how much code is affected and the performance impact. In a local test (time git log --grep=is -p), best of 5 timings went from 23.009s to 22.997s (i.e. the performance impact was *well* lost in the noise). This fixes https://github.com/git-for-windows/git/issues/501 Signed-off-by: Johannes Schindelin --- config.mak.uname | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config.mak.uname b/config.mak.uname index c112af1282..55f0d50bf8 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -646,7 +646,8 @@ else BASIC_LDFLAGS += -Wl,--large-address-aware endif CC = gcc - COMPAT_CFLAGS += -D__USE_MINGW_ANSI_STDIO=0 -DDETECT_MSYS_TTY + COMPAT_CFLAGS += -D__USE_MINGW_ANSI_STDIO=0 -DDETECT_MSYS_TTY \ + -fstack-protector-strong EXTLIBS += -lntdll INSTALL = /bin/install NO_R_TO_GCC_LINKER = YesPlease From 319fc293adcf2ef8268994b00566e7c01a278eba Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 6 Apr 2015 10:37:04 +0100 Subject: [PATCH 06/41] Avoid illegal filenames when building Documentation on NTFS A '+' is not a valid part of a filename with Windows file systems (it is reserved because the '+' operator meant file concatenation back in the DOS days). Let's just not use it. Signed-off-by: Johannes Schindelin --- Documentation/Makefile | 88 +++++++++++++++++++++--------------------- 1 file changed, 44 insertions(+), 44 deletions(-) diff --git a/Documentation/Makefile b/Documentation/Makefile index 26a2342bea..fdd4a4b410 100644 --- a/Documentation/Makefile +++ b/Documentation/Makefile @@ -294,9 +294,9 @@ docdep_prereqs = \ cmd-list.made $(cmds_txt) doc.dep : $(docdep_prereqs) $(wildcard *.txt) $(wildcard config/*.txt) build-docdep.perl - $(QUIET_GEN)$(RM) $@+ $@ && \ - $(PERL_PATH) ./build-docdep.perl >$@+ $(QUIET_STDERR) && \ - mv $@+ $@ + $(QUIET_GEN)$(RM) $@.new $@ && \ + $(PERL_PATH) ./build-docdep.perl >$@.new $(QUIET_STDERR) && \ + mv $@.new $@ -include doc.dep @@ -332,8 +332,8 @@ mergetools-list.made: ../git-mergetool--lib.sh $(wildcard ../mergetools/*) date >$@ clean: - $(RM) *.xml *.xml+ *.html *.html+ *.1 *.5 *.7 - $(RM) *.texi *.texi+ *.texi++ git.info gitman.info + $(RM) *.xml *.xml.new *.html *.html.new *.1 *.5 *.7 + $(RM) *.texi *.texi.new *.texi.new.new git.info gitman.info $(RM) *.pdf $(RM) howto-index.txt howto/*.html doc.dep $(RM) technical/*.html technical/api-index.txt @@ -342,14 +342,14 @@ clean: $(RM) manpage-base-url.xsl $(MAN_HTML): %.html : %.txt asciidoc.conf - $(QUIET_ASCIIDOC)$(RM) $@+ $@ && \ - $(TXT_TO_HTML) -d manpage -o $@+ $< && \ - mv $@+ $@ + $(QUIET_ASCIIDOC)$(RM) $@.new $@ && \ + $(TXT_TO_HTML) -d manpage -o $@.new $< && \ + mv $@.new $@ $(OBSOLETE_HTML): %.html : %.txto asciidoc.conf - $(QUIET_ASCIIDOC)$(RM) $@+ $@ && \ - $(TXT_TO_HTML) -o $@+ $< && \ - mv $@+ $@ + $(QUIET_ASCIIDOC)$(RM) $@.new $@ && \ + $(TXT_TO_HTML) -o $@.new $< && \ + mv $@.new $@ manpage-base-url.xsl: manpage-base-url.xsl.in $(QUIET_GEN)sed "s|@@MAN_BASE_URL@@|$(MAN_BASE_URL)|" $< > $@ @@ -359,14 +359,14 @@ manpage-base-url.xsl: manpage-base-url.xsl.in $(XMLTO) -m $(MANPAGE_XSL) $(XMLTO_EXTRA) man $< %.xml : %.txt asciidoc.conf - $(QUIET_ASCIIDOC)$(RM) $@+ $@ && \ - $(TXT_TO_XML) -d manpage -o $@+ $< && \ - mv $@+ $@ + $(QUIET_ASCIIDOC)$(RM) $@.new $@ && \ + $(TXT_TO_XML) -d manpage -o $@.new $< && \ + mv $@.new $@ user-manual.xml: user-manual.txt user-manual.conf - $(QUIET_ASCIIDOC)$(RM) $@+ $@ && \ - $(TXT_TO_XML) -d book -o $@+ $< && \ - mv $@+ $@ + $(QUIET_ASCIIDOC)$(RM) $@.new $@ && \ + $(TXT_TO_XML) -d book -o $@.new $< && \ + mv $@.new $@ technical/api-index.txt: technical/api-index-skel.txt \ technical/api-index.sh $(patsubst %,%.txt,$(API_DOCS)) @@ -383,46 +383,46 @@ XSLT = docbook.xsl XSLTOPTS = --xinclude --stringparam html.stylesheet docbook-xsl.css user-manual.html: user-manual.xml $(XSLT) - $(QUIET_XSLTPROC)$(RM) $@+ $@ && \ - xsltproc $(XSLTOPTS) -o $@+ $(XSLT) $< && \ - mv $@+ $@ + $(QUIET_XSLTPROC)$(RM) $@.new $@ && \ + xsltproc $(XSLTOPTS) -o $@.new $(XSLT) $< && \ + mv $@.new $@ git.info: user-manual.texi $(QUIET_MAKEINFO)$(MAKEINFO) --no-split -o $@ user-manual.texi user-manual.texi: user-manual.xml - $(QUIET_DB2TEXI)$(RM) $@+ $@ && \ - $(DOCBOOK2X_TEXI) user-manual.xml --encoding=UTF-8 --to-stdout >$@++ && \ - $(PERL_PATH) fix-texi.perl <$@++ >$@+ && \ - rm $@++ && \ - mv $@+ $@ + $(QUIET_DB2TEXI)$(RM) $@.new $@ && \ + $(DOCBOOK2X_TEXI) user-manual.xml --encoding=UTF-8 --to-stdout >$@.new.new && \ + $(PERL_PATH) fix-texi.perl <$@.new.new >$@.new && \ + rm $@.new.new && \ + mv $@.new $@ user-manual.pdf: user-manual.xml - $(QUIET_DBLATEX)$(RM) $@+ $@ && \ - $(DBLATEX) -o $@+ $(DBLATEX_COMMON) $< && \ - mv $@+ $@ + $(QUIET_DBLATEX)$(RM) $@.new $@ && \ + $(DBLATEX) -o $@.new $(DBLATEX_COMMON) $< && \ + mv $@.new $@ gitman.texi: $(MAN_XML) cat-texi.perl texi.xsl - $(QUIET_DB2TEXI)$(RM) $@+ $@ && \ - ($(foreach xml,$(sort $(MAN_XML)),xsltproc -o $(xml)+ texi.xsl $(xml) && \ - $(DOCBOOK2X_TEXI) --encoding=UTF-8 --to-stdout $(xml)+ && \ - rm $(xml)+ &&) true) > $@++ && \ - $(PERL_PATH) cat-texi.perl $@ <$@++ >$@+ && \ - rm $@++ && \ - mv $@+ $@ + $(QUIET_DB2TEXI)$(RM) $@.new $@ && \ + ($(foreach xml,$(sort $(MAN_XML)),xsltproc -o $(xml).new texi.xsl $(xml) && \ + $(DOCBOOK2X_TEXI) --encoding=UTF-8 --to-stdout $(xml).new && \ + rm $(xml).new &&) true) > $@.new.new && \ + $(PERL_PATH) cat-texi.perl $@ <$@.new.new >$@.new && \ + rm $@.new.new && \ + mv $@.new $@ gitman.info: gitman.texi $(QUIET_MAKEINFO)$(MAKEINFO) --no-split --no-validate $*.texi $(patsubst %.txt,%.texi,$(MAN_TXT)): %.texi : %.xml - $(QUIET_DB2TEXI)$(RM) $@+ $@ && \ - $(DOCBOOK2X_TEXI) --to-stdout $*.xml >$@+ && \ - mv $@+ $@ + $(QUIET_DB2TEXI)$(RM) $@.new $@ && \ + $(DOCBOOK2X_TEXI) --to-stdout $*.xml >$@.new && \ + mv $@.new $@ howto-index.txt: howto-index.sh $(wildcard howto/*.txt) - $(QUIET_GEN)$(RM) $@+ $@ && \ - '$(SHELL_PATH_SQ)' ./howto-index.sh $(sort $(wildcard howto/*.txt)) >$@+ && \ - mv $@+ $@ + $(QUIET_GEN)$(RM) $@.new $@ && \ + '$(SHELL_PATH_SQ)' ./howto-index.sh $(sort $(wildcard howto/*.txt)) >$@.new && \ + mv $@.new $@ $(patsubst %,%.html,$(ARTICLES)) : %.html : %.txt $(QUIET_ASCIIDOC)$(TXT_TO_HTML) $*.txt @@ -431,10 +431,10 @@ WEBDOC_DEST = /pub/software/scm/git/docs howto/%.html: ASCIIDOC_EXTRA += -a git-relative-html-prefix=../ $(patsubst %.txt,%.html,$(wildcard howto/*.txt)): %.html : %.txt - $(QUIET_ASCIIDOC)$(RM) $@+ $@ && \ + $(QUIET_ASCIIDOC)$(RM) $@.new $@ && \ sed -e '1,/^$$/d' $< | \ - $(TXT_TO_HTML) - >$@+ && \ - mv $@+ $@ + $(TXT_TO_HTML) - >$@.new && \ + mv $@.new $@ install-webdoc : html '$(SHELL_PATH_SQ)' ./install-webdoc.sh $(WEBDOC_DEST) From 33157b706ff9a79b9c4fc51a05f439917307d5e6 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Thu, 9 Apr 2015 16:19:56 +0100 Subject: [PATCH 07/41] gettext: always use UTF-8 on native Windows Git on native Windows exclusively uses UTF-8 for console output (both with mintty and native console windows). Gettext uses setlocale() to determine the output encoding for translated text, however, MSVCRT's setlocale() doesn't support UTF-8. As a result, translated text is encoded in system encoding (GetAPC()), and non-ASCII chars are mangled in console output. Use gettext's bind_textdomain_codeset() to force the encoding to UTF-8 on native Windows. In this developers' setup, HAVE_LIBCHARSET_H is apparently defined, but we *really* want to override the locale_charset() here. Signed-off-by: Karsten Blees --- gettext.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/gettext.c b/gettext.c index d4021d690c..d8423e5c41 100644 --- a/gettext.c +++ b/gettext.c @@ -12,7 +12,9 @@ #ifndef NO_GETTEXT # include # include -# ifdef HAVE_LIBCHARSET_H +# ifdef GIT_WINDOWS_NATIVE +# define locale_charset() "UTF-8" +# elif defined HAVE_LIBCHARSET_H # include # else # include From fcb1c8a034ebf16a09285d0797d75a4ce7290c1e Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Mon, 6 Apr 2015 21:37:18 +0200 Subject: [PATCH 08/41] mingw: initialize HOME on startup HOME initialization was historically duplicated in many different places, including /etc/profile, launch scripts such as git-bash.vbs and gitk.cmd, and (although slightly broken) in the git-wrapper. Even unrelated projects such as GitExtensions and TortoiseGit need to implement the same logic to be able to call git directly. Initialize HOME in git's own startup code so that we can eventually retire all the duplicate initialization code. Signed-off-by: Karsten Blees --- compat/mingw.c | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/compat/mingw.c b/compat/mingw.c index e041be6db8..c70fb2522e 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2381,6 +2381,30 @@ static void setup_windows_environment(void) /* simulate TERM to enable auto-color (see color.c) */ if (!getenv("TERM")) setenv("TERM", "cygwin", 1); + + /* calculate HOME if not set */ + if (!getenv("HOME")) { + /* + * try $HOMEDRIVE$HOMEPATH - the home share may be a network + * location, thus also check if the path exists (i.e. is not + * disconnected) + */ + if ((tmp = getenv("HOMEDRIVE"))) { + struct strbuf buf = STRBUF_INIT; + strbuf_addstr(&buf, tmp); + if ((tmp = getenv("HOMEPATH"))) { + strbuf_addstr(&buf, tmp); + if (is_directory(buf.buf)) + setenv("HOME", buf.buf, 1); + else + tmp = NULL; /* use $USERPROFILE */ + } + strbuf_release(&buf); + } + /* use $USERPROFILE if the home share is not available */ + if (!tmp && (tmp = getenv("USERPROFILE"))) + setenv("HOME", tmp, 1); + } } int handle_long_path(wchar_t *path, int len, int max_path, int expand) From c40fc4041ebfa2abcf4c2079dc4164c015e7ddc3 Mon Sep 17 00:00:00 2001 From: nalla Date: Thu, 16 Apr 2015 11:45:05 +0100 Subject: [PATCH 09/41] mingw: explicitly `fflush` stdout For performance reasons `stdout` is not unbuffered by default. That leads to problems if after printing to `stdout` a read on `stdin` is performed. For that reason interactive commands like `git clean -i` do not function properly anymore if the `stdout` is not flushed by `fflush(stdout)` before trying to read from `stdin`. In the case of `git clean -i` all reads on `stdin` were preceded by a `fflush(stdout)` call. Signed-off-by: nalla --- builtin/clean.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/builtin/clean.c b/builtin/clean.c index aaba4af3c2..4368af6a8a 100644 --- a/builtin/clean.c +++ b/builtin/clean.c @@ -578,6 +578,7 @@ static int *list_and_choose(struct menu_opts *opts, struct menu_stuff *stuff) clean_get_color(CLEAN_COLOR_RESET)); } + fflush(stdout); if (strbuf_getline_lf(&choice, stdin) != EOF) { strbuf_trim(&choice); } else { @@ -660,6 +661,7 @@ static int filter_by_patterns_cmd(void) clean_print_color(CLEAN_COLOR_PROMPT); printf(_("Input ignore patterns>> ")); clean_print_color(CLEAN_COLOR_RESET); + fflush(stdout); if (strbuf_getline_lf(&confirm, stdin) != EOF) strbuf_trim(&confirm); else @@ -758,6 +760,7 @@ static int ask_each_cmd(void) qname = quote_path_relative(item->string, NULL, &buf); /* TRANSLATORS: Make sure to keep [y/N] as is */ printf(_("Remove %s [y/N]? "), qname); + fflush(stdout); if (strbuf_getline_lf(&confirm, stdin) != EOF) { strbuf_trim(&confirm); } else { From 58c23b695b6d069ee7a4ce28606c4002004e7d53 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Thu, 19 Mar 2015 16:33:44 +0100 Subject: [PATCH 10/41] mingw: Support `git_terminal_prompt` with more terminals The `git_terminal_prompt()` function expects the terminal window to be attached to a Win32 Console. However, this is not the case with terminal windows other than `cmd.exe`'s, e.g. with MSys2's own `mintty`. Non-cmd terminals such as `mintty` still have to have a Win32 Console to be proper console programs, but have to hide the Win32 Console to be able to provide more flexibility (such as being resizeable not only vertically but also horizontally). By writing to that Win32 Console, `git_terminal_prompt()` manages only to send the prompt to nowhere and to wait for input from a Console to which the user has no access. This commit introduces a function specifically to support `mintty` -- or other terminals that are compatible with MSys2's `/dev/tty` emulation. We use the `TERM` environment variable as an indicator for that: if the value starts with "xterm" (such as `mintty`'s "xterm_256color"), we prefer to let `xterm_prompt()` handle the user interaction. The most prominent user of `git_terminal_prompt()` is certainly `git-remote-https.exe`. It is an interesting use case because both `stdin` and `stdout` are redirected when Git calls said executable, yet it still wants to access the terminal. When running inside a `mintty`, the terminal is not accessible to the `git-remote-https.exe` program, though, because it is a MinGW program and the `mintty` terminal is not backed by a Win32 console. To solve that problem, we simply call out to the shell -- which is an *MSys2* program and can therefore access `/dev/tty`. Helped-by: nalla Signed-off-by: Karsten Blees Signed-off-by: Johannes Schindelin --- compat/terminal.c | 59 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/compat/terminal.c b/compat/terminal.c index fa13ee672d..069f4061ed 100644 --- a/compat/terminal.c +++ b/compat/terminal.c @@ -1,7 +1,12 @@ +#ifndef NO_INTTYPES_H +#include +#endif #include "git-compat-util.h" +#include "run-command.h" #include "compat/terminal.h" #include "sigchain.h" #include "strbuf.h" +#include "cache.h" #if defined(HAVE_DEV_TTY) || defined(GIT_WINDOWS_NATIVE) @@ -91,6 +96,54 @@ static int disable_echo(void) return 0; } +static char *shell_prompt(const char *prompt, int echo) +{ + const char *read_input[] = { + /* Note: call 'bash' explicitly, as 'read -s' is bash-specific */ + "bash", "-c", echo ? + "cat >/dev/tty && read -r line /dev/tty && read -r -s line /dev/tty", + NULL + }; + struct child_process child = CHILD_PROCESS_INIT; + static struct strbuf buffer = STRBUF_INIT; + int prompt_len = strlen(prompt), len = -1, code; + + child.argv = read_input; + child.in = -1; + child.out = -1; + + if (start_command(&child)) + return NULL; + + if (write_in_full(child.in, prompt, prompt_len) != prompt_len) { + error("could not write to prompt script"); + close(child.in); + goto ret; + } + close(child.in); + + strbuf_reset(&buffer); + len = strbuf_read(&buffer, child.out, 1024); + if (len < 0) { + error("could not read from prompt script"); + goto ret; + } + + strbuf_strip_suffix(&buffer, "\n"); + strbuf_strip_suffix(&buffer, "\r"); + +ret: + close(child.out); + code = finish_command(&child); + if (code) { + error("failed to execute prompt script (exit code %d)", code); + return NULL; + } + + return len < 0 ? NULL : buffer.buf; +} + #endif #ifndef FORCE_TEXT @@ -102,6 +155,12 @@ char *git_terminal_prompt(const char *prompt, int echo) static struct strbuf buf = STRBUF_INIT; int r; FILE *input_fh, *output_fh; +#ifdef GIT_WINDOWS_NATIVE + const char *term = getenv("TERM"); + + if (term && starts_with(term, "xterm")) + return shell_prompt(prompt, echo); +#endif input_fh = fopen(INPUT_PATH, "r" FORCE_TEXT); if (!input_fh) From 2c046c72c28b7777bf2196540620becdd5a8a622 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sat, 9 May 2015 02:11:48 +0200 Subject: [PATCH 11/41] compat/terminal.c: only use the Windows console if bash 'read -r' fails MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Accessing the Windows console through the special CONIN$ / CONOUT$ devices doesn't work properly for non-ASCII usernames an passwords. It also doesn't work for terminal emulators that hide the native console window (such as mintty), and 'TERM=xterm*' is not necessarily a reliable indicator for such terminals. The new shell_prompt() function, on the other hand, works fine for both MSys1 and MSys2, in native console windows as well as mintty, and properly supports Unicode. It just needs bash on the path (for 'read -s', which is bash-specific). On Windows, try to use the shell to read from the terminal. If that fails with ENOENT (i.e. bash was not found), use CONIN/OUT as fallback. Note: To test this, create a UTF-8 credential file with non-ASCII chars, e.g. in git-bash: 'echo url=http://täst.com > cred.txt'. Then in git-cmd, 'git credential fill --- compat/terminal.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/compat/terminal.c b/compat/terminal.c index 069f4061ed..d9d3945afa 100644 --- a/compat/terminal.c +++ b/compat/terminal.c @@ -112,6 +112,7 @@ static char *shell_prompt(const char *prompt, int echo) child.argv = read_input; child.in = -1; child.out = -1; + child.silent_exec_failure = 1; if (start_command(&child)) return NULL; @@ -155,11 +156,14 @@ char *git_terminal_prompt(const char *prompt, int echo) static struct strbuf buf = STRBUF_INIT; int r; FILE *input_fh, *output_fh; -#ifdef GIT_WINDOWS_NATIVE - const char *term = getenv("TERM"); - if (term && starts_with(term, "xterm")) - return shell_prompt(prompt, echo); +#ifdef GIT_WINDOWS_NATIVE + + /* try shell_prompt first, fall back to CONIN/OUT if bash is missing */ + char *result = shell_prompt(prompt, echo); + if (result || errno != ENOENT) + return result; + #endif input_fh = fopen(INPUT_PATH, "r" FORCE_TEXT); From 607518dadc7fe024bacfc7c7a71e8a4de4f237a2 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 10 Jan 2017 23:14:20 +0100 Subject: [PATCH 12/41] winansi: simplify loading the GetCurrentConsoleFontEx() function We introduced helper macros to simplify loading functions dynamically. Might just as well use them. Signed-off-by: Johannes Schindelin --- compat/winansi.c | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/compat/winansi.c b/compat/winansi.c index 11cd9b82cc..efc0abcdac 100644 --- a/compat/winansi.c +++ b/compat/winansi.c @@ -7,6 +7,7 @@ #include #include #include "win32.h" +#include "win32/lazyload.h" static int fd_is_interactive[3] = { 0, 0, 0 }; #define FD_CONSOLE 0x1 @@ -41,26 +42,21 @@ typedef struct _CONSOLE_FONT_INFOEX { #endif #endif -typedef BOOL (WINAPI *PGETCURRENTCONSOLEFONTEX)(HANDLE, BOOL, - PCONSOLE_FONT_INFOEX); - static void warn_if_raster_font(void) { DWORD fontFamily = 0; - PGETCURRENTCONSOLEFONTEX pGetCurrentConsoleFontEx; + DECLARE_PROC_ADDR(kernel32.dll, BOOL, GetCurrentConsoleFontEx, + HANDLE, BOOL, PCONSOLE_FONT_INFOEX); /* don't bother if output was ascii only */ if (!non_ascii_used) return; /* GetCurrentConsoleFontEx is available since Vista */ - pGetCurrentConsoleFontEx = (PGETCURRENTCONSOLEFONTEX) GetProcAddress( - GetModuleHandle("kernel32.dll"), - "GetCurrentConsoleFontEx"); - if (pGetCurrentConsoleFontEx) { + if (INIT_PROC_ADDR(GetCurrentConsoleFontEx)) { CONSOLE_FONT_INFOEX cfi; cfi.cbSize = sizeof(cfi); - if (pGetCurrentConsoleFontEx(console, 0, &cfi)) + if (GetCurrentConsoleFontEx(console, 0, &cfi)) fontFamily = cfi.FontFamily; } else { /* pre-Vista: check default console font in registry */ From f2ea9ffa77176a0317091360258b95baa1eaa4f9 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 6 Sep 2016 09:50:33 +0200 Subject: [PATCH 13/41] Unbreak interactive GPG prompt upon signing With the recent update in efee955 (gpg-interface: check gpg signature creation status, 2016-06-17), we ask GPG to send all status updates to stderr, and then catch the stderr in an strbuf. But GPG might fail, and send error messages to stderr. And we simply do not show them to the user. Even worse: this swallows any interactive prompt for a passphrase. And detaches stderr from the tty so that the passphrase cannot be read. So while the first problem could be fixed (by printing the captured stderr upon error), the second problem cannot be easily fixed, and presents a major regression. So let's just revert commit efee9553a4f97b2ecd8f49be19606dd4cf7d9c28. This fixes https://github.com/git-for-windows/git/issues/871 Cc: Michael J Gruber Signed-off-by: Johannes Schindelin --- gpg-interface.c | 8 ++------ t/t7004-tag.sh | 13 ------------- 2 files changed, 2 insertions(+), 19 deletions(-) diff --git a/gpg-interface.c b/gpg-interface.c index 8ed274533f..24348691f8 100644 --- a/gpg-interface.c +++ b/gpg-interface.c @@ -293,11 +293,9 @@ int sign_buffer(struct strbuf *buffer, struct strbuf *signature, const char *sig struct child_process gpg = CHILD_PROCESS_INIT; int ret; size_t i, j, bottom; - struct strbuf gpg_status = STRBUF_INIT; argv_array_pushl(&gpg.args, use_format->program, - "--status-fd=2", "-bsau", signing_key, NULL); @@ -309,12 +307,10 @@ int sign_buffer(struct strbuf *buffer, struct strbuf *signature, const char *sig */ sigchain_push(SIGPIPE, SIG_IGN); ret = pipe_command(&gpg, buffer->buf, buffer->len, - signature, 1024, &gpg_status, 0); + signature, 1024, NULL, 0); sigchain_pop(SIGPIPE); - ret |= !strstr(gpg_status.buf, "\n[GNUPG:] SIG_CREATED "); - strbuf_release(&gpg_status); - if (ret) + if (ret || signature->len == bottom) return error(_("gpg failed to sign the data")); /* Strip CR from the line endings, in case we are on Windows. */ diff --git a/t/t7004-tag.sh b/t/t7004-tag.sh index 0b01862c23..a05df0d7b6 100755 --- a/t/t7004-tag.sh +++ b/t/t7004-tag.sh @@ -1345,12 +1345,6 @@ test_expect_success GPG \ 'test_config user.signingkey BobTheMouse && test_must_fail git tag -s -m tail tag-gpg-failure' -# try to produce invalid signature -test_expect_success GPG \ - 'git tag -s fails if gpg is misconfigured (bad signature format)' \ - 'test_config gpg.program echo && - test_must_fail git tag -s -m tail tag-gpg-failure' - # try to sign with bad user.signingkey test_expect_success GPGSM \ 'git tag -s fails if gpgsm is misconfigured (bad key)' \ @@ -1358,13 +1352,6 @@ test_expect_success GPGSM \ test_config gpg.format x509 && test_must_fail git tag -s -m tail tag-gpg-failure' -# try to produce invalid signature -test_expect_success GPGSM \ - 'git tag -s fails if gpgsm is misconfigured (bad signature format)' \ - 'test_config gpg.x509.program echo && - test_config gpg.format x509 && - test_must_fail git tag -s -m tail tag-gpg-failure' - # try to verify without gpg: rm -rf gpghome From f80609c7d2f51850c0785d78c4316fb3bf2e3bdf Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 21 Feb 2017 13:28:58 +0100 Subject: [PATCH 14/41] mingw: ensure valid CTYPE A change between versions 2.4.1 and 2.6.0 of the MSYS2 runtime modified how Cygwin's runtime (and hence Git for Windows' MSYS2 runtime derivative) handles locales: d16a56306d (Consolidate wctomb/mbtowc calls for POSIX-1.2008, 2016-07-20). An unintended side-effect is that "cold-calling" into the POSIX emulation will start with a locale based on the current code page, something that Git for Windows is very ill-prepared for, as it expects to be able to pass a command-line containing non-ASCII characters to the shell without having those characters munged. One symptom of this behavior: when `git clone` or `git fetch` shell out to call `git-upload-pack` with a path that contains non-ASCII characters, the shell tried to interpret the entire command-line (including command-line parameters) as executable path, which obviously must fail. This fixes https://github.com/git-for-windows/git/issues/1036 Signed-off-by: Johannes Schindelin --- compat/mingw.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/compat/mingw.c b/compat/mingw.c index c70fb2522e..71d892e89a 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2405,6 +2405,9 @@ static void setup_windows_environment(void) if (!tmp && (tmp = getenv("USERPROFILE"))) setenv("HOME", tmp, 1); } + + if (!getenv("LC_ALL") && !getenv("LC_CTYPE") && !getenv("LANG")) + setenv("LC_CTYPE", "C", 1); } int handle_long_path(wchar_t *path, int len, int max_path, int expand) From 67364b569439a2f033af25beda9d25ce38a69ce8 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 7 Feb 2019 13:39:21 +0100 Subject: [PATCH 15/41] mingw: drop MakeMaker reference In 20d2a30f8ffe (Makefile: replace perl/Makefile.PL with simple make rules, 2017-12-10), Git stopped using MakeMaker. Therefore, that definition in the MINGW-specific section became useless. Signed-off-by: Johannes Schindelin --- config.mak.uname | 1 - 1 file changed, 1 deletion(-) diff --git a/config.mak.uname b/config.mak.uname index 55f0d50bf8..2f7308229b 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -586,7 +586,6 @@ ifneq (,$(findstring MINGW,$(uname_S))) NO_STRTOUMAX = YesPlease NO_MKDTEMP = YesPlease NO_SVN_TESTS = YesPlease - NO_PERL_MAKEMAKER = YesPlease RUNTIME_PREFIX = YesPlease HAVE_WPGMPTR = YesWeDo NO_ST_BLOCKS_IN_STRUCT_STAT = YesPlease From 417578000c6d7a2d81c229ddb361af3638efed07 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 21 Feb 2017 20:34:38 +0100 Subject: [PATCH 16/41] mingw: make is_hidden tests in t0001/t5611 more robust We should not actually expect the first `attrib.exe` in the PATH to be the one we are looking for. Or that it is in the PATH, for that matter. Signed-off-by: Johannes Schindelin --- t/t0001-init.sh | 2 +- t/t5611-clone-config.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/t/t0001-init.sh b/t/t0001-init.sh index 4d04e6a863..a6c5fe9e14 100755 --- a/t/t0001-init.sh +++ b/t/t0001-init.sh @@ -412,7 +412,7 @@ test_expect_success SYMLINKS 're-init to move gitdir symlink' ' # Tests for the hidden file attribute on windows is_hidden () { # Use the output of `attrib`, ignore the absolute path - case "$(attrib "$1")" in *H*?:*) return 0;; esac + case "$("$SYSTEMROOT"/system32/attrib "$1")" in *H*?:*) return 0;; esac return 1 } diff --git a/t/t5611-clone-config.sh b/t/t5611-clone-config.sh index 60c1ba951b..87b8073cd7 100755 --- a/t/t5611-clone-config.sh +++ b/t/t5611-clone-config.sh @@ -95,7 +95,7 @@ test_expect_success 'clone -c remote..fetch= --origin=' ' # Tests for the hidden file attribute on windows is_hidden () { # Use the output of `attrib`, ignore the absolute path - case "$(attrib "$1")" in *H*?:*) return 0;; esac + case "$("$SYSTEMROOT"/system32/attrib "$1")" in *H*?:*) return 0;; esac return 1 } From 477fbb5a9fce08093d9a1abd2bf82536b9808c7c Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 5 Oct 2017 11:48:16 +0200 Subject: [PATCH 17/41] diff: munmap() file contents before running external diff When running an external diff from, say, a diff tool, it is safe to assume that we want to write the files in question. On Windows, that means that there cannot be any other process holding an open handle to said files. So let's make sure that `git diff` itself is not holding any open handle to the files in question. This fixes https://github.com/git-for-windows/git/issues/1315 Signed-off-by: Johannes Schindelin --- diff.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/diff.c b/diff.c index 5306c48652..4d931179bb 100644 --- a/diff.c +++ b/diff.c @@ -4207,6 +4207,10 @@ static void run_external_diff(const char *pgm, argv_array_pushf(&env, "GIT_DIFF_PATH_COUNTER=%d", ++o->diff_path_counter); argv_array_pushf(&env, "GIT_DIFF_PATH_TOTAL=%d", q->nr); + if (one && one->should_munmap) + diff_free_filespec_data(one); + if (two && two->should_munmap) + diff_free_filespec_data(two); if (run_command_v_opt_cd_env(argv.argv, RUN_USING_SHELL, NULL, env.argv)) die(_("external diff died, stopping at %s"), name); From 925141afa16e9c8ce5f4ef9347c4fc83a7d3bd7b Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 23 Feb 2015 15:55:47 +0000 Subject: [PATCH 18/41] mingw: disable t9020 POSIX-to-Windows path mangling would make it fail. Symptoms: ++ init_git ++ rm -fr .git ++ git init Initialized empty Git repository in [...] ++ git remote add svnsim testsvn::sim:///usr/src/git/wip5/t/t9154/svn.dump ++ git remote add svnfile testsvn::file:///usr/src/git/wip5/t/t9154/svn.dump ++ git fetch svnsim progress Imported commit 1. fatal: Write to frontend failed: Bad file descriptor fast-import: dumping crash report to .git/fast_import_crash_23356 fatal: error while running fast-import fatal: unexpected end of fast-import feedback error: last command exited with $?=128 not ok 1 - simple fetch Since the remote-svn project seems to be dormant at the moment (and not complete enough to be used, which is a pity), let's just skip this test on Windows. Signed-off-by: Johannes Schindelin --- t/t9020-remote-svn.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/t/t9020-remote-svn.sh b/t/t9020-remote-svn.sh index 6fca08e5e3..76d9be2e1d 100755 --- a/t/t9020-remote-svn.sh +++ b/t/t9020-remote-svn.sh @@ -12,6 +12,12 @@ then test_done fi +if test_have_prereq MINGW +then + skip_all='skipping remote-svn tests for lack of POSIX' + test_done +fi + # Override svnrdump with our simulator PATH="$HOME:$PATH" export PATH PYTHON_PATH GIT_BUILD_DIR From 2620a9f3945ea8cdea8fca68b9cbfb7d94d3ca93 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 11 Jan 2017 21:08:15 +0100 Subject: [PATCH 19/41] t9001, t9116: avoid pipes When grepping through the output of a command in the test suite, there is always a chance that something goes wrong, in which case there would not be anything useful to debug. Let's redirect the output into a file instead, and grep that file, so that the log can be inspected easily if the grep fails. Signed-off-by: Johannes Schindelin --- t/t9001-send-email.sh | 4 ++-- t/t9116-git-svn-log.sh | 12 ++++++++---- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/t/t9001-send-email.sh b/t/t9001-send-email.sh index ee1efcc59d..748e263169 100755 --- a/t/t9001-send-email.sh +++ b/t/t9001-send-email.sh @@ -1180,8 +1180,8 @@ test_expect_success $PREREQ 'in-reply-to but no threading' ' --to=nobody@example.com \ --in-reply-to="" \ --no-thread \ - $patches | - grep "In-Reply-To: " + $patches >out && + grep "In-Reply-To: " out ' test_expect_success $PREREQ 'no in-reply-to and no threading' ' diff --git a/t/t9116-git-svn-log.sh b/t/t9116-git-svn-log.sh index 45773ee560..0a9f1ef366 100755 --- a/t/t9116-git-svn-log.sh +++ b/t/t9116-git-svn-log.sh @@ -43,14 +43,18 @@ test_expect_success 'setup repository and import' ' test_expect_success 'run log' " git reset --hard origin/a && - git svn log -r2 origin/trunk | grep ^r2 && - git svn log -r4 origin/trunk | grep ^r4 && - git svn log -r3 | grep ^r3 + git svn log -r2 origin/trunk >out && + grep ^r2 out && + git svn log -r4 origin/trunk >out && + grep ^r4 out && + git svn log -r3 >out && + grep ^r3 out " test_expect_success 'run log against a from trunk' " git reset --hard origin/trunk && - git svn log -r3 origin/a | grep ^r3 + git svn log -r3 origin/a >out && + grep ^r3 out " printf 'r1 \nr2 \nr4 \n' > expected-range-r1-r2-r4 From 184499328fba3a62d809be90a598a3a7d63281fa Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Mon, 11 May 2015 19:54:23 +0200 Subject: [PATCH 20/41] strbuf_readlink: don't call readlink twice if hint is the exact link size strbuf_readlink() calls readlink() twice if the hint argument specifies the exact size of the link target (e.g. by passing stat.st_size as returned by lstat()). This is necessary because 'readlink(..., hint) == hint' could mean that the buffer was too small. Use hint + 1 as buffer size to prevent this. Signed-off-by: Karsten Blees --- strbuf.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/strbuf.c b/strbuf.c index f6a6cf78b9..82fcc6fd35 100644 --- a/strbuf.c +++ b/strbuf.c @@ -480,12 +480,12 @@ int strbuf_readlink(struct strbuf *sb, const char *path, size_t hint) while (hint < STRBUF_MAXLINK) { ssize_t len; - strbuf_grow(sb, hint); - len = readlink(path, sb->buf, hint); + strbuf_grow(sb, hint + 1); + len = readlink(path, sb->buf, hint + 1); if (len < 0) { if (errno != ERANGE) break; - } else if (len < hint) { + } else if (len <= hint) { strbuf_setlen(sb, len); return 0; } From 1bdec10715c233960ba8b7e1c971565b584b5b7d Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Mon, 11 May 2015 22:15:40 +0200 Subject: [PATCH 21/41] strbuf_readlink: support link targets that exceed PATH_MAX strbuf_readlink() refuses to read link targets that exceed PATH_MAX (even if a sufficient size was specified by the caller). As some platforms support longer paths, remove this restriction (similar to strbuf_getcwd()). Signed-off-by: Karsten Blees --- strbuf.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/strbuf.c b/strbuf.c index 82fcc6fd35..9be7fe0ca1 100644 --- a/strbuf.c +++ b/strbuf.c @@ -468,8 +468,6 @@ ssize_t strbuf_write(struct strbuf *sb, FILE *f) } -#define STRBUF_MAXLINK (2*PATH_MAX) - int strbuf_readlink(struct strbuf *sb, const char *path, size_t hint) { size_t oldalloc = sb->alloc; @@ -477,7 +475,7 @@ int strbuf_readlink(struct strbuf *sb, const char *path, size_t hint) if (hint < 32) hint = 32; - while (hint < STRBUF_MAXLINK) { + for (;;) { ssize_t len; strbuf_grow(sb, hint + 1); From 0ba09934d1802ae672f6467e6eb2a17e03e6a28c Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Mon, 11 May 2015 19:58:14 +0200 Subject: [PATCH 22/41] lockfile.c: use is_dir_sep() instead of hardcoded '/' checks Signed-off-by: Karsten Blees --- lockfile.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lockfile.c b/lockfile.c index 8e8ab4f29f..3704a603f6 100644 --- a/lockfile.c +++ b/lockfile.c @@ -17,14 +17,14 @@ static void trim_last_path_component(struct strbuf *path) int i = path->len; /* back up past trailing slashes, if any */ - while (i && path->buf[i - 1] == '/') + while (i && is_dir_sep(path->buf[i - 1])) i--; /* * then go backwards until a slash, or the beginning of the * string */ - while (i && path->buf[i - 1] != '/') + while (i && !is_dir_sep(path->buf[i - 1])) i--; strbuf_setlen(path, i); From 313b414caab14d113327fddc8931ac7b2dd4f128 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Tue, 12 May 2015 11:09:01 +0200 Subject: [PATCH 23/41] Win32: don't call GetFileAttributes twice in mingw_lstat() GetFileAttributes cannot handle paths with trailing dir separator. The current [l]stat implementation calls GetFileAttributes twice if the path has trailing slashes (first with the original path passed to [l]stat, and and a second time with a path copy with trailing '/' removed). With Unicode conversion, we get the length of the path for free and also have a (wide char) buffer that can be modified. Remove trailing directory separators before calling the Win32 API. Signed-off-by: Karsten Blees --- compat/mingw.c | 48 ++++++++++++------------------------------------ 1 file changed, 12 insertions(+), 36 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 71d892e89a..22f48aaeec 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -716,9 +716,18 @@ static int do_lstat(int follow, const char *file_name, struct stat *buf) { WIN32_FILE_ATTRIBUTE_DATA fdata; wchar_t wfilename[MAX_LONG_PATH]; - if (xutftowcs_long_path(wfilename, file_name) < 0) + int wlen = xutftowcs_long_path(wfilename, file_name); + if (wlen < 0) return -1; + /* strip trailing '/', or GetFileAttributes will fail */ + while (wlen && is_dir_sep(wfilename[wlen - 1])) + wfilename[--wlen] = 0; + if (!wlen) { + errno = ENOENT; + return -1; + } + if (GetFileAttributesExW(wfilename, GetFileExInfoStandard, &fdata)) { buf->st_ino = 0; buf->st_gid = 0; @@ -778,39 +787,6 @@ static int do_lstat(int follow, const char *file_name, struct stat *buf) return -1; } -/* We provide our own lstat/fstat functions, since the provided - * lstat/fstat functions are so slow. These stat functions are - * tailored for Git's usage (read: fast), and are not meant to be - * complete. Note that Git stat()s are redirected to mingw_lstat() - * too, since Windows doesn't really handle symlinks that well. - */ -static int do_stat_internal(int follow, const char *file_name, struct stat *buf) -{ - int namelen; - char alt_name[MAX_LONG_PATH]; - - if (!do_lstat(follow, file_name, buf)) - return 0; - - /* if file_name ended in a '/', Windows returned ENOENT; - * try again without trailing slashes - */ - if (errno != ENOENT) - return -1; - - namelen = strlen(file_name); - if (namelen && file_name[namelen-1] != '/') - return -1; - while (namelen && file_name[namelen-1] == '/') - --namelen; - if (!namelen || namelen >= MAX_LONG_PATH) - return -1; - - memcpy(alt_name, file_name, namelen); - alt_name[namelen] = 0; - return do_lstat(follow, alt_name, buf); -} - int (*lstat)(const char *file_name, struct stat *buf) = mingw_lstat; static int get_file_info_by_handle(HANDLE hnd, struct stat *buf) @@ -838,11 +814,11 @@ static int get_file_info_by_handle(HANDLE hnd, struct stat *buf) int mingw_lstat(const char *file_name, struct stat *buf) { - return do_stat_internal(0, file_name, buf); + return do_lstat(0, file_name, buf); } int mingw_stat(const char *file_name, struct stat *buf) { - return do_stat_internal(1, file_name, buf); + return do_lstat(1, file_name, buf); } int mingw_fstat(int fd, struct stat *buf) From 5a1b752377c72df2d124045160699fc9db913816 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sat, 16 May 2015 01:18:14 +0200 Subject: [PATCH 24/41] Win32: implement stat() with symlink support With respect to symlinks, the current stat() implementation is almost the same as lstat(): except for the file type (st_mode & S_IFMT), it returns information about the link rather than the target. Implement stat by opening the file with as little permissions as possible and calling GetFileInformationByHandle on it. This way, all link resoltion is handled by the Windows file system layer. If symlinks are disabled, use lstat() as before, but fail with ELOOP if a symlink would have to be resolved. Signed-off-by: Karsten Blees Signed-off-by: Johannes Schindelin --- compat/mingw.c | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/compat/mingw.c b/compat/mingw.c index 22f48aaeec..65e39b9e91 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -816,9 +816,26 @@ int mingw_lstat(const char *file_name, struct stat *buf) { return do_lstat(0, file_name, buf); } + int mingw_stat(const char *file_name, struct stat *buf) { - return do_lstat(1, file_name, buf); + wchar_t wfile_name[MAX_LONG_PATH]; + HANDLE hnd; + int result; + + /* open the file and let Windows resolve the links */ + if (xutftowcs_long_path(wfile_name, file_name) < 0) + return -1; + hnd = CreateFileW(wfile_name, 0, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, NULL, + OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); + if (hnd == INVALID_HANDLE_VALUE) { + errno = err_win_to_posix(GetLastError()); + return -1; + } + result = get_file_info_by_handle(hnd, buf); + CloseHandle(hnd); + return result; } int mingw_fstat(int fd, struct stat *buf) From 81d2915eee79dab58e5da0f88ebe75fa7d86e3e2 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Tue, 12 May 2015 00:58:39 +0200 Subject: [PATCH 25/41] Win32: remove separate do_lstat() function With the new mingw_stat() implementation, do_lstat() is only called from mingw_lstat() (with follow == 0). Remove the extra function and the old mingw_stat()-specific (follow == 1) logic. Signed-off-by: Karsten Blees --- compat/mingw.c | 22 ++-------------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 65e39b9e91..c8c703e5d4 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -705,14 +705,7 @@ static int has_valid_directory_prefix(wchar_t *wfilename) return 1; } -/* We keep the do_lstat code in a separate function to avoid recursion. - * When a path ends with a slash, the stat will fail with ENOENT. In - * this case, we strip the trailing slashes and stat again. - * - * If follow is true then act like stat() and report on the link - * target. Otherwise report on the link itself. - */ -static int do_lstat(int follow, const char *file_name, struct stat *buf) +int mingw_lstat(const char *file_name, struct stat *buf) { WIN32_FILE_ATTRIBUTE_DATA fdata; wchar_t wfilename[MAX_LONG_PATH]; @@ -746,13 +739,7 @@ static int do_lstat(int follow, const char *file_name, struct stat *buf) if (handle != INVALID_HANDLE_VALUE) { if ((findbuf.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) && (findbuf.dwReserved0 == IO_REPARSE_TAG_SYMLINK)) { - if (follow) { - char buffer[MAXIMUM_REPARSE_DATA_BUFFER_SIZE]; - buf->st_size = readlink(file_name, buffer, MAXIMUM_REPARSE_DATA_BUFFER_SIZE); - } else { - buf->st_mode = S_IFLNK; - } - buf->st_mode |= S_IREAD; + buf->st_mode = S_IFLNK | S_IREAD; if (!(findbuf.dwFileAttributes & FILE_ATTRIBUTE_READONLY)) buf->st_mode |= S_IWRITE; } @@ -812,11 +799,6 @@ static int get_file_info_by_handle(HANDLE hnd, struct stat *buf) return 0; } -int mingw_lstat(const char *file_name, struct stat *buf) -{ - return do_lstat(0, file_name, buf); -} - int mingw_stat(const char *file_name, struct stat *buf) { wchar_t wfile_name[MAX_LONG_PATH]; From 01128e4559905e82c198ae4e29813e0cd318d6a6 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sun, 24 May 2015 00:17:56 +0200 Subject: [PATCH 26/41] Win32: let mingw_lstat() error early upon problems with reparse points When obtaining lstat information for reparse points, we need to call FindFirstFile() in addition to GetFileInformationEx() to obtain the type of the reparse point (symlink, mount point etc.). However, currently there is no error handling whatsoever if FindFirstFile() fails. Call FindFirstFile() before modifying the stat *buf output parameter and error out if the call fails. Note: The FindFirstFile() return value includes all the data that we get from GetFileAttributesEx(), so we could replace GetFileAttributesEx() with FindFirstFile(). We don't do that because GetFileAttributesEx() is about twice as fast for single files. I.e. we only pay the extra cost of calling FindFirstFile() in the rare case that we encounter a reparse point. Note: The indentation of the remaining reparse point code will be fixed in the next patch. Signed-off-by: Karsten Blees --- compat/mingw.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index c8c703e5d4..a74a67f15a 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -708,6 +708,7 @@ static int has_valid_directory_prefix(wchar_t *wfilename) int mingw_lstat(const char *file_name, struct stat *buf) { WIN32_FILE_ATTRIBUTE_DATA fdata; + WIN32_FIND_DATAW findbuf = { 0 }; wchar_t wfilename[MAX_LONG_PATH]; int wlen = xutftowcs_long_path(wfilename, file_name); if (wlen < 0) @@ -722,6 +723,13 @@ int mingw_lstat(const char *file_name, struct stat *buf) } if (GetFileAttributesExW(wfilename, GetFileExInfoStandard, &fdata)) { + /* for reparse points, use FindFirstFile to get the reparse tag */ + if (fdata.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) { + HANDLE handle = FindFirstFileW(wfilename, &findbuf); + if (handle == INVALID_HANDLE_VALUE) + goto error; + FindClose(handle); + } buf->st_ino = 0; buf->st_gid = 0; buf->st_uid = 0; @@ -734,20 +742,16 @@ int mingw_lstat(const char *file_name, struct stat *buf) filetime_to_timespec(&(fdata.ftLastWriteTime), &(buf->st_mtim)); filetime_to_timespec(&(fdata.ftCreationTime), &(buf->st_ctim)); if (fdata.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) { - WIN32_FIND_DATAW findbuf; - HANDLE handle = FindFirstFileW(wfilename, &findbuf); - if (handle != INVALID_HANDLE_VALUE) { if ((findbuf.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) && (findbuf.dwReserved0 == IO_REPARSE_TAG_SYMLINK)) { buf->st_mode = S_IFLNK | S_IREAD; if (!(findbuf.dwFileAttributes & FILE_ATTRIBUTE_READONLY)) buf->st_mode |= S_IWRITE; } - FindClose(handle); - } } return 0; } +error: switch (GetLastError()) { case ERROR_ACCESS_DENIED: case ERROR_SHARING_VIOLATION: From 85126df62dc7a60e3368695b5bd6a99f12161f03 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Tue, 10 Jan 2017 23:21:56 +0100 Subject: [PATCH 27/41] Win32: teach fscache and dirent about symlinks Move S_IFLNK detection to file_attr_to_st_mode() and reuse it in fscache. Implement DT_LNK detection in dirent.c and the fscache readdir version. Signed-off-by: Karsten Blees Signed-off-by: Johannes Schindelin --- compat/mingw.c | 13 +++---------- compat/win32.h | 6 ++++-- compat/win32/dirent.c | 5 ++++- compat/win32/fscache.c | 6 ++++-- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index a74a67f15a..e2783b61aa 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -734,21 +734,14 @@ int mingw_lstat(const char *file_name, struct stat *buf) buf->st_gid = 0; buf->st_uid = 0; buf->st_nlink = 1; - buf->st_mode = file_attr_to_st_mode(fdata.dwFileAttributes); + buf->st_mode = file_attr_to_st_mode(fdata.dwFileAttributes, + findbuf.dwReserved0); buf->st_size = fdata.nFileSizeLow | (((off_t)fdata.nFileSizeHigh)<<32); buf->st_dev = buf->st_rdev = 0; /* not used by Git */ filetime_to_timespec(&(fdata.ftLastAccessTime), &(buf->st_atim)); filetime_to_timespec(&(fdata.ftLastWriteTime), &(buf->st_mtim)); filetime_to_timespec(&(fdata.ftCreationTime), &(buf->st_ctim)); - if (fdata.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) { - if ((findbuf.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) && - (findbuf.dwReserved0 == IO_REPARSE_TAG_SYMLINK)) { - buf->st_mode = S_IFLNK | S_IREAD; - if (!(findbuf.dwFileAttributes & FILE_ATTRIBUTE_READONLY)) - buf->st_mode |= S_IWRITE; - } - } return 0; } error: @@ -793,7 +786,7 @@ static int get_file_info_by_handle(HANDLE hnd, struct stat *buf) buf->st_gid = 0; buf->st_uid = 0; buf->st_nlink = 1; - buf->st_mode = file_attr_to_st_mode(fdata.dwFileAttributes); + buf->st_mode = file_attr_to_st_mode(fdata.dwFileAttributes, 0); buf->st_size = fdata.nFileSizeLow | (((off_t)fdata.nFileSizeHigh)<<32); buf->st_dev = buf->st_rdev = 0; /* not used by Git */ diff --git a/compat/win32.h b/compat/win32.h index a97e880757..671bcc81f9 100644 --- a/compat/win32.h +++ b/compat/win32.h @@ -6,10 +6,12 @@ #include #endif -static inline int file_attr_to_st_mode (DWORD attr) +static inline int file_attr_to_st_mode (DWORD attr, DWORD tag) { int fMode = S_IREAD; - if (attr & FILE_ATTRIBUTE_DIRECTORY) + if ((attr & FILE_ATTRIBUTE_REPARSE_POINT) && tag == IO_REPARSE_TAG_SYMLINK) + fMode |= S_IFLNK; + else if (attr & FILE_ATTRIBUTE_DIRECTORY) fMode |= S_IFDIR; else fMode |= S_IFREG; diff --git a/compat/win32/dirent.c b/compat/win32/dirent.c index b3bd8d7af7..8c654d722b 100644 --- a/compat/win32/dirent.c +++ b/compat/win32/dirent.c @@ -16,7 +16,10 @@ static inline void finddata2dirent(struct dirent *ent, WIN32_FIND_DATAW *fdata) xwcstoutf(ent->d_name, fdata->cFileName, MAX_PATH * 3); /* Set file type, based on WIN32_FIND_DATA */ - if (fdata->dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) + if ((fdata->dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) + && fdata->dwReserved0 == IO_REPARSE_TAG_SYMLINK) + ent->d_type = DT_LNK; + else if (fdata->dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) ent->d_type = DT_DIR; else ent->d_type = DT_REG; diff --git a/compat/win32/fscache.c b/compat/win32/fscache.c index 4ebd15e426..60aeeb1293 100644 --- a/compat/win32/fscache.c +++ b/compat/win32/fscache.c @@ -149,7 +149,8 @@ static struct fsentry *fseentry_create_entry(struct fsentry *list, fse = fsentry_alloc(list, buf, len); - fse->st_mode = file_attr_to_st_mode(fdata->dwFileAttributes); + fse->st_mode = file_attr_to_st_mode(fdata->dwFileAttributes, + fdata->dwReserved0); fse->st_size = (((off64_t) (fdata->nFileSizeHigh)) << 32) | fdata->nFileSizeLow; filetime_to_timespec(&(fdata->ftLastAccessTime), &(fse->st_atim)); @@ -442,7 +443,8 @@ static struct dirent *fscache_readdir(DIR *base_dir) if (!next) return NULL; dir->pfsentry = next; - dir->dirent.d_type = S_ISDIR(next->st_mode) ? DT_DIR : DT_REG; + dir->dirent.d_type = S_ISREG(next->st_mode) ? DT_REG : + S_ISDIR(next->st_mode) ? DT_DIR : DT_LNK; dir->dirent.d_name = (char*) next->name; return &(dir->dirent); } From f3daf219ae90ce8af0408d83506bf392a7678f82 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sat, 16 May 2015 01:11:37 +0200 Subject: [PATCH 28/41] Win32: lstat(): return adequate stat.st_size for symlinks Git typically doesn't trust the stat.st_size member of symlinks (e.g. see strbuf_readlink()). However, some functions take shortcuts if st_size is 0 (e.g. diff_populate_filespec()). In mingw_lstat() and fscache_lstat(), make sure to return an adequate size. The extra overhead of opening and reading the reparse point to calculate the exact size is not necessary, as git doesn't rely on the value anyway. Signed-off-by: Karsten Blees Signed-off-by: Johannes Schindelin --- compat/mingw.c | 4 ++-- compat/win32/fscache.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index e2783b61aa..3109b0bd1a 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -736,8 +736,8 @@ int mingw_lstat(const char *file_name, struct stat *buf) buf->st_nlink = 1; buf->st_mode = file_attr_to_st_mode(fdata.dwFileAttributes, findbuf.dwReserved0); - buf->st_size = fdata.nFileSizeLow | - (((off_t)fdata.nFileSizeHigh)<<32); + buf->st_size = S_ISLNK(buf->st_mode) ? MAX_LONG_PATH : + fdata.nFileSizeLow | (((off_t) fdata.nFileSizeHigh) << 32); buf->st_dev = buf->st_rdev = 0; /* not used by Git */ filetime_to_timespec(&(fdata.ftLastAccessTime), &(buf->st_atim)); filetime_to_timespec(&(fdata.ftLastWriteTime), &(buf->st_mtim)); diff --git a/compat/win32/fscache.c b/compat/win32/fscache.c index 60aeeb1293..345d7b226b 100644 --- a/compat/win32/fscache.c +++ b/compat/win32/fscache.c @@ -151,8 +151,8 @@ static struct fsentry *fseentry_create_entry(struct fsentry *list, fse->st_mode = file_attr_to_st_mode(fdata->dwFileAttributes, fdata->dwReserved0); - fse->st_size = (((off64_t) (fdata->nFileSizeHigh)) << 32) - | fdata->nFileSizeLow; + fse->st_size = S_ISLNK(fse->st_mode) ? MAX_LONG_PATH : + fdata->nFileSizeLow | (((off_t) fdata->nFileSizeHigh) << 32); filetime_to_timespec(&(fdata->ftLastAccessTime), &(fse->st_atim)); filetime_to_timespec(&(fdata->ftLastWriteTime), &(fse->st_mtim)); filetime_to_timespec(&(fdata->ftCreationTime), &(fse->st_ctim)); From 43a0a00b7c75000ff60ce8faecc1ba6cedaa87f9 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Tue, 19 May 2015 21:48:55 +0200 Subject: [PATCH 29/41] Win32: factor out retry logic The retry pattern is duplicated in three places. It also seems to be too hard to use: mingw_unlink() and mingw_rmdir() duplicate the code to retry, and both of them do so incompletely. They also do not restore errno if the user answers 'no'. Introduce a retry_ask_yes_no() helper function that handles retry with small delay, asking the user, and restoring errno. mingw_unlink: include _wchmod in the retry loop (which may fail if the file is locked exclusively). mingw_rmdir: include special error handling in the retry loop. Signed-off-by: Karsten Blees --- compat/mingw.c | 98 ++++++++++++++++++++++---------------------------- 1 file changed, 43 insertions(+), 55 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 3109b0bd1a..78c8053c74 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -11,8 +11,6 @@ #define HCAST(type, handle) ((type)(intptr_t)handle) -static const int delay[] = { 0, 1, 10, 20, 40 }; - void open_in_gdb(void) { static struct child_process cp = CHILD_PROCESS_INIT; @@ -188,15 +186,12 @@ static int read_yes_no_answer(void) return -1; } -static int ask_yes_no_if_possible(const char *format, ...) +static int ask_yes_no_if_possible(const char *format, va_list args) { char question[4096]; const char *retry_hook[] = { NULL, NULL, NULL }; - va_list args; - va_start(args, format); vsnprintf(question, sizeof(question), format, args); - va_end(args); if ((retry_hook[0] = mingw_getenv("GIT_ASK_YESNO"))) { retry_hook[1] = question; @@ -218,6 +213,31 @@ static int ask_yes_no_if_possible(const char *format, ...) } } +static int retry_ask_yes_no(int *tries, const char *format, ...) +{ + static const int delay[] = { 0, 1, 10, 20, 40 }; + va_list args; + int result, saved_errno = errno; + + if ((*tries) < ARRAY_SIZE(delay)) { + /* + * We assume that some other process had the file open at the wrong + * moment and retry. In order to give the other process a higher + * chance to complete its operation, we give up our time slice now. + * If we have to retry again, we do sleep a bit. + */ + Sleep(delay[*tries]); + (*tries)++; + return 1; + } + + va_start(args, format); + result = ask_yes_no_if_possible(format, args); + va_end(args); + errno = saved_errno; + return result; +} + /* Windows only */ enum hide_dotfiles_type { HIDE_DOTFILES_FALSE = 0, @@ -286,31 +306,21 @@ static wchar_t *normalize_ntpath(wchar_t *wbuf) int mingw_unlink(const char *pathname) { - int ret, tries = 0; + int tries = 0; wchar_t wpathname[MAX_LONG_PATH]; if (xutftowcs_long_path(wpathname, pathname) < 0) return -1; - /* read-only files cannot be removed */ - _wchmod(wpathname, 0666); - while ((ret = _wunlink(wpathname)) == -1 && tries < ARRAY_SIZE(delay)) { + do { + /* read-only files cannot be removed */ + _wchmod(wpathname, 0666); + if (!_wunlink(wpathname)) + return 0; if (!is_file_in_use_error(GetLastError())) break; - /* - * We assume that some other process had the source or - * destination file open at the wrong moment and retry. - * In order to give the other process a higher chance to - * complete its operation, we give up our time slice now. - * If we have to retry again, we do sleep a bit. - */ - Sleep(delay[tries]); - tries++; - } - while (ret == -1 && is_file_in_use_error(GetLastError()) && - ask_yes_no_if_possible("Unlink of file '%s' failed. " - "Should I try again?", pathname)) - ret = _wunlink(wpathname); - return ret; + } while (retry_ask_yes_no(&tries, "Unlink of file '%s' failed. " + "Should I try again?", pathname)); + return -1; } static int is_dir_empty(const wchar_t *wpath) @@ -337,12 +347,14 @@ static int is_dir_empty(const wchar_t *wpath) int mingw_rmdir(const char *pathname) { - int ret, tries = 0; + int tries = 0; wchar_t wpathname[MAX_LONG_PATH]; if (xutftowcs_long_path(wpathname, pathname) < 0) return -1; - while ((ret = _wrmdir(wpathname)) == -1 && tries < ARRAY_SIZE(delay)) { + do { + if (!_wrmdir(wpathname)) + return 0; if (!is_file_in_use_error(GetLastError())) errno = err_win_to_posix(GetLastError()); if (errno != EACCES) @@ -351,21 +363,9 @@ int mingw_rmdir(const char *pathname) errno = ENOTEMPTY; break; } - /* - * We assume that some other process had the source or - * destination file open at the wrong moment and retry. - * In order to give the other process a higher chance to - * complete its operation, we give up our time slice now. - * If we have to retry again, we do sleep a bit. - */ - Sleep(delay[tries]); - tries++; - } - while (ret == -1 && errno == EACCES && is_file_in_use_error(GetLastError()) && - ask_yes_no_if_possible("Deletion of directory '%s' failed. " - "Should I try again?", pathname)) - ret = _wrmdir(wpathname); - return ret; + } while (retry_ask_yes_no(&tries, "Deletion of directory '%s' failed. " + "Should I try again?", pathname)); + return -1; } static inline int needs_hiding(const char *path) @@ -1904,20 +1904,8 @@ repeat: SetFileAttributesW(wpnew, attrs); } } - if (tries < ARRAY_SIZE(delay) && gle == ERROR_ACCESS_DENIED) { - /* - * We assume that some other process had the source or - * destination file open at the wrong moment and retry. - * In order to give the other process a higher chance to - * complete its operation, we give up our time slice now. - * If we have to retry again, we do sleep a bit. - */ - Sleep(delay[tries]); - tries++; - goto repeat; - } if (gle == ERROR_ACCESS_DENIED && - ask_yes_no_if_possible("Rename from '%s' to '%s' failed. " + retry_ask_yes_no(&tries, "Rename from '%s' to '%s' failed. " "Should I try again?", pold, pnew)) goto repeat; From e24dc386fcf10113ebcf9665798dd75d5f8988f0 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sun, 24 May 2015 01:55:05 +0200 Subject: [PATCH 30/41] Win32: change default of 'core.symlinks' to false Symlinks on Windows don't work the same way as on Unix systems. E.g. there are different types of symlinks for directories and files, creating symlinks requires administrative privileges etc. By default, disable symlink support on Windows. I.e. users explicitly have to enable it with 'git config [--system|--global] core.symlinks true'. The test suite ignores system / global config files. Allow testing *with* symlink support by checking if native symlinks are enabled in MSys2 (via 'MSYS=winsymlinks:nativestrict'). Reminder: This would need to be changed if / when we find a way to run the test suite in a non-MSys-based shell (e.g. dash). Signed-off-by: Karsten Blees --- compat/mingw.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/compat/mingw.c b/compat/mingw.c index 78c8053c74..922241a979 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2366,6 +2366,15 @@ static void setup_windows_environment(void) setenv("HOME", tmp, 1); } + /* + * Change 'core.symlinks' default to false, unless native symlinks are + * enabled in MSys2 (via 'MSYS=winsymlinks:nativestrict'). Thus we can + * run the test suite (which doesn't obey config files) with or without + * symlink support. + */ + if (!(tmp = getenv("MSYS")) || !strstr(tmp, "winsymlinks:nativestrict")) + has_symlinks = 0; + if (!getenv("LC_ALL") && !getenv("LC_CTYPE") && !getenv("LANG")) setenv("LC_CTYPE", "C", 1); } From e55a5a8c65f11174f97817ac566b7573283cce0d Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sat, 16 May 2015 00:32:03 +0200 Subject: [PATCH 31/41] Win32: add symlink-specific error codes Signed-off-by: Karsten Blees --- compat/mingw.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/compat/mingw.c b/compat/mingw.c index 922241a979..75170a1fcc 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -86,6 +86,7 @@ int err_win_to_posix(DWORD winerr) case ERROR_INVALID_PARAMETER: error = EINVAL; break; case ERROR_INVALID_PASSWORD: error = EPERM; break; case ERROR_INVALID_PRIMARY_GROUP: error = EINVAL; break; + case ERROR_INVALID_REPARSE_DATA: error = EINVAL; break; case ERROR_INVALID_SIGNAL_NUMBER: error = EINVAL; break; case ERROR_INVALID_TARGET_HANDLE: error = EIO; break; case ERROR_INVALID_WORKSTATION: error = EACCES; break; @@ -100,6 +101,7 @@ int err_win_to_posix(DWORD winerr) case ERROR_NEGATIVE_SEEK: error = ESPIPE; break; case ERROR_NOACCESS: error = EFAULT; break; case ERROR_NONE_MAPPED: error = EINVAL; break; + case ERROR_NOT_A_REPARSE_POINT: error = EINVAL; break; case ERROR_NOT_ENOUGH_MEMORY: error = ENOMEM; break; case ERROR_NOT_READY: error = EAGAIN; break; case ERROR_NOT_SAME_DEVICE: error = EXDEV; break; @@ -120,6 +122,9 @@ int err_win_to_posix(DWORD winerr) case ERROR_PIPE_NOT_CONNECTED: error = EPIPE; break; case ERROR_PRIVILEGE_NOT_HELD: error = EACCES; break; case ERROR_READ_FAULT: error = EIO; break; + case ERROR_REPARSE_ATTRIBUTE_CONFLICT: error = EINVAL; break; + case ERROR_REPARSE_TAG_INVALID: error = EINVAL; break; + case ERROR_REPARSE_TAG_MISMATCH: error = EINVAL; break; case ERROR_SEEK: error = EIO; break; case ERROR_SEEK_ON_DEVICE: error = ESPIPE; break; case ERROR_SHARING_BUFFER_EXCEEDED: error = ENFILE; break; From 003c92a1b9ca49bc92f74314cbf95f1917c11f87 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sun, 24 May 2015 01:06:10 +0200 Subject: [PATCH 32/41] Win32: mingw_unlink: support symlinks to directories _wunlink() / DeleteFileW() refuses to delete symlinks to directories. If _wunlink() fails with ERROR_ACCESS_DENIED, try _wrmdir() as well. Signed-off-by: Karsten Blees --- compat/mingw.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/compat/mingw.c b/compat/mingw.c index 75170a1fcc..2ad018769a 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -323,6 +323,13 @@ int mingw_unlink(const char *pathname) return 0; if (!is_file_in_use_error(GetLastError())) break; + /* + * _wunlink() / DeleteFileW() for directory symlinks fails with + * ERROR_ACCESS_DENIED (EACCES), so try _wrmdir() as well. This is the + * same error we get if a file is in use (already checked above). + */ + if (!_wrmdir(wpathname)) + return 0; } while (retry_ask_yes_no(&tries, "Unlink of file '%s' failed. " "Should I try again?", pathname)); return -1; From 2989d2a8395cb82a2f7fbe16091c43a8e2f076e0 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Tue, 19 May 2015 22:42:48 +0200 Subject: [PATCH 33/41] Win32: mingw_rename: support renaming symlinks MSVCRT's _wrename() cannot rename symlinks over existing files: it returns success without doing anything. Newer MSVCR*.dll versions probably do not have this problem: according to CRT sources, they just call MoveFileEx() with the MOVEFILE_COPY_ALLOWED flag. Get rid of _wrename() and call MoveFileEx() with proper error handling. Signed-off-by: Karsten Blees --- compat/mingw.c | 38 +++++++++++++++++--------------------- 1 file changed, 17 insertions(+), 21 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 2ad018769a..90875f6145 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -1876,27 +1876,29 @@ int mingw_accept(int sockfd1, struct sockaddr *sa, socklen_t *sz) #undef rename int mingw_rename(const char *pold, const char *pnew) { - DWORD attrs, gle; + DWORD attrs = INVALID_FILE_ATTRIBUTES, gle; int tries = 0; wchar_t wpold[MAX_LONG_PATH], wpnew[MAX_LONG_PATH]; if (xutftowcs_long_path(wpold, pold) < 0 || xutftowcs_long_path(wpnew, pnew) < 0) return -1; - /* - * Try native rename() first to get errno right. - * It is based on MoveFile(), which cannot overwrite existing files. - */ - if (!_wrename(wpold, wpnew)) - return 0; - if (errno != EEXIST) - return -1; repeat: - if (MoveFileExW(wpold, wpnew, MOVEFILE_REPLACE_EXISTING)) + if (MoveFileExW(wpold, wpnew, + MOVEFILE_REPLACE_EXISTING | MOVEFILE_COPY_ALLOWED)) return 0; - /* TODO: translate more errors */ gle = GetLastError(); - if (gle == ERROR_ACCESS_DENIED && + + /* revert file attributes on failure */ + if (attrs != INVALID_FILE_ATTRIBUTES) + SetFileAttributesW(wpnew, attrs); + + if (!is_file_in_use_error(gle)) { + errno = err_win_to_posix(gle); + return -1; + } + + if (attrs == INVALID_FILE_ATTRIBUTES && (attrs = GetFileAttributesW(wpnew)) != INVALID_FILE_ATTRIBUTES) { if (attrs & FILE_ATTRIBUTE_DIRECTORY) { DWORD attrsold = GetFileAttributesW(wpold); @@ -1908,16 +1910,10 @@ repeat: return -1; } if ((attrs & FILE_ATTRIBUTE_READONLY) && - SetFileAttributesW(wpnew, attrs & ~FILE_ATTRIBUTE_READONLY)) { - if (MoveFileExW(wpold, wpnew, MOVEFILE_REPLACE_EXISTING)) - return 0; - gle = GetLastError(); - /* revert file attributes on failure */ - SetFileAttributesW(wpnew, attrs); - } + SetFileAttributesW(wpnew, attrs & ~FILE_ATTRIBUTE_READONLY)) + goto repeat; } - if (gle == ERROR_ACCESS_DENIED && - retry_ask_yes_no(&tries, "Rename from '%s' to '%s' failed. " + if (retry_ask_yes_no(&tries, "Rename from '%s' to '%s' failed. " "Should I try again?", pold, pnew)) goto repeat; From a6aae1d360e3da4090da98fd32079ed96e842ac7 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sun, 24 May 2015 01:17:31 +0200 Subject: [PATCH 34/41] Win32: mingw_chdir: change to symlink-resolved directory If symlinks are enabled, resolve all symlinks when changing directories, as required by POSIX. Note: Git's real_path() function bases its link resolution algorithm on this property of chdir(). Unfortunately, the current directory on Windows is limited to only MAX_PATH (260) characters. Therefore using symlinks and long paths in combination may be problematic. Signed-off-by: Karsten Blees Signed-off-by: Johannes Schindelin --- compat/mingw.c | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/compat/mingw.c b/compat/mingw.c index 90875f6145..bcc30589d2 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -671,7 +671,24 @@ int mingw_chdir(const char *dirname) wchar_t wdirname[MAX_LONG_PATH]; if (xutftowcs_long_path(wdirname, dirname) < 0) return -1; - result = _wchdir(wdirname); + + if (has_symlinks) { + HANDLE hnd = CreateFileW(wdirname, 0, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, NULL, + OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); + if (hnd == INVALID_HANDLE_VALUE) { + errno = err_win_to_posix(GetLastError()); + return -1; + } + if (!GetFinalPathNameByHandleW(hnd, wdirname, ARRAY_SIZE(wdirname), 0)) { + errno = err_win_to_posix(GetLastError()); + CloseHandle(hnd); + return -1; + } + CloseHandle(hnd); + } + + result = _wchdir(normalize_ntpath(wdirname)); current_directory_len = GetCurrentDirectoryW(0, NULL); return result; } From 9d9231171610143be0ecc719e4b1d64304a2d693 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sun, 24 May 2015 01:24:41 +0200 Subject: [PATCH 35/41] Win32: implement readlink() Implement readlink() by reading NTFS reparse points. Works for symlinks and directory junctions. If symlinks are disabled, fail with ENOSYS. Signed-off-by: Karsten Blees --- compat/mingw.c | 98 ++++++++++++++++++++++++++++++++++++++++++++++++++ compat/mingw.h | 3 +- 2 files changed, 99 insertions(+), 2 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index bcc30589d2..699577c797 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2,6 +2,7 @@ #include "win32.h" #include #include +#include #include "../strbuf.h" #include "../run-command.h" #include "../cache.h" @@ -2202,6 +2203,103 @@ int link(const char *oldpath, const char *newpath) return 0; } +#ifndef _WINNT_H +/* + * The REPARSE_DATA_BUFFER structure is defined in the Windows DDK (in + * ntifs.h) and in MSYS1's winnt.h (which defines _WINNT_H). So define + * it ourselves if we are on MSYS2 (whose winnt.h defines _WINNT_). + */ +typedef struct _REPARSE_DATA_BUFFER { + DWORD ReparseTag; + WORD ReparseDataLength; + WORD Reserved; +#ifndef _MSC_VER + _ANONYMOUS_UNION +#endif + union { + struct { + WORD SubstituteNameOffset; + WORD SubstituteNameLength; + WORD PrintNameOffset; + WORD PrintNameLength; + ULONG Flags; + WCHAR PathBuffer[1]; + } SymbolicLinkReparseBuffer; + struct { + WORD SubstituteNameOffset; + WORD SubstituteNameLength; + WORD PrintNameOffset; + WORD PrintNameLength; + WCHAR PathBuffer[1]; + } MountPointReparseBuffer; + struct { + BYTE DataBuffer[1]; + } GenericReparseBuffer; + } DUMMYUNIONNAME; +} REPARSE_DATA_BUFFER, *PREPARSE_DATA_BUFFER; +#endif + +int readlink(const char *path, char *buf, size_t bufsiz) +{ + HANDLE handle; + WCHAR wpath[MAX_LONG_PATH], *wbuf; + REPARSE_DATA_BUFFER *b = alloca(MAXIMUM_REPARSE_DATA_BUFFER_SIZE); + DWORD dummy; + char tmpbuf[MAX_LONG_PATH]; + int len; + + if (xutftowcs_long_path(wpath, path) < 0) + return -1; + + /* read reparse point data */ + handle = CreateFileW(wpath, 0, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, NULL, + OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT, NULL); + if (handle == INVALID_HANDLE_VALUE) { + errno = err_win_to_posix(GetLastError()); + return -1; + } + if (!DeviceIoControl(handle, FSCTL_GET_REPARSE_POINT, NULL, 0, b, + MAXIMUM_REPARSE_DATA_BUFFER_SIZE, &dummy, NULL)) { + errno = err_win_to_posix(GetLastError()); + CloseHandle(handle); + return -1; + } + CloseHandle(handle); + + /* get target path for symlinks or mount points (aka 'junctions') */ + switch (b->ReparseTag) { + case IO_REPARSE_TAG_SYMLINK: + wbuf = (WCHAR*) (((char*) b->SymbolicLinkReparseBuffer.PathBuffer) + + b->SymbolicLinkReparseBuffer.SubstituteNameOffset); + *(WCHAR*) (((char*) wbuf) + + b->SymbolicLinkReparseBuffer.SubstituteNameLength) = 0; + break; + case IO_REPARSE_TAG_MOUNT_POINT: + wbuf = (WCHAR*) (((char*) b->MountPointReparseBuffer.PathBuffer) + + b->MountPointReparseBuffer.SubstituteNameOffset); + *(WCHAR*) (((char*) wbuf) + + b->MountPointReparseBuffer.SubstituteNameLength) = 0; + break; + default: + errno = EINVAL; + return -1; + } + + /* + * Adapt to strange readlink() API: Copy up to bufsiz *bytes*, potentially + * cutting off a UTF-8 sequence. Insufficient bufsize is *not* a failure + * condition. There is no conversion function that produces invalid UTF-8, + * so convert to a (hopefully large enough) temporary buffer, then memcpy + * the requested number of bytes (including '\0' for robustness). + */ + if ((len = xwcstoutf(tmpbuf, normalize_ntpath(wbuf), MAX_LONG_PATH)) < 0) + return -1; + memcpy(buf, tmpbuf, min(bufsiz, len + 1)); + return min(bufsiz, len); +} + pid_t waitpid(pid_t pid, int *status, int options) { HANDLE h = OpenProcess(SYNCHRONIZE | PROCESS_QUERY_INFORMATION, diff --git a/compat/mingw.h b/compat/mingw.h index b154150791..06463b4aeb 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -142,8 +142,6 @@ struct utsname { * trivial stubs */ -static inline int readlink(const char *path, char *buf, size_t bufsiz) -{ errno = ENOSYS; return -1; } static inline int symlink(const char *oldpath, const char *newpath) { errno = ENOSYS; return -1; } static inline int fchmod(int fildes, mode_t mode) @@ -237,6 +235,7 @@ int setitimer(int type, struct itimerval *in, struct itimerval *out); int sigaction(int sig, struct sigaction *in, struct sigaction *out); int link(const char *oldpath, const char *newpath); int uname(struct utsname *buf); +int readlink(const char *path, char *buf, size_t bufsiz); /* * replacements of existing functions From 7997eaee9cf8d20951ebd851cd3dee6e2996c359 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sun, 24 May 2015 01:32:03 +0200 Subject: [PATCH 36/41] Win32: implement basic symlink() functionality (file symlinks only) Implement symlink() that always creates file symlinks. Fails with ENOSYS if symlinks are disabled or unsupported. Note: CreateSymbolicLinkW() was introduced with symlink support in Windows Vista. For compatibility with Windows XP, we need to load it dynamically and fail gracefully if it isnt's available. Signed-off-by: Karsten Blees --- compat/mingw.c | 28 ++++++++++++++++++++++++++++ compat/mingw.h | 3 +-- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 699577c797..ce4ebcf1b5 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2203,6 +2203,34 @@ int link(const char *oldpath, const char *newpath) return 0; } +int symlink(const char *target, const char *link) +{ + wchar_t wtarget[MAX_LONG_PATH], wlink[MAX_LONG_PATH]; + int len; + + /* fail if symlinks are disabled or API is not supported (WinXP) */ + if (!has_symlinks) { + errno = ENOSYS; + return -1; + } + + if ((len = xutftowcs_long_path(wtarget, target)) < 0 + || xutftowcs_long_path(wlink, link) < 0) + return -1; + + /* convert target dir separators to backslashes */ + while (len--) + if (wtarget[len] == '/') + wtarget[len] = '\\'; + + /* create file symlink */ + if (!CreateSymbolicLinkW(wlink, wtarget, 0)) { + errno = err_win_to_posix(GetLastError()); + return -1; + } + return 0; +} + #ifndef _WINNT_H /* * The REPARSE_DATA_BUFFER structure is defined in the Windows DDK (in diff --git a/compat/mingw.h b/compat/mingw.h index 06463b4aeb..3024bd80f1 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -142,8 +142,6 @@ struct utsname { * trivial stubs */ -static inline int symlink(const char *oldpath, const char *newpath) -{ errno = ENOSYS; return -1; } static inline int fchmod(int fildes, mode_t mode) { errno = ENOSYS; return -1; } #ifndef __MINGW64_VERSION_MAJOR @@ -235,6 +233,7 @@ int setitimer(int type, struct itimerval *in, struct itimerval *out); int sigaction(int sig, struct sigaction *in, struct sigaction *out); int link(const char *oldpath, const char *newpath); int uname(struct utsname *buf); +int symlink(const char *target, const char *link); int readlink(const char *path, char *buf, size_t bufsiz); /* From e93c8f1445a602b9778da47614e91d0670ed9145 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sun, 24 May 2015 01:48:35 +0200 Subject: [PATCH 37/41] Win32: symlink: add support for symlinks to directories Symlinks on Windows have a flag that indicates whether the target is a file or a directory. Symlinks of wrong type simply don't work. This even affects core Win32 APIs (e.g. DeleteFile() refuses to delete directory symlinks). However, CreateFile() with FILE_FLAG_BACKUP_SEMANTICS doesn't seem to care. Check the target type by first creating a tentative file symlink, opening it, and checking the type of the resulting handle. If it is a directory, recreate the symlink with the directory flag set. It is possible to create symlinks before the target exists (or in case of symlinks to symlinks: before the target type is known). If this happens, create a tentative file symlink and postpone the directory decision: keep a list of phantom symlinks to be processed whenever a new directory is created in mingw_mkdir(). Limitations: This algorithm may fail if a link target changes from file to directory or vice versa, or if the target directory is created in another process. Signed-off-by: Karsten Blees Signed-off-by: Johannes Schindelin --- compat/mingw.c | 164 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 164 insertions(+) diff --git a/compat/mingw.c b/compat/mingw.c index ce4ebcf1b5..ddc8e3095d 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -285,6 +285,131 @@ int mingw_core_config(const char *var, const char *value, void *cb) return 0; } +enum phantom_symlink_result { + PHANTOM_SYMLINK_RETRY, + PHANTOM_SYMLINK_DONE, + PHANTOM_SYMLINK_DIRECTORY +}; + +static inline int is_wdir_sep(wchar_t wchar) +{ + return wchar == L'/' || wchar == L'\\'; +} + +static const wchar_t *make_relative_to(const wchar_t *path, + const wchar_t *relative_to, wchar_t *out, + size_t size) +{ + size_t i = wcslen(relative_to), len; + + /* Is `path` already absolute? */ + if (is_wdir_sep(path[0]) || + (iswalpha(path[0]) && path[1] == L':' && is_wdir_sep(path[2]))) + return path; + + while (i > 0 && !is_wdir_sep(relative_to[i - 1])) + i--; + + /* Is `relative_to` in the current directory? */ + if (!i) + return path; + + len = wcslen(path); + if (i + len + 1 > size) { + error("Could not make '%S' relative to '%S' (too large)", + path, relative_to); + return NULL; + } + + memcpy(out, relative_to, i * sizeof(wchar_t)); + wcscpy(out + i, path); + return out; +} + +/* + * Changes a file symlink to a directory symlink if the target exists and is a + * directory. + */ +static enum phantom_symlink_result +process_phantom_symlink(const wchar_t *wtarget, const wchar_t *wlink) +{ + HANDLE hnd; + BY_HANDLE_FILE_INFORMATION fdata; + wchar_t relative[MAX_LONG_PATH]; + const wchar_t *rel; + + /* check that wlink is still a file symlink */ + if ((GetFileAttributesW(wlink) + & (FILE_ATTRIBUTE_REPARSE_POINT | FILE_ATTRIBUTE_DIRECTORY)) + != FILE_ATTRIBUTE_REPARSE_POINT) + return PHANTOM_SYMLINK_DONE; + + /* make it relative, if necessary */ + rel = make_relative_to(wtarget, wlink, relative, ARRAY_SIZE(relative)); + if (!rel) + return PHANTOM_SYMLINK_DONE; + + /* let Windows resolve the link by opening it */ + hnd = CreateFileW(rel, 0, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, NULL, + OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); + if (hnd == INVALID_HANDLE_VALUE) { + errno = err_win_to_posix(GetLastError()); + return PHANTOM_SYMLINK_RETRY; + } + + if (!GetFileInformationByHandle(hnd, &fdata)) { + errno = err_win_to_posix(GetLastError()); + CloseHandle(hnd); + return PHANTOM_SYMLINK_RETRY; + } + CloseHandle(hnd); + + /* if target exists and is a file, we're done */ + if (!(fdata.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)) + return PHANTOM_SYMLINK_DONE; + + /* otherwise recreate the symlink with directory flag */ + if (DeleteFileW(wlink) && CreateSymbolicLinkW(wlink, wtarget, 1)) + return PHANTOM_SYMLINK_DIRECTORY; + + errno = err_win_to_posix(GetLastError()); + return PHANTOM_SYMLINK_RETRY; +} + +/* keep track of newly created symlinks to non-existing targets */ +struct phantom_symlink_info { + struct phantom_symlink_info *next; + wchar_t *wlink; + wchar_t *wtarget; +}; + +static struct phantom_symlink_info *phantom_symlinks = NULL; +static CRITICAL_SECTION phantom_symlinks_cs; + +static void process_phantom_symlinks(void) +{ + struct phantom_symlink_info *current, **psi; + EnterCriticalSection(&phantom_symlinks_cs); + /* process phantom symlinks list */ + psi = &phantom_symlinks; + while ((current = *psi)) { + enum phantom_symlink_result result = process_phantom_symlink( + current->wtarget, current->wlink); + if (result == PHANTOM_SYMLINK_RETRY) { + psi = ¤t->next; + } else { + /* symlink was processed, remove from list */ + *psi = current->next; + free(current); + /* if symlink was a directory, start over */ + if (result == PHANTOM_SYMLINK_DIRECTORY) + psi = &phantom_symlinks; + } + } + LeaveCriticalSection(&phantom_symlinks_cs); +} + /* Normalizes NT paths as returned by some low-level APIs. */ static wchar_t *normalize_ntpath(wchar_t *wbuf) { @@ -434,6 +559,8 @@ int mingw_mkdir(const char *path, int mode) return -1; ret = _wmkdir(wpath); + if (!ret) + process_phantom_symlinks(); if (!ret && needs_hiding(path)) return set_hidden_flag(wpath, 1); return ret; @@ -2228,6 +2355,42 @@ int symlink(const char *target, const char *link) errno = err_win_to_posix(GetLastError()); return -1; } + + /* convert to directory symlink if target exists */ + switch (process_phantom_symlink(wtarget, wlink)) { + case PHANTOM_SYMLINK_RETRY: { + /* if target doesn't exist, add to phantom symlinks list */ + wchar_t wfullpath[MAX_LONG_PATH]; + struct phantom_symlink_info *psi; + + /* convert to absolute path to be independent of cwd */ + len = GetFullPathNameW(wlink, MAX_LONG_PATH, wfullpath, NULL); + if (!len || len >= MAX_LONG_PATH) { + errno = err_win_to_posix(GetLastError()); + return -1; + } + + /* over-allocate and fill phantom_symlink_info structure */ + psi = xmalloc(sizeof(struct phantom_symlink_info) + + sizeof(wchar_t) * (len + wcslen(wtarget) + 2)); + psi->wlink = (wchar_t *)(psi + 1); + wcscpy(psi->wlink, wfullpath); + psi->wtarget = psi->wlink + len + 1; + wcscpy(psi->wtarget, wtarget); + + EnterCriticalSection(&phantom_symlinks_cs); + psi->next = phantom_symlinks; + phantom_symlinks = psi; + LeaveCriticalSection(&phantom_symlinks_cs); + break; + } + case PHANTOM_SYMLINK_DIRECTORY: + /* if we created a dir symlink, process other phantom symlinks */ + process_phantom_symlinks(); + break; + default: + break; + } return 0; } @@ -2737,6 +2900,7 @@ int wmain(int argc, const wchar_t **wargv) /* initialize critical section for waitpid pinfo_t list */ InitializeCriticalSection(&pinfo_cs); + InitializeCriticalSection(&phantom_symlinks_cs); /* set up default file mode and file modes for stdin/out/err */ _fmode = _O_BINARY; From f4f24741bcd6e27f85335612c4f0d170ae4416d3 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 30 May 2017 21:50:57 +0200 Subject: [PATCH 38/41] mingw: try to create symlinks without elevated permissions With Windows 10 Build 14972 in Developer Mode, a new flag is supported by CreateSymbolicLink() to create symbolic links even when running outside of an elevated session (which was previously required). This new flag is called SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE and has the numeric value 0x02. Previous Windows 10 versions will not understand that flag and return an ERROR_INVALID_PARAMETER, therefore we have to be careful to try passing that flag only when the build number indicates that it is supported. For more information about the new flag, see this blog post: https://blogs.windows.com/buildingapps/2016/12/02/symlinks-windows-10/ This patch is loosely based on the patch submitted by Samuel D. Leslie as https://github.com/git-for-windows/git/pull/1184. Signed-off-by: Johannes Schindelin --- compat/mingw.c | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index ddc8e3095d..ab6a99d16a 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -285,6 +285,8 @@ int mingw_core_config(const char *var, const char *value, void *cb) return 0; } +static DWORD symlink_file_flags = 0, symlink_directory_flags = 1; + enum phantom_symlink_result { PHANTOM_SYMLINK_RETRY, PHANTOM_SYMLINK_DONE, @@ -370,7 +372,8 @@ process_phantom_symlink(const wchar_t *wtarget, const wchar_t *wlink) return PHANTOM_SYMLINK_DONE; /* otherwise recreate the symlink with directory flag */ - if (DeleteFileW(wlink) && CreateSymbolicLinkW(wlink, wtarget, 1)) + if (DeleteFileW(wlink) && + CreateSymbolicLinkW(wlink, wtarget, symlink_directory_flags)) return PHANTOM_SYMLINK_DIRECTORY; errno = err_win_to_posix(GetLastError()); @@ -2351,7 +2354,7 @@ int symlink(const char *target, const char *link) wtarget[len] = '\\'; /* create file symlink */ - if (!CreateSymbolicLinkW(wlink, wtarget, 0)) { + if (!CreateSymbolicLinkW(wlink, wtarget, symlink_file_flags)) { errno = err_win_to_posix(GetLastError()); return -1; } @@ -2839,6 +2842,24 @@ static void maybe_redirect_std_handles(void) GENERIC_WRITE, FILE_FLAG_NO_BUFFERING); } +static void adjust_symlink_flags(void) +{ + /* + * Starting with Windows 10 Build 14972, symbolic links can be created + * using CreateSymbolicLink() without elevation by passing the flag + * SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE (0x02) as last + * parameter, provided the Developer Mode has been enabled. Some + * earlier Windows versions complain about this flag with an + * ERROR_INVALID_PARAMETER, hence we have to test the build number + * specifically. + */ + if (GetVersion() >= 14972 << 16) { + symlink_file_flags |= 2; + symlink_directory_flags |= 2; + } + +} + #ifdef _MSC_VER #ifdef _DEBUG #include @@ -2871,6 +2892,7 @@ int wmain(int argc, const wchar_t **wargv) #endif maybe_redirect_std_handles(); + adjust_symlink_flags(); /* determine size of argv and environ conversion buffer */ maxlen = wcslen(wargv[0]); From 3847c8f0a20861b1236c2ab0bfdac339e06ad099 Mon Sep 17 00:00:00 2001 From: Bert Belder Date: Fri, 26 Oct 2018 11:13:45 +0200 Subject: [PATCH 39/41] Win32: symlink: move phantom symlink creation to a separate function Signed-off-by: Bert Belder --- compat/mingw.c | 91 +++++++++++++++++++++++++++----------------------- 1 file changed, 49 insertions(+), 42 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index ab6a99d16a..a6177e5f46 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -413,6 +413,54 @@ static void process_phantom_symlinks(void) LeaveCriticalSection(&phantom_symlinks_cs); } +static int create_phantom_symlink(wchar_t *wtarget, wchar_t *wlink) +{ + int len; + + /* create file symlink */ + if (!CreateSymbolicLinkW(wlink, wtarget, symlink_file_flags)) { + errno = err_win_to_posix(GetLastError()); + return -1; + } + + /* convert to directory symlink if target exists */ + switch (process_phantom_symlink(wtarget, wlink)) { + case PHANTOM_SYMLINK_RETRY: { + /* if target doesn't exist, add to phantom symlinks list */ + wchar_t wfullpath[MAX_LONG_PATH]; + struct phantom_symlink_info *psi; + + /* convert to absolute path to be independent of cwd */ + len = GetFullPathNameW(wlink, MAX_LONG_PATH, wfullpath, NULL); + if (!len || len >= MAX_LONG_PATH) { + errno = err_win_to_posix(GetLastError()); + return -1; + } + + /* over-allocate and fill phantom_symlink_info structure */ + psi = xmalloc(sizeof(struct phantom_symlink_info) + + sizeof(wchar_t) * (len + wcslen(wtarget) + 2)); + psi->wlink = (wchar_t *)(psi + 1); + wcscpy(psi->wlink, wfullpath); + psi->wtarget = psi->wlink + len + 1; + wcscpy(psi->wtarget, wtarget); + + EnterCriticalSection(&phantom_symlinks_cs); + psi->next = phantom_symlinks; + phantom_symlinks = psi; + LeaveCriticalSection(&phantom_symlinks_cs); + break; + } + case PHANTOM_SYMLINK_DIRECTORY: + /* if we created a dir symlink, process other phantom symlinks */ + process_phantom_symlinks(); + break; + default: + break; + } + return 0; +} + /* Normalizes NT paths as returned by some low-level APIs. */ static wchar_t *normalize_ntpath(wchar_t *wbuf) { @@ -2353,48 +2401,7 @@ int symlink(const char *target, const char *link) if (wtarget[len] == '/') wtarget[len] = '\\'; - /* create file symlink */ - if (!CreateSymbolicLinkW(wlink, wtarget, symlink_file_flags)) { - errno = err_win_to_posix(GetLastError()); - return -1; - } - - /* convert to directory symlink if target exists */ - switch (process_phantom_symlink(wtarget, wlink)) { - case PHANTOM_SYMLINK_RETRY: { - /* if target doesn't exist, add to phantom symlinks list */ - wchar_t wfullpath[MAX_LONG_PATH]; - struct phantom_symlink_info *psi; - - /* convert to absolute path to be independent of cwd */ - len = GetFullPathNameW(wlink, MAX_LONG_PATH, wfullpath, NULL); - if (!len || len >= MAX_LONG_PATH) { - errno = err_win_to_posix(GetLastError()); - return -1; - } - - /* over-allocate and fill phantom_symlink_info structure */ - psi = xmalloc(sizeof(struct phantom_symlink_info) - + sizeof(wchar_t) * (len + wcslen(wtarget) + 2)); - psi->wlink = (wchar_t *)(psi + 1); - wcscpy(psi->wlink, wfullpath); - psi->wtarget = psi->wlink + len + 1; - wcscpy(psi->wtarget, wtarget); - - EnterCriticalSection(&phantom_symlinks_cs); - psi->next = phantom_symlinks; - phantom_symlinks = psi; - LeaveCriticalSection(&phantom_symlinks_cs); - break; - } - case PHANTOM_SYMLINK_DIRECTORY: - /* if we created a dir symlink, process other phantom symlinks */ - process_phantom_symlinks(); - break; - default: - break; - } - return 0; + return create_phantom_symlink(wtarget, wlink); } #ifndef _WINNT_H From c2894b6e609e47d4aa4c5f346956592a24bb7744 Mon Sep 17 00:00:00 2001 From: Bert Belder Date: Fri, 26 Oct 2018 11:51:51 +0200 Subject: [PATCH 40/41] Win32: symlink: specify symlink type in .gitattributes On Windows, symbolic links have a type: a "file symlink" must point at a file, and a "directory symlink" must point at a directory. If the type of symlink does not match its target, it doesn't work. Git does not record the type of symlink in the index or in a tree. On checkout it'll guess the type, which only works if the target exists at the time the symlink is created. This may often not be the case, for example when the link points at a directory inside a submodule. By specifying `symlink=file` or `symlink=dir` the user can specify what type of symlink Git should create, so Git doesn't have to rely on unreliable heuristics. Signed-off-by: Bert Belder Signed-off-by: Johannes Schindelin --- Documentation/gitattributes.txt | 30 ++++++++++++++++++ compat/mingw.c | 54 ++++++++++++++++++++++++++++++++- 2 files changed, 83 insertions(+), 1 deletion(-) diff --git a/Documentation/gitattributes.txt b/Documentation/gitattributes.txt index 9b41f81c06..43abfb0e0b 100644 --- a/Documentation/gitattributes.txt +++ b/Documentation/gitattributes.txt @@ -382,6 +382,36 @@ sign `$` upon checkout. Any byte sequence that begins with with `$Id$` upon check-in. +`symlink` +^^^^^^^^^ + +On Windows, symbolic links have a type: a "file symlink" must point at +a file, and a "directory symlink" must point at a directory. If the +type of symlink does not match its target, it doesn't work. + +Git does not record the type of symlink in the index or in a tree. On +checkout it'll guess the type, which only works if the target exists +at the time the symlink is created. This may often not be the case, +for example when the link points at a directory inside a submodule. + +The `symlink` attribute allows you to explicitly set the type of symlink +to `file` or `dir`, so Git doesn't have to guess. If you have a set of +symlinks that point at other files, you can do: + +------------------------ +*.gif symlink=file +------------------------ + +To tell Git that a symlink points at a directory, use: + +------------------------ +tools_folder symlink=dir +------------------------ + +The `symlink` attribute is ignored on platforms other than Windows, +since they don't distinguish between different types of symlinks. + + `filter` ^^^^^^^^ diff --git a/compat/mingw.c b/compat/mingw.c index a6177e5f46..3980eb1255 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -9,6 +9,7 @@ #include "win32/lazyload.h" #include "../config.h" #include "dir.h" +#include "../attr.h" #define HCAST(type, handle) ((type)(intptr_t)handle) @@ -2381,6 +2382,33 @@ int link(const char *oldpath, const char *newpath) return 0; } +enum symlink_type { + SYMLINK_TYPE_UNSPECIFIED = 0, + SYMLINK_TYPE_FILE, + SYMLINK_TYPE_DIRECTORY, +}; + +static enum symlink_type check_symlink_attr(const char *link) +{ + static struct attr_check *check; + const char *value; + + if (!check) + check = attr_check_initl("symlink", NULL); + + git_check_attr(the_repository->index, link, check); + + value = check->items[0].value; + if (value == NULL) + ; + else if (!strcmp(value, "file")) + return SYMLINK_TYPE_FILE; + else if (!strcmp(value, "dir")) + return SYMLINK_TYPE_DIRECTORY; + + return SYMLINK_TYPE_UNSPECIFIED; +} + int symlink(const char *target, const char *link) { wchar_t wtarget[MAX_LONG_PATH], wlink[MAX_LONG_PATH]; @@ -2401,7 +2429,31 @@ int symlink(const char *target, const char *link) if (wtarget[len] == '/') wtarget[len] = '\\'; - return create_phantom_symlink(wtarget, wlink); + switch (check_symlink_attr(link)) { + case SYMLINK_TYPE_UNSPECIFIED: + /* Create a phantom symlink: it is initially created as a file + * symlink, but may change to a directory symlink later if/when + * the target exists. */ + return create_phantom_symlink(wtarget, wlink); + case SYMLINK_TYPE_FILE: + if (!CreateSymbolicLinkW(wlink, wtarget, symlink_file_flags)) + break; + return 0; + case SYMLINK_TYPE_DIRECTORY: + if (!CreateSymbolicLinkW(wlink, wtarget, + symlink_directory_flags)) + break; + /* There may be dangling phantom symlinks that point at this + * one, which should now morph into directory symlinks. */ + process_phantom_symlinks(); + return 0; + default: + BUG("unhandled symlink type"); + } + + /* CreateSymbolicLinkW failed. */ + errno = err_win_to_posix(GetLastError()); + return -1; } #ifndef _WINNT_H From a759d2af8540cad304d2360d5b0b20a0680d2c7f Mon Sep 17 00:00:00 2001 From: Bert Belder Date: Fri, 26 Oct 2018 23:42:09 +0200 Subject: [PATCH 41/41] Win32: symlink: add test for `symlink` attribute Signed-off-by: Bert Belder --- t/t2040-checkout-symlink-attr.sh | 51 ++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100755 t/t2040-checkout-symlink-attr.sh diff --git a/t/t2040-checkout-symlink-attr.sh b/t/t2040-checkout-symlink-attr.sh new file mode 100755 index 0000000000..6b8a15116e --- /dev/null +++ b/t/t2040-checkout-symlink-attr.sh @@ -0,0 +1,51 @@ +#!/bin/sh + +test_description='checkout symlinks with `symlink` attribute on Windows + +Ensures that Git for Windows creates symlinks of the right type, +as specified by the `symlink` attribute in `.gitattributes`.' + +# Tell MSYS to create native symlinks. Without this flag test-lib's +# prerequisite detection for SYMLINKS doesn't detect the right thing. +MSYS=winsymlinks:nativestrict && export MSYS + +. ./test-lib.sh + +if ! test_have_prereq MINGW,SYMLINKS +then + skip_all='skipping $0: MinGW-only test, which requires symlink support.' + test_done +fi + +# Adds a symlink to the index without clobbering the work tree. +cache_symlink () { + sha=$(printf '%s' "$1" | git hash-object --stdin -w) && + git update-index --add --cacheinfo 120000,$sha,"$2" +} + +# MSYS2 is very forgiving, it will resolve symlinks even if the +# symlink type isn't correct. To make this test meaningful, try +# them with a native, non-MSYS executable. +cat_native () { + filename=$(cygpath -w "$1") && + cmd.exe /c "type \"$filename\"" +} + +test_expect_success 'checkout symlinks with attr' ' + cache_symlink file1 file-link && + cache_symlink dir dir-link && + + printf "file-link symlink=file\ndir-link symlink=dir\n" >.gitattributes && + git add .gitattributes && + + git checkout . && + + mkdir dir && + echo "contents1" >file1 && + echo "contents2" >dir/file2 && + + test "$(cat_native file-link)" = "contents1" && + test "$(cat_native dir-link/file2)" = "contents2" +' + +test_done