From e03056f47fb071e744eef6144aa7312109398565 Mon Sep 17 00:00:00 2001 From: nalla Date: Wed, 15 Apr 2015 07:54:39 +0100 Subject: [PATCH 01/59] asciidoctor: Fix user-manual to be built by `asciidoctor` The `user-manual.txt` ist designed as a `book` but the `Makefile` wants to build it as an `article`. This seems to be a problem when building the documentation with `asciidoctor`. Furthermore the parts *Git Glossary* and *Apendix B* had no subsections which is not allowed when building with `asciidoctor`. So lets add a *dummy* section. Signed-off-by: nalla --- Documentation/Makefile | 2 +- Documentation/user-manual.txt | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/Documentation/Makefile b/Documentation/Makefile index 3e39e2815b..df3901d83a 100644 --- a/Documentation/Makefile +++ b/Documentation/Makefile @@ -335,7 +335,7 @@ manpage-base-url.xsl: manpage-base-url.xsl.in user-manual.xml: user-manual.txt user-manual.conf $(QUIET_ASCIIDOC)$(RM) $@+ $@ && \ - $(TXT_TO_XML) -d article -o $@+ $< && \ + $(TXT_TO_XML) -d book -o $@+ $< && \ mv $@+ $@ technical/api-index.txt: technical/api-index-skel.txt \ diff --git a/Documentation/user-manual.txt b/Documentation/user-manual.txt index 68978f5338..908ac855a6 100644 --- a/Documentation/user-manual.txt +++ b/Documentation/user-manual.txt @@ -3424,7 +3424,7 @@ just missing one particular blob version. [[the-index]] The index ------------ +--------- The index is a binary file (generally kept in `.git/index`) containing a sorted list of path names, each with permissions and the SHA-1 of a blob @@ -4366,6 +4366,10 @@ itself! Git Glossary ============ +[[git-explained]] +Git explained +------------- + include::glossary-content.txt[] [[git-quick-start]] @@ -4607,6 +4611,10 @@ $ git gc Appendix B: Notes and todo list for this manual =============================================== +[[todo-list]] +Todo list +--------- + This is a work in progress. The basic requirements: From 2ec7ba1baa9e2f4f330a3a927a779d1baf840150 Mon Sep 17 00:00:00 2001 From: nalla Date: Wed, 15 Apr 2015 08:01:07 +0100 Subject: [PATCH 02/59] asciidoctor: Fix `giteveryday.txt` to be built with asciidoctor. When building the `doc` with `asciidoctor`, `asciidoctor` complains about a nested code block in a callout list. This is a really dirty solution to restore the callout list to function properly. There is a minimal visual sideeffect; the *immitated* codeblock has no overall greyish background. Instead the individual lines have it. Note: When building this patch with `asciidoc` the background is totally gone but the font is still monospaced. Signed-off-by: nalla --- Documentation/giteveryday.txt | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/Documentation/giteveryday.txt b/Documentation/giteveryday.txt index 7be6e64846..276fa033ac 100644 --- a/Documentation/giteveryday.txt +++ b/Documentation/giteveryday.txt @@ -308,18 +308,15 @@ master or exposed as a part of a stable branch. <10> create a signed tag. <11> make sure master was not accidentally rewound beyond that already pushed out. `ko` shorthand points at the Git maintainer's -repository at kernel.org, and looks like this: -+ ------------- -(in .git/config) -[remote "ko"] - url = kernel.org:/pub/scm/git/git.git - fetch = refs/heads/*:refs/remotes/ko/* - push = refs/heads/master - push = refs/heads/next - push = +refs/heads/pu - push = refs/heads/maint ------------- +repository at kernel.org, and looks like this: + +++(in .git/config) + +{startsb}remote "ko"{endsb} + +{nbsp}{nbsp}{nbsp}{nbsp}url = kernel.org:/pub/scm/git/git.git + +{nbsp}{nbsp}{nbsp}{nbsp}fetch = refs/heads/\*:refs/remotes/ko/* + +{nbsp}{nbsp}{nbsp}{nbsp}push = refs/heads/master + +{nbsp}{nbsp}{nbsp}{nbsp}push = refs/heads/next + +{nbsp}{nbsp}{nbsp}{nbsp}push = +refs/heads/pu + +{nbsp}{nbsp}{nbsp}{nbsp}push = refs/heads/maint++ + <12> In the output from `git show-branch`, `master` should have everything `ko/master` has, and `next` should have From 6e07a191f237a08a7cc64290910ac393163e0042 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sat, 6 Jul 2013 02:09:35 +0200 Subject: [PATCH 03/59] Win32: make FILETIME conversion functions public Signed-off-by: Karsten Blees --- compat/mingw.c | 16 ---------------- compat/mingw.h | 16 ++++++++++++++++ 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 7d04a4fcc6..102d6a633b 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -474,22 +474,6 @@ int mingw_chmod(const char *filename, int mode) return _wchmod(wfilename, mode); } -/* - * The unit of FILETIME is 100-nanoseconds since January 1, 1601, UTC. - * Returns the 100-nanoseconds ("hekto nanoseconds") since the epoch. - */ -static inline long long filetime_to_hnsec(const FILETIME *ft) -{ - long long winTime = ((long long)ft->dwHighDateTime << 32) + ft->dwLowDateTime; - /* Windows to Unix Epoch conversion */ - return winTime - 116444736000000000LL; -} - -static inline time_t filetime_to_time_t(const FILETIME *ft) -{ - return (time_t)(filetime_to_hnsec(ft) / 10000000); -} - /* We keep the do_lstat code in a separate function to avoid recursion. * When a path ends with a slash, the stat will fail with ENOENT. In * this case, we strip the trailing slashes and stat again. diff --git a/compat/mingw.h b/compat/mingw.h index 738865c6c0..e7e8368d0b 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -295,6 +295,22 @@ static inline int getrlimit(int resource, struct rlimit *rlp) return 0; } +/* + * The unit of FILETIME is 100-nanoseconds since January 1, 1601, UTC. + * Returns the 100-nanoseconds ("hekto nanoseconds") since the epoch. + */ +static inline long long filetime_to_hnsec(const FILETIME *ft) +{ + long long winTime = ((long long)ft->dwHighDateTime << 32) + ft->dwLowDateTime; + /* Windows to Unix Epoch conversion */ + return winTime - 116444736000000000LL; +} + +static inline time_t filetime_to_time_t(const FILETIME *ft) +{ + return (time_t)(filetime_to_hnsec(ft) / 10000000); +} + /* * Use mingw specific stat()/lstat()/fstat() implementations on Windows. */ From 34f791f9f4ba3c5072d4c0643783e5060cf27725 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sun, 8 Sep 2013 14:17:31 +0200 Subject: [PATCH 04/59] Win32: dirent.c: Move opendir down Move opendir down in preparation for the next patch. Signed-off-by: Karsten Blees --- compat/win32/dirent.c | 68 +++++++++++++++++++++---------------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/compat/win32/dirent.c b/compat/win32/dirent.c index 52420ec7d4..2603a0fa39 100644 --- a/compat/win32/dirent.c +++ b/compat/win32/dirent.c @@ -18,40 +18,6 @@ static inline void finddata2dirent(struct dirent *ent, WIN32_FIND_DATAW *fdata) ent->d_type = DT_REG; } -DIR *opendir(const char *name) -{ - wchar_t pattern[MAX_PATH + 2]; /* + 2 for '/' '*' */ - WIN32_FIND_DATAW fdata; - HANDLE h; - int len; - DIR *dir; - - /* convert name to UTF-16 and check length < MAX_PATH */ - if ((len = xutftowcs_path(pattern, name)) < 0) - return NULL; - - /* append optional '/' and wildcard '*' */ - if (len && !is_dir_sep(pattern[len - 1])) - pattern[len++] = '/'; - pattern[len++] = '*'; - pattern[len] = 0; - - /* open find handle */ - h = FindFirstFileW(pattern, &fdata); - if (h == INVALID_HANDLE_VALUE) { - DWORD err = GetLastError(); - errno = (err == ERROR_DIRECTORY) ? ENOTDIR : err_win_to_posix(err); - return NULL; - } - - /* initialize DIR structure and copy first dir entry */ - dir = xmalloc(sizeof(DIR)); - dir->dd_handle = h; - dir->dd_stat = 0; - finddata2dirent(&dir->dd_dir, &fdata); - return dir; -} - struct dirent *readdir(DIR *dir) { if (!dir) { @@ -90,3 +56,37 @@ int closedir(DIR *dir) free(dir); return 0; } + +DIR *opendir(const char *name) +{ + wchar_t pattern[MAX_PATH + 2]; /* + 2 for '/' '*' */ + WIN32_FIND_DATAW fdata; + HANDLE h; + int len; + DIR *dir; + + /* convert name to UTF-16 and check length < MAX_PATH */ + if ((len = xutftowcs_path(pattern, name)) < 0) + return NULL; + + /* append optional '/' and wildcard '*' */ + if (len && !is_dir_sep(pattern[len - 1])) + pattern[len++] = '/'; + pattern[len++] = '*'; + pattern[len] = 0; + + /* open find handle */ + h = FindFirstFileW(pattern, &fdata); + if (h == INVALID_HANDLE_VALUE) { + DWORD err = GetLastError(); + errno = (err == ERROR_DIRECTORY) ? ENOTDIR : err_win_to_posix(err); + return NULL; + } + + /* initialize DIR structure and copy first dir entry */ + dir = xmalloc(sizeof(DIR)); + dir->dd_handle = h; + dir->dd_stat = 0; + finddata2dirent(&dir->dd_dir, &fdata); + return dir; +} From 28744a20c0f37653317d4327cf451e6653f93887 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sun, 8 Sep 2013 14:18:40 +0200 Subject: [PATCH 05/59] Win32: Make the dirent implementation pluggable Emulating the POSIX dirent API on Windows via FindFirstFile/FindNextFile is pretty staightforward, however, most of the information provided in the WIN32_FIND_DATA structure is thrown away in the process. A more sophisticated implementation may cache this data, e.g. for later reuse in calls to lstat. Make the dirent implementation pluggable so that it can be switched at runtime, e.g. based on a config option. Define a base DIR structure with pointers to readdir/closedir that match the opendir implementation (i.e. similar to vtable pointers in OOP). Define readdir/closedir so that they call the function pointers in the DIR structure. This allows to choose the opendir implementation on a call-by-call basis. Move the fixed sized dirent.d_name buffer to the dirent-specific DIR structure, as d_name may be implementation specific (e.g. a caching implementation may just set d_name to point into the cache instead of copying the entire file name string). Signed-off-by: Karsten Blees --- compat/win32/dirent.c | 27 +++++++++++++++++---------- compat/win32/dirent.h | 26 +++++++++++++++++++------- 2 files changed, 36 insertions(+), 17 deletions(-) diff --git a/compat/win32/dirent.c b/compat/win32/dirent.c index 2603a0fa39..6b87042182 100644 --- a/compat/win32/dirent.c +++ b/compat/win32/dirent.c @@ -1,15 +1,19 @@ #include "../../git-compat-util.h" -struct DIR { +typedef struct dirent_DIR { + struct DIR base_dir; /* extend base struct DIR */ struct dirent dd_dir; /* includes d_type */ HANDLE dd_handle; /* FindFirstFile handle */ int dd_stat; /* 0-based index */ -}; + char dd_name[MAX_PATH * 3]; /* file name (* 3 for UTF-8 conversion) */ +} dirent_DIR; + +DIR *(*opendir)(const char *dirname) = dirent_opendir; static inline void finddata2dirent(struct dirent *ent, WIN32_FIND_DATAW *fdata) { - /* convert UTF-16 name to UTF-8 */ - xwcstoutf(ent->d_name, fdata->cFileName, sizeof(ent->d_name)); + /* convert UTF-16 name to UTF-8 (d_name points to dirent_DIR.dd_name) */ + xwcstoutf(ent->d_name, fdata->cFileName, MAX_PATH * 3); /* Set file type, based on WIN32_FIND_DATA */ if (fdata->dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) @@ -18,7 +22,7 @@ static inline void finddata2dirent(struct dirent *ent, WIN32_FIND_DATAW *fdata) ent->d_type = DT_REG; } -struct dirent *readdir(DIR *dir) +static struct dirent *dirent_readdir(dirent_DIR *dir) { if (!dir) { errno = EBADF; /* No set_errno for mingw */ @@ -45,7 +49,7 @@ struct dirent *readdir(DIR *dir) return &dir->dd_dir; } -int closedir(DIR *dir) +static int dirent_closedir(dirent_DIR *dir) { if (!dir) { errno = EBADF; @@ -57,13 +61,13 @@ int closedir(DIR *dir) return 0; } -DIR *opendir(const char *name) +DIR *dirent_opendir(const char *name) { wchar_t pattern[MAX_PATH + 2]; /* + 2 for '/' '*' */ WIN32_FIND_DATAW fdata; HANDLE h; int len; - DIR *dir; + dirent_DIR *dir; /* convert name to UTF-16 and check length < MAX_PATH */ if ((len = xutftowcs_path(pattern, name)) < 0) @@ -84,9 +88,12 @@ DIR *opendir(const char *name) } /* initialize DIR structure and copy first dir entry */ - dir = xmalloc(sizeof(DIR)); + dir = xmalloc(sizeof(dirent_DIR)); + dir->base_dir.preaddir = (struct dirent *(*)(DIR *dir)) dirent_readdir; + dir->base_dir.pclosedir = (int (*)(DIR *dir)) dirent_closedir; + dir->dd_dir.d_name = dir->dd_name; dir->dd_handle = h; dir->dd_stat = 0; finddata2dirent(&dir->dd_dir, &fdata); - return dir; + return (DIR*) dir; } diff --git a/compat/win32/dirent.h b/compat/win32/dirent.h index 058207e4bf..6b3ddee51b 100644 --- a/compat/win32/dirent.h +++ b/compat/win32/dirent.h @@ -1,20 +1,32 @@ #ifndef DIRENT_H #define DIRENT_H -typedef struct DIR DIR; - #define DT_UNKNOWN 0 #define DT_DIR 1 #define DT_REG 2 #define DT_LNK 3 struct dirent { - unsigned char d_type; /* file type to prevent lstat after readdir */ - char d_name[MAX_PATH * 3]; /* file name (* 3 for UTF-8 conversion) */ + unsigned char d_type; /* file type to prevent lstat after readdir */ + char *d_name; /* file name */ }; -DIR *opendir(const char *dirname); -struct dirent *readdir(DIR *dir); -int closedir(DIR *dir); +/* + * Base DIR structure, contains pointers to readdir/closedir implementations so + * that opendir may choose a concrete implementation on a call-by-call basis. + */ +typedef struct DIR { + struct dirent *(*preaddir)(struct DIR *dir); + int (*pclosedir)(struct DIR *dir); +} DIR; + +/* default dirent implementation */ +extern DIR *dirent_opendir(const char *dirname); + +/* current dirent implementation */ +extern DIR *(*opendir)(const char *dirname); + +#define readdir(dir) (dir->preaddir(dir)) +#define closedir(dir) (dir->pclosedir(dir)) #endif /* DIRENT_H */ From 19d862af0229b29972d6bf479dc8fd6b244e6b21 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sun, 8 Sep 2013 14:21:30 +0200 Subject: [PATCH 06/59] Win32: make the lstat implementation pluggable Emulating the POSIX lstat API on Windows via GetFileAttributes[Ex] is quite slow. Windows operating system APIs seem to be much better at scanning the status of entire directories than checking single files. A caching implementation may improve performance by bulk-reading entire directories or reusing data obtained via opendir / readdir. Make the lstat implementation pluggable so that it can be switched at runtime, e.g. based on a config option. Signed-off-by: Karsten Blees --- compat/mingw.c | 2 ++ compat/mingw.h | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/compat/mingw.c b/compat/mingw.c index 102d6a633b..44c05d2cd3 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -574,6 +574,8 @@ static int do_stat_internal(int follow, const char *file_name, struct stat *buf) return do_lstat(follow, alt_name, buf); } +int (*lstat)(const char *file_name, struct stat *buf) = mingw_lstat; + int mingw_lstat(const char *file_name, struct stat *buf) { return do_stat_internal(0, file_name, buf); diff --git a/compat/mingw.h b/compat/mingw.h index e7e8368d0b..d8905cc155 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -332,7 +332,7 @@ int mingw_fstat(int fd, struct stat *buf); #ifdef lstat #undef lstat #endif -#define lstat mingw_lstat +extern int (*lstat)(const char *file_name, struct stat *buf); #ifndef _stati64 # define _stati64(x,y) mingw_stat(x,y) From 3f3307aaffd232280211b7f9df24c2bc42d5c247 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sun, 8 Sep 2013 14:23:27 +0200 Subject: [PATCH 07/59] add infrastructure for read-only file system level caches Add a macro to mark code sections that only read from the file system, along with a config option and documentation. This facilitates implementation of relatively simple file system level caches without the need to synchronize with the file system. Enable read-only sections for 'git status' and preload_index. Signed-off-by: Karsten Blees --- Documentation/config.txt | 6 ++++++ builtin/commit.c | 1 + cache.h | 2 ++ config.c | 5 +++++ environment.c | 1 + git-compat-util.h | 15 +++++++++++++++ preload-index.c | 2 ++ 7 files changed, 32 insertions(+) diff --git a/Documentation/config.txt b/Documentation/config.txt index b4514cdda6..00ff3f5af6 100644 --- a/Documentation/config.txt +++ b/Documentation/config.txt @@ -713,6 +713,12 @@ relatively high IO latencies. When enabled, Git will do the index comparison to the filesystem data in parallel, allowing overlapping IO's. Defaults to true. +core.fscache:: + Enable additional caching of file system data for some operations. ++ +Git for Windows uses this to bulk-read and cache lstat data of entire +directories (instead of doing lstat file by file). + core.createObject:: You can set this to 'link', in which case a hardlink followed by a delete of the source are used to make sure that object creation diff --git a/builtin/commit.c b/builtin/commit.c index 1692620e25..922703caee 100644 --- a/builtin/commit.c +++ b/builtin/commit.c @@ -1360,6 +1360,7 @@ int cmd_status(int argc, const char **argv, const char *prefix) PATHSPEC_PREFER_FULL, prefix, argv); + enable_fscache(1); read_cache_preload(&s.pathspec); refresh_index(&the_index, REFRESH_QUIET|REFRESH_UNMERGED, &s.pathspec, NULL, NULL); diff --git a/cache.h b/cache.h index 759e069d92..424d495765 100644 --- a/cache.h +++ b/cache.h @@ -653,6 +653,8 @@ enum hide_dotfiles_type { }; extern enum hide_dotfiles_type hide_dotfiles; +extern int core_fscache; + enum branch_track { BRANCH_TRACK_UNSPECIFIED = -1, BRANCH_TRACK_NEVER = 0, diff --git a/config.c b/config.c index f03cc7edc6..306e562e64 100644 --- a/config.c +++ b/config.c @@ -920,6 +920,11 @@ static int git_default_core_config(const char *var, const char *value) return 0; } + if (!strcmp(var, "core.fscache")) { + core_fscache = git_config_bool(var, value); + return 0; + } + /* Add other config variables here and to Documentation/config.txt. */ return 0; } diff --git a/environment.c b/environment.c index 2e2f89505b..8803d2ebc6 100644 --- a/environment.c +++ b/environment.c @@ -65,6 +65,7 @@ int precomposed_unicode = -1; /* see probe_utf8_pathname_composition() */ struct startup_info *startup_info; unsigned long pack_size_limit_cfg; enum hide_dotfiles_type hide_dotfiles = HIDE_DOTFILES_DOTGITONLY; +int core_fscache; #ifndef PROTECT_HFS_DEFAULT #define PROTECT_HFS_DEFAULT 0 diff --git a/git-compat-util.h b/git-compat-util.h index 2eba88f8dd..5d2995d509 100644 --- a/git-compat-util.h +++ b/git-compat-util.h @@ -945,4 +945,19 @@ struct tm *git_gmtime_r(const time_t *, struct tm *); #define mark_as_git_dir(x) /* noop */ #endif +/* + * Enable/disable a read-only cache for file system data on platforms that + * support it. + * + * Implementing a live-cache is complicated and requires special platform + * support (inotify, ReadDirectoryChangesW...). enable_fscache shall be used + * to mark sections of git code that extensively read from the file system + * without modifying anything. Implementations can use this to cache e.g. stat + * data or even file content without the need to synchronize with the file + * system. + */ +#ifndef enable_fscache +#define enable_fscache(x) /* noop */ +#endif + #endif diff --git a/preload-index.c b/preload-index.c index c1fe3a3ef9..c7970d7cb9 100644 --- a/preload-index.c +++ b/preload-index.c @@ -84,6 +84,7 @@ static void preload_index(struct index_state *index, offset = 0; work = DIV_ROUND_UP(index->cache_nr, threads); memset(&data, 0, sizeof(data)); + enable_fscache(1); for (i = 0; i < threads; i++) { struct thread_data *p = data+i; p->index = index; @@ -100,6 +101,7 @@ static void preload_index(struct index_state *index, if (pthread_join(p->pthread, NULL)) die("unable to join threaded lstat"); } + enable_fscache(0); } #endif From 271768fc74201b02cfc4e673e63155c51bce5fe7 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Tue, 1 Oct 2013 12:51:54 +0200 Subject: [PATCH 08/59] Win32: add a cache below mingw's lstat and dirent implementations Checking the work tree status is quite slow on Windows, due to slow lstat emulation (git calls lstat once for each file in the index). Windows operating system APIs seem to be much better at scanning the status of entire directories than checking single files. Add an lstat implementation that uses a cache for lstat data. Cache misses read the entire parent directory and add it to the cache. Subsequent lstat calls for the same directory are served directly from the cache. Also implement opendir / readdir / closedir so that they create and use directory listings in the cache. The cache doesn't track file system changes and doesn't plug into any modifying file APIs, so it has to be explicitly enabled for git functions that don't modify the working copy. Note: in an earlier version of this patch, the cache was always active and tracked file system changes via ReadDirectoryChangesW. However, this was much more complex and had negative impact on the performance of modifying git commands such as 'git checkout'. Signed-off-by: Karsten Blees --- compat/win32/fscache.c | 458 +++++++++++++++++++++++++++++++++++++++++ compat/win32/fscache.h | 10 + config.mak.uname | 4 +- git-compat-util.h | 2 + 4 files changed, 472 insertions(+), 2 deletions(-) create mode 100644 compat/win32/fscache.c create mode 100644 compat/win32/fscache.h diff --git a/compat/win32/fscache.c b/compat/win32/fscache.c new file mode 100644 index 0000000000..5388d64377 --- /dev/null +++ b/compat/win32/fscache.c @@ -0,0 +1,458 @@ +#include "../../cache.h" +#include "../../hashmap.h" +#include "../win32.h" +#include "fscache.h" + +static int initialized; +static volatile long enabled; +static struct hashmap map; +static CRITICAL_SECTION mutex; + +/* + * An entry in the file system cache. Used for both entire directory listings + * and file entries. + */ +struct fsentry { + struct hashmap_entry ent; + mode_t st_mode; + /* Length of name. */ + unsigned short len; + /* + * Name of the entry. For directory listings: relative path of the + * directory, without trailing '/' (empty for cwd()). For file entries: + * name of the file. Typically points to the end of the structure if + * the fsentry is allocated on the heap (see fsentry_alloc), or to a + * local variable if on the stack (see fsentry_init). + */ + const char *name; + /* Pointer to the directory listing, or NULL for the listing itself. */ + struct fsentry *list; + /* Pointer to the next file entry of the list. */ + struct fsentry *next; + + union { + /* Reference count of the directory listing. */ + volatile long refcnt; + struct { + /* More stat members (only used for file entries). */ + off64_t st_size; + time_t st_atime; + time_t st_mtime; + time_t st_ctime; + }; + }; +}; + +/* + * Compares the paths of two fsentry structures for equality. + */ +static int fsentry_cmp(const struct fsentry *fse1, const struct fsentry *fse2) +{ + int res; + if (fse1 == fse2) + return 0; + + /* compare the list parts first */ + if (fse1->list != fse2->list && (res = fsentry_cmp( + fse1->list ? fse1->list : fse1, + fse2->list ? fse2->list : fse2))) + return res; + + /* if list parts are equal, compare len and name */ + if (fse1->len != fse2->len) + return fse1->len - fse2->len; + return strnicmp(fse1->name, fse2->name, fse1->len); +} + +/* + * Calculates the hash code of an fsentry structure's path. + */ +static unsigned int fsentry_hash(const struct fsentry *fse) +{ + unsigned int hash = fse->list ? fse->list->ent.hash : 0; + return hash ^ memihash(fse->name, fse->len); +} + +/* + * Initialize an fsentry structure for use by fsentry_hash and fsentry_cmp. + */ +static void fsentry_init(struct fsentry *fse, struct fsentry *list, + const char *name, size_t len) +{ + fse->list = list; + fse->name = name; + fse->len = len; + hashmap_entry_init(fse, fsentry_hash(fse)); +} + +/* + * Allocate an fsentry structure on the heap. + */ +static struct fsentry *fsentry_alloc(struct fsentry *list, const char *name, + size_t len) +{ + /* overallocate fsentry and copy the name to the end */ + struct fsentry *fse = xmalloc(sizeof(struct fsentry) + len + 1); + char *nm = ((char*) fse) + sizeof(struct fsentry); + memcpy(nm, name, len); + nm[len] = 0; + /* init the rest of the structure */ + fsentry_init(fse, list, nm, len); + fse->next = NULL; + fse->refcnt = 1; + return fse; +} + +/* + * Add a reference to an fsentry. + */ +inline static void fsentry_addref(struct fsentry *fse) +{ + if (fse->list) + fse = fse->list; + + InterlockedIncrement(&(fse->refcnt)); +} + +/* + * Release the reference to an fsentry, frees the memory if its the last ref. + */ +static void fsentry_release(struct fsentry *fse) +{ + if (fse->list) + fse = fse->list; + + if (InterlockedDecrement(&(fse->refcnt))) + return; + + while (fse) { + struct fsentry *next = fse->next; + free(fse); + fse = next; + } +} + +/* + * Allocate and initialize an fsentry from a WIN32_FIND_DATA structure. + */ +static struct fsentry *fseentry_create_entry(struct fsentry *list, + const WIN32_FIND_DATAW *fdata) +{ + char buf[MAX_PATH * 3]; + int len; + struct fsentry *fse; + len = xwcstoutf(buf, fdata->cFileName, ARRAY_SIZE(buf)); + + fse = fsentry_alloc(list, buf, len); + + fse->st_mode = file_attr_to_st_mode(fdata->dwFileAttributes); + fse->st_size = (((off64_t) (fdata->nFileSizeHigh)) << 32) + | fdata->nFileSizeLow; + fse->st_atime = filetime_to_time_t(&(fdata->ftLastAccessTime)); + fse->st_mtime = filetime_to_time_t(&(fdata->ftLastWriteTime)); + fse->st_ctime = filetime_to_time_t(&(fdata->ftCreationTime)); + + return fse; +} + +/* + * Create an fsentry-based directory listing (similar to opendir / readdir). + * Dir should not contain trailing '/'. Use an empty string for the current + * directory (not "."!). + */ +static struct fsentry *fsentry_create_list(const struct fsentry *dir) +{ + wchar_t pattern[MAX_PATH + 2]; /* + 2 for '/' '*' */ + WIN32_FIND_DATAW fdata; + HANDLE h; + int wlen; + struct fsentry *list, **phead; + DWORD err; + + /* convert name to UTF-16 and check length < MAX_PATH */ + if ((wlen = xutftowcsn(pattern, dir->name, MAX_PATH, dir->len)) < 0) { + if (errno == ERANGE) + errno = ENAMETOOLONG; + return NULL; + } + + /* append optional '/' and wildcard '*' */ + if (wlen) + pattern[wlen++] = '/'; + pattern[wlen++] = '*'; + pattern[wlen] = 0; + + /* open find handle */ + h = FindFirstFileW(pattern, &fdata); + if (h == INVALID_HANDLE_VALUE) { + err = GetLastError(); + errno = (err == ERROR_DIRECTORY) ? ENOTDIR : err_win_to_posix(err); + return NULL; + } + + /* allocate object to hold directory listing */ + list = fsentry_alloc(NULL, dir->name, dir->len); + + /* walk directory and build linked list of fsentry structures */ + phead = &list->next; + do { + *phead = fseentry_create_entry(list, &fdata); + phead = &(*phead)->next; + } while (FindNextFileW(h, &fdata)); + + /* remember result of last FindNextFile, then close find handle */ + err = GetLastError(); + FindClose(h); + + /* return the list if we've got all the files */ + if (err == ERROR_NO_MORE_FILES) + return list; + + /* otherwise free the list and return error */ + fsentry_release(list); + errno = err_win_to_posix(err); + return NULL; +} + +/* + * Adds a directory listing to the cache. + */ +static void fscache_add(struct fsentry *fse) +{ + if (fse->list) + fse = fse->list; + + for (; fse; fse = fse->next) + hashmap_add(&map, fse); +} + +/* + * Removes a directory listing from the cache. + */ +static void fscache_remove(struct fsentry *fse) +{ + if (fse->list) + fse = fse->list; + + for (; fse; fse = fse->next) + hashmap_remove(&map, fse, NULL); +} + +/* + * Clears the cache. + */ +static void fscache_clear() +{ + struct hashmap_iter iter; + struct fsentry *fse; + while ((fse = hashmap_iter_first(&map, &iter))) { + fscache_remove(fse); + fsentry_release(fse); + } +} + +/* + * Checks if the cache is enabled for the given path. + */ +static inline int fscache_enabled(const char *path) +{ + return enabled > 0 && !is_absolute_path(path); +} + +/* + * Looks up or creates a cache entry for the specified key. + */ +static struct fsentry *fscache_get(struct fsentry *key) +{ + struct fsentry *fse; + + EnterCriticalSection(&mutex); + /* check if entry is in cache */ + fse = hashmap_get(&map, key, NULL); + if (fse) { + fsentry_addref(fse); + LeaveCriticalSection(&mutex); + return fse; + } + /* if looking for a file, check if directory listing is in cache */ + if (!fse && key->list) { + fse = hashmap_get(&map, key->list, NULL); + if (fse) { + LeaveCriticalSection(&mutex); + /* dir entry without file entry -> file doesn't exist */ + errno = ENOENT; + return NULL; + } + } + + /* create the directory listing (outside mutex!) */ + LeaveCriticalSection(&mutex); + fse = fsentry_create_list(key->list ? key->list : key); + if (!fse) + return NULL; + + EnterCriticalSection(&mutex); + /* add directory listing if it hasn't been added by some other thread */ + if (!hashmap_get(&map, key, NULL)) + fscache_add(fse); + + /* lookup file entry if requested (fse already points to directory) */ + if (key->list) + fse = hashmap_get(&map, key, NULL); + + /* return entry or ENOENT */ + if (fse) + fsentry_addref(fse); + else + errno = ENOENT; + + LeaveCriticalSection(&mutex); + return fse; +} + +/* + * Enables or disables the cache. Note that the cache is read-only, changes to + * the working directory are NOT reflected in the cache while enabled. + */ +int fscache_enable(int enable) +{ + int result; + + if (!initialized) { + /* allow the cache to be disabled entirely */ + if (!core_fscache) + return 0; + + InitializeCriticalSection(&mutex); + hashmap_init(&map, (hashmap_cmp_fn) fsentry_cmp, 0); + initialized = 1; + } + + result = enable ? InterlockedIncrement(&enabled) + : InterlockedDecrement(&enabled); + + if (enable && result == 1) { + /* redirect opendir and lstat to the fscache implementations */ + opendir = fscache_opendir; + lstat = fscache_lstat; + } else if (!enable && !result) { + /* reset opendir and lstat to the original implementations */ + opendir = dirent_opendir; + lstat = mingw_lstat; + EnterCriticalSection(&mutex); + fscache_clear(); + LeaveCriticalSection(&mutex); + } + return result; +} + +/* + * Lstat replacement, uses the cache if enabled, otherwise redirects to + * mingw_lstat. + */ +int fscache_lstat(const char *filename, struct stat *st) +{ + int dirlen, base, len; + struct fsentry key[2], *fse; + + if (!fscache_enabled(filename)) + return mingw_lstat(filename, st); + + /* split filename into path + name */ + len = strlen(filename); + if (len && is_dir_sep(filename[len - 1])) + len--; + base = len; + while (base && !is_dir_sep(filename[base - 1])) + base--; + dirlen = base ? base - 1 : 0; + + /* lookup entry for path + name in cache */ + fsentry_init(key, NULL, filename, dirlen); + fsentry_init(key + 1, key, filename + base, len - base); + fse = fscache_get(key + 1); + if (!fse) + return -1; + + /* copy stat data */ + st->st_ino = 0; + st->st_gid = 0; + st->st_uid = 0; + st->st_dev = 0; + st->st_rdev = 0; + st->st_nlink = 1; + st->st_mode = fse->st_mode; + st->st_size = fse->st_size; + st->st_atime = fse->st_atime; + st->st_mtime = fse->st_mtime; + st->st_ctime = fse->st_ctime; + + /* don't forget to release fsentry */ + fsentry_release(fse); + return 0; +} + +typedef struct fscache_DIR { + struct DIR base_dir; /* extend base struct DIR */ + struct fsentry *pfsentry; + struct dirent dirent; +} fscache_DIR; + +/* + * Readdir replacement. + */ +static struct dirent *fscache_readdir(DIR *base_dir) +{ + fscache_DIR *dir = (fscache_DIR*) base_dir; + struct fsentry *next = dir->pfsentry->next; + if (!next) + return NULL; + dir->pfsentry = next; + dir->dirent.d_type = S_ISDIR(next->st_mode) ? DT_DIR : DT_REG; + dir->dirent.d_name = (char*) next->name; + return &(dir->dirent); +} + +/* + * Closedir replacement. + */ +static int fscache_closedir(DIR *base_dir) +{ + fscache_DIR *dir = (fscache_DIR*) base_dir; + fsentry_release(dir->pfsentry); + free(dir); + return 0; +} + +/* + * Opendir replacement, uses a directory listing from the cache if enabled, + * otherwise calls original dirent implementation. + */ +DIR *fscache_opendir(const char *dirname) +{ + struct fsentry key, *list; + fscache_DIR *dir; + int len; + + if (!fscache_enabled(dirname)) + return dirent_opendir(dirname); + + /* prepare name (strip trailing '/', replace '.') */ + len = strlen(dirname); + if ((len == 1 && dirname[0] == '.') || + (len && is_dir_sep(dirname[len - 1]))) + len--; + + /* get directory listing from cache */ + fsentry_init(&key, NULL, dirname, len); + list = fscache_get(&key); + if (!list) + return NULL; + + /* alloc and return DIR structure */ + dir = (fscache_DIR*) xmalloc(sizeof(fscache_DIR)); + dir->base_dir.preaddir = fscache_readdir; + dir->base_dir.pclosedir = fscache_closedir; + dir->pfsentry = list; + return (DIR*) dir; +} diff --git a/compat/win32/fscache.h b/compat/win32/fscache.h new file mode 100644 index 0000000000..ed518b422d --- /dev/null +++ b/compat/win32/fscache.h @@ -0,0 +1,10 @@ +#ifndef FSCACHE_H +#define FSCACHE_H + +int fscache_enable(int enable); +#define enable_fscache(x) fscache_enable(x) + +DIR *fscache_opendir(const char *dir); +int fscache_lstat(const char *file_name, struct stat *buf); + +#endif diff --git a/config.mak.uname b/config.mak.uname index 943c43965e..903eccd5c9 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -378,7 +378,7 @@ ifeq ($(uname_S),Windows) BASIC_CFLAGS = -nologo -I. -I../zlib -Icompat/vcbuild -Icompat/vcbuild/include -DWIN32 -D_CONSOLE -DHAVE_STRING_H -D_CRT_SECURE_NO_WARNINGS -D_CRT_NONSTDC_NO_DEPRECATE COMPAT_OBJS = compat/msvc.o compat/winansi.o \ compat/win32/pthread.o compat/win32/syslog.o \ - compat/win32/dirent.o + compat/win32/dirent.o compat/win32/fscache.o COMPAT_CFLAGS = -D__USE_MINGW_ACCESS -DNOGDI -DHAVE_STRING_H -Icompat -Icompat/regex -Icompat/win32 -DSTRIP_EXTENSION=\".exe\" BASIC_LDFLAGS = -IGNORE:4217 -IGNORE:4049 -NOLOGO -SUBSYSTEM:CONSOLE EXTLIBS = user32.lib advapi32.lib shell32.lib wininet.lib ws2_32.lib invalidcontinue.obj @@ -525,7 +525,7 @@ ifneq (,$(findstring MINGW,$(uname_S))) COMPAT_CFLAGS += -DSTRIP_EXTENSION=\".exe\" COMPAT_OBJS += compat/mingw.o compat/winansi.o \ compat/win32/pthread.o compat/win32/syslog.o \ - compat/win32/dirent.o + compat/win32/dirent.o compat/win32/fscache.o BASIC_CFLAGS += -DPROTECT_NTFS_DEFAULT=1 BASIC_LDFLAGS += -Wl,--large-address-aware EXTLIBS += -lws2_32 diff --git a/git-compat-util.h b/git-compat-util.h index 5d2995d509..420a50718c 100644 --- a/git-compat-util.h +++ b/git-compat-util.h @@ -184,8 +184,10 @@ #if defined(__MINGW32__) /* pull in Windows compatibility stuff */ #include "compat/mingw.h" +#include "compat/win32/fscache.h" #elif defined(_MSC_VER) #include "compat/msvc.h" +#include "compat/win32/fscache.h" #else #include #include From 452921a9f2191cc5cfcaef060a2374c175733a6b Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Tue, 24 Jun 2014 13:22:35 +0200 Subject: [PATCH 09/59] fscache: load directories only once If multiple threads access a directory that is not yet in the cache, the directory will be loaded by each thread. Only one of the results is added to the cache, all others are leaked. This wastes performance and memory. On cache miss, add a future object to the cache to indicate that the directory is currently being loaded. Subsequent threads register themselves with the future object and wait. When the first thread has loaded the directory, it replaces the future object with the result and notifies waiting threads. Signed-off-by: Karsten Blees --- compat/win32/fscache.c | 67 +++++++++++++++++++++++++++++++++++------- 1 file changed, 57 insertions(+), 10 deletions(-) diff --git a/compat/win32/fscache.c b/compat/win32/fscache.c index 5388d64377..05a2fa9663 100644 --- a/compat/win32/fscache.c +++ b/compat/win32/fscache.c @@ -33,6 +33,8 @@ struct fsentry { union { /* Reference count of the directory listing. */ volatile long refcnt; + /* Handle to wait on the loading thread. */ + HANDLE hwait; struct { /* More stat members (only used for file entries). */ off64_t st_size; @@ -259,16 +261,43 @@ static inline int fscache_enabled(const char *path) return enabled > 0 && !is_absolute_path(path); } +/* + * Looks up a cache entry, waits if its being loaded by another thread. + * The mutex must be owned by the calling thread. + */ +static struct fsentry *fscache_get_wait(struct fsentry *key) +{ + struct fsentry *fse = hashmap_get(&map, key, NULL); + + /* return if its a 'real' entry (future entries have refcnt == 0) */ + if (!fse || fse->list || fse->refcnt) + return fse; + + /* create an event and link our key to the future entry */ + key->hwait = CreateEvent(NULL, TRUE, FALSE, NULL); + key->next = fse->next; + fse->next = key; + + /* wait for the loading thread to signal us */ + LeaveCriticalSection(&mutex); + WaitForSingleObject(key->hwait, INFINITE); + CloseHandle(key->hwait); + EnterCriticalSection(&mutex); + + /* repeat cache lookup */ + return hashmap_get(&map, key, NULL); +} + /* * Looks up or creates a cache entry for the specified key. */ static struct fsentry *fscache_get(struct fsentry *key) { - struct fsentry *fse; + struct fsentry *fse, *future, *waiter; EnterCriticalSection(&mutex); /* check if entry is in cache */ - fse = hashmap_get(&map, key, NULL); + fse = fscache_get_wait(key); if (fse) { fsentry_addref(fse); LeaveCriticalSection(&mutex); @@ -276,7 +305,7 @@ static struct fsentry *fscache_get(struct fsentry *key) } /* if looking for a file, check if directory listing is in cache */ if (!fse && key->list) { - fse = hashmap_get(&map, key->list, NULL); + fse = fscache_get_wait(key->list); if (fse) { LeaveCriticalSection(&mutex); /* dir entry without file entry -> file doesn't exist */ @@ -285,16 +314,34 @@ static struct fsentry *fscache_get(struct fsentry *key) } } + /* add future entry to indicate that we're loading it */ + future = key->list ? key->list : key; + future->next = NULL; + future->refcnt = 0; + hashmap_add(&map, future); + /* create the directory listing (outside mutex!) */ LeaveCriticalSection(&mutex); - fse = fsentry_create_list(key->list ? key->list : key); - if (!fse) - return NULL; - + fse = fsentry_create_list(future); EnterCriticalSection(&mutex); - /* add directory listing if it hasn't been added by some other thread */ - if (!hashmap_get(&map, key, NULL)) - fscache_add(fse); + + /* remove future entry and signal waiting threads */ + hashmap_remove(&map, future, NULL); + waiter = future->next; + while (waiter) { + HANDLE h = waiter->hwait; + waiter = waiter->next; + SetEvent(h); + } + + /* leave on error (errno set by fsentry_create_list) */ + if (!fse) { + LeaveCriticalSection(&mutex); + return NULL; + } + + /* add directory listing to the cache */ + fscache_add(fse); /* lookup file entry if requested (fse already points to directory) */ if (key->list) From d5f0e2191e250f23fe1901833d58bbc7bbad7de8 Mon Sep 17 00:00:00 2001 From: Doug Kelly Date: Wed, 8 Jan 2014 20:28:15 -0600 Subject: [PATCH 10/59] Add a test demonstrating a problem with long submodule paths [jes: adusted test number to avoid conflicts, fixed non-portable use of the 'export' statement] Signed-off-by: Johannes Schindelin --- t/t7411-submodule-long-path.sh | 101 +++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100755 t/t7411-submodule-long-path.sh diff --git a/t/t7411-submodule-long-path.sh b/t/t7411-submodule-long-path.sh new file mode 100755 index 0000000000..6bb641cd4b --- /dev/null +++ b/t/t7411-submodule-long-path.sh @@ -0,0 +1,101 @@ +#!/bin/sh +# +# Copyright (c) 2013 Doug Kelly +# + +test_description='Test submodules with a path near PATH_MAX + +This test verifies that "git submodule" initialization, update and clones work, including with recursive submodules and paths approaching PATH_MAX (260 characters on Windows) +' + +TEST_NO_CREATE_REPO=1 +. ./test-lib.sh + +longpath="" +for (( i=0; i<4; i++ )); do + longpath="0123456789abcdefghijklmnopqrstuvwxyz$longpath" +done +# Pick a substring maximum of 90 characters +# This should be good, since we'll add on a lot for temp directories +longpath=${longpath:0:90}; export longpath + +test_expect_failure 'submodule with a long path' ' + git init --bare remote && + test_create_repo bundle1 && + ( + cd bundle1 && + test_commit "shoot" && + git rev-parse --verify HEAD >../expect + ) && + mkdir home && + ( + cd home && + git clone ../remote test && + cd test && + git submodule add ../bundle1 $longpath && + test_commit "sogood" && + ( + cd $longpath && + git rev-parse --verify HEAD >actual && + test_cmp ../../../expect actual + ) && + git push origin master + ) && + mkdir home2 && + ( + cd home2 && + git clone ../remote test && + cd test && + git checkout master && + git submodule update --init && + ( + cd $longpath && + git rev-parse --verify HEAD >actual && + test_cmp ../../../expect actual + ) + ) +' + +test_expect_failure 'recursive submodule with a long path' ' + git init --bare super && + test_create_repo child && + ( + cd child && + test_commit "shoot" && + git rev-parse --verify HEAD >../expect + ) && + test_create_repo parent && + ( + cd parent && + git submodule add ../child $longpath && + test_commit "aim" + ) && + mkdir home3 && + ( + cd home3 && + git clone ../super test && + cd test && + git submodule add ../parent foo && + git submodule update --init --recursive + test_commit "sogood" && + ( + cd foo/$longpath && + git rev-parse --verify HEAD >actual && + test_cmp ../../../../expect actual + ) && + git push origin master + ) && + mkdir home4 && + ( + cd home4 && + git clone ../super test --recursive && + ( + cd test/foo/$longpath && + git rev-parse --verify HEAD >actual && + test_cmp ../../../../expect actual + ) + ) +' +unset longpath + +test_done From 497c47519ecb2cfb8211b75b6f7141a4c162ac3b Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 28 Jul 2015 21:07:41 +0200 Subject: [PATCH 11/59] Win32: support long paths Windows paths are typically limited to MAX_PATH = 260 characters, even though the underlying NTFS file system supports paths up to 32,767 chars. This limitation is also evident in Windows Explorer, cmd.exe and many other applications (including IDEs). Particularly annoying is that most Windows APIs return bogus error codes if a relative path only barely exceeds MAX_PATH in conjunction with the current directory, e.g. ERROR_PATH_NOT_FOUND / ENOENT instead of the infinitely more helpful ERROR_FILENAME_EXCED_RANGE / ENAMETOOLONG. Many Windows wide char APIs support longer than MAX_PATH paths through the file namespace prefix ('\\?\' or '\\?\UNC\') followed by an absolute path. Notable exceptions include functions dealing with executables and the current directory (CreateProcess, LoadLibrary, Get/SetCurrentDirectory) as well as the entire shell API (ShellExecute, SHGetSpecialFolderPath...). Introduce a handle_long_path function to check the length of a specified path properly (and fail with ENAMETOOLONG), and to optionally expand long paths using the '\\?\' file namespace prefix. Short paths will not be modified, so we don't need to worry about device names (NUL, CON, AUX). Contrary to MSDN docs, the GetFullPathNameW function doesn't seem to be limited to MAX_PATH (at least not on Win7), so we can use it to do the heavy lifting of the conversion (translate '/' to '\', eliminate '.' and '..', and make an absolute path). Add long path error checking to xutftowcs_path for APIs with hard MAX_PATH limit. Add a new MAX_LONG_PATH constant and xutftowcs_long_path function for APIs that support long paths. While improved error checking is always active, long paths support must be explicitly enabled via 'core.longpaths' option. This is to prevent end users to shoot themselves in the foot by checking out files that Windows Explorer, cmd/bash or their favorite IDE cannot handle. Test suite: Test the case is when the full pathname length of a dir is close to 260 (MAX_PATH). Bug report and an original reproducer by Andrey Rogozhnikov: https://github.com/msysgit/git/pull/122#issuecomment-43604199 [jes: adjusted test number to avoid conflicts] Thanks-to: Martin W. Kirst Thanks-to: Doug Kelly Signed-off-by: Karsten Blees Original-test-by: Andrey Rogozhnikov Signed-off-by: Stepan Kasal Signed-off-by: Johannes Schindelin --- Documentation/config.txt | 7 ++ cache.h | 2 + compat/mingw.c | 136 ++++++++++++++++++++++++++------- compat/mingw.h | 77 +++++++++++++++++-- compat/win32/dirent.c | 14 ++-- compat/win32/fscache.c | 17 +++-- config.c | 5 ++ environment.c | 1 + t/t2027-checkout-long-paths.sh | 99 ++++++++++++++++++++++++ t/t7411-submodule-long-path.sh | 24 +++--- 10 files changed, 325 insertions(+), 57 deletions(-) create mode 100755 t/t2027-checkout-long-paths.sh diff --git a/Documentation/config.txt b/Documentation/config.txt index 00ff3f5af6..89986582ed 100644 --- a/Documentation/config.txt +++ b/Documentation/config.txt @@ -719,6 +719,13 @@ core.fscache:: Git for Windows uses this to bulk-read and cache lstat data of entire directories (instead of doing lstat file by file). +core.longpaths:: + Enable long path (> 260) support for builtin commands in Git for + Windows. This is disabled by default, as long paths are not supported + by Windows Explorer, cmd.exe and the Git for Windows tool chain + (msys, bash, tcl, perl...). Only enable this if you know what you're + doing and are prepared to live with a few quirks. + core.createObject:: You can set this to 'link', in which case a hardlink followed by a delete of the source are used to make sure that object creation diff --git a/cache.h b/cache.h index 424d495765..5be3cea358 100644 --- a/cache.h +++ b/cache.h @@ -655,6 +655,8 @@ extern enum hide_dotfiles_type hide_dotfiles; extern int core_fscache; +extern int core_long_paths; + enum branch_track { BRANCH_TRACK_UNSPECIFIED = -1, BRANCH_TRACK_NEVER = 0, diff --git a/compat/mingw.c b/compat/mingw.c index 44c05d2cd3..22e8e1bce3 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -204,8 +204,8 @@ static int ask_yes_no_if_possible(const char *format, ...) int mingw_unlink(const char *pathname) { int ret, tries = 0; - wchar_t wpathname[MAX_PATH]; - if (xutftowcs_path(wpathname, pathname) < 0) + wchar_t wpathname[MAX_LONG_PATH]; + if (xutftowcs_long_path(wpathname, pathname) < 0) return -1; /* read-only files cannot be removed */ @@ -234,7 +234,7 @@ static int is_dir_empty(const wchar_t *wpath) { WIN32_FIND_DATAW findbuf; HANDLE handle; - wchar_t wbuf[MAX_PATH + 2]; + wchar_t wbuf[MAX_LONG_PATH + 2]; wcscpy(wbuf, wpath); wcscat(wbuf, L"\\*"); handle = FindFirstFileW(wbuf, &findbuf); @@ -255,8 +255,8 @@ static int is_dir_empty(const wchar_t *wpath) int mingw_rmdir(const char *pathname) { int ret, tries = 0; - wchar_t wpathname[MAX_PATH]; - if (xutftowcs_path(wpathname, pathname) < 0) + wchar_t wpathname[MAX_LONG_PATH]; + if (xutftowcs_long_path(wpathname, pathname) < 0) return -1; while ((ret = _wrmdir(wpathname)) == -1 && tries < ARRAY_SIZE(delay)) { @@ -296,9 +296,9 @@ static int make_hidden(const wchar_t *path) void mingw_mark_as_git_dir(const char *dir) { - wchar_t wdir[MAX_PATH]; + wchar_t wdir[MAX_LONG_PATH]; if (hide_dotfiles != HIDE_DOTFILES_FALSE && !is_bare_repository()) - if (xutftowcs_path(wdir, dir) < 0 || make_hidden(wdir)) + if (xutftowcs_long_path(wdir, dir) < 0 || make_hidden(wdir)) warning("Failed to make '%s' hidden", dir); git_config_set("core.hideDotFiles", hide_dotfiles == HIDE_DOTFILES_FALSE ? "false" : @@ -309,9 +309,12 @@ void mingw_mark_as_git_dir(const char *dir) int mingw_mkdir(const char *path, int mode) { int ret; - wchar_t wpath[MAX_PATH]; - if (xutftowcs_path(wpath, path) < 0) + wchar_t wpath[MAX_LONG_PATH]; + /* CreateDirectoryW path limit is 248 (MAX_PATH - 8.3 file name) */ + if (xutftowcs_path_ex(wpath, path, MAX_LONG_PATH, -1, 248, + core_long_paths) < 0) return -1; + ret = _wmkdir(wpath); if (!ret && hide_dotfiles == HIDE_DOTFILES_TRUE) { /* @@ -331,7 +334,7 @@ int mingw_open (const char *filename, int oflags, ...) va_list args; unsigned mode; int fd; - wchar_t wfilename[MAX_PATH]; + wchar_t wfilename[MAX_LONG_PATH]; va_start(args, oflags); mode = va_arg(args, int); @@ -340,7 +343,7 @@ int mingw_open (const char *filename, int oflags, ...) if (filename && !strcmp(filename, "/dev/null")) filename = "nul"; - if (xutftowcs_path(wfilename, filename) < 0) + if (xutftowcs_long_path(wfilename, filename) < 0) return -1; fd = _wopen(wfilename, oflags, mode); @@ -393,13 +396,13 @@ FILE *mingw_fopen (const char *filename, const char *otype) { int hide = 0; FILE *file; - wchar_t wfilename[MAX_PATH], wotype[4]; + wchar_t wfilename[MAX_LONG_PATH], wotype[4]; if (hide_dotfiles == HIDE_DOTFILES_TRUE && basename((char*)filename)[0] == '.') hide = access(filename, F_OK); if (filename && !strcmp(filename, "/dev/null")) filename = "nul"; - if (xutftowcs_path(wfilename, filename) < 0 || + if (xutftowcs_long_path(wfilename, filename) < 0 || xutftowcs(wotype, otype, ARRAY_SIZE(wotype)) < 0) return NULL; file = _wfopen(wfilename, wotype); @@ -412,13 +415,13 @@ FILE *mingw_freopen (const char *filename, const char *otype, FILE *stream) { int hide = 0; FILE *file; - wchar_t wfilename[MAX_PATH], wotype[4]; + wchar_t wfilename[MAX_LONG_PATH], wotype[4]; if (hide_dotfiles == HIDE_DOTFILES_TRUE && basename((char*)filename)[0] == '.') hide = access(filename, F_OK); if (filename && !strcmp(filename, "/dev/null")) filename = "nul"; - if (xutftowcs_path(wfilename, filename) < 0 || + if (xutftowcs_long_path(wfilename, filename) < 0 || xutftowcs(wotype, otype, ARRAY_SIZE(wotype)) < 0) return NULL; file = _wfreopen(wfilename, wotype, stream); @@ -451,25 +454,32 @@ int mingw_fflush(FILE *stream) int mingw_access(const char *filename, int mode) { - wchar_t wfilename[MAX_PATH]; - if (xutftowcs_path(wfilename, filename) < 0) + wchar_t wfilename[MAX_LONG_PATH]; + if (xutftowcs_long_path(wfilename, filename) < 0) return -1; /* X_OK is not supported by the MSVCRT version */ return _waccess(wfilename, mode & ~X_OK); } +/* cached length of current directory for handle_long_path */ +static int current_directory_len = 0; + int mingw_chdir(const char *dirname) { + int result; wchar_t wdirname[MAX_PATH]; + /* SetCurrentDirectoryW doesn't support long paths */ if (xutftowcs_path(wdirname, dirname) < 0) return -1; - return _wchdir(wdirname); + result = _wchdir(wdirname); + current_directory_len = GetCurrentDirectoryW(0, NULL); + return result; } int mingw_chmod(const char *filename, int mode) { - wchar_t wfilename[MAX_PATH]; - if (xutftowcs_path(wfilename, filename) < 0) + wchar_t wfilename[MAX_LONG_PATH]; + if (xutftowcs_long_path(wfilename, filename) < 0) return -1; return _wchmod(wfilename, mode); } @@ -484,8 +494,8 @@ int mingw_chmod(const char *filename, int mode) static int do_lstat(int follow, const char *file_name, struct stat *buf) { WIN32_FILE_ATTRIBUTE_DATA fdata; - wchar_t wfilename[MAX_PATH]; - if (xutftowcs_path(wfilename, file_name) < 0) + wchar_t wfilename[MAX_LONG_PATH]; + if (xutftowcs_long_path(wfilename, file_name) < 0) return -1; if (GetFileAttributesExW(wfilename, GetFileExInfoStandard, &fdata)) { @@ -628,8 +638,8 @@ int mingw_utime (const char *file_name, const struct utimbuf *times) FILETIME mft, aft; int fh, rc; DWORD attrs; - wchar_t wfilename[MAX_PATH]; - if (xutftowcs_path(wfilename, file_name) < 0) + wchar_t wfilename[MAX_LONG_PATH]; + if (xutftowcs_long_path(wfilename, file_name) < 0) return -1; /* must have write permission */ @@ -677,6 +687,7 @@ unsigned int sleep (unsigned int seconds) char *mingw_mktemp(char *template) { wchar_t wtemplate[MAX_PATH]; + /* we need to return the path, thus no long paths here! */ if (xutftowcs_path(wtemplate, template) < 0) return NULL; if (!_wmktemp(wtemplate)) @@ -1030,6 +1041,7 @@ static pid_t mingw_spawnve_fd(const char *cmd, const char **argv, char **deltaen si.hStdOutput = winansi_get_osfhandle(fhout); si.hStdError = winansi_get_osfhandle(fherr); + /* executables and the current directory don't support long paths */ if (xutftowcs_path(wcmd, cmd) < 0) return -1; if (dir && xutftowcs_path(wdir, dir) < 0) @@ -1605,8 +1617,9 @@ int mingw_rename(const char *pold, const char *pnew) { DWORD attrs, gle; int tries = 0; - wchar_t wpold[MAX_PATH], wpnew[MAX_PATH]; - if (xutftowcs_path(wpold, pold) < 0 || xutftowcs_path(wpnew, pnew) < 0) + wchar_t wpold[MAX_LONG_PATH], wpnew[MAX_LONG_PATH]; + if (xutftowcs_long_path(wpold, pold) < 0 || + xutftowcs_long_path(wpnew, pnew) < 0) return -1; /* @@ -1882,9 +1895,9 @@ int link(const char *oldpath, const char *newpath) { typedef BOOL (WINAPI *T)(LPCWSTR, LPCWSTR, LPSECURITY_ATTRIBUTES); static T create_hard_link = NULL; - wchar_t woldpath[MAX_PATH], wnewpath[MAX_PATH]; - if (xutftowcs_path(woldpath, oldpath) < 0 || - xutftowcs_path(wnewpath, newpath) < 0) + wchar_t woldpath[MAX_LONG_PATH], wnewpath[MAX_LONG_PATH]; + if (xutftowcs_long_path(woldpath, oldpath) < 0 || + xutftowcs_long_path(wnewpath, newpath) < 0) return -1; if (!create_hard_link) { @@ -2065,6 +2078,68 @@ int xwcstoutf(char *utf, const wchar_t *wcs, size_t utflen) return -1; } +int handle_long_path(wchar_t *path, int len, int max_path, int expand) +{ + int result; + wchar_t buf[MAX_LONG_PATH]; + + /* + * we don't need special handling if path is relative to the current + * directory, and current directory + path don't exceed the desired + * max_path limit. This should cover > 99 % of cases with minimal + * performance impact (git almost always uses relative paths). + */ + if ((len < 2 || (!is_dir_sep(path[0]) && path[1] != ':')) && + (current_directory_len + len < max_path)) + return len; + + /* + * handle everything else: + * - absolute paths: "C:\dir\file" + * - absolute UNC paths: "\\server\share\dir\file" + * - absolute paths on current drive: "\dir\file" + * - relative paths on other drive: "X:file" + * - prefixed paths: "\\?\...", "\\.\..." + */ + + /* convert to absolute path using GetFullPathNameW */ + result = GetFullPathNameW(path, MAX_LONG_PATH, buf, NULL); + if (!result) { + errno = err_win_to_posix(GetLastError()); + return -1; + } + + /* + * return absolute path if it fits within max_path (even if + * "cwd + path" doesn't due to '..' components) + */ + if (result < max_path) { + wcscpy(path, buf); + return result; + } + + /* error out if we shouldn't expand the path or buf is too small */ + if (!expand || result >= MAX_LONG_PATH - 6) { + errno = ENAMETOOLONG; + return -1; + } + + /* prefix full path with "\\?\" or "\\?\UNC\" */ + if (buf[0] == '\\') { + /* ...unless already prefixed */ + if (buf[1] == '\\' && (buf[2] == '?' || buf[2] == '.')) + return len; + + wcscpy(path, L"\\\\?\\UNC\\"); + wcscpy(path + 8, buf + 2); + return result + 6; + } else { + wcscpy(path, L"\\\\?\\"); + wcscpy(path + 4, buf); + return result + 4; + } +} + /* * Disable MSVCRT command line wildcard expansion (__getmainargs called from * mingw startup code, see init.c in mingw runtime). @@ -2168,6 +2243,9 @@ void mingw_startup() /* initialize Unicode console */ winansi_init(); + + /* init length of current directory for handle_long_path */ + current_directory_len = GetCurrentDirectoryW(0, NULL); } int uname(struct utsname *buf) diff --git a/compat/mingw.h b/compat/mingw.h index d8905cc155..cc404fa4b3 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -397,6 +397,42 @@ void mingw_open_html(const char *path); void mingw_mark_as_git_dir(const char *dir); #define mark_as_git_dir mingw_mark_as_git_dir +/** + * Max length of long paths (exceeding MAX_PATH). The actual maximum supported + * by NTFS is 32,767 (* sizeof(wchar_t)), but we choose an arbitrary smaller + * value to limit required stack memory. + */ +#define MAX_LONG_PATH 4096 + +/** + * Handles paths that would exceed the MAX_PATH limit of Windows Unicode APIs. + * + * With expand == false, the function checks for over-long paths and fails + * with ENAMETOOLONG. The path parameter is not modified, except if cwd + path + * exceeds max_path, but the resulting absolute path doesn't (e.g. due to + * eliminating '..' components). The path parameter must point to a buffer + * of max_path wide characters. + * + * With expand == true, an over-long path is automatically converted in place + * to an absolute path prefixed with '\\?\', and the new length is returned. + * The path parameter must point to a buffer of MAX_LONG_PATH wide characters. + * + * Parameters: + * path: path to check and / or convert + * len: size of path on input (number of wide chars without \0) + * max_path: max short path length to check (usually MAX_PATH = 260, but just + * 248 for CreateDirectoryW) + * expand: false to only check the length, true to expand the path to a + * '\\?\'-prefixed absolute path + * + * Return: + * length of the resulting path, or -1 on failure + * + * Errors: + * ENAMETOOLONG if path is too long + */ +int handle_long_path(wchar_t *path, int len, int max_path, int expand); + /** * Converts UTF-8 encoded string to UTF-16LE. * @@ -454,17 +490,48 @@ static inline int xutftowcs(wchar_t *wcs, const char *utf, size_t wcslen) return xutftowcsn(wcs, utf, wcslen, -1); } +/** + * Simplified file system specific wrapper of xutftowcsn and handle_long_path. + * Converts ERANGE to ENAMETOOLONG. If expand is true, wcs must be at least + * MAX_LONG_PATH wide chars (see handle_long_path). + */ +static inline int xutftowcs_path_ex(wchar_t *wcs, const char *utf, + size_t wcslen, int utflen, int max_path, int expand) +{ + int result = xutftowcsn(wcs, utf, wcslen, utflen); + if (result < 0 && errno == ERANGE) + errno = ENAMETOOLONG; + if (result >= 0) + result = handle_long_path(wcs, result, max_path, expand); + return result; +} + /** * Simplified file system specific variant of xutftowcsn, assumes output * buffer size is MAX_PATH wide chars and input string is \0-terminated, - * fails with ENAMETOOLONG if input string is too long. + * fails with ENAMETOOLONG if input string is too long. Typically used for + * Windows APIs that don't support long paths, e.g. SetCurrentDirectory, + * LoadLibrary, CreateProcess... */ static inline int xutftowcs_path(wchar_t *wcs, const char *utf) { - int result = xutftowcsn(wcs, utf, MAX_PATH, -1); - if (result < 0 && errno == ERANGE) - errno = ENAMETOOLONG; - return result; + return xutftowcs_path_ex(wcs, utf, MAX_PATH, -1, MAX_PATH, 0); +} + +/* need to re-declare that here as mingw.h is included before cache.h */ +extern int core_long_paths; + +/** + * Simplified file system specific variant of xutftowcsn for Windows APIs + * that support long paths via '\\?\'-prefix, assumes output buffer size is + * MAX_LONG_PATH wide chars, fails with ENAMETOOLONG if input string is too + * long. The 'core.longpaths' git-config option controls whether the path + * is only checked or expanded to a long path. + */ +static inline int xutftowcs_long_path(wchar_t *wcs, const char *utf) +{ + return xutftowcs_path_ex(wcs, utf, MAX_LONG_PATH, -1, MAX_PATH, + core_long_paths); } /** diff --git a/compat/win32/dirent.c b/compat/win32/dirent.c index 6b87042182..b3bd8d7af7 100644 --- a/compat/win32/dirent.c +++ b/compat/win32/dirent.c @@ -63,19 +63,23 @@ static int dirent_closedir(dirent_DIR *dir) DIR *dirent_opendir(const char *name) { - wchar_t pattern[MAX_PATH + 2]; /* + 2 for '/' '*' */ + wchar_t pattern[MAX_LONG_PATH + 2]; /* + 2 for "\*" */ WIN32_FIND_DATAW fdata; HANDLE h; int len; dirent_DIR *dir; - /* convert name to UTF-16 and check length < MAX_PATH */ - if ((len = xutftowcs_path(pattern, name)) < 0) + /* convert name to UTF-16 and check length */ + if ((len = xutftowcs_path_ex(pattern, name, MAX_LONG_PATH, -1, + MAX_PATH - 2, core_long_paths)) < 0) return NULL; - /* append optional '/' and wildcard '*' */ + /* + * append optional '\' and wildcard '*'. Note: we need to use '\' as + * Windows doesn't translate '/' to '\' for "\\?\"-prefixed paths. + */ if (len && !is_dir_sep(pattern[len - 1])) - pattern[len++] = '/'; + pattern[len++] = '\\'; pattern[len++] = '*'; pattern[len] = 0; diff --git a/compat/win32/fscache.c b/compat/win32/fscache.c index 05a2fa9663..9e7e36ed39 100644 --- a/compat/win32/fscache.c +++ b/compat/win32/fscache.c @@ -164,23 +164,24 @@ static struct fsentry *fseentry_create_entry(struct fsentry *list, */ static struct fsentry *fsentry_create_list(const struct fsentry *dir) { - wchar_t pattern[MAX_PATH + 2]; /* + 2 for '/' '*' */ + wchar_t pattern[MAX_LONG_PATH + 2]; /* + 2 for "\*" */ WIN32_FIND_DATAW fdata; HANDLE h; int wlen; struct fsentry *list, **phead; DWORD err; - /* convert name to UTF-16 and check length < MAX_PATH */ - if ((wlen = xutftowcsn(pattern, dir->name, MAX_PATH, dir->len)) < 0) { - if (errno == ERANGE) - errno = ENAMETOOLONG; + /* convert name to UTF-16 and check length */ + if ((wlen = xutftowcs_path_ex(pattern, dir->name, MAX_LONG_PATH, + dir->len, MAX_PATH - 2, core_long_paths)) < 0) return NULL; - } - /* append optional '/' and wildcard '*' */ + /* + * append optional '\' and wildcard '*'. Note: we need to use '\' as + * Windows doesn't translate '/' to '\' for "\\?\"-prefixed paths. + */ if (wlen) - pattern[wlen++] = '/'; + pattern[wlen++] = '\\'; pattern[wlen++] = '*'; pattern[wlen] = 0; diff --git a/config.c b/config.c index 306e562e64..40a814cd7b 100644 --- a/config.c +++ b/config.c @@ -925,6 +925,11 @@ static int git_default_core_config(const char *var, const char *value) return 0; } + if (!strcmp(var, "core.longpaths")) { + core_long_paths = git_config_bool(var, value); + return 0; + } + /* Add other config variables here and to Documentation/config.txt. */ return 0; } diff --git a/environment.c b/environment.c index 8803d2ebc6..3ddf13a763 100644 --- a/environment.c +++ b/environment.c @@ -66,6 +66,7 @@ struct startup_info *startup_info; unsigned long pack_size_limit_cfg; enum hide_dotfiles_type hide_dotfiles = HIDE_DOTFILES_DOTGITONLY; int core_fscache; +int core_long_paths; #ifndef PROTECT_HFS_DEFAULT #define PROTECT_HFS_DEFAULT 0 diff --git a/t/t2027-checkout-long-paths.sh b/t/t2027-checkout-long-paths.sh new file mode 100755 index 0000000000..e65a38da4d --- /dev/null +++ b/t/t2027-checkout-long-paths.sh @@ -0,0 +1,99 @@ +#!/bin/sh + +test_description='checkout long paths on Windows + +Ensures that Git for Windows can deal with long paths (>260) enabled via core.longpaths' + +. ./test-lib.sh + +if test_have_prereq !MINGW +then + skip_all='skipping MINGW specific long paths test' + test_done +fi + +test_expect_success setup ' + p=longpathxx && # -> 10 + p=$p$p$p$p$p && # -> 50 + p=$p$p$p$p$p && # -> 250 + + path=${p}/longtestfile && # -> 263 (MAX_PATH = 260) + + blob=$(echo foobar | git hash-object -w --stdin) && + + printf "100644 %s 0\t%s\n" "$blob" "$path" | + git update-index --add --index-info && + git commit -m initial -q +' + +test_expect_success 'checkout of long paths without core.longpaths fails' ' + git config core.longpaths false && + test_must_fail git checkout -f 2>error && + grep -q "Filename too long" error && + test_path_is_missing longpa~1/longtestfile +' + +test_expect_success 'checkout of long paths with core.longpaths works' ' + git config core.longpaths true && + git checkout -f && + test_path_is_file longpa~1/longtestfile +' + +test_expect_success 'update of long paths' ' + echo frotz >> longpa~1/longtestfile && + echo $path > expect && + git ls-files -m > actual && + test_cmp expect actual && + git add $path && + git commit -m second && + git grep "frotz" HEAD -- $path +' + +test_expect_success cleanup ' + # bash cannot delete the trash dir if it contains a long path + # lets help cleaning up (unless in debug mode) + test ! -z "$debug" || rm -rf longpa~1 +' + +# check that the template used in the test won't be too long: +abspath="$(pwd -W)"/testdir +test ${#abspath} -gt 230 || +test_set_prereq SHORTABSPATH + +test_expect_success SHORTABSPATH 'clean up path close to MAX_PATH' ' + p=/123456789abcdef/123456789abcdef/123456789abcdef/123456789abc/ef && + p=y$p$p$p$p && + subdir="x$(echo "$p" | tail -c $((253 - ${#abspath})) - )" && + # Now, $abspath/$subdir has exactly 254 characters, and is inside CWD + p2="$abspath/$subdir" && + test 254 = ${#p2} && + + # Be careful to overcome path limitations of the MSys tools and split + # the $subdir into two parts. ($subdir2 has to contain 16 chars and a + # slash somewhere following; that is why we asked for abspath <= 230 and + # why we placed a slash near the end of the $subdir template.) + subdir2=${subdir#????????????????*/} && + subdir1=testdir/${subdir%/$subdir2} && + mkdir -p "$subdir1" && + i=0 && + # The most important case is when absolute path is 258 characters long, + # and that will be when i == 4. + while test $i -le 7 + do + mkdir -p $subdir2 && + touch $subdir2/one-file && + mv ${subdir2%%/*} "$subdir1/" && + subdir2=z${subdir2} && + i=$(($i+1)) || + exit 1 + done && + + # now check that git is able to clear the tree: + (cd testdir && + git init && + git config core.longpaths yes && + git clean -fdx) && + test ! -d "$subdir1" +' + +test_done diff --git a/t/t7411-submodule-long-path.sh b/t/t7411-submodule-long-path.sh index 6bb641cd4b..59b3d347db 100755 --- a/t/t7411-submodule-long-path.sh +++ b/t/t7411-submodule-long-path.sh @@ -11,15 +11,20 @@ This test verifies that "git submodule" initialization, update and clones work, TEST_NO_CREATE_REPO=1 . ./test-lib.sh -longpath="" -for (( i=0; i<4; i++ )); do - longpath="0123456789abcdefghijklmnopqrstuvwxyz$longpath" -done -# Pick a substring maximum of 90 characters -# This should be good, since we'll add on a lot for temp directories -longpath=${longpath:0:90}; export longpath +# cloning a submodule calls is_git_directory("$path/../.git/modules/$path"), +# which effectively limits the maximum length to PATH_MAX / 2 minus some +# overhead; start with 3 * 36 = 108 chars (test 2 fails if >= 110) +longpath36=0123456789abcdefghijklmnopqrstuvwxyz +longpath180=$longpath36$longpath36$longpath36$longpath36$longpath36 -test_expect_failure 'submodule with a long path' ' +# the git database must fit within PATH_MAX, which limits the submodule name +# to PATH_MAX - len(pwd) - ~90 (= len("/objects//") + 40-byte sha1 + some +# overhead from the test case) +pwd=$(pwd) +pwdlen=$(echo "$pwd" | wc -c) +longpath=$(echo $longpath180 | cut -c 1-$((170-$pwdlen))) + +test_expect_success 'submodule with a long path' ' git init --bare remote && test_create_repo bundle1 && ( @@ -56,7 +61,7 @@ test_expect_failure 'submodule with a long path' ' ) ' -test_expect_failure 'recursive submodule with a long path' ' +test_expect_success 'recursive submodule with a long path' ' git init --bare super && test_create_repo child && ( @@ -96,6 +101,5 @@ test_expect_failure 'recursive submodule with a long path' ' ) ) ' -unset longpath test_done From 84f813718af87ca4b27db415d242a50bc3fb999c Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sun, 30 Aug 2015 18:49:05 +0200 Subject: [PATCH 12/59] Win32: support long paths Windows paths are typically limited to MAX_PATH = 260 characters, even though the underlying NTFS file system supports paths up to 32,767 chars. This limitation is also evident in Windows Explorer, cmd.exe and many other applications (including IDEs). Particularly annoying is that most Windows APIs return bogus error codes if a relative path only barely exceeds MAX_PATH in conjunction with the current directory, e.g. ERROR_PATH_NOT_FOUND / ENOENT instead of the infinitely more helpful ERROR_FILENAME_EXCED_RANGE / ENAMETOOLONG. Many Windows wide char APIs support longer than MAX_PATH paths through the file namespace prefix ('\\?\' or '\\?\UNC\') followed by an absolute path. Notable exceptions include functions dealing with executables and the current directory (CreateProcess, LoadLibrary, Get/SetCurrentDirectory) as well as the entire shell API (ShellExecute, SHGetSpecialFolderPath...). Introduce a handle_long_path function to check the length of a specified path properly (and fail with ENAMETOOLONG), and to optionally expand long paths using the '\\?\' file namespace prefix. Short paths will not be modified, so we don't need to worry about device names (NUL, CON, AUX). Contrary to MSDN docs, the GetFullPathNameW function doesn't seem to be limited to MAX_PATH (at least not on Win7), so we can use it to do the heavy lifting of the conversion (translate '/' to '\', eliminate '.' and '..', and make an absolute path). Add long path error checking to xutftowcs_path for APIs with hard MAX_PATH limit. Add a new MAX_LONG_PATH constant and xutftowcs_long_path function for APIs that support long paths. While improved error checking is always active, long paths support must be explicitly enabled via 'core.longpaths' option. This is to prevent end users to shoot themselves in the foot by checking out files that Windows Explorer, cmd/bash or their favorite IDE cannot handle. Test suite: Test the case is when the full pathname length of a dir is close to 260 (MAX_PATH). Bug report and an original reproducer by Andrey Rogozhnikov: https://github.com/msysgit/git/pull/122#issuecomment-43604199 [jes: adjusted test number to avoid conflicts, reinstated && chain] Thanks-to: Martin W. Kirst Thanks-to: Doug Kelly Signed-off-by: Karsten Blees Original-test-by: Andrey Rogozhnikov Signed-off-by: Stepan Kasal Signed-off-by: Johannes Schindelin --- t/t2027-checkout-long-paths.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/t/t2027-checkout-long-paths.sh b/t/t2027-checkout-long-paths.sh index e65a38da4d..79345ed7fd 100755 --- a/t/t2027-checkout-long-paths.sh +++ b/t/t2027-checkout-long-paths.sh @@ -52,7 +52,10 @@ test_expect_success 'update of long paths' ' test_expect_success cleanup ' # bash cannot delete the trash dir if it contains a long path # lets help cleaning up (unless in debug mode) - test ! -z "$debug" || rm -rf longpa~1 + if test -z "$debug" + then + rm -rf longpa~1 + fi ' # check that the template used in the test won't be too long: From 5946057db8fe2af270adb6ee2c25f474cbfa9b9f Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sat, 5 Jul 2014 00:00:36 +0200 Subject: [PATCH 13/59] Win32: fix 'lstat("dir/")' with long paths Use a suffciently large buffer to strip the trailing slash. Signed-off-by: Karsten Blees --- compat/mingw.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 22e8e1bce3..4c1e039228 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -560,7 +560,7 @@ static int do_lstat(int follow, const char *file_name, struct stat *buf) static int do_stat_internal(int follow, const char *file_name, struct stat *buf) { int namelen; - char alt_name[PATH_MAX]; + char alt_name[MAX_LONG_PATH]; if (!do_lstat(follow, file_name, buf)) return 0; @@ -576,7 +576,7 @@ static int do_stat_internal(int follow, const char *file_name, struct stat *buf) return -1; while (namelen && file_name[namelen-1] == '/') --namelen; - if (!namelen || namelen >= PATH_MAX) + if (!namelen || namelen >= MAX_LONG_PATH) return -1; memcpy(alt_name, file_name, namelen); From 8dbb92a7f41d75b9c251f1bfdb980b79f9557961 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sun, 26 Jul 2009 05:08:42 +0200 Subject: [PATCH 14/59] Handle http.* config variables pointing to files gracefully on Windows On Windows, we would like to be able to have a default http.sslCAinfo that points to an MSys path (i.e. relative to the installation root of Git). As Git is a MinGW program, it has to handle the conversion of the MSys path into a MinGW32 path itself. Since system_path() considers paths starting with '/' as absolute, we have to convince it to make a Windows path by stripping the leading slash. Signed-off-by: Johannes Schindelin --- http.c | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/http.c b/http.c index e9c6fdd835..2bd1755dee 100644 --- a/http.c +++ b/http.c @@ -9,6 +9,7 @@ #include "version.h" #include "pkt-line.h" #include "gettext.h" +#include "exec_cmd.h" int active_requests; int http_is_verbose; @@ -182,6 +183,18 @@ static void process_curl_messages(void) } #endif +static int git_config_path(const char **result, + const char *var, const char *value) +{ + if (git_config_string(result, var, value)) + return 1; +#ifdef __MINGW32__ + if (**result == '/') + *result = system_path((*result) + 1); +#endif + return 0; +} + static int http_options(const char *var, const char *value, void *cb) { if (!strcmp("http.sslverify", var)) { @@ -191,17 +204,17 @@ static int http_options(const char *var, const char *value, void *cb) if (!strcmp("http.sslcipherlist", var)) return git_config_string(&ssl_cipherlist, var, value); if (!strcmp("http.sslcert", var)) - return git_config_string(&ssl_cert, var, value); + return git_config_path(&ssl_cert, var, value); #if LIBCURL_VERSION_NUM >= 0x070903 if (!strcmp("http.sslkey", var)) - return git_config_string(&ssl_key, var, value); + return git_config_path(&ssl_key, var, value); #endif #if LIBCURL_VERSION_NUM >= 0x070908 if (!strcmp("http.sslcapath", var)) - return git_config_string(&ssl_capath, var, value); + return git_config_path(&ssl_capath, var, value); #endif if (!strcmp("http.sslcainfo", var)) - return git_config_string(&ssl_cainfo, var, value); + return git_config_path(&ssl_cainfo, var, value); if (!strcmp("http.sslcertpasswordprotected", var)) { ssl_cert_password_required = git_config_bool(var, value); return 0; From 8a759ef22377e6f2b82dd4c39cc5c95b3c89c96e Mon Sep 17 00:00:00 2001 From: Sebastian Schuberth Date: Thu, 29 Aug 2013 13:09:27 +0200 Subject: [PATCH 15/59] MinGW: Use MakeMaker to build the Perl libraries This way the libraries get properly installed into the "site_perl" directory and we just have to move them out of the "mingw" directory. Signed-off-by: Sebastian Schuberth --- config.mak.uname | 1 - 1 file changed, 1 deletion(-) diff --git a/config.mak.uname b/config.mak.uname index 903eccd5c9..10c769344e 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -504,7 +504,6 @@ ifneq (,$(findstring MINGW,$(uname_S))) NO_MKDTEMP = YesPlease NO_MKSTEMPS = YesPlease NO_SVN_TESTS = YesPlease - NO_PERL_MAKEMAKER = YesPlease RUNTIME_PREFIX = YesPlease NO_ST_BLOCKS_IN_STRUCT_STAT = YesPlease NO_NSEC = YesPlease From 5f625d3bd5ed4269f7b1f2e87df53241d80e6d89 Mon Sep 17 00:00:00 2001 From: Sebastian Schuberth Date: Wed, 4 Sep 2013 18:18:49 +0200 Subject: [PATCH 16/59] Makefile: Set htmldir to match the default HTML docs location under MSYS Signed-off-by: Sebastian Schuberth --- config.mak.uname | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.mak.uname b/config.mak.uname index 10c769344e..ed2cb2fb54 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -535,7 +535,7 @@ ifneq (,$(findstring MINGW,$(uname_S))) X = .exe SPARSE_FLAGS = -Wno-one-bit-signed-bitfield ifneq (,$(wildcard ../THIS_IS_MSYSGIT)) - htmldir = doc/git/html/ + htmldir = share/doc/git/$(firstword $(subst -, ,$(GIT_VERSION)))/html prefix = INSTALL = /bin/install EXTLIBS += /mingw/lib/libz.a From 7dee5c1ff04f6f4b658e2e515a63a71f0f5e7f2b Mon Sep 17 00:00:00 2001 From: Thomas Braun Date: Wed, 3 Sep 2014 23:37:54 +0200 Subject: [PATCH 17/59] Enable support for perl regular expressions (LIBPCRE) Signed-off-by: Thomas Braun --- config.mak.uname | 1 + 1 file changed, 1 insertion(+) diff --git a/config.mak.uname b/config.mak.uname index ed2cb2fb54..981eb0be0e 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -543,6 +543,7 @@ ifneq (,$(wildcard ../THIS_IS_MSYSGIT)) INTERNAL_QSORT = YesPlease HAVE_LIBCHARSET_H = YesPlease NO_GETTEXT = YesPlease + USE_LIBPCRE= YesPlease else NO_CURL = YesPlease endif From fda72ca0b7198b7b5bbcf5da1dd6c77966798ed7 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 20 Feb 2015 09:52:07 +0000 Subject: [PATCH 18/59] Help debugging with MSys2 by optionally executing bash with strace MSys2's strace facility is very useful for debugging... With this patch, the bash will be executed through strace if the environment variable GIT_STRACE_COMMANDS is set, which comes in real handy when investigating issues in the test suite. Signed-off-by: Johannes Schindelin --- compat/mingw.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/compat/mingw.c b/compat/mingw.c index 4c1e039228..949ecefdd4 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -1064,6 +1064,16 @@ static pid_t mingw_spawnve_fd(const char *cmd, const char **argv, char **deltaen free(quoted); } + if (getenv("GIT_STRACE_COMMANDS")) { + char **path = get_path_split(); + cmd = path_lookup("strace.exe", path, 1); + if (!cmd) + return error("strace not found!"); + if (xutftowcs_path(wcmd, cmd) < 0) + return -1; + strbuf_insert(&args, 0, "strace ", 7); + } + wargs = xmalloc((2 * args.len + 1) * sizeof(wchar_t)); xutftowcs(wargs, args.buf, 2 * args.len + 1); strbuf_release(&args); From bfe8cc7a1490c0dd55b9bbc3be4c1619459c103f Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 13 Feb 2015 12:18:54 +0000 Subject: [PATCH 19/59] Assorted header fixes to support MSys2-based MinGW build The excellent MSys2 project brings a substantially updated MinGW environment including newer GCC versions and new headers. To support compiling Git, let's special-case the new MinGW (tell-tale: the _MINGW64_VERSION_MAJOR constant is defined). Signed-off-by: Johannes Schindelin --- compat/mingw.h | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/compat/mingw.h b/compat/mingw.h index cc404fa4b3..9df3723f76 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -1,14 +1,13 @@ +#include +#include #include #include - /* * things that are not available in header files */ -typedef int pid_t; typedef int uid_t; typedef int socklen_t; -#define hstrerror strerror #define S_IFLNK 0120000 /* Symbolic link */ #define S_ISLNK(x) (((x) & S_IFMT) == S_IFLNK) @@ -100,8 +99,10 @@ static inline int symlink(const char *oldpath, const char *newpath) { errno = ENOSYS; return -1; } static inline int fchmod(int fildes, mode_t mode) { errno = ENOSYS; return -1; } +#ifndef __MINGW64_VERSION_MAJOR static inline pid_t fork(void) { errno = ENOSYS; return -1; } +#endif static inline unsigned int alarm(unsigned int seconds) { return 0; } static inline int fsync(int fd) @@ -176,8 +177,10 @@ int pipe(int filedes[2]); unsigned int sleep (unsigned int seconds); int mkstemp(char *template); int gettimeofday(struct timeval *tv, void *tz); +#ifndef __MINGW64_VERSION_MAJOR struct tm *gmtime_r(const time_t *timep, struct tm *result); struct tm *localtime_r(const time_t *timep, struct tm *result); +#endif int getpagesize(void); /* defined in MinGW's libgcc.a */ struct passwd *getpwuid(uid_t uid); int setitimer(int type, struct itimerval *in, struct itimerval *out); @@ -314,8 +317,10 @@ static inline time_t filetime_to_time_t(const FILETIME *ft) /* * Use mingw specific stat()/lstat()/fstat() implementations on Windows. */ +#ifndef __MINGW64_VERSION_MAJOR #define off_t off64_t #define lseek _lseeki64 +#endif /* use struct stat with 64 bit st_size */ #ifdef stat @@ -388,8 +393,12 @@ static inline char *mingw_find_last_dir_sep(const char *path) int mingw_offset_1st_component(const char *path); #define offset_1st_component mingw_offset_1st_component #define PATH_SEP ';' +#ifndef __MINGW64_VERSION_MAJOR #define PRIuMAX "I64u" #define PRId64 "I64d" +#else +#include +#endif void mingw_open_html(const char *path); #define open_html mingw_open_html From 44b6f9c35f8aa54f4d7147d79980aa538ba3a453 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 13 Feb 2015 12:19:48 +0000 Subject: [PATCH 20/59] Avoid redefining S_* constants Signed-off-by: Johannes Schindelin --- compat/mingw.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/compat/mingw.h b/compat/mingw.h index 9df3723f76..be27d6cb9d 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -13,14 +13,18 @@ typedef int socklen_t; #define S_ISLNK(x) (((x) & S_IFMT) == S_IFLNK) #define S_ISSOCK(x) 0 +#ifndef S_IRWXG #define S_IRGRP 0 #define S_IWGRP 0 #define S_IXGRP 0 #define S_IRWXG (S_IRGRP | S_IWGRP | S_IXGRP) +#endif +#ifndef S_IRWXO #define S_IROTH 0 #define S_IWOTH 0 #define S_IXOTH 0 #define S_IRWXO (S_IROTH | S_IWOTH | S_IXOTH) +#endif #define S_ISUID 0004000 #define S_ISGID 0002000 From a0bf0b157ecb7558dacdd3fd604d2d7588f875b3 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 13 Feb 2015 12:23:19 +0000 Subject: [PATCH 21/59] Do not re-define _CONSOLE_FONT_INFOEX when compiling with MSys2 MSys2 already has that structure. Signed-off-by: Johannes Schindelin --- compat/winansi.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compat/winansi.c b/compat/winansi.c index efc5bb3a4b..aac430a952 100644 --- a/compat/winansi.c +++ b/compat/winansi.c @@ -23,6 +23,7 @@ static HANDLE hthread, hread, hwrite; static HANDLE hconsole1, hconsole2; #ifdef __MINGW32__ +#if !defined(__MINGW64_VERSION_MAJOR) || __MINGW64_VERSION_MAJOR < 5 typedef struct _CONSOLE_FONT_INFOEX { ULONG cbSize; DWORD nFont; @@ -32,6 +33,7 @@ typedef struct _CONSOLE_FONT_INFOEX { WCHAR FaceName[LF_FACESIZE]; } CONSOLE_FONT_INFOEX, *PCONSOLE_FONT_INFOEX; #endif +#endif typedef BOOL (WINAPI *PGETCURRENTCONSOLEFONTEX)(HANDLE, BOOL, PCONSOLE_FONT_INFOEX); From a667219c4606be15347efc80829587388c60f6e7 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 3 Apr 2015 19:58:28 +0100 Subject: [PATCH 22/59] mingw: allow compiling with MSys2's compiler With MSys2's GCC, `ReadWriteBarrier` is already defined, and FORCEINLINE unfortunately gets defined incorrectly. Let's work around both problems, using the MSys2-specific __MINGW64_VERSION_MAJOR constant to guard them. Signed-off-by: Johannes Schindelin --- compat/nedmalloc/malloc.c.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/compat/nedmalloc/malloc.c.h b/compat/nedmalloc/malloc.c.h index f216a2a7d3..18634e3a3b 100644 --- a/compat/nedmalloc/malloc.c.h +++ b/compat/nedmalloc/malloc.c.h @@ -720,6 +720,9 @@ struct mallinfo { inlining are defined as macros, so these aren't used for them. */ +#ifdef __MINGW64_VERSION_MAJOR +#undef FORCEINLINE +#endif #ifndef FORCEINLINE #if defined(__GNUC__) #define FORCEINLINE __inline __attribute__ ((always_inline)) @@ -1382,6 +1385,9 @@ LONG __cdecl _InterlockedExchange(LONG volatile *Target, LONG Value); /*** Atomic operations ***/ #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100 + #ifdef __MINGW64_VERSION_MAJOR + #undef _ReadWriteBarrier + #endif #define _ReadWriteBarrier() __sync_synchronize() #else static __inline__ __attribute__((always_inline)) long __sync_lock_test_and_set(volatile long * const Target, const long Value) From c54851c02f69f69cf5d7272043b5fe8a4e50f39e Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 13 Feb 2015 12:35:10 +0000 Subject: [PATCH 23/59] config.mak.uname: support MSys2 Git for Windows lags a little bit behind with the 2.x releases because the Git for Windows developers wanted to let that big jump coincide with a well-needed overhaul of the context within which Git for Windows is developed. To understand why this is such a big issue, it needs to be noted that many parts of Git are not written in portable C, but instead relies on a POSIX shell and Perl to be available. Even in the portable C part, there is the ingrained notion that we can work with UTF-8 encoded strings. To support the scripts, Git for Windows has to ship a minimal POSIX emulation layer with Bash and Perl thrown in, and when the Git for Windows effort started originally, in August 2007, we settled on using MSys, a stripped down version of Cygwin. Consequently, the original name of the project was "msysGit" (which, sadly, caused a *lot* of confusion because few Windows users know about MSys, and even less care). To compile the C code of Git for Windows, we used MSys, too: it sports an additional version of the GNU C Compiler that targets the plain Win32 API (with a few convenience functions thrown in) instead of the POSIX emulation layer that would require the MSys runtime to run the compiled programs. That way, Git for Windows' executable(s) are really just Win32 programs. To discern executables requiring the POSIX emulation layer from the ones that do not, the latter are called MinGW (Minimal GNU for Windows) when the former are called MSys executables. This reliance on MSys incurred challenges, too, though: some of our changes to the MSys runtime -- necessary to support Git for Windows better -- were not accepted upstream, the MSys runtime was not developed further to support e.g. UTF-8 or 64-bit, and apart from not having a package management system until much later (when mingw-get was introduced), many packages provided by the MSys/MinGW project lag behind the respective source code versions, in particular Bash and OpenSSL. For a while, the Git for Windows project tried to remedy the situation by trying to build newer versions of those packages, but the situation quickly became untenable, especially with problems like the Heartbleed bug requiring swift action and Git for Windows contributors being scarce -- despite millions of downloads suggesting that there are many users. After a brief push in the direction of mingw-get, thanks to the long-time contributor and co-maintainer Sebastian Schuberth, it became clear that we need to look for alternatives. Happily, in the meantime the MSys2 project (https://msys2.github.io/) emerged, and was chosen to be the base of the Git for Windows 2.x. MSys2 is a rewrite of the spirit of MSys: it is again a stripped down version of Cygwin, but it is actively kept up-to-date with Cygwin's source code. Thereby, it already supports Unicode internally, and it also offers the 64-bit support that we yearned for since the beginning of the Git for Windows project. MSys2 also ported the Pacman package management system from Arch Linux and uses it heavily. This brings the same convenience to which Linux users are used to from `yum` or `apt-get`, and to which MacOSX users are used to from Homebrew or MacPorts, or BSD users from the Ports system, to MSys2: a simple `pacman -Syu` will update all installed packages to the newest versions currently available. MSys2 is also *very* active, typically providing package updates multiple times per week. It still required a two-month effort to bring everything to a state where Git's test suite passes, and a couple of patches await their submission to the respective upstream projects. Yet without MSys2, the modernization of Git for Windows would simply not have happened. This commit lays the ground work to supporting MSys2-based Git builds. Assisted-by: Waldek Maleska Signed-off-by: Johannes Schindelin squash! config.mak.uname: support MSys2 Make sure that the nedmalloc patch is applied before. --- config.mak.uname | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/config.mak.uname b/config.mak.uname index 981eb0be0e..7599b79c2d 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -520,7 +520,7 @@ ifneq (,$(findstring MINGW,$(uname_S))) NO_POSIX_GOODIES = UnfortunatelyYes DEFAULT_HELP_FORMAT = html NO_D_INO_IN_DIRENT = YesPlease - COMPAT_CFLAGS += -D__USE_MINGW_ACCESS -D_USE_32BIT_TIME_T -DNOGDI -Icompat -Icompat/win32 + COMPAT_CFLAGS += -D_USE_32BIT_TIME_T -DNOGDI -Icompat -Icompat/win32 COMPAT_CFLAGS += -DSTRIP_EXTENSION=\".exe\" COMPAT_OBJS += compat/mingw.o compat/winansi.o \ compat/win32/pthread.o compat/win32/syslog.o \ @@ -543,9 +543,25 @@ ifneq (,$(wildcard ../THIS_IS_MSYSGIT)) INTERNAL_QSORT = YesPlease HAVE_LIBCHARSET_H = YesPlease NO_GETTEXT = YesPlease - USE_LIBPCRE= YesPlease + COMPAT_CLFAGS += -D__USE_MINGW_ACCESS else - NO_CURL = YesPlease + ifeq ($(shell expr "$(uname_R)" : '2\.'),2) + # MSys2 + CC = gcc + prefix = /mingw32 + COMPAT_CFLAGS += -D__USE_MINGW_ANSI_STDIO=0 + INSTALL = /bin/install + NO_R_TO_GCC_LINKER = YesPlease + INTERNAL_QSORT = YesPlease + HAVE_LIBCHARSET_H = YesPlease + NO_GETTEXT = YesPlease + USE_LIBPCRE= YesPlease + NO_CURL = + USE_NED_ALLOCATOR = YesPlease + else + COMPAT_CFLAGS += -D__USE_MINGW_ANSI_STDIO + NO_CURL = YesPlease + endif endif endif ifeq ($(uname_S),QNX) From 95554ad7c3011c9e876be2c7aeee7a8984f87d7f Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 13 Feb 2015 12:45:37 +0000 Subject: [PATCH 24/59] config.mak.uname: supporting 64-bit MSys2 This just makes things compile, the test suite needs extra tender loving care in addition to this change. We will address these issues in later commits. While at it, also allow building MSys2 Git (i.e. a Git that uses MSys2's POSIX emulation layer). Signed-off-by: Johannes Schindelin --- config.mak.uname | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/config.mak.uname b/config.mak.uname index 7599b79c2d..f6548ee3bf 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -520,13 +520,12 @@ ifneq (,$(findstring MINGW,$(uname_S))) NO_POSIX_GOODIES = UnfortunatelyYes DEFAULT_HELP_FORMAT = html NO_D_INO_IN_DIRENT = YesPlease - COMPAT_CFLAGS += -D_USE_32BIT_TIME_T -DNOGDI -Icompat -Icompat/win32 + COMPAT_CFLAGS += -DNOGDI -Icompat -Icompat/win32 COMPAT_CFLAGS += -DSTRIP_EXTENSION=\".exe\" COMPAT_OBJS += compat/mingw.o compat/winansi.o \ compat/win32/pthread.o compat/win32/syslog.o \ compat/win32/dirent.o compat/win32/fscache.o BASIC_CFLAGS += -DPROTECT_NTFS_DEFAULT=1 - BASIC_LDFLAGS += -Wl,--large-address-aware EXTLIBS += -lws2_32 GITLIBS += git.res PTHREAD_LIBS = @@ -547,8 +546,17 @@ ifneq (,$(wildcard ../THIS_IS_MSYSGIT)) else ifeq ($(shell expr "$(uname_R)" : '2\.'),2) # MSys2 + prefix = /usr/ + ifeq (MINGW32,$(MSYSTEM)) + prefix = /mingw32 + endif + ifeq (MINGW64,$(MSYSTEM)) + prefix = /mingw64 + else + COMPAT_CFLAGS += -D_USE_32BIT_TIME_T + BASIC_LDFLAGS += -Wl,--large-address-aware + endif CC = gcc - prefix = /mingw32 COMPAT_CFLAGS += -D__USE_MINGW_ANSI_STDIO=0 INSTALL = /bin/install NO_R_TO_GCC_LINKER = YesPlease From 5a03a1b1addf03e162e3615e0c81557a603668fa Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 16 Feb 2015 14:06:59 +0100 Subject: [PATCH 25/59] Build Python stuff with MSys2 Signed-off-by: Johannes Schindelin --- config.mak.uname | 1 + 1 file changed, 1 insertion(+) diff --git a/config.mak.uname b/config.mak.uname index f6548ee3bf..a53f5cd074 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -566,6 +566,7 @@ else USE_LIBPCRE= YesPlease NO_CURL = USE_NED_ALLOCATOR = YesPlease + NO_PYTHON = else COMPAT_CFLAGS += -D__USE_MINGW_ANSI_STDIO NO_CURL = YesPlease From 82ae17ae0580fd3d678a78ae18755032d254c0f9 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 19 Feb 2015 17:03:46 +0000 Subject: [PATCH 26/59] Let's use gettext with MSys2 Signed-off-by: Johannes Schindelin --- config.mak.uname | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.mak.uname b/config.mak.uname index a53f5cd074..ea8697d712 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -562,7 +562,7 @@ else NO_R_TO_GCC_LINKER = YesPlease INTERNAL_QSORT = YesPlease HAVE_LIBCHARSET_H = YesPlease - NO_GETTEXT = YesPlease + NO_GETTEXT = USE_LIBPCRE= YesPlease NO_CURL = USE_NED_ALLOCATOR = YesPlease From 0d62f67e5ca99da9f58ef4867bc15e686d2739d5 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 24 Feb 2015 15:09:31 +0000 Subject: [PATCH 27/59] Do not trust MSys2's MinGW gettext.sh It does not quite work because it produces DOS line endings which the shell does not like at all. This lets t3406, t3903, t4254, t7400, t7401, t7406 and t7407 pass. Signed-off-by: Johannes Schindelin --- config.mak.uname | 1 + 1 file changed, 1 insertion(+) diff --git a/config.mak.uname b/config.mak.uname index ea8697d712..c61899c49f 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -563,6 +563,7 @@ else INTERNAL_QSORT = YesPlease HAVE_LIBCHARSET_H = YesPlease NO_GETTEXT = + USE_GETTEXT_SCHEME = fallthrough USE_LIBPCRE= YesPlease NO_CURL = USE_NED_ALLOCATOR = YesPlease From 19f8a0477bfeec79d5325e79e2ed92b2421bddbb Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sat, 14 Feb 2015 15:03:58 +0000 Subject: [PATCH 28/59] Squash compile warning with MSys2 Signed-off-by: Johannes Schindelin --- compat/win32/pthread.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compat/win32/pthread.h b/compat/win32/pthread.h index 8ad187344f..1ade961769 100644 --- a/compat/win32/pthread.h +++ b/compat/win32/pthread.h @@ -77,7 +77,7 @@ extern pthread_t pthread_self(void); static inline int pthread_exit(void *ret) { - ExitThread((DWORD)ret); + ExitThread((DWORD)(size_t)ret); } typedef DWORD pthread_key_t; From 99578c56172524b248166b029e3034f772b14f3d Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sun, 15 Feb 2015 09:56:35 +0000 Subject: [PATCH 29/59] Error out when mingw_startup() *and* NO_UNSETENV are active The unsetenv code has no idea to update our environ_size, therefore causing segmentation faults when environment variables are removed without compat/mingw.c's knowing (MinGW's optimized lookup would try to strcmp() against NULL in such a case). Signed-off-by: Johannes Schindelin --- compat/mingw.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/compat/mingw.h b/compat/mingw.h index be27d6cb9d..3b9eb21e6f 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -233,6 +233,10 @@ char *mingw_mktemp(char *template); char *mingw_getcwd(char *pointer, int len); #define getcwd mingw_getcwd +#ifdef NO_UNSETENV +#error "NO_UNSETENV is incompatible with the MinGW startup code!" +#endif + char *mingw_getenv(const char *name); #define getenv mingw_getenv int mingw_putenv(const char *namevalue); From 21b5f9939396b591204daee5ac8103b5b13f5da7 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sun, 15 Feb 2015 16:39:10 +0100 Subject: [PATCH 30/59] Special-case the MSYS2_TZ environment variable With MSys2, the "TZ" environment variable gets filtered out when calling non-MSys2 executables. The reason is that Windows' time zone handling is substantially different from the POSIX one. However, we just taught Git for Windows' fork of the MSys2 runtime to pass on the timezone in a different environment variable, MSYS2_TZ for the sole purpose of Git being able to reinterpret it correctly. Signed-off-by: Johannes Schindelin --- compat/mingw.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/compat/mingw.c b/compat/mingw.c index 949ecefdd4..b04dad55e1 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2219,8 +2219,14 @@ void mingw_startup() __argv[0] = wcstoutfdup_startup(buffer, _wpgmptr, maxlen); for (i = 1; i < argc; i++) __argv[i] = wcstoutfdup_startup(buffer, wargv[i], maxlen); - for (i = 0; wenv[i]; i++) + for (i = 0; wenv[i]; i++) { environ[i] = wcstoutfdup_startup(buffer, wenv[i], maxlen); + if (!strncasecmp(environ[i], "MSYS2_TZ=", 9)) { + char *to_free = environ[i]; + environ[i] = xstrdup(to_free + 6); + free(to_free); + } + } environ[i] = NULL; free(buffer); From bf37861ce3df9d26594ff9955022a1c6ff85850a Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 18 Feb 2015 11:55:48 +0000 Subject: [PATCH 31/59] Tests (MinGW): Do not bother to test funny file names MSys2 actually allows to create files or directories whose names contain tabs, newlines or colors, even if plain Win32 API cannot access them. As we are using an MSys2 bash to run the tests, such files or directories are created successfully, but Git has no chance to work with them because it is a regular Windows program, hence limited by the Win32 API. Signed-off-by: Johannes Schindelin --- t/t3300-funny-names.sh | 1 + t/t3600-rm.sh | 3 ++- t/t3703-add-magic-pathspec.sh | 2 +- t/t3902-quoted.sh | 1 + t/t4016-diff-quote.sh | 1 + t/t4135-apply-weird-filenames.sh | 3 ++- t/t9903-bash-prompt.sh | 2 +- 7 files changed, 9 insertions(+), 4 deletions(-) diff --git a/t/t3300-funny-names.sh b/t/t3300-funny-names.sh index 9a146f1335..04de03cad0 100755 --- a/t/t3300-funny-names.sh +++ b/t/t3300-funny-names.sh @@ -13,6 +13,7 @@ tree, index, and tree objects. HT=' ' +test_have_prereq MINGW || echo 2>/dev/null > "Name with an${HT}HT" if ! test -f "Name with an${HT}HT" then diff --git a/t/t3600-rm.sh b/t/t3600-rm.sh index 9d90d2c935..6f762b4404 100755 --- a/t/t3600-rm.sh +++ b/t/t3600-rm.sh @@ -14,7 +14,8 @@ test_expect_success \ git add -- foo bar baz 'space embedded' -q && git commit -m 'add normal files'" -if touch -- 'tab embedded' 'newline + +if ! test_have_prereq MINGW && touch -- 'tab embedded' 'newline embedded' 2>/dev/null then test_set_prereq FUNNYNAMES diff --git a/t/t3703-add-magic-pathspec.sh b/t/t3703-add-magic-pathspec.sh index 5115de7036..aaff784659 100755 --- a/t/t3703-add-magic-pathspec.sh +++ b/t/t3703-add-magic-pathspec.sh @@ -38,7 +38,7 @@ cat >expected </dev/null +if ! test_have_prereq MINGW && mkdir ":" 2>/dev/null then test_set_prereq COLON_DIR fi diff --git a/t/t3902-quoted.sh b/t/t3902-quoted.sh index 892f567844..f528008c36 100755 --- a/t/t3902-quoted.sh +++ b/t/t3902-quoted.sh @@ -12,6 +12,7 @@ GN='純' HT=' ' DQ='"' +test_have_prereq MINGW || echo foo 2>/dev/null > "Name and an${HT}HT" if ! test -f "Name and an${HT}HT" then diff --git a/t/t4016-diff-quote.sh b/t/t4016-diff-quote.sh index cd543ecc54..9c48e5c2c9 100755 --- a/t/t4016-diff-quote.sh +++ b/t/t4016-diff-quote.sh @@ -13,6 +13,7 @@ P1='pathname with HT' P2='pathname with SP' P3='pathname with LF' +test_have_prereq !MINGW && echo 2>/dev/null >"$P1" && test -f "$P1" && rm -f "$P1" || { skip_all='Your filesystem does not allow tabs in filenames' test_done diff --git a/t/t4135-apply-weird-filenames.sh b/t/t4135-apply-weird-filenames.sh index bf5dc57286..6d6b96d5d3 100755 --- a/t/t4135-apply-weird-filenames.sh +++ b/t/t4135-apply-weird-filenames.sh @@ -19,7 +19,8 @@ test_expect_success 'setup' ' test_when_finished "rm -f \"tab embedded.txt\"" && test_when_finished "rm -f '\''\"quoteembedded\".txt'\''" && - if touch -- "tab embedded.txt" '\''"quoteembedded".txt'\'' + if ! test_have_prereq MINGW && + touch -- "tab embedded.txt" '\''"quoteembedded".txt'\'' then test_set_prereq FUNNYNAMES fi diff --git a/t/t9903-bash-prompt.sh b/t/t9903-bash-prompt.sh index 49d58e6726..2e1acef483 100755 --- a/t/t9903-bash-prompt.sh +++ b/t/t9903-bash-prompt.sh @@ -67,7 +67,7 @@ repo_with_newline='repo with newline' -if mkdir "$repo_with_newline" 2>/dev/null +if ! test_have_prereq MINGW && mkdir "$repo_with_newline" 2>/dev/null then test_set_prereq FUNNYNAMES else From f217f6f2a99dcaee40ff595c2e736687945b860f Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 19 Feb 2015 12:25:42 +0000 Subject: [PATCH 32/59] Tests (MinGW): Disable mkfifo-based tests With MSys2, there is actually an implementation of mkfifo available. The only problem is that it is only emulating named pipes through the MSys2 runtime; The Win32 API has no idea about named pipes, hence the Git executable cannot access those pipes either. The symptom is that Git fails with a ': No such file or directory' because MSys2 emulates named pipes through special-crafted '.lnk' files. The solution is to tell the test suite explicitly that we cannot use named pipes when we want to test a MinGW Git. Signed-off-by: Johannes Schindelin --- t/test-lib.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/test-lib.sh b/t/test-lib.sh index 16c4d7b516..32ac1a6936 100644 --- a/t/test-lib.sh +++ b/t/test-lib.sh @@ -998,7 +998,7 @@ test_i18ngrep () { test_lazy_prereq PIPE ' # test whether the filesystem supports FIFOs case $(uname -s) in - CYGWIN*) + CYGWIN*|MINGW*) false ;; *) From 01f7f22488e2e16ea6d1847e6b5b62bb584cb851 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 19 Feb 2015 15:20:16 +0000 Subject: [PATCH 33/59] t5516: override MinGW-specific pwd override This test is susceptible to MSys2's posix-to-windows path mangling; Let's just use POSIX paths throughout and let the tests pass. Signed-off-by: Johannes Schindelin --- t/t5516-fetch-push.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/t/t5516-fetch-push.sh b/t/t5516-fetch-push.sh index ec22c98445..a6f4c889b1 100755 --- a/t/t5516-fetch-push.sh +++ b/t/t5516-fetch-push.sh @@ -16,6 +16,14 @@ This test checks the following functionality: . ./test-lib.sh +if test_have_prereq MINGW +then + # Avoid posix-to-windows path mangling + pwd () { + builtin pwd + } +fi + D=`pwd` mk_empty () { From 64d71aa43efa7c529ebdd692969cbecae4634150 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 20 Feb 2015 14:21:04 +0000 Subject: [PATCH 34/59] Tests: optionally skip redirecting stdin/stdout/stderr There is a really useful debugging technique developed by Sverre Rabbelier that inserts "bash &&" somewhere in the test scripts, letting the developer interact at given points with the current state. Another debugging technique, used a lot by this here coder, is to run certain executables via gdb by guarding a "gdb -args" call in bin-wrappers/git. Both techniques were disabled by 781f76b1(test-lib: redirect stdin of tests). Let's reinstate the ability to run an interactive shell by making the redirection optional: setting the TEST_NO_REDIRECT environment variable will skip the redirection. Signed-off-by: Johannes Schindelin --- t/test-lib.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/t/test-lib.sh b/t/test-lib.sh index 32ac1a6936..a946f74338 100644 --- a/t/test-lib.sh +++ b/t/test-lib.sh @@ -555,6 +555,20 @@ test_eval_ () { # # The test itself is run with stderr put back to &4 (so either to # /dev/null, or to the original stderr if --verbose was used). + if test -n "$TEST_NO_REDIRECT" + then + test_eval_inner_ "$@" + test_eval_ret_=$? + if test "$trace" = t + then + set +x + if test "$test_eval_ret_" != 0 + then + say_color error >&4 "error: last command exited with \$?=$test_eval_ret_" + fi + fi + return $test_eval_ret_ + fi { test_eval_inner_ "$@" &3 2>&4 test_eval_ret_=$? From f13d606091fd87d6a5ddf03a8edd9929d12f69a7 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 20 Feb 2015 15:41:20 +0000 Subject: [PATCH 35/59] mingw: Prepare the TMP environment variable for shell scripts When shell scripts access a $TMP variable containing backslashes, they will be mistaken for escape characters. Let's not let that happen by converting them to forward slashes. This fixes t7800 with MSys2. Signed-off-by: Johannes Schindelin --- compat/mingw.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/compat/mingw.c b/compat/mingw.c index b04dad55e1..7cb8939d11 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2226,6 +2226,18 @@ void mingw_startup() environ[i] = xstrdup(to_free + 6); free(to_free); } + if (!strncasecmp(environ[i], "TMP=", 4)) { + /* + * Convert all dir separators to forward slashes, + * to help shell commands called from the Git + * executable (by not mistaking the dir separators + * for escape characters). + */ + char *p; + for (p = environ[i]; *p; p++) + if (*p == '\\') + *p = '/'; + } } environ[i] = NULL; free(buffer); From e682e0b79b1328f4e85dc42973a0adcec8417589 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sun, 22 Feb 2015 13:33:35 +0000 Subject: [PATCH 36/59] Git.pm: stop assuming that absolute paths start with a slash This fixes t7800 with MSys2. Signed-off-by: Johannes Schindelin --- perl/Git.pm | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/perl/Git.pm b/perl/Git.pm index 9026a7bb98..d1e4c48202 100644 --- a/perl/Git.pm +++ b/perl/Git.pm @@ -188,7 +188,8 @@ sub repository { }; if ($dir) { - $dir =~ m#^/# or $dir = $opts{Directory} . '/' . $dir; + _verify_require(); + File::Spec->file_name_is_absolute($dir) or $dir = $opts{Directory} . '/' . $dir; $opts{Repository} = abs_path($dir); # If --git-dir went ok, this shouldn't die either. From 80e0f5c9dffbf5afa95cff6a7e7a1ccac9617401 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sun, 22 Feb 2015 16:08:36 +0000 Subject: [PATCH 37/59] MSys2: Fix t9700 assumption about directory separators This test assumed that there are no two equivalent directory separators. However, on Windows, the back slash and the forward slash *are* equivalent. Let's paper over this issue by converting the backward slashes to forward ones in the test that fails with MSys2 otherwise. Signed-off-by: Johannes Schindelin --- t/t9700/test.pl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/t/t9700/test.pl b/t/t9700/test.pl index 1140767b50..a01665473d 100755 --- a/t/t9700/test.pl +++ b/t/t9700/test.pl @@ -33,7 +33,9 @@ is($r->config_int("test.int"), 2048, "config_int: integer"); is($r->config_int("test.nonexistent"), undef, "config_int: nonexistent"); ok($r->config_bool("test.booltrue"), "config_bool: true"); ok(!$r->config_bool("test.boolfalse"), "config_bool: false"); -is($r->config_path("test.path"), $r->config("test.pathexpanded"), +our $test_path = $r->config_path("test.path"); +$test_path =~ s/\\/\//g if ($^O eq 'msys'); +is($test_path, $r->config("test.pathexpanded"), "config_path: ~/foo expansion"); is_deeply([$r->config_path("test.pathmulti")], ["foo", "bar"], "config_path: multiple values"); From f6ad82f5daac963fa97dcb9e6a88e17fb138e6d0 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sun, 22 Feb 2015 19:55:26 +0000 Subject: [PATCH 38/59] mingw: Make sure sigset_t is defined With MSys2, the sigset_t type is defined in sys/types.h, therefore we need to #include said file. Signed-off-by: Johannes Schindelin --- compat/mingw.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/compat/mingw.h b/compat/mingw.h index 3b9eb21e6f..46c6c913b4 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -1,5 +1,9 @@ #include #include +#include +#ifndef _POSIX +typedef _sigset_t sigset_t; +#endif #include #include /* From 84add3a1d8b9c1bf5e8c16cb8220f2e4bf646efc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EB=A7=88=EB=88=84=EC=97=98?= Date: Fri, 6 Mar 2015 12:09:51 +0100 Subject: [PATCH 39/59] mingw: Try to delete target directory first. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the rename function tries to move a directory it fails if the target directory exists. It should check if it can delete the (possibly empty) target directory and then try again to move the directory. Helped-by: Johannes Schindelin Signed-off-by: 마누엘 --- compat/mingw.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/compat/mingw.c b/compat/mingw.c index 7cb8939d11..cda9c97dc0 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -1648,7 +1648,12 @@ repeat: if (gle == ERROR_ACCESS_DENIED && (attrs = GetFileAttributesW(wpnew)) != INVALID_FILE_ATTRIBUTES) { if (attrs & FILE_ATTRIBUTE_DIRECTORY) { - errno = EISDIR; + DWORD attrsold = GetFileAttributesW(wpold); + if (attrsold == INVALID_FILE_ATTRIBUTES || + !(attrsold & FILE_ATTRIBUTE_DIRECTORY)) + errno = EISDIR; + else if (!_wrmdir(wpnew)) + goto repeat; return -1; } if ((attrs & FILE_ATTRIBUTE_READONLY) && From 0e508c0e105450976d57a18427fb665d5d50bb6a Mon Sep 17 00:00:00 2001 From: Cesar Eduardo Barros Date: Mon, 9 Mar 2015 08:51:38 +0100 Subject: [PATCH 40/59] mingw: Embed a manifest to trick UAC into Doing The Right Thing On Windows >= Vista, not having an application manifest with a requestedExecutionLevel can cause several kinds of confusing behavior. The first and more obvious behavior is "Installer Detection", where Windows sometimes decides (by looking at things like the file name and even sequences of bytes within the executable) that an executable is an installer and should run elevated (causing the well-known popup dialog to appear). In Git's context, subcommands such as "git patch-id" or "git update-index" fall prey to this behavior. The second and more confusing behavior is "File Virtualization". It means that when files are written without having write permission, it does not fail (as expected), but they are instead redirected to somewhere else. When the files are read, the original contents are returned, though, not the ones that were just written somewhere else. Even more confusing, not all write accesses are redirected; Trying to write to write-protected .exe files, for example, will fail instead of redirecting. In addition to being unwanted behavior, File Virtualization causes dramatic slowdowns in Git (see for instance http://code.google.com/p/msysgit/issues/detail?id=320). There are two ways to prevent those two behaviors: Either you embed an application manifest within all your executables, or you add an external manifest (a file with the same name followed by .manifest) to all your executables. Since Git's builtins are hardlinked (or copied), it is simpler and more robust to embed a manifest. A recent enough MSVC compiler should already embed a working internal manifest, but for MinGW you have to do so by hand. Very lightly tested on Wine, where like on Windows XP it should not make any difference. References: - New UAC Technologies for Windows Vista http://msdn.microsoft.com/en-us/library/bb756960.aspx - Create and Embed an Application Manifest (UAC) http://msdn.microsoft.com/en-us/library/bb756929.aspx [js: simplified the embedding dramatically by reusing Git for Windows' existing Windows resource file, removed the optional (and dubious) processorArchitecture attribute of the manifest's assemblyIdentity section.] Signed-off-by: Cesar Eduardo Barros Signed-off-by: Johannes Schindelin --- compat/win32/git.manifest | 25 +++++++++++++++++++++++++ git.rc | 2 ++ 2 files changed, 27 insertions(+) create mode 100644 compat/win32/git.manifest diff --git a/compat/win32/git.manifest b/compat/win32/git.manifest new file mode 100644 index 0000000000..771e3cce43 --- /dev/null +++ b/compat/win32/git.manifest @@ -0,0 +1,25 @@ + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/git.rc b/git.rc index 33aafb786c..4aa4f9973e 100644 --- a/git.rc +++ b/git.rc @@ -20,3 +20,5 @@ BEGIN VALUE "Translation", 0x409, 1200 END END + +1 RT_MANIFEST "compat/win32/git.manifest" From be0f9bec037731293f784a0f26a4f8437385cc25 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Mon, 20 Apr 2015 19:24:29 +0200 Subject: [PATCH 41/59] mingw: make isatty() recognize MSys pseudo terminals (/dev/pty*) MSys2 emulates pseudo terminals via named pipes, and isatty() returns 0 for such file descriptors. Therefore, some interactive functionality (such as launching a pager, asking if a failed unlink should be repeated etc.) doesn't work when run in a terminal emulator that uses MSys ptys (such as mintty). However, MSys uses special names for its pty pipes ('msys-*-pty*'), which allows us to distinguish them from normal piped input / output. On startup, check if stdin / stdout / stderr are connected to such pipes using the NtQueryObject API from NTDll.dll. If the names match, adjust the flags in MSVCRT's ioinfo structure accordingly. Signed-off-by: Karsten Blees --- compat/winansi.c | 58 ++++++++++++++++++++++++++++++++++++++++++++---- config.mak.uname | 3 ++- 2 files changed, 56 insertions(+), 5 deletions(-) diff --git a/compat/winansi.c b/compat/winansi.c index aac430a952..23b0fe7e86 100644 --- a/compat/winansi.c +++ b/compat/winansi.c @@ -482,6 +482,7 @@ static size_t sizeof_ioinfo = 0; #define IOINFO_L2E 5 #define IOINFO_ARRAY_ELTS (1 << IOINFO_L2E) +#define FPIPE 0x08 #define FDEV 0x40 static inline ioinfo* _pioinfo(int fd) @@ -529,6 +530,45 @@ static HANDLE swap_osfhnd(int fd, HANDLE new_handle) return old_handle; } +#ifdef DETECT_MSYS_TTY + +#include +#include + +static void detect_msys_tty(int fd) +{ + ULONG result; + BYTE buffer[1024]; + POBJECT_NAME_INFORMATION nameinfo = (POBJECT_NAME_INFORMATION) buffer; + PWSTR name; + + /* check if fd is a pipe */ + HANDLE h = (HANDLE) _get_osfhandle(fd); + if (GetFileType(h) != FILE_TYPE_PIPE) + return; + + /* get pipe name */ + if (!NT_SUCCESS(NtQueryObject(h, ObjectNameInformation, + buffer, sizeof(buffer) - 2, &result))) + return; + name = nameinfo->Name.Buffer; + name[nameinfo->Name.Length] = 0; + + /* check if this could be a msys pty pipe ('msys-XXXX-ptyN-XX') */ + if (!wcsstr(name, L"msys-") || !wcsstr(name, L"-pty")) + return; + + /* init ioinfo size if we haven't done so */ + if (init_sizeof_ioinfo()) + return; + + /* set FDEV flag, reset FPIPE flag */ + _pioinfo(fd)->osflags &= ~FPIPE; + _pioinfo(fd)->osflags |= FDEV; +} + +#endif + void winansi_init(void) { int con1, con2; @@ -537,8 +577,15 @@ void winansi_init(void) /* check if either stdout or stderr is a console output screen buffer */ con1 = is_console(1); con2 = is_console(2); - if (!con1 && !con2) + if (!con1 && !con2) { +#ifdef DETECT_MSYS_TTY + /* check if stdin / stdout / stderr are msys pty pipes */ + detect_msys_tty(0); + detect_msys_tty(1); + detect_msys_tty(2); +#endif return; + } /* create a named pipe to communicate with the console thread */ sprintf(name, "\\\\.\\pipe\\winansi%lu", GetCurrentProcessId()); @@ -574,8 +621,11 @@ void winansi_init(void) HANDLE winansi_get_osfhandle(int fd) { HANDLE hnd = (HANDLE) _get_osfhandle(fd); - if ((fd == 1 || fd == 2) && isatty(fd) - && GetFileType(hnd) == FILE_TYPE_PIPE) - return (fd == 1) ? hconsole1 : hconsole2; + if (isatty(fd) && GetFileType(hnd) == FILE_TYPE_PIPE) { + if (fd == 1 && hconsole1) + return hconsole1; + else if (fd == 2 && hconsole2) + return hconsole2; + } return hnd; } diff --git a/config.mak.uname b/config.mak.uname index c61899c49f..2569b23fa9 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -557,7 +557,8 @@ else BASIC_LDFLAGS += -Wl,--large-address-aware endif CC = gcc - COMPAT_CFLAGS += -D__USE_MINGW_ANSI_STDIO=0 + COMPAT_CFLAGS += -D__USE_MINGW_ANSI_STDIO=0 -DDETECT_MSYS_TTY + EXTLIBS += -lntdll INSTALL = /bin/install NO_R_TO_GCC_LINKER = YesPlease INTERNAL_QSORT = YesPlease From e7941b8276236f7dd1f2fddb7ff8bb94445e606b Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 17 Mar 2015 19:41:02 +0100 Subject: [PATCH 42/59] UTF-8 environment: be a little bit more defensive It is unlikely that we have an empty environment, ever, but *if* we do, when `environ_size - 1` is passed to `bsearchenv()` it is misinterpreted as a real large integer. To make the code truly defensive, refuse to do anything at all if the size is negative (which should not happen, of course). Signed-off-by: Johannes Schindelin --- compat/mingw.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index cda9c97dc0..28f1456802 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -1292,7 +1292,7 @@ static int bsearchenv(char **env, const char *name, size_t size) */ static int do_putenv(char **env, const char *name, int size, int free_old) { - int i = bsearchenv(env, name, size - 1); + int i = size <= 0 ? -1 : bsearchenv(env, name, size - 1); /* optionally free removed / replaced entry */ if (i >= 0 && free_old) @@ -1317,7 +1317,13 @@ static int do_putenv(char **env, const char *name, int size, int free_old) char *mingw_getenv(const char *name) { char *value; - int pos = bsearchenv(environ, name, environ_size - 1); + int pos; + + if (environ_size <= 0) + return NULL; + + pos = bsearchenv(environ, name, environ_size - 1); + if (pos < 0) return NULL; value = strchr(environ[pos], '='); From 5478f4d38c6fb56af1adabb814b0471bf8f8664a Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 18 Mar 2015 10:57:52 +0100 Subject: [PATCH 43/59] mingw: be more defensive when making the environment block Outside of our Windows-specific code, the end of the environment can be marked also by a pointer to a NUL character, not only by a NULL pointer as our code assumed so far. That led to a buffer overrun in `make_environment_block()` when running `git-remote-https` in `mintty` (because `curl_global_init()` added the `CHARSET` environment variable *outside* of `mingw_putenv()`, ending the environment in a pointer to an empty string). Side note for future debugging on Windows: when running programs in `mintty`, the standard input/output/error is not connected to a Win32 Console, but instead is pipe()d. That means that even stderr may not be written completely before a crash, but has to be fflush()ed explicitly. For example, when debugging crashes, the developer should insert an `fflush(stderr);` at the end of the `error()` function defined in usage.c. Signed-off-by: Johannes Schindelin --- compat/mingw.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 28f1456802..e9eef228c2 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -968,7 +968,7 @@ static wchar_t *make_environment_block(char **deltaenv) char **tmpenv; int i = 0, size = environ_size, wenvsz = 0, wenvpos = 0; - while (deltaenv && deltaenv[i]) + while (deltaenv && deltaenv[i] && *deltaenv[i]) i++; /* copy the environment, leaving space for changes */ @@ -976,11 +976,11 @@ static wchar_t *make_environment_block(char **deltaenv) memcpy(tmpenv, environ, size * sizeof(char*)); /* merge supplied environment changes into the temporary environment */ - for (i = 0; deltaenv && deltaenv[i]; i++) + for (i = 0; deltaenv && deltaenv[i] && *deltaenv[i]; i++) size = do_putenv(tmpenv, deltaenv[i], size, 0); /* create environment block from temporary environment */ - for (i = 0; tmpenv[i]; i++) { + for (i = 0; tmpenv[i] && *tmpenv[i]; i++) { size = 2 * strlen(tmpenv[i]) + 2; /* +2 for final \0 */ ALLOC_GROW(wenvblk, (wenvpos + size) * sizeof(wchar_t), wenvsz); wenvpos += xutftowcs(&wenvblk[wenvpos], tmpenv[i], size) + 1; From bd469f34ef5c1597f2ad7b7a02259dfea39df845 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 17 Mar 2015 19:41:31 +0100 Subject: [PATCH 44/59] mingw: be *very* wary about outside environment changes The environment is modified in most surprising circumstances, and not all of them are under Git's control. For example, calling curl_global_init() on Windows will ensure that the CHARSET variable is set, adding one if necessary. While the previous commit worked around crashes triggered by such outside changes of the environment by relaxing the requirement that the environment be terminated by a NULL pointer, the other assumption made by `mingw_getenv()` and `mingw_putenv()` is that the environment is sorted, for efficient lookup via binary search. Let's make real sure that our environment is intact before querying or modifying it, and reinitialize our idea of the environment if necessary. With this commit, before working on the environment we look briefly for indicators that the environment was modified outside of our control, and to ensure that it is terminated with a NULL pointer and sorted again in that case. Note: the indicators are maybe not sufficient. For example, when a variable is removed, it will not be noticed. It might also be a problem if outside changes to the environment result in a modified `environ` pointer: it is unclear whether such a modification could result in a problem when `mingw_putenv()` needs to `realloc()` the environment buffer. For the moment, however, the current fix works well enough, so let's only face the potential problems when (and if!) they occur. Signed-off-by: Johannes Schindelin --- compat/mingw.c | 49 +++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index e9eef228c2..7893a36b06 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -957,6 +957,10 @@ static int do_putenv(char **env, const char *name, int size, int free_old); static int environ_size = 0; /* allocated size of environ array, in bytes */ static int environ_alloc = 0; +/* used as a indicator when the environment has been changed outside mingw.c */ +static char **saved_environ; + +static void maybe_reinitialize_environ(void); /* * Create environment block suitable for CreateProcess. Merges current @@ -966,7 +970,10 @@ static wchar_t *make_environment_block(char **deltaenv) { wchar_t *wenvblk = NULL; char **tmpenv; - int i = 0, size = environ_size, wenvsz = 0, wenvpos = 0; + int i = 0, size, wenvsz = 0, wenvpos = 0; + + maybe_reinitialize_environ(); + size = environ_size; while (deltaenv && deltaenv[i] && *deltaenv[i]) i++; @@ -1269,6 +1276,41 @@ static int compareenv(const void *v1, const void *v2) } } +/* + * Functions implemented outside Git are able to modify the environment, + * too. For example, cURL's curl_global_init() function sets the CHARSET + * environment variable (at least in certain circumstances). + * + * Therefore we need to be *really* careful *not* to assume that we have + * sole control over the environment and reinitalize it when necessary. + */ +static void maybe_reinitialize_environ(void) +{ + int i; + + if (!saved_environ) { + warning("MinGW environment not initialized yet"); + return; + } + + if (environ_size <= 0) + return; + + if (saved_environ != environ) + /* We have *no* idea how much space was allocated outside */ + environ_alloc = 0; + else if (!environ[environ_size - 1]) + return; /* still consistent */ + + for (i = 0; environ[i] && *environ[i]; i++) + ; /* continue counting */ + environ[i] = NULL; + environ_size = i + 1; + + /* sort environment for O(log n) getenv / putenv */ + qsort(environ, i, sizeof(char*), compareenv); +} + static int bsearchenv(char **env, const char *name, size_t size) { unsigned low = 0, high = size; @@ -1322,6 +1364,7 @@ char *mingw_getenv(const char *name) if (environ_size <= 0) return NULL; + maybe_reinitialize_environ(); pos = bsearchenv(environ, name, environ_size - 1); if (pos < 0) @@ -1332,7 +1375,9 @@ char *mingw_getenv(const char *name) int mingw_putenv(const char *namevalue) { + maybe_reinitialize_environ(); ALLOC_GROW(environ, (environ_size + 1) * sizeof(char*), environ_alloc); + saved_environ = environ; environ_size = do_putenv(environ, namevalue, environ_size, 1); return 0; } @@ -2220,7 +2265,7 @@ void mingw_startup() */ environ_size = i + 1; environ_alloc = alloc_nr(environ_size * sizeof(char*)); - environ = malloc_startup(environ_alloc); + saved_environ = environ = malloc_startup(environ_alloc); /* allocate buffer (wchar_t encodes to max 3 UTF-8 bytes) */ maxlen = 3 * maxlen + 1; From 9a67f5f26413e1bde57c253050fca2a1011e8f00 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 31 Mar 2015 17:39:54 +0100 Subject: [PATCH 45/59] mingw: uglify pthread_mutex_init definition to shut up warning When the result of a (a, 0) expression is not used, GCC now finds it necessary to complain with a warning: right-hand operand of comma expression has no effect Let's just pretend to use the 0 value and have a peaceful and quiet life again. Signed-off-by: Johannes Schindelin --- compat/nedmalloc/malloc.c.h | 3 ++- compat/win32/pthread.h | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/compat/nedmalloc/malloc.c.h b/compat/nedmalloc/malloc.c.h index 18634e3a3b..9df8aaed87 100644 --- a/compat/nedmalloc/malloc.c.h +++ b/compat/nedmalloc/malloc.c.h @@ -1804,9 +1804,10 @@ struct win32_mlock_t volatile long threadid; }; +static inline int return_0(int i) { return 0; } #define MLOCK_T struct win32_mlock_t #define CURRENT_THREAD win32_getcurrentthreadid() -#define INITIAL_LOCK(sl) (memset(sl, 0, sizeof(MLOCK_T)), 0) +#define INITIAL_LOCK(sl) (memset(sl, 0, sizeof(MLOCK_T)), return_0(0)) #define ACQUIRE_LOCK(sl) win32_acquire_lock(sl) #define RELEASE_LOCK(sl) win32_release_lock(sl) #define TRY_LOCK(sl) win32_try_lock(sl) diff --git a/compat/win32/pthread.h b/compat/win32/pthread.h index 1ade961769..62151d7550 100644 --- a/compat/win32/pthread.h +++ b/compat/win32/pthread.h @@ -18,7 +18,10 @@ */ #define pthread_mutex_t CRITICAL_SECTION -#define pthread_mutex_init(a,b) (InitializeCriticalSection((a)), 0) +static inline int return_0(int i) { + return 0; +} +#define pthread_mutex_init(a,b) return_0((InitializeCriticalSection((a)), 0)) #define pthread_mutex_destroy(a) DeleteCriticalSection((a)) #define pthread_mutex_lock EnterCriticalSection #define pthread_mutex_unlock LeaveCriticalSection From 7aadde68b29f4922029e80872796292d3821301c Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 31 Mar 2015 17:51:57 +0100 Subject: [PATCH 46/59] mingw: avoid warnings when casting HANDLEs to int HANDLE is defined internally as a void *, but in many cases it is actually guaranteed to be a 32-bit integer. In these cases, GCC should not warn about a cast of a pointer to an integer of a different type because we know exactly what we are doing. Signed-off-by: Johannes Schindelin --- compat/mingw.c | 9 ++++++--- compat/poll/poll.c | 2 +- compat/winansi.c | 3 ++- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 7893a36b06..f200a3cf70 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -6,6 +6,8 @@ #include "../run-command.h" #include "../cache.h" +#define HCAST(type, handle) ((type)(intptr_t)handle) + static const int delay[] = { 0, 1, 10, 20, 40 }; unsigned int _CRT_fmode = _O_BINARY; @@ -726,13 +728,13 @@ int pipe(int filedes[2]) errno = err_win_to_posix(GetLastError()); return -1; } - filedes[0] = _open_osfhandle((int)h[0], O_NOINHERIT); + filedes[0] = _open_osfhandle(HCAST(int, h[0]), O_NOINHERIT); if (filedes[0] < 0) { CloseHandle(h[0]); CloseHandle(h[1]); return -1; } - filedes[1] = _open_osfhandle((int)h[1], O_NOINHERIT); + filedes[1] = _open_osfhandle(HCAST(int, h[1]), O_NOINHERIT); if (filedes[1] < 0) { close(filedes[0]); CloseHandle(h[1]); @@ -1949,7 +1951,8 @@ void mingw_open_html(const char *unixpath) die("cannot run browser"); printf("Launching default browser to display HTML ...\n"); - r = (int)ShellExecute(NULL, "open", htmlpath, NULL, "\\", SW_SHOWNORMAL); + r = HCAST(int, ShellExecute(NULL, "open", htmlpath, + NULL, "\\", SW_SHOWNORMAL)); FreeLibrary(shell32); /* see the MSDN documentation referring to the result codes here */ if (r <= 32) { diff --git a/compat/poll/poll.c b/compat/poll/poll.c index a9b41d89f4..300e95f12f 100644 --- a/compat/poll/poll.c +++ b/compat/poll/poll.c @@ -76,7 +76,7 @@ #ifdef WIN32_NATIVE -#define IsConsoleHandle(h) (((long) (h) & 3) == 3) +#define IsConsoleHandle(h) (((long) (intptr_t) (h) & 3) == 3) static BOOL IsSocketHandle (HANDLE h) diff --git a/compat/winansi.c b/compat/winansi.c index 23b0fe7e86..33910badb9 100644 --- a/compat/winansi.c +++ b/compat/winansi.c @@ -454,7 +454,8 @@ static HANDLE duplicate_handle(HANDLE hnd) HANDLE hresult, hproc = GetCurrentProcess(); if (!DuplicateHandle(hproc, hnd, hproc, &hresult, 0, TRUE, DUPLICATE_SAME_ACCESS)) - die_lasterr("DuplicateHandle(%li) failed", (long) hnd); + die_lasterr("DuplicateHandle(%li) failed", + (long) (intptr_t) hnd); return hresult; } From 68b876e7b51177a8dc17dadfecc9320793019544 Mon Sep 17 00:00:00 2001 From: Sebastian Schuberth Date: Fri, 23 Mar 2012 10:58:37 +0100 Subject: [PATCH 47/59] am: Use cat instead of echo to avoid DOS line-endings (fixes t4150) Along the lines of 05d0e3b and f33946d, use cat instead of echo to avoid line ending mismatches in the test result of "am empty-file does not infloop" which make the test fail. Signed-off-by: Sebastian Schuberth --- git-am.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/git-am.sh b/git-am.sh index 455ce4fca0..f4061e11ea 100755 --- a/git-am.sh +++ b/git-am.sh @@ -188,7 +188,9 @@ It does not apply to blobs recorded in its index.")" } clean_abort () { - test $# = 0 || echo >&2 "$@" + test $# = 0 || cat >&2 < Date: Mon, 9 Apr 2012 21:09:49 -0500 Subject: [PATCH 48/59] Windows: make sure that merge-octopus only outputs LF line endings This happens to shut up t7602 on Windows which would otherwise take the different line endings for a sign that the merge failed. Signed-off-by: Johannes Schindelin --- git-merge-octopus.sh | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/git-merge-octopus.sh b/git-merge-octopus.sh index 8643f74cb0..2226eeba99 100755 --- a/git-merge-octopus.sh +++ b/git-merge-octopus.sh @@ -71,7 +71,9 @@ do case "$LF$common$LF" in *"$LF$SHA1$LF"*) - echo "Already up-to-date with $pretty_name" + cat << EOF +Already up-to-date with $pretty_name +EOF continue ;; esac @@ -83,7 +85,9 @@ do # tree as the intermediate result of the merge. # We still need to count this as part of the parent set. - echo "Fast-forwarding to: $pretty_name" + cat << EOF +Fast-forwarding to: $pretty_name +EOF git read-tree -u -m $head $SHA1 || exit MRC=$SHA1 MRT=$(git write-tree) continue @@ -91,7 +95,9 @@ do NON_FF_MERGE=1 - echo "Trying simple merge with $pretty_name" + cat << EOF +Trying simple merge with $pretty_name +EOF git read-tree -u -m --aggressive $common $MRT $SHA1 || exit 2 next=$(git write-tree 2>/dev/null) if test $? -ne 0 From a4171a997b94e04394b1478433bd3987e007b513 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 6 Apr 2015 14:34:59 +0100 Subject: [PATCH 49/59] git-gui (MinGW): make use of MSys2's msgfmt When Git for Windows was still based on MSys1, we had no gettext, ergo no msgfmt, either. Therefore, we introduced a small and simple Tcl script to perform the same task. However, with MSys2, we no longer need that because we have a proper msgfmt executable. Plus, the po2msg.sh script somehow manages to hang when run in parallel. Two reasons to use real msgfmt.exe instead. Signed-off-by: Johannes Schindelin --- git-gui/Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/git-gui/Makefile b/git-gui/Makefile index 4f00bdd3d6..de1d17491f 100644 --- a/git-gui/Makefile +++ b/git-gui/Makefile @@ -161,7 +161,9 @@ ifeq ($(uname_S),Darwin) endif endif ifneq (,$(findstring MINGW,$(uname_S))) +ifeq ($(shell expr "$(uname_R)" : '1\.'),2) NO_MSGFMT=1 +endif GITGUI_WINDOWS_WRAPPER := YesPlease GITGUI_RELATIVE := 1 endif From 11f0f3d8a888b4819b32433faf5f228991312d37 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 6 Apr 2015 10:37:04 +0100 Subject: [PATCH 50/59] Avoid illegal filenames when building Documentation on NTFS A '+' is not a valid part of a filename with Windows file systems (it is reserved because the '+' operator meant file concatenation back in the DOS days). Let's just not use it. Signed-off-by: Johannes Schindelin --- Documentation/Makefile | 84 +++++++++++++++++++++--------------------- 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/Documentation/Makefile b/Documentation/Makefile index df3901d83a..d1afd45e9a 100644 --- a/Documentation/Makefile +++ b/Documentation/Makefile @@ -265,9 +265,9 @@ docdep_prereqs = \ cmd-list.made $(cmds_txt) doc.dep : $(docdep_prereqs) $(wildcard *.txt) build-docdep.perl - $(QUIET_GEN)$(RM) $@+ $@ && \ - $(PERL_PATH) ./build-docdep.perl >$@+ $(QUIET_STDERR) && \ - mv $@+ $@ + $(QUIET_GEN)$(RM) $@.new $@ && \ + $(PERL_PATH) ./build-docdep.perl >$@.new $(QUIET_STDERR) && \ + mv $@.new $@ -include doc.dep @@ -303,8 +303,8 @@ mergetools-list.made: ../git-mergetool--lib.sh $(wildcard ../mergetools/*) date >$@ clean: - $(RM) *.xml *.xml+ *.html *.html+ *.1 *.5 *.7 - $(RM) *.texi *.texi+ *.texi++ git.info gitman.info + $(RM) *.xml *.xml.new *.html *.html.new *.1 *.5 *.7 + $(RM) *.texi *.texi.new *.texi.new.new git.info gitman.info $(RM) *.pdf $(RM) howto-index.txt howto/*.html doc.dep $(RM) technical/*.html technical/api-index.txt @@ -312,14 +312,14 @@ clean: $(RM) manpage-base-url.xsl $(MAN_HTML): %.html : %.txt asciidoc.conf - $(QUIET_ASCIIDOC)$(RM) $@+ $@ && \ - $(TXT_TO_HTML) -d manpage -o $@+ $< && \ - mv $@+ $@ + $(QUIET_ASCIIDOC)$(RM) $@.new $@ && \ + $(TXT_TO_HTML) -d manpage -o $@.new $< && \ + mv $@.new $@ $(OBSOLETE_HTML): %.html : %.txto asciidoc.conf - $(QUIET_ASCIIDOC)$(RM) $@+ $@ && \ - $(TXT_TO_HTML) -o $@+ $< && \ - mv $@+ $@ + $(QUIET_ASCIIDOC)$(RM) $@.new $@ && \ + $(TXT_TO_HTML) -o $@.new $< && \ + mv $@.new $@ manpage-base-url.xsl: manpage-base-url.xsl.in sed "s|@@MAN_BASE_URL@@|$(MAN_BASE_URL)|" $< > $@ @@ -329,14 +329,14 @@ manpage-base-url.xsl: manpage-base-url.xsl.in $(XMLTO) -m $(MANPAGE_XSL) $(XMLTO_EXTRA) man $< %.xml : %.txt asciidoc.conf - $(QUIET_ASCIIDOC)$(RM) $@+ $@ && \ - $(TXT_TO_XML) -d manpage -o $@+ $< && \ - mv $@+ $@ + $(QUIET_ASCIIDOC)$(RM) $@.new $@ && \ + $(TXT_TO_XML) -d manpage -o $@.new $< && \ + mv $@.new $@ user-manual.xml: user-manual.txt user-manual.conf - $(QUIET_ASCIIDOC)$(RM) $@+ $@ && \ - $(TXT_TO_XML) -d book -o $@+ $< && \ - mv $@+ $@ + $(QUIET_ASCIIDOC)$(RM) $@.new $@ && \ + $(TXT_TO_XML) -d book -o $@.new $< && \ + mv $@.new $@ technical/api-index.txt: technical/api-index-skel.txt \ technical/api-index.sh $(patsubst %,%.txt,$(API_DOCS)) @@ -350,45 +350,45 @@ XSLT = docbook.xsl XSLTOPTS = --xinclude --stringparam html.stylesheet docbook-xsl.css user-manual.html: user-manual.xml $(XSLT) - $(QUIET_XSLTPROC)$(RM) $@+ $@ && \ - xsltproc $(XSLTOPTS) -o $@+ $(XSLT) $< && \ - mv $@+ $@ + $(QUIET_XSLTPROC)$(RM) $@.new $@ && \ + xsltproc $(XSLTOPTS) -o $@.new $(XSLT) $< && \ + mv $@.new $@ git.info: user-manual.texi $(QUIET_MAKEINFO)$(MAKEINFO) --no-split -o $@ user-manual.texi user-manual.texi: user-manual.xml - $(QUIET_DB2TEXI)$(RM) $@+ $@ && \ - $(DOCBOOK2X_TEXI) user-manual.xml --encoding=UTF-8 --to-stdout >$@++ && \ - $(PERL_PATH) fix-texi.perl <$@++ >$@+ && \ - rm $@++ && \ - mv $@+ $@ + $(QUIET_DB2TEXI)$(RM) $@.new $@ && \ + $(DOCBOOK2X_TEXI) user-manual.xml --encoding=UTF-8 --to-stdout >$@.new.new && \ + $(PERL_PATH) fix-texi.perl <$@.new.new >$@.new && \ + rm $@.new.new && \ + mv $@.new $@ user-manual.pdf: user-manual.xml - $(QUIET_DBLATEX)$(RM) $@+ $@ && \ - $(DBLATEX) -o $@+ -p $(ASCIIDOC_DBLATEX_DIR)/asciidoc-dblatex.xsl -s $(ASCIIDOC_DBLATEX_DIR)/asciidoc-dblatex.sty $< && \ - mv $@+ $@ + $(QUIET_DBLATEX)$(RM) $@.new $@ && \ + $(DBLATEX) -o $@.new -p $(ASCIIDOC_DBLATEX_DIR)/asciidoc-dblatex.xsl -s $(ASCIIDOC_DBLATEX_DIR)/asciidoc-dblatex.sty $< && \ + mv $@.new $@ gitman.texi: $(MAN_XML) cat-texi.perl - $(QUIET_DB2TEXI)$(RM) $@+ $@ && \ + $(QUIET_DB2TEXI)$(RM) $@.new $@ && \ ($(foreach xml,$(MAN_XML),$(DOCBOOK2X_TEXI) --encoding=UTF-8 \ - --to-stdout $(xml) &&) true) > $@++ && \ - $(PERL_PATH) cat-texi.perl $@ <$@++ >$@+ && \ - rm $@++ && \ - mv $@+ $@ + --to-stdout $(xml) &&) true) > $@.new.new && \ + $(PERL_PATH) cat-texi.perl $@ <$@.new.new >$@.new && \ + rm $@.new.new && \ + mv $@.new $@ gitman.info: gitman.texi $(QUIET_MAKEINFO)$(MAKEINFO) --no-split --no-validate $*.texi $(patsubst %.txt,%.texi,$(MAN_TXT)): %.texi : %.xml - $(QUIET_DB2TEXI)$(RM) $@+ $@ && \ - $(DOCBOOK2X_TEXI) --to-stdout $*.xml >$@+ && \ - mv $@+ $@ + $(QUIET_DB2TEXI)$(RM) $@.new $@ && \ + $(DOCBOOK2X_TEXI) --to-stdout $*.xml >$@.new && \ + mv $@.new $@ howto-index.txt: howto-index.sh $(wildcard howto/*.txt) - $(QUIET_GEN)$(RM) $@+ $@ && \ - '$(SHELL_PATH_SQ)' ./howto-index.sh $(sort $(wildcard howto/*.txt)) >$@+ && \ - mv $@+ $@ + $(QUIET_GEN)$(RM) $@.new $@ && \ + '$(SHELL_PATH_SQ)' ./howto-index.sh $(sort $(wildcard howto/*.txt)) >$@.new && \ + mv $@.new $@ $(patsubst %,%.html,$(ARTICLES)) : %.html : %.txt $(QUIET_ASCIIDOC)$(TXT_TO_HTML) $*.txt @@ -397,10 +397,10 @@ WEBDOC_DEST = /pub/software/scm/git/docs howto/%.html: ASCIIDOC_EXTRA += -a git-relative-html-prefix=../ $(patsubst %.txt,%.html,$(wildcard howto/*.txt)): %.html : %.txt - $(QUIET_ASCIIDOC)$(RM) $@+ $@ && \ + $(QUIET_ASCIIDOC)$(RM) $@.new $@ && \ sed -e '1,/^$$/d' $< | \ - $(TXT_TO_HTML) - >$@+ && \ - mv $@+ $@ + $(TXT_TO_HTML) - >$@.new && \ + mv $@.new $@ install-webdoc : html '$(SHELL_PATH_SQ)' ./install-webdoc.sh $(WEBDOC_DEST) From 14f3626223318304fa8fd7e56d6e79fd7ff270c2 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Thu, 9 Apr 2015 16:19:56 +0100 Subject: [PATCH 51/59] gettext: always use UTF-8 on native Windows Git on native Windows exclusively uses UTF-8 for console output (both with mintty and native console windows). Gettext uses setlocale() to determine the output encoding for translated text, however, MSVCRT's setlocale() doesn't support UTF-8. As a result, translated text is encoded in system encoding (GetAPC()), and non-ASCII chars are mangled in console output. Use gettext's bind_textdomain_codeset() to force the encoding to UTF-8 on native Windows. In this developers' setup, HAVE_LIBCHARSET_H is apparently defined, but we *really* want to override the locale_charset() here. Signed-off-by: Karsten Blees --- gettext.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/gettext.c b/gettext.c index 7378ba287f..0a432d3492 100644 --- a/gettext.c +++ b/gettext.c @@ -10,7 +10,9 @@ #ifndef NO_GETTEXT # include # include -# ifdef HAVE_LIBCHARSET_H +# ifdef GIT_WINDOWS_NATIVE +# define locale_charset() "UTF-8" +# elif defined HAVE_LIBCHARSET_H # include # else # include From d80b693a8161ceb40d5c986c8c95b90053c9a19b Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 7 Apr 2015 10:24:55 +0100 Subject: [PATCH 52/59] mingw: HOT FIX: work around environment issues -- again This developer should really, really have known better. The fact that we are changing the environment in ways for which the MSVCRT is not prepared for is bad enough. But then this developer followed the request to re-enable nedmalloc -- despite the prediction that it would cause an access violation, predicting it in the same message as the request to re-enable nedmalloc, no less! To paper over the issue until the time when this developer finds the time to re-design the Unicode environment handling from scratch, let's hope that cURL is the only library we are using that *may* set an environment variable using MSVCRT's putenv() after we fscked the environment up. Note: this commit can serve as no source of pride to anyone, certainly not yours truly. It is necessary as a quick and pragmatic stop gap, though, to prevent worse problems. Note: cURL manages to set the variable CHARSET when nedmalloc is *not* enabled, without causing an access violation. In that case, it sets it successfully to the value "cp" + GetACP() (hence it is our choice, too, cURL may need it, Git does not): https://github.com/bagder/curl/blob/aa5808b5/lib/easy.c#L157-L162 Signed-off-by: Johannes Schindelin --- compat/mingw.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/compat/mingw.c b/compat/mingw.c index f200a3cf70..57dfb22d3d 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2319,6 +2319,17 @@ void mingw_startup() if (!getenv("TERM")) setenv("TERM", "cygwin", 1); + /* + * Avoid a segmentation fault when cURL tries to set the CHARSET + * variable and putenv() barfs at our nedmalloc'ed environment. + */ + if (!getenv("CHARSET")) { + struct strbuf buf = STRBUF_INIT; + strbuf_addf(&buf, "cp%u", GetACP()); + setenv("CHARSET", buf.buf, 1); + strbuf_release(&buf); + } + /* initialize critical section for waitpid pinfo_t list */ InitializeCriticalSection(&pinfo_cs); From da50ac12e1853c263d9bfcb9cfe3405ea77d495e Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Thu, 2 Apr 2015 19:37:08 +0200 Subject: [PATCH 53/59] mingw: factor out Windows specific environment setup Signed-off-by: Karsten Blees --- compat/mingw.c | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 57dfb22d3d..fa601dc399 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2209,6 +2209,22 @@ int handle_long_path(wchar_t *path, int len, int max_path, int expand) } } +static void setup_windows_environment() +{ + /* on Windows it is TMP and TEMP */ + if (!getenv("TMPDIR")) { + const char *tmp = getenv("TMP"); + if (!tmp) + tmp = getenv("TEMP"); + if (tmp) + setenv("TMPDIR", tmp, 1); + } + + /* simulate TERM to enable auto-color (see color.c) */ + if (!getenv("TERM")) + setenv("TERM", "cygwin", 1); +} + /* * Disable MSVCRT command line wildcard expansion (__getmainargs called from * mingw startup code, see init.c in mingw runtime). @@ -2305,19 +2321,7 @@ void mingw_startup() qsort(environ, i, sizeof(char*), compareenv); /* fix Windows specific environment settings */ - - /* on Windows it is TMP and TEMP */ - if (!mingw_getenv("TMPDIR")) { - const char *tmp = mingw_getenv("TMP"); - if (!tmp) - tmp = mingw_getenv("TEMP"); - if (tmp) - setenv("TMPDIR", tmp, 1); - } - - /* simulate TERM to enable auto-color (see color.c) */ - if (!getenv("TERM")) - setenv("TERM", "cygwin", 1); + setup_windows_environment(); /* * Avoid a segmentation fault when cURL tries to set the CHARSET From 65d86875a9c410c30c29535735db26f1fb4beab2 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Mon, 6 Apr 2015 21:47:07 +0200 Subject: [PATCH 54/59] mingw: move MSys2 specific environment tweaks to setup_windows_environment Lets keep the environment initialization and conversion section as lean as possible and move recently added tweaks to setup_windows_environment(). This fixes the following potential problems: * Prevent duplicate TZ variables if both TZ and MSYS2_TZ are set. * Some of the higher level x* APIs from wrapper.c require a working getenv(), using e.g. xstrdup() during initialization is dangerous. * Slashifying the Windows TMP variable may break native Windows programs, use POSIX TMPDIR instead. * Properly slashify TMPDIR even if it is already set, and also if we only have TEMP, but not TMP. * Reduce complexity from O(n) to O(log n). Signed-off-by: Karsten Blees --- compat/mingw.c | 40 +++++++++++++++++++--------------------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index fa601dc399..b36da8d503 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2211,15 +2211,31 @@ int handle_long_path(wchar_t *path, int len, int max_path, int expand) static void setup_windows_environment() { + char *tmp; + /* on Windows it is TMP and TEMP */ if (!getenv("TMPDIR")) { - const char *tmp = getenv("TMP"); - if (!tmp) + if (!(tmp = getenv("TMP"))) tmp = getenv("TEMP"); if (tmp) setenv("TMPDIR", tmp, 1); } + if ((tmp = getenv("TMPDIR"))) { + /* + * Convert all dir separators to forward slashes, + * to help shell commands called from the Git + * executable (by not mistaking the dir separators + * for escape characters). + */ + for (; *tmp; tmp++) + if (*tmp == '\\') + *tmp = '/'; + } + + if (!getenv("TZ") && (tmp = getenv("MSYS2_TZ"))) + setenv("TZ", tmp, 1); + /* simulate TERM to enable auto-color (see color.c) */ if (!getenv("TERM")) setenv("TERM", "cygwin", 1); @@ -2294,26 +2310,8 @@ void mingw_startup() __argv[0] = wcstoutfdup_startup(buffer, _wpgmptr, maxlen); for (i = 1; i < argc; i++) __argv[i] = wcstoutfdup_startup(buffer, wargv[i], maxlen); - for (i = 0; wenv[i]; i++) { + for (i = 0; wenv[i]; i++) environ[i] = wcstoutfdup_startup(buffer, wenv[i], maxlen); - if (!strncasecmp(environ[i], "MSYS2_TZ=", 9)) { - char *to_free = environ[i]; - environ[i] = xstrdup(to_free + 6); - free(to_free); - } - if (!strncasecmp(environ[i], "TMP=", 4)) { - /* - * Convert all dir separators to forward slashes, - * to help shell commands called from the Git - * executable (by not mistaking the dir separators - * for escape characters). - */ - char *p; - for (p = environ[i]; *p; p++) - if (*p == '\\') - *p = '/'; - } - } environ[i] = NULL; free(buffer); From 288e740b82d15dfd5e36535a9b8d9e735e506dfe Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Mon, 6 Apr 2015 21:37:18 +0200 Subject: [PATCH 55/59] mingw: initialize HOME on startup HOME initialization was historically duplicated in many different places, including /etc/profile, launch scripts such as git-bash.vbs and gitk.cmd, and (although slightly broken) in the git-wrapper. Even unrelated projects such as GitExtensions and TortoiseGit need to implement the same logic to be able to call git directly. Initialize HOME in git's own startup code so that we can eventually retire all the duplicate initialization code. Signed-off-by: Karsten Blees --- compat/mingw.c | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/compat/mingw.c b/compat/mingw.c index b36da8d503..24ef6ee0aa 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2239,6 +2239,30 @@ static void setup_windows_environment() /* simulate TERM to enable auto-color (see color.c) */ if (!getenv("TERM")) setenv("TERM", "cygwin", 1); + + /* calculate HOME if not set */ + if (!getenv("HOME")) { + /* + * try $HOMEDRIVE$HOMEPATH - the home share may be a network + * location, thus also check if the path exists (i.e. is not + * disconnected) + */ + if ((tmp = getenv("HOMEDRIVE"))) { + struct strbuf buf = STRBUF_INIT; + strbuf_addstr(&buf, tmp); + if ((tmp = getenv("HOMEPATH"))) { + strbuf_addstr(&buf, tmp); + if (is_directory(buf.buf)) + setenv("HOME", buf.buf, 1); + else + tmp = NULL; /* use $USERPROFILE */ + } + strbuf_release(&buf); + } + /* use $USERPROFILE if the home share is not available */ + if (!tmp && (tmp = getenv("USERPROFILE"))) + setenv("HOME", tmp, 1); + } } /* From a768cfc06a78f19272799bfee374214513b1b519 Mon Sep 17 00:00:00 2001 From: nalla Date: Thu, 16 Apr 2015 11:45:05 +0100 Subject: [PATCH 56/59] mingw: explicitly `fflush` stdout For performance reasons `stdout` is not unbuffered by default. That leads to problems if after printing to `stdout` a read on `stdin` is performed. For that reason interactive commands like `git clean -i` do not function properly anymore if the `stdout` is not flushed by `fflush(stdout)` before trying to read from `stdin`. In the case of `git clean -i` all reads on `stdin` were preceded by a `fflush(stdout)` call. Signed-off-by: nalla --- builtin/clean.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/builtin/clean.c b/builtin/clean.c index 6dcb72e644..78cd1008d8 100644 --- a/builtin/clean.c +++ b/builtin/clean.c @@ -574,6 +574,7 @@ static int *list_and_choose(struct menu_opts *opts, struct menu_stuff *stuff) clean_get_color(CLEAN_COLOR_RESET)); } + fflush(stdout); if (strbuf_getline(&choice, stdin, '\n') != EOF) { strbuf_trim(&choice); } else { @@ -656,6 +657,7 @@ static int filter_by_patterns_cmd(void) clean_print_color(CLEAN_COLOR_PROMPT); printf(_("Input ignore patterns>> ")); clean_print_color(CLEAN_COLOR_RESET); + fflush(stdout); if (strbuf_getline(&confirm, stdin, '\n') != EOF) strbuf_trim(&confirm); else @@ -754,6 +756,7 @@ static int ask_each_cmd(void) qname = quote_path_relative(item->string, NULL, &buf); /* TRANSLATORS: Make sure to keep [y/N] as is */ printf(_("Remove %s [y/N]? "), qname); + fflush(stdout); if (strbuf_getline(&confirm, stdin, '\n') != EOF) { strbuf_trim(&confirm); } else { From 34b63d6842ddd33a0b9beae68f4534626b6cb1c1 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Thu, 19 Mar 2015 16:33:44 +0100 Subject: [PATCH 57/59] mingw: Support `git_terminal_prompt` with more terminals The `git_terminal_prompt()` function expects the terminal window to be attached to a Win32 Console. However, this is not the case with terminal windows other than `cmd.exe`'s, e.g. with MSys2's own `mintty`. Non-cmd terminals such as `mintty` still have to have a Win32 Console to be proper console programs, but have to hide the Win32 Console to be able to provide more flexibility (such as being resizeable not only vertically but also horizontally). By writing to that Win32 Console, `git_terminal_prompt()` manages only to send the prompt to nowhere and to wait for input from a Console to which the user has no access. This commit introduces a function specifically to support `mintty` -- or other terminals that are compatible with MSys2's `/dev/tty` emulation. We use the `TERM` environment variable as an indicator for that: if the value starts with "xterm" (such as `mintty`'s "xterm_256color"), we prefer to let `xterm_prompt()` handle the user interaction. The most prominent user of `git_terminal_prompt()` is certainly `git-remote-https.exe`. It is an interesting use case because both `stdin` and `stdout` are redirected when Git calls said executable, yet it still wants to access the terminal. When running inside a `mintty`, the terminal is not accessible to the `git-remote-https.exe` program, though, because it is a MinGW program and the `mintty` terminal is not backed by a Win32 console. To solve that problem, we simply call out to the shell -- which is an *MSys2* program and can therefore access `/dev/tty`. Helped-by: nalla Signed-off-by: Karsten Blees Signed-off-by: Johannes Schindelin --- compat/terminal.c | 57 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/compat/terminal.c b/compat/terminal.c index 313897d581..a923417d28 100644 --- a/compat/terminal.c +++ b/compat/terminal.c @@ -1,7 +1,10 @@ +#include #include "git-compat-util.h" +#include "run-command.h" #include "compat/terminal.h" #include "sigchain.h" #include "strbuf.h" +#include "cache.h" #if defined(HAVE_DEV_TTY) || defined(GIT_WINDOWS_NATIVE) @@ -91,6 +94,54 @@ static int disable_echo(void) return 0; } +static char *shell_prompt(const char *prompt, int echo) +{ + const char *read_input[] = { + /* Note: call 'bash' explicitly, as 'read -s' is bash-specific */ + "bash", "-c", echo ? + "cat >/dev/tty && read -r line /dev/tty && read -r -s line /dev/tty", + NULL + }; + struct child_process child = CHILD_PROCESS_INIT; + static struct strbuf buffer = STRBUF_INIT; + int prompt_len = strlen(prompt), len = -1, code; + + child.argv = read_input; + child.in = -1; + child.out = -1; + + if (start_command(&child)) + return NULL; + + if (write_in_full(child.in, prompt, prompt_len) != prompt_len) { + error("could not write to prompt script"); + close(child.in); + goto ret; + } + close(child.in); + + strbuf_reset(&buffer); + len = strbuf_read(&buffer, child.out, 1024); + if (len < 0) { + error("could not read from prompt script"); + goto ret; + } + + strbuf_strip_suffix(&buffer, "\n"); + strbuf_strip_suffix(&buffer, "\r"); + +ret: + close(child.out); + code = finish_command(&child); + if (code) { + error("failed to execute prompt script (exit code %d)", code); + return NULL; + } + + return len < 0 ? NULL : buffer.buf; +} + #endif #ifndef FORCE_TEXT @@ -102,6 +153,12 @@ char *git_terminal_prompt(const char *prompt, int echo) static struct strbuf buf = STRBUF_INIT; int r; FILE *input_fh, *output_fh; +#ifdef GIT_WINDOWS_NATIVE + const char *term = getenv("TERM"); + + if (term && starts_with(term, "xterm")) + return shell_prompt(prompt, echo); +#endif input_fh = fopen(INPUT_PATH, "r" FORCE_TEXT); if (!input_fh) From 2f5aee17137ddcd6150f424be08cee7b9f250c79 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sat, 9 May 2015 02:11:48 +0200 Subject: [PATCH 58/59] compat/terminal.c: only use the Windows console if bash 'read -r' fails MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Accessing the Windows console through the special CONIN$ / CONOUT$ devices doesn't work properly for non-ASCII usernames an passwords. It also doesn't work for terminal emulators that hide the native console window (such as mintty), and 'TERM=xterm*' is not necessarily a reliable indicator for such terminals. The new shell_prompt() function, on the other hand, works fine for both MSys1 and MSys2, in native console windows as well as mintty, and properly supports Unicode. It just needs bash on the path (for 'read -s', which is bash-specific). On Windows, try to use the shell to read from the terminal. If that fails with ENOENT (i.e. bash was not found), use CONIN/OUT as fallback. Note: To test this, create a UTF-8 credential file with non-ASCII chars, e.g. in git-bash: 'echo url=http://täst.com > cred.txt'. Then in git-cmd, 'git credential fill --- compat/terminal.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/compat/terminal.c b/compat/terminal.c index a923417d28..8479f6049a 100644 --- a/compat/terminal.c +++ b/compat/terminal.c @@ -110,6 +110,7 @@ static char *shell_prompt(const char *prompt, int echo) child.argv = read_input; child.in = -1; child.out = -1; + child.silent_exec_failure = 1; if (start_command(&child)) return NULL; @@ -153,11 +154,14 @@ char *git_terminal_prompt(const char *prompt, int echo) static struct strbuf buf = STRBUF_INIT; int r; FILE *input_fh, *output_fh; -#ifdef GIT_WINDOWS_NATIVE - const char *term = getenv("TERM"); - if (term && starts_with(term, "xterm")) - return shell_prompt(prompt, echo); +#ifdef GIT_WINDOWS_NATIVE + + /* try shell_prompt first, fall back to CONIN/OUT if bash is missing */ + char *result = shell_prompt(prompt, echo); + if (result || errno != ENOENT) + return result; + #endif input_fh = fopen(INPUT_PATH, "r" FORCE_TEXT); From 5884eec75f7cbd2a5470b3542dca9cf9cd2c518e Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 10 Jun 2015 15:52:22 +0000 Subject: [PATCH 59/59] mingw: do not special-case .exe files anymore Since baaf233 (connect: improve check for plink to reduce false positives, 2015-04-26), t5601 writes out a `plink.exe` for testing that is actually a shell script. So the assumption that the `.exe` extension implies that the file is *not* a shell script is now wrong. The original idea to special-case `.exe` files was probably to help performance, but since we are in a code path that involves spawning a new process (which in and of itself is pretty slow on Windows anyway), we pursue a better idea to improve performance elsewhere: we try to convert scripts into builtins and to reduce the number of spawned processes by adding more internal API calls. Signed-off-by: Johannes Schindelin --- compat/mingw.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 24ef6ee0aa..900afce781 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -836,11 +836,6 @@ static const char *parse_interpreter(const char *cmd) char *p, *opt; int n, fd; - /* don't even try a .exe */ - n = strlen(cmd); - if (n >= 4 && !strcasecmp(cmd+n-4, ".exe")) - return NULL; - fd = open(cmd, O_RDONLY); if (fd < 0) return NULL;