From a7824e696da904ec0e91bc38c4a40ee49f9b014e Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sat, 6 Jul 2013 02:09:35 +0200 Subject: [PATCH 01/38] Win32: make FILETIME conversion functions public Signed-off-by: Karsten Blees --- compat/mingw.c | 16 ---------------- compat/mingw.h | 16 ++++++++++++++++ 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 3fbfda5978..d4ed08e5e1 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -512,22 +512,6 @@ int mingw_chmod(const char *filename, int mode) return _wchmod(wfilename, mode); } -/* - * The unit of FILETIME is 100-nanoseconds since January 1, 1601, UTC. - * Returns the 100-nanoseconds ("hekto nanoseconds") since the epoch. - */ -static inline long long filetime_to_hnsec(const FILETIME *ft) -{ - long long winTime = ((long long)ft->dwHighDateTime << 32) + ft->dwLowDateTime; - /* Windows to Unix Epoch conversion */ - return winTime - 116444736000000000LL; -} - -static inline time_t filetime_to_time_t(const FILETIME *ft) -{ - return (time_t)(filetime_to_hnsec(ft) / 10000000); -} - /** * Verifies that safe_create_leading_directories() would succeed. */ diff --git a/compat/mingw.h b/compat/mingw.h index 034fff9479..5e00c6da25 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -326,6 +326,22 @@ static inline int getrlimit(int resource, struct rlimit *rlp) return 0; } +/* + * The unit of FILETIME is 100-nanoseconds since January 1, 1601, UTC. + * Returns the 100-nanoseconds ("hekto nanoseconds") since the epoch. + */ +static inline long long filetime_to_hnsec(const FILETIME *ft) +{ + long long winTime = ((long long)ft->dwHighDateTime << 32) + ft->dwLowDateTime; + /* Windows to Unix Epoch conversion */ + return winTime - 116444736000000000LL; +} + +static inline time_t filetime_to_time_t(const FILETIME *ft) +{ + return (time_t)(filetime_to_hnsec(ft) / 10000000); +} + /* * Use mingw specific stat()/lstat()/fstat() implementations on Windows. */ From 4f17108d90625c4b201a4a6c3a9aa5a92b2ef6fe Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sun, 8 Sep 2013 14:17:31 +0200 Subject: [PATCH 02/38] Win32: dirent.c: Move opendir down Move opendir down in preparation for the next patch. Signed-off-by: Karsten Blees --- compat/win32/dirent.c | 68 +++++++++++++++++++++---------------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/compat/win32/dirent.c b/compat/win32/dirent.c index 52420ec7d4..2603a0fa39 100644 --- a/compat/win32/dirent.c +++ b/compat/win32/dirent.c @@ -18,40 +18,6 @@ static inline void finddata2dirent(struct dirent *ent, WIN32_FIND_DATAW *fdata) ent->d_type = DT_REG; } -DIR *opendir(const char *name) -{ - wchar_t pattern[MAX_PATH + 2]; /* + 2 for '/' '*' */ - WIN32_FIND_DATAW fdata; - HANDLE h; - int len; - DIR *dir; - - /* convert name to UTF-16 and check length < MAX_PATH */ - if ((len = xutftowcs_path(pattern, name)) < 0) - return NULL; - - /* append optional '/' and wildcard '*' */ - if (len && !is_dir_sep(pattern[len - 1])) - pattern[len++] = '/'; - pattern[len++] = '*'; - pattern[len] = 0; - - /* open find handle */ - h = FindFirstFileW(pattern, &fdata); - if (h == INVALID_HANDLE_VALUE) { - DWORD err = GetLastError(); - errno = (err == ERROR_DIRECTORY) ? ENOTDIR : err_win_to_posix(err); - return NULL; - } - - /* initialize DIR structure and copy first dir entry */ - dir = xmalloc(sizeof(DIR)); - dir->dd_handle = h; - dir->dd_stat = 0; - finddata2dirent(&dir->dd_dir, &fdata); - return dir; -} - struct dirent *readdir(DIR *dir) { if (!dir) { @@ -90,3 +56,37 @@ int closedir(DIR *dir) free(dir); return 0; } + +DIR *opendir(const char *name) +{ + wchar_t pattern[MAX_PATH + 2]; /* + 2 for '/' '*' */ + WIN32_FIND_DATAW fdata; + HANDLE h; + int len; + DIR *dir; + + /* convert name to UTF-16 and check length < MAX_PATH */ + if ((len = xutftowcs_path(pattern, name)) < 0) + return NULL; + + /* append optional '/' and wildcard '*' */ + if (len && !is_dir_sep(pattern[len - 1])) + pattern[len++] = '/'; + pattern[len++] = '*'; + pattern[len] = 0; + + /* open find handle */ + h = FindFirstFileW(pattern, &fdata); + if (h == INVALID_HANDLE_VALUE) { + DWORD err = GetLastError(); + errno = (err == ERROR_DIRECTORY) ? ENOTDIR : err_win_to_posix(err); + return NULL; + } + + /* initialize DIR structure and copy first dir entry */ + dir = xmalloc(sizeof(DIR)); + dir->dd_handle = h; + dir->dd_stat = 0; + finddata2dirent(&dir->dd_dir, &fdata); + return dir; +} From 0b6319a92caad6065181041623987c12081aaf4d Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sun, 8 Sep 2013 14:18:40 +0200 Subject: [PATCH 03/38] Win32: Make the dirent implementation pluggable Emulating the POSIX dirent API on Windows via FindFirstFile/FindNextFile is pretty staightforward, however, most of the information provided in the WIN32_FIND_DATA structure is thrown away in the process. A more sophisticated implementation may cache this data, e.g. for later reuse in calls to lstat. Make the dirent implementation pluggable so that it can be switched at runtime, e.g. based on a config option. Define a base DIR structure with pointers to readdir/closedir that match the opendir implementation (i.e. similar to vtable pointers in OOP). Define readdir/closedir so that they call the function pointers in the DIR structure. This allows to choose the opendir implementation on a call-by-call basis. Move the fixed sized dirent.d_name buffer to the dirent-specific DIR structure, as d_name may be implementation specific (e.g. a caching implementation may just set d_name to point into the cache instead of copying the entire file name string). Signed-off-by: Karsten Blees --- compat/win32/dirent.c | 27 +++++++++++++++++---------- compat/win32/dirent.h | 26 +++++++++++++++++++------- 2 files changed, 36 insertions(+), 17 deletions(-) diff --git a/compat/win32/dirent.c b/compat/win32/dirent.c index 2603a0fa39..6b87042182 100644 --- a/compat/win32/dirent.c +++ b/compat/win32/dirent.c @@ -1,15 +1,19 @@ #include "../../git-compat-util.h" -struct DIR { +typedef struct dirent_DIR { + struct DIR base_dir; /* extend base struct DIR */ struct dirent dd_dir; /* includes d_type */ HANDLE dd_handle; /* FindFirstFile handle */ int dd_stat; /* 0-based index */ -}; + char dd_name[MAX_PATH * 3]; /* file name (* 3 for UTF-8 conversion) */ +} dirent_DIR; + +DIR *(*opendir)(const char *dirname) = dirent_opendir; static inline void finddata2dirent(struct dirent *ent, WIN32_FIND_DATAW *fdata) { - /* convert UTF-16 name to UTF-8 */ - xwcstoutf(ent->d_name, fdata->cFileName, sizeof(ent->d_name)); + /* convert UTF-16 name to UTF-8 (d_name points to dirent_DIR.dd_name) */ + xwcstoutf(ent->d_name, fdata->cFileName, MAX_PATH * 3); /* Set file type, based on WIN32_FIND_DATA */ if (fdata->dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) @@ -18,7 +22,7 @@ static inline void finddata2dirent(struct dirent *ent, WIN32_FIND_DATAW *fdata) ent->d_type = DT_REG; } -struct dirent *readdir(DIR *dir) +static struct dirent *dirent_readdir(dirent_DIR *dir) { if (!dir) { errno = EBADF; /* No set_errno for mingw */ @@ -45,7 +49,7 @@ struct dirent *readdir(DIR *dir) return &dir->dd_dir; } -int closedir(DIR *dir) +static int dirent_closedir(dirent_DIR *dir) { if (!dir) { errno = EBADF; @@ -57,13 +61,13 @@ int closedir(DIR *dir) return 0; } -DIR *opendir(const char *name) +DIR *dirent_opendir(const char *name) { wchar_t pattern[MAX_PATH + 2]; /* + 2 for '/' '*' */ WIN32_FIND_DATAW fdata; HANDLE h; int len; - DIR *dir; + dirent_DIR *dir; /* convert name to UTF-16 and check length < MAX_PATH */ if ((len = xutftowcs_path(pattern, name)) < 0) @@ -84,9 +88,12 @@ DIR *opendir(const char *name) } /* initialize DIR structure and copy first dir entry */ - dir = xmalloc(sizeof(DIR)); + dir = xmalloc(sizeof(dirent_DIR)); + dir->base_dir.preaddir = (struct dirent *(*)(DIR *dir)) dirent_readdir; + dir->base_dir.pclosedir = (int (*)(DIR *dir)) dirent_closedir; + dir->dd_dir.d_name = dir->dd_name; dir->dd_handle = h; dir->dd_stat = 0; finddata2dirent(&dir->dd_dir, &fdata); - return dir; + return (DIR*) dir; } diff --git a/compat/win32/dirent.h b/compat/win32/dirent.h index 058207e4bf..6b3ddee51b 100644 --- a/compat/win32/dirent.h +++ b/compat/win32/dirent.h @@ -1,20 +1,32 @@ #ifndef DIRENT_H #define DIRENT_H -typedef struct DIR DIR; - #define DT_UNKNOWN 0 #define DT_DIR 1 #define DT_REG 2 #define DT_LNK 3 struct dirent { - unsigned char d_type; /* file type to prevent lstat after readdir */ - char d_name[MAX_PATH * 3]; /* file name (* 3 for UTF-8 conversion) */ + unsigned char d_type; /* file type to prevent lstat after readdir */ + char *d_name; /* file name */ }; -DIR *opendir(const char *dirname); -struct dirent *readdir(DIR *dir); -int closedir(DIR *dir); +/* + * Base DIR structure, contains pointers to readdir/closedir implementations so + * that opendir may choose a concrete implementation on a call-by-call basis. + */ +typedef struct DIR { + struct dirent *(*preaddir)(struct DIR *dir); + int (*pclosedir)(struct DIR *dir); +} DIR; + +/* default dirent implementation */ +extern DIR *dirent_opendir(const char *dirname); + +/* current dirent implementation */ +extern DIR *(*opendir)(const char *dirname); + +#define readdir(dir) (dir->preaddir(dir)) +#define closedir(dir) (dir->pclosedir(dir)) #endif /* DIRENT_H */ From 3097f584e40bcc107b6ed9e49272d0a242c18043 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sun, 8 Sep 2013 14:21:30 +0200 Subject: [PATCH 04/38] Win32: make the lstat implementation pluggable Emulating the POSIX lstat API on Windows via GetFileAttributes[Ex] is quite slow. Windows operating system APIs seem to be much better at scanning the status of entire directories than checking single files. A caching implementation may improve performance by bulk-reading entire directories or reusing data obtained via opendir / readdir. Make the lstat implementation pluggable so that it can be switched at runtime, e.g. based on a config option. Signed-off-by: Karsten Blees --- compat/mingw.c | 2 ++ compat/mingw.h | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/compat/mingw.c b/compat/mingw.c index d4ed08e5e1..954009e067 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -651,6 +651,8 @@ static int do_stat_internal(int follow, const char *file_name, struct stat *buf) return do_lstat(follow, alt_name, buf); } +int (*lstat)(const char *file_name, struct stat *buf) = mingw_lstat; + int mingw_lstat(const char *file_name, struct stat *buf) { return do_stat_internal(0, file_name, buf); diff --git a/compat/mingw.h b/compat/mingw.h index 5e00c6da25..e5bd433b2d 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -365,7 +365,7 @@ int mingw_fstat(int fd, struct stat *buf); #ifdef lstat #undef lstat #endif -#define lstat mingw_lstat +extern int (*lstat)(const char *file_name, struct stat *buf); #ifndef _stati64 # define _stati64(x,y) mingw_stat(x,y) From fce640064d5dbf0afa63c43ac8b0365daf3d690b Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sun, 8 Sep 2013 14:23:27 +0200 Subject: [PATCH 05/38] add infrastructure for read-only file system level caches Add a macro to mark code sections that only read from the file system, along with a config option and documentation. This facilitates implementation of relatively simple file system level caches without the need to synchronize with the file system. Enable read-only sections for 'git status' and preload_index. Signed-off-by: Karsten Blees --- Documentation/config.txt | 6 ++++++ builtin/commit.c | 1 + cache.h | 2 ++ config.c | 5 +++++ environment.c | 1 + git-compat-util.h | 15 +++++++++++++++ preload-index.c | 2 ++ 7 files changed, 32 insertions(+) diff --git a/Documentation/config.txt b/Documentation/config.txt index 0bcb6790d6..1e787319df 100644 --- a/Documentation/config.txt +++ b/Documentation/config.txt @@ -760,6 +760,12 @@ relatively high IO latencies. When enabled, Git will do the index comparison to the filesystem data in parallel, allowing overlapping IO's. Defaults to true. +core.fscache:: + Enable additional caching of file system data for some operations. ++ +Git for Windows uses this to bulk-read and cache lstat data of entire +directories (instead of doing lstat file by file). + core.createObject:: You can set this to 'link', in which case a hardlink followed by a delete of the source are used to make sure that object creation diff --git a/builtin/commit.c b/builtin/commit.c index 77e3dc8494..71d521d332 100644 --- a/builtin/commit.c +++ b/builtin/commit.c @@ -1374,6 +1374,7 @@ int cmd_status(int argc, const char **argv, const char *prefix) PATHSPEC_PREFER_FULL, prefix, argv); + enable_fscache(1); read_cache_preload(&s.pathspec); refresh_index(&the_index, REFRESH_QUIET|REFRESH_UNMERGED, &s.pathspec, NULL, NULL); diff --git a/cache.h b/cache.h index b780a91a56..1fa9491be5 100644 --- a/cache.h +++ b/cache.h @@ -710,6 +710,8 @@ enum hide_dotfiles_type { }; extern enum hide_dotfiles_type hide_dotfiles; +extern int core_fscache; + enum branch_track { BRANCH_TRACK_UNSPECIFIED = -1, BRANCH_TRACK_NEVER = 0, diff --git a/config.c b/config.c index 0dfed682b8..91ff686add 100644 --- a/config.c +++ b/config.c @@ -1007,6 +1007,11 @@ static int git_default_core_config(const char *var, const char *value) return 0; } + if (!strcmp(var, "core.fscache")) { + core_fscache = git_config_bool(var, value); + return 0; + } + /* Add other config variables here and to Documentation/config.txt. */ return 0; } diff --git a/environment.c b/environment.c index ca72464a98..d513d78484 100644 --- a/environment.c +++ b/environment.c @@ -65,6 +65,7 @@ int merge_log_config = -1; int precomposed_unicode = -1; /* see probe_utf8_pathname_composition() */ unsigned long pack_size_limit_cfg; enum hide_dotfiles_type hide_dotfiles = HIDE_DOTFILES_DOTGITONLY; +int core_fscache; #ifndef PROTECT_HFS_DEFAULT #define PROTECT_HFS_DEFAULT 0 diff --git a/git-compat-util.h b/git-compat-util.h index db89ba7748..62c27e2412 100644 --- a/git-compat-util.h +++ b/git-compat-util.h @@ -1065,6 +1065,21 @@ struct tm *git_gmtime_r(const time_t *, struct tm *); #define getc_unlocked(fh) getc(fh) #endif +/* + * Enable/disable a read-only cache for file system data on platforms that + * support it. + * + * Implementing a live-cache is complicated and requires special platform + * support (inotify, ReadDirectoryChangesW...). enable_fscache shall be used + * to mark sections of git code that extensively read from the file system + * without modifying anything. Implementations can use this to cache e.g. stat + * data or even file content without the need to synchronize with the file + * system. + */ +#ifndef enable_fscache +#define enable_fscache(x) /* noop */ +#endif + #endif extern int cmd_main(int, const char **); diff --git a/preload-index.c b/preload-index.c index c1fe3a3ef9..c7970d7cb9 100644 --- a/preload-index.c +++ b/preload-index.c @@ -84,6 +84,7 @@ static void preload_index(struct index_state *index, offset = 0; work = DIV_ROUND_UP(index->cache_nr, threads); memset(&data, 0, sizeof(data)); + enable_fscache(1); for (i = 0; i < threads; i++) { struct thread_data *p = data+i; p->index = index; @@ -100,6 +101,7 @@ static void preload_index(struct index_state *index, if (pthread_join(p->pthread, NULL)) die("unable to join threaded lstat"); } + enable_fscache(0); } #endif From e887698e9a695a27d226626b1a68c09aa844eba8 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Tue, 1 Oct 2013 12:51:54 +0200 Subject: [PATCH 06/38] Win32: add a cache below mingw's lstat and dirent implementations Checking the work tree status is quite slow on Windows, due to slow lstat emulation (git calls lstat once for each file in the index). Windows operating system APIs seem to be much better at scanning the status of entire directories than checking single files. Add an lstat implementation that uses a cache for lstat data. Cache misses read the entire parent directory and add it to the cache. Subsequent lstat calls for the same directory are served directly from the cache. Also implement opendir / readdir / closedir so that they create and use directory listings in the cache. The cache doesn't track file system changes and doesn't plug into any modifying file APIs, so it has to be explicitly enabled for git functions that don't modify the working copy. Note: in an earlier version of this patch, the cache was always active and tracked file system changes via ReadDirectoryChangesW. However, this was much more complex and had negative impact on the performance of modifying git commands such as 'git checkout'. Signed-off-by: Karsten Blees --- compat/win32/fscache.c | 458 +++++++++++++++++++++++++++++++++++++++++ compat/win32/fscache.h | 10 + config.mak.uname | 4 +- git-compat-util.h | 2 + 4 files changed, 472 insertions(+), 2 deletions(-) create mode 100644 compat/win32/fscache.c create mode 100644 compat/win32/fscache.h diff --git a/compat/win32/fscache.c b/compat/win32/fscache.c new file mode 100644 index 0000000000..28d22791bd --- /dev/null +++ b/compat/win32/fscache.c @@ -0,0 +1,458 @@ +#include "../../cache.h" +#include "../../hashmap.h" +#include "../win32.h" +#include "fscache.h" + +static int initialized; +static volatile long enabled; +static struct hashmap map; +static CRITICAL_SECTION mutex; + +/* + * An entry in the file system cache. Used for both entire directory listings + * and file entries. + */ +struct fsentry { + struct hashmap_entry ent; + mode_t st_mode; + /* Length of name. */ + unsigned short len; + /* + * Name of the entry. For directory listings: relative path of the + * directory, without trailing '/' (empty for cwd()). For file entries: + * name of the file. Typically points to the end of the structure if + * the fsentry is allocated on the heap (see fsentry_alloc), or to a + * local variable if on the stack (see fsentry_init). + */ + const char *name; + /* Pointer to the directory listing, or NULL for the listing itself. */ + struct fsentry *list; + /* Pointer to the next file entry of the list. */ + struct fsentry *next; + + union { + /* Reference count of the directory listing. */ + volatile long refcnt; + struct { + /* More stat members (only used for file entries). */ + off64_t st_size; + time_t st_atime; + time_t st_mtime; + time_t st_ctime; + }; + }; +}; + +/* + * Compares the paths of two fsentry structures for equality. + */ +static int fsentry_cmp(const struct fsentry *fse1, const struct fsentry *fse2) +{ + int res; + if (fse1 == fse2) + return 0; + + /* compare the list parts first */ + if (fse1->list != fse2->list && (res = fsentry_cmp( + fse1->list ? fse1->list : fse1, + fse2->list ? fse2->list : fse2))) + return res; + + /* if list parts are equal, compare len and name */ + if (fse1->len != fse2->len) + return fse1->len - fse2->len; + return strnicmp(fse1->name, fse2->name, fse1->len); +} + +/* + * Calculates the hash code of an fsentry structure's path. + */ +static unsigned int fsentry_hash(const struct fsentry *fse) +{ + unsigned int hash = fse->list ? fse->list->ent.hash : 0; + return hash ^ memihash(fse->name, fse->len); +} + +/* + * Initialize an fsentry structure for use by fsentry_hash and fsentry_cmp. + */ +static void fsentry_init(struct fsentry *fse, struct fsentry *list, + const char *name, size_t len) +{ + fse->list = list; + fse->name = name; + fse->len = len; + hashmap_entry_init(fse, fsentry_hash(fse)); +} + +/* + * Allocate an fsentry structure on the heap. + */ +static struct fsentry *fsentry_alloc(struct fsentry *list, const char *name, + size_t len) +{ + /* overallocate fsentry and copy the name to the end */ + struct fsentry *fse = xmalloc(sizeof(struct fsentry) + len + 1); + char *nm = ((char*) fse) + sizeof(struct fsentry); + memcpy(nm, name, len); + nm[len] = 0; + /* init the rest of the structure */ + fsentry_init(fse, list, nm, len); + fse->next = NULL; + fse->refcnt = 1; + return fse; +} + +/* + * Add a reference to an fsentry. + */ +inline static void fsentry_addref(struct fsentry *fse) +{ + if (fse->list) + fse = fse->list; + + InterlockedIncrement(&(fse->refcnt)); +} + +/* + * Release the reference to an fsentry, frees the memory if its the last ref. + */ +static void fsentry_release(struct fsentry *fse) +{ + if (fse->list) + fse = fse->list; + + if (InterlockedDecrement(&(fse->refcnt))) + return; + + while (fse) { + struct fsentry *next = fse->next; + free(fse); + fse = next; + } +} + +/* + * Allocate and initialize an fsentry from a WIN32_FIND_DATA structure. + */ +static struct fsentry *fseentry_create_entry(struct fsentry *list, + const WIN32_FIND_DATAW *fdata) +{ + char buf[MAX_PATH * 3]; + int len; + struct fsentry *fse; + len = xwcstoutf(buf, fdata->cFileName, ARRAY_SIZE(buf)); + + fse = fsentry_alloc(list, buf, len); + + fse->st_mode = file_attr_to_st_mode(fdata->dwFileAttributes); + fse->st_size = (((off64_t) (fdata->nFileSizeHigh)) << 32) + | fdata->nFileSizeLow; + fse->st_atime = filetime_to_time_t(&(fdata->ftLastAccessTime)); + fse->st_mtime = filetime_to_time_t(&(fdata->ftLastWriteTime)); + fse->st_ctime = filetime_to_time_t(&(fdata->ftCreationTime)); + + return fse; +} + +/* + * Create an fsentry-based directory listing (similar to opendir / readdir). + * Dir should not contain trailing '/'. Use an empty string for the current + * directory (not "."!). + */ +static struct fsentry *fsentry_create_list(const struct fsentry *dir) +{ + wchar_t pattern[MAX_PATH + 2]; /* + 2 for '/' '*' */ + WIN32_FIND_DATAW fdata; + HANDLE h; + int wlen; + struct fsentry *list, **phead; + DWORD err; + + /* convert name to UTF-16 and check length < MAX_PATH */ + if ((wlen = xutftowcsn(pattern, dir->name, MAX_PATH, dir->len)) < 0) { + if (errno == ERANGE) + errno = ENAMETOOLONG; + return NULL; + } + + /* append optional '/' and wildcard '*' */ + if (wlen) + pattern[wlen++] = '/'; + pattern[wlen++] = '*'; + pattern[wlen] = 0; + + /* open find handle */ + h = FindFirstFileW(pattern, &fdata); + if (h == INVALID_HANDLE_VALUE) { + err = GetLastError(); + errno = (err == ERROR_DIRECTORY) ? ENOTDIR : err_win_to_posix(err); + return NULL; + } + + /* allocate object to hold directory listing */ + list = fsentry_alloc(NULL, dir->name, dir->len); + + /* walk directory and build linked list of fsentry structures */ + phead = &list->next; + do { + *phead = fseentry_create_entry(list, &fdata); + phead = &(*phead)->next; + } while (FindNextFileW(h, &fdata)); + + /* remember result of last FindNextFile, then close find handle */ + err = GetLastError(); + FindClose(h); + + /* return the list if we've got all the files */ + if (err == ERROR_NO_MORE_FILES) + return list; + + /* otherwise free the list and return error */ + fsentry_release(list); + errno = err_win_to_posix(err); + return NULL; +} + +/* + * Adds a directory listing to the cache. + */ +static void fscache_add(struct fsentry *fse) +{ + if (fse->list) + fse = fse->list; + + for (; fse; fse = fse->next) + hashmap_add(&map, fse); +} + +/* + * Removes a directory listing from the cache. + */ +static void fscache_remove(struct fsentry *fse) +{ + if (fse->list) + fse = fse->list; + + for (; fse; fse = fse->next) + hashmap_remove(&map, fse, NULL); +} + +/* + * Clears the cache. + */ +static void fscache_clear(void) +{ + struct hashmap_iter iter; + struct fsentry *fse; + while ((fse = hashmap_iter_first(&map, &iter))) { + fscache_remove(fse); + fsentry_release(fse); + } +} + +/* + * Checks if the cache is enabled for the given path. + */ +static inline int fscache_enabled(const char *path) +{ + return enabled > 0 && !is_absolute_path(path); +} + +/* + * Looks up or creates a cache entry for the specified key. + */ +static struct fsentry *fscache_get(struct fsentry *key) +{ + struct fsentry *fse; + + EnterCriticalSection(&mutex); + /* check if entry is in cache */ + fse = hashmap_get(&map, key, NULL); + if (fse) { + fsentry_addref(fse); + LeaveCriticalSection(&mutex); + return fse; + } + /* if looking for a file, check if directory listing is in cache */ + if (!fse && key->list) { + fse = hashmap_get(&map, key->list, NULL); + if (fse) { + LeaveCriticalSection(&mutex); + /* dir entry without file entry -> file doesn't exist */ + errno = ENOENT; + return NULL; + } + } + + /* create the directory listing (outside mutex!) */ + LeaveCriticalSection(&mutex); + fse = fsentry_create_list(key->list ? key->list : key); + if (!fse) + return NULL; + + EnterCriticalSection(&mutex); + /* add directory listing if it hasn't been added by some other thread */ + if (!hashmap_get(&map, key, NULL)) + fscache_add(fse); + + /* lookup file entry if requested (fse already points to directory) */ + if (key->list) + fse = hashmap_get(&map, key, NULL); + + /* return entry or ENOENT */ + if (fse) + fsentry_addref(fse); + else + errno = ENOENT; + + LeaveCriticalSection(&mutex); + return fse; +} + +/* + * Enables or disables the cache. Note that the cache is read-only, changes to + * the working directory are NOT reflected in the cache while enabled. + */ +int fscache_enable(int enable) +{ + int result; + + if (!initialized) { + /* allow the cache to be disabled entirely */ + if (!core_fscache) + return 0; + + InitializeCriticalSection(&mutex); + hashmap_init(&map, (hashmap_cmp_fn) fsentry_cmp, 0); + initialized = 1; + } + + result = enable ? InterlockedIncrement(&enabled) + : InterlockedDecrement(&enabled); + + if (enable && result == 1) { + /* redirect opendir and lstat to the fscache implementations */ + opendir = fscache_opendir; + lstat = fscache_lstat; + } else if (!enable && !result) { + /* reset opendir and lstat to the original implementations */ + opendir = dirent_opendir; + lstat = mingw_lstat; + EnterCriticalSection(&mutex); + fscache_clear(); + LeaveCriticalSection(&mutex); + } + return result; +} + +/* + * Lstat replacement, uses the cache if enabled, otherwise redirects to + * mingw_lstat. + */ +int fscache_lstat(const char *filename, struct stat *st) +{ + int dirlen, base, len; + struct fsentry key[2], *fse; + + if (!fscache_enabled(filename)) + return mingw_lstat(filename, st); + + /* split filename into path + name */ + len = strlen(filename); + if (len && is_dir_sep(filename[len - 1])) + len--; + base = len; + while (base && !is_dir_sep(filename[base - 1])) + base--; + dirlen = base ? base - 1 : 0; + + /* lookup entry for path + name in cache */ + fsentry_init(key, NULL, filename, dirlen); + fsentry_init(key + 1, key, filename + base, len - base); + fse = fscache_get(key + 1); + if (!fse) + return -1; + + /* copy stat data */ + st->st_ino = 0; + st->st_gid = 0; + st->st_uid = 0; + st->st_dev = 0; + st->st_rdev = 0; + st->st_nlink = 1; + st->st_mode = fse->st_mode; + st->st_size = fse->st_size; + st->st_atime = fse->st_atime; + st->st_mtime = fse->st_mtime; + st->st_ctime = fse->st_ctime; + + /* don't forget to release fsentry */ + fsentry_release(fse); + return 0; +} + +typedef struct fscache_DIR { + struct DIR base_dir; /* extend base struct DIR */ + struct fsentry *pfsentry; + struct dirent dirent; +} fscache_DIR; + +/* + * Readdir replacement. + */ +static struct dirent *fscache_readdir(DIR *base_dir) +{ + fscache_DIR *dir = (fscache_DIR*) base_dir; + struct fsentry *next = dir->pfsentry->next; + if (!next) + return NULL; + dir->pfsentry = next; + dir->dirent.d_type = S_ISDIR(next->st_mode) ? DT_DIR : DT_REG; + dir->dirent.d_name = (char*) next->name; + return &(dir->dirent); +} + +/* + * Closedir replacement. + */ +static int fscache_closedir(DIR *base_dir) +{ + fscache_DIR *dir = (fscache_DIR*) base_dir; + fsentry_release(dir->pfsentry); + free(dir); + return 0; +} + +/* + * Opendir replacement, uses a directory listing from the cache if enabled, + * otherwise calls original dirent implementation. + */ +DIR *fscache_opendir(const char *dirname) +{ + struct fsentry key, *list; + fscache_DIR *dir; + int len; + + if (!fscache_enabled(dirname)) + return dirent_opendir(dirname); + + /* prepare name (strip trailing '/', replace '.') */ + len = strlen(dirname); + if ((len == 1 && dirname[0] == '.') || + (len && is_dir_sep(dirname[len - 1]))) + len--; + + /* get directory listing from cache */ + fsentry_init(&key, NULL, dirname, len); + list = fscache_get(&key); + if (!list) + return NULL; + + /* alloc and return DIR structure */ + dir = (fscache_DIR*) xmalloc(sizeof(fscache_DIR)); + dir->base_dir.preaddir = fscache_readdir; + dir->base_dir.pclosedir = fscache_closedir; + dir->pfsentry = list; + return (DIR*) dir; +} diff --git a/compat/win32/fscache.h b/compat/win32/fscache.h new file mode 100644 index 0000000000..ed518b422d --- /dev/null +++ b/compat/win32/fscache.h @@ -0,0 +1,10 @@ +#ifndef FSCACHE_H +#define FSCACHE_H + +int fscache_enable(int enable); +#define enable_fscache(x) fscache_enable(x) + +DIR *fscache_opendir(const char *dir); +int fscache_lstat(const char *file_name, struct stat *buf); + +#endif diff --git a/config.mak.uname b/config.mak.uname index b232908f8c..c517b378cf 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -382,7 +382,7 @@ ifeq ($(uname_S),Windows) BASIC_CFLAGS = -nologo -I. -I../zlib -Icompat/vcbuild -Icompat/vcbuild/include -DWIN32 -D_CONSOLE -DHAVE_STRING_H -D_CRT_SECURE_NO_WARNINGS -D_CRT_NONSTDC_NO_DEPRECATE COMPAT_OBJS = compat/msvc.o compat/winansi.o \ compat/win32/pthread.o compat/win32/syslog.o \ - compat/win32/dirent.o + compat/win32/dirent.o compat/win32/fscache.o COMPAT_CFLAGS = -D__USE_MINGW_ACCESS -DNOGDI -DHAVE_STRING_H -Icompat -Icompat/regex -Icompat/win32 -DSTRIP_EXTENSION=\".exe\" BASIC_LDFLAGS = -IGNORE:4217 -IGNORE:4049 -NOLOGO -SUBSYSTEM:CONSOLE EXTLIBS = user32.lib advapi32.lib shell32.lib wininet.lib ws2_32.lib invalidcontinue.obj @@ -528,7 +528,7 @@ ifneq (,$(findstring MINGW,$(uname_S))) COMPAT_CFLAGS += -DSTRIP_EXTENSION=\".exe\" COMPAT_OBJS += compat/mingw.o compat/winansi.o \ compat/win32/pthread.o compat/win32/syslog.o \ - compat/win32/dirent.o + compat/win32/dirent.o compat/win32/fscache.o BASIC_CFLAGS += -DPROTECT_NTFS_DEFAULT=1 EXTLIBS += -lws2_32 GITLIBS += git.res diff --git a/git-compat-util.h b/git-compat-util.h index 62c27e2412..e46b37fc25 100644 --- a/git-compat-util.h +++ b/git-compat-util.h @@ -192,8 +192,10 @@ #if defined(__MINGW32__) /* pull in Windows compatibility stuff */ #include "compat/mingw.h" +#include "compat/win32/fscache.h" #elif defined(_MSC_VER) #include "compat/msvc.h" +#include "compat/win32/fscache.h" #else #include #include From 8a17793959ef8959be54a5f7b2bfd91270f5898a Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Tue, 24 Jun 2014 13:22:35 +0200 Subject: [PATCH 07/38] fscache: load directories only once If multiple threads access a directory that is not yet in the cache, the directory will be loaded by each thread. Only one of the results is added to the cache, all others are leaked. This wastes performance and memory. On cache miss, add a future object to the cache to indicate that the directory is currently being loaded. Subsequent threads register themselves with the future object and wait. When the first thread has loaded the directory, it replaces the future object with the result and notifies waiting threads. Signed-off-by: Karsten Blees --- compat/win32/fscache.c | 67 +++++++++++++++++++++++++++++++++++------- 1 file changed, 57 insertions(+), 10 deletions(-) diff --git a/compat/win32/fscache.c b/compat/win32/fscache.c index 28d22791bd..cbfcb18474 100644 --- a/compat/win32/fscache.c +++ b/compat/win32/fscache.c @@ -33,6 +33,8 @@ struct fsentry { union { /* Reference count of the directory listing. */ volatile long refcnt; + /* Handle to wait on the loading thread. */ + HANDLE hwait; struct { /* More stat members (only used for file entries). */ off64_t st_size; @@ -259,16 +261,43 @@ static inline int fscache_enabled(const char *path) return enabled > 0 && !is_absolute_path(path); } +/* + * Looks up a cache entry, waits if its being loaded by another thread. + * The mutex must be owned by the calling thread. + */ +static struct fsentry *fscache_get_wait(struct fsentry *key) +{ + struct fsentry *fse = hashmap_get(&map, key, NULL); + + /* return if its a 'real' entry (future entries have refcnt == 0) */ + if (!fse || fse->list || fse->refcnt) + return fse; + + /* create an event and link our key to the future entry */ + key->hwait = CreateEvent(NULL, TRUE, FALSE, NULL); + key->next = fse->next; + fse->next = key; + + /* wait for the loading thread to signal us */ + LeaveCriticalSection(&mutex); + WaitForSingleObject(key->hwait, INFINITE); + CloseHandle(key->hwait); + EnterCriticalSection(&mutex); + + /* repeat cache lookup */ + return hashmap_get(&map, key, NULL); +} + /* * Looks up or creates a cache entry for the specified key. */ static struct fsentry *fscache_get(struct fsentry *key) { - struct fsentry *fse; + struct fsentry *fse, *future, *waiter; EnterCriticalSection(&mutex); /* check if entry is in cache */ - fse = hashmap_get(&map, key, NULL); + fse = fscache_get_wait(key); if (fse) { fsentry_addref(fse); LeaveCriticalSection(&mutex); @@ -276,7 +305,7 @@ static struct fsentry *fscache_get(struct fsentry *key) } /* if looking for a file, check if directory listing is in cache */ if (!fse && key->list) { - fse = hashmap_get(&map, key->list, NULL); + fse = fscache_get_wait(key->list); if (fse) { LeaveCriticalSection(&mutex); /* dir entry without file entry -> file doesn't exist */ @@ -285,16 +314,34 @@ static struct fsentry *fscache_get(struct fsentry *key) } } + /* add future entry to indicate that we're loading it */ + future = key->list ? key->list : key; + future->next = NULL; + future->refcnt = 0; + hashmap_add(&map, future); + /* create the directory listing (outside mutex!) */ LeaveCriticalSection(&mutex); - fse = fsentry_create_list(key->list ? key->list : key); - if (!fse) - return NULL; - + fse = fsentry_create_list(future); EnterCriticalSection(&mutex); - /* add directory listing if it hasn't been added by some other thread */ - if (!hashmap_get(&map, key, NULL)) - fscache_add(fse); + + /* remove future entry and signal waiting threads */ + hashmap_remove(&map, future, NULL); + waiter = future->next; + while (waiter) { + HANDLE h = waiter->hwait; + waiter = waiter->next; + SetEvent(h); + } + + /* leave on error (errno set by fsentry_create_list) */ + if (!fse) { + LeaveCriticalSection(&mutex); + return NULL; + } + + /* add directory listing to the cache */ + fscache_add(fse); /* lookup file entry if requested (fse already points to directory) */ if (key->list) From cd0250051a7a10567ad92807c41b50834f7ff86f Mon Sep 17 00:00:00 2001 From: Doug Kelly Date: Wed, 8 Jan 2014 20:28:15 -0600 Subject: [PATCH 08/38] Add a test demonstrating a problem with long submodule paths [jes: adusted test number to avoid conflicts, fixed non-portable use of the 'export' statement] Signed-off-by: Johannes Schindelin --- t/t7413-submodule-long-path.sh | 101 +++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100755 t/t7413-submodule-long-path.sh diff --git a/t/t7413-submodule-long-path.sh b/t/t7413-submodule-long-path.sh new file mode 100755 index 0000000000..6bb641cd4b --- /dev/null +++ b/t/t7413-submodule-long-path.sh @@ -0,0 +1,101 @@ +#!/bin/sh +# +# Copyright (c) 2013 Doug Kelly +# + +test_description='Test submodules with a path near PATH_MAX + +This test verifies that "git submodule" initialization, update and clones work, including with recursive submodules and paths approaching PATH_MAX (260 characters on Windows) +' + +TEST_NO_CREATE_REPO=1 +. ./test-lib.sh + +longpath="" +for (( i=0; i<4; i++ )); do + longpath="0123456789abcdefghijklmnopqrstuvwxyz$longpath" +done +# Pick a substring maximum of 90 characters +# This should be good, since we'll add on a lot for temp directories +longpath=${longpath:0:90}; export longpath + +test_expect_failure 'submodule with a long path' ' + git init --bare remote && + test_create_repo bundle1 && + ( + cd bundle1 && + test_commit "shoot" && + git rev-parse --verify HEAD >../expect + ) && + mkdir home && + ( + cd home && + git clone ../remote test && + cd test && + git submodule add ../bundle1 $longpath && + test_commit "sogood" && + ( + cd $longpath && + git rev-parse --verify HEAD >actual && + test_cmp ../../../expect actual + ) && + git push origin master + ) && + mkdir home2 && + ( + cd home2 && + git clone ../remote test && + cd test && + git checkout master && + git submodule update --init && + ( + cd $longpath && + git rev-parse --verify HEAD >actual && + test_cmp ../../../expect actual + ) + ) +' + +test_expect_failure 'recursive submodule with a long path' ' + git init --bare super && + test_create_repo child && + ( + cd child && + test_commit "shoot" && + git rev-parse --verify HEAD >../expect + ) && + test_create_repo parent && + ( + cd parent && + git submodule add ../child $longpath && + test_commit "aim" + ) && + mkdir home3 && + ( + cd home3 && + git clone ../super test && + cd test && + git submodule add ../parent foo && + git submodule update --init --recursive + test_commit "sogood" && + ( + cd foo/$longpath && + git rev-parse --verify HEAD >actual && + test_cmp ../../../../expect actual + ) && + git push origin master + ) && + mkdir home4 && + ( + cd home4 && + git clone ../super test --recursive && + ( + cd test/foo/$longpath && + git rev-parse --verify HEAD >actual && + test_cmp ../../../../expect actual + ) + ) +' +unset longpath + +test_done From 5d8c439882715b418e16ec900a00875ef52bcc7e Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 28 Jul 2015 21:07:41 +0200 Subject: [PATCH 09/38] Win32: support long paths Windows paths are typically limited to MAX_PATH = 260 characters, even though the underlying NTFS file system supports paths up to 32,767 chars. This limitation is also evident in Windows Explorer, cmd.exe and many other applications (including IDEs). Particularly annoying is that most Windows APIs return bogus error codes if a relative path only barely exceeds MAX_PATH in conjunction with the current directory, e.g. ERROR_PATH_NOT_FOUND / ENOENT instead of the infinitely more helpful ERROR_FILENAME_EXCED_RANGE / ENAMETOOLONG. Many Windows wide char APIs support longer than MAX_PATH paths through the file namespace prefix ('\\?\' or '\\?\UNC\') followed by an absolute path. Notable exceptions include functions dealing with executables and the current directory (CreateProcess, LoadLibrary, Get/SetCurrentDirectory) as well as the entire shell API (ShellExecute, SHGetSpecialFolderPath...). Introduce a handle_long_path function to check the length of a specified path properly (and fail with ENAMETOOLONG), and to optionally expand long paths using the '\\?\' file namespace prefix. Short paths will not be modified, so we don't need to worry about device names (NUL, CON, AUX). Contrary to MSDN docs, the GetFullPathNameW function doesn't seem to be limited to MAX_PATH (at least not on Win7), so we can use it to do the heavy lifting of the conversion (translate '/' to '\', eliminate '.' and '..', and make an absolute path). Add long path error checking to xutftowcs_path for APIs with hard MAX_PATH limit. Add a new MAX_LONG_PATH constant and xutftowcs_long_path function for APIs that support long paths. While improved error checking is always active, long paths support must be explicitly enabled via 'core.longpaths' option. This is to prevent end users to shoot themselves in the foot by checking out files that Windows Explorer, cmd/bash or their favorite IDE cannot handle. Test suite: Test the case is when the full pathname length of a dir is close to 260 (MAX_PATH). Bug report and an original reproducer by Andrey Rogozhnikov: https://github.com/msysgit/git/pull/122#issuecomment-43604199 [jes: adjusted test number to avoid conflicts] Thanks-to: Martin W. Kirst Thanks-to: Doug Kelly Signed-off-by: Karsten Blees Original-test-by: Andrey Rogozhnikov Signed-off-by: Stepan Kasal Signed-off-by: Johannes Schindelin --- Documentation/config.txt | 7 ++ cache.h | 2 + compat/mingw.c | 132 ++++++++++++++++++++++++++------- compat/mingw.h | 77 +++++++++++++++++-- compat/win32/dirent.c | 14 ++-- compat/win32/fscache.c | 17 +++-- config.c | 5 ++ environment.c | 1 + t/t2027-checkout-long-paths.sh | 99 +++++++++++++++++++++++++ t/t7413-submodule-long-path.sh | 24 +++--- 10 files changed, 323 insertions(+), 55 deletions(-) create mode 100755 t/t2027-checkout-long-paths.sh diff --git a/Documentation/config.txt b/Documentation/config.txt index 1e787319df..ef7ad3d039 100644 --- a/Documentation/config.txt +++ b/Documentation/config.txt @@ -766,6 +766,13 @@ core.fscache:: Git for Windows uses this to bulk-read and cache lstat data of entire directories (instead of doing lstat file by file). +core.longpaths:: + Enable long path (> 260) support for builtin commands in Git for + Windows. This is disabled by default, as long paths are not supported + by Windows Explorer, cmd.exe and the Git for Windows tool chain + (msys, bash, tcl, perl...). Only enable this if you know what you're + doing and are prepared to live with a few quirks. + core.createObject:: You can set this to 'link', in which case a hardlink followed by a delete of the source are used to make sure that object creation diff --git a/cache.h b/cache.h index 1fa9491be5..aee050e730 100644 --- a/cache.h +++ b/cache.h @@ -712,6 +712,8 @@ extern enum hide_dotfiles_type hide_dotfiles; extern int core_fscache; +extern int core_long_paths; + enum branch_track { BRANCH_TRACK_UNSPECIFIED = -1, BRANCH_TRACK_NEVER = 0, diff --git a/compat/mingw.c b/compat/mingw.c index 954009e067..51f5f8aef6 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -205,8 +205,8 @@ static int ask_yes_no_if_possible(const char *format, ...) int mingw_unlink(const char *pathname) { int ret, tries = 0; - wchar_t wpathname[MAX_PATH]; - if (xutftowcs_path(wpathname, pathname) < 0) + wchar_t wpathname[MAX_LONG_PATH]; + if (xutftowcs_long_path(wpathname, pathname) < 0) return -1; /* read-only files cannot be removed */ @@ -235,7 +235,7 @@ static int is_dir_empty(const wchar_t *wpath) { WIN32_FIND_DATAW findbuf; HANDLE handle; - wchar_t wbuf[MAX_PATH + 2]; + wchar_t wbuf[MAX_LONG_PATH + 2]; wcscpy(wbuf, wpath); wcscat(wbuf, L"\\*"); handle = FindFirstFileW(wbuf, &findbuf); @@ -256,8 +256,8 @@ static int is_dir_empty(const wchar_t *wpath) int mingw_rmdir(const char *pathname) { int ret, tries = 0; - wchar_t wpathname[MAX_PATH]; - if (xutftowcs_path(wpathname, pathname) < 0) + wchar_t wpathname[MAX_LONG_PATH]; + if (xutftowcs_long_path(wpathname, pathname) < 0) return -1; while ((ret = _wrmdir(wpathname)) == -1 && tries < ARRAY_SIZE(delay)) { @@ -332,9 +332,12 @@ static int set_hidden_flag(const wchar_t *path, int set) int mingw_mkdir(const char *path, int mode) { int ret; - wchar_t wpath[MAX_PATH]; - if (xutftowcs_path(wpath, path) < 0) + wchar_t wpath[MAX_LONG_PATH]; + /* CreateDirectoryW path limit is 248 (MAX_PATH - 8.3 file name) */ + if (xutftowcs_path_ex(wpath, path, MAX_LONG_PATH, -1, 248, + core_long_paths) < 0) return -1; + ret = _wmkdir(wpath); if (!ret && needs_hiding(path)) return set_hidden_flag(wpath, 1); @@ -346,7 +349,7 @@ int mingw_open (const char *filename, int oflags, ...) va_list args; unsigned mode; int fd; - wchar_t wfilename[MAX_PATH]; + wchar_t wfilename[MAX_LONG_PATH]; va_start(args, oflags); mode = va_arg(args, int); @@ -355,7 +358,7 @@ int mingw_open (const char *filename, int oflags, ...) if (filename && !strcmp(filename, "/dev/null")) filename = "nul"; - if (xutftowcs_path(wfilename, filename) < 0) + if (xutftowcs_long_path(wfilename, filename) < 0) return -1; fd = _wopen(wfilename, oflags, mode); @@ -412,10 +415,10 @@ FILE *mingw_fopen (const char *filename, const char *otype) { int hide = needs_hiding(filename); FILE *file; - wchar_t wfilename[MAX_PATH], wotype[4]; + wchar_t wfilename[MAX_LONG_PATH], wotype[4]; if (filename && !strcmp(filename, "/dev/null")) filename = "nul"; - if (xutftowcs_path(wfilename, filename) < 0 || + if (xutftowcs_long_path(wfilename, filename) < 0 || xutftowcs(wotype, otype, ARRAY_SIZE(wotype)) < 0) return NULL; if (hide && !access(filename, F_OK) && set_hidden_flag(wfilename, 0)) { @@ -432,10 +435,10 @@ FILE *mingw_freopen (const char *filename, const char *otype, FILE *stream) { int hide = needs_hiding(filename); FILE *file; - wchar_t wfilename[MAX_PATH], wotype[4]; + wchar_t wfilename[MAX_LONG_PATH], wotype[4]; if (filename && !strcmp(filename, "/dev/null")) filename = "nul"; - if (xutftowcs_path(wfilename, filename) < 0 || + if (xutftowcs_long_path(wfilename, filename) < 0 || xutftowcs(wotype, otype, ARRAY_SIZE(wotype)) < 0) return NULL; if (hide && !access(filename, F_OK) && set_hidden_flag(wfilename, 0)) { @@ -489,25 +492,32 @@ ssize_t mingw_write(int fd, const void *buf, size_t len) int mingw_access(const char *filename, int mode) { - wchar_t wfilename[MAX_PATH]; - if (xutftowcs_path(wfilename, filename) < 0) + wchar_t wfilename[MAX_LONG_PATH]; + if (xutftowcs_long_path(wfilename, filename) < 0) return -1; /* X_OK is not supported by the MSVCRT version */ return _waccess(wfilename, mode & ~X_OK); } +/* cached length of current directory for handle_long_path */ +static int current_directory_len = 0; + int mingw_chdir(const char *dirname) { + int result; wchar_t wdirname[MAX_PATH]; + /* SetCurrentDirectoryW doesn't support long paths */ if (xutftowcs_path(wdirname, dirname) < 0) return -1; - return _wchdir(wdirname); + result = _wchdir(wdirname); + current_directory_len = GetCurrentDirectoryW(0, NULL); + return result; } int mingw_chmod(const char *filename, int mode) { - wchar_t wfilename[MAX_PATH]; - if (xutftowcs_path(wfilename, filename) < 0) + wchar_t wfilename[MAX_LONG_PATH]; + if (xutftowcs_long_path(wfilename, filename) < 0) return -1; return _wchmod(wfilename, mode); } @@ -555,8 +565,8 @@ static int has_valid_directory_prefix(wchar_t *wfilename) static int do_lstat(int follow, const char *file_name, struct stat *buf) { WIN32_FILE_ATTRIBUTE_DATA fdata; - wchar_t wfilename[MAX_PATH]; - if (xutftowcs_path(wfilename, file_name) < 0) + wchar_t wfilename[MAX_LONG_PATH]; + if (xutftowcs_long_path(wfilename, file_name) < 0) return -1; if (GetFileAttributesExW(wfilename, GetFileExInfoStandard, &fdata)) { @@ -705,8 +715,8 @@ int mingw_utime (const char *file_name, const struct utimbuf *times) FILETIME mft, aft; int fh, rc; DWORD attrs; - wchar_t wfilename[MAX_PATH]; - if (xutftowcs_path(wfilename, file_name) < 0) + wchar_t wfilename[MAX_LONG_PATH]; + if (xutftowcs_long_path(wfilename, file_name) < 0) return -1; /* must have write permission */ @@ -754,6 +764,7 @@ unsigned int sleep (unsigned int seconds) char *mingw_mktemp(char *template) { wchar_t wtemplate[MAX_PATH]; + /* we need to return the path, thus no long paths here! */ if (xutftowcs_path(wtemplate, template) < 0) return NULL; if (!_wmktemp(wtemplate)) @@ -1104,6 +1115,7 @@ static pid_t mingw_spawnve_fd(const char *cmd, const char **argv, char **deltaen si.hStdOutput = winansi_get_osfhandle(fhout); si.hStdError = winansi_get_osfhandle(fherr); + /* executables and the current directory don't support long paths */ if (xutftowcs_path(wcmd, cmd) < 0) return -1; if (dir && xutftowcs_path(wdir, dir) < 0) @@ -1679,8 +1691,9 @@ int mingw_rename(const char *pold, const char *pnew) { DWORD attrs, gle; int tries = 0; - wchar_t wpold[MAX_PATH], wpnew[MAX_PATH]; - if (xutftowcs_path(wpold, pold) < 0 || xutftowcs_path(wpnew, pnew) < 0) + wchar_t wpold[MAX_LONG_PATH], wpnew[MAX_LONG_PATH]; + if (xutftowcs_long_path(wpold, pold) < 0 || + xutftowcs_long_path(wpnew, pnew) < 0) return -1; /* @@ -1920,9 +1933,9 @@ int link(const char *oldpath, const char *newpath) { typedef BOOL (WINAPI *T)(LPCWSTR, LPCWSTR, LPSECURITY_ATTRIBUTES); static T create_hard_link = NULL; - wchar_t woldpath[MAX_PATH], wnewpath[MAX_PATH]; - if (xutftowcs_path(woldpath, oldpath) < 0 || - xutftowcs_path(wnewpath, newpath) < 0) + wchar_t woldpath[MAX_LONG_PATH], wnewpath[MAX_LONG_PATH]; + if (xutftowcs_long_path(woldpath, oldpath) < 0 || + xutftowcs_long_path(wnewpath, newpath) < 0) return -1; if (!create_hard_link) { @@ -2135,6 +2148,68 @@ static void setup_windows_environment(void) setenv("TERM", "cygwin", 1); } +int handle_long_path(wchar_t *path, int len, int max_path, int expand) +{ + int result; + wchar_t buf[MAX_LONG_PATH]; + + /* + * we don't need special handling if path is relative to the current + * directory, and current directory + path don't exceed the desired + * max_path limit. This should cover > 99 % of cases with minimal + * performance impact (git almost always uses relative paths). + */ + if ((len < 2 || (!is_dir_sep(path[0]) && path[1] != ':')) && + (current_directory_len + len < max_path)) + return len; + + /* + * handle everything else: + * - absolute paths: "C:\dir\file" + * - absolute UNC paths: "\\server\share\dir\file" + * - absolute paths on current drive: "\dir\file" + * - relative paths on other drive: "X:file" + * - prefixed paths: "\\?\...", "\\.\..." + */ + + /* convert to absolute path using GetFullPathNameW */ + result = GetFullPathNameW(path, MAX_LONG_PATH, buf, NULL); + if (!result) { + errno = err_win_to_posix(GetLastError()); + return -1; + } + + /* + * return absolute path if it fits within max_path (even if + * "cwd + path" doesn't due to '..' components) + */ + if (result < max_path) { + wcscpy(path, buf); + return result; + } + + /* error out if we shouldn't expand the path or buf is too small */ + if (!expand || result >= MAX_LONG_PATH - 6) { + errno = ENAMETOOLONG; + return -1; + } + + /* prefix full path with "\\?\" or "\\?\UNC\" */ + if (buf[0] == '\\') { + /* ...unless already prefixed */ + if (buf[1] == '\\' && (buf[2] == '?' || buf[2] == '.')) + return len; + + wcscpy(path, L"\\\\?\\UNC\\"); + wcscpy(path + 8, buf + 2); + return result + 6; + } else { + wcscpy(path, L"\\\\?\\"); + wcscpy(path + 4, buf); + return result + 4; + } +} + /* * Disable MSVCRT command line wildcard expansion (__getmainargs called from * mingw startup code, see init.c in mingw runtime). @@ -2226,6 +2301,9 @@ void mingw_startup(void) /* initialize Unicode console */ winansi_init(); + + /* init length of current directory for handle_long_path */ + current_directory_len = GetCurrentDirectoryW(0, NULL); } int uname(struct utsname *buf) diff --git a/compat/mingw.h b/compat/mingw.h index e5bd433b2d..90caac8a6c 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -437,6 +437,42 @@ int mingw_offset_1st_component(const char *path); #include #endif +/** + * Max length of long paths (exceeding MAX_PATH). The actual maximum supported + * by NTFS is 32,767 (* sizeof(wchar_t)), but we choose an arbitrary smaller + * value to limit required stack memory. + */ +#define MAX_LONG_PATH 4096 + +/** + * Handles paths that would exceed the MAX_PATH limit of Windows Unicode APIs. + * + * With expand == false, the function checks for over-long paths and fails + * with ENAMETOOLONG. The path parameter is not modified, except if cwd + path + * exceeds max_path, but the resulting absolute path doesn't (e.g. due to + * eliminating '..' components). The path parameter must point to a buffer + * of max_path wide characters. + * + * With expand == true, an over-long path is automatically converted in place + * to an absolute path prefixed with '\\?\', and the new length is returned. + * The path parameter must point to a buffer of MAX_LONG_PATH wide characters. + * + * Parameters: + * path: path to check and / or convert + * len: size of path on input (number of wide chars without \0) + * max_path: max short path length to check (usually MAX_PATH = 260, but just + * 248 for CreateDirectoryW) + * expand: false to only check the length, true to expand the path to a + * '\\?\'-prefixed absolute path + * + * Return: + * length of the resulting path, or -1 on failure + * + * Errors: + * ENAMETOOLONG if path is too long + */ +int handle_long_path(wchar_t *path, int len, int max_path, int expand); + /** * Converts UTF-8 encoded string to UTF-16LE. * @@ -494,17 +530,48 @@ static inline int xutftowcs(wchar_t *wcs, const char *utf, size_t wcslen) return xutftowcsn(wcs, utf, wcslen, -1); } +/** + * Simplified file system specific wrapper of xutftowcsn and handle_long_path. + * Converts ERANGE to ENAMETOOLONG. If expand is true, wcs must be at least + * MAX_LONG_PATH wide chars (see handle_long_path). + */ +static inline int xutftowcs_path_ex(wchar_t *wcs, const char *utf, + size_t wcslen, int utflen, int max_path, int expand) +{ + int result = xutftowcsn(wcs, utf, wcslen, utflen); + if (result < 0 && errno == ERANGE) + errno = ENAMETOOLONG; + if (result >= 0) + result = handle_long_path(wcs, result, max_path, expand); + return result; +} + /** * Simplified file system specific variant of xutftowcsn, assumes output * buffer size is MAX_PATH wide chars and input string is \0-terminated, - * fails with ENAMETOOLONG if input string is too long. + * fails with ENAMETOOLONG if input string is too long. Typically used for + * Windows APIs that don't support long paths, e.g. SetCurrentDirectory, + * LoadLibrary, CreateProcess... */ static inline int xutftowcs_path(wchar_t *wcs, const char *utf) { - int result = xutftowcsn(wcs, utf, MAX_PATH, -1); - if (result < 0 && errno == ERANGE) - errno = ENAMETOOLONG; - return result; + return xutftowcs_path_ex(wcs, utf, MAX_PATH, -1, MAX_PATH, 0); +} + +/* need to re-declare that here as mingw.h is included before cache.h */ +extern int core_long_paths; + +/** + * Simplified file system specific variant of xutftowcsn for Windows APIs + * that support long paths via '\\?\'-prefix, assumes output buffer size is + * MAX_LONG_PATH wide chars, fails with ENAMETOOLONG if input string is too + * long. The 'core.longpaths' git-config option controls whether the path + * is only checked or expanded to a long path. + */ +static inline int xutftowcs_long_path(wchar_t *wcs, const char *utf) +{ + return xutftowcs_path_ex(wcs, utf, MAX_LONG_PATH, -1, MAX_PATH, + core_long_paths); } /** diff --git a/compat/win32/dirent.c b/compat/win32/dirent.c index 6b87042182..b3bd8d7af7 100644 --- a/compat/win32/dirent.c +++ b/compat/win32/dirent.c @@ -63,19 +63,23 @@ static int dirent_closedir(dirent_DIR *dir) DIR *dirent_opendir(const char *name) { - wchar_t pattern[MAX_PATH + 2]; /* + 2 for '/' '*' */ + wchar_t pattern[MAX_LONG_PATH + 2]; /* + 2 for "\*" */ WIN32_FIND_DATAW fdata; HANDLE h; int len; dirent_DIR *dir; - /* convert name to UTF-16 and check length < MAX_PATH */ - if ((len = xutftowcs_path(pattern, name)) < 0) + /* convert name to UTF-16 and check length */ + if ((len = xutftowcs_path_ex(pattern, name, MAX_LONG_PATH, -1, + MAX_PATH - 2, core_long_paths)) < 0) return NULL; - /* append optional '/' and wildcard '*' */ + /* + * append optional '\' and wildcard '*'. Note: we need to use '\' as + * Windows doesn't translate '/' to '\' for "\\?\"-prefixed paths. + */ if (len && !is_dir_sep(pattern[len - 1])) - pattern[len++] = '/'; + pattern[len++] = '\\'; pattern[len++] = '*'; pattern[len] = 0; diff --git a/compat/win32/fscache.c b/compat/win32/fscache.c index cbfcb18474..63d522f49a 100644 --- a/compat/win32/fscache.c +++ b/compat/win32/fscache.c @@ -164,23 +164,24 @@ static struct fsentry *fseentry_create_entry(struct fsentry *list, */ static struct fsentry *fsentry_create_list(const struct fsentry *dir) { - wchar_t pattern[MAX_PATH + 2]; /* + 2 for '/' '*' */ + wchar_t pattern[MAX_LONG_PATH + 2]; /* + 2 for "\*" */ WIN32_FIND_DATAW fdata; HANDLE h; int wlen; struct fsentry *list, **phead; DWORD err; - /* convert name to UTF-16 and check length < MAX_PATH */ - if ((wlen = xutftowcsn(pattern, dir->name, MAX_PATH, dir->len)) < 0) { - if (errno == ERANGE) - errno = ENAMETOOLONG; + /* convert name to UTF-16 and check length */ + if ((wlen = xutftowcs_path_ex(pattern, dir->name, MAX_LONG_PATH, + dir->len, MAX_PATH - 2, core_long_paths)) < 0) return NULL; - } - /* append optional '/' and wildcard '*' */ + /* + * append optional '\' and wildcard '*'. Note: we need to use '\' as + * Windows doesn't translate '/' to '\' for "\\?\"-prefixed paths. + */ if (wlen) - pattern[wlen++] = '/'; + pattern[wlen++] = '\\'; pattern[wlen++] = '*'; pattern[wlen] = 0; diff --git a/config.c b/config.c index 91ff686add..c910b06837 100644 --- a/config.c +++ b/config.c @@ -1012,6 +1012,11 @@ static int git_default_core_config(const char *var, const char *value) return 0; } + if (!strcmp(var, "core.longpaths")) { + core_long_paths = git_config_bool(var, value); + return 0; + } + /* Add other config variables here and to Documentation/config.txt. */ return 0; } diff --git a/environment.c b/environment.c index d513d78484..6911f85a7d 100644 --- a/environment.c +++ b/environment.c @@ -66,6 +66,7 @@ int precomposed_unicode = -1; /* see probe_utf8_pathname_composition() */ unsigned long pack_size_limit_cfg; enum hide_dotfiles_type hide_dotfiles = HIDE_DOTFILES_DOTGITONLY; int core_fscache; +int core_long_paths; #ifndef PROTECT_HFS_DEFAULT #define PROTECT_HFS_DEFAULT 0 diff --git a/t/t2027-checkout-long-paths.sh b/t/t2027-checkout-long-paths.sh new file mode 100755 index 0000000000..e65a38da4d --- /dev/null +++ b/t/t2027-checkout-long-paths.sh @@ -0,0 +1,99 @@ +#!/bin/sh + +test_description='checkout long paths on Windows + +Ensures that Git for Windows can deal with long paths (>260) enabled via core.longpaths' + +. ./test-lib.sh + +if test_have_prereq !MINGW +then + skip_all='skipping MINGW specific long paths test' + test_done +fi + +test_expect_success setup ' + p=longpathxx && # -> 10 + p=$p$p$p$p$p && # -> 50 + p=$p$p$p$p$p && # -> 250 + + path=${p}/longtestfile && # -> 263 (MAX_PATH = 260) + + blob=$(echo foobar | git hash-object -w --stdin) && + + printf "100644 %s 0\t%s\n" "$blob" "$path" | + git update-index --add --index-info && + git commit -m initial -q +' + +test_expect_success 'checkout of long paths without core.longpaths fails' ' + git config core.longpaths false && + test_must_fail git checkout -f 2>error && + grep -q "Filename too long" error && + test_path_is_missing longpa~1/longtestfile +' + +test_expect_success 'checkout of long paths with core.longpaths works' ' + git config core.longpaths true && + git checkout -f && + test_path_is_file longpa~1/longtestfile +' + +test_expect_success 'update of long paths' ' + echo frotz >> longpa~1/longtestfile && + echo $path > expect && + git ls-files -m > actual && + test_cmp expect actual && + git add $path && + git commit -m second && + git grep "frotz" HEAD -- $path +' + +test_expect_success cleanup ' + # bash cannot delete the trash dir if it contains a long path + # lets help cleaning up (unless in debug mode) + test ! -z "$debug" || rm -rf longpa~1 +' + +# check that the template used in the test won't be too long: +abspath="$(pwd -W)"/testdir +test ${#abspath} -gt 230 || +test_set_prereq SHORTABSPATH + +test_expect_success SHORTABSPATH 'clean up path close to MAX_PATH' ' + p=/123456789abcdef/123456789abcdef/123456789abcdef/123456789abc/ef && + p=y$p$p$p$p && + subdir="x$(echo "$p" | tail -c $((253 - ${#abspath})) - )" && + # Now, $abspath/$subdir has exactly 254 characters, and is inside CWD + p2="$abspath/$subdir" && + test 254 = ${#p2} && + + # Be careful to overcome path limitations of the MSys tools and split + # the $subdir into two parts. ($subdir2 has to contain 16 chars and a + # slash somewhere following; that is why we asked for abspath <= 230 and + # why we placed a slash near the end of the $subdir template.) + subdir2=${subdir#????????????????*/} && + subdir1=testdir/${subdir%/$subdir2} && + mkdir -p "$subdir1" && + i=0 && + # The most important case is when absolute path is 258 characters long, + # and that will be when i == 4. + while test $i -le 7 + do + mkdir -p $subdir2 && + touch $subdir2/one-file && + mv ${subdir2%%/*} "$subdir1/" && + subdir2=z${subdir2} && + i=$(($i+1)) || + exit 1 + done && + + # now check that git is able to clear the tree: + (cd testdir && + git init && + git config core.longpaths yes && + git clean -fdx) && + test ! -d "$subdir1" +' + +test_done diff --git a/t/t7413-submodule-long-path.sh b/t/t7413-submodule-long-path.sh index 6bb641cd4b..59b3d347db 100755 --- a/t/t7413-submodule-long-path.sh +++ b/t/t7413-submodule-long-path.sh @@ -11,15 +11,20 @@ This test verifies that "git submodule" initialization, update and clones work, TEST_NO_CREATE_REPO=1 . ./test-lib.sh -longpath="" -for (( i=0; i<4; i++ )); do - longpath="0123456789abcdefghijklmnopqrstuvwxyz$longpath" -done -# Pick a substring maximum of 90 characters -# This should be good, since we'll add on a lot for temp directories -longpath=${longpath:0:90}; export longpath +# cloning a submodule calls is_git_directory("$path/../.git/modules/$path"), +# which effectively limits the maximum length to PATH_MAX / 2 minus some +# overhead; start with 3 * 36 = 108 chars (test 2 fails if >= 110) +longpath36=0123456789abcdefghijklmnopqrstuvwxyz +longpath180=$longpath36$longpath36$longpath36$longpath36$longpath36 -test_expect_failure 'submodule with a long path' ' +# the git database must fit within PATH_MAX, which limits the submodule name +# to PATH_MAX - len(pwd) - ~90 (= len("/objects//") + 40-byte sha1 + some +# overhead from the test case) +pwd=$(pwd) +pwdlen=$(echo "$pwd" | wc -c) +longpath=$(echo $longpath180 | cut -c 1-$((170-$pwdlen))) + +test_expect_success 'submodule with a long path' ' git init --bare remote && test_create_repo bundle1 && ( @@ -56,7 +61,7 @@ test_expect_failure 'submodule with a long path' ' ) ' -test_expect_failure 'recursive submodule with a long path' ' +test_expect_success 'recursive submodule with a long path' ' git init --bare super && test_create_repo child && ( @@ -96,6 +101,5 @@ test_expect_failure 'recursive submodule with a long path' ' ) ) ' -unset longpath test_done From b9f3808a823b64efa62c70bc45d2066c7e0e76c3 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sun, 30 Aug 2015 18:49:05 +0200 Subject: [PATCH 10/38] Win32: support long paths Windows paths are typically limited to MAX_PATH = 260 characters, even though the underlying NTFS file system supports paths up to 32,767 chars. This limitation is also evident in Windows Explorer, cmd.exe and many other applications (including IDEs). Particularly annoying is that most Windows APIs return bogus error codes if a relative path only barely exceeds MAX_PATH in conjunction with the current directory, e.g. ERROR_PATH_NOT_FOUND / ENOENT instead of the infinitely more helpful ERROR_FILENAME_EXCED_RANGE / ENAMETOOLONG. Many Windows wide char APIs support longer than MAX_PATH paths through the file namespace prefix ('\\?\' or '\\?\UNC\') followed by an absolute path. Notable exceptions include functions dealing with executables and the current directory (CreateProcess, LoadLibrary, Get/SetCurrentDirectory) as well as the entire shell API (ShellExecute, SHGetSpecialFolderPath...). Introduce a handle_long_path function to check the length of a specified path properly (and fail with ENAMETOOLONG), and to optionally expand long paths using the '\\?\' file namespace prefix. Short paths will not be modified, so we don't need to worry about device names (NUL, CON, AUX). Contrary to MSDN docs, the GetFullPathNameW function doesn't seem to be limited to MAX_PATH (at least not on Win7), so we can use it to do the heavy lifting of the conversion (translate '/' to '\', eliminate '.' and '..', and make an absolute path). Add long path error checking to xutftowcs_path for APIs with hard MAX_PATH limit. Add a new MAX_LONG_PATH constant and xutftowcs_long_path function for APIs that support long paths. While improved error checking is always active, long paths support must be explicitly enabled via 'core.longpaths' option. This is to prevent end users to shoot themselves in the foot by checking out files that Windows Explorer, cmd/bash or their favorite IDE cannot handle. Test suite: Test the case is when the full pathname length of a dir is close to 260 (MAX_PATH). Bug report and an original reproducer by Andrey Rogozhnikov: https://github.com/msysgit/git/pull/122#issuecomment-43604199 Note that the test cannot rely on the presence of short names, as they are not enabled by default except on the system drive. [jes: adjusted test number to avoid conflicts, reinstated && chain, adjusted test to work without short names] Thanks-to: Martin W. Kirst Thanks-to: Doug Kelly Signed-off-by: Karsten Blees Original-test-by: Andrey Rogozhnikov Signed-off-by: Stepan Kasal Signed-off-by: Johannes Schindelin --- compat/mingw.c | 5 ++--- ...t-long-paths.sh => t2029-checkout-long-paths.sh} | 13 +++++++++---- 2 files changed, 11 insertions(+), 7 deletions(-) rename t/{t2027-checkout-long-paths.sh => t2029-checkout-long-paths.sh} (93%) diff --git a/compat/mingw.c b/compat/mingw.c index 51f5f8aef6..958903e432 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -505,9 +505,8 @@ static int current_directory_len = 0; int mingw_chdir(const char *dirname) { int result; - wchar_t wdirname[MAX_PATH]; - /* SetCurrentDirectoryW doesn't support long paths */ - if (xutftowcs_path(wdirname, dirname) < 0) + wchar_t wdirname[MAX_LONG_PATH]; + if (xutftowcs_long_path(wdirname, dirname) < 0) return -1; result = _wchdir(wdirname); current_directory_len = GetCurrentDirectoryW(0, NULL); diff --git a/t/t2027-checkout-long-paths.sh b/t/t2029-checkout-long-paths.sh similarity index 93% rename from t/t2027-checkout-long-paths.sh rename to t/t2029-checkout-long-paths.sh index e65a38da4d..18e5eabe15 100755 --- a/t/t2027-checkout-long-paths.sh +++ b/t/t2029-checkout-long-paths.sh @@ -30,17 +30,19 @@ test_expect_success 'checkout of long paths without core.longpaths fails' ' git config core.longpaths false && test_must_fail git checkout -f 2>error && grep -q "Filename too long" error && - test_path_is_missing longpa~1/longtestfile + test ! -d longpa* ' test_expect_success 'checkout of long paths with core.longpaths works' ' git config core.longpaths true && git checkout -f && - test_path_is_file longpa~1/longtestfile + (cd longpa* && + test_path_is_file longtestfile) ' test_expect_success 'update of long paths' ' - echo frotz >> longpa~1/longtestfile && + (cd longpa* && + echo frotz >> longtestfile) && echo $path > expect && git ls-files -m > actual && test_cmp expect actual && @@ -52,7 +54,10 @@ test_expect_success 'update of long paths' ' test_expect_success cleanup ' # bash cannot delete the trash dir if it contains a long path # lets help cleaning up (unless in debug mode) - test ! -z "$debug" || rm -rf longpa~1 + if test -z "$debug" + then + rm -rf longpa~1 + fi ' # check that the template used in the test won't be too long: From 2f6de043af47c9aec1f609f05139b9fa310e97ed Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sat, 5 Jul 2014 00:00:36 +0200 Subject: [PATCH 11/38] Win32: fix 'lstat("dir/")' with long paths Use a suffciently large buffer to strip the trailing slash. Signed-off-by: Karsten Blees --- compat/mingw.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 958903e432..4a1b4a51eb 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -636,7 +636,7 @@ static int do_lstat(int follow, const char *file_name, struct stat *buf) static int do_stat_internal(int follow, const char *file_name, struct stat *buf) { int namelen; - char alt_name[PATH_MAX]; + char alt_name[MAX_LONG_PATH]; if (!do_lstat(follow, file_name, buf)) return 0; @@ -652,7 +652,7 @@ static int do_stat_internal(int follow, const char *file_name, struct stat *buf) return -1; while (namelen && file_name[namelen-1] == '/') --namelen; - if (!namelen || namelen >= PATH_MAX) + if (!namelen || namelen >= MAX_LONG_PATH) return -1; memcpy(alt_name, file_name, namelen); From a36968d268200c3180ce8ff0269b96f2f197844f Mon Sep 17 00:00:00 2001 From: nalla Date: Wed, 15 Apr 2015 07:54:39 +0100 Subject: [PATCH 12/38] asciidoctor: Fix user-manual to be built by `asciidoctor` The `user-manual.txt` ist designed as a `book` but the `Makefile` wants to build it as an `article`. This seems to be a problem when building the documentation with `asciidoctor`. Furthermore the parts *Git Glossary* and *Apendix B* had no subsections which is not allowed when building with `asciidoctor`. So lets add a *dummy* section. Signed-off-by: nalla --- Documentation/Makefile | 2 +- Documentation/user-manual.txt | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/Documentation/Makefile b/Documentation/Makefile index b43d66eae6..a9fb497b83 100644 --- a/Documentation/Makefile +++ b/Documentation/Makefile @@ -337,7 +337,7 @@ manpage-base-url.xsl: manpage-base-url.xsl.in user-manual.xml: user-manual.txt user-manual.conf $(QUIET_ASCIIDOC)$(RM) $@+ $@ && \ - $(TXT_TO_XML) -d article -o $@+ $< && \ + $(TXT_TO_XML) -d book -o $@+ $< && \ mv $@+ $@ technical/api-index.txt: technical/api-index-skel.txt \ diff --git a/Documentation/user-manual.txt b/Documentation/user-manual.txt index 5e07454572..bc29298678 100644 --- a/Documentation/user-manual.txt +++ b/Documentation/user-manual.txt @@ -4395,6 +4395,10 @@ itself! Git Glossary ============ +[[git-explained]] +Git explained +------------- + include::glossary-content.txt[] [[git-quick-start]] @@ -4636,6 +4640,10 @@ $ git gc Appendix B: Notes and todo list for this manual =============================================== +[[todo-list]] +Todo list +--------- + This is a work in progress. The basic requirements: From 56c28ba5238b5daea2a5e2d5fd815c91afe26e26 Mon Sep 17 00:00:00 2001 From: nalla Date: Wed, 15 Apr 2015 08:01:07 +0100 Subject: [PATCH 13/38] asciidoctor: Fix `giteveryday.txt` to be built with asciidoctor. When building the `doc` with `asciidoctor`, `asciidoctor` complains about a nested code block in a callout list. This is a really dirty solution to restore the callout list to function properly. There is a minimal visual sideeffect; the *immitated* codeblock has no overall greyish background. Instead the individual lines have it. Note: When building this patch with `asciidoc` the background is totally gone but the font is still monospaced. Signed-off-by: nalla --- Documentation/giteveryday.txt | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/Documentation/giteveryday.txt b/Documentation/giteveryday.txt index 35473ad02f..bd57ea2c1a 100644 --- a/Documentation/giteveryday.txt +++ b/Documentation/giteveryday.txt @@ -308,18 +308,15 @@ master or exposed as a part of a stable branch. <10> create a signed tag. <11> make sure master was not accidentally rewound beyond that already pushed out. `ko` shorthand points at the Git maintainer's -repository at kernel.org, and looks like this: -+ ------------- -(in .git/config) -[remote "ko"] - url = kernel.org:/pub/scm/git/git.git - fetch = refs/heads/*:refs/remotes/ko/* - push = refs/heads/master - push = refs/heads/next - push = +refs/heads/pu - push = refs/heads/maint ------------- +repository at kernel.org, and looks like this: + +++(in .git/config) + +{startsb}remote "ko"{endsb} + +{nbsp}{nbsp}{nbsp}{nbsp}url = kernel.org:/pub/scm/git/git.git + +{nbsp}{nbsp}{nbsp}{nbsp}fetch = refs/heads/\*:refs/remotes/ko/* + +{nbsp}{nbsp}{nbsp}{nbsp}push = refs/heads/master + +{nbsp}{nbsp}{nbsp}{nbsp}push = refs/heads/next + +{nbsp}{nbsp}{nbsp}{nbsp}push = +refs/heads/pu + +{nbsp}{nbsp}{nbsp}{nbsp}push = refs/heads/maint++ + <12> In the output from `git show-branch`, `master` should have everything `ko/master` has, and `next` should have From 818957ea79f1b58d7a6c16314f3f63168f4d26ef Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 14 Apr 2016 16:10:59 +0200 Subject: [PATCH 14/38] mingw: include the full version information in the resources This fixes https://github.com/git-for-windows/git/issues/723 Signed-off-by: Johannes Schindelin --- Makefile | 9 +++++++-- git.rc | 4 ++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index d96ecb7141..a1509422d6 100644 --- a/Makefile +++ b/Makefile @@ -1805,9 +1805,14 @@ $(SCRIPT_LIB) : % : %.sh GIT-SCRIPT-DEFINES $(QUIET_GEN)$(cmd_munge_script) && \ mv $@+ $@ +ifeq (,$(findstring .windows.,$(GIT_VERSION))) +RC_VERSION_DEFS := $(join -DMAJOR= -DMINOR= -DMICRO=, $(wordlist 1,3,$(subst -, ,$(subst ., ,$(subst .windows., ,$(GIT_VERSION)))))) -DPATCHLEVEL=0 +else +RC_VERSION_DEFS := $(join -DMAJOR= -DMINOR= -DMICRO= -DPATCHLEVEL=, $(wordlist 1,4,$(subst -, ,$(subst ., ,$(subst .windows., ,$(GIT_VERSION)))))) +endif + git.res: git.rc GIT-VERSION-FILE - $(QUIET_RC)$(RC) \ - $(join -DMAJOR= -DMINOR=, $(wordlist 1,2,$(subst -, ,$(subst ., ,$(GIT_VERSION))))) \ + $(QUIET_RC)$(RC) $(RC_VERSION_DEFS) \ -DGIT_VERSION="\\\"$(GIT_VERSION)\\\"" $< -o $@ # This makes sure we depend on the NO_PERL setting itself. diff --git a/git.rc b/git.rc index 33aafb786c..49002e0d54 100644 --- a/git.rc +++ b/git.rc @@ -1,6 +1,6 @@ 1 VERSIONINFO -FILEVERSION MAJOR,MINOR,0,0 -PRODUCTVERSION MAJOR,MINOR,0,0 +FILEVERSION MAJOR,MINOR,MICRO,PATCHLEVEL +PRODUCTVERSION MAJOR,MINOR,MICRO,PATCHLEVEL BEGIN BLOCK "StringFileInfo" BEGIN From fd56094429331be394edcb6b35ff2ce88ef95f2f Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 11 Dec 2015 06:59:13 +0100 Subject: [PATCH 15/38] mingw: handle absolute paths in expand_user_path() On Windows, an absolute POSIX path needs to be turned into a Windows one. Signed-off-by: Johannes Schindelin --- path.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/path.c b/path.c index fe3c4d96c6..44ceb95746 100644 --- a/path.c +++ b/path.c @@ -6,6 +6,7 @@ #include "string-list.h" #include "dir.h" #include "worktree.h" +#include "exec_cmd.h" static int get_st_mode_bits(const char *path, int *mode) { @@ -618,6 +619,10 @@ char *expand_user_path(const char *path) if (path == NULL) goto return_null; +#ifdef __MINGW32__ + if (path[0] == '/') + return system_path(path + 1); +#endif if (path[0] == '~') { const char *first_slash = strchrnul(path, '/'); const char *username = path + 1; From b41eafe767297361f56a2a955d8364d6403dcf14 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 11 Dec 2015 07:00:48 +0100 Subject: [PATCH 16/38] http: also treat config options sslCert and sslKey as paths This is a companion patch to bf9acba (http: treat config options sslCAPath and sslCAInfo as paths, 2015-11-23). Signed-off-by: Johannes Schindelin --- http.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/http.c b/http.c index cd40b012f8..cc6f7ca11b 100644 --- a/http.c +++ b/http.c @@ -242,10 +242,10 @@ static int http_options(const char *var, const char *value, void *cb) if (!strcmp("http.sslversion", var)) return git_config_string(&ssl_version, var, value); if (!strcmp("http.sslcert", var)) - return git_config_string(&ssl_cert, var, value); + return git_config_pathname(&ssl_cert, var, value); #if LIBCURL_VERSION_NUM >= 0x070903 if (!strcmp("http.sslkey", var)) - return git_config_string(&ssl_key, var, value); + return git_config_pathname(&ssl_key, var, value); #endif #if LIBCURL_VERSION_NUM >= 0x070908 if (!strcmp("http.sslcapath", var)) From 6929007c70fc88a5d5c75d34e733a81cd5a718fd Mon Sep 17 00:00:00 2001 From: Sebastian Schuberth Date: Thu, 29 Aug 2013 13:09:27 +0200 Subject: [PATCH 17/38] MinGW: Use MakeMaker to build the Perl libraries This way the libraries get properly installed into the "site_perl" directory and we just have to move them out of the "mingw" directory. Signed-off-by: Sebastian Schuberth --- config.mak.uname | 1 - 1 file changed, 1 deletion(-) diff --git a/config.mak.uname b/config.mak.uname index c517b378cf..359e152b60 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -508,7 +508,6 @@ ifneq (,$(findstring MINGW,$(uname_S))) NO_MKDTEMP = YesPlease NO_MKSTEMPS = YesPlease NO_SVN_TESTS = YesPlease - NO_PERL_MAKEMAKER = YesPlease RUNTIME_PREFIX = YesPlease NO_ST_BLOCKS_IN_STRUCT_STAT = YesPlease NO_NSEC = YesPlease From c42526eb50dafb580ad64d6c57f2c811a5272022 Mon Sep 17 00:00:00 2001 From: Sebastian Schuberth Date: Wed, 4 Sep 2013 18:18:49 +0200 Subject: [PATCH 18/38] Makefile: Set htmldir to match the default HTML docs location under MSYS Signed-off-by: Sebastian Schuberth --- config.mak.uname | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.mak.uname b/config.mak.uname index 359e152b60..01be19b69c 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -537,7 +537,7 @@ ifneq (,$(findstring MINGW,$(uname_S))) X = .exe SPARSE_FLAGS = -Wno-one-bit-signed-bitfield ifneq (,$(wildcard ../THIS_IS_MSYSGIT)) - htmldir = doc/git/html/ + htmldir = share/doc/git/$(firstword $(subst -, ,$(GIT_VERSION)))/html prefix = INSTALL = /bin/install EXTLIBS += /mingw/lib/libz.a From 8875d4391dfc8e63059c2b91bb5d02641e87d174 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 20 Feb 2015 09:52:07 +0000 Subject: [PATCH 19/38] Help debugging with MSys2 by optionally executing bash with strace MSys2's strace facility is very useful for debugging... With this patch, the bash will be executed through strace if the environment variable GIT_STRACE_COMMANDS is set, which comes in real handy when investigating issues in the test suite. Signed-off-by: Johannes Schindelin --- compat/mingw.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/compat/mingw.c b/compat/mingw.c index 4a1b4a51eb..ffc8f4cfa6 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -1137,6 +1137,16 @@ static pid_t mingw_spawnve_fd(const char *cmd, const char **argv, char **deltaen free(quoted); } + if (getenv("GIT_STRACE_COMMANDS")) { + char **path = get_path_split(); + cmd = path_lookup("strace.exe", path, 1); + if (!cmd) + return error("strace not found!"); + if (xutftowcs_path(wcmd, cmd) < 0) + return -1; + strbuf_insert(&args, 0, "strace ", 7); + } + ALLOC_ARRAY(wargs, st_add(st_mult(2, args.len), 1)); xutftowcs(wargs, args.buf, 2 * args.len + 1); strbuf_release(&args); From 3c17cc3dd37906933bd877b0282fc4a0a01ad1ff Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 16 Feb 2015 14:06:59 +0100 Subject: [PATCH 20/38] Build Python stuff with MSys2 Signed-off-by: Johannes Schindelin --- config.mak.uname | 1 + 1 file changed, 1 insertion(+) diff --git a/config.mak.uname b/config.mak.uname index 01be19b69c..c3ffc7c17f 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -571,6 +571,7 @@ else USE_LIBPCRE= YesPlease NO_CURL = USE_NED_ALLOCATOR = YesPlease + NO_PYTHON = else COMPAT_CFLAGS += -D__USE_MINGW_ANSI_STDIO NO_CURL = YesPlease From c208fa333c2bbbd63ed4bf7736533f6d02eadd1b Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sun, 15 Feb 2015 09:56:35 +0000 Subject: [PATCH 21/38] Error out when mingw_startup() *and* NO_UNSETENV are active The unsetenv code has no idea to update our environ_size, therefore causing segmentation faults when environment variables are removed without compat/mingw.c's knowing (MinGW's optimized lookup would try to strcmp() against NULL in such a case). Signed-off-by: Johannes Schindelin --- compat/mingw.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/compat/mingw.h b/compat/mingw.h index 90caac8a6c..7166679779 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -257,6 +257,10 @@ char *mingw_mktemp(char *template); char *mingw_getcwd(char *pointer, int len); #define getcwd mingw_getcwd +#ifdef NO_UNSETENV +#error "NO_UNSETENV is incompatible with the MinGW startup code!" +#endif + char *mingw_getenv(const char *name); #define getenv mingw_getenv int mingw_putenv(const char *namevalue); From a47c2bf89c3993ee104ed296d561194bf563b1b0 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 20 Feb 2015 14:21:04 +0000 Subject: [PATCH 22/38] Tests: optionally skip redirecting stdin/stdout/stderr There is a really useful debugging technique developed by Sverre Rabbelier that inserts "bash &&" somewhere in the test scripts, letting the developer interact at given points with the current state. Another debugging technique, used a lot by this here coder, is to run certain executables via gdb by guarding a "gdb -args" call in bin-wrappers/git. Both techniques were disabled by 781f76b1(test-lib: redirect stdin of tests). Let's reinstate the ability to run an interactive shell by making the redirection optional: setting the TEST_NO_REDIRECT environment variable will skip the redirection. Signed-off-by: Johannes Schindelin --- t/test-lib.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/t/test-lib.sh b/t/test-lib.sh index d731d66e36..c3e721afa3 100644 --- a/t/test-lib.sh +++ b/t/test-lib.sh @@ -571,6 +571,20 @@ test_eval_ () { # # The test itself is run with stderr put back to &4 (so either to # /dev/null, or to the original stderr if --verbose was used). + if test -n "$TEST_NO_REDIRECT" + then + test_eval_inner_ "$@" + test_eval_ret_=$? + if test "$trace" = t + then + set +x + if test "$test_eval_ret_" != 0 + then + say_color error >&4 "error: last command exited with \$?=$test_eval_ret_" + fi + fi + return $test_eval_ret_ + fi { test_eval_inner_ "$@" &3 2>&4 test_eval_ret_=$? From 4cfa2f3f2abbf07a4da5712e2597be160da232c7 Mon Sep 17 00:00:00 2001 From: Cesar Eduardo Barros Date: Mon, 9 Mar 2015 08:51:38 +0100 Subject: [PATCH 23/38] mingw: Embed a manifest to trick UAC into Doing The Right Thing On Windows >= Vista, not having an application manifest with a requestedExecutionLevel can cause several kinds of confusing behavior. The first and more obvious behavior is "Installer Detection", where Windows sometimes decides (by looking at things like the file name and even sequences of bytes within the executable) that an executable is an installer and should run elevated (causing the well-known popup dialog to appear). In Git's context, subcommands such as "git patch-id" or "git update-index" fall prey to this behavior. The second and more confusing behavior is "File Virtualization". It means that when files are written without having write permission, it does not fail (as expected), but they are instead redirected to somewhere else. When the files are read, the original contents are returned, though, not the ones that were just written somewhere else. Even more confusing, not all write accesses are redirected; Trying to write to write-protected .exe files, for example, will fail instead of redirecting. In addition to being unwanted behavior, File Virtualization causes dramatic slowdowns in Git (see for instance http://code.google.com/p/msysgit/issues/detail?id=320). There are two ways to prevent those two behaviors: Either you embed an application manifest within all your executables, or you add an external manifest (a file with the same name followed by .manifest) to all your executables. Since Git's builtins are hardlinked (or copied), it is simpler and more robust to embed a manifest. A recent enough MSVC compiler should already embed a working internal manifest, but for MinGW you have to do so by hand. Very lightly tested on Wine, where like on Windows XP it should not make any difference. References: - New UAC Technologies for Windows Vista http://msdn.microsoft.com/en-us/library/bb756960.aspx - Create and Embed an Application Manifest (UAC) http://msdn.microsoft.com/en-us/library/bb756929.aspx [js: simplified the embedding dramatically by reusing Git for Windows' existing Windows resource file, removed the optional (and dubious) processorArchitecture attribute of the manifest's assemblyIdentity section.] Signed-off-by: Cesar Eduardo Barros Signed-off-by: Johannes Schindelin --- compat/win32/git.manifest | 25 +++++++++++++++++++++++++ git.rc | 2 ++ 2 files changed, 27 insertions(+) create mode 100644 compat/win32/git.manifest diff --git a/compat/win32/git.manifest b/compat/win32/git.manifest new file mode 100644 index 0000000000..771e3cce43 --- /dev/null +++ b/compat/win32/git.manifest @@ -0,0 +1,25 @@ + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/git.rc b/git.rc index 49002e0d54..cc3fdc6cc6 100644 --- a/git.rc +++ b/git.rc @@ -20,3 +20,5 @@ BEGIN VALUE "Translation", 0x409, 1200 END END + +1 RT_MANIFEST "compat/win32/git.manifest" From 81afd91d00a286b511fb2187eafb9a509f6df58e Mon Sep 17 00:00:00 2001 From: Alan Davies Date: Thu, 13 Aug 2015 11:15:44 +0100 Subject: [PATCH 24/38] winansi.c: Fix colourization on Cygwin pseudo terminals. Git only colours the output and uses pagination if isatty() returns 1. MSys and Cygwin emulate pseudo terminals via named pipes, meaning that isatty() returns 0. Commit 3adef8de5513390459e1caa999e7774adb89a44d fixed this for MSys terminals, but not Cygwin. The named pipes that Cygwin and Msys use are very similar. MSys PTY pipes are called 'msys-*-pty*' and Cygwin uses 'cygwin-*-pty*'. This commit modifies the existing check to allow both MSys and Cygwin PTY pipes to be identified as TTYs. Note that Pagination is still broken on Cygwin. less.exe is spawned (as seen in Process Explorer and using GIT_TRACE=1), but the output is not being piped into it. This partially fixes https://github.com/git-for-windows/git/issues/267 Signed-off-by: Alan Davies --- compat/winansi.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/compat/winansi.c b/compat/winansi.c index db4a5b0a37..54af2c5cf0 100644 --- a/compat/winansi.c +++ b/compat/winansi.c @@ -555,8 +555,12 @@ static void detect_msys_tty(int fd) name = nameinfo->Name.Buffer; name[nameinfo->Name.Length] = 0; - /* check if this could be a MSYS2 pty pipe ('msys-XXXX-ptyN-XX') */ - if (!wcsstr(name, L"msys-") || !wcsstr(name, L"-pty")) + /* + * Check if this could be a MSYS2 pty pipe ('msys-XXXX-ptyN-XX') + * or a cygwin pty pipe ('cygwin-XXXX-ptyN-XX') + */ + if ((!wcsstr(name, L"msys-") && !wcsstr(name, L"cygwin-")) || + !wcsstr(name, L"-pty")) return; /* init ioinfo size if we haven't done so */ From be917a3219bb23075b51104f772e578b3752b992 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 2 Nov 2015 08:41:10 +0100 Subject: [PATCH 25/38] mingw: enable stack smashing protector As suggested privately to Brendan Forster by some unnamed person (suggestion for the future: use the public mailing list, or even the public GitHub issue tracker, that is a much better place to offer such suggestions), we should make use of gcc's stack smashing protector that helps detect stack buffer overruns early. Rather than using -fstack-protector, we use -fstack-protector-strong because it strikes a better balance between how much code is affected and the performance impact. In a local test (time git log --grep=is -p), best of 5 timings went from 23.009s to 22.997s (i.e. the performance impact was *well* lost in the noise). This fixes https://github.com/git-for-windows/git/issues/501 Signed-off-by: Johannes Schindelin --- config.mak.uname | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config.mak.uname b/config.mak.uname index c3ffc7c17f..acd4a785c7 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -560,7 +560,8 @@ else BASIC_LDFLAGS += -Wl,--large-address-aware endif CC = gcc - COMPAT_CFLAGS += -D__USE_MINGW_ANSI_STDIO=0 -DDETECT_MSYS_TTY + COMPAT_CFLAGS += -D__USE_MINGW_ANSI_STDIO=0 -DDETECT_MSYS_TTY \ + -fstack-protector-strong EXTLIBS += -lntdll INSTALL = /bin/install NO_R_TO_GCC_LINKER = YesPlease From 91072e7d368d8bb413acfc486e8abf09c708cf3b Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 17 Mar 2015 19:41:02 +0100 Subject: [PATCH 26/38] UTF-8 environment: be a little bit more defensive It is unlikely that we have an empty environment, ever, but *if* we do, when `environ_size - 1` is passed to `bsearchenv()` it is misinterpreted as a real large integer. To make the code truly defensive, refuse to do anything at all if the size is negative (which should not happen, of course). Signed-off-by: Johannes Schindelin --- compat/mingw.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index ffc8f4cfa6..89be11ec49 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -1365,7 +1365,7 @@ static int bsearchenv(char **env, const char *name, size_t size) */ static int do_putenv(char **env, const char *name, int size, int free_old) { - int i = bsearchenv(env, name, size - 1); + int i = size <= 0 ? -1 : bsearchenv(env, name, size - 1); /* optionally free removed / replaced entry */ if (i >= 0 && free_old) @@ -1390,7 +1390,13 @@ static int do_putenv(char **env, const char *name, int size, int free_old) char *mingw_getenv(const char *name) { char *value; - int pos = bsearchenv(environ, name, environ_size - 1); + int pos; + + if (environ_size <= 0) + return NULL; + + pos = bsearchenv(environ, name, environ_size - 1); + if (pos < 0) return NULL; value = strchr(environ[pos], '='); From 868f874b1caab40a064dd5a13f0c97e8161feaec Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 18 Mar 2015 10:57:52 +0100 Subject: [PATCH 27/38] mingw: be more defensive when making the environment block Outside of our Windows-specific code, the end of the environment can be marked also by a pointer to a NUL character, not only by a NULL pointer as our code assumed so far. That led to a buffer overrun in `make_environment_block()` when running `git-remote-https` in `mintty` (because `curl_global_init()` added the `CHARSET` environment variable *outside* of `mingw_putenv()`, ending the environment in a pointer to an empty string). Side note for future debugging on Windows: when running programs in `mintty`, the standard input/output/error is not connected to a Win32 Console, but instead is pipe()d. That means that even stderr may not be written completely before a crash, but has to be fflush()ed explicitly. For example, when debugging crashes, the developer should insert an `fflush(stderr);` at the end of the `error()` function defined in usage.c. Signed-off-by: Johannes Schindelin --- compat/mingw.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 89be11ec49..1748414af8 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -1041,7 +1041,7 @@ static wchar_t *make_environment_block(char **deltaenv) char **tmpenv; int i = 0, size = environ_size, wenvsz = 0, wenvpos = 0; - while (deltaenv && deltaenv[i]) + while (deltaenv && deltaenv[i] && *deltaenv[i]) i++; /* copy the environment, leaving space for changes */ @@ -1049,11 +1049,11 @@ static wchar_t *make_environment_block(char **deltaenv) memcpy(tmpenv, environ, size * sizeof(char*)); /* merge supplied environment changes into the temporary environment */ - for (i = 0; deltaenv && deltaenv[i]; i++) + for (i = 0; deltaenv && deltaenv[i] && *deltaenv[i]; i++) size = do_putenv(tmpenv, deltaenv[i], size, 0); /* create environment block from temporary environment */ - for (i = 0; tmpenv[i]; i++) { + for (i = 0; tmpenv[i] && *tmpenv[i]; i++) { size = 2 * strlen(tmpenv[i]) + 2; /* +2 for final \0 */ ALLOC_GROW(wenvblk, (wenvpos + size) * sizeof(wchar_t), wenvsz); wenvpos += xutftowcs(&wenvblk[wenvpos], tmpenv[i], size) + 1; From b4af700685a4a1d4bafe3745e20277d10a71dab3 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 17 Mar 2015 19:41:31 +0100 Subject: [PATCH 28/38] mingw: be *very* wary about outside environment changes The environment is modified in most surprising circumstances, and not all of them are under Git's control. For example, calling curl_global_init() on Windows will ensure that the CHARSET variable is set, adding one if necessary. While the previous commit worked around crashes triggered by such outside changes of the environment by relaxing the requirement that the environment be terminated by a NULL pointer, the other assumption made by `mingw_getenv()` and `mingw_putenv()` is that the environment is sorted, for efficient lookup via binary search. Let's make real sure that our environment is intact before querying or modifying it, and reinitialize our idea of the environment if necessary. With this commit, before working on the environment we look briefly for indicators that the environment was modified outside of our control, and to ensure that it is terminated with a NULL pointer and sorted again in that case. Note: the indicators are maybe not sufficient. For example, when a variable is removed, it will not be noticed. It might also be a problem if outside changes to the environment result in a modified `environ` pointer: it is unclear whether such a modification could result in a problem when `mingw_putenv()` needs to `realloc()` the environment buffer. For the moment, however, the current fix works well enough, so let's only face the potential problems when (and if!) they occur. Signed-off-by: Johannes Schindelin --- compat/mingw.c | 49 +++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 1748414af8..543aa779dd 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -1030,6 +1030,10 @@ static int do_putenv(char **env, const char *name, int size, int free_old); static int environ_size = 0; /* allocated size of environ array, in bytes */ static int environ_alloc = 0; +/* used as a indicator when the environment has been changed outside mingw.c */ +static char **saved_environ; + +static void maybe_reinitialize_environ(void); /* * Create environment block suitable for CreateProcess. Merges current @@ -1039,7 +1043,10 @@ static wchar_t *make_environment_block(char **deltaenv) { wchar_t *wenvblk = NULL; char **tmpenv; - int i = 0, size = environ_size, wenvsz = 0, wenvpos = 0; + int i = 0, size, wenvsz = 0, wenvpos = 0; + + maybe_reinitialize_environ(); + size = environ_size; while (deltaenv && deltaenv[i] && *deltaenv[i]) i++; @@ -1342,6 +1349,41 @@ static int compareenv(const void *v1, const void *v2) } } +/* + * Functions implemented outside Git are able to modify the environment, + * too. For example, cURL's curl_global_init() function sets the CHARSET + * environment variable (at least in certain circumstances). + * + * Therefore we need to be *really* careful *not* to assume that we have + * sole control over the environment and reinitalize it when necessary. + */ +static void maybe_reinitialize_environ(void) +{ + int i; + + if (!saved_environ) { + warning("MinGW environment not initialized yet"); + return; + } + + if (environ_size <= 0) + return; + + if (saved_environ != environ) + /* We have *no* idea how much space was allocated outside */ + environ_alloc = 0; + else if (!environ[environ_size - 1]) + return; /* still consistent */ + + for (i = 0; environ[i] && *environ[i]; i++) + ; /* continue counting */ + environ[i] = NULL; + environ_size = i + 1; + + /* sort environment for O(log n) getenv / putenv */ + qsort(environ, i, sizeof(char*), compareenv); +} + static int bsearchenv(char **env, const char *name, size_t size) { unsigned low = 0, high = size; @@ -1395,6 +1437,7 @@ char *mingw_getenv(const char *name) if (environ_size <= 0) return NULL; + maybe_reinitialize_environ(); pos = bsearchenv(environ, name, environ_size - 1); if (pos < 0) @@ -1405,7 +1448,9 @@ char *mingw_getenv(const char *name) int mingw_putenv(const char *namevalue) { + maybe_reinitialize_environ(); ALLOC_GROW(environ, (environ_size + 1) * sizeof(char*), environ_alloc); + saved_environ = environ; environ_size = do_putenv(environ, namevalue, environ_size, 1); return 0; } @@ -2284,7 +2329,7 @@ void mingw_startup(void) */ environ_size = i + 1; environ_alloc = alloc_nr(environ_size * sizeof(char*)); - environ = malloc_startup(environ_alloc); + saved_environ = environ = malloc_startup(environ_alloc); /* allocate buffer (wchar_t encodes to max 3 UTF-8 bytes) */ maxlen = 3 * maxlen + 1; From 7dc407e57c10d5bcbc21412d3f50cf9ba3ae6f3b Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 6 Apr 2015 14:34:59 +0100 Subject: [PATCH 29/38] git-gui (MinGW): make use of MSys2's msgfmt When Git for Windows was still based on MSys1, we had no gettext, ergo no msgfmt, either. Therefore, we introduced a small and simple Tcl script to perform the same task. However, with MSys2, we no longer need that because we have a proper msgfmt executable. Plus, the po2msg.sh script somehow manages to hang when run in parallel. Two reasons to use real msgfmt.exe instead. Signed-off-by: Johannes Schindelin --- git-gui/Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/git-gui/Makefile b/git-gui/Makefile index 4f00bdd3d6..de1d17491f 100644 --- a/git-gui/Makefile +++ b/git-gui/Makefile @@ -161,7 +161,9 @@ ifeq ($(uname_S),Darwin) endif endif ifneq (,$(findstring MINGW,$(uname_S))) +ifeq ($(shell expr "$(uname_R)" : '1\.'),2) NO_MSGFMT=1 +endif GITGUI_WINDOWS_WRAPPER := YesPlease GITGUI_RELATIVE := 1 endif From 4131792e9ee3e19e455b02208ba25e30741e97c2 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 6 Apr 2015 10:37:04 +0100 Subject: [PATCH 30/38] Avoid illegal filenames when building Documentation on NTFS A '+' is not a valid part of a filename with Windows file systems (it is reserved because the '+' operator meant file concatenation back in the DOS days). Let's just not use it. Signed-off-by: Johannes Schindelin --- Documentation/Makefile | 84 +++++++++++++++++++++--------------------- 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/Documentation/Makefile b/Documentation/Makefile index a9fb497b83..6517b59c6b 100644 --- a/Documentation/Makefile +++ b/Documentation/Makefile @@ -267,9 +267,9 @@ docdep_prereqs = \ cmd-list.made $(cmds_txt) doc.dep : $(docdep_prereqs) $(wildcard *.txt) build-docdep.perl - $(QUIET_GEN)$(RM) $@+ $@ && \ - $(PERL_PATH) ./build-docdep.perl >$@+ $(QUIET_STDERR) && \ - mv $@+ $@ + $(QUIET_GEN)$(RM) $@.new $@ && \ + $(PERL_PATH) ./build-docdep.perl >$@.new $(QUIET_STDERR) && \ + mv $@.new $@ -include doc.dep @@ -305,8 +305,8 @@ mergetools-list.made: ../git-mergetool--lib.sh $(wildcard ../mergetools/*) date >$@ clean: - $(RM) *.xml *.xml+ *.html *.html+ *.1 *.5 *.7 - $(RM) *.texi *.texi+ *.texi++ git.info gitman.info + $(RM) *.xml *.xml.new *.html *.html.new *.1 *.5 *.7 + $(RM) *.texi *.texi.new *.texi.new.new git.info gitman.info $(RM) *.pdf $(RM) howto-index.txt howto/*.html doc.dep $(RM) technical/*.html technical/api-index.txt @@ -314,14 +314,14 @@ clean: $(RM) manpage-base-url.xsl $(MAN_HTML): %.html : %.txt asciidoc.conf - $(QUIET_ASCIIDOC)$(RM) $@+ $@ && \ - $(TXT_TO_HTML) -d manpage -o $@+ $< && \ - mv $@+ $@ + $(QUIET_ASCIIDOC)$(RM) $@.new $@ && \ + $(TXT_TO_HTML) -d manpage -o $@.new $< && \ + mv $@.new $@ $(OBSOLETE_HTML): %.html : %.txto asciidoc.conf - $(QUIET_ASCIIDOC)$(RM) $@+ $@ && \ - $(TXT_TO_HTML) -o $@+ $< && \ - mv $@+ $@ + $(QUIET_ASCIIDOC)$(RM) $@.new $@ && \ + $(TXT_TO_HTML) -o $@.new $< && \ + mv $@.new $@ manpage-base-url.xsl: manpage-base-url.xsl.in sed "s|@@MAN_BASE_URL@@|$(MAN_BASE_URL)|" $< > $@ @@ -331,14 +331,14 @@ manpage-base-url.xsl: manpage-base-url.xsl.in $(XMLTO) -m $(MANPAGE_XSL) $(XMLTO_EXTRA) man $< %.xml : %.txt asciidoc.conf - $(QUIET_ASCIIDOC)$(RM) $@+ $@ && \ - $(TXT_TO_XML) -d manpage -o $@+ $< && \ - mv $@+ $@ + $(QUIET_ASCIIDOC)$(RM) $@.new $@ && \ + $(TXT_TO_XML) -d manpage -o $@.new $< && \ + mv $@.new $@ user-manual.xml: user-manual.txt user-manual.conf - $(QUIET_ASCIIDOC)$(RM) $@+ $@ && \ - $(TXT_TO_XML) -d book -o $@+ $< && \ - mv $@+ $@ + $(QUIET_ASCIIDOC)$(RM) $@.new $@ && \ + $(TXT_TO_XML) -d book -o $@.new $< && \ + mv $@.new $@ technical/api-index.txt: technical/api-index-skel.txt \ technical/api-index.sh $(patsubst %,%.txt,$(API_DOCS)) @@ -352,45 +352,45 @@ XSLT = docbook.xsl XSLTOPTS = --xinclude --stringparam html.stylesheet docbook-xsl.css user-manual.html: user-manual.xml $(XSLT) - $(QUIET_XSLTPROC)$(RM) $@+ $@ && \ - xsltproc $(XSLTOPTS) -o $@+ $(XSLT) $< && \ - mv $@+ $@ + $(QUIET_XSLTPROC)$(RM) $@.new $@ && \ + xsltproc $(XSLTOPTS) -o $@.new $(XSLT) $< && \ + mv $@.new $@ git.info: user-manual.texi $(QUIET_MAKEINFO)$(MAKEINFO) --no-split -o $@ user-manual.texi user-manual.texi: user-manual.xml - $(QUIET_DB2TEXI)$(RM) $@+ $@ && \ - $(DOCBOOK2X_TEXI) user-manual.xml --encoding=UTF-8 --to-stdout >$@++ && \ - $(PERL_PATH) fix-texi.perl <$@++ >$@+ && \ - rm $@++ && \ - mv $@+ $@ + $(QUIET_DB2TEXI)$(RM) $@.new $@ && \ + $(DOCBOOK2X_TEXI) user-manual.xml --encoding=UTF-8 --to-stdout >$@.new.new && \ + $(PERL_PATH) fix-texi.perl <$@.new.new >$@.new && \ + rm $@.new.new && \ + mv $@.new $@ user-manual.pdf: user-manual.xml - $(QUIET_DBLATEX)$(RM) $@+ $@ && \ - $(DBLATEX) -o $@+ -p $(ASCIIDOC_DBLATEX_DIR)/asciidoc-dblatex.xsl -s $(ASCIIDOC_DBLATEX_DIR)/asciidoc-dblatex.sty $< && \ - mv $@+ $@ + $(QUIET_DBLATEX)$(RM) $@.new $@ && \ + $(DBLATEX) -o $@.new -p $(ASCIIDOC_DBLATEX_DIR)/asciidoc-dblatex.xsl -s $(ASCIIDOC_DBLATEX_DIR)/asciidoc-dblatex.sty $< && \ + mv $@.new $@ gitman.texi: $(MAN_XML) cat-texi.perl - $(QUIET_DB2TEXI)$(RM) $@+ $@ && \ + $(QUIET_DB2TEXI)$(RM) $@.new $@ && \ ($(foreach xml,$(MAN_XML),$(DOCBOOK2X_TEXI) --encoding=UTF-8 \ - --to-stdout $(xml) &&) true) > $@++ && \ - $(PERL_PATH) cat-texi.perl $@ <$@++ >$@+ && \ - rm $@++ && \ - mv $@+ $@ + --to-stdout $(xml) &&) true) > $@.new.new && \ + $(PERL_PATH) cat-texi.perl $@ <$@.new.new >$@.new && \ + rm $@.new.new && \ + mv $@.new $@ gitman.info: gitman.texi $(QUIET_MAKEINFO)$(MAKEINFO) --no-split --no-validate $*.texi $(patsubst %.txt,%.texi,$(MAN_TXT)): %.texi : %.xml - $(QUIET_DB2TEXI)$(RM) $@+ $@ && \ - $(DOCBOOK2X_TEXI) --to-stdout $*.xml >$@+ && \ - mv $@+ $@ + $(QUIET_DB2TEXI)$(RM) $@.new $@ && \ + $(DOCBOOK2X_TEXI) --to-stdout $*.xml >$@.new && \ + mv $@.new $@ howto-index.txt: howto-index.sh $(wildcard howto/*.txt) - $(QUIET_GEN)$(RM) $@+ $@ && \ - '$(SHELL_PATH_SQ)' ./howto-index.sh $(sort $(wildcard howto/*.txt)) >$@+ && \ - mv $@+ $@ + $(QUIET_GEN)$(RM) $@.new $@ && \ + '$(SHELL_PATH_SQ)' ./howto-index.sh $(sort $(wildcard howto/*.txt)) >$@.new && \ + mv $@.new $@ $(patsubst %,%.html,$(ARTICLES)) : %.html : %.txt $(QUIET_ASCIIDOC)$(TXT_TO_HTML) $*.txt @@ -399,10 +399,10 @@ WEBDOC_DEST = /pub/software/scm/git/docs howto/%.html: ASCIIDOC_EXTRA += -a git-relative-html-prefix=../ $(patsubst %.txt,%.html,$(wildcard howto/*.txt)): %.html : %.txt - $(QUIET_ASCIIDOC)$(RM) $@+ $@ && \ + $(QUIET_ASCIIDOC)$(RM) $@.new $@ && \ sed -e '1,/^$$/d' $< | \ - $(TXT_TO_HTML) - >$@+ && \ - mv $@+ $@ + $(TXT_TO_HTML) - >$@.new && \ + mv $@.new $@ install-webdoc : html '$(SHELL_PATH_SQ)' ./install-webdoc.sh $(WEBDOC_DEST) From 5a5d9f9b96fadd87b55b9fe8431eb688f18e6af8 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Thu, 9 Apr 2015 16:19:56 +0100 Subject: [PATCH 31/38] gettext: always use UTF-8 on native Windows Git on native Windows exclusively uses UTF-8 for console output (both with mintty and native console windows). Gettext uses setlocale() to determine the output encoding for translated text, however, MSVCRT's setlocale() doesn't support UTF-8. As a result, translated text is encoded in system encoding (GetAPC()), and non-ASCII chars are mangled in console output. Use gettext's bind_textdomain_codeset() to force the encoding to UTF-8 on native Windows. In this developers' setup, HAVE_LIBCHARSET_H is apparently defined, but we *really* want to override the locale_charset() here. Signed-off-by: Karsten Blees --- gettext.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/gettext.c b/gettext.c index db727ea020..7d9e2c27ef 100644 --- a/gettext.c +++ b/gettext.c @@ -10,7 +10,9 @@ #ifndef NO_GETTEXT # include # include -# ifdef HAVE_LIBCHARSET_H +# ifdef GIT_WINDOWS_NATIVE +# define locale_charset() "UTF-8" +# elif defined HAVE_LIBCHARSET_H # include # else # include From 1204d53e15d99d23afdce3f3072ee0b841162a79 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 7 Apr 2015 10:24:55 +0100 Subject: [PATCH 32/38] mingw: HOT FIX: work around environment issues -- again This developer should really, really have known better. The fact that we are changing the environment in ways for which the MSVCRT is not prepared for is bad enough. But then this developer followed the request to re-enable nedmalloc -- despite the prediction that it would cause an access violation, predicting it in the same message as the request to re-enable nedmalloc, no less! To paper over the issue until the time when this developer finds the time to re-design the Unicode environment handling from scratch, let's hope that cURL is the only library we are using that *may* set an environment variable using MSVCRT's putenv() after we fscked the environment up. Note: this commit can serve as no source of pride to anyone, certainly not yours truly. It is necessary as a quick and pragmatic stop gap, though, to prevent worse problems. Note: cURL manages to set the variable CHARSET when nedmalloc is *not* enabled, without causing an access violation. In that case, it sets it successfully to the value "cp" + GetACP() (hence it is our choice, too, cURL may need it, Git does not): https://github.com/bagder/curl/blob/aa5808b5/lib/easy.c#L157-L162 Signed-off-by: Johannes Schindelin --- compat/mingw.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/compat/mingw.c b/compat/mingw.c index 543aa779dd..ba6b2f8a78 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2350,6 +2350,17 @@ void mingw_startup(void) /* fix Windows specific environment settings */ setup_windows_environment(); + /* + * Avoid a segmentation fault when cURL tries to set the CHARSET + * variable and putenv() barfs at our nedmalloc'ed environment. + */ + if (!getenv("CHARSET")) { + struct strbuf buf = STRBUF_INIT; + strbuf_addf(&buf, "cp%u", GetACP()); + setenv("CHARSET", buf.buf, 1); + strbuf_release(&buf); + } + /* initialize critical section for waitpid pinfo_t list */ InitializeCriticalSection(&pinfo_cs); From 8682273a0de423bac1305e66b081d75876301f65 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Mon, 6 Apr 2015 21:37:18 +0200 Subject: [PATCH 33/38] mingw: initialize HOME on startup HOME initialization was historically duplicated in many different places, including /etc/profile, launch scripts such as git-bash.vbs and gitk.cmd, and (although slightly broken) in the git-wrapper. Even unrelated projects such as GitExtensions and TortoiseGit need to implement the same logic to be able to call git directly. Initialize HOME in git's own startup code so that we can eventually retire all the duplicate initialization code. Signed-off-by: Karsten Blees --- compat/mingw.c | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/compat/mingw.c b/compat/mingw.c index ba6b2f8a78..79666ab33d 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2206,6 +2206,30 @@ static void setup_windows_environment(void) /* simulate TERM to enable auto-color (see color.c) */ if (!getenv("TERM")) setenv("TERM", "cygwin", 1); + + /* calculate HOME if not set */ + if (!getenv("HOME")) { + /* + * try $HOMEDRIVE$HOMEPATH - the home share may be a network + * location, thus also check if the path exists (i.e. is not + * disconnected) + */ + if ((tmp = getenv("HOMEDRIVE"))) { + struct strbuf buf = STRBUF_INIT; + strbuf_addstr(&buf, tmp); + if ((tmp = getenv("HOMEPATH"))) { + strbuf_addstr(&buf, tmp); + if (is_directory(buf.buf)) + setenv("HOME", buf.buf, 1); + else + tmp = NULL; /* use $USERPROFILE */ + } + strbuf_release(&buf); + } + /* use $USERPROFILE if the home share is not available */ + if (!tmp && (tmp = getenv("USERPROFILE"))) + setenv("HOME", tmp, 1); + } } int handle_long_path(wchar_t *path, int len, int max_path, int expand) From 031fae571aeedf309ee67de49acbd221db79ef5f Mon Sep 17 00:00:00 2001 From: nalla Date: Thu, 16 Apr 2015 11:45:05 +0100 Subject: [PATCH 34/38] mingw: explicitly `fflush` stdout For performance reasons `stdout` is not unbuffered by default. That leads to problems if after printing to `stdout` a read on `stdin` is performed. For that reason interactive commands like `git clean -i` do not function properly anymore if the `stdout` is not flushed by `fflush(stdout)` before trying to read from `stdin`. In the case of `git clean -i` all reads on `stdin` were preceded by a `fflush(stdout)` call. Signed-off-by: nalla --- builtin/clean.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/builtin/clean.c b/builtin/clean.c index 0371010afb..f54bcba497 100644 --- a/builtin/clean.c +++ b/builtin/clean.c @@ -570,6 +570,7 @@ static int *list_and_choose(struct menu_opts *opts, struct menu_stuff *stuff) clean_get_color(CLEAN_COLOR_RESET)); } + fflush(stdout); if (strbuf_getline_lf(&choice, stdin) != EOF) { strbuf_trim(&choice); } else { @@ -652,6 +653,7 @@ static int filter_by_patterns_cmd(void) clean_print_color(CLEAN_COLOR_PROMPT); printf(_("Input ignore patterns>> ")); clean_print_color(CLEAN_COLOR_RESET); + fflush(stdout); if (strbuf_getline_lf(&confirm, stdin) != EOF) strbuf_trim(&confirm); else @@ -750,6 +752,7 @@ static int ask_each_cmd(void) qname = quote_path_relative(item->string, NULL, &buf); /* TRANSLATORS: Make sure to keep [y/N] as is */ printf(_("Remove %s [y/N]? "), qname); + fflush(stdout); if (strbuf_getline_lf(&confirm, stdin) != EOF) { strbuf_trim(&confirm); } else { From fb33d68fdaf8481ae74f7a1fee7705c55c1181b0 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Thu, 19 Mar 2015 16:33:44 +0100 Subject: [PATCH 35/38] mingw: Support `git_terminal_prompt` with more terminals The `git_terminal_prompt()` function expects the terminal window to be attached to a Win32 Console. However, this is not the case with terminal windows other than `cmd.exe`'s, e.g. with MSys2's own `mintty`. Non-cmd terminals such as `mintty` still have to have a Win32 Console to be proper console programs, but have to hide the Win32 Console to be able to provide more flexibility (such as being resizeable not only vertically but also horizontally). By writing to that Win32 Console, `git_terminal_prompt()` manages only to send the prompt to nowhere and to wait for input from a Console to which the user has no access. This commit introduces a function specifically to support `mintty` -- or other terminals that are compatible with MSys2's `/dev/tty` emulation. We use the `TERM` environment variable as an indicator for that: if the value starts with "xterm" (such as `mintty`'s "xterm_256color"), we prefer to let `xterm_prompt()` handle the user interaction. The most prominent user of `git_terminal_prompt()` is certainly `git-remote-https.exe`. It is an interesting use case because both `stdin` and `stdout` are redirected when Git calls said executable, yet it still wants to access the terminal. When running inside a `mintty`, the terminal is not accessible to the `git-remote-https.exe` program, though, because it is a MinGW program and the `mintty` terminal is not backed by a Win32 console. To solve that problem, we simply call out to the shell -- which is an *MSys2* program and can therefore access `/dev/tty`. Helped-by: nalla Signed-off-by: Karsten Blees Signed-off-by: Johannes Schindelin --- compat/terminal.c | 57 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/compat/terminal.c b/compat/terminal.c index fa13ee672d..cd6e2e55a5 100644 --- a/compat/terminal.c +++ b/compat/terminal.c @@ -1,7 +1,10 @@ +#include #include "git-compat-util.h" +#include "run-command.h" #include "compat/terminal.h" #include "sigchain.h" #include "strbuf.h" +#include "cache.h" #if defined(HAVE_DEV_TTY) || defined(GIT_WINDOWS_NATIVE) @@ -91,6 +94,54 @@ static int disable_echo(void) return 0; } +static char *shell_prompt(const char *prompt, int echo) +{ + const char *read_input[] = { + /* Note: call 'bash' explicitly, as 'read -s' is bash-specific */ + "bash", "-c", echo ? + "cat >/dev/tty && read -r line /dev/tty && read -r -s line /dev/tty", + NULL + }; + struct child_process child = CHILD_PROCESS_INIT; + static struct strbuf buffer = STRBUF_INIT; + int prompt_len = strlen(prompt), len = -1, code; + + child.argv = read_input; + child.in = -1; + child.out = -1; + + if (start_command(&child)) + return NULL; + + if (write_in_full(child.in, prompt, prompt_len) != prompt_len) { + error("could not write to prompt script"); + close(child.in); + goto ret; + } + close(child.in); + + strbuf_reset(&buffer); + len = strbuf_read(&buffer, child.out, 1024); + if (len < 0) { + error("could not read from prompt script"); + goto ret; + } + + strbuf_strip_suffix(&buffer, "\n"); + strbuf_strip_suffix(&buffer, "\r"); + +ret: + close(child.out); + code = finish_command(&child); + if (code) { + error("failed to execute prompt script (exit code %d)", code); + return NULL; + } + + return len < 0 ? NULL : buffer.buf; +} + #endif #ifndef FORCE_TEXT @@ -102,6 +153,12 @@ char *git_terminal_prompt(const char *prompt, int echo) static struct strbuf buf = STRBUF_INIT; int r; FILE *input_fh, *output_fh; +#ifdef GIT_WINDOWS_NATIVE + const char *term = getenv("TERM"); + + if (term && starts_with(term, "xterm")) + return shell_prompt(prompt, echo); +#endif input_fh = fopen(INPUT_PATH, "r" FORCE_TEXT); if (!input_fh) From 2f63b55a37d10eade2198f9c8c4ecd6ea9a8de32 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sat, 9 May 2015 02:11:48 +0200 Subject: [PATCH 36/38] compat/terminal.c: only use the Windows console if bash 'read -r' fails MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Accessing the Windows console through the special CONIN$ / CONOUT$ devices doesn't work properly for non-ASCII usernames an passwords. It also doesn't work for terminal emulators that hide the native console window (such as mintty), and 'TERM=xterm*' is not necessarily a reliable indicator for such terminals. The new shell_prompt() function, on the other hand, works fine for both MSys1 and MSys2, in native console windows as well as mintty, and properly supports Unicode. It just needs bash on the path (for 'read -s', which is bash-specific). On Windows, try to use the shell to read from the terminal. If that fails with ENOENT (i.e. bash was not found), use CONIN/OUT as fallback. Note: To test this, create a UTF-8 credential file with non-ASCII chars, e.g. in git-bash: 'echo url=http://täst.com > cred.txt'. Then in git-cmd, 'git credential fill --- compat/terminal.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/compat/terminal.c b/compat/terminal.c index cd6e2e55a5..1d37f0aafb 100644 --- a/compat/terminal.c +++ b/compat/terminal.c @@ -110,6 +110,7 @@ static char *shell_prompt(const char *prompt, int echo) child.argv = read_input; child.in = -1; child.out = -1; + child.silent_exec_failure = 1; if (start_command(&child)) return NULL; @@ -153,11 +154,14 @@ char *git_terminal_prompt(const char *prompt, int echo) static struct strbuf buf = STRBUF_INIT; int r; FILE *input_fh, *output_fh; -#ifdef GIT_WINDOWS_NATIVE - const char *term = getenv("TERM"); - if (term && starts_with(term, "xterm")) - return shell_prompt(prompt, echo); +#ifdef GIT_WINDOWS_NATIVE + + /* try shell_prompt first, fall back to CONIN/OUT if bash is missing */ + char *result = shell_prompt(prompt, echo); + if (result || errno != ENOENT) + return result; + #endif input_fh = fopen(INPUT_PATH, "r" FORCE_TEXT); From 2f7a7706e5d03235b1c9b2c71c2956313c78e8b8 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 13 Aug 2015 15:25:59 +0000 Subject: [PATCH 37/38] mingw: Suppress warning that :.gitattributes does not exist On Windows, a file name containing a colon is illegal. We should therefore expect the corresponding errno when `fopen()` is called for a path of the form :.gitattributes. This fixes https://github.com/git-for-windows/git/issues/255. Signed-off-by: Johannes Schindelin --- attr.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/attr.c b/attr.c index eec5d7d15a..256d9cf1fd 100644 --- a/attr.c +++ b/attr.c @@ -373,7 +373,7 @@ static struct attr_stack *read_attr_from_file(const char *path, int macro_ok) int lineno = 0; if (!fp) { - if (errno != ENOENT && errno != ENOTDIR) + if (errno != ENOENT && errno != ENOTDIR && errno != EINVAL) warn_on_inaccessible(path); return NULL; } From 3a1302933a7869edb6cb5eba7816327276979d0d Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 9 Dec 2015 13:56:11 +0100 Subject: [PATCH 38/38] Windows: force-recompile git.res for differing architectures When git.rc is compiled into git.res, the result is actually dependent on the architecture. That is, you cannot simply link a 32-bit git.res into a 64-bit git.exe. Therefore, to allow 32-bit and 64-bit builds in the same directory, we let git.res depend on GIT-PREFIX so that it gets recompiled when compiling for a different architecture (this works because the exec path changes based on the architecture: /mingw32/libexec/git-core for 32-bit and /mingw64/libexec/git-core for 64-bit). Signed-off-by: Johannes Schindelin --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index a1509422d6..25a81aa2c9 100644 --- a/Makefile +++ b/Makefile @@ -1811,7 +1811,7 @@ else RC_VERSION_DEFS := $(join -DMAJOR= -DMINOR= -DMICRO= -DPATCHLEVEL=, $(wordlist 1,4,$(subst -, ,$(subst ., ,$(subst .windows., ,$(GIT_VERSION)))))) endif -git.res: git.rc GIT-VERSION-FILE +git.res: git.rc GIT-VERSION-FILE GIT-PREFIX $(QUIET_RC)$(RC) $(RC_VERSION_DEFS) \ -DGIT_VERSION="\\\"$(GIT_VERSION)\\\"" $< -o $@