From 01e27562da1be9e3639da577128e9e611fbf8c35 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sat, 6 Jul 2013 02:09:35 +0200 Subject: [PATCH 01/55] Win32: make FILETIME conversion functions public Signed-off-by: Karsten Blees --- compat/mingw.c | 16 ---------------- compat/mingw.h | 16 ++++++++++++++++ 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index ef4556fe32..f2b96032ef 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -474,22 +474,6 @@ int mingw_chmod(const char *filename, int mode) return _wchmod(wfilename, mode); } -/* - * The unit of FILETIME is 100-nanoseconds since January 1, 1601, UTC. - * Returns the 100-nanoseconds ("hekto nanoseconds") since the epoch. - */ -static inline long long filetime_to_hnsec(const FILETIME *ft) -{ - long long winTime = ((long long)ft->dwHighDateTime << 32) + ft->dwLowDateTime; - /* Windows to Unix Epoch conversion */ - return winTime - 116444736000000000LL; -} - -static inline time_t filetime_to_time_t(const FILETIME *ft) -{ - return (time_t)(filetime_to_hnsec(ft) / 10000000); -} - /* We keep the do_lstat code in a separate function to avoid recursion. * When a path ends with a slash, the stat will fail with ENOENT. In * this case, we strip the trailing slashes and stat again. diff --git a/compat/mingw.h b/compat/mingw.h index 5e499cfb71..f2a78b471c 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -282,6 +282,22 @@ static inline int getrlimit(int resource, struct rlimit *rlp) return 0; } +/* + * The unit of FILETIME is 100-nanoseconds since January 1, 1601, UTC. + * Returns the 100-nanoseconds ("hekto nanoseconds") since the epoch. + */ +static inline long long filetime_to_hnsec(const FILETIME *ft) +{ + long long winTime = ((long long)ft->dwHighDateTime << 32) + ft->dwLowDateTime; + /* Windows to Unix Epoch conversion */ + return winTime - 116444736000000000LL; +} + +static inline time_t filetime_to_time_t(const FILETIME *ft) +{ + return (time_t)(filetime_to_hnsec(ft) / 10000000); +} + /* * Use mingw specific stat()/lstat()/fstat() implementations on Windows. */ From 2e3a8ea2c5182b4ad83953f98e963083f2d4247d Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sun, 8 Sep 2013 14:17:31 +0200 Subject: [PATCH 02/55] Win32: dirent.c: Move opendir down Move opendir down in preparation for the next patch. Signed-off-by: Karsten Blees --- compat/win32/dirent.c | 68 +++++++++++++++++++++---------------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/compat/win32/dirent.c b/compat/win32/dirent.c index 52420ec7d4..2603a0fa39 100644 --- a/compat/win32/dirent.c +++ b/compat/win32/dirent.c @@ -18,40 +18,6 @@ static inline void finddata2dirent(struct dirent *ent, WIN32_FIND_DATAW *fdata) ent->d_type = DT_REG; } -DIR *opendir(const char *name) -{ - wchar_t pattern[MAX_PATH + 2]; /* + 2 for '/' '*' */ - WIN32_FIND_DATAW fdata; - HANDLE h; - int len; - DIR *dir; - - /* convert name to UTF-16 and check length < MAX_PATH */ - if ((len = xutftowcs_path(pattern, name)) < 0) - return NULL; - - /* append optional '/' and wildcard '*' */ - if (len && !is_dir_sep(pattern[len - 1])) - pattern[len++] = '/'; - pattern[len++] = '*'; - pattern[len] = 0; - - /* open find handle */ - h = FindFirstFileW(pattern, &fdata); - if (h == INVALID_HANDLE_VALUE) { - DWORD err = GetLastError(); - errno = (err == ERROR_DIRECTORY) ? ENOTDIR : err_win_to_posix(err); - return NULL; - } - - /* initialize DIR structure and copy first dir entry */ - dir = xmalloc(sizeof(DIR)); - dir->dd_handle = h; - dir->dd_stat = 0; - finddata2dirent(&dir->dd_dir, &fdata); - return dir; -} - struct dirent *readdir(DIR *dir) { if (!dir) { @@ -90,3 +56,37 @@ int closedir(DIR *dir) free(dir); return 0; } + +DIR *opendir(const char *name) +{ + wchar_t pattern[MAX_PATH + 2]; /* + 2 for '/' '*' */ + WIN32_FIND_DATAW fdata; + HANDLE h; + int len; + DIR *dir; + + /* convert name to UTF-16 and check length < MAX_PATH */ + if ((len = xutftowcs_path(pattern, name)) < 0) + return NULL; + + /* append optional '/' and wildcard '*' */ + if (len && !is_dir_sep(pattern[len - 1])) + pattern[len++] = '/'; + pattern[len++] = '*'; + pattern[len] = 0; + + /* open find handle */ + h = FindFirstFileW(pattern, &fdata); + if (h == INVALID_HANDLE_VALUE) { + DWORD err = GetLastError(); + errno = (err == ERROR_DIRECTORY) ? ENOTDIR : err_win_to_posix(err); + return NULL; + } + + /* initialize DIR structure and copy first dir entry */ + dir = xmalloc(sizeof(DIR)); + dir->dd_handle = h; + dir->dd_stat = 0; + finddata2dirent(&dir->dd_dir, &fdata); + return dir; +} From 13304841e2c11ce9451782fdfa3068e527acf6b5 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sun, 8 Sep 2013 14:18:40 +0200 Subject: [PATCH 03/55] Win32: Make the dirent implementation pluggable Emulating the POSIX dirent API on Windows via FindFirstFile/FindNextFile is pretty staightforward, however, most of the information provided in the WIN32_FIND_DATA structure is thrown away in the process. A more sophisticated implementation may cache this data, e.g. for later reuse in calls to lstat. Make the dirent implementation pluggable so that it can be switched at runtime, e.g. based on a config option. Define a base DIR structure with pointers to readdir/closedir that match the opendir implementation (i.e. similar to vtable pointers in OOP). Define readdir/closedir so that they call the function pointers in the DIR structure. This allows to choose the opendir implementation on a call-by-call basis. Move the fixed sized dirent.d_name buffer to the dirent-specific DIR structure, as d_name may be implementation specific (e.g. a caching implementation may just set d_name to point into the cache instead of copying the entire file name string). Signed-off-by: Karsten Blees --- compat/win32/dirent.c | 27 +++++++++++++++++---------- compat/win32/dirent.h | 26 +++++++++++++++++++------- 2 files changed, 36 insertions(+), 17 deletions(-) diff --git a/compat/win32/dirent.c b/compat/win32/dirent.c index 2603a0fa39..6b87042182 100644 --- a/compat/win32/dirent.c +++ b/compat/win32/dirent.c @@ -1,15 +1,19 @@ #include "../../git-compat-util.h" -struct DIR { +typedef struct dirent_DIR { + struct DIR base_dir; /* extend base struct DIR */ struct dirent dd_dir; /* includes d_type */ HANDLE dd_handle; /* FindFirstFile handle */ int dd_stat; /* 0-based index */ -}; + char dd_name[MAX_PATH * 3]; /* file name (* 3 for UTF-8 conversion) */ +} dirent_DIR; + +DIR *(*opendir)(const char *dirname) = dirent_opendir; static inline void finddata2dirent(struct dirent *ent, WIN32_FIND_DATAW *fdata) { - /* convert UTF-16 name to UTF-8 */ - xwcstoutf(ent->d_name, fdata->cFileName, sizeof(ent->d_name)); + /* convert UTF-16 name to UTF-8 (d_name points to dirent_DIR.dd_name) */ + xwcstoutf(ent->d_name, fdata->cFileName, MAX_PATH * 3); /* Set file type, based on WIN32_FIND_DATA */ if (fdata->dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) @@ -18,7 +22,7 @@ static inline void finddata2dirent(struct dirent *ent, WIN32_FIND_DATAW *fdata) ent->d_type = DT_REG; } -struct dirent *readdir(DIR *dir) +static struct dirent *dirent_readdir(dirent_DIR *dir) { if (!dir) { errno = EBADF; /* No set_errno for mingw */ @@ -45,7 +49,7 @@ struct dirent *readdir(DIR *dir) return &dir->dd_dir; } -int closedir(DIR *dir) +static int dirent_closedir(dirent_DIR *dir) { if (!dir) { errno = EBADF; @@ -57,13 +61,13 @@ int closedir(DIR *dir) return 0; } -DIR *opendir(const char *name) +DIR *dirent_opendir(const char *name) { wchar_t pattern[MAX_PATH + 2]; /* + 2 for '/' '*' */ WIN32_FIND_DATAW fdata; HANDLE h; int len; - DIR *dir; + dirent_DIR *dir; /* convert name to UTF-16 and check length < MAX_PATH */ if ((len = xutftowcs_path(pattern, name)) < 0) @@ -84,9 +88,12 @@ DIR *opendir(const char *name) } /* initialize DIR structure and copy first dir entry */ - dir = xmalloc(sizeof(DIR)); + dir = xmalloc(sizeof(dirent_DIR)); + dir->base_dir.preaddir = (struct dirent *(*)(DIR *dir)) dirent_readdir; + dir->base_dir.pclosedir = (int (*)(DIR *dir)) dirent_closedir; + dir->dd_dir.d_name = dir->dd_name; dir->dd_handle = h; dir->dd_stat = 0; finddata2dirent(&dir->dd_dir, &fdata); - return dir; + return (DIR*) dir; } diff --git a/compat/win32/dirent.h b/compat/win32/dirent.h index 058207e4bf..6b3ddee51b 100644 --- a/compat/win32/dirent.h +++ b/compat/win32/dirent.h @@ -1,20 +1,32 @@ #ifndef DIRENT_H #define DIRENT_H -typedef struct DIR DIR; - #define DT_UNKNOWN 0 #define DT_DIR 1 #define DT_REG 2 #define DT_LNK 3 struct dirent { - unsigned char d_type; /* file type to prevent lstat after readdir */ - char d_name[MAX_PATH * 3]; /* file name (* 3 for UTF-8 conversion) */ + unsigned char d_type; /* file type to prevent lstat after readdir */ + char *d_name; /* file name */ }; -DIR *opendir(const char *dirname); -struct dirent *readdir(DIR *dir); -int closedir(DIR *dir); +/* + * Base DIR structure, contains pointers to readdir/closedir implementations so + * that opendir may choose a concrete implementation on a call-by-call basis. + */ +typedef struct DIR { + struct dirent *(*preaddir)(struct DIR *dir); + int (*pclosedir)(struct DIR *dir); +} DIR; + +/* default dirent implementation */ +extern DIR *dirent_opendir(const char *dirname); + +/* current dirent implementation */ +extern DIR *(*opendir)(const char *dirname); + +#define readdir(dir) (dir->preaddir(dir)) +#define closedir(dir) (dir->pclosedir(dir)) #endif /* DIRENT_H */ From 3a9194d06c52a0219b6835e1ef6600fd2d0f3822 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sun, 8 Sep 2013 14:21:30 +0200 Subject: [PATCH 04/55] Win32: make the lstat implementation pluggable Emulating the POSIX lstat API on Windows via GetFileAttributes[Ex] is quite slow. Windows operating system APIs seem to be much better at scanning the status of entire directories than checking single files. A caching implementation may improve performance by bulk-reading entire directories or reusing data obtained via opendir / readdir. Make the lstat implementation pluggable so that it can be switched at runtime, e.g. based on a config option. Signed-off-by: Karsten Blees --- compat/mingw.c | 2 ++ compat/mingw.h | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/compat/mingw.c b/compat/mingw.c index f2b96032ef..47dae5a5ac 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -574,6 +574,8 @@ static int do_stat_internal(int follow, const char *file_name, struct stat *buf) return do_lstat(follow, alt_name, buf); } +int (*lstat)(const char *file_name, struct stat *buf) = mingw_lstat; + int mingw_lstat(const char *file_name, struct stat *buf) { return do_stat_internal(0, file_name, buf); diff --git a/compat/mingw.h b/compat/mingw.h index f2a78b471c..ca9358e93c 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -319,7 +319,7 @@ int mingw_fstat(int fd, struct stat *buf); #ifdef lstat #undef lstat #endif -#define lstat mingw_lstat +extern int (*lstat)(const char *file_name, struct stat *buf); #ifndef _stati64 # define _stati64(x,y) mingw_stat(x,y) From e4c6557897d3abef9c6804c56573b5607d9718e5 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sun, 8 Sep 2013 14:23:27 +0200 Subject: [PATCH 05/55] add infrastructure for read-only file system level caches Add a macro to mark code sections that only read from the file system, along with a config option and documentation. This facilitates implementation of relatively simple file system level caches without the need to synchronize with the file system. Enable read-only sections for 'git status' and preload_index. Signed-off-by: Karsten Blees --- Documentation/config.txt | 6 ++++++ builtin/commit.c | 1 + cache.h | 2 ++ config.c | 5 +++++ environment.c | 1 + git-compat-util.h | 15 +++++++++++++++ preload-index.c | 2 ++ 7 files changed, 32 insertions(+) diff --git a/Documentation/config.txt b/Documentation/config.txt index 17d6bdc335..100f85c21d 100644 --- a/Documentation/config.txt +++ b/Documentation/config.txt @@ -706,6 +706,12 @@ relatively high IO latencies. When enabled, Git will do the index comparison to the filesystem data in parallel, allowing overlapping IO's. Defaults to true. +core.fscache:: + Enable additional caching of file system data for some operations. ++ +Git for Windows uses this to bulk-read and cache lstat data of entire +directories (instead of doing lstat file by file). + core.createObject:: You can set this to 'link', in which case a hardlink followed by a delete of the source are used to make sure that object creation diff --git a/builtin/commit.c b/builtin/commit.c index 714638c5d6..37fc6fcae4 100644 --- a/builtin/commit.c +++ b/builtin/commit.c @@ -1360,6 +1360,7 @@ int cmd_status(int argc, const char **argv, const char *prefix) PATHSPEC_PREFER_FULL, prefix, argv); + enable_fscache(1); read_cache_preload(&s.pathspec); refresh_index(&the_index, REFRESH_QUIET|REFRESH_UNMERGED, &s.pathspec, NULL, NULL); diff --git a/cache.h b/cache.h index 479d2b0952..c5b16d39b3 100644 --- a/cache.h +++ b/cache.h @@ -635,6 +635,8 @@ enum hide_dotfiles_type { }; extern enum hide_dotfiles_type hide_dotfiles; +extern int core_fscache; + enum branch_track { BRANCH_TRACK_UNSPECIFIED = -1, BRANCH_TRACK_NEVER = 0, diff --git a/config.c b/config.c index de856f317a..8c5333d591 100644 --- a/config.c +++ b/config.c @@ -920,6 +920,11 @@ static int git_default_core_config(const char *var, const char *value) return 0; } + if (!strcmp(var, "core.fscache")) { + core_fscache = git_config_bool(var, value); + return 0; + } + /* Add other config variables here and to Documentation/config.txt. */ return 0; } diff --git a/environment.c b/environment.c index 6bbf66f17b..83b7cbc3da 100644 --- a/environment.c +++ b/environment.c @@ -65,6 +65,7 @@ int precomposed_unicode = -1; /* see probe_utf8_pathname_composition() */ struct startup_info *startup_info; unsigned long pack_size_limit_cfg; enum hide_dotfiles_type hide_dotfiles = HIDE_DOTFILES_DOTGITONLY; +int core_fscache; #ifndef PROTECT_HFS_DEFAULT #define PROTECT_HFS_DEFAULT 0 diff --git a/git-compat-util.h b/git-compat-util.h index 3ea5393514..ae34bf2544 100644 --- a/git-compat-util.h +++ b/git-compat-util.h @@ -876,4 +876,19 @@ struct tm *git_gmtime_r(const time_t *, struct tm *); #define mark_as_git_dir(x) /* noop */ #endif +/* + * Enable/disable a read-only cache for file system data on platforms that + * support it. + * + * Implementing a live-cache is complicated and requires special platform + * support (inotify, ReadDirectoryChangesW...). enable_fscache shall be used + * to mark sections of git code that extensively read from the file system + * without modifying anything. Implementations can use this to cache e.g. stat + * data or even file content without the need to synchronize with the file + * system. + */ +#ifndef enable_fscache +#define enable_fscache(x) /* noop */ +#endif + #endif diff --git a/preload-index.c b/preload-index.c index c1fe3a3ef9..c7970d7cb9 100644 --- a/preload-index.c +++ b/preload-index.c @@ -84,6 +84,7 @@ static void preload_index(struct index_state *index, offset = 0; work = DIV_ROUND_UP(index->cache_nr, threads); memset(&data, 0, sizeof(data)); + enable_fscache(1); for (i = 0; i < threads; i++) { struct thread_data *p = data+i; p->index = index; @@ -100,6 +101,7 @@ static void preload_index(struct index_state *index, if (pthread_join(p->pthread, NULL)) die("unable to join threaded lstat"); } + enable_fscache(0); } #endif From d006a7d4ef28e91ffb8e9edffa9614f9706bd58c Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Tue, 1 Oct 2013 12:51:54 +0200 Subject: [PATCH 06/55] Win32: add a cache below mingw's lstat and dirent implementations Checking the work tree status is quite slow on Windows, due to slow lstat emulation (git calls lstat once for each file in the index). Windows operating system APIs seem to be much better at scanning the status of entire directories than checking single files. Add an lstat implementation that uses a cache for lstat data. Cache misses read the entire parent directory and add it to the cache. Subsequent lstat calls for the same directory are served directly from the cache. Also implement opendir / readdir / closedir so that they create and use directory listings in the cache. The cache doesn't track file system changes and doesn't plug into any modifying file APIs, so it has to be explicitly enabled for git functions that don't modify the working copy. Note: in an earlier version of this patch, the cache was always active and tracked file system changes via ReadDirectoryChangesW. However, this was much more complex and had negative impact on the performance of modifying git commands such as 'git checkout'. Signed-off-by: Karsten Blees --- compat/win32/fscache.c | 458 +++++++++++++++++++++++++++++++++++++++++ compat/win32/fscache.h | 10 + config.mak.uname | 4 +- git-compat-util.h | 2 + 4 files changed, 472 insertions(+), 2 deletions(-) create mode 100644 compat/win32/fscache.c create mode 100644 compat/win32/fscache.h diff --git a/compat/win32/fscache.c b/compat/win32/fscache.c new file mode 100644 index 0000000000..5388d64377 --- /dev/null +++ b/compat/win32/fscache.c @@ -0,0 +1,458 @@ +#include "../../cache.h" +#include "../../hashmap.h" +#include "../win32.h" +#include "fscache.h" + +static int initialized; +static volatile long enabled; +static struct hashmap map; +static CRITICAL_SECTION mutex; + +/* + * An entry in the file system cache. Used for both entire directory listings + * and file entries. + */ +struct fsentry { + struct hashmap_entry ent; + mode_t st_mode; + /* Length of name. */ + unsigned short len; + /* + * Name of the entry. For directory listings: relative path of the + * directory, without trailing '/' (empty for cwd()). For file entries: + * name of the file. Typically points to the end of the structure if + * the fsentry is allocated on the heap (see fsentry_alloc), or to a + * local variable if on the stack (see fsentry_init). + */ + const char *name; + /* Pointer to the directory listing, or NULL for the listing itself. */ + struct fsentry *list; + /* Pointer to the next file entry of the list. */ + struct fsentry *next; + + union { + /* Reference count of the directory listing. */ + volatile long refcnt; + struct { + /* More stat members (only used for file entries). */ + off64_t st_size; + time_t st_atime; + time_t st_mtime; + time_t st_ctime; + }; + }; +}; + +/* + * Compares the paths of two fsentry structures for equality. + */ +static int fsentry_cmp(const struct fsentry *fse1, const struct fsentry *fse2) +{ + int res; + if (fse1 == fse2) + return 0; + + /* compare the list parts first */ + if (fse1->list != fse2->list && (res = fsentry_cmp( + fse1->list ? fse1->list : fse1, + fse2->list ? fse2->list : fse2))) + return res; + + /* if list parts are equal, compare len and name */ + if (fse1->len != fse2->len) + return fse1->len - fse2->len; + return strnicmp(fse1->name, fse2->name, fse1->len); +} + +/* + * Calculates the hash code of an fsentry structure's path. + */ +static unsigned int fsentry_hash(const struct fsentry *fse) +{ + unsigned int hash = fse->list ? fse->list->ent.hash : 0; + return hash ^ memihash(fse->name, fse->len); +} + +/* + * Initialize an fsentry structure for use by fsentry_hash and fsentry_cmp. + */ +static void fsentry_init(struct fsentry *fse, struct fsentry *list, + const char *name, size_t len) +{ + fse->list = list; + fse->name = name; + fse->len = len; + hashmap_entry_init(fse, fsentry_hash(fse)); +} + +/* + * Allocate an fsentry structure on the heap. + */ +static struct fsentry *fsentry_alloc(struct fsentry *list, const char *name, + size_t len) +{ + /* overallocate fsentry and copy the name to the end */ + struct fsentry *fse = xmalloc(sizeof(struct fsentry) + len + 1); + char *nm = ((char*) fse) + sizeof(struct fsentry); + memcpy(nm, name, len); + nm[len] = 0; + /* init the rest of the structure */ + fsentry_init(fse, list, nm, len); + fse->next = NULL; + fse->refcnt = 1; + return fse; +} + +/* + * Add a reference to an fsentry. + */ +inline static void fsentry_addref(struct fsentry *fse) +{ + if (fse->list) + fse = fse->list; + + InterlockedIncrement(&(fse->refcnt)); +} + +/* + * Release the reference to an fsentry, frees the memory if its the last ref. + */ +static void fsentry_release(struct fsentry *fse) +{ + if (fse->list) + fse = fse->list; + + if (InterlockedDecrement(&(fse->refcnt))) + return; + + while (fse) { + struct fsentry *next = fse->next; + free(fse); + fse = next; + } +} + +/* + * Allocate and initialize an fsentry from a WIN32_FIND_DATA structure. + */ +static struct fsentry *fseentry_create_entry(struct fsentry *list, + const WIN32_FIND_DATAW *fdata) +{ + char buf[MAX_PATH * 3]; + int len; + struct fsentry *fse; + len = xwcstoutf(buf, fdata->cFileName, ARRAY_SIZE(buf)); + + fse = fsentry_alloc(list, buf, len); + + fse->st_mode = file_attr_to_st_mode(fdata->dwFileAttributes); + fse->st_size = (((off64_t) (fdata->nFileSizeHigh)) << 32) + | fdata->nFileSizeLow; + fse->st_atime = filetime_to_time_t(&(fdata->ftLastAccessTime)); + fse->st_mtime = filetime_to_time_t(&(fdata->ftLastWriteTime)); + fse->st_ctime = filetime_to_time_t(&(fdata->ftCreationTime)); + + return fse; +} + +/* + * Create an fsentry-based directory listing (similar to opendir / readdir). + * Dir should not contain trailing '/'. Use an empty string for the current + * directory (not "."!). + */ +static struct fsentry *fsentry_create_list(const struct fsentry *dir) +{ + wchar_t pattern[MAX_PATH + 2]; /* + 2 for '/' '*' */ + WIN32_FIND_DATAW fdata; + HANDLE h; + int wlen; + struct fsentry *list, **phead; + DWORD err; + + /* convert name to UTF-16 and check length < MAX_PATH */ + if ((wlen = xutftowcsn(pattern, dir->name, MAX_PATH, dir->len)) < 0) { + if (errno == ERANGE) + errno = ENAMETOOLONG; + return NULL; + } + + /* append optional '/' and wildcard '*' */ + if (wlen) + pattern[wlen++] = '/'; + pattern[wlen++] = '*'; + pattern[wlen] = 0; + + /* open find handle */ + h = FindFirstFileW(pattern, &fdata); + if (h == INVALID_HANDLE_VALUE) { + err = GetLastError(); + errno = (err == ERROR_DIRECTORY) ? ENOTDIR : err_win_to_posix(err); + return NULL; + } + + /* allocate object to hold directory listing */ + list = fsentry_alloc(NULL, dir->name, dir->len); + + /* walk directory and build linked list of fsentry structures */ + phead = &list->next; + do { + *phead = fseentry_create_entry(list, &fdata); + phead = &(*phead)->next; + } while (FindNextFileW(h, &fdata)); + + /* remember result of last FindNextFile, then close find handle */ + err = GetLastError(); + FindClose(h); + + /* return the list if we've got all the files */ + if (err == ERROR_NO_MORE_FILES) + return list; + + /* otherwise free the list and return error */ + fsentry_release(list); + errno = err_win_to_posix(err); + return NULL; +} + +/* + * Adds a directory listing to the cache. + */ +static void fscache_add(struct fsentry *fse) +{ + if (fse->list) + fse = fse->list; + + for (; fse; fse = fse->next) + hashmap_add(&map, fse); +} + +/* + * Removes a directory listing from the cache. + */ +static void fscache_remove(struct fsentry *fse) +{ + if (fse->list) + fse = fse->list; + + for (; fse; fse = fse->next) + hashmap_remove(&map, fse, NULL); +} + +/* + * Clears the cache. + */ +static void fscache_clear() +{ + struct hashmap_iter iter; + struct fsentry *fse; + while ((fse = hashmap_iter_first(&map, &iter))) { + fscache_remove(fse); + fsentry_release(fse); + } +} + +/* + * Checks if the cache is enabled for the given path. + */ +static inline int fscache_enabled(const char *path) +{ + return enabled > 0 && !is_absolute_path(path); +} + +/* + * Looks up or creates a cache entry for the specified key. + */ +static struct fsentry *fscache_get(struct fsentry *key) +{ + struct fsentry *fse; + + EnterCriticalSection(&mutex); + /* check if entry is in cache */ + fse = hashmap_get(&map, key, NULL); + if (fse) { + fsentry_addref(fse); + LeaveCriticalSection(&mutex); + return fse; + } + /* if looking for a file, check if directory listing is in cache */ + if (!fse && key->list) { + fse = hashmap_get(&map, key->list, NULL); + if (fse) { + LeaveCriticalSection(&mutex); + /* dir entry without file entry -> file doesn't exist */ + errno = ENOENT; + return NULL; + } + } + + /* create the directory listing (outside mutex!) */ + LeaveCriticalSection(&mutex); + fse = fsentry_create_list(key->list ? key->list : key); + if (!fse) + return NULL; + + EnterCriticalSection(&mutex); + /* add directory listing if it hasn't been added by some other thread */ + if (!hashmap_get(&map, key, NULL)) + fscache_add(fse); + + /* lookup file entry if requested (fse already points to directory) */ + if (key->list) + fse = hashmap_get(&map, key, NULL); + + /* return entry or ENOENT */ + if (fse) + fsentry_addref(fse); + else + errno = ENOENT; + + LeaveCriticalSection(&mutex); + return fse; +} + +/* + * Enables or disables the cache. Note that the cache is read-only, changes to + * the working directory are NOT reflected in the cache while enabled. + */ +int fscache_enable(int enable) +{ + int result; + + if (!initialized) { + /* allow the cache to be disabled entirely */ + if (!core_fscache) + return 0; + + InitializeCriticalSection(&mutex); + hashmap_init(&map, (hashmap_cmp_fn) fsentry_cmp, 0); + initialized = 1; + } + + result = enable ? InterlockedIncrement(&enabled) + : InterlockedDecrement(&enabled); + + if (enable && result == 1) { + /* redirect opendir and lstat to the fscache implementations */ + opendir = fscache_opendir; + lstat = fscache_lstat; + } else if (!enable && !result) { + /* reset opendir and lstat to the original implementations */ + opendir = dirent_opendir; + lstat = mingw_lstat; + EnterCriticalSection(&mutex); + fscache_clear(); + LeaveCriticalSection(&mutex); + } + return result; +} + +/* + * Lstat replacement, uses the cache if enabled, otherwise redirects to + * mingw_lstat. + */ +int fscache_lstat(const char *filename, struct stat *st) +{ + int dirlen, base, len; + struct fsentry key[2], *fse; + + if (!fscache_enabled(filename)) + return mingw_lstat(filename, st); + + /* split filename into path + name */ + len = strlen(filename); + if (len && is_dir_sep(filename[len - 1])) + len--; + base = len; + while (base && !is_dir_sep(filename[base - 1])) + base--; + dirlen = base ? base - 1 : 0; + + /* lookup entry for path + name in cache */ + fsentry_init(key, NULL, filename, dirlen); + fsentry_init(key + 1, key, filename + base, len - base); + fse = fscache_get(key + 1); + if (!fse) + return -1; + + /* copy stat data */ + st->st_ino = 0; + st->st_gid = 0; + st->st_uid = 0; + st->st_dev = 0; + st->st_rdev = 0; + st->st_nlink = 1; + st->st_mode = fse->st_mode; + st->st_size = fse->st_size; + st->st_atime = fse->st_atime; + st->st_mtime = fse->st_mtime; + st->st_ctime = fse->st_ctime; + + /* don't forget to release fsentry */ + fsentry_release(fse); + return 0; +} + +typedef struct fscache_DIR { + struct DIR base_dir; /* extend base struct DIR */ + struct fsentry *pfsentry; + struct dirent dirent; +} fscache_DIR; + +/* + * Readdir replacement. + */ +static struct dirent *fscache_readdir(DIR *base_dir) +{ + fscache_DIR *dir = (fscache_DIR*) base_dir; + struct fsentry *next = dir->pfsentry->next; + if (!next) + return NULL; + dir->pfsentry = next; + dir->dirent.d_type = S_ISDIR(next->st_mode) ? DT_DIR : DT_REG; + dir->dirent.d_name = (char*) next->name; + return &(dir->dirent); +} + +/* + * Closedir replacement. + */ +static int fscache_closedir(DIR *base_dir) +{ + fscache_DIR *dir = (fscache_DIR*) base_dir; + fsentry_release(dir->pfsentry); + free(dir); + return 0; +} + +/* + * Opendir replacement, uses a directory listing from the cache if enabled, + * otherwise calls original dirent implementation. + */ +DIR *fscache_opendir(const char *dirname) +{ + struct fsentry key, *list; + fscache_DIR *dir; + int len; + + if (!fscache_enabled(dirname)) + return dirent_opendir(dirname); + + /* prepare name (strip trailing '/', replace '.') */ + len = strlen(dirname); + if ((len == 1 && dirname[0] == '.') || + (len && is_dir_sep(dirname[len - 1]))) + len--; + + /* get directory listing from cache */ + fsentry_init(&key, NULL, dirname, len); + list = fscache_get(&key); + if (!list) + return NULL; + + /* alloc and return DIR structure */ + dir = (fscache_DIR*) xmalloc(sizeof(fscache_DIR)); + dir->base_dir.preaddir = fscache_readdir; + dir->base_dir.pclosedir = fscache_closedir; + dir->pfsentry = list; + return (DIR*) dir; +} diff --git a/compat/win32/fscache.h b/compat/win32/fscache.h new file mode 100644 index 0000000000..ed518b422d --- /dev/null +++ b/compat/win32/fscache.h @@ -0,0 +1,10 @@ +#ifndef FSCACHE_H +#define FSCACHE_H + +int fscache_enable(int enable); +#define enable_fscache(x) fscache_enable(x) + +DIR *fscache_opendir(const char *dir); +int fscache_lstat(const char *file_name, struct stat *buf); + +#endif diff --git a/config.mak.uname b/config.mak.uname index b64b63c34c..2dae474513 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -369,7 +369,7 @@ ifeq ($(uname_S),Windows) BASIC_CFLAGS = -nologo -I. -I../zlib -Icompat/vcbuild -Icompat/vcbuild/include -DWIN32 -D_CONSOLE -DHAVE_STRING_H -D_CRT_SECURE_NO_WARNINGS -D_CRT_NONSTDC_NO_DEPRECATE COMPAT_OBJS = compat/msvc.o compat/winansi.o \ compat/win32/pthread.o compat/win32/syslog.o \ - compat/win32/dirent.o + compat/win32/dirent.o compat/win32/fscache.o COMPAT_CFLAGS = -D__USE_MINGW_ACCESS -DNOGDI -DHAVE_STRING_H -Icompat -Icompat/regex -Icompat/win32 -DSTRIP_EXTENSION=\".exe\" BASIC_LDFLAGS = -IGNORE:4217 -IGNORE:4049 -NOLOGO -SUBSYSTEM:CONSOLE EXTLIBS = user32.lib advapi32.lib shell32.lib wininet.lib ws2_32.lib invalidcontinue.obj @@ -516,7 +516,7 @@ ifneq (,$(findstring MINGW,$(uname_S))) COMPAT_CFLAGS += -DSTRIP_EXTENSION=\".exe\" COMPAT_OBJS += compat/mingw.o compat/winansi.o \ compat/win32/pthread.o compat/win32/syslog.o \ - compat/win32/dirent.o + compat/win32/dirent.o compat/win32/fscache.o BASIC_CFLAGS += -DPROTECT_NTFS_DEFAULT=1 BASIC_LDFLAGS += -Wl,--large-address-aware EXTLIBS += -lws2_32 diff --git a/git-compat-util.h b/git-compat-util.h index ae34bf2544..6373d93679 100644 --- a/git-compat-util.h +++ b/git-compat-util.h @@ -131,8 +131,10 @@ #if defined(__MINGW32__) /* pull in Windows compatibility stuff */ #include "compat/mingw.h" +#include "compat/win32/fscache.h" #elif defined(_MSC_VER) #include "compat/msvc.h" +#include "compat/win32/fscache.h" #else #include #include From 2974ef96e8e59343adfb3061d505f58aeb5458c1 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Tue, 24 Jun 2014 13:22:35 +0200 Subject: [PATCH 07/55] fscache: load directories only once If multiple threads access a directory that is not yet in the cache, the directory will be loaded by each thread. Only one of the results is added to the cache, all others are leaked. This wastes performance and memory. On cache miss, add a future object to the cache to indicate that the directory is currently being loaded. Subsequent threads register themselves with the future object and wait. When the first thread has loaded the directory, it replaces the future object with the result and notifies waiting threads. Signed-off-by: Karsten Blees --- compat/win32/fscache.c | 67 +++++++++++++++++++++++++++++++++++------- 1 file changed, 57 insertions(+), 10 deletions(-) diff --git a/compat/win32/fscache.c b/compat/win32/fscache.c index 5388d64377..05a2fa9663 100644 --- a/compat/win32/fscache.c +++ b/compat/win32/fscache.c @@ -33,6 +33,8 @@ struct fsentry { union { /* Reference count of the directory listing. */ volatile long refcnt; + /* Handle to wait on the loading thread. */ + HANDLE hwait; struct { /* More stat members (only used for file entries). */ off64_t st_size; @@ -259,16 +261,43 @@ static inline int fscache_enabled(const char *path) return enabled > 0 && !is_absolute_path(path); } +/* + * Looks up a cache entry, waits if its being loaded by another thread. + * The mutex must be owned by the calling thread. + */ +static struct fsentry *fscache_get_wait(struct fsentry *key) +{ + struct fsentry *fse = hashmap_get(&map, key, NULL); + + /* return if its a 'real' entry (future entries have refcnt == 0) */ + if (!fse || fse->list || fse->refcnt) + return fse; + + /* create an event and link our key to the future entry */ + key->hwait = CreateEvent(NULL, TRUE, FALSE, NULL); + key->next = fse->next; + fse->next = key; + + /* wait for the loading thread to signal us */ + LeaveCriticalSection(&mutex); + WaitForSingleObject(key->hwait, INFINITE); + CloseHandle(key->hwait); + EnterCriticalSection(&mutex); + + /* repeat cache lookup */ + return hashmap_get(&map, key, NULL); +} + /* * Looks up or creates a cache entry for the specified key. */ static struct fsentry *fscache_get(struct fsentry *key) { - struct fsentry *fse; + struct fsentry *fse, *future, *waiter; EnterCriticalSection(&mutex); /* check if entry is in cache */ - fse = hashmap_get(&map, key, NULL); + fse = fscache_get_wait(key); if (fse) { fsentry_addref(fse); LeaveCriticalSection(&mutex); @@ -276,7 +305,7 @@ static struct fsentry *fscache_get(struct fsentry *key) } /* if looking for a file, check if directory listing is in cache */ if (!fse && key->list) { - fse = hashmap_get(&map, key->list, NULL); + fse = fscache_get_wait(key->list); if (fse) { LeaveCriticalSection(&mutex); /* dir entry without file entry -> file doesn't exist */ @@ -285,16 +314,34 @@ static struct fsentry *fscache_get(struct fsentry *key) } } + /* add future entry to indicate that we're loading it */ + future = key->list ? key->list : key; + future->next = NULL; + future->refcnt = 0; + hashmap_add(&map, future); + /* create the directory listing (outside mutex!) */ LeaveCriticalSection(&mutex); - fse = fsentry_create_list(key->list ? key->list : key); - if (!fse) - return NULL; - + fse = fsentry_create_list(future); EnterCriticalSection(&mutex); - /* add directory listing if it hasn't been added by some other thread */ - if (!hashmap_get(&map, key, NULL)) - fscache_add(fse); + + /* remove future entry and signal waiting threads */ + hashmap_remove(&map, future, NULL); + waiter = future->next; + while (waiter) { + HANDLE h = waiter->hwait; + waiter = waiter->next; + SetEvent(h); + } + + /* leave on error (errno set by fsentry_create_list) */ + if (!fse) { + LeaveCriticalSection(&mutex); + return NULL; + } + + /* add directory listing to the cache */ + fscache_add(fse); /* lookup file entry if requested (fse already points to directory) */ if (key->list) From 6814848aa2fc93984170c3361747f7337cd43121 Mon Sep 17 00:00:00 2001 From: Doug Kelly Date: Wed, 8 Jan 2014 20:28:15 -0600 Subject: [PATCH 08/55] Add a test demonstrating a problem with long submodule paths Signed-off-by: Johannes Schindelin --- t/t7410-submodule-long-path.sh | 101 +++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100755 t/t7410-submodule-long-path.sh diff --git a/t/t7410-submodule-long-path.sh b/t/t7410-submodule-long-path.sh new file mode 100755 index 0000000000..ee4f3c139e --- /dev/null +++ b/t/t7410-submodule-long-path.sh @@ -0,0 +1,101 @@ +#!/bin/sh +# +# Copyright (c) 2013 Doug Kelly +# + +test_description='Test submodules with a path near PATH_MAX + +This test verifies that "git submodule" initialization, update and clones work, including with recursive submodules and paths approaching PATH_MAX (260 characters on Windows) +' + +TEST_NO_CREATE_REPO=1 +. ./test-lib.sh + +longpath="" +for (( i=0; i<4; i++ )); do + longpath="0123456789abcdefghijklmnopqrstuvwxyz$longpath" +done +# Pick a substring maximum of 90 characters +# This should be good, since we'll add on a lot for temp directories +export longpath=${longpath:0:90} + +test_expect_failure 'submodule with a long path' ' + git init --bare remote && + test_create_repo bundle1 && + ( + cd bundle1 && + test_commit "shoot" && + git rev-parse --verify HEAD >../expect + ) && + mkdir home && + ( + cd home && + git clone ../remote test && + cd test && + git submodule add ../bundle1 $longpath && + test_commit "sogood" && + ( + cd $longpath && + git rev-parse --verify HEAD >actual && + test_cmp ../../../expect actual + ) && + git push origin master + ) && + mkdir home2 && + ( + cd home2 && + git clone ../remote test && + cd test && + git checkout master && + git submodule update --init && + ( + cd $longpath && + git rev-parse --verify HEAD >actual && + test_cmp ../../../expect actual + ) + ) +' + +test_expect_failure 'recursive submodule with a long path' ' + git init --bare super && + test_create_repo child && + ( + cd child && + test_commit "shoot" && + git rev-parse --verify HEAD >../expect + ) && + test_create_repo parent && + ( + cd parent && + git submodule add ../child $longpath && + test_commit "aim" + ) && + mkdir home3 && + ( + cd home3 && + git clone ../super test && + cd test && + git submodule add ../parent foo && + git submodule update --init --recursive + test_commit "sogood" && + ( + cd foo/$longpath && + git rev-parse --verify HEAD >actual && + test_cmp ../../../../expect actual + ) && + git push origin master + ) && + mkdir home4 && + ( + cd home4 && + git clone ../super test --recursive && + ( + cd test/foo/$longpath && + git rev-parse --verify HEAD >actual && + test_cmp ../../../../expect actual + ) + ) +' +unset longpath + +test_done From 1eb6ed688b5cae3cc39576af376c8ada71ab5c37 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Tue, 11 Feb 2014 20:05:00 +0100 Subject: [PATCH 09/55] Win32: support long paths Windows paths are typically limited to MAX_PATH = 260 characters, even though the underlying NTFS file system supports paths up to 32,767 chars. This limitation is also evident in Windows Explorer, cmd.exe and many other applications (including IDEs). Particularly annoying is that most Windows APIs return bogus error codes if a relative path only barely exceeds MAX_PATH in conjunction with the current directory, e.g. ERROR_PATH_NOT_FOUND / ENOENT instead of the infinitely more helpful ERROR_FILENAME_EXCED_RANGE / ENAMETOOLONG. Many Windows wide char APIs support longer than MAX_PATH paths through the file namespace prefix ('\\?\' or '\\?\UNC\') followed by an absolute path. Notable exceptions include functions dealing with executables and the current directory (CreateProcess, LoadLibrary, Get/SetCurrentDirectory) as well as the entire shell API (ShellExecute, SHGetSpecialFolderPath...). Introduce a handle_long_path function to check the length of a specified path properly (and fail with ENAMETOOLONG), and to optionally expand long paths using the '\\?\' file namespace prefix. Short paths will not be modified, so we don't need to worry about device names (NUL, CON, AUX). Contrary to MSDN docs, the GetFullPathNameW function doesn't seem to be limited to MAX_PATH (at least not on Win7), so we can use it to do the heavy lifting of the conversion (translate '/' to '\', eliminate '.' and '..', and make an absolute path). Add long path error checking to xutftowcs_path for APIs with hard MAX_PATH limit. Add a new MAX_LONG_PATH constant and xutftowcs_long_path function for APIs that support long paths. While improved error checking is always active, long paths support must be explicitly enabled via 'core.longpaths' option. This is to prevent end users to shoot themselves in the foot by checking out files that Windows Explorer, cmd/bash or their favorite IDE cannot handle. Test suite: Test the case is when the full pathname length of a dir is close to 260 (MAX_PATH). Bug report and an original reproducer by Andrey Rogozhnikov: https://github.com/msysgit/git/pull/122#issuecomment-43604199 Thanks-to: Martin W. Kirst Thanks-to: Doug Kelly Signed-off-by: Karsten Blees Original-test-by: Andrey Rogozhnikov Signed-off-by: Stepan Kasal Signed-off-by: Johannes Schindelin --- Documentation/config.txt | 7 ++ cache.h | 2 + compat/mingw.c | 136 ++++++++++++++++++++++++++------- compat/mingw.h | 77 +++++++++++++++++-- compat/win32/dirent.c | 14 ++-- compat/win32/fscache.c | 17 +++-- config.c | 5 ++ environment.c | 1 + t/t2025-checkout-long-paths.sh | 99 ++++++++++++++++++++++++ t/t7410-submodule-long-path.sh | 24 +++--- 10 files changed, 325 insertions(+), 57 deletions(-) create mode 100755 t/t2025-checkout-long-paths.sh diff --git a/Documentation/config.txt b/Documentation/config.txt index 100f85c21d..fe3533b3d8 100644 --- a/Documentation/config.txt +++ b/Documentation/config.txt @@ -712,6 +712,13 @@ core.fscache:: Git for Windows uses this to bulk-read and cache lstat data of entire directories (instead of doing lstat file by file). +core.longpaths:: + Enable long path (> 260) support for builtin commands in Git for + Windows. This is disabled by default, as long paths are not supported + by Windows Explorer, cmd.exe and the Git for Windows tool chain + (msys, bash, tcl, perl...). Only enable this if you know what you're + doing and are prepared to live with a few quirks. + core.createObject:: You can set this to 'link', in which case a hardlink followed by a delete of the source are used to make sure that object creation diff --git a/cache.h b/cache.h index c5b16d39b3..c1b24e67c0 100644 --- a/cache.h +++ b/cache.h @@ -637,6 +637,8 @@ extern enum hide_dotfiles_type hide_dotfiles; extern int core_fscache; +extern int core_long_paths; + enum branch_track { BRANCH_TRACK_UNSPECIFIED = -1, BRANCH_TRACK_NEVER = 0, diff --git a/compat/mingw.c b/compat/mingw.c index 47dae5a5ac..5049d37433 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -204,8 +204,8 @@ static int ask_yes_no_if_possible(const char *format, ...) int mingw_unlink(const char *pathname) { int ret, tries = 0; - wchar_t wpathname[MAX_PATH]; - if (xutftowcs_path(wpathname, pathname) < 0) + wchar_t wpathname[MAX_LONG_PATH]; + if (xutftowcs_long_path(wpathname, pathname) < 0) return -1; /* read-only files cannot be removed */ @@ -234,7 +234,7 @@ static int is_dir_empty(const wchar_t *wpath) { WIN32_FIND_DATAW findbuf; HANDLE handle; - wchar_t wbuf[MAX_PATH + 2]; + wchar_t wbuf[MAX_LONG_PATH + 2]; wcscpy(wbuf, wpath); wcscat(wbuf, L"\\*"); handle = FindFirstFileW(wbuf, &findbuf); @@ -255,8 +255,8 @@ static int is_dir_empty(const wchar_t *wpath) int mingw_rmdir(const char *pathname) { int ret, tries = 0; - wchar_t wpathname[MAX_PATH]; - if (xutftowcs_path(wpathname, pathname) < 0) + wchar_t wpathname[MAX_LONG_PATH]; + if (xutftowcs_long_path(wpathname, pathname) < 0) return -1; while ((ret = _wrmdir(wpathname)) == -1 && tries < ARRAY_SIZE(delay)) { @@ -296,9 +296,9 @@ static int make_hidden(const wchar_t *path) void mingw_mark_as_git_dir(const char *dir) { - wchar_t wdir[MAX_PATH]; + wchar_t wdir[MAX_LONG_PATH]; if (hide_dotfiles != HIDE_DOTFILES_FALSE && !is_bare_repository()) - if (xutftowcs_path(wdir, dir) < 0 || make_hidden(wdir)) + if (xutftowcs_long_path(wdir, dir) < 0 || make_hidden(wdir)) warning("Failed to make '%s' hidden", dir); git_config_set("core.hideDotFiles", hide_dotfiles == HIDE_DOTFILES_FALSE ? "false" : @@ -309,9 +309,12 @@ void mingw_mark_as_git_dir(const char *dir) int mingw_mkdir(const char *path, int mode) { int ret; - wchar_t wpath[MAX_PATH]; - if (xutftowcs_path(wpath, path) < 0) + wchar_t wpath[MAX_LONG_PATH]; + /* CreateDirectoryW path limit is 248 (MAX_PATH - 8.3 file name) */ + if (xutftowcs_path_ex(wpath, path, MAX_LONG_PATH, -1, 248, + core_long_paths) < 0) return -1; + ret = _wmkdir(wpath); if (!ret && hide_dotfiles == HIDE_DOTFILES_TRUE) { /* @@ -331,7 +334,7 @@ int mingw_open (const char *filename, int oflags, ...) va_list args; unsigned mode; int fd; - wchar_t wfilename[MAX_PATH]; + wchar_t wfilename[MAX_LONG_PATH]; va_start(args, oflags); mode = va_arg(args, int); @@ -340,7 +343,7 @@ int mingw_open (const char *filename, int oflags, ...) if (filename && !strcmp(filename, "/dev/null")) filename = "nul"; - if (xutftowcs_path(wfilename, filename) < 0) + if (xutftowcs_long_path(wfilename, filename) < 0) return -1; fd = _wopen(wfilename, oflags, mode); @@ -393,13 +396,13 @@ FILE *mingw_fopen (const char *filename, const char *otype) { int hide = 0; FILE *file; - wchar_t wfilename[MAX_PATH], wotype[4]; + wchar_t wfilename[MAX_LONG_PATH], wotype[4]; if (hide_dotfiles == HIDE_DOTFILES_TRUE && basename((char*)filename)[0] == '.') hide = access(filename, F_OK); if (filename && !strcmp(filename, "/dev/null")) filename = "nul"; - if (xutftowcs_path(wfilename, filename) < 0 || + if (xutftowcs_long_path(wfilename, filename) < 0 || xutftowcs(wotype, otype, ARRAY_SIZE(wotype)) < 0) return NULL; file = _wfopen(wfilename, wotype); @@ -412,13 +415,13 @@ FILE *mingw_freopen (const char *filename, const char *otype, FILE *stream) { int hide = 0; FILE *file; - wchar_t wfilename[MAX_PATH], wotype[4]; + wchar_t wfilename[MAX_LONG_PATH], wotype[4]; if (hide_dotfiles == HIDE_DOTFILES_TRUE && basename((char*)filename)[0] == '.') hide = access(filename, F_OK); if (filename && !strcmp(filename, "/dev/null")) filename = "nul"; - if (xutftowcs_path(wfilename, filename) < 0 || + if (xutftowcs_long_path(wfilename, filename) < 0 || xutftowcs(wotype, otype, ARRAY_SIZE(wotype)) < 0) return NULL; file = _wfreopen(wfilename, wotype, stream); @@ -451,25 +454,32 @@ int mingw_fflush(FILE *stream) int mingw_access(const char *filename, int mode) { - wchar_t wfilename[MAX_PATH]; - if (xutftowcs_path(wfilename, filename) < 0) + wchar_t wfilename[MAX_LONG_PATH]; + if (xutftowcs_long_path(wfilename, filename) < 0) return -1; /* X_OK is not supported by the MSVCRT version */ return _waccess(wfilename, mode & ~X_OK); } +/* cached length of current directory for handle_long_path */ +static int current_directory_len = 0; + int mingw_chdir(const char *dirname) { + int result; wchar_t wdirname[MAX_PATH]; + /* SetCurrentDirectoryW doesn't support long paths */ if (xutftowcs_path(wdirname, dirname) < 0) return -1; - return _wchdir(wdirname); + result = _wchdir(wdirname); + current_directory_len = GetCurrentDirectoryW(0, NULL); + return result; } int mingw_chmod(const char *filename, int mode) { - wchar_t wfilename[MAX_PATH]; - if (xutftowcs_path(wfilename, filename) < 0) + wchar_t wfilename[MAX_LONG_PATH]; + if (xutftowcs_long_path(wfilename, filename) < 0) return -1; return _wchmod(wfilename, mode); } @@ -484,8 +494,8 @@ int mingw_chmod(const char *filename, int mode) static int do_lstat(int follow, const char *file_name, struct stat *buf) { WIN32_FILE_ATTRIBUTE_DATA fdata; - wchar_t wfilename[MAX_PATH]; - if (xutftowcs_path(wfilename, file_name) < 0) + wchar_t wfilename[MAX_LONG_PATH]; + if (xutftowcs_long_path(wfilename, file_name) < 0) return -1; if (GetFileAttributesExW(wfilename, GetFileExInfoStandard, &fdata)) { @@ -628,8 +638,8 @@ int mingw_utime (const char *file_name, const struct utimbuf *times) FILETIME mft, aft; int fh, rc; DWORD attrs; - wchar_t wfilename[MAX_PATH]; - if (xutftowcs_path(wfilename, file_name) < 0) + wchar_t wfilename[MAX_LONG_PATH]; + if (xutftowcs_long_path(wfilename, file_name) < 0) return -1; /* must have write permission */ @@ -677,6 +687,7 @@ unsigned int sleep (unsigned int seconds) char *mingw_mktemp(char *template) { wchar_t wtemplate[MAX_PATH]; + /* we need to return the path, thus no long paths here! */ if (xutftowcs_path(wtemplate, template) < 0) return NULL; if (!_wmktemp(wtemplate)) @@ -1030,6 +1041,7 @@ static pid_t mingw_spawnve_fd(const char *cmd, const char **argv, char **deltaen si.hStdOutput = winansi_get_osfhandle(fhout); si.hStdError = winansi_get_osfhandle(fherr); + /* executables and the current directory don't support long paths */ if (xutftowcs_path(wcmd, cmd) < 0) return -1; if (dir && xutftowcs_path(wdir, dir) < 0) @@ -1605,8 +1617,9 @@ int mingw_rename(const char *pold, const char *pnew) { DWORD attrs, gle; int tries = 0; - wchar_t wpold[MAX_PATH], wpnew[MAX_PATH]; - if (xutftowcs_path(wpold, pold) < 0 || xutftowcs_path(wpnew, pnew) < 0) + wchar_t wpold[MAX_LONG_PATH], wpnew[MAX_LONG_PATH]; + if (xutftowcs_long_path(wpold, pold) < 0 || + xutftowcs_long_path(wpnew, pnew) < 0) return -1; /* @@ -1882,9 +1895,9 @@ int link(const char *oldpath, const char *newpath) { typedef BOOL (WINAPI *T)(LPCWSTR, LPCWSTR, LPSECURITY_ATTRIBUTES); static T create_hard_link = NULL; - wchar_t woldpath[MAX_PATH], wnewpath[MAX_PATH]; - if (xutftowcs_path(woldpath, oldpath) < 0 || - xutftowcs_path(wnewpath, newpath) < 0) + wchar_t woldpath[MAX_LONG_PATH], wnewpath[MAX_LONG_PATH]; + if (xutftowcs_long_path(woldpath, oldpath) < 0 || + xutftowcs_long_path(wnewpath, newpath) < 0) return -1; if (!create_hard_link) { @@ -2065,6 +2078,68 @@ int xwcstoutf(char *utf, const wchar_t *wcs, size_t utflen) return -1; } +int handle_long_path(wchar_t *path, int len, int max_path, int expand) +{ + int result; + wchar_t buf[MAX_LONG_PATH]; + + /* + * we don't need special handling if path is relative to the current + * directory, and current directory + path don't exceed the desired + * max_path limit. This should cover > 99 % of cases with minimal + * performance impact (git almost always uses relative paths). + */ + if ((len < 2 || (!is_dir_sep(path[0]) && path[1] != ':')) && + (current_directory_len + len < max_path)) + return len; + + /* + * handle everything else: + * - absolute paths: "C:\dir\file" + * - absolute UNC paths: "\\server\share\dir\file" + * - absolute paths on current drive: "\dir\file" + * - relative paths on other drive: "X:file" + * - prefixed paths: "\\?\...", "\\.\..." + */ + + /* convert to absolute path using GetFullPathNameW */ + result = GetFullPathNameW(path, MAX_LONG_PATH, buf, NULL); + if (!result) { + errno = err_win_to_posix(GetLastError()); + return -1; + } + + /* + * return absolute path if it fits within max_path (even if + * "cwd + path" doesn't due to '..' components) + */ + if (result < max_path) { + wcscpy(path, buf); + return result; + } + + /* error out if we shouldn't expand the path or buf is too small */ + if (!expand || result >= MAX_LONG_PATH - 6) { + errno = ENAMETOOLONG; + return -1; + } + + /* prefix full path with "\\?\" or "\\?\UNC\" */ + if (buf[0] == '\\') { + /* ...unless already prefixed */ + if (buf[1] == '\\' && (buf[2] == '?' || buf[2] == '.')) + return len; + + wcscpy(path, L"\\\\?\\UNC\\"); + wcscpy(path + 8, buf + 2); + return result + 6; + } else { + wcscpy(path, L"\\\\?\\"); + wcscpy(path + 4, buf); + return result + 4; + } +} + /* * Disable MSVCRT command line wildcard expansion (__getmainargs called from * mingw startup code, see init.c in mingw runtime). @@ -2168,4 +2243,7 @@ void mingw_startup() /* initialize Unicode console */ winansi_init(); + + /* init length of current directory for handle_long_path */ + current_directory_len = GetCurrentDirectoryW(0, NULL); } diff --git a/compat/mingw.h b/compat/mingw.h index ca9358e93c..b614ac5af4 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -384,6 +384,42 @@ void mingw_open_html(const char *path); void mingw_mark_as_git_dir(const char *dir); #define mark_as_git_dir mingw_mark_as_git_dir +/** + * Max length of long paths (exceeding MAX_PATH). The actual maximum supported + * by NTFS is 32,767 (* sizeof(wchar_t)), but we choose an arbitrary smaller + * value to limit required stack memory. + */ +#define MAX_LONG_PATH 4096 + +/** + * Handles paths that would exceed the MAX_PATH limit of Windows Unicode APIs. + * + * With expand == false, the function checks for over-long paths and fails + * with ENAMETOOLONG. The path parameter is not modified, except if cwd + path + * exceeds max_path, but the resulting absolute path doesn't (e.g. due to + * eliminating '..' components). The path parameter must point to a buffer + * of max_path wide characters. + * + * With expand == true, an over-long path is automatically converted in place + * to an absolute path prefixed with '\\?\', and the new length is returned. + * The path parameter must point to a buffer of MAX_LONG_PATH wide characters. + * + * Parameters: + * path: path to check and / or convert + * len: size of path on input (number of wide chars without \0) + * max_path: max short path length to check (usually MAX_PATH = 260, but just + * 248 for CreateDirectoryW) + * expand: false to only check the length, true to expand the path to a + * '\\?\'-prefixed absolute path + * + * Return: + * length of the resulting path, or -1 on failure + * + * Errors: + * ENAMETOOLONG if path is too long + */ +int handle_long_path(wchar_t *path, int len, int max_path, int expand); + /** * Converts UTF-8 encoded string to UTF-16LE. * @@ -441,17 +477,48 @@ static inline int xutftowcs(wchar_t *wcs, const char *utf, size_t wcslen) return xutftowcsn(wcs, utf, wcslen, -1); } +/** + * Simplified file system specific wrapper of xutftowcsn and handle_long_path. + * Converts ERANGE to ENAMETOOLONG. If expand is true, wcs must be at least + * MAX_LONG_PATH wide chars (see handle_long_path). + */ +static inline int xutftowcs_path_ex(wchar_t *wcs, const char *utf, + size_t wcslen, int utflen, int max_path, int expand) +{ + int result = xutftowcsn(wcs, utf, wcslen, utflen); + if (result < 0 && errno == ERANGE) + errno = ENAMETOOLONG; + if (result >= 0) + result = handle_long_path(wcs, result, max_path, expand); + return result; +} + /** * Simplified file system specific variant of xutftowcsn, assumes output * buffer size is MAX_PATH wide chars and input string is \0-terminated, - * fails with ENAMETOOLONG if input string is too long. + * fails with ENAMETOOLONG if input string is too long. Typically used for + * Windows APIs that don't support long paths, e.g. SetCurrentDirectory, + * LoadLibrary, CreateProcess... */ static inline int xutftowcs_path(wchar_t *wcs, const char *utf) { - int result = xutftowcsn(wcs, utf, MAX_PATH, -1); - if (result < 0 && errno == ERANGE) - errno = ENAMETOOLONG; - return result; + return xutftowcs_path_ex(wcs, utf, MAX_PATH, -1, MAX_PATH, 0); +} + +/* need to re-declare that here as mingw.h is included before cache.h */ +extern int core_long_paths; + +/** + * Simplified file system specific variant of xutftowcsn for Windows APIs + * that support long paths via '\\?\'-prefix, assumes output buffer size is + * MAX_LONG_PATH wide chars, fails with ENAMETOOLONG if input string is too + * long. The 'core.longpaths' git-config option controls whether the path + * is only checked or expanded to a long path. + */ +static inline int xutftowcs_long_path(wchar_t *wcs, const char *utf) +{ + return xutftowcs_path_ex(wcs, utf, MAX_LONG_PATH, -1, MAX_PATH, + core_long_paths); } /** diff --git a/compat/win32/dirent.c b/compat/win32/dirent.c index 6b87042182..b3bd8d7af7 100644 --- a/compat/win32/dirent.c +++ b/compat/win32/dirent.c @@ -63,19 +63,23 @@ static int dirent_closedir(dirent_DIR *dir) DIR *dirent_opendir(const char *name) { - wchar_t pattern[MAX_PATH + 2]; /* + 2 for '/' '*' */ + wchar_t pattern[MAX_LONG_PATH + 2]; /* + 2 for "\*" */ WIN32_FIND_DATAW fdata; HANDLE h; int len; dirent_DIR *dir; - /* convert name to UTF-16 and check length < MAX_PATH */ - if ((len = xutftowcs_path(pattern, name)) < 0) + /* convert name to UTF-16 and check length */ + if ((len = xutftowcs_path_ex(pattern, name, MAX_LONG_PATH, -1, + MAX_PATH - 2, core_long_paths)) < 0) return NULL; - /* append optional '/' and wildcard '*' */ + /* + * append optional '\' and wildcard '*'. Note: we need to use '\' as + * Windows doesn't translate '/' to '\' for "\\?\"-prefixed paths. + */ if (len && !is_dir_sep(pattern[len - 1])) - pattern[len++] = '/'; + pattern[len++] = '\\'; pattern[len++] = '*'; pattern[len] = 0; diff --git a/compat/win32/fscache.c b/compat/win32/fscache.c index 05a2fa9663..9e7e36ed39 100644 --- a/compat/win32/fscache.c +++ b/compat/win32/fscache.c @@ -164,23 +164,24 @@ static struct fsentry *fseentry_create_entry(struct fsentry *list, */ static struct fsentry *fsentry_create_list(const struct fsentry *dir) { - wchar_t pattern[MAX_PATH + 2]; /* + 2 for '/' '*' */ + wchar_t pattern[MAX_LONG_PATH + 2]; /* + 2 for "\*" */ WIN32_FIND_DATAW fdata; HANDLE h; int wlen; struct fsentry *list, **phead; DWORD err; - /* convert name to UTF-16 and check length < MAX_PATH */ - if ((wlen = xutftowcsn(pattern, dir->name, MAX_PATH, dir->len)) < 0) { - if (errno == ERANGE) - errno = ENAMETOOLONG; + /* convert name to UTF-16 and check length */ + if ((wlen = xutftowcs_path_ex(pattern, dir->name, MAX_LONG_PATH, + dir->len, MAX_PATH - 2, core_long_paths)) < 0) return NULL; - } - /* append optional '/' and wildcard '*' */ + /* + * append optional '\' and wildcard '*'. Note: we need to use '\' as + * Windows doesn't translate '/' to '\' for "\\?\"-prefixed paths. + */ if (wlen) - pattern[wlen++] = '/'; + pattern[wlen++] = '\\'; pattern[wlen++] = '*'; pattern[wlen] = 0; diff --git a/config.c b/config.c index 8c5333d591..423be73c59 100644 --- a/config.c +++ b/config.c @@ -925,6 +925,11 @@ static int git_default_core_config(const char *var, const char *value) return 0; } + if (!strcmp(var, "core.longpaths")) { + core_long_paths = git_config_bool(var, value); + return 0; + } + /* Add other config variables here and to Documentation/config.txt. */ return 0; } diff --git a/environment.c b/environment.c index 83b7cbc3da..e2230267c0 100644 --- a/environment.c +++ b/environment.c @@ -66,6 +66,7 @@ struct startup_info *startup_info; unsigned long pack_size_limit_cfg; enum hide_dotfiles_type hide_dotfiles = HIDE_DOTFILES_DOTGITONLY; int core_fscache; +int core_long_paths; #ifndef PROTECT_HFS_DEFAULT #define PROTECT_HFS_DEFAULT 0 diff --git a/t/t2025-checkout-long-paths.sh b/t/t2025-checkout-long-paths.sh new file mode 100755 index 0000000000..0b8dd633c7 --- /dev/null +++ b/t/t2025-checkout-long-paths.sh @@ -0,0 +1,99 @@ +#!/bin/sh + +test_description='checkout long paths on Windows + +Ensures that Git for Windows can deal with long paths (>260) enabled via core.longpaths' + +. ./test-lib.sh + +if test_have_prereq !MINGW +then + skip_all='skipping MINGW specific long paths test' + test_done +fi + +test_expect_success setup ' + p=longpathxx && # -> 10 + p=$p$p$p$p$p && # -> 50 + p=$p$p$p$p$p && # -> 250 + + path=${p}/longtestfile && # -> 263 (MAX_PATH = 260) + + blob=$(echo foobar | git hash-object -w --stdin) && + + printf "100644 %s 0\t%s\n" "$blob" "$path" | + git update-index --add --index-info && + git commit -m initial -q +' + +test_expect_success 'checkout of long paths without core.longpaths fails' ' + git config core.longpaths false && + test_must_fail git checkout -f 2>error && + grep -q "Filename too long" error && + test_path_is_missing longpa~1/longtestfile +' + +test_expect_success 'checkout of long paths with core.longpaths works' ' + git config core.longpaths true && + git checkout -f && + test_path_is_file longpa~1/longtestfile +' + +test_expect_success 'update of long paths' ' + echo frotz >> longpa~1/longtestfile && + echo $path > expect && + git ls-files -m > actual && + test_cmp expect actual && + git add $path && + git commit -m second && + git grep "frotz" HEAD -- $path +' + +test_expect_success cleanup ' + # bash cannot delete the trash dir if it contains a long path + # lets help cleaning up (unless in debug mode) + test ! -z "$debug" || rm -rf longpa~1 +' + +# check that the template used in the test won't be too long: +abspath="$(pwd -W)"/testdir +test ${#abspath} -gt 230 || +test_set_prereq SHORTABSPATH + +test_expect_success SHORTABSPATH 'clean up path close to MAX_PATH' ' + p=/123456789abcdef/123456789abcdef/123456789abcdef/123456789abc/ef && + p=y$p$p$p$p && + subdir="x$(echo "$p" | tail -c $((253 - ${#abspath})))" && + # Now, $abspath/$subdir has exactly 254 characters, and is inside CWD + p2="$abspath/$subdir" && + test 254 = ${#p2} && + + # Be careful to overcome path limitations of the MSys tools and split + # the $subdir into two parts. ($subdir2 has to contain 16 chars and a + # slash somewhere following; that is why we asked for abspath <= 230 and + # why we placed a slash near the end of the $subdir template.) + subdir2=${subdir#????????????????*/} && + subdir1=testdir/${subdir%/$subdir2} && + mkdir -p "$subdir1" && + i=0 && + # The most important case is when absolute path is 258 characters long, + # and that will be when i == 4. + while test $i -le 7 + do + mkdir -p $subdir2 && + touch $subdir2/one-file && + mv ${subdir2%%/*} "$subdir1/" && + subdir2=z${subdir2} && + i=$(($i+1)) || + exit 1 + done && + + # now check that git is able to clear the tree: + (cd testdir && + git init && + git config core.longpaths yes && + git clean -fdx) && + test ! -d "$subdir1" +' + +test_done diff --git a/t/t7410-submodule-long-path.sh b/t/t7410-submodule-long-path.sh index ee4f3c139e..59b3d347db 100755 --- a/t/t7410-submodule-long-path.sh +++ b/t/t7410-submodule-long-path.sh @@ -11,15 +11,20 @@ This test verifies that "git submodule" initialization, update and clones work, TEST_NO_CREATE_REPO=1 . ./test-lib.sh -longpath="" -for (( i=0; i<4; i++ )); do - longpath="0123456789abcdefghijklmnopqrstuvwxyz$longpath" -done -# Pick a substring maximum of 90 characters -# This should be good, since we'll add on a lot for temp directories -export longpath=${longpath:0:90} +# cloning a submodule calls is_git_directory("$path/../.git/modules/$path"), +# which effectively limits the maximum length to PATH_MAX / 2 minus some +# overhead; start with 3 * 36 = 108 chars (test 2 fails if >= 110) +longpath36=0123456789abcdefghijklmnopqrstuvwxyz +longpath180=$longpath36$longpath36$longpath36$longpath36$longpath36 -test_expect_failure 'submodule with a long path' ' +# the git database must fit within PATH_MAX, which limits the submodule name +# to PATH_MAX - len(pwd) - ~90 (= len("/objects//") + 40-byte sha1 + some +# overhead from the test case) +pwd=$(pwd) +pwdlen=$(echo "$pwd" | wc -c) +longpath=$(echo $longpath180 | cut -c 1-$((170-$pwdlen))) + +test_expect_success 'submodule with a long path' ' git init --bare remote && test_create_repo bundle1 && ( @@ -56,7 +61,7 @@ test_expect_failure 'submodule with a long path' ' ) ' -test_expect_failure 'recursive submodule with a long path' ' +test_expect_success 'recursive submodule with a long path' ' git init --bare super && test_create_repo child && ( @@ -96,6 +101,5 @@ test_expect_failure 'recursive submodule with a long path' ' ) ) ' -unset longpath test_done From b6516722078c56eb3d0a8d809617b21f977b6f62 Mon Sep 17 00:00:00 2001 From: Thomas Braun Date: Wed, 24 Sep 2014 19:42:44 +0200 Subject: [PATCH 10/55] t2025: Tell tail explicitly to read from stdin Our current version of bash 3.1.17(5) can not parse the following snippet correctly p=abcd abspath=/$p subdir="x$(echo "$p" | tail -c $((253 - ${#abspath})))" as it returns tail: cannot open `253' for reading: No such file or directory This is fixed in bash 3.1.20(4), I did not check earlier versions. Signed-off-by: Thomas Braun --- t/t2025-checkout-long-paths.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/t2025-checkout-long-paths.sh b/t/t2025-checkout-long-paths.sh index 0b8dd633c7..e65a38da4d 100755 --- a/t/t2025-checkout-long-paths.sh +++ b/t/t2025-checkout-long-paths.sh @@ -63,7 +63,7 @@ test_set_prereq SHORTABSPATH test_expect_success SHORTABSPATH 'clean up path close to MAX_PATH' ' p=/123456789abcdef/123456789abcdef/123456789abcdef/123456789abc/ef && p=y$p$p$p$p && - subdir="x$(echo "$p" | tail -c $((253 - ${#abspath})))" && + subdir="x$(echo "$p" | tail -c $((253 - ${#abspath})) - )" && # Now, $abspath/$subdir has exactly 254 characters, and is inside CWD p2="$abspath/$subdir" && test 254 = ${#p2} && From a57dcd87484c6d9e44c984315d08f225b99dd177 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sat, 5 Jul 2014 00:00:36 +0200 Subject: [PATCH 11/55] Win32: fix 'lstat("dir/")' with long paths Use a suffciently large buffer to strip the trailing slash. Signed-off-by: Karsten Blees --- compat/mingw.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 5049d37433..4be201b491 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -560,7 +560,7 @@ static int do_lstat(int follow, const char *file_name, struct stat *buf) static int do_stat_internal(int follow, const char *file_name, struct stat *buf) { int namelen; - char alt_name[PATH_MAX]; + char alt_name[MAX_LONG_PATH]; if (!do_lstat(follow, file_name, buf)) return 0; @@ -576,7 +576,7 @@ static int do_stat_internal(int follow, const char *file_name, struct stat *buf) return -1; while (namelen && file_name[namelen-1] == '/') --namelen; - if (!namelen || namelen >= PATH_MAX) + if (!namelen || namelen >= MAX_LONG_PATH) return -1; memcpy(alt_name, file_name, namelen); From b48a2419775bfbff1749bbe7317085b6b5247699 Mon Sep 17 00:00:00 2001 From: nalla Date: Wed, 15 Apr 2015 07:54:39 +0100 Subject: [PATCH 12/55] asciidoctor: Fix user-manual to be built by `asciidoctor` The `user-manual.txt` ist designed as a `book` but the `Makefile` wants to build it as an `article`. This seems to be a problem when building the documentation with `asciidoctor`. Furthermore the parts *Git Glossary* and *Apendix B* had no subsections which is not allowed when building with `asciidoctor`. So lets add a *dummy* section. Signed-off-by: nalla --- Documentation/Makefile | 2 +- Documentation/user-manual.txt | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/Documentation/Makefile b/Documentation/Makefile index 3e39e2815b..df3901d83a 100644 --- a/Documentation/Makefile +++ b/Documentation/Makefile @@ -335,7 +335,7 @@ manpage-base-url.xsl: manpage-base-url.xsl.in user-manual.xml: user-manual.txt user-manual.conf $(QUIET_ASCIIDOC)$(RM) $@+ $@ && \ - $(TXT_TO_XML) -d article -o $@+ $< && \ + $(TXT_TO_XML) -d book -o $@+ $< && \ mv $@+ $@ technical/api-index.txt: technical/api-index-skel.txt \ diff --git a/Documentation/user-manual.txt b/Documentation/user-manual.txt index 7330d880f3..a5a8fdb0cd 100644 --- a/Documentation/user-manual.txt +++ b/Documentation/user-manual.txt @@ -3424,7 +3424,7 @@ just missing one particular blob version. [[the-index]] The index ------------ +--------- The index is a binary file (generally kept in `.git/index`) containing a sorted list of path names, each with permissions and the SHA-1 of a blob @@ -4366,6 +4366,10 @@ itself! Git Glossary ============ +[[git-explained]] +Git explained +------------- + include::glossary-content.txt[] [[git-quick-start]] @@ -4607,6 +4611,10 @@ $ git gc Appendix B: Notes and todo list for this manual =============================================== +[[todo-list]] +Todo list +--------- + This is a work in progress. The basic requirements: From bcfaee4d6a71df16503727b7eee04c00e48c74cb Mon Sep 17 00:00:00 2001 From: nalla Date: Wed, 15 Apr 2015 08:01:07 +0100 Subject: [PATCH 13/55] asciidoctor: Fix `giteveryday.txt` to be built with asciidoctor. When building the `doc` with `asciidoctor`, `asciidoctor` complains about a nested code block in a callout list. This is a really dirty solution to restore the callout list to function properly. There is a minimal visual sideeffect; the *immitated* codeblock has no overall greyish background. Instead the individual lines have it. Note: When building this patch with `asciidoc` the background is totally gone but the font is still monospaced. Signed-off-by: nalla --- Documentation/giteveryday.txt | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/Documentation/giteveryday.txt b/Documentation/giteveryday.txt index 7be6e64846..276fa033ac 100644 --- a/Documentation/giteveryday.txt +++ b/Documentation/giteveryday.txt @@ -308,18 +308,15 @@ master or exposed as a part of a stable branch. <10> create a signed tag. <11> make sure master was not accidentally rewound beyond that already pushed out. `ko` shorthand points at the Git maintainer's -repository at kernel.org, and looks like this: -+ ------------- -(in .git/config) -[remote "ko"] - url = kernel.org:/pub/scm/git/git.git - fetch = refs/heads/*:refs/remotes/ko/* - push = refs/heads/master - push = refs/heads/next - push = +refs/heads/pu - push = refs/heads/maint ------------- +repository at kernel.org, and looks like this: + +++(in .git/config) + +{startsb}remote "ko"{endsb} + +{nbsp}{nbsp}{nbsp}{nbsp}url = kernel.org:/pub/scm/git/git.git + +{nbsp}{nbsp}{nbsp}{nbsp}fetch = refs/heads/\*:refs/remotes/ko/* + +{nbsp}{nbsp}{nbsp}{nbsp}push = refs/heads/master + +{nbsp}{nbsp}{nbsp}{nbsp}push = refs/heads/next + +{nbsp}{nbsp}{nbsp}{nbsp}push = +refs/heads/pu + +{nbsp}{nbsp}{nbsp}{nbsp}push = refs/heads/maint++ + <12> In the output from `git show-branch`, `master` should have everything `ko/master` has, and `next` should have From 96e298add4f0ac61a6bcd81e3ad555ffd6d59aa5 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sun, 26 Jul 2009 05:08:42 +0200 Subject: [PATCH 14/55] Handle http.* config variables pointing to files gracefully on Windows On Windows, we would like to be able to have a default http.sslCAinfo that points to an MSys path (i.e. relative to the installation root of Git). As Git is a MinGW program, it has to handle the conversion of the MSys path into a MinGW32 path itself. Since system_path() considers paths starting with '/' as absolute, we have to convince it to make a Windows path by stripping the leading slash. Signed-off-by: Johannes Schindelin --- http.c | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/http.c b/http.c index 6798620065..4564a63bcb 100644 --- a/http.c +++ b/http.c @@ -8,6 +8,7 @@ #include "credential.h" #include "version.h" #include "pkt-line.h" +#include "exec_cmd.h" int active_requests; int http_is_verbose; @@ -147,6 +148,18 @@ static void process_curl_messages(void) } #endif +static int git_config_path(const char **result, + const char *var, const char *value) +{ + if (git_config_string(result, var, value)) + return 1; +#ifdef __MINGW32__ + if (**result == '/') + *result = system_path((*result) + 1); +#endif + return 0; +} + static int http_options(const char *var, const char *value, void *cb) { if (!strcmp("http.sslverify", var)) { @@ -154,17 +167,17 @@ static int http_options(const char *var, const char *value, void *cb) return 0; } if (!strcmp("http.sslcert", var)) - return git_config_string(&ssl_cert, var, value); + return git_config_path(&ssl_cert, var, value); #if LIBCURL_VERSION_NUM >= 0x070903 if (!strcmp("http.sslkey", var)) - return git_config_string(&ssl_key, var, value); + return git_config_path(&ssl_key, var, value); #endif #if LIBCURL_VERSION_NUM >= 0x070908 if (!strcmp("http.sslcapath", var)) - return git_config_string(&ssl_capath, var, value); + return git_config_path(&ssl_capath, var, value); #endif if (!strcmp("http.sslcainfo", var)) - return git_config_string(&ssl_cainfo, var, value); + return git_config_path(&ssl_cainfo, var, value); if (!strcmp("http.sslcertpasswordprotected", var)) { ssl_cert_password_required = git_config_bool(var, value); return 0; From 039dca68844235b9e4d03815e8a83e8f5f58067d Mon Sep 17 00:00:00 2001 From: Sebastian Schuberth Date: Thu, 29 Aug 2013 13:09:27 +0200 Subject: [PATCH 15/55] MinGW: Use MakeMaker to build the Perl libraries This way the libraries get properly installed into the "site_perl" directory and we just have to move them out of the "mingw" directory. Signed-off-by: Sebastian Schuberth --- config.mak.uname | 1 - 1 file changed, 1 deletion(-) diff --git a/config.mak.uname b/config.mak.uname index 2dae474513..3fca864353 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -495,7 +495,6 @@ ifneq (,$(findstring MINGW,$(uname_S))) NO_MKDTEMP = YesPlease NO_MKSTEMPS = YesPlease NO_SVN_TESTS = YesPlease - NO_PERL_MAKEMAKER = YesPlease RUNTIME_PREFIX = YesPlease NO_ST_BLOCKS_IN_STRUCT_STAT = YesPlease NO_NSEC = YesPlease From d58e45fe82f45ca662082d12452c79bf302f1ca8 Mon Sep 17 00:00:00 2001 From: Sebastian Schuberth Date: Wed, 4 Sep 2013 18:18:49 +0200 Subject: [PATCH 16/55] Makefile: Set htmldir to match the default HTML docs location under MSYS Signed-off-by: Sebastian Schuberth --- config.mak.uname | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.mak.uname b/config.mak.uname index 3fca864353..b58b5db025 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -526,7 +526,7 @@ ifneq (,$(findstring MINGW,$(uname_S))) X = .exe SPARSE_FLAGS = -Wno-one-bit-signed-bitfield ifneq (,$(wildcard ../THIS_IS_MSYSGIT)) - htmldir = doc/git/html/ + htmldir = share/doc/git/$(firstword $(subst -, ,$(GIT_VERSION)))/html prefix = INSTALL = /bin/install EXTLIBS += /mingw/lib/libz.a From 1086a222bc97fd63ae25993f3c5a70332f9d4ffc Mon Sep 17 00:00:00 2001 From: Thomas Braun Date: Wed, 3 Sep 2014 23:37:54 +0200 Subject: [PATCH 17/55] Enable support for perl regular expressions (LIBPCRE) Signed-off-by: Thomas Braun --- config.mak.uname | 1 + 1 file changed, 1 insertion(+) diff --git a/config.mak.uname b/config.mak.uname index b58b5db025..cde13fa472 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -534,6 +534,7 @@ ifneq (,$(wildcard ../THIS_IS_MSYSGIT)) INTERNAL_QSORT = YesPlease HAVE_LIBCHARSET_H = YesPlease NO_GETTEXT = YesPlease + USE_LIBPCRE= YesPlease else NO_CURL = YesPlease endif From 7710ba1c0280290c17637ba7d6ce6e70391fb621 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 20 Feb 2015 09:52:07 +0000 Subject: [PATCH 18/55] Help debugging with MSys2 by optionally executing bash with strace MSys2's strace facility is very useful for debugging... With this patch, the bash will be executed through strace if the environment variable GIT_STRACE_COMMANDS is set, which comes in real handy when investigating issues in the test suite. Signed-off-by: Johannes Schindelin --- compat/mingw.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/compat/mingw.c b/compat/mingw.c index 4be201b491..ecaaa2d176 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -1064,6 +1064,16 @@ static pid_t mingw_spawnve_fd(const char *cmd, const char **argv, char **deltaen free(quoted); } + if (getenv("GIT_STRACE_COMMANDS")) { + char **path = get_path_split(); + cmd = path_lookup("strace.exe", path, 1); + if (!cmd) + return error("strace not found!"); + if (xutftowcs_path(wcmd, cmd) < 0) + return -1; + strbuf_insert(&args, 0, "strace ", 7); + } + wargs = xmalloc((2 * args.len + 1) * sizeof(wchar_t)); xutftowcs(wargs, args.buf, 2 * args.len + 1); strbuf_release(&args); From f05932baaea4b94da875a9894f240f95c4086481 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 13 Feb 2015 12:18:54 +0000 Subject: [PATCH 19/55] Assorted header fixes to support MSys2-based MinGW build The excellent MSys2 project brings a substantially updated MinGW environment including newer GCC versions and new headers. To support compiling Git, let's special-case the new MinGW (tell-tale: the _MINGW64_VERSION_MAJOR constant is defined). Signed-off-by: Johannes Schindelin --- compat/mingw.h | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/compat/mingw.h b/compat/mingw.h index b614ac5af4..f4d6595121 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -1,14 +1,13 @@ +#include +#include #include #include - /* * things that are not available in header files */ -typedef int pid_t; typedef int uid_t; typedef int socklen_t; -#define hstrerror strerror #define S_IFLNK 0120000 /* Symbolic link */ #define S_ISLNK(x) (((x) & S_IFMT) == S_IFLNK) @@ -92,8 +91,10 @@ static inline int symlink(const char *oldpath, const char *newpath) { errno = ENOSYS; return -1; } static inline int fchmod(int fildes, mode_t mode) { errno = ENOSYS; return -1; } +#ifndef __MINGW64_VERSION_MAJOR static inline pid_t fork(void) { errno = ENOSYS; return -1; } +#endif static inline unsigned int alarm(unsigned int seconds) { return 0; } static inline int fsync(int fd) @@ -164,8 +165,10 @@ int pipe(int filedes[2]); unsigned int sleep (unsigned int seconds); int mkstemp(char *template); int gettimeofday(struct timeval *tv, void *tz); +#ifndef __MINGW64_VERSION_MAJOR struct tm *gmtime_r(const time_t *timep, struct tm *result); struct tm *localtime_r(const time_t *timep, struct tm *result); +#endif int getpagesize(void); /* defined in MinGW's libgcc.a */ struct passwd *getpwuid(uid_t uid); int setitimer(int type, struct itimerval *in, struct itimerval *out); @@ -301,8 +304,10 @@ static inline time_t filetime_to_time_t(const FILETIME *ft) /* * Use mingw specific stat()/lstat()/fstat() implementations on Windows. */ +#ifndef __MINGW64_VERSION_MAJOR #define off_t off64_t #define lseek _lseeki64 +#endif /* use struct stat with 64 bit st_size */ #ifdef stat @@ -375,8 +380,12 @@ static inline char *mingw_find_last_dir_sep(const char *path) int mingw_offset_1st_component(const char *path); #define offset_1st_component mingw_offset_1st_component #define PATH_SEP ';' +#ifndef __MINGW64_VERSION_MAJOR #define PRIuMAX "I64u" #define PRId64 "I64d" +#else +#include +#endif void mingw_open_html(const char *path); #define open_html mingw_open_html From 47eac89f5420b07125c1b209e47b8ae8f8b57fcc Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 13 Feb 2015 12:19:48 +0000 Subject: [PATCH 20/55] Avoid redefining S_* constants Signed-off-by: Johannes Schindelin --- compat/mingw.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/compat/mingw.h b/compat/mingw.h index f4d6595121..ea398123e6 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -13,14 +13,18 @@ typedef int socklen_t; #define S_ISLNK(x) (((x) & S_IFMT) == S_IFLNK) #define S_ISSOCK(x) 0 +#ifndef S_IRWXG #define S_IRGRP 0 #define S_IWGRP 0 #define S_IXGRP 0 #define S_IRWXG (S_IRGRP | S_IWGRP | S_IXGRP) +#endif +#ifndef S_IRWXO #define S_IROTH 0 #define S_IWOTH 0 #define S_IXOTH 0 #define S_IRWXO (S_IROTH | S_IWOTH | S_IXOTH) +#endif #define S_ISUID 0004000 #define S_ISGID 0002000 From c78295ca3002496d26353c88c7d23faf7bccba34 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 13 Feb 2015 12:23:19 +0000 Subject: [PATCH 21/55] Do not re-define _CONSOLE_FONT_INFOEX when compiling with MSys2 MSys2 already has that structure. Signed-off-by: Johannes Schindelin --- compat/winansi.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compat/winansi.c b/compat/winansi.c index efc5bb3a4b..aac430a952 100644 --- a/compat/winansi.c +++ b/compat/winansi.c @@ -23,6 +23,7 @@ static HANDLE hthread, hread, hwrite; static HANDLE hconsole1, hconsole2; #ifdef __MINGW32__ +#if !defined(__MINGW64_VERSION_MAJOR) || __MINGW64_VERSION_MAJOR < 5 typedef struct _CONSOLE_FONT_INFOEX { ULONG cbSize; DWORD nFont; @@ -32,6 +33,7 @@ typedef struct _CONSOLE_FONT_INFOEX { WCHAR FaceName[LF_FACESIZE]; } CONSOLE_FONT_INFOEX, *PCONSOLE_FONT_INFOEX; #endif +#endif typedef BOOL (WINAPI *PGETCURRENTCONSOLEFONTEX)(HANDLE, BOOL, PCONSOLE_FONT_INFOEX); From 8dd5256716e66c58fdde1ce97139988f8e274acb Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 3 Apr 2015 19:58:28 +0100 Subject: [PATCH 22/55] mingw: allow compiling with MSys2's compiler With MSys2's GCC, `ReadWriteBarrier` is already defined, and FORCEINLINE unfortunately gets defined incorrectly. Let's work around both problems, using the MSys2-specific __MINGW64_VERSION_MAJOR constant to guard them. Signed-off-by: Johannes Schindelin --- compat/nedmalloc/malloc.c.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/compat/nedmalloc/malloc.c.h b/compat/nedmalloc/malloc.c.h index f216a2a7d3..18634e3a3b 100644 --- a/compat/nedmalloc/malloc.c.h +++ b/compat/nedmalloc/malloc.c.h @@ -720,6 +720,9 @@ struct mallinfo { inlining are defined as macros, so these aren't used for them. */ +#ifdef __MINGW64_VERSION_MAJOR +#undef FORCEINLINE +#endif #ifndef FORCEINLINE #if defined(__GNUC__) #define FORCEINLINE __inline __attribute__ ((always_inline)) @@ -1382,6 +1385,9 @@ LONG __cdecl _InterlockedExchange(LONG volatile *Target, LONG Value); /*** Atomic operations ***/ #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100 + #ifdef __MINGW64_VERSION_MAJOR + #undef _ReadWriteBarrier + #endif #define _ReadWriteBarrier() __sync_synchronize() #else static __inline__ __attribute__((always_inline)) long __sync_lock_test_and_set(volatile long * const Target, const long Value) From 9d451168aae22242f9b1370bcbc9ff4d505457a8 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 13 Feb 2015 12:35:10 +0000 Subject: [PATCH 23/55] config.mak.uname: support MSys2 Git for Windows lags a little bit behind with the 2.x releases because the Git for Windows developers wanted to let that big jump coincide with a well-needed overhaul of the context within which Git for Windows is developed. To understand why this is such a big issue, it needs to be noted that many parts of Git are not written in portable C, but instead relies on a POSIX shell and Perl to be available. Even in the portable C part, there is the ingrained notion that we can work with UTF-8 encoded strings. To support the scripts, Git for Windows has to ship a minimal POSIX emulation layer with Bash and Perl thrown in, and when the Git for Windows effort started originally, in August 2007, we settled on using MSys, a stripped down version of Cygwin. Consequently, the original name of the project was "msysGit" (which, sadly, caused a *lot* of confusion because few Windows users know about MSys, and even less care). To compile the C code of Git for Windows, we used MSys, too: it sports an additional version of the GNU C Compiler that targets the plain Win32 API (with a few convenience functions thrown in) instead of the POSIX emulation layer that would require the MSys runtime to run the compiled programs. That way, Git for Windows' executable(s) are really just Win32 programs. To discern executables requiring the POSIX emulation layer from the ones that do not, the latter are called MinGW (Minimal GNU for Windows) when the former are called MSys executables. This reliance on MSys incurred challenges, too, though: some of our changes to the MSys runtime -- necessary to support Git for Windows better -- were not accepted upstream, the MSys runtime was not developed further to support e.g. UTF-8 or 64-bit, and apart from not having a package management system until much later (when mingw-get was introduced), many packages provided by the MSys/MinGW project lag behind the respective source code versions, in particular Bash and OpenSSL. For a while, the Git for Windows project tried to remedy the situation by trying to build newer versions of those packages, but the situation quickly became untenable, especially with problems like the Heartbleed bug requiring swift action and Git for Windows contributors being scarce -- despite millions of downloads suggesting that there are many users. After a brief push in the direction of mingw-get, thanks to the long-time contributor and co-maintainer Sebastian Schuberth, it became clear that we need to look for alternatives. Happily, in the meantime the MSys2 project (https://msys2.github.io/) emerged, and was chosen to be the base of the Git for Windows 2.x. MSys2 is a rewrite of the spirit of MSys: it is again a stripped down version of Cygwin, but it is actively kept up-to-date with Cygwin's source code. Thereby, it already supports Unicode internally, and it also offers the 64-bit support that we yearned for since the beginning of the Git for Windows project. MSys2 also ported the Pacman package management system from Arch Linux and uses it heavily. This brings the same convenience to which Linux users are used to from `yum` or `apt-get`, and to which MacOSX users are used to from Homebrew or MacPorts, or BSD users from the Ports system, to MSys2: a simple `pacman -Syu` will update all installed packages to the newest versions currently available. MSys2 is also *very* active, typically providing package updates multiple times per week. It still required a two-month effort to bring everything to a state where Git's test suite passes, and a couple of patches await their submission to the respective upstream projects. Yet without MSys2, the modernization of Git for Windows would simply not have happened. This commit lays the ground work to supporting MSys2-based Git builds. Assisted-by: Waldek Maleska Signed-off-by: Johannes Schindelin squash! config.mak.uname: support MSys2 Make sure that the nedmalloc patch is applied before. --- config.mak.uname | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/config.mak.uname b/config.mak.uname index cde13fa472..b4fd040d45 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -511,7 +511,7 @@ ifneq (,$(findstring MINGW,$(uname_S))) NO_POSIX_GOODIES = UnfortunatelyYes DEFAULT_HELP_FORMAT = html NO_D_INO_IN_DIRENT = YesPlease - COMPAT_CFLAGS += -D__USE_MINGW_ACCESS -D_USE_32BIT_TIME_T -DNOGDI -Icompat -Icompat/win32 + COMPAT_CFLAGS += -D_USE_32BIT_TIME_T -DNOGDI -Icompat -Icompat/win32 COMPAT_CFLAGS += -DSTRIP_EXTENSION=\".exe\" COMPAT_OBJS += compat/mingw.o compat/winansi.o \ compat/win32/pthread.o compat/win32/syslog.o \ @@ -534,9 +534,25 @@ ifneq (,$(wildcard ../THIS_IS_MSYSGIT)) INTERNAL_QSORT = YesPlease HAVE_LIBCHARSET_H = YesPlease NO_GETTEXT = YesPlease - USE_LIBPCRE= YesPlease + COMPAT_CLFAGS += -D__USE_MINGW_ACCESS else - NO_CURL = YesPlease + ifeq ($(shell expr "$(uname_R)" : '2\.'),2) + # MSys2 + CC = gcc + prefix = /mingw32 + COMPAT_CFLAGS += -D__USE_MINGW_ANSI_STDIO=0 + INSTALL = /bin/install + NO_R_TO_GCC_LINKER = YesPlease + INTERNAL_QSORT = YesPlease + HAVE_LIBCHARSET_H = YesPlease + NO_GETTEXT = YesPlease + USE_LIBPCRE= YesPlease + NO_CURL = + USE_NED_ALLOCATOR = YesPlease + else + COMPAT_CFLAGS += -D__USE_MINGW_ANSI_STDIO + NO_CURL = YesPlease + endif endif endif ifeq ($(uname_S),QNX) From 10b86a845ce2a329a0ad6e1b4c6f8be5be0d3819 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 13 Feb 2015 12:45:37 +0000 Subject: [PATCH 24/55] config.mak.uname: supporting 64-bit MSys2 This just makes things compile, the test suite needs extra tender loving care in addition to this change. We will address these issues in later commits. While at it, also allow building MSys2 Git (i.e. a Git that uses MSys2's POSIX emulation layer). Signed-off-by: Johannes Schindelin --- config.mak.uname | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/config.mak.uname b/config.mak.uname index b4fd040d45..755fc62693 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -511,13 +511,12 @@ ifneq (,$(findstring MINGW,$(uname_S))) NO_POSIX_GOODIES = UnfortunatelyYes DEFAULT_HELP_FORMAT = html NO_D_INO_IN_DIRENT = YesPlease - COMPAT_CFLAGS += -D_USE_32BIT_TIME_T -DNOGDI -Icompat -Icompat/win32 + COMPAT_CFLAGS += -DNOGDI -Icompat -Icompat/win32 COMPAT_CFLAGS += -DSTRIP_EXTENSION=\".exe\" COMPAT_OBJS += compat/mingw.o compat/winansi.o \ compat/win32/pthread.o compat/win32/syslog.o \ compat/win32/dirent.o compat/win32/fscache.o BASIC_CFLAGS += -DPROTECT_NTFS_DEFAULT=1 - BASIC_LDFLAGS += -Wl,--large-address-aware EXTLIBS += -lws2_32 GITLIBS += git.res PTHREAD_LIBS = @@ -538,8 +537,17 @@ ifneq (,$(wildcard ../THIS_IS_MSYSGIT)) else ifeq ($(shell expr "$(uname_R)" : '2\.'),2) # MSys2 + prefix = /usr/ + ifeq (MINGW32,$(MSYSTEM)) + prefix = /mingw32 + endif + ifeq (MINGW64,$(MSYSTEM)) + prefix = /mingw64 + else + COMPAT_CFLAGS += -D_USE_32BIT_TIME_T + BASIC_LDFLAGS += -Wl,--large-address-aware + endif CC = gcc - prefix = /mingw32 COMPAT_CFLAGS += -D__USE_MINGW_ANSI_STDIO=0 INSTALL = /bin/install NO_R_TO_GCC_LINKER = YesPlease From 9ae8e25eb17d6969665ac21813342f7bc9367509 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 16 Feb 2015 14:06:59 +0100 Subject: [PATCH 25/55] Build Python stuff with MSys2 Signed-off-by: Johannes Schindelin --- config.mak.uname | 1 + 1 file changed, 1 insertion(+) diff --git a/config.mak.uname b/config.mak.uname index 755fc62693..fca6ee42fe 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -557,6 +557,7 @@ else USE_LIBPCRE= YesPlease NO_CURL = USE_NED_ALLOCATOR = YesPlease + NO_PYTHON = else COMPAT_CFLAGS += -D__USE_MINGW_ANSI_STDIO NO_CURL = YesPlease From 8722f37fdcf699b8469b10f9667f8f5466734a27 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 19 Feb 2015 17:03:46 +0000 Subject: [PATCH 26/55] Let's use gettext with MSys2 Signed-off-by: Johannes Schindelin --- config.mak.uname | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.mak.uname b/config.mak.uname index fca6ee42fe..129cd06eaa 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -553,7 +553,7 @@ else NO_R_TO_GCC_LINKER = YesPlease INTERNAL_QSORT = YesPlease HAVE_LIBCHARSET_H = YesPlease - NO_GETTEXT = YesPlease + NO_GETTEXT = USE_LIBPCRE= YesPlease NO_CURL = USE_NED_ALLOCATOR = YesPlease From 6044f3487908c6aeac22d2b4120649750f928598 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 24 Feb 2015 15:09:31 +0000 Subject: [PATCH 27/55] Do not trust MSys2's MinGW gettext.sh It does not quite work because it produces DOS line endings which the shell does not like at all. This lets t3406, t3903, t4254, t7400, t7401, t7406 and t7407 pass. Signed-off-by: Johannes Schindelin --- config.mak.uname | 1 + 1 file changed, 1 insertion(+) diff --git a/config.mak.uname b/config.mak.uname index 129cd06eaa..74272022ec 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -554,6 +554,7 @@ else INTERNAL_QSORT = YesPlease HAVE_LIBCHARSET_H = YesPlease NO_GETTEXT = + USE_GETTEXT_SCHEME = fallthrough USE_LIBPCRE= YesPlease NO_CURL = USE_NED_ALLOCATOR = YesPlease From d86328334f95420de2b2eff7a267ab3b96c0093a Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sat, 14 Feb 2015 15:03:58 +0000 Subject: [PATCH 28/55] Squash compile warning with MSys2 Signed-off-by: Johannes Schindelin --- compat/win32/pthread.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compat/win32/pthread.h b/compat/win32/pthread.h index 8ad187344f..1ade961769 100644 --- a/compat/win32/pthread.h +++ b/compat/win32/pthread.h @@ -77,7 +77,7 @@ extern pthread_t pthread_self(void); static inline int pthread_exit(void *ret) { - ExitThread((DWORD)ret); + ExitThread((DWORD)(size_t)ret); } typedef DWORD pthread_key_t; From b361ea79c81ca219959fd0fd37af950ced2c7f86 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sun, 15 Feb 2015 09:56:35 +0000 Subject: [PATCH 29/55] Error out when mingw_startup() *and* NO_UNSETENV are active The unsetenv code has no idea to update our environ_size, therefore causing segmentation faults when environment variables are removed without compat/mingw.c's knowing (MinGW's optimized lookup would try to strcmp() against NULL in such a case). Signed-off-by: Johannes Schindelin --- compat/mingw.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/compat/mingw.h b/compat/mingw.h index ea398123e6..7ca2e5287a 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -220,6 +220,10 @@ char *mingw_mktemp(char *template); char *mingw_getcwd(char *pointer, int len); #define getcwd mingw_getcwd +#ifdef NO_UNSETENV +#error "NO_UNSETENV is incompatible with the MinGW startup code!" +#endif + char *mingw_getenv(const char *name); #define getenv mingw_getenv int mingw_putenv(const char *namevalue); From 6771f226778c99cabfebbe410e4b38dcca297b4e Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sun, 15 Feb 2015 16:39:10 +0100 Subject: [PATCH 30/55] Special-case the MSYS2_TZ environment variable With MSys2, the "TZ" environment variable gets filtered out when calling non-MSys2 executables. The reason is that Windows' time zone handling is substantially different from the POSIX one. However, we just taught Git for Windows' fork of the MSys2 runtime to pass on the timezone in a different environment variable, MSYS2_TZ for the sole purpose of Git being able to reinterpret it correctly. Signed-off-by: Johannes Schindelin --- compat/mingw.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/compat/mingw.c b/compat/mingw.c index ecaaa2d176..fa35c44e6a 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2219,8 +2219,14 @@ void mingw_startup() __argv[0] = wcstoutfdup_startup(buffer, _wpgmptr, maxlen); for (i = 1; i < argc; i++) __argv[i] = wcstoutfdup_startup(buffer, wargv[i], maxlen); - for (i = 0; wenv[i]; i++) + for (i = 0; wenv[i]; i++) { environ[i] = wcstoutfdup_startup(buffer, wenv[i], maxlen); + if (!strncasecmp(environ[i], "MSYS2_TZ=", 9)) { + char *to_free = environ[i]; + environ[i] = xstrdup(to_free + 6); + free(to_free); + } + } environ[i] = NULL; free(buffer); From b6fff2e1f7eccbdb7b212fd809d700e220a9caeb Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 18 Feb 2015 11:55:48 +0000 Subject: [PATCH 31/55] Tests (MinGW): Do not bother to test funny file names MSys2 actually allows to create files or directories whose names contain tabs, newlines or colors, even if plain Win32 API cannot access them. As we are using an MSys2 bash to run the tests, such files or directories are created successfully, but Git has no chance to work with them because it is a regular Windows program, hence limited by the Win32 API. Signed-off-by: Johannes Schindelin --- t/t3300-funny-names.sh | 1 + t/t3600-rm.sh | 3 ++- t/t3703-add-magic-pathspec.sh | 2 +- t/t3902-quoted.sh | 1 + t/t4016-diff-quote.sh | 1 + t/t4135-apply-weird-filenames.sh | 3 ++- t/t9903-bash-prompt.sh | 2 +- 7 files changed, 9 insertions(+), 4 deletions(-) diff --git a/t/t3300-funny-names.sh b/t/t3300-funny-names.sh index 9a146f1335..04de03cad0 100755 --- a/t/t3300-funny-names.sh +++ b/t/t3300-funny-names.sh @@ -13,6 +13,7 @@ tree, index, and tree objects. HT=' ' +test_have_prereq MINGW || echo 2>/dev/null > "Name with an${HT}HT" if ! test -f "Name with an${HT}HT" then diff --git a/t/t3600-rm.sh b/t/t3600-rm.sh index e00d7d2b61..83b1df9e3c 100755 --- a/t/t3600-rm.sh +++ b/t/t3600-rm.sh @@ -14,7 +14,8 @@ test_expect_success \ git add -- foo bar baz 'space embedded' -q && git commit -m 'add normal files'" -if touch -- 'tab embedded' 'newline + +if ! test_have_prereq MINGW && touch -- 'tab embedded' 'newline embedded' 2>/dev/null then test_set_prereq FUNNYNAMES diff --git a/t/t3703-add-magic-pathspec.sh b/t/t3703-add-magic-pathspec.sh index 5115de7036..aaff784659 100755 --- a/t/t3703-add-magic-pathspec.sh +++ b/t/t3703-add-magic-pathspec.sh @@ -38,7 +38,7 @@ cat >expected </dev/null +if ! test_have_prereq MINGW && mkdir ":" 2>/dev/null then test_set_prereq COLON_DIR fi diff --git a/t/t3902-quoted.sh b/t/t3902-quoted.sh index 892f567844..f528008c36 100755 --- a/t/t3902-quoted.sh +++ b/t/t3902-quoted.sh @@ -12,6 +12,7 @@ GN='純' HT=' ' DQ='"' +test_have_prereq MINGW || echo foo 2>/dev/null > "Name and an${HT}HT" if ! test -f "Name and an${HT}HT" then diff --git a/t/t4016-diff-quote.sh b/t/t4016-diff-quote.sh index cd543ecc54..9c48e5c2c9 100755 --- a/t/t4016-diff-quote.sh +++ b/t/t4016-diff-quote.sh @@ -13,6 +13,7 @@ P1='pathname with HT' P2='pathname with SP' P3='pathname with LF' +test_have_prereq !MINGW && echo 2>/dev/null >"$P1" && test -f "$P1" && rm -f "$P1" || { skip_all='Your filesystem does not allow tabs in filenames' test_done diff --git a/t/t4135-apply-weird-filenames.sh b/t/t4135-apply-weird-filenames.sh index bf5dc57286..6d6b96d5d3 100755 --- a/t/t4135-apply-weird-filenames.sh +++ b/t/t4135-apply-weird-filenames.sh @@ -19,7 +19,8 @@ test_expect_success 'setup' ' test_when_finished "rm -f \"tab embedded.txt\"" && test_when_finished "rm -f '\''\"quoteembedded\".txt'\''" && - if touch -- "tab embedded.txt" '\''"quoteembedded".txt'\'' + if ! test_have_prereq MINGW && + touch -- "tab embedded.txt" '\''"quoteembedded".txt'\'' then test_set_prereq FUNNYNAMES fi diff --git a/t/t9903-bash-prompt.sh b/t/t9903-bash-prompt.sh index 46d7d37a51..a1749f44ed 100755 --- a/t/t9903-bash-prompt.sh +++ b/t/t9903-bash-prompt.sh @@ -67,7 +67,7 @@ repo_with_newline='repo with newline' -if mkdir "$repo_with_newline" 2>/dev/null +if ! test_have_prereq MINGW && mkdir "$repo_with_newline" 2>/dev/null then test_set_prereq FUNNYNAMES else From 6a5bf70a1590ea742fc5b5ee02d91d345fd65440 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 19 Feb 2015 12:25:42 +0000 Subject: [PATCH 32/55] Tests (MinGW): Disable mkfifo-based tests With MSys2, there is actually an implementation of mkfifo available. The only problem is that it is only emulating named pipes through the MSys2 runtime; The Win32 API has no idea about named pipes, hence the Git executable cannot access those pipes either. The symptom is that Git fails with a ': No such file or directory' because MSys2 emulates named pipes through special-crafted '.lnk' files. The solution is to tell the test suite explicitly that we cannot use named pipes when we want to test a MinGW Git. Signed-off-by: Johannes Schindelin --- t/test-lib.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/test-lib.sh b/t/test-lib.sh index 9914d3e1cf..5cbdf4df04 100644 --- a/t/test-lib.sh +++ b/t/test-lib.sh @@ -978,7 +978,7 @@ test_i18ngrep () { test_lazy_prereq PIPE ' # test whether the filesystem supports FIFOs case $(uname -s) in - CYGWIN*) + CYGWIN*|MINGW*) false ;; *) From f50eb7417a099efef17730f1390c3f5158c2b7e3 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 19 Feb 2015 15:20:16 +0000 Subject: [PATCH 33/55] t5516: override MinGW-specific pwd override This test is susceptible to MSys2's posix-to-windows path mangling; Let's just use POSIX paths throughout and let the tests pass. Signed-off-by: Johannes Schindelin --- t/t5516-fetch-push.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/t/t5516-fetch-push.sh b/t/t5516-fetch-push.sh index 050877f181..a8c2b3dd36 100755 --- a/t/t5516-fetch-push.sh +++ b/t/t5516-fetch-push.sh @@ -16,6 +16,14 @@ This test checks the following functionality: . ./test-lib.sh +if test_have_prereq MINGW +then + # Avoid posix-to-windows path mangling + pwd () { + builtin pwd + } +fi + D=`pwd` mk_empty () { From 3ae7272df1d4d6442b322c2eaacf357301e4ebeb Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 20 Feb 2015 14:21:04 +0000 Subject: [PATCH 34/55] Tests: optionally skip redirecting stdin/stdout/stderr There is a really useful debugging technique developed by Sverre Rabbelier that inserts "bash &&" somewhere in the test scripts, letting the developer interact at given points with the current state. Another debugging technique, used a lot by this here coder, is to run certain executables via gdb by guarding a "gdb -args" call in bin-wrappers/git. Both techniques were disabled by 781f76b1(test-lib: redirect stdin of tests). Let's reinstate the ability to run an interactive shell by making the redirection optional: setting the TEST_NO_REDIRECT environment variable will skip the redirection. Signed-off-by: Johannes Schindelin --- t/test-lib.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/t/test-lib.sh b/t/test-lib.sh index 5cbdf4df04..f5db7337b8 100644 --- a/t/test-lib.sh +++ b/t/test-lib.sh @@ -504,6 +504,20 @@ test_eval_ () { # # The test itself is run with stderr put back to &4 (so either to # /dev/null, or to the original stderr if --verbose was used). + if test -n "$TEST_NO_REDIRECT" + then + test_eval_inner_ "$@" + test_eval_ret_=$? + if test "$trace" = t + then + set +x + if test "$test_eval_ret_" != 0 + then + say_color error >&4 "error: last command exited with \$?=$test_eval_ret_" + fi + fi + return $test_eval_ret_ + fi { test_eval_inner_ "$@" &3 2>&4 test_eval_ret_=$? From 594564da52656c5b47642e15f3dcd3aada3a5656 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 20 Feb 2015 15:41:20 +0000 Subject: [PATCH 35/55] mingw: Prepare the TMP environment variable for shell scripts When shell scripts access a $TMP variable containing backslashes, they will be mistaken for escape characters. Let's not let that happen by converting them to forward slashes. This fixes t7800 with MSys2. Signed-off-by: Johannes Schindelin --- compat/mingw.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/compat/mingw.c b/compat/mingw.c index fa35c44e6a..27989789b6 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2226,6 +2226,18 @@ void mingw_startup() environ[i] = xstrdup(to_free + 6); free(to_free); } + if (!strncasecmp(environ[i], "TMP=", 4)) { + /* + * Convert all dir separators to forward slashes, + * to help shell commands called from the Git + * executable (by not mistaking the dir separators + * for escape characters). + */ + char *p; + for (p = environ[i]; *p; p++) + if (*p == '\\') + *p = '/'; + } } environ[i] = NULL; free(buffer); From 6e975ed897c82c05147a23721bdac5dd2363e176 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sun, 22 Feb 2015 13:33:35 +0000 Subject: [PATCH 36/55] Git.pm: stop assuming that absolute paths start with a slash This fixes t7800 with MSys2. Signed-off-by: Johannes Schindelin --- perl/Git.pm | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/perl/Git.pm b/perl/Git.pm index 9026a7bb98..d1e4c48202 100644 --- a/perl/Git.pm +++ b/perl/Git.pm @@ -188,7 +188,8 @@ sub repository { }; if ($dir) { - $dir =~ m#^/# or $dir = $opts{Directory} . '/' . $dir; + _verify_require(); + File::Spec->file_name_is_absolute($dir) or $dir = $opts{Directory} . '/' . $dir; $opts{Repository} = abs_path($dir); # If --git-dir went ok, this shouldn't die either. From 753e84f8fcbe63bbfaefd20a313d5b3bd316d7f1 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sun, 22 Feb 2015 16:08:36 +0000 Subject: [PATCH 37/55] MSys2: Fix t9700 assumption about directory separators This test assumed that there are no two equivalent directory separators. However, on Windows, the back slash and the forward slash *are* equivalent. Let's paper over this issue by converting the backward slashes to forward ones in the test that fails with MSys2 otherwise. Signed-off-by: Johannes Schindelin --- t/t9700/test.pl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/t/t9700/test.pl b/t/t9700/test.pl index 1140767b50..a01665473d 100755 --- a/t/t9700/test.pl +++ b/t/t9700/test.pl @@ -33,7 +33,9 @@ is($r->config_int("test.int"), 2048, "config_int: integer"); is($r->config_int("test.nonexistent"), undef, "config_int: nonexistent"); ok($r->config_bool("test.booltrue"), "config_bool: true"); ok(!$r->config_bool("test.boolfalse"), "config_bool: false"); -is($r->config_path("test.path"), $r->config("test.pathexpanded"), +our $test_path = $r->config_path("test.path"); +$test_path =~ s/\\/\//g if ($^O eq 'msys'); +is($test_path, $r->config("test.pathexpanded"), "config_path: ~/foo expansion"); is_deeply([$r->config_path("test.pathmulti")], ["foo", "bar"], "config_path: multiple values"); From 671bd6a0a45e94fd0cfe0c9afa65ca4479fd26da Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sun, 22 Feb 2015 19:55:26 +0000 Subject: [PATCH 38/55] mingw: Make sure sigset_t is defined With MSys2, the sigset_t type is defined in sys/types.h, therefore we need to #include said file. Signed-off-by: Johannes Schindelin --- compat/mingw.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/compat/mingw.h b/compat/mingw.h index 7ca2e5287a..a628b2d48d 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -1,5 +1,9 @@ #include #include +#include +#ifndef _POSIX +typedef _sigset_t sigset_t; +#endif #include #include /* From 86425acb29d85a4ffff771c9c29b384e209359dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EB=A7=88=EB=88=84=EC=97=98?= Date: Fri, 6 Mar 2015 12:09:51 +0100 Subject: [PATCH 39/55] mingw: Try to delete target directory first. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the rename function tries to move a directory it fails if the target directory exists. It should check if it can delete the (possibly empty) target directory and then try again to move the directory. Helped-by: Johannes Schindelin Signed-off-by: 마누엘 --- compat/mingw.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/compat/mingw.c b/compat/mingw.c index 27989789b6..a78ab64768 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -1648,7 +1648,12 @@ repeat: if (gle == ERROR_ACCESS_DENIED && (attrs = GetFileAttributesW(wpnew)) != INVALID_FILE_ATTRIBUTES) { if (attrs & FILE_ATTRIBUTE_DIRECTORY) { - errno = EISDIR; + DWORD attrsold = GetFileAttributesW(wpold); + if (attrsold == INVALID_FILE_ATTRIBUTES || + !(attrsold & FILE_ATTRIBUTE_DIRECTORY)) + errno = EISDIR; + else if (!_wrmdir(wpnew)) + goto repeat; return -1; } if ((attrs & FILE_ATTRIBUTE_READONLY) && From df0145d9ef722d26f48d073c681844657ac08edc Mon Sep 17 00:00:00 2001 From: Cesar Eduardo Barros Date: Mon, 9 Mar 2015 08:51:38 +0100 Subject: [PATCH 40/55] mingw: Embed a manifest to trick UAC into Doing The Right Thing On Windows >= Vista, not having an application manifest with a requestedExecutionLevel can cause several kinds of confusing behavior. The first and more obvious behavior is "Installer Detection", where Windows sometimes decides (by looking at things like the file name and even sequences of bytes within the executable) that an executable is an installer and should run elevated (causing the well-known popup dialog to appear). In Git's context, subcommands such as "git patch-id" or "git update-index" fall prey to this behavior. The second and more confusing behavior is "File Virtualization". It means that when files are written without having write permission, it does not fail (as expected), but they are instead redirected to somewhere else. When the files are read, the original contents are returned, though, not the ones that were just written somewhere else. Even more confusing, not all write accesses are redirected; Trying to write to write-protected .exe files, for example, will fail instead of redirecting. In addition to being unwanted behavior, File Virtualization causes dramatic slowdowns in Git (see for instance http://code.google.com/p/msysgit/issues/detail?id=320). There are two ways to prevent those two behaviors: Either you embed an application manifest within all your executables, or you add an external manifest (a file with the same name followed by .manifest) to all your executables. Since Git's builtins are hardlinked (or copied), it is simpler and more robust to embed a manifest. A recent enough MSVC compiler should already embed a working internal manifest, but for MinGW you have to do so by hand. Very lightly tested on Wine, where like on Windows XP it should not make any difference. References: - New UAC Technologies for Windows Vista http://msdn.microsoft.com/en-us/library/bb756960.aspx - Create and Embed an Application Manifest (UAC) http://msdn.microsoft.com/en-us/library/bb756929.aspx [js: simplified the embedding dramatically by reusing Git for Windows' existing Windows resource file, removed the optional (and dubious) processorArchitecture attribute of the manifest's assemblyIdentity section.] Signed-off-by: Cesar Eduardo Barros Signed-off-by: Johannes Schindelin --- compat/win32/git.manifest | 11 +++++++++++ git.rc | 2 ++ 2 files changed, 13 insertions(+) create mode 100644 compat/win32/git.manifest diff --git a/compat/win32/git.manifest b/compat/win32/git.manifest new file mode 100644 index 0000000000..f691172a8c --- /dev/null +++ b/compat/win32/git.manifest @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/git.rc b/git.rc index 33aafb786c..4aa4f9973e 100644 --- a/git.rc +++ b/git.rc @@ -20,3 +20,5 @@ BEGIN VALUE "Translation", 0x409, 1200 END END + +1 RT_MANIFEST "compat/win32/git.manifest" From 03570445e74103cc8e34135376a9c606c9b6e019 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Mon, 20 Apr 2015 19:24:29 +0200 Subject: [PATCH 41/55] mingw: make isatty() recognize MSys pseudo terminals (/dev/pty*) MSys2 emulates pseudo terminals via named pipes, and isatty() returns 0 for such file descriptors. Therefore, some interactive functionality (such as launching a pager, asking if a failed unlink should be repeated etc.) doesn't work when run in a terminal emulator that uses MSys ptys (such as mintty). However, MSys uses special names for its pty pipes ('msys-*-pty*'), which allows us to distinguish them from normal piped input / output. On startup, check if stdin / stdout / stderr are connected to such pipes using the NtQueryObject API from NTDll.dll. If the names match, adjust the flags in MSVCRT's ioinfo structure accordingly. Signed-off-by: Karsten Blees --- compat/winansi.c | 58 ++++++++++++++++++++++++++++++++++++++++++++---- config.mak.uname | 3 ++- 2 files changed, 56 insertions(+), 5 deletions(-) diff --git a/compat/winansi.c b/compat/winansi.c index aac430a952..23b0fe7e86 100644 --- a/compat/winansi.c +++ b/compat/winansi.c @@ -482,6 +482,7 @@ static size_t sizeof_ioinfo = 0; #define IOINFO_L2E 5 #define IOINFO_ARRAY_ELTS (1 << IOINFO_L2E) +#define FPIPE 0x08 #define FDEV 0x40 static inline ioinfo* _pioinfo(int fd) @@ -529,6 +530,45 @@ static HANDLE swap_osfhnd(int fd, HANDLE new_handle) return old_handle; } +#ifdef DETECT_MSYS_TTY + +#include +#include + +static void detect_msys_tty(int fd) +{ + ULONG result; + BYTE buffer[1024]; + POBJECT_NAME_INFORMATION nameinfo = (POBJECT_NAME_INFORMATION) buffer; + PWSTR name; + + /* check if fd is a pipe */ + HANDLE h = (HANDLE) _get_osfhandle(fd); + if (GetFileType(h) != FILE_TYPE_PIPE) + return; + + /* get pipe name */ + if (!NT_SUCCESS(NtQueryObject(h, ObjectNameInformation, + buffer, sizeof(buffer) - 2, &result))) + return; + name = nameinfo->Name.Buffer; + name[nameinfo->Name.Length] = 0; + + /* check if this could be a msys pty pipe ('msys-XXXX-ptyN-XX') */ + if (!wcsstr(name, L"msys-") || !wcsstr(name, L"-pty")) + return; + + /* init ioinfo size if we haven't done so */ + if (init_sizeof_ioinfo()) + return; + + /* set FDEV flag, reset FPIPE flag */ + _pioinfo(fd)->osflags &= ~FPIPE; + _pioinfo(fd)->osflags |= FDEV; +} + +#endif + void winansi_init(void) { int con1, con2; @@ -537,8 +577,15 @@ void winansi_init(void) /* check if either stdout or stderr is a console output screen buffer */ con1 = is_console(1); con2 = is_console(2); - if (!con1 && !con2) + if (!con1 && !con2) { +#ifdef DETECT_MSYS_TTY + /* check if stdin / stdout / stderr are msys pty pipes */ + detect_msys_tty(0); + detect_msys_tty(1); + detect_msys_tty(2); +#endif return; + } /* create a named pipe to communicate with the console thread */ sprintf(name, "\\\\.\\pipe\\winansi%lu", GetCurrentProcessId()); @@ -574,8 +621,11 @@ void winansi_init(void) HANDLE winansi_get_osfhandle(int fd) { HANDLE hnd = (HANDLE) _get_osfhandle(fd); - if ((fd == 1 || fd == 2) && isatty(fd) - && GetFileType(hnd) == FILE_TYPE_PIPE) - return (fd == 1) ? hconsole1 : hconsole2; + if (isatty(fd) && GetFileType(hnd) == FILE_TYPE_PIPE) { + if (fd == 1 && hconsole1) + return hconsole1; + else if (fd == 2 && hconsole2) + return hconsole2; + } return hnd; } diff --git a/config.mak.uname b/config.mak.uname index 74272022ec..05259a95e9 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -548,7 +548,8 @@ else BASIC_LDFLAGS += -Wl,--large-address-aware endif CC = gcc - COMPAT_CFLAGS += -D__USE_MINGW_ANSI_STDIO=0 + COMPAT_CFLAGS += -D__USE_MINGW_ANSI_STDIO=0 -DDETECT_MSYS_TTY + EXTLIBS += -lntdll INSTALL = /bin/install NO_R_TO_GCC_LINKER = YesPlease INTERNAL_QSORT = YesPlease From b44734537a4ee1d6ffcc549749d1530ae36499d8 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 17 Mar 2015 19:41:02 +0100 Subject: [PATCH 42/55] UTF-8 environment: be a little bit more defensive It is unlikely that we have an empty environment, ever, but *if* we do, when `environ_size - 1` is passed to `bsearchenv()` it is misinterpreted as a real large integer. To make the code truly defensive, refuse to do anything at all if the size is negative (which should not happen, of course). Signed-off-by: Johannes Schindelin --- compat/mingw.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index a78ab64768..d60a48b8e0 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -1292,7 +1292,7 @@ static int bsearchenv(char **env, const char *name, size_t size) */ static int do_putenv(char **env, const char *name, int size, int free_old) { - int i = bsearchenv(env, name, size - 1); + int i = size <= 0 ? -1 : bsearchenv(env, name, size - 1); /* optionally free removed / replaced entry */ if (i >= 0 && free_old) @@ -1317,7 +1317,13 @@ static int do_putenv(char **env, const char *name, int size, int free_old) char *mingw_getenv(const char *name) { char *value; - int pos = bsearchenv(environ, name, environ_size - 1); + int pos; + + if (environ_size <= 0) + return NULL; + + pos = bsearchenv(environ, name, environ_size - 1); + if (pos < 0) return NULL; value = strchr(environ[pos], '='); From 8228e4cb4444e51c19b57869b90333a7f531d6cc Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 18 Mar 2015 10:57:52 +0100 Subject: [PATCH 43/55] mingw: be more defensive when making the environment block Outside of our Windows-specific code, the end of the environment can be marked also by a pointer to a NUL character, not only by a NULL pointer as our code assumed so far. That led to a buffer overrun in `make_environment_block()` when running `git-remote-https` in `mintty` (because `curl_global_init()` added the `CHARSET` environment variable *outside* of `mingw_putenv()`, ending the environment in a pointer to an empty string). Side note for future debugging on Windows: when running programs in `mintty`, the standard input/output/error is not connected to a Win32 Console, but instead is pipe()d. That means that even stderr may not be written completely before a crash, but has to be fflush()ed explicitly. For example, when debugging crashes, the developer should insert an `fflush(stderr);` at the end of the `error()` function defined in usage.c. Signed-off-by: Johannes Schindelin --- compat/mingw.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index d60a48b8e0..087e8a9a09 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -968,7 +968,7 @@ static wchar_t *make_environment_block(char **deltaenv) char **tmpenv; int i = 0, size = environ_size, wenvsz = 0, wenvpos = 0; - while (deltaenv && deltaenv[i]) + while (deltaenv && deltaenv[i] && *deltaenv[i]) i++; /* copy the environment, leaving space for changes */ @@ -976,11 +976,11 @@ static wchar_t *make_environment_block(char **deltaenv) memcpy(tmpenv, environ, size * sizeof(char*)); /* merge supplied environment changes into the temporary environment */ - for (i = 0; deltaenv && deltaenv[i]; i++) + for (i = 0; deltaenv && deltaenv[i] && *deltaenv[i]; i++) size = do_putenv(tmpenv, deltaenv[i], size, 0); /* create environment block from temporary environment */ - for (i = 0; tmpenv[i]; i++) { + for (i = 0; tmpenv[i] && *tmpenv[i]; i++) { size = 2 * strlen(tmpenv[i]) + 2; /* +2 for final \0 */ ALLOC_GROW(wenvblk, (wenvpos + size) * sizeof(wchar_t), wenvsz); wenvpos += xutftowcs(&wenvblk[wenvpos], tmpenv[i], size) + 1; From f650ef0409a8f35019e723fdc0952ef835793428 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 17 Mar 2015 19:41:31 +0100 Subject: [PATCH 44/55] mingw: be *very* wary about outside environment changes The environment is modified in most surprising circumstances, and not all of them are under Git's control. For example, calling curl_global_init() on Windows will ensure that the CHARSET variable is set, adding one if necessary. While the previous commit worked around crashes triggered by such outside changes of the environment by relaxing the requirement that the environment be terminated by a NULL pointer, the other assumption made by `mingw_getenv()` and `mingw_putenv()` is that the environment is sorted, for efficient lookup via binary search. Let's make real sure that our environment is intact before querying or modifying it, and reinitialize our idea of the environment if necessary. With this commit, before working on the environment we look briefly for indicators that the environment was modified outside of our control, and to ensure that it is terminated with a NULL pointer and sorted again in that case. Note: the indicators are maybe not sufficient. For example, when a variable is removed, it will not be noticed. It might also be a problem if outside changes to the environment result in a modified `environ` pointer: it is unclear whether such a modification could result in a problem when `mingw_putenv()` needs to `realloc()` the environment buffer. For the moment, however, the current fix works well enough, so let's only face the potential problems when (and if!) they occur. Signed-off-by: Johannes Schindelin --- compat/mingw.c | 49 +++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 087e8a9a09..c3d0eb1668 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -957,6 +957,10 @@ static int do_putenv(char **env, const char *name, int size, int free_old); static int environ_size = 0; /* allocated size of environ array, in bytes */ static int environ_alloc = 0; +/* used as a indicator when the environment has been changed outside mingw.c */ +static char **saved_environ; + +static void maybe_reinitialize_environ(void); /* * Create environment block suitable for CreateProcess. Merges current @@ -966,7 +970,10 @@ static wchar_t *make_environment_block(char **deltaenv) { wchar_t *wenvblk = NULL; char **tmpenv; - int i = 0, size = environ_size, wenvsz = 0, wenvpos = 0; + int i = 0, size, wenvsz = 0, wenvpos = 0; + + maybe_reinitialize_environ(); + size = environ_size; while (deltaenv && deltaenv[i] && *deltaenv[i]) i++; @@ -1269,6 +1276,41 @@ static int compareenv(const void *v1, const void *v2) } } +/* + * Functions implemented outside Git are able to modify the environment, + * too. For example, cURL's curl_global_init() function sets the CHARSET + * environment variable (at least in certain circumstances). + * + * Therefore we need to be *really* careful *not* to assume that we have + * sole control over the environment and reinitalize it when necessary. + */ +static void maybe_reinitialize_environ(void) +{ + int i; + + if (!saved_environ) { + warning("MinGW environment not initialized yet"); + return; + } + + if (environ_size <= 0) + return; + + if (saved_environ != environ) + /* We have *no* idea how much space was allocated outside */ + environ_alloc = 0; + else if (!environ[environ_size - 1]) + return; /* still consistent */ + + for (i = 0; environ[i] && *environ[i]; i++) + ; /* continue counting */ + environ[i] = NULL; + environ_size = i + 1; + + /* sort environment for O(log n) getenv / putenv */ + qsort(environ, i, sizeof(char*), compareenv); +} + static int bsearchenv(char **env, const char *name, size_t size) { unsigned low = 0, high = size; @@ -1322,6 +1364,7 @@ char *mingw_getenv(const char *name) if (environ_size <= 0) return NULL; + maybe_reinitialize_environ(); pos = bsearchenv(environ, name, environ_size - 1); if (pos < 0) @@ -1332,7 +1375,9 @@ char *mingw_getenv(const char *name) int mingw_putenv(const char *namevalue) { + maybe_reinitialize_environ(); ALLOC_GROW(environ, (environ_size + 1) * sizeof(char*), environ_alloc); + saved_environ = environ; environ_size = do_putenv(environ, namevalue, environ_size, 1); return 0; } @@ -2220,7 +2265,7 @@ void mingw_startup() */ environ_size = i + 1; environ_alloc = alloc_nr(environ_size * sizeof(char*)); - environ = malloc_startup(environ_alloc); + saved_environ = environ = malloc_startup(environ_alloc); /* allocate buffer (wchar_t encodes to max 3 UTF-8 bytes) */ maxlen = 3 * maxlen + 1; From 93e69e1442c280b88b2f6e452984d58113ebdec5 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 31 Mar 2015 17:39:54 +0100 Subject: [PATCH 45/55] mingw: uglify pthread_mutex_init definition to shut up warning When the result of a (a, 0) expression is not used, GCC now finds it necessary to complain with a warning: right-hand operand of comma expression has no effect Let's just pretend to use the 0 value and have a peaceful and quiet life again. Signed-off-by: Johannes Schindelin --- compat/nedmalloc/malloc.c.h | 3 ++- compat/win32/pthread.h | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/compat/nedmalloc/malloc.c.h b/compat/nedmalloc/malloc.c.h index 18634e3a3b..9df8aaed87 100644 --- a/compat/nedmalloc/malloc.c.h +++ b/compat/nedmalloc/malloc.c.h @@ -1804,9 +1804,10 @@ struct win32_mlock_t volatile long threadid; }; +static inline int return_0(int i) { return 0; } #define MLOCK_T struct win32_mlock_t #define CURRENT_THREAD win32_getcurrentthreadid() -#define INITIAL_LOCK(sl) (memset(sl, 0, sizeof(MLOCK_T)), 0) +#define INITIAL_LOCK(sl) (memset(sl, 0, sizeof(MLOCK_T)), return_0(0)) #define ACQUIRE_LOCK(sl) win32_acquire_lock(sl) #define RELEASE_LOCK(sl) win32_release_lock(sl) #define TRY_LOCK(sl) win32_try_lock(sl) diff --git a/compat/win32/pthread.h b/compat/win32/pthread.h index 1ade961769..62151d7550 100644 --- a/compat/win32/pthread.h +++ b/compat/win32/pthread.h @@ -18,7 +18,10 @@ */ #define pthread_mutex_t CRITICAL_SECTION -#define pthread_mutex_init(a,b) (InitializeCriticalSection((a)), 0) +static inline int return_0(int i) { + return 0; +} +#define pthread_mutex_init(a,b) return_0((InitializeCriticalSection((a)), 0)) #define pthread_mutex_destroy(a) DeleteCriticalSection((a)) #define pthread_mutex_lock EnterCriticalSection #define pthread_mutex_unlock LeaveCriticalSection From 1c9e3444b00b423e2e3bea3eafdde2f043eaed94 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 31 Mar 2015 17:51:57 +0100 Subject: [PATCH 46/55] mingw: avoid warnings when casting HANDLEs to int HANDLE is defined internally as a void *, but in many cases it is actually guaranteed to be a 32-bit integer. In these cases, GCC should not warn about a cast of a pointer to an integer of a different type because we know exactly what we are doing. Signed-off-by: Johannes Schindelin --- compat/mingw.c | 9 ++++++--- compat/poll/poll.c | 2 +- compat/winansi.c | 3 ++- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index c3d0eb1668..61e8d02caf 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -6,6 +6,8 @@ #include "../run-command.h" #include "../cache.h" +#define HCAST(type, handle) ((type)(intptr_t)handle) + static const int delay[] = { 0, 1, 10, 20, 40 }; unsigned int _CRT_fmode = _O_BINARY; @@ -726,13 +728,13 @@ int pipe(int filedes[2]) errno = err_win_to_posix(GetLastError()); return -1; } - filedes[0] = _open_osfhandle((int)h[0], O_NOINHERIT); + filedes[0] = _open_osfhandle(HCAST(int, h[0]), O_NOINHERIT); if (filedes[0] < 0) { CloseHandle(h[0]); CloseHandle(h[1]); return -1; } - filedes[1] = _open_osfhandle((int)h[1], O_NOINHERIT); + filedes[1] = _open_osfhandle(HCAST(int, h[1]), O_NOINHERIT); if (filedes[0] < 0) { close(filedes[0]); CloseHandle(h[1]); @@ -1949,7 +1951,8 @@ void mingw_open_html(const char *unixpath) die("cannot run browser"); printf("Launching default browser to display HTML ...\n"); - r = (int)ShellExecute(NULL, "open", htmlpath, NULL, "\\", SW_SHOWNORMAL); + r = HCAST(int, ShellExecute(NULL, "open", htmlpath, + NULL, "\\", SW_SHOWNORMAL)); FreeLibrary(shell32); /* see the MSDN documentation referring to the result codes here */ if (r <= 32) { diff --git a/compat/poll/poll.c b/compat/poll/poll.c index a9b41d89f4..300e95f12f 100644 --- a/compat/poll/poll.c +++ b/compat/poll/poll.c @@ -76,7 +76,7 @@ #ifdef WIN32_NATIVE -#define IsConsoleHandle(h) (((long) (h) & 3) == 3) +#define IsConsoleHandle(h) (((long) (intptr_t) (h) & 3) == 3) static BOOL IsSocketHandle (HANDLE h) diff --git a/compat/winansi.c b/compat/winansi.c index 23b0fe7e86..33910badb9 100644 --- a/compat/winansi.c +++ b/compat/winansi.c @@ -454,7 +454,8 @@ static HANDLE duplicate_handle(HANDLE hnd) HANDLE hresult, hproc = GetCurrentProcess(); if (!DuplicateHandle(hproc, hnd, hproc, &hresult, 0, TRUE, DUPLICATE_SAME_ACCESS)) - die_lasterr("DuplicateHandle(%li) failed", (long) hnd); + die_lasterr("DuplicateHandle(%li) failed", + (long) (intptr_t) hnd); return hresult; } From 9b0bd18277678d18e52a6bf279f856bb1288e9ff Mon Sep 17 00:00:00 2001 From: Sebastian Schuberth Date: Fri, 23 Mar 2012 10:58:37 +0100 Subject: [PATCH 47/55] am: Use cat instead of echo to avoid DOS line-endings (fixes t4150) Along the lines of 05d0e3b and f33946d, use cat instead of echo to avoid line ending mismatches in the test result of "am empty-file does not infloop" which make the test fail. Signed-off-by: Sebastian Schuberth --- git-am.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/git-am.sh b/git-am.sh index a67d0f9898..2aa0392c47 100755 --- a/git-am.sh +++ b/git-am.sh @@ -185,7 +185,9 @@ It does not apply to blobs recorded in its index.")" } clean_abort () { - test $# = 0 || echo >&2 "$@" + test $# = 0 || cat >&2 < Date: Mon, 9 Apr 2012 21:09:49 -0500 Subject: [PATCH 48/55] Windows: make sure that merge-octopus only outputs LF line endings This happens to shut up t7602 on Windows which would otherwise take the different line endings for a sign that the merge failed. Signed-off-by: Johannes Schindelin --- git-merge-octopus.sh | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/git-merge-octopus.sh b/git-merge-octopus.sh index 8643f74cb0..2226eeba99 100755 --- a/git-merge-octopus.sh +++ b/git-merge-octopus.sh @@ -71,7 +71,9 @@ do case "$LF$common$LF" in *"$LF$SHA1$LF"*) - echo "Already up-to-date with $pretty_name" + cat << EOF +Already up-to-date with $pretty_name +EOF continue ;; esac @@ -83,7 +85,9 @@ do # tree as the intermediate result of the merge. # We still need to count this as part of the parent set. - echo "Fast-forwarding to: $pretty_name" + cat << EOF +Fast-forwarding to: $pretty_name +EOF git read-tree -u -m $head $SHA1 || exit MRC=$SHA1 MRT=$(git write-tree) continue @@ -91,7 +95,9 @@ do NON_FF_MERGE=1 - echo "Trying simple merge with $pretty_name" + cat << EOF +Trying simple merge with $pretty_name +EOF git read-tree -u -m --aggressive $common $MRT $SHA1 || exit 2 next=$(git write-tree 2>/dev/null) if test $? -ne 0 From 3bbb099d799ae95b143e0a958ed0b11b3e41082d Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 6 Apr 2015 14:34:59 +0100 Subject: [PATCH 49/55] git-gui (MinGW): make use of MSys2's msgfmt When Git for Windows was still based on MSys1, we had no gettext, ergo no msgfmt, either. Therefore, we introduced a small and simple Tcl script to perform the same task. However, with MSys2, we no longer need that because we have a proper msgfmt executable. Plus, the po2msg.sh script somehow manages to hang when run in parallel. Two reasons to use real msgfmt.exe instead. Signed-off-by: Johannes Schindelin --- git-gui/Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/git-gui/Makefile b/git-gui/Makefile index cde8b2ea31..f1a46222ac 100644 --- a/git-gui/Makefile +++ b/git-gui/Makefile @@ -161,7 +161,9 @@ ifeq ($(uname_S),Darwin) endif endif ifneq (,$(findstring MINGW,$(uname_S))) +ifeq ($(shell expr "$(uname_R)" : '1\.'),2) NO_MSGFMT=1 +endif GITGUI_WINDOWS_WRAPPER := YesPlease GITGUI_RELATIVE := 1 endif From 73ab2cd98bc1d01e41c2eb0db81da3e82ba0edb4 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 6 Apr 2015 10:37:04 +0100 Subject: [PATCH 50/55] Avoid illegal filenames when building Documentation on NTFS A '+' is not a valid part of a filename with Windows file systems (it is reserved because the '+' operator meant file concatenation back in the DOS days). Let's just not use it. Signed-off-by: Johannes Schindelin --- Documentation/Makefile | 84 +++++++++++++++++++++--------------------- 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/Documentation/Makefile b/Documentation/Makefile index df3901d83a..d1afd45e9a 100644 --- a/Documentation/Makefile +++ b/Documentation/Makefile @@ -265,9 +265,9 @@ docdep_prereqs = \ cmd-list.made $(cmds_txt) doc.dep : $(docdep_prereqs) $(wildcard *.txt) build-docdep.perl - $(QUIET_GEN)$(RM) $@+ $@ && \ - $(PERL_PATH) ./build-docdep.perl >$@+ $(QUIET_STDERR) && \ - mv $@+ $@ + $(QUIET_GEN)$(RM) $@.new $@ && \ + $(PERL_PATH) ./build-docdep.perl >$@.new $(QUIET_STDERR) && \ + mv $@.new $@ -include doc.dep @@ -303,8 +303,8 @@ mergetools-list.made: ../git-mergetool--lib.sh $(wildcard ../mergetools/*) date >$@ clean: - $(RM) *.xml *.xml+ *.html *.html+ *.1 *.5 *.7 - $(RM) *.texi *.texi+ *.texi++ git.info gitman.info + $(RM) *.xml *.xml.new *.html *.html.new *.1 *.5 *.7 + $(RM) *.texi *.texi.new *.texi.new.new git.info gitman.info $(RM) *.pdf $(RM) howto-index.txt howto/*.html doc.dep $(RM) technical/*.html technical/api-index.txt @@ -312,14 +312,14 @@ clean: $(RM) manpage-base-url.xsl $(MAN_HTML): %.html : %.txt asciidoc.conf - $(QUIET_ASCIIDOC)$(RM) $@+ $@ && \ - $(TXT_TO_HTML) -d manpage -o $@+ $< && \ - mv $@+ $@ + $(QUIET_ASCIIDOC)$(RM) $@.new $@ && \ + $(TXT_TO_HTML) -d manpage -o $@.new $< && \ + mv $@.new $@ $(OBSOLETE_HTML): %.html : %.txto asciidoc.conf - $(QUIET_ASCIIDOC)$(RM) $@+ $@ && \ - $(TXT_TO_HTML) -o $@+ $< && \ - mv $@+ $@ + $(QUIET_ASCIIDOC)$(RM) $@.new $@ && \ + $(TXT_TO_HTML) -o $@.new $< && \ + mv $@.new $@ manpage-base-url.xsl: manpage-base-url.xsl.in sed "s|@@MAN_BASE_URL@@|$(MAN_BASE_URL)|" $< > $@ @@ -329,14 +329,14 @@ manpage-base-url.xsl: manpage-base-url.xsl.in $(XMLTO) -m $(MANPAGE_XSL) $(XMLTO_EXTRA) man $< %.xml : %.txt asciidoc.conf - $(QUIET_ASCIIDOC)$(RM) $@+ $@ && \ - $(TXT_TO_XML) -d manpage -o $@+ $< && \ - mv $@+ $@ + $(QUIET_ASCIIDOC)$(RM) $@.new $@ && \ + $(TXT_TO_XML) -d manpage -o $@.new $< && \ + mv $@.new $@ user-manual.xml: user-manual.txt user-manual.conf - $(QUIET_ASCIIDOC)$(RM) $@+ $@ && \ - $(TXT_TO_XML) -d book -o $@+ $< && \ - mv $@+ $@ + $(QUIET_ASCIIDOC)$(RM) $@.new $@ && \ + $(TXT_TO_XML) -d book -o $@.new $< && \ + mv $@.new $@ technical/api-index.txt: technical/api-index-skel.txt \ technical/api-index.sh $(patsubst %,%.txt,$(API_DOCS)) @@ -350,45 +350,45 @@ XSLT = docbook.xsl XSLTOPTS = --xinclude --stringparam html.stylesheet docbook-xsl.css user-manual.html: user-manual.xml $(XSLT) - $(QUIET_XSLTPROC)$(RM) $@+ $@ && \ - xsltproc $(XSLTOPTS) -o $@+ $(XSLT) $< && \ - mv $@+ $@ + $(QUIET_XSLTPROC)$(RM) $@.new $@ && \ + xsltproc $(XSLTOPTS) -o $@.new $(XSLT) $< && \ + mv $@.new $@ git.info: user-manual.texi $(QUIET_MAKEINFO)$(MAKEINFO) --no-split -o $@ user-manual.texi user-manual.texi: user-manual.xml - $(QUIET_DB2TEXI)$(RM) $@+ $@ && \ - $(DOCBOOK2X_TEXI) user-manual.xml --encoding=UTF-8 --to-stdout >$@++ && \ - $(PERL_PATH) fix-texi.perl <$@++ >$@+ && \ - rm $@++ && \ - mv $@+ $@ + $(QUIET_DB2TEXI)$(RM) $@.new $@ && \ + $(DOCBOOK2X_TEXI) user-manual.xml --encoding=UTF-8 --to-stdout >$@.new.new && \ + $(PERL_PATH) fix-texi.perl <$@.new.new >$@.new && \ + rm $@.new.new && \ + mv $@.new $@ user-manual.pdf: user-manual.xml - $(QUIET_DBLATEX)$(RM) $@+ $@ && \ - $(DBLATEX) -o $@+ -p $(ASCIIDOC_DBLATEX_DIR)/asciidoc-dblatex.xsl -s $(ASCIIDOC_DBLATEX_DIR)/asciidoc-dblatex.sty $< && \ - mv $@+ $@ + $(QUIET_DBLATEX)$(RM) $@.new $@ && \ + $(DBLATEX) -o $@.new -p $(ASCIIDOC_DBLATEX_DIR)/asciidoc-dblatex.xsl -s $(ASCIIDOC_DBLATEX_DIR)/asciidoc-dblatex.sty $< && \ + mv $@.new $@ gitman.texi: $(MAN_XML) cat-texi.perl - $(QUIET_DB2TEXI)$(RM) $@+ $@ && \ + $(QUIET_DB2TEXI)$(RM) $@.new $@ && \ ($(foreach xml,$(MAN_XML),$(DOCBOOK2X_TEXI) --encoding=UTF-8 \ - --to-stdout $(xml) &&) true) > $@++ && \ - $(PERL_PATH) cat-texi.perl $@ <$@++ >$@+ && \ - rm $@++ && \ - mv $@+ $@ + --to-stdout $(xml) &&) true) > $@.new.new && \ + $(PERL_PATH) cat-texi.perl $@ <$@.new.new >$@.new && \ + rm $@.new.new && \ + mv $@.new $@ gitman.info: gitman.texi $(QUIET_MAKEINFO)$(MAKEINFO) --no-split --no-validate $*.texi $(patsubst %.txt,%.texi,$(MAN_TXT)): %.texi : %.xml - $(QUIET_DB2TEXI)$(RM) $@+ $@ && \ - $(DOCBOOK2X_TEXI) --to-stdout $*.xml >$@+ && \ - mv $@+ $@ + $(QUIET_DB2TEXI)$(RM) $@.new $@ && \ + $(DOCBOOK2X_TEXI) --to-stdout $*.xml >$@.new && \ + mv $@.new $@ howto-index.txt: howto-index.sh $(wildcard howto/*.txt) - $(QUIET_GEN)$(RM) $@+ $@ && \ - '$(SHELL_PATH_SQ)' ./howto-index.sh $(sort $(wildcard howto/*.txt)) >$@+ && \ - mv $@+ $@ + $(QUIET_GEN)$(RM) $@.new $@ && \ + '$(SHELL_PATH_SQ)' ./howto-index.sh $(sort $(wildcard howto/*.txt)) >$@.new && \ + mv $@.new $@ $(patsubst %,%.html,$(ARTICLES)) : %.html : %.txt $(QUIET_ASCIIDOC)$(TXT_TO_HTML) $*.txt @@ -397,10 +397,10 @@ WEBDOC_DEST = /pub/software/scm/git/docs howto/%.html: ASCIIDOC_EXTRA += -a git-relative-html-prefix=../ $(patsubst %.txt,%.html,$(wildcard howto/*.txt)): %.html : %.txt - $(QUIET_ASCIIDOC)$(RM) $@+ $@ && \ + $(QUIET_ASCIIDOC)$(RM) $@.new $@ && \ sed -e '1,/^$$/d' $< | \ - $(TXT_TO_HTML) - >$@+ && \ - mv $@+ $@ + $(TXT_TO_HTML) - >$@.new && \ + mv $@.new $@ install-webdoc : html '$(SHELL_PATH_SQ)' ./install-webdoc.sh $(WEBDOC_DEST) From c3389bd482956367a47b482a4dcf37879e16b820 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Thu, 9 Apr 2015 16:19:56 +0100 Subject: [PATCH 51/55] gettext: always use UTF-8 on native Windows Git on native Windows exclusively uses UTF-8 for console output (both with mintty and native console windows). Gettext uses setlocale() to determine the output encoding for translated text, however, MSVCRT's setlocale() doesn't support UTF-8. As a result, translated text is encoded in system encoding (GetAPC()), and non-ASCII chars are mangled in console output. Use gettext's bind_textdomain_codeset() to force the encoding to UTF-8 on native Windows. In this developers' setup, HAVE_LIBCHARSET_H is apparently defined, but we *really* want to override the locale_charset() here. Signed-off-by: Karsten Blees --- gettext.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/gettext.c b/gettext.c index 8b2da4641f..df669648d8 100644 --- a/gettext.c +++ b/gettext.c @@ -10,7 +10,9 @@ #ifndef NO_GETTEXT # include # include -# ifdef HAVE_LIBCHARSET_H +# ifdef GIT_WINDOWS_NATIVE +# define locale_charset() "UTF-8" +# elif defined HAVE_LIBCHARSET_H # include # else # include From b5bf51779a1ba037e9f2e9440b038e6855e72af6 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 7 Apr 2015 10:24:55 +0100 Subject: [PATCH 52/55] mingw: HOT FIX: work around environment issues -- again This developer should really, really have known better. The fact that we are changing the environment in ways for which the MSVCRT is not prepared for is bad enough. But then this developer followed the request to re-enable nedmalloc -- despite the prediction that it would cause an access violation, predicting it in the same message as the request to re-enable nedmalloc, no less! To paper over the issue until the time when this developer finds the time to re-design the Unicode environment handling from scratch, let's hope that cURL is the only library we are using that *may* set an environment variable using MSVCRT's putenv() after we fscked the environment up. Note: this commit can serve as no source of pride to anyone, certainly not yours truly. It is necessary as a quick and pragmatic stop gap, though, to prevent worse problems. Note: cURL manages to set the variable CHARSET when nedmalloc is *not* enabled, without causing an access violation. In that case, it sets it successfully to the value "cp" + GetACP() (hence it is our choice, too, cURL may need it, Git does not): https://github.com/bagder/curl/blob/aa5808b5/lib/easy.c#L157-L162 Signed-off-by: Johannes Schindelin --- compat/mingw.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/compat/mingw.c b/compat/mingw.c index 61e8d02caf..020344c15e 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2319,6 +2319,17 @@ void mingw_startup() if (!getenv("TERM")) setenv("TERM", "cygwin", 1); + /* + * Avoid a segmentation fault when cURL tries to set the CHARSET + * variable and putenv() barfs at our nedmalloc'ed environment. + */ + if (!getenv("CHARSET")) { + struct strbuf buf = STRBUF_INIT; + strbuf_addf(&buf, "cp%u", GetACP()); + setenv("CHARSET", buf.buf, 1); + strbuf_release(&buf); + } + /* initialize critical section for waitpid pinfo_t list */ InitializeCriticalSection(&pinfo_cs); From 216bd1831fa7c9292a2543cd30d3375c82ba2cc4 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Thu, 2 Apr 2015 19:37:08 +0200 Subject: [PATCH 53/55] mingw: factor out Windows specific environment setup Signed-off-by: Karsten Blees --- compat/mingw.c | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 020344c15e..de6b52b409 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2209,6 +2209,22 @@ int handle_long_path(wchar_t *path, int len, int max_path, int expand) } } +static void setup_windows_environment() +{ + /* on Windows it is TMP and TEMP */ + if (!getenv("TMPDIR")) { + const char *tmp = getenv("TMP"); + if (!tmp) + tmp = getenv("TEMP"); + if (tmp) + setenv("TMPDIR", tmp, 1); + } + + /* simulate TERM to enable auto-color (see color.c) */ + if (!getenv("TERM")) + setenv("TERM", "cygwin", 1); +} + /* * Disable MSVCRT command line wildcard expansion (__getmainargs called from * mingw startup code, see init.c in mingw runtime). @@ -2305,19 +2321,7 @@ void mingw_startup() qsort(environ, i, sizeof(char*), compareenv); /* fix Windows specific environment settings */ - - /* on Windows it is TMP and TEMP */ - if (!mingw_getenv("TMPDIR")) { - const char *tmp = mingw_getenv("TMP"); - if (!tmp) - tmp = mingw_getenv("TEMP"); - if (tmp) - setenv("TMPDIR", tmp, 1); - } - - /* simulate TERM to enable auto-color (see color.c) */ - if (!getenv("TERM")) - setenv("TERM", "cygwin", 1); + setup_windows_environment(); /* * Avoid a segmentation fault when cURL tries to set the CHARSET From 15b152e458810838a46006f055d07c39ee9a1f1e Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Mon, 6 Apr 2015 21:47:07 +0200 Subject: [PATCH 54/55] mingw: move MSys2 specific environment tweaks to setup_windows_environment Lets keep the environment initialization and conversion section as lean as possible and move recently added tweaks to setup_windows_environment(). This fixes the following potential problems: * Prevent duplicate TZ variables if both TZ and MSYS2_TZ are set. * Some of the higher level x* APIs from wrapper.c require a working getenv(), using e.g. xstrdup() during initialization is dangerous. * Slashifying the Windows TMP variable may break native Windows programs, use POSIX TMPDIR instead. * Properly slashify TMPDIR even if it is already set, and also if we only have TEMP, but not TMP. * Reduce complexity from O(n) to O(log n). Signed-off-by: Karsten Blees --- compat/mingw.c | 40 +++++++++++++++++++--------------------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index de6b52b409..9af14ea9b7 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2211,15 +2211,31 @@ int handle_long_path(wchar_t *path, int len, int max_path, int expand) static void setup_windows_environment() { + char *tmp; + /* on Windows it is TMP and TEMP */ if (!getenv("TMPDIR")) { - const char *tmp = getenv("TMP"); - if (!tmp) + if (!(tmp = getenv("TMP"))) tmp = getenv("TEMP"); if (tmp) setenv("TMPDIR", tmp, 1); } + if ((tmp = getenv("TMPDIR"))) { + /* + * Convert all dir separators to forward slashes, + * to help shell commands called from the Git + * executable (by not mistaking the dir separators + * for escape characters). + */ + for (; *tmp; tmp++) + if (*tmp == '\\') + *tmp = '/'; + } + + if (!getenv("TZ") && (tmp = getenv("MSYS2_TZ"))) + setenv("TZ", tmp, 1); + /* simulate TERM to enable auto-color (see color.c) */ if (!getenv("TERM")) setenv("TERM", "cygwin", 1); @@ -2294,26 +2310,8 @@ void mingw_startup() __argv[0] = wcstoutfdup_startup(buffer, _wpgmptr, maxlen); for (i = 1; i < argc; i++) __argv[i] = wcstoutfdup_startup(buffer, wargv[i], maxlen); - for (i = 0; wenv[i]; i++) { + for (i = 0; wenv[i]; i++) environ[i] = wcstoutfdup_startup(buffer, wenv[i], maxlen); - if (!strncasecmp(environ[i], "MSYS2_TZ=", 9)) { - char *to_free = environ[i]; - environ[i] = xstrdup(to_free + 6); - free(to_free); - } - if (!strncasecmp(environ[i], "TMP=", 4)) { - /* - * Convert all dir separators to forward slashes, - * to help shell commands called from the Git - * executable (by not mistaking the dir separators - * for escape characters). - */ - char *p; - for (p = environ[i]; *p; p++) - if (*p == '\\') - *p = '/'; - } - } environ[i] = NULL; free(buffer); From 4c0c4bfbc631215468267d035f4bb911078042ab Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Mon, 6 Apr 2015 21:37:18 +0200 Subject: [PATCH 55/55] mingw: initialize HOME on startup HOME initialization was historically duplicated in many different places, including /etc/profile, launch scripts such as git-bash.vbs and gitk.cmd, and (although slightly broken) in the git-wrapper. Even unrelated projects such as GitExtensions and TortoiseGit need to implement the same logic to be able to call git directly. Initialize HOME in git's own startup code so that we can eventually retire all the duplicate initialization code. Signed-off-by: Karsten Blees --- compat/mingw.c | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/compat/mingw.c b/compat/mingw.c index 9af14ea9b7..f11a399611 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2239,6 +2239,30 @@ static void setup_windows_environment() /* simulate TERM to enable auto-color (see color.c) */ if (!getenv("TERM")) setenv("TERM", "cygwin", 1); + + /* calculate HOME if not set */ + if (!getenv("HOME")) { + /* + * try $HOMEDRIVE$HOMEPATH - the home share may be a network + * location, thus also check if the path exists (i.e. is not + * disconnected) + */ + if ((tmp = getenv("HOMEDRIVE"))) { + struct strbuf buf = STRBUF_INIT; + strbuf_addstr(&buf, tmp); + if ((tmp = getenv("HOMEPATH"))) { + strbuf_addstr(&buf, tmp); + if (is_directory(buf.buf)) + setenv("HOME", buf.buf, 1); + else + tmp = NULL; /* use $USERPROFILE */ + } + strbuf_release(&buf); + } + /* use $USERPROFILE if the home share is not available */ + if (!tmp && (tmp = getenv("USERPROFILE"))) + setenv("HOME", tmp, 1); + } } /*