From 7ca583891f0e7b69b27933c8efeaa17be6cc5acf Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sat, 6 Jul 2013 02:09:35 +0200 Subject: [PATCH 01/36] Win32: make FILETIME conversion functions public Signed-off-by: Karsten Blees --- compat/mingw.c | 16 ---------------- compat/mingw.h | 16 ++++++++++++++++ 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 00191fe9db..2d20e17715 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -580,22 +580,6 @@ int mingw_chmod(const char *filename, int mode) return _wchmod(wfilename, mode); } -/* - * The unit of FILETIME is 100-nanoseconds since January 1, 1601, UTC. - * Returns the 100-nanoseconds ("hekto nanoseconds") since the epoch. - */ -static inline long long filetime_to_hnsec(const FILETIME *ft) -{ - long long winTime = ((long long)ft->dwHighDateTime << 32) + ft->dwLowDateTime; - /* Windows to Unix Epoch conversion */ - return winTime - 116444736000000000LL; -} - -static inline time_t filetime_to_time_t(const FILETIME *ft) -{ - return (time_t)(filetime_to_hnsec(ft) / 10000000); -} - /** * Verifies that safe_create_leading_directories() would succeed. */ diff --git a/compat/mingw.h b/compat/mingw.h index 908279decc..8c603c8443 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -329,6 +329,22 @@ static inline int getrlimit(int resource, struct rlimit *rlp) return 0; } +/* + * The unit of FILETIME is 100-nanoseconds since January 1, 1601, UTC. + * Returns the 100-nanoseconds ("hekto nanoseconds") since the epoch. + */ +static inline long long filetime_to_hnsec(const FILETIME *ft) +{ + long long winTime = ((long long)ft->dwHighDateTime << 32) + ft->dwLowDateTime; + /* Windows to Unix Epoch conversion */ + return winTime - 116444736000000000LL; +} + +static inline time_t filetime_to_time_t(const FILETIME *ft) +{ + return (time_t)(filetime_to_hnsec(ft) / 10000000); +} + /* * Use mingw specific stat()/lstat()/fstat() implementations on Windows. */ From 4fcdefd04344df9e9815f8f886f6255ad7bb4dee Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sun, 8 Sep 2013 14:17:31 +0200 Subject: [PATCH 02/36] Win32: dirent.c: Move opendir down Move opendir down in preparation for the next patch. Signed-off-by: Karsten Blees --- compat/win32/dirent.c | 68 +++++++++++++++++++++---------------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/compat/win32/dirent.c b/compat/win32/dirent.c index 52420ec7d4..2603a0fa39 100644 --- a/compat/win32/dirent.c +++ b/compat/win32/dirent.c @@ -18,40 +18,6 @@ static inline void finddata2dirent(struct dirent *ent, WIN32_FIND_DATAW *fdata) ent->d_type = DT_REG; } -DIR *opendir(const char *name) -{ - wchar_t pattern[MAX_PATH + 2]; /* + 2 for '/' '*' */ - WIN32_FIND_DATAW fdata; - HANDLE h; - int len; - DIR *dir; - - /* convert name to UTF-16 and check length < MAX_PATH */ - if ((len = xutftowcs_path(pattern, name)) < 0) - return NULL; - - /* append optional '/' and wildcard '*' */ - if (len && !is_dir_sep(pattern[len - 1])) - pattern[len++] = '/'; - pattern[len++] = '*'; - pattern[len] = 0; - - /* open find handle */ - h = FindFirstFileW(pattern, &fdata); - if (h == INVALID_HANDLE_VALUE) { - DWORD err = GetLastError(); - errno = (err == ERROR_DIRECTORY) ? ENOTDIR : err_win_to_posix(err); - return NULL; - } - - /* initialize DIR structure and copy first dir entry */ - dir = xmalloc(sizeof(DIR)); - dir->dd_handle = h; - dir->dd_stat = 0; - finddata2dirent(&dir->dd_dir, &fdata); - return dir; -} - struct dirent *readdir(DIR *dir) { if (!dir) { @@ -90,3 +56,37 @@ int closedir(DIR *dir) free(dir); return 0; } + +DIR *opendir(const char *name) +{ + wchar_t pattern[MAX_PATH + 2]; /* + 2 for '/' '*' */ + WIN32_FIND_DATAW fdata; + HANDLE h; + int len; + DIR *dir; + + /* convert name to UTF-16 and check length < MAX_PATH */ + if ((len = xutftowcs_path(pattern, name)) < 0) + return NULL; + + /* append optional '/' and wildcard '*' */ + if (len && !is_dir_sep(pattern[len - 1])) + pattern[len++] = '/'; + pattern[len++] = '*'; + pattern[len] = 0; + + /* open find handle */ + h = FindFirstFileW(pattern, &fdata); + if (h == INVALID_HANDLE_VALUE) { + DWORD err = GetLastError(); + errno = (err == ERROR_DIRECTORY) ? ENOTDIR : err_win_to_posix(err); + return NULL; + } + + /* initialize DIR structure and copy first dir entry */ + dir = xmalloc(sizeof(DIR)); + dir->dd_handle = h; + dir->dd_stat = 0; + finddata2dirent(&dir->dd_dir, &fdata); + return dir; +} From ce9a20460552eca71abf94a2bbbc75563414346a Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 30 Aug 2017 15:26:35 +0200 Subject: [PATCH 03/36] config: rename `dummy` parameter to `cb` in git_default_config() This is the convention elsewhere (and prepares for the case where we may need to pass callback data). Signed-off-by: Johannes Schindelin --- config.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config.c b/config.c index 3461993f0a..f2d89ae2e7 100644 --- a/config.c +++ b/config.c @@ -1448,13 +1448,13 @@ static int git_default_mailmap_config(const char *var, const char *value) return 0; } -int git_default_config(const char *var, const char *value, void *dummy) +int git_default_config(const char *var, const char *value, void *cb) { if (starts_with(var, "core.")) return git_default_core_config(var, value); if (starts_with(var, "user.")) - return git_ident_config(var, value, dummy); + return git_ident_config(var, value, cb); if (starts_with(var, "i18n.")) return git_default_i18n_config(var, value); From 256271ba7bae019a99aeb88efcf7cba1d4f925fa Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sun, 8 Sep 2013 14:18:40 +0200 Subject: [PATCH 04/36] Win32: Make the dirent implementation pluggable Emulating the POSIX dirent API on Windows via FindFirstFile/FindNextFile is pretty staightforward, however, most of the information provided in the WIN32_FIND_DATA structure is thrown away in the process. A more sophisticated implementation may cache this data, e.g. for later reuse in calls to lstat. Make the dirent implementation pluggable so that it can be switched at runtime, e.g. based on a config option. Define a base DIR structure with pointers to readdir/closedir that match the opendir implementation (i.e. similar to vtable pointers in OOP). Define readdir/closedir so that they call the function pointers in the DIR structure. This allows to choose the opendir implementation on a call-by-call basis. Move the fixed sized dirent.d_name buffer to the dirent-specific DIR structure, as d_name may be implementation specific (e.g. a caching implementation may just set d_name to point into the cache instead of copying the entire file name string). Signed-off-by: Karsten Blees --- compat/win32/dirent.c | 27 +++++++++++++++++---------- compat/win32/dirent.h | 26 +++++++++++++++++++------- 2 files changed, 36 insertions(+), 17 deletions(-) diff --git a/compat/win32/dirent.c b/compat/win32/dirent.c index 2603a0fa39..6b87042182 100644 --- a/compat/win32/dirent.c +++ b/compat/win32/dirent.c @@ -1,15 +1,19 @@ #include "../../git-compat-util.h" -struct DIR { +typedef struct dirent_DIR { + struct DIR base_dir; /* extend base struct DIR */ struct dirent dd_dir; /* includes d_type */ HANDLE dd_handle; /* FindFirstFile handle */ int dd_stat; /* 0-based index */ -}; + char dd_name[MAX_PATH * 3]; /* file name (* 3 for UTF-8 conversion) */ +} dirent_DIR; + +DIR *(*opendir)(const char *dirname) = dirent_opendir; static inline void finddata2dirent(struct dirent *ent, WIN32_FIND_DATAW *fdata) { - /* convert UTF-16 name to UTF-8 */ - xwcstoutf(ent->d_name, fdata->cFileName, sizeof(ent->d_name)); + /* convert UTF-16 name to UTF-8 (d_name points to dirent_DIR.dd_name) */ + xwcstoutf(ent->d_name, fdata->cFileName, MAX_PATH * 3); /* Set file type, based on WIN32_FIND_DATA */ if (fdata->dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) @@ -18,7 +22,7 @@ static inline void finddata2dirent(struct dirent *ent, WIN32_FIND_DATAW *fdata) ent->d_type = DT_REG; } -struct dirent *readdir(DIR *dir) +static struct dirent *dirent_readdir(dirent_DIR *dir) { if (!dir) { errno = EBADF; /* No set_errno for mingw */ @@ -45,7 +49,7 @@ struct dirent *readdir(DIR *dir) return &dir->dd_dir; } -int closedir(DIR *dir) +static int dirent_closedir(dirent_DIR *dir) { if (!dir) { errno = EBADF; @@ -57,13 +61,13 @@ int closedir(DIR *dir) return 0; } -DIR *opendir(const char *name) +DIR *dirent_opendir(const char *name) { wchar_t pattern[MAX_PATH + 2]; /* + 2 for '/' '*' */ WIN32_FIND_DATAW fdata; HANDLE h; int len; - DIR *dir; + dirent_DIR *dir; /* convert name to UTF-16 and check length < MAX_PATH */ if ((len = xutftowcs_path(pattern, name)) < 0) @@ -84,9 +88,12 @@ DIR *opendir(const char *name) } /* initialize DIR structure and copy first dir entry */ - dir = xmalloc(sizeof(DIR)); + dir = xmalloc(sizeof(dirent_DIR)); + dir->base_dir.preaddir = (struct dirent *(*)(DIR *dir)) dirent_readdir; + dir->base_dir.pclosedir = (int (*)(DIR *dir)) dirent_closedir; + dir->dd_dir.d_name = dir->dd_name; dir->dd_handle = h; dir->dd_stat = 0; finddata2dirent(&dir->dd_dir, &fdata); - return dir; + return (DIR*) dir; } diff --git a/compat/win32/dirent.h b/compat/win32/dirent.h index 058207e4bf..6b3ddee51b 100644 --- a/compat/win32/dirent.h +++ b/compat/win32/dirent.h @@ -1,20 +1,32 @@ #ifndef DIRENT_H #define DIRENT_H -typedef struct DIR DIR; - #define DT_UNKNOWN 0 #define DT_DIR 1 #define DT_REG 2 #define DT_LNK 3 struct dirent { - unsigned char d_type; /* file type to prevent lstat after readdir */ - char d_name[MAX_PATH * 3]; /* file name (* 3 for UTF-8 conversion) */ + unsigned char d_type; /* file type to prevent lstat after readdir */ + char *d_name; /* file name */ }; -DIR *opendir(const char *dirname); -struct dirent *readdir(DIR *dir); -int closedir(DIR *dir); +/* + * Base DIR structure, contains pointers to readdir/closedir implementations so + * that opendir may choose a concrete implementation on a call-by-call basis. + */ +typedef struct DIR { + struct dirent *(*preaddir)(struct DIR *dir); + int (*pclosedir)(struct DIR *dir); +} DIR; + +/* default dirent implementation */ +extern DIR *dirent_opendir(const char *dirname); + +/* current dirent implementation */ +extern DIR *(*opendir)(const char *dirname); + +#define readdir(dir) (dir->preaddir(dir)) +#define closedir(dir) (dir->pclosedir(dir)) #endif /* DIRENT_H */ From 91354c0a2c998023eac2428ef12359d10c14ac45 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 17 Nov 2016 17:24:10 +0100 Subject: [PATCH 05/36] Allow for platform-specific core.* config settings In the Git for Windows project, we have ample precendent for config settings that apply to Windows, and to Windows only. Let's formalize this concept by introducing a platform_core_config() function that can be #define'd in a platform-specific manner. This will allow us to contain platform-specific code better, as the corresponding variables no longer need to be exported so that they can be defined in environment.c and be set in config.c Signed-off-by: Johannes Schindelin --- compat/mingw.c | 5 +++++ compat/mingw.h | 3 +++ config.c | 6 +++--- git-compat-util.h | 8 ++++++++ 4 files changed, 19 insertions(+), 3 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 858ca14a57..303c2e518c 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -202,6 +202,11 @@ static int ask_yes_no_if_possible(const char *format, ...) } } +int mingw_core_config(const char *var, const char *value, void *cb) +{ + return 0; +} + int mingw_unlink(const char *pathname) { int ret, tries = 0; diff --git a/compat/mingw.h b/compat/mingw.h index 571019d0bd..908279decc 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -11,6 +11,9 @@ typedef _sigset_t sigset_t; #undef _POSIX_THREAD_SAFE_FUNCTIONS #endif +extern int mingw_core_config(const char *var, const char *value, void *cb); +#define platform_core_config mingw_core_config + /* * things that are not available in header files */ diff --git a/config.c b/config.c index f2d89ae2e7..8b3e0f0c66 100644 --- a/config.c +++ b/config.c @@ -1093,7 +1093,7 @@ int git_config_color(char *dest, const char *var, const char *value) return 0; } -static int git_default_core_config(const char *var, const char *value) +static int git_default_core_config(const char *var, const char *value, void *cb) { /* This needs a better name */ if (!strcmp(var, "core.filemode")) { @@ -1363,7 +1363,7 @@ static int git_default_core_config(const char *var, const char *value) } /* Add other config variables here and to Documentation/config.txt. */ - return 0; + return platform_core_config(var, value, cb); } static int git_default_i18n_config(const char *var, const char *value) @@ -1451,7 +1451,7 @@ static int git_default_mailmap_config(const char *var, const char *value) int git_default_config(const char *var, const char *value, void *cb) { if (starts_with(var, "core.")) - return git_default_core_config(var, value); + return git_default_core_config(var, value, cb); if (starts_with(var, "user.")) return git_ident_config(var, value, cb); diff --git a/git-compat-util.h b/git-compat-util.h index 5f2e90932f..7513ec4949 100644 --- a/git-compat-util.h +++ b/git-compat-util.h @@ -342,6 +342,14 @@ typedef uintmax_t timestamp_t; #define _PATH_DEFPATH "/usr/local/bin:/usr/bin:/bin" #endif +#ifndef platform_core_config +static inline int noop_core_config(const char *var, const char *value, void *cb) +{ + return 0; +} +#define platform_core_config noop_core_config +#endif + #ifndef has_dos_drive_prefix static inline int git_has_dos_drive_prefix(const char *path) { From 34f4dc3dc4182685893d0a1dd2880bc34996de20 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sun, 8 Sep 2013 14:21:30 +0200 Subject: [PATCH 06/36] Win32: make the lstat implementation pluggable Emulating the POSIX lstat API on Windows via GetFileAttributes[Ex] is quite slow. Windows operating system APIs seem to be much better at scanning the status of entire directories than checking single files. A caching implementation may improve performance by bulk-reading entire directories or reusing data obtained via opendir / readdir. Make the lstat implementation pluggable so that it can be switched at runtime, e.g. based on a config option. Signed-off-by: Karsten Blees --- compat/mingw.c | 2 ++ compat/mingw.h | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/compat/mingw.c b/compat/mingw.c index 2d20e17715..146a5069bd 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -719,6 +719,8 @@ static int do_stat_internal(int follow, const char *file_name, struct stat *buf) return do_lstat(follow, alt_name, buf); } +int (*lstat)(const char *file_name, struct stat *buf) = mingw_lstat; + int mingw_lstat(const char *file_name, struct stat *buf) { return do_stat_internal(0, file_name, buf); diff --git a/compat/mingw.h b/compat/mingw.h index 8c603c8443..33544e141a 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -368,7 +368,7 @@ int mingw_fstat(int fd, struct stat *buf); #ifdef lstat #undef lstat #endif -#define lstat mingw_lstat +extern int (*lstat)(const char *file_name, struct stat *buf); #ifndef _stati64 # define _stati64(x,y) mingw_stat(x,y) From f93953411e1702b8ea045ef6480098a19eda4299 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 17 Nov 2016 17:33:06 +0100 Subject: [PATCH 07/36] Move Windows-specific config settings into compat/mingw.c Signed-off-by: Johannes Schindelin --- cache.h | 8 -------- compat/mingw.c | 17 +++++++++++++++++ config.c | 8 -------- environment.c | 1 - 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/cache.h b/cache.h index 4d014541ab..f49d9fbac8 100644 --- a/cache.h +++ b/cache.h @@ -901,14 +901,6 @@ int use_optional_locks(void); extern char comment_line_char; extern int auto_comment_line_char; -/* Windows only */ -enum hide_dotfiles_type { - HIDE_DOTFILES_FALSE = 0, - HIDE_DOTFILES_TRUE, - HIDE_DOTFILES_DOTGITONLY -}; -extern enum hide_dotfiles_type hide_dotfiles; - enum log_refs_config { LOG_REFS_UNSET = -1, LOG_REFS_NONE = 0, diff --git a/compat/mingw.c b/compat/mingw.c index 303c2e518c..276a8d73f4 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -202,8 +202,25 @@ static int ask_yes_no_if_possible(const char *format, ...) } } +/* Windows only */ +enum hide_dotfiles_type { + HIDE_DOTFILES_FALSE = 0, + HIDE_DOTFILES_TRUE, + HIDE_DOTFILES_DOTGITONLY +}; + +static enum hide_dotfiles_type hide_dotfiles = HIDE_DOTFILES_DOTGITONLY; + int mingw_core_config(const char *var, const char *value, void *cb) { + if (!strcmp(var, "core.hidedotfiles")) { + if (value && !strcasecmp(value, "dotgitonly")) + hide_dotfiles = HIDE_DOTFILES_DOTGITONLY; + else + hide_dotfiles = git_config_bool(var, value); + return 0; + } + return 0; } diff --git a/config.c b/config.c index 8b3e0f0c66..221de4bc0f 100644 --- a/config.c +++ b/config.c @@ -1344,14 +1344,6 @@ static int git_default_core_config(const char *var, const char *value, void *cb) return 0; } - if (!strcmp(var, "core.hidedotfiles")) { - if (value && !strcasecmp(value, "dotgitonly")) - hide_dotfiles = HIDE_DOTFILES_DOTGITONLY; - else - hide_dotfiles = git_config_bool(var, value); - return 0; - } - if (!strcmp(var, "core.partialclonefilter")) { return git_config_string(&core_partial_clone_filter_default, var, value); diff --git a/environment.c b/environment.c index 3f3c8746c2..30ecd4e784 100644 --- a/environment.c +++ b/environment.c @@ -71,7 +71,6 @@ int core_apply_sparse_checkout; int merge_log_config = -1; int precomposed_unicode = -1; /* see probe_utf8_pathname_composition() */ unsigned long pack_size_limit_cfg; -enum hide_dotfiles_type hide_dotfiles = HIDE_DOTFILES_DOTGITONLY; enum log_refs_config log_all_ref_updates = LOG_REFS_UNSET; #ifndef PROTECT_HFS_DEFAULT From 6462b6911699a5f1d0a273e5bc2fb209e28b05f2 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sun, 8 Sep 2013 14:23:27 +0200 Subject: [PATCH 08/36] add infrastructure for read-only file system level caches Add a macro to mark code sections that only read from the file system, along with a config option and documentation. This facilitates implementation of relatively simple file system level caches without the need to synchronize with the file system. Enable read-only sections for 'git status' and preload_index. Signed-off-by: Karsten Blees --- Documentation/config.txt | 6 ++++++ builtin/commit.c | 1 + compat/mingw.c | 6 ++++++ compat/mingw.h | 2 ++ git-compat-util.h | 15 +++++++++++++++ preload-index.c | 2 ++ 6 files changed, 32 insertions(+) diff --git a/Documentation/config.txt b/Documentation/config.txt index b25f1c0dc4..3aa9df09ed 100644 --- a/Documentation/config.txt +++ b/Documentation/config.txt @@ -909,6 +909,12 @@ relatively high IO latencies. When enabled, Git will do the index comparison to the filesystem data in parallel, allowing overlapping IO's. Defaults to true. +core.fscache:: + Enable additional caching of file system data for some operations. ++ +Git for Windows uses this to bulk-read and cache lstat data of entire +directories (instead of doing lstat file by file). + core.unsetenvvars:: EXPERIMENTAL, Windows-only: comma-separated list of environment variables' names that need to be unset before spawning any other diff --git a/builtin/commit.c b/builtin/commit.c index 0d9828e29e..f6be863a0a 100644 --- a/builtin/commit.c +++ b/builtin/commit.c @@ -1355,6 +1355,7 @@ int cmd_status(int argc, const char **argv, const char *prefix) PATHSPEC_PREFER_FULL, prefix, argv); + enable_fscache(1); read_cache_preload(&s.pathspec); refresh_index(&the_index, REFRESH_QUIET|REFRESH_UNMERGED, &s.pathspec, NULL, NULL); diff --git a/compat/mingw.c b/compat/mingw.c index 146a5069bd..ba146ad270 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -211,6 +211,7 @@ enum hide_dotfiles_type { static enum hide_dotfiles_type hide_dotfiles = HIDE_DOTFILES_DOTGITONLY; static char *unset_environment_variables; +int core_fscache; int mingw_core_config(const char *var, const char *value, void *cb) { @@ -222,6 +223,11 @@ int mingw_core_config(const char *var, const char *value, void *cb) return 0; } + if (!strcmp(var, "core.fscache")) { + core_fscache = git_config_bool(var, value); + return 0; + } + if (!strcmp(var, "core.unsetenvvars")) { free(unset_environment_variables); unset_environment_variables = xstrdup(value); diff --git a/compat/mingw.h b/compat/mingw.h index 33544e141a..616a5066cd 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -11,6 +11,8 @@ typedef _sigset_t sigset_t; #undef _POSIX_THREAD_SAFE_FUNCTIONS #endif +extern int core_fscache; + extern int mingw_core_config(const char *var, const char *value, void *cb); #define platform_core_config mingw_core_config diff --git a/git-compat-util.h b/git-compat-util.h index 7513ec4949..9fed029e7e 100644 --- a/git-compat-util.h +++ b/git-compat-util.h @@ -1225,6 +1225,21 @@ static inline int is_missing_file_error(int errno_) return (errno_ == ENOENT || errno_ == ENOTDIR); } +/* + * Enable/disable a read-only cache for file system data on platforms that + * support it. + * + * Implementing a live-cache is complicated and requires special platform + * support (inotify, ReadDirectoryChangesW...). enable_fscache shall be used + * to mark sections of git code that extensively read from the file system + * without modifying anything. Implementations can use this to cache e.g. stat + * data or even file content without the need to synchronize with the file + * system. + */ +#ifndef enable_fscache +#define enable_fscache(x) /* noop */ +#endif + extern int cmd_main(int, const char **); /* diff --git a/preload-index.c b/preload-index.c index 71cd2437a3..9c22a0705f 100644 --- a/preload-index.c +++ b/preload-index.c @@ -93,6 +93,7 @@ static void preload_index(struct index_state *index, offset = 0; work = DIV_ROUND_UP(index->cache_nr, threads); memset(&data, 0, sizeof(data)); + enable_fscache(1); for (i = 0; i < threads; i++) { struct thread_data *p = data+i; p->index = index; @@ -110,6 +111,7 @@ static void preload_index(struct index_state *index, die("unable to join threaded lstat"); } trace_performance_since(start, "preload index"); + enable_fscache(0); } #endif From b008d70020dfa8c2ae3f954233627e6d8fcd798b Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 17 Nov 2016 18:05:04 +0100 Subject: [PATCH 09/36] mingw: unset PERL5LIB by default Git for Windows ships with its own Perl interpreter, and insists on using it, so it will most likely wreak havoc if PERL5LIB is set before launching Git. Let's just unset that environment variables when spawning processes. To make this feature extensible (and overrideable), there is a new config setting `core.unsetenvvars` that allows specifying a comma-separated list of names to unset before spawning processes. Reported by Gabriel Fuhrmann. Signed-off-by: Johannes Schindelin --- Documentation/config.txt | 6 ++++++ compat/mingw.c | 35 ++++++++++++++++++++++++++++++++++- t/t0029-core-unsetenvvars.sh | 30 ++++++++++++++++++++++++++++++ 3 files changed, 70 insertions(+), 1 deletion(-) create mode 100755 t/t0029-core-unsetenvvars.sh diff --git a/Documentation/config.txt b/Documentation/config.txt index eb66a11975..b25f1c0dc4 100644 --- a/Documentation/config.txt +++ b/Documentation/config.txt @@ -909,6 +909,12 @@ relatively high IO latencies. When enabled, Git will do the index comparison to the filesystem data in parallel, allowing overlapping IO's. Defaults to true. +core.unsetenvvars:: + EXPERIMENTAL, Windows-only: comma-separated list of environment + variables' names that need to be unset before spawning any other + process. Defaults to `PERL5LIB` to account for the fact that Git + for Windows insists on using its own Perl interpreter. + core.createObject:: You can set this to 'link', in which case a hardlink followed by a delete of the source are used to make sure that object creation diff --git a/compat/mingw.c b/compat/mingw.c index 276a8d73f4..00191fe9db 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -210,6 +210,7 @@ enum hide_dotfiles_type { }; static enum hide_dotfiles_type hide_dotfiles = HIDE_DOTFILES_DOTGITONLY; +static char *unset_environment_variables; int mingw_core_config(const char *var, const char *value, void *cb) { @@ -221,6 +222,12 @@ int mingw_core_config(const char *var, const char *value, void *cb) return 0; } + if (!strcmp(var, "core.unsetenvvars")) { + free(unset_environment_variables); + unset_environment_variables = xstrdup(value); + return 0; + } + return 0; } @@ -1102,6 +1109,27 @@ static wchar_t *make_environment_block(char **deltaenv) return wenvblk; } +static void do_unset_environment_variables(void) +{ + static int done; + char *p = unset_environment_variables; + + if (done || !p) + return; + done = 1; + + for (;;) { + char *comma = strchr(p, ','); + + if (comma) + *comma = '\0'; + unsetenv(p); + if (!comma) + break; + p = comma + 1; + } +} + struct pinfo_t { struct pinfo_t *next; pid_t pid; @@ -1120,9 +1148,12 @@ static pid_t mingw_spawnve_fd(const char *cmd, const char **argv, char **deltaen wchar_t wcmd[MAX_PATH], wdir[MAX_PATH], *wargs, *wenvblk = NULL; unsigned flags = CREATE_UNICODE_ENVIRONMENT; BOOL ret; + HANDLE cons; + + do_unset_environment_variables(); /* Determine whether or not we are associated to a console */ - HANDLE cons = CreateFile("CONOUT$", GENERIC_WRITE, + cons = CreateFile("CONOUT$", GENERIC_WRITE, FILE_SHARE_WRITE, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); if (cons == INVALID_HANDLE_VALUE) { @@ -2313,6 +2344,8 @@ void mingw_startup(void) /* fix Windows specific environment settings */ setup_windows_environment(); + unset_environment_variables = xstrdup("PERL5LIB"); + /* initialize critical section for waitpid pinfo_t list */ InitializeCriticalSection(&pinfo_cs); diff --git a/t/t0029-core-unsetenvvars.sh b/t/t0029-core-unsetenvvars.sh new file mode 100755 index 0000000000..24ce46a6ea --- /dev/null +++ b/t/t0029-core-unsetenvvars.sh @@ -0,0 +1,30 @@ +#!/bin/sh + +test_description='test the Windows-only core.unsetenvvars setting' + +. ./test-lib.sh + +if ! test_have_prereq MINGW +then + skip_all='skipping Windows-specific tests' + test_done +fi + +test_expect_success 'setup' ' + mkdir -p "$TRASH_DIRECTORY/.git/hooks" && + write_script "$TRASH_DIRECTORY/.git/hooks/pre-commit" <<-\EOF + echo $HOBBES >&2 + EOF +' + +test_expect_success 'core.unsetenvvars works' ' + HOBBES=Calvin && + export HOBBES && + git commit --allow-empty -m with 2>err && + grep Calvin err && + git -c core.unsetenvvars=FINDUS,HOBBES,CALVIN \ + commit --allow-empty -m without 2>err && + ! grep Calvin err +' + +test_done From 66f51ba51228244a6a903192e06ab7b4c21b0eff Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Tue, 1 Oct 2013 12:51:54 +0200 Subject: [PATCH 10/36] Win32: add a cache below mingw's lstat and dirent implementations Checking the work tree status is quite slow on Windows, due to slow lstat emulation (git calls lstat once for each file in the index). Windows operating system APIs seem to be much better at scanning the status of entire directories than checking single files. Add an lstat implementation that uses a cache for lstat data. Cache misses read the entire parent directory and add it to the cache. Subsequent lstat calls for the same directory are served directly from the cache. Also implement opendir / readdir / closedir so that they create and use directory listings in the cache. The cache doesn't track file system changes and doesn't plug into any modifying file APIs, so it has to be explicitly enabled for git functions that don't modify the working copy. Note: in an earlier version of this patch, the cache was always active and tracked file system changes via ReadDirectoryChangesW. However, this was much more complex and had negative impact on the performance of modifying git commands such as 'git checkout'. Signed-off-by: Karsten Blees --- compat/win32/fscache.c | 444 +++++++++++++++++++++++++++++++++++++++++ compat/win32/fscache.h | 10 + config.mak.uname | 4 +- git-compat-util.h | 2 + 4 files changed, 458 insertions(+), 2 deletions(-) create mode 100644 compat/win32/fscache.c create mode 100644 compat/win32/fscache.h diff --git a/compat/win32/fscache.c b/compat/win32/fscache.c new file mode 100644 index 0000000000..0efa7663c5 --- /dev/null +++ b/compat/win32/fscache.c @@ -0,0 +1,444 @@ +#include "../../cache.h" +#include "../../hashmap.h" +#include "../win32.h" +#include "fscache.h" + +static int initialized; +static volatile long enabled; +static struct hashmap map; +static CRITICAL_SECTION mutex; + +/* + * An entry in the file system cache. Used for both entire directory listings + * and file entries. + */ +struct fsentry { + struct hashmap_entry ent; + mode_t st_mode; + /* Length of name. */ + unsigned short len; + /* + * Name of the entry. For directory listings: relative path of the + * directory, without trailing '/' (empty for cwd()). For file entries: + * name of the file. Typically points to the end of the structure if + * the fsentry is allocated on the heap (see fsentry_alloc), or to a + * local variable if on the stack (see fsentry_init). + */ + const char *name; + /* Pointer to the directory listing, or NULL for the listing itself. */ + struct fsentry *list; + /* Pointer to the next file entry of the list. */ + struct fsentry *next; + + union { + /* Reference count of the directory listing. */ + volatile long refcnt; + struct { + /* More stat members (only used for file entries). */ + off64_t st_size; + time_t st_atime; + time_t st_mtime; + time_t st_ctime; + }; + }; +}; + +/* + * Compares the paths of two fsentry structures for equality. + */ +static int fsentry_cmp(void *unused_cmp_data, + const struct fsentry *fse1, const struct fsentry *fse2, + void *unused_keydata) +{ + int res; + if (fse1 == fse2) + return 0; + + /* compare the list parts first */ + if (fse1->list != fse2->list && + (res = fsentry_cmp(NULL, fse1->list ? fse1->list : fse1, + fse2->list ? fse2->list : fse2, NULL))) + return res; + + /* if list parts are equal, compare len and name */ + if (fse1->len != fse2->len) + return fse1->len - fse2->len; + return strnicmp(fse1->name, fse2->name, fse1->len); +} + +/* + * Calculates the hash code of an fsentry structure's path. + */ +static unsigned int fsentry_hash(const struct fsentry *fse) +{ + unsigned int hash = fse->list ? fse->list->ent.hash : 0; + return hash ^ memihash(fse->name, fse->len); +} + +/* + * Initialize an fsentry structure for use by fsentry_hash and fsentry_cmp. + */ +static void fsentry_init(struct fsentry *fse, struct fsentry *list, + const char *name, size_t len) +{ + fse->list = list; + fse->name = name; + fse->len = len; + hashmap_entry_init(fse, fsentry_hash(fse)); +} + +/* + * Allocate an fsentry structure on the heap. + */ +static struct fsentry *fsentry_alloc(struct fsentry *list, const char *name, + size_t len) +{ + /* overallocate fsentry and copy the name to the end */ + struct fsentry *fse = xmalloc(sizeof(struct fsentry) + len + 1); + char *nm = ((char*) fse) + sizeof(struct fsentry); + memcpy(nm, name, len); + nm[len] = 0; + /* init the rest of the structure */ + fsentry_init(fse, list, nm, len); + fse->next = NULL; + fse->refcnt = 1; + return fse; +} + +/* + * Add a reference to an fsentry. + */ +inline static void fsentry_addref(struct fsentry *fse) +{ + if (fse->list) + fse = fse->list; + + InterlockedIncrement(&(fse->refcnt)); +} + +/* + * Release the reference to an fsentry, frees the memory if its the last ref. + */ +static void fsentry_release(struct fsentry *fse) +{ + if (fse->list) + fse = fse->list; + + if (InterlockedDecrement(&(fse->refcnt))) + return; + + while (fse) { + struct fsentry *next = fse->next; + free(fse); + fse = next; + } +} + +/* + * Allocate and initialize an fsentry from a WIN32_FIND_DATA structure. + */ +static struct fsentry *fseentry_create_entry(struct fsentry *list, + const WIN32_FIND_DATAW *fdata) +{ + char buf[MAX_PATH * 3]; + int len; + struct fsentry *fse; + len = xwcstoutf(buf, fdata->cFileName, ARRAY_SIZE(buf)); + + fse = fsentry_alloc(list, buf, len); + + fse->st_mode = file_attr_to_st_mode(fdata->dwFileAttributes); + fse->st_size = (((off64_t) (fdata->nFileSizeHigh)) << 32) + | fdata->nFileSizeLow; + fse->st_atime = filetime_to_time_t(&(fdata->ftLastAccessTime)); + fse->st_mtime = filetime_to_time_t(&(fdata->ftLastWriteTime)); + fse->st_ctime = filetime_to_time_t(&(fdata->ftCreationTime)); + + return fse; +} + +/* + * Create an fsentry-based directory listing (similar to opendir / readdir). + * Dir should not contain trailing '/'. Use an empty string for the current + * directory (not "."!). + */ +static struct fsentry *fsentry_create_list(const struct fsentry *dir) +{ + wchar_t pattern[MAX_PATH + 2]; /* + 2 for '/' '*' */ + WIN32_FIND_DATAW fdata; + HANDLE h; + int wlen; + struct fsentry *list, **phead; + DWORD err; + + /* convert name to UTF-16 and check length < MAX_PATH */ + if ((wlen = xutftowcsn(pattern, dir->name, MAX_PATH, dir->len)) < 0) { + if (errno == ERANGE) + errno = ENAMETOOLONG; + return NULL; + } + + /* append optional '/' and wildcard '*' */ + if (wlen) + pattern[wlen++] = '/'; + pattern[wlen++] = '*'; + pattern[wlen] = 0; + + /* open find handle */ + h = FindFirstFileW(pattern, &fdata); + if (h == INVALID_HANDLE_VALUE) { + err = GetLastError(); + errno = (err == ERROR_DIRECTORY) ? ENOTDIR : err_win_to_posix(err); + return NULL; + } + + /* allocate object to hold directory listing */ + list = fsentry_alloc(NULL, dir->name, dir->len); + + /* walk directory and build linked list of fsentry structures */ + phead = &list->next; + do { + *phead = fseentry_create_entry(list, &fdata); + phead = &(*phead)->next; + } while (FindNextFileW(h, &fdata)); + + /* remember result of last FindNextFile, then close find handle */ + err = GetLastError(); + FindClose(h); + + /* return the list if we've got all the files */ + if (err == ERROR_NO_MORE_FILES) + return list; + + /* otherwise free the list and return error */ + fsentry_release(list); + errno = err_win_to_posix(err); + return NULL; +} + +/* + * Adds a directory listing to the cache. + */ +static void fscache_add(struct fsentry *fse) +{ + if (fse->list) + fse = fse->list; + + for (; fse; fse = fse->next) + hashmap_add(&map, fse); +} + +/* + * Clears the cache. + */ +static void fscache_clear(void) +{ + hashmap_free(&map, 1); + hashmap_init(&map, (hashmap_cmp_fn)fsentry_cmp, NULL, 0); +} + +/* + * Checks if the cache is enabled for the given path. + */ +static inline int fscache_enabled(const char *path) +{ + return enabled > 0 && !is_absolute_path(path); +} + +/* + * Looks up or creates a cache entry for the specified key. + */ +static struct fsentry *fscache_get(struct fsentry *key) +{ + struct fsentry *fse; + + EnterCriticalSection(&mutex); + /* check if entry is in cache */ + fse = hashmap_get(&map, key, NULL); + if (fse) { + fsentry_addref(fse); + LeaveCriticalSection(&mutex); + return fse; + } + /* if looking for a file, check if directory listing is in cache */ + if (!fse && key->list) { + fse = hashmap_get(&map, key->list, NULL); + if (fse) { + LeaveCriticalSection(&mutex); + /* dir entry without file entry -> file doesn't exist */ + errno = ENOENT; + return NULL; + } + } + + /* create the directory listing (outside mutex!) */ + LeaveCriticalSection(&mutex); + fse = fsentry_create_list(key->list ? key->list : key); + if (!fse) + return NULL; + + EnterCriticalSection(&mutex); + /* add directory listing if it hasn't been added by some other thread */ + if (!hashmap_get(&map, key, NULL)) + fscache_add(fse); + + /* lookup file entry if requested (fse already points to directory) */ + if (key->list) + fse = hashmap_get(&map, key, NULL); + + /* return entry or ENOENT */ + if (fse) + fsentry_addref(fse); + else + errno = ENOENT; + + LeaveCriticalSection(&mutex); + return fse; +} + +/* + * Enables or disables the cache. Note that the cache is read-only, changes to + * the working directory are NOT reflected in the cache while enabled. + */ +int fscache_enable(int enable) +{ + int result; + + if (!initialized) { + /* allow the cache to be disabled entirely */ + if (!core_fscache) + return 0; + + InitializeCriticalSection(&mutex); + hashmap_init(&map, (hashmap_cmp_fn) fsentry_cmp, NULL, 0); + initialized = 1; + } + + result = enable ? InterlockedIncrement(&enabled) + : InterlockedDecrement(&enabled); + + if (enable && result == 1) { + /* redirect opendir and lstat to the fscache implementations */ + opendir = fscache_opendir; + lstat = fscache_lstat; + } else if (!enable && !result) { + /* reset opendir and lstat to the original implementations */ + opendir = dirent_opendir; + lstat = mingw_lstat; + EnterCriticalSection(&mutex); + fscache_clear(); + LeaveCriticalSection(&mutex); + } + return result; +} + +/* + * Lstat replacement, uses the cache if enabled, otherwise redirects to + * mingw_lstat. + */ +int fscache_lstat(const char *filename, struct stat *st) +{ + int dirlen, base, len; + struct fsentry key[2], *fse; + + if (!fscache_enabled(filename)) + return mingw_lstat(filename, st); + + /* split filename into path + name */ + len = strlen(filename); + if (len && is_dir_sep(filename[len - 1])) + len--; + base = len; + while (base && !is_dir_sep(filename[base - 1])) + base--; + dirlen = base ? base - 1 : 0; + + /* lookup entry for path + name in cache */ + fsentry_init(key, NULL, filename, dirlen); + fsentry_init(key + 1, key, filename + base, len - base); + fse = fscache_get(key + 1); + if (!fse) + return -1; + + /* copy stat data */ + st->st_ino = 0; + st->st_gid = 0; + st->st_uid = 0; + st->st_dev = 0; + st->st_rdev = 0; + st->st_nlink = 1; + st->st_mode = fse->st_mode; + st->st_size = fse->st_size; + st->st_atime = fse->st_atime; + st->st_mtime = fse->st_mtime; + st->st_ctime = fse->st_ctime; + + /* don't forget to release fsentry */ + fsentry_release(fse); + return 0; +} + +typedef struct fscache_DIR { + struct DIR base_dir; /* extend base struct DIR */ + struct fsentry *pfsentry; + struct dirent dirent; +} fscache_DIR; + +/* + * Readdir replacement. + */ +static struct dirent *fscache_readdir(DIR *base_dir) +{ + fscache_DIR *dir = (fscache_DIR*) base_dir; + struct fsentry *next = dir->pfsentry->next; + if (!next) + return NULL; + dir->pfsentry = next; + dir->dirent.d_type = S_ISDIR(next->st_mode) ? DT_DIR : DT_REG; + dir->dirent.d_name = (char*) next->name; + return &(dir->dirent); +} + +/* + * Closedir replacement. + */ +static int fscache_closedir(DIR *base_dir) +{ + fscache_DIR *dir = (fscache_DIR*) base_dir; + fsentry_release(dir->pfsentry); + free(dir); + return 0; +} + +/* + * Opendir replacement, uses a directory listing from the cache if enabled, + * otherwise calls original dirent implementation. + */ +DIR *fscache_opendir(const char *dirname) +{ + struct fsentry key, *list; + fscache_DIR *dir; + int len; + + if (!fscache_enabled(dirname)) + return dirent_opendir(dirname); + + /* prepare name (strip trailing '/', replace '.') */ + len = strlen(dirname); + if ((len == 1 && dirname[0] == '.') || + (len && is_dir_sep(dirname[len - 1]))) + len--; + + /* get directory listing from cache */ + fsentry_init(&key, NULL, dirname, len); + list = fscache_get(&key); + if (!list) + return NULL; + + /* alloc and return DIR structure */ + dir = (fscache_DIR*) xmalloc(sizeof(fscache_DIR)); + dir->base_dir.preaddir = fscache_readdir; + dir->base_dir.pclosedir = fscache_closedir; + dir->pfsentry = list; + return (DIR*) dir; +} diff --git a/compat/win32/fscache.h b/compat/win32/fscache.h new file mode 100644 index 0000000000..ed518b422d --- /dev/null +++ b/compat/win32/fscache.h @@ -0,0 +1,10 @@ +#ifndef FSCACHE_H +#define FSCACHE_H + +int fscache_enable(int enable); +#define enable_fscache(x) fscache_enable(x) + +DIR *fscache_opendir(const char *dir); +int fscache_lstat(const char *file_name, struct stat *buf); + +#endif diff --git a/config.mak.uname b/config.mak.uname index e47af72e01..6646aa303e 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -393,7 +393,7 @@ ifeq ($(uname_S),Windows) BASIC_CFLAGS = -nologo -I. -I../zlib -Icompat/vcbuild -Icompat/vcbuild/include -DWIN32 -D_CONSOLE -DHAVE_STRING_H -D_CRT_SECURE_NO_WARNINGS -D_CRT_NONSTDC_NO_DEPRECATE COMPAT_OBJS = compat/msvc.o compat/winansi.o \ compat/win32/pthread.o compat/win32/syslog.o \ - compat/win32/dirent.o + compat/win32/dirent.o compat/win32/fscache.o COMPAT_CFLAGS = -D__USE_MINGW_ACCESS -DNOGDI -DHAVE_STRING_H -Icompat -Icompat/regex -Icompat/win32 -DSTRIP_EXTENSION=\".exe\" BASIC_LDFLAGS = -IGNORE:4217 -IGNORE:4049 -NOLOGO -SUBSYSTEM:CONSOLE EXTLIBS = user32.lib advapi32.lib shell32.lib wininet.lib ws2_32.lib invalidcontinue.obj @@ -537,7 +537,7 @@ ifneq (,$(findstring MINGW,$(uname_S))) COMPAT_CFLAGS += -DSTRIP_EXTENSION=\".exe\" COMPAT_OBJS += compat/mingw.o compat/winansi.o \ compat/win32/pthread.o compat/win32/syslog.o \ - compat/win32/dirent.o + compat/win32/dirent.o compat/win32/fscache.o BASIC_CFLAGS += -DWIN32 -DPROTECT_NTFS_DEFAULT=1 EXTLIBS += -lws2_32 GITLIBS += git.res diff --git a/git-compat-util.h b/git-compat-util.h index 9fed029e7e..d0d249b931 100644 --- a/git-compat-util.h +++ b/git-compat-util.h @@ -195,8 +195,10 @@ #if defined(__MINGW32__) /* pull in Windows compatibility stuff */ #include "compat/mingw.h" +#include "compat/win32/fscache.h" #elif defined(_MSC_VER) #include "compat/msvc.h" +#include "compat/win32/fscache.h" #else #include #include From 72be485574a02b2bee92c08c8a1cda63a387eec5 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Tue, 24 Jun 2014 13:22:35 +0200 Subject: [PATCH 11/36] fscache: load directories only once If multiple threads access a directory that is not yet in the cache, the directory will be loaded by each thread. Only one of the results is added to the cache, all others are leaked. This wastes performance and memory. On cache miss, add a future object to the cache to indicate that the directory is currently being loaded. Subsequent threads register themselves with the future object and wait. When the first thread has loaded the directory, it replaces the future object with the result and notifies waiting threads. Signed-off-by: Karsten Blees --- compat/win32/fscache.c | 67 +++++++++++++++++++++++++++++++++++------- 1 file changed, 57 insertions(+), 10 deletions(-) diff --git a/compat/win32/fscache.c b/compat/win32/fscache.c index 0efa7663c5..8094d0a8c6 100644 --- a/compat/win32/fscache.c +++ b/compat/win32/fscache.c @@ -33,6 +33,8 @@ struct fsentry { union { /* Reference count of the directory listing. */ volatile long refcnt; + /* Handle to wait on the loading thread. */ + HANDLE hwait; struct { /* More stat members (only used for file entries). */ off64_t st_size; @@ -245,16 +247,43 @@ static inline int fscache_enabled(const char *path) return enabled > 0 && !is_absolute_path(path); } +/* + * Looks up a cache entry, waits if its being loaded by another thread. + * The mutex must be owned by the calling thread. + */ +static struct fsentry *fscache_get_wait(struct fsentry *key) +{ + struct fsentry *fse = hashmap_get(&map, key, NULL); + + /* return if its a 'real' entry (future entries have refcnt == 0) */ + if (!fse || fse->list || fse->refcnt) + return fse; + + /* create an event and link our key to the future entry */ + key->hwait = CreateEvent(NULL, TRUE, FALSE, NULL); + key->next = fse->next; + fse->next = key; + + /* wait for the loading thread to signal us */ + LeaveCriticalSection(&mutex); + WaitForSingleObject(key->hwait, INFINITE); + CloseHandle(key->hwait); + EnterCriticalSection(&mutex); + + /* repeat cache lookup */ + return hashmap_get(&map, key, NULL); +} + /* * Looks up or creates a cache entry for the specified key. */ static struct fsentry *fscache_get(struct fsentry *key) { - struct fsentry *fse; + struct fsentry *fse, *future, *waiter; EnterCriticalSection(&mutex); /* check if entry is in cache */ - fse = hashmap_get(&map, key, NULL); + fse = fscache_get_wait(key); if (fse) { fsentry_addref(fse); LeaveCriticalSection(&mutex); @@ -262,7 +291,7 @@ static struct fsentry *fscache_get(struct fsentry *key) } /* if looking for a file, check if directory listing is in cache */ if (!fse && key->list) { - fse = hashmap_get(&map, key->list, NULL); + fse = fscache_get_wait(key->list); if (fse) { LeaveCriticalSection(&mutex); /* dir entry without file entry -> file doesn't exist */ @@ -271,16 +300,34 @@ static struct fsentry *fscache_get(struct fsentry *key) } } + /* add future entry to indicate that we're loading it */ + future = key->list ? key->list : key; + future->next = NULL; + future->refcnt = 0; + hashmap_add(&map, future); + /* create the directory listing (outside mutex!) */ LeaveCriticalSection(&mutex); - fse = fsentry_create_list(key->list ? key->list : key); - if (!fse) - return NULL; - + fse = fsentry_create_list(future); EnterCriticalSection(&mutex); - /* add directory listing if it hasn't been added by some other thread */ - if (!hashmap_get(&map, key, NULL)) - fscache_add(fse); + + /* remove future entry and signal waiting threads */ + hashmap_remove(&map, future, NULL); + waiter = future->next; + while (waiter) { + HANDLE h = waiter->hwait; + waiter = waiter->next; + SetEvent(h); + } + + /* leave on error (errno set by fsentry_create_list) */ + if (!fse) { + LeaveCriticalSection(&mutex); + return NULL; + } + + /* add directory listing to the cache */ + fscache_add(fse); /* lookup file entry if requested (fse already points to directory) */ if (key->list) From 78f6a9b6f42678aa4a1cc7542168a81be60c0693 Mon Sep 17 00:00:00 2001 From: Doug Kelly Date: Wed, 8 Jan 2014 20:28:15 -0600 Subject: [PATCH 12/36] Add a test demonstrating a problem with long submodule paths [jes: adjusted test number to avoid conflicts, fixed non-portable use of the 'export' statement, fixed broken && chain] Signed-off-by: Doug Kelly Signed-off-by: Johannes Schindelin --- t/t7418-submodule-long-path.sh | 101 +++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100755 t/t7418-submodule-long-path.sh diff --git a/t/t7418-submodule-long-path.sh b/t/t7418-submodule-long-path.sh new file mode 100755 index 0000000000..9f9d2ea446 --- /dev/null +++ b/t/t7418-submodule-long-path.sh @@ -0,0 +1,101 @@ +#!/bin/sh +# +# Copyright (c) 2013 Doug Kelly +# + +test_description='Test submodules with a path near PATH_MAX + +This test verifies that "git submodule" initialization, update and clones work, including with recursive submodules and paths approaching PATH_MAX (260 characters on Windows) +' + +TEST_NO_CREATE_REPO=1 +. ./test-lib.sh + +longpath="" +for (( i=0; i<4; i++ )); do + longpath="0123456789abcdefghijklmnopqrstuvwxyz$longpath" +done +# Pick a substring maximum of 90 characters +# This should be good, since we'll add on a lot for temp directories +longpath=${longpath:0:90}; export longpath + +test_expect_failure 'submodule with a long path' ' + git init --bare remote && + test_create_repo bundle1 && + ( + cd bundle1 && + test_commit "shoot" && + git rev-parse --verify HEAD >../expect + ) && + mkdir home && + ( + cd home && + git clone ../remote test && + cd test && + git submodule add ../bundle1 $longpath && + test_commit "sogood" && + ( + cd $longpath && + git rev-parse --verify HEAD >actual && + test_cmp ../../../expect actual + ) && + git push origin master + ) && + mkdir home2 && + ( + cd home2 && + git clone ../remote test && + cd test && + git checkout master && + git submodule update --init && + ( + cd $longpath && + git rev-parse --verify HEAD >actual && + test_cmp ../../../expect actual + ) + ) +' + +test_expect_failure 'recursive submodule with a long path' ' + git init --bare super && + test_create_repo child && + ( + cd child && + test_commit "shoot" && + git rev-parse --verify HEAD >../expect + ) && + test_create_repo parent && + ( + cd parent && + git submodule add ../child $longpath && + test_commit "aim" + ) && + mkdir home3 && + ( + cd home3 && + git clone ../super test && + cd test && + git submodule add ../parent foo && + git submodule update --init --recursive && + test_commit "sogood" && + ( + cd foo/$longpath && + git rev-parse --verify HEAD >actual && + test_cmp ../../../../expect actual + ) && + git push origin master + ) && + mkdir home4 && + ( + cd home4 && + git clone ../super test --recursive && + ( + cd test/foo/$longpath && + git rev-parse --verify HEAD >actual && + test_cmp ../../../../expect actual + ) + ) +' +unset longpath + +test_done From 292dd63d02594dc244fb9beeb7b3ef1b15207007 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 28 Jul 2015 21:07:41 +0200 Subject: [PATCH 13/36] Win32: support long paths Windows paths are typically limited to MAX_PATH = 260 characters, even though the underlying NTFS file system supports paths up to 32,767 chars. This limitation is also evident in Windows Explorer, cmd.exe and many other applications (including IDEs). Particularly annoying is that most Windows APIs return bogus error codes if a relative path only barely exceeds MAX_PATH in conjunction with the current directory, e.g. ERROR_PATH_NOT_FOUND / ENOENT instead of the infinitely more helpful ERROR_FILENAME_EXCED_RANGE / ENAMETOOLONG. Many Windows wide char APIs support longer than MAX_PATH paths through the file namespace prefix ('\\?\' or '\\?\UNC\') followed by an absolute path. Notable exceptions include functions dealing with executables and the current directory (CreateProcess, LoadLibrary, Get/SetCurrentDirectory) as well as the entire shell API (ShellExecute, SHGetSpecialFolderPath...). Introduce a handle_long_path function to check the length of a specified path properly (and fail with ENAMETOOLONG), and to optionally expand long paths using the '\\?\' file namespace prefix. Short paths will not be modified, so we don't need to worry about device names (NUL, CON, AUX). Contrary to MSDN docs, the GetFullPathNameW function doesn't seem to be limited to MAX_PATH (at least not on Win7), so we can use it to do the heavy lifting of the conversion (translate '/' to '\', eliminate '.' and '..', and make an absolute path). Add long path error checking to xutftowcs_path for APIs with hard MAX_PATH limit. Add a new MAX_LONG_PATH constant and xutftowcs_long_path function for APIs that support long paths. While improved error checking is always active, long paths support must be explicitly enabled via 'core.longpaths' option. This is to prevent end users to shoot themselves in the foot by checking out files that Windows Explorer, cmd/bash or their favorite IDE cannot handle. Test suite: Test the case is when the full pathname length of a dir is close to 260 (MAX_PATH). Bug report and an original reproducer by Andrey Rogozhnikov: https://github.com/msysgit/git/pull/122#issuecomment-43604199 [jes: adjusted test number to avoid conflicts] Thanks-to: Martin W. Kirst Thanks-to: Doug Kelly Signed-off-by: Karsten Blees Original-test-by: Andrey Rogozhnikov Signed-off-by: Stepan Kasal Signed-off-by: Johannes Schindelin --- Documentation/config.txt | 7 ++ compat/mingw.c | 138 ++++++++++++++++++++++++++------- compat/mingw.h | 75 ++++++++++++++++-- compat/win32/dirent.c | 14 ++-- compat/win32/fscache.c | 17 ++-- t/t2027-checkout-long-paths.sh | 99 +++++++++++++++++++++++ t/t7418-submodule-long-path.sh | 24 +++--- 7 files changed, 319 insertions(+), 55 deletions(-) create mode 100755 t/t2027-checkout-long-paths.sh diff --git a/Documentation/config.txt b/Documentation/config.txt index 3aa9df09ed..b28004b383 100644 --- a/Documentation/config.txt +++ b/Documentation/config.txt @@ -915,6 +915,13 @@ core.fscache:: Git for Windows uses this to bulk-read and cache lstat data of entire directories (instead of doing lstat file by file). +core.longpaths:: + Enable long path (> 260) support for builtin commands in Git for + Windows. This is disabled by default, as long paths are not supported + by Windows Explorer, cmd.exe and the Git for Windows tool chain + (msys, bash, tcl, perl...). Only enable this if you know what you're + doing and are prepared to live with a few quirks. + core.unsetenvvars:: EXPERIMENTAL, Windows-only: comma-separated list of environment variables' names that need to be unset before spawning any other diff --git a/compat/mingw.c b/compat/mingw.c index ba146ad270..efac32906a 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -212,6 +212,7 @@ enum hide_dotfiles_type { static enum hide_dotfiles_type hide_dotfiles = HIDE_DOTFILES_DOTGITONLY; static char *unset_environment_variables; int core_fscache; +int core_long_paths; int mingw_core_config(const char *var, const char *value, void *cb) { @@ -228,6 +229,11 @@ int mingw_core_config(const char *var, const char *value, void *cb) return 0; } + if (!strcmp(var, "core.longpaths")) { + core_long_paths = git_config_bool(var, value); + return 0; + } + if (!strcmp(var, "core.unsetenvvars")) { free(unset_environment_variables); unset_environment_variables = xstrdup(value); @@ -240,8 +246,8 @@ int mingw_core_config(const char *var, const char *value, void *cb) int mingw_unlink(const char *pathname) { int ret, tries = 0; - wchar_t wpathname[MAX_PATH]; - if (xutftowcs_path(wpathname, pathname) < 0) + wchar_t wpathname[MAX_LONG_PATH]; + if (xutftowcs_long_path(wpathname, pathname) < 0) return -1; /* read-only files cannot be removed */ @@ -270,7 +276,7 @@ static int is_dir_empty(const wchar_t *wpath) { WIN32_FIND_DATAW findbuf; HANDLE handle; - wchar_t wbuf[MAX_PATH + 2]; + wchar_t wbuf[MAX_LONG_PATH + 2]; wcscpy(wbuf, wpath); wcscat(wbuf, L"\\*"); handle = FindFirstFileW(wbuf, &findbuf); @@ -291,8 +297,8 @@ static int is_dir_empty(const wchar_t *wpath) int mingw_rmdir(const char *pathname) { int ret, tries = 0; - wchar_t wpathname[MAX_PATH]; - if (xutftowcs_path(wpathname, pathname) < 0) + wchar_t wpathname[MAX_LONG_PATH]; + if (xutftowcs_long_path(wpathname, pathname) < 0) return -1; while ((ret = _wrmdir(wpathname)) == -1 && tries < ARRAY_SIZE(delay)) { @@ -367,9 +373,12 @@ static int set_hidden_flag(const wchar_t *path, int set) int mingw_mkdir(const char *path, int mode) { int ret; - wchar_t wpath[MAX_PATH]; - if (xutftowcs_path(wpath, path) < 0) + wchar_t wpath[MAX_LONG_PATH]; + /* CreateDirectoryW path limit is 248 (MAX_PATH - 8.3 file name) */ + if (xutftowcs_path_ex(wpath, path, MAX_LONG_PATH, -1, 248, + core_long_paths) < 0) return -1; + ret = _wmkdir(wpath); if (!ret && needs_hiding(path)) return set_hidden_flag(wpath, 1); @@ -412,7 +421,7 @@ int mingw_open (const char *filename, int oflags, ...) va_list args; unsigned mode; int fd; - wchar_t wfilename[MAX_PATH]; + wchar_t wfilename[MAX_LONG_PATH]; open_fn_t open_fn; va_start(args, oflags); @@ -427,7 +436,7 @@ int mingw_open (const char *filename, int oflags, ...) else open_fn = _wopen; - if (xutftowcs_path(wfilename, filename) < 0) + if (xutftowcs_long_path(wfilename, filename) < 0) return -1; fd = open_fn(wfilename, oflags, mode); @@ -484,10 +493,10 @@ FILE *mingw_fopen (const char *filename, const char *otype) { int hide = needs_hiding(filename); FILE *file; - wchar_t wfilename[MAX_PATH], wotype[4]; + wchar_t wfilename[MAX_LONG_PATH], wotype[4]; if (filename && !strcmp(filename, "/dev/null")) filename = "nul"; - if (xutftowcs_path(wfilename, filename) < 0 || + if (xutftowcs_long_path(wfilename, filename) < 0 || xutftowcs(wotype, otype, ARRAY_SIZE(wotype)) < 0) return NULL; if (hide && !access(filename, F_OK) && set_hidden_flag(wfilename, 0)) { @@ -506,10 +515,10 @@ FILE *mingw_freopen (const char *filename, const char *otype, FILE *stream) { int hide = needs_hiding(filename); FILE *file; - wchar_t wfilename[MAX_PATH], wotype[4]; + wchar_t wfilename[MAX_LONG_PATH], wotype[4]; if (filename && !strcmp(filename, "/dev/null")) filename = "nul"; - if (xutftowcs_path(wfilename, filename) < 0 || + if (xutftowcs_long_path(wfilename, filename) < 0 || xutftowcs(wotype, otype, ARRAY_SIZE(wotype)) < 0) return NULL; if (hide && !access(filename, F_OK) && set_hidden_flag(wfilename, 0)) { @@ -563,25 +572,32 @@ ssize_t mingw_write(int fd, const void *buf, size_t len) int mingw_access(const char *filename, int mode) { - wchar_t wfilename[MAX_PATH]; - if (xutftowcs_path(wfilename, filename) < 0) + wchar_t wfilename[MAX_LONG_PATH]; + if (xutftowcs_long_path(wfilename, filename) < 0) return -1; /* X_OK is not supported by the MSVCRT version */ return _waccess(wfilename, mode & ~X_OK); } +/* cached length of current directory for handle_long_path */ +static int current_directory_len = 0; + int mingw_chdir(const char *dirname) { + int result; wchar_t wdirname[MAX_PATH]; + /* SetCurrentDirectoryW doesn't support long paths */ if (xutftowcs_path(wdirname, dirname) < 0) return -1; - return _wchdir(wdirname); + result = _wchdir(wdirname); + current_directory_len = GetCurrentDirectoryW(0, NULL); + return result; } int mingw_chmod(const char *filename, int mode) { - wchar_t wfilename[MAX_PATH]; - if (xutftowcs_path(wfilename, filename) < 0) + wchar_t wfilename[MAX_LONG_PATH]; + if (xutftowcs_long_path(wfilename, filename) < 0) return -1; return _wchmod(wfilename, mode); } @@ -629,8 +645,8 @@ static int has_valid_directory_prefix(wchar_t *wfilename) static int do_lstat(int follow, const char *file_name, struct stat *buf) { WIN32_FILE_ATTRIBUTE_DATA fdata; - wchar_t wfilename[MAX_PATH]; - if (xutftowcs_path(wfilename, file_name) < 0) + wchar_t wfilename[MAX_LONG_PATH]; + if (xutftowcs_long_path(wfilename, file_name) < 0) return -1; if (GetFileAttributesExW(wfilename, GetFileExInfoStandard, &fdata)) { @@ -779,8 +795,8 @@ int mingw_utime (const char *file_name, const struct utimbuf *times) FILETIME mft, aft; int fh, rc; DWORD attrs; - wchar_t wfilename[MAX_PATH]; - if (xutftowcs_path(wfilename, file_name) < 0) + wchar_t wfilename[MAX_LONG_PATH]; + if (xutftowcs_long_path(wfilename, file_name) < 0) return -1; /* must have write permission */ @@ -839,6 +855,7 @@ unsigned int sleep (unsigned int seconds) char *mingw_mktemp(char *template) { wchar_t wtemplate[MAX_PATH]; + /* we need to return the path, thus no long paths here! */ if (xutftowcs_path(wtemplate, template) < 0) return NULL; if (!_wmktemp(wtemplate)) @@ -1175,6 +1192,7 @@ static pid_t mingw_spawnve_fd(const char *cmd, const char **argv, char **deltaen si.hStdOutput = winansi_get_osfhandle(fhout); si.hStdError = winansi_get_osfhandle(fherr); + /* executables and the current directory don't support long paths */ if (xutftowcs_path(wcmd, cmd) < 0) return -1; if (dir && xutftowcs_path(wdir, dir) < 0) @@ -1743,8 +1761,9 @@ int mingw_rename(const char *pold, const char *pnew) { DWORD attrs, gle; int tries = 0; - wchar_t wpold[MAX_PATH], wpnew[MAX_PATH]; - if (xutftowcs_path(wpold, pold) < 0 || xutftowcs_path(wpnew, pnew) < 0) + wchar_t wpold[MAX_LONG_PATH], wpnew[MAX_LONG_PATH]; + if (xutftowcs_long_path(wpold, pold) < 0 || + xutftowcs_long_path(wpnew, pnew) < 0) return -1; /* @@ -1984,9 +2003,9 @@ int link(const char *oldpath, const char *newpath) { typedef BOOL (WINAPI *T)(LPCWSTR, LPCWSTR, LPSECURITY_ATTRIBUTES); static T create_hard_link = NULL; - wchar_t woldpath[MAX_PATH], wnewpath[MAX_PATH]; - if (xutftowcs_path(woldpath, oldpath) < 0 || - xutftowcs_path(wnewpath, newpath) < 0) + wchar_t woldpath[MAX_LONG_PATH], wnewpath[MAX_LONG_PATH]; + if (xutftowcs_long_path(woldpath, oldpath) < 0 || + xutftowcs_long_path(wnewpath, newpath) < 0) return -1; if (!create_hard_link) { @@ -2199,6 +2218,68 @@ static void setup_windows_environment(void) setenv("TERM", "cygwin", 1); } +int handle_long_path(wchar_t *path, int len, int max_path, int expand) +{ + int result; + wchar_t buf[MAX_LONG_PATH]; + + /* + * we don't need special handling if path is relative to the current + * directory, and current directory + path don't exceed the desired + * max_path limit. This should cover > 99 % of cases with minimal + * performance impact (git almost always uses relative paths). + */ + if ((len < 2 || (!is_dir_sep(path[0]) && path[1] != ':')) && + (current_directory_len + len < max_path)) + return len; + + /* + * handle everything else: + * - absolute paths: "C:\dir\file" + * - absolute UNC paths: "\\server\share\dir\file" + * - absolute paths on current drive: "\dir\file" + * - relative paths on other drive: "X:file" + * - prefixed paths: "\\?\...", "\\.\..." + */ + + /* convert to absolute path using GetFullPathNameW */ + result = GetFullPathNameW(path, MAX_LONG_PATH, buf, NULL); + if (!result) { + errno = err_win_to_posix(GetLastError()); + return -1; + } + + /* + * return absolute path if it fits within max_path (even if + * "cwd + path" doesn't due to '..' components) + */ + if (result < max_path) { + wcscpy(path, buf); + return result; + } + + /* error out if we shouldn't expand the path or buf is too small */ + if (!expand || result >= MAX_LONG_PATH - 6) { + errno = ENAMETOOLONG; + return -1; + } + + /* prefix full path with "\\?\" or "\\?\UNC\" */ + if (buf[0] == '\\') { + /* ...unless already prefixed */ + if (buf[1] == '\\' && (buf[2] == '?' || buf[2] == '.')) + return len; + + wcscpy(path, L"\\\\?\\UNC\\"); + wcscpy(path + 8, buf + 2); + return result + 6; + } else { + wcscpy(path, L"\\\\?\\"); + wcscpy(path + 4, buf); + return result + 4; + } +} + /* * Disable MSVCRT command line wildcard expansion (__getmainargs called from * mingw startup code, see init.c in mingw runtime). @@ -2349,6 +2430,9 @@ void mingw_startup(void) /* initialize Unicode console */ winansi_init(); + + /* init length of current directory for handle_long_path */ + current_directory_len = GetCurrentDirectoryW(0, NULL); } int uname(struct utsname *buf) diff --git a/compat/mingw.h b/compat/mingw.h index 616a5066cd..0a4281ab14 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -12,6 +12,7 @@ typedef _sigset_t sigset_t; #endif extern int core_fscache; +extern int core_long_paths; extern int mingw_core_config(const char *var, const char *value, void *cb); #define platform_core_config mingw_core_config @@ -452,6 +453,42 @@ int mingw_offset_1st_component(const char *path); #include #endif +/** + * Max length of long paths (exceeding MAX_PATH). The actual maximum supported + * by NTFS is 32,767 (* sizeof(wchar_t)), but we choose an arbitrary smaller + * value to limit required stack memory. + */ +#define MAX_LONG_PATH 4096 + +/** + * Handles paths that would exceed the MAX_PATH limit of Windows Unicode APIs. + * + * With expand == false, the function checks for over-long paths and fails + * with ENAMETOOLONG. The path parameter is not modified, except if cwd + path + * exceeds max_path, but the resulting absolute path doesn't (e.g. due to + * eliminating '..' components). The path parameter must point to a buffer + * of max_path wide characters. + * + * With expand == true, an over-long path is automatically converted in place + * to an absolute path prefixed with '\\?\', and the new length is returned. + * The path parameter must point to a buffer of MAX_LONG_PATH wide characters. + * + * Parameters: + * path: path to check and / or convert + * len: size of path on input (number of wide chars without \0) + * max_path: max short path length to check (usually MAX_PATH = 260, but just + * 248 for CreateDirectoryW) + * expand: false to only check the length, true to expand the path to a + * '\\?\'-prefixed absolute path + * + * Return: + * length of the resulting path, or -1 on failure + * + * Errors: + * ENAMETOOLONG if path is too long + */ +int handle_long_path(wchar_t *path, int len, int max_path, int expand); + /** * Converts UTF-8 encoded string to UTF-16LE. * @@ -509,17 +546,45 @@ static inline int xutftowcs(wchar_t *wcs, const char *utf, size_t wcslen) return xutftowcsn(wcs, utf, wcslen, -1); } +/** + * Simplified file system specific wrapper of xutftowcsn and handle_long_path. + * Converts ERANGE to ENAMETOOLONG. If expand is true, wcs must be at least + * MAX_LONG_PATH wide chars (see handle_long_path). + */ +static inline int xutftowcs_path_ex(wchar_t *wcs, const char *utf, + size_t wcslen, int utflen, int max_path, int expand) +{ + int result = xutftowcsn(wcs, utf, wcslen, utflen); + if (result < 0 && errno == ERANGE) + errno = ENAMETOOLONG; + if (result >= 0) + result = handle_long_path(wcs, result, max_path, expand); + return result; +} + /** * Simplified file system specific variant of xutftowcsn, assumes output * buffer size is MAX_PATH wide chars and input string is \0-terminated, - * fails with ENAMETOOLONG if input string is too long. + * fails with ENAMETOOLONG if input string is too long. Typically used for + * Windows APIs that don't support long paths, e.g. SetCurrentDirectory, + * LoadLibrary, CreateProcess... */ static inline int xutftowcs_path(wchar_t *wcs, const char *utf) { - int result = xutftowcsn(wcs, utf, MAX_PATH, -1); - if (result < 0 && errno == ERANGE) - errno = ENAMETOOLONG; - return result; + return xutftowcs_path_ex(wcs, utf, MAX_PATH, -1, MAX_PATH, 0); +} + +/** + * Simplified file system specific variant of xutftowcsn for Windows APIs + * that support long paths via '\\?\'-prefix, assumes output buffer size is + * MAX_LONG_PATH wide chars, fails with ENAMETOOLONG if input string is too + * long. The 'core.longpaths' git-config option controls whether the path + * is only checked or expanded to a long path. + */ +static inline int xutftowcs_long_path(wchar_t *wcs, const char *utf) +{ + return xutftowcs_path_ex(wcs, utf, MAX_LONG_PATH, -1, MAX_PATH, + core_long_paths); } /** diff --git a/compat/win32/dirent.c b/compat/win32/dirent.c index 6b87042182..b3bd8d7af7 100644 --- a/compat/win32/dirent.c +++ b/compat/win32/dirent.c @@ -63,19 +63,23 @@ static int dirent_closedir(dirent_DIR *dir) DIR *dirent_opendir(const char *name) { - wchar_t pattern[MAX_PATH + 2]; /* + 2 for '/' '*' */ + wchar_t pattern[MAX_LONG_PATH + 2]; /* + 2 for "\*" */ WIN32_FIND_DATAW fdata; HANDLE h; int len; dirent_DIR *dir; - /* convert name to UTF-16 and check length < MAX_PATH */ - if ((len = xutftowcs_path(pattern, name)) < 0) + /* convert name to UTF-16 and check length */ + if ((len = xutftowcs_path_ex(pattern, name, MAX_LONG_PATH, -1, + MAX_PATH - 2, core_long_paths)) < 0) return NULL; - /* append optional '/' and wildcard '*' */ + /* + * append optional '\' and wildcard '*'. Note: we need to use '\' as + * Windows doesn't translate '/' to '\' for "\\?\"-prefixed paths. + */ if (len && !is_dir_sep(pattern[len - 1])) - pattern[len++] = '/'; + pattern[len++] = '\\'; pattern[len++] = '*'; pattern[len] = 0; diff --git a/compat/win32/fscache.c b/compat/win32/fscache.c index 8094d0a8c6..162fbed09b 100644 --- a/compat/win32/fscache.c +++ b/compat/win32/fscache.c @@ -166,23 +166,24 @@ static struct fsentry *fseentry_create_entry(struct fsentry *list, */ static struct fsentry *fsentry_create_list(const struct fsentry *dir) { - wchar_t pattern[MAX_PATH + 2]; /* + 2 for '/' '*' */ + wchar_t pattern[MAX_LONG_PATH + 2]; /* + 2 for "\*" */ WIN32_FIND_DATAW fdata; HANDLE h; int wlen; struct fsentry *list, **phead; DWORD err; - /* convert name to UTF-16 and check length < MAX_PATH */ - if ((wlen = xutftowcsn(pattern, dir->name, MAX_PATH, dir->len)) < 0) { - if (errno == ERANGE) - errno = ENAMETOOLONG; + /* convert name to UTF-16 and check length */ + if ((wlen = xutftowcs_path_ex(pattern, dir->name, MAX_LONG_PATH, + dir->len, MAX_PATH - 2, core_long_paths)) < 0) return NULL; - } - /* append optional '/' and wildcard '*' */ + /* + * append optional '\' and wildcard '*'. Note: we need to use '\' as + * Windows doesn't translate '/' to '\' for "\\?\"-prefixed paths. + */ if (wlen) - pattern[wlen++] = '/'; + pattern[wlen++] = '\\'; pattern[wlen++] = '*'; pattern[wlen] = 0; diff --git a/t/t2027-checkout-long-paths.sh b/t/t2027-checkout-long-paths.sh new file mode 100755 index 0000000000..e65a38da4d --- /dev/null +++ b/t/t2027-checkout-long-paths.sh @@ -0,0 +1,99 @@ +#!/bin/sh + +test_description='checkout long paths on Windows + +Ensures that Git for Windows can deal with long paths (>260) enabled via core.longpaths' + +. ./test-lib.sh + +if test_have_prereq !MINGW +then + skip_all='skipping MINGW specific long paths test' + test_done +fi + +test_expect_success setup ' + p=longpathxx && # -> 10 + p=$p$p$p$p$p && # -> 50 + p=$p$p$p$p$p && # -> 250 + + path=${p}/longtestfile && # -> 263 (MAX_PATH = 260) + + blob=$(echo foobar | git hash-object -w --stdin) && + + printf "100644 %s 0\t%s\n" "$blob" "$path" | + git update-index --add --index-info && + git commit -m initial -q +' + +test_expect_success 'checkout of long paths without core.longpaths fails' ' + git config core.longpaths false && + test_must_fail git checkout -f 2>error && + grep -q "Filename too long" error && + test_path_is_missing longpa~1/longtestfile +' + +test_expect_success 'checkout of long paths with core.longpaths works' ' + git config core.longpaths true && + git checkout -f && + test_path_is_file longpa~1/longtestfile +' + +test_expect_success 'update of long paths' ' + echo frotz >> longpa~1/longtestfile && + echo $path > expect && + git ls-files -m > actual && + test_cmp expect actual && + git add $path && + git commit -m second && + git grep "frotz" HEAD -- $path +' + +test_expect_success cleanup ' + # bash cannot delete the trash dir if it contains a long path + # lets help cleaning up (unless in debug mode) + test ! -z "$debug" || rm -rf longpa~1 +' + +# check that the template used in the test won't be too long: +abspath="$(pwd -W)"/testdir +test ${#abspath} -gt 230 || +test_set_prereq SHORTABSPATH + +test_expect_success SHORTABSPATH 'clean up path close to MAX_PATH' ' + p=/123456789abcdef/123456789abcdef/123456789abcdef/123456789abc/ef && + p=y$p$p$p$p && + subdir="x$(echo "$p" | tail -c $((253 - ${#abspath})) - )" && + # Now, $abspath/$subdir has exactly 254 characters, and is inside CWD + p2="$abspath/$subdir" && + test 254 = ${#p2} && + + # Be careful to overcome path limitations of the MSys tools and split + # the $subdir into two parts. ($subdir2 has to contain 16 chars and a + # slash somewhere following; that is why we asked for abspath <= 230 and + # why we placed a slash near the end of the $subdir template.) + subdir2=${subdir#????????????????*/} && + subdir1=testdir/${subdir%/$subdir2} && + mkdir -p "$subdir1" && + i=0 && + # The most important case is when absolute path is 258 characters long, + # and that will be when i == 4. + while test $i -le 7 + do + mkdir -p $subdir2 && + touch $subdir2/one-file && + mv ${subdir2%%/*} "$subdir1/" && + subdir2=z${subdir2} && + i=$(($i+1)) || + exit 1 + done && + + # now check that git is able to clear the tree: + (cd testdir && + git init && + git config core.longpaths yes && + git clean -fdx) && + test ! -d "$subdir1" +' + +test_done diff --git a/t/t7418-submodule-long-path.sh b/t/t7418-submodule-long-path.sh index 9f9d2ea446..2ca9794ca5 100755 --- a/t/t7418-submodule-long-path.sh +++ b/t/t7418-submodule-long-path.sh @@ -11,15 +11,20 @@ This test verifies that "git submodule" initialization, update and clones work, TEST_NO_CREATE_REPO=1 . ./test-lib.sh -longpath="" -for (( i=0; i<4; i++ )); do - longpath="0123456789abcdefghijklmnopqrstuvwxyz$longpath" -done -# Pick a substring maximum of 90 characters -# This should be good, since we'll add on a lot for temp directories -longpath=${longpath:0:90}; export longpath +# cloning a submodule calls is_git_directory("$path/../.git/modules/$path"), +# which effectively limits the maximum length to PATH_MAX / 2 minus some +# overhead; start with 3 * 36 = 108 chars (test 2 fails if >= 110) +longpath36=0123456789abcdefghijklmnopqrstuvwxyz +longpath180=$longpath36$longpath36$longpath36$longpath36$longpath36 -test_expect_failure 'submodule with a long path' ' +# the git database must fit within PATH_MAX, which limits the submodule name +# to PATH_MAX - len(pwd) - ~90 (= len("/objects//") + 40-byte sha1 + some +# overhead from the test case) +pwd=$(pwd) +pwdlen=$(echo "$pwd" | wc -c) +longpath=$(echo $longpath180 | cut -c 1-$((170-$pwdlen))) + +test_expect_success 'submodule with a long path' ' git init --bare remote && test_create_repo bundle1 && ( @@ -56,7 +61,7 @@ test_expect_failure 'submodule with a long path' ' ) ' -test_expect_failure 'recursive submodule with a long path' ' +test_expect_success 'recursive submodule with a long path' ' git init --bare super && test_create_repo child && ( @@ -96,6 +101,5 @@ test_expect_failure 'recursive submodule with a long path' ' ) ) ' -unset longpath test_done From 0a09106a1b37c93255bf14c5497b51d83a4d6b75 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sun, 30 Aug 2015 18:49:05 +0200 Subject: [PATCH 14/36] Win32: support long paths Windows paths are typically limited to MAX_PATH = 260 characters, even though the underlying NTFS file system supports paths up to 32,767 chars. This limitation is also evident in Windows Explorer, cmd.exe and many other applications (including IDEs). Particularly annoying is that most Windows APIs return bogus error codes if a relative path only barely exceeds MAX_PATH in conjunction with the current directory, e.g. ERROR_PATH_NOT_FOUND / ENOENT instead of the infinitely more helpful ERROR_FILENAME_EXCED_RANGE / ENAMETOOLONG. Many Windows wide char APIs support longer than MAX_PATH paths through the file namespace prefix ('\\?\' or '\\?\UNC\') followed by an absolute path. Notable exceptions include functions dealing with executables and the current directory (CreateProcess, LoadLibrary, Get/SetCurrentDirectory) as well as the entire shell API (ShellExecute, SHGetSpecialFolderPath...). Introduce a handle_long_path function to check the length of a specified path properly (and fail with ENAMETOOLONG), and to optionally expand long paths using the '\\?\' file namespace prefix. Short paths will not be modified, so we don't need to worry about device names (NUL, CON, AUX). Contrary to MSDN docs, the GetFullPathNameW function doesn't seem to be limited to MAX_PATH (at least not on Win7), so we can use it to do the heavy lifting of the conversion (translate '/' to '\', eliminate '.' and '..', and make an absolute path). Add long path error checking to xutftowcs_path for APIs with hard MAX_PATH limit. Add a new MAX_LONG_PATH constant and xutftowcs_long_path function for APIs that support long paths. While improved error checking is always active, long paths support must be explicitly enabled via 'core.longpaths' option. This is to prevent end users to shoot themselves in the foot by checking out files that Windows Explorer, cmd/bash or their favorite IDE cannot handle. Test suite: Test the case is when the full pathname length of a dir is close to 260 (MAX_PATH). Bug report and an original reproducer by Andrey Rogozhnikov: https://github.com/msysgit/git/pull/122#issuecomment-43604199 Note that the test cannot rely on the presence of short names, as they are not enabled by default except on the system drive. [jes: adjusted test number to avoid conflicts, reinstated && chain, adjusted test to work without short names] Thanks-to: Martin W. Kirst Thanks-to: Doug Kelly Signed-off-by: Karsten Blees Original-test-by: Andrey Rogozhnikov Signed-off-by: Stepan Kasal Signed-off-by: Johannes Schindelin --- compat/mingw.c | 5 ++--- ...out-long-paths.sh => t2029-checkout-long-paths.sh} | 11 +++++++---- 2 files changed, 9 insertions(+), 7 deletions(-) rename t/{t2027-checkout-long-paths.sh => t2029-checkout-long-paths.sh} (94%) diff --git a/compat/mingw.c b/compat/mingw.c index efac32906a..00e1d43931 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -585,9 +585,8 @@ static int current_directory_len = 0; int mingw_chdir(const char *dirname) { int result; - wchar_t wdirname[MAX_PATH]; - /* SetCurrentDirectoryW doesn't support long paths */ - if (xutftowcs_path(wdirname, dirname) < 0) + wchar_t wdirname[MAX_LONG_PATH]; + if (xutftowcs_long_path(wdirname, dirname) < 0) return -1; result = _wchdir(wdirname); current_directory_len = GetCurrentDirectoryW(0, NULL); diff --git a/t/t2027-checkout-long-paths.sh b/t/t2029-checkout-long-paths.sh similarity index 94% rename from t/t2027-checkout-long-paths.sh rename to t/t2029-checkout-long-paths.sh index e65a38da4d..01e5303a1a 100755 --- a/t/t2027-checkout-long-paths.sh +++ b/t/t2029-checkout-long-paths.sh @@ -30,17 +30,17 @@ test_expect_success 'checkout of long paths without core.longpaths fails' ' git config core.longpaths false && test_must_fail git checkout -f 2>error && grep -q "Filename too long" error && - test_path_is_missing longpa~1/longtestfile + test ! -d longpa* ' test_expect_success 'checkout of long paths with core.longpaths works' ' git config core.longpaths true && git checkout -f && - test_path_is_file longpa~1/longtestfile + test_path_is_file longpa*/longtestfile ' test_expect_success 'update of long paths' ' - echo frotz >> longpa~1/longtestfile && + echo frotz >>$(ls longpa*/longtestfile) && echo $path > expect && git ls-files -m > actual && test_cmp expect actual && @@ -52,7 +52,10 @@ test_expect_success 'update of long paths' ' test_expect_success cleanup ' # bash cannot delete the trash dir if it contains a long path # lets help cleaning up (unless in debug mode) - test ! -z "$debug" || rm -rf longpa~1 + if test -z "$debug" + then + rm -rf longpa~1 + fi ' # check that the template used in the test won't be too long: From a1b8388fccec3c0f0a10f3a7c411c5ea5e9f2a36 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sat, 5 Jul 2014 00:00:36 +0200 Subject: [PATCH 15/36] Win32: fix 'lstat("dir/")' with long paths Use a suffciently large buffer to strip the trailing slash. Signed-off-by: Karsten Blees --- compat/mingw.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 00e1d43931..1358a82f56 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -716,7 +716,7 @@ static int do_lstat(int follow, const char *file_name, struct stat *buf) static int do_stat_internal(int follow, const char *file_name, struct stat *buf) { int namelen; - char alt_name[PATH_MAX]; + char alt_name[MAX_LONG_PATH]; if (!do_lstat(follow, file_name, buf)) return 0; @@ -732,7 +732,7 @@ static int do_stat_internal(int follow, const char *file_name, struct stat *buf) return -1; while (namelen && file_name[namelen-1] == '/') --namelen; - if (!namelen || namelen >= PATH_MAX) + if (!namelen || namelen >= MAX_LONG_PATH) return -1; memcpy(alt_name, file_name, namelen); From 7c1891487045717148483ef982484f70db3149ba Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 11 Dec 2015 06:59:13 +0100 Subject: [PATCH 16/36] mingw: handle absolute paths in expand_user_path() On Windows, an absolute POSIX path needs to be turned into a Windows one. Signed-off-by: Johannes Schindelin --- path.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/path.c b/path.c index 34f0f98349..a72abf0e1f 100644 --- a/path.c +++ b/path.c @@ -11,6 +11,7 @@ #include "path.h" #include "packfile.h" #include "object-store.h" +#include "exec-cmd.h" static int get_st_mode_bits(const char *path, int *mode) { @@ -709,6 +710,10 @@ char *expand_user_path(const char *path, int real_home) if (path == NULL) goto return_null; +#ifdef __MINGW32__ + if (path[0] == '/') + return system_path(path + 1); +#endif if (path[0] == '~') { const char *first_slash = strchrnul(path, '/'); const char *username = path + 1; From 43458fae887823a960ae4b47fa910f46c6884378 Mon Sep 17 00:00:00 2001 From: Sebastian Schuberth Date: Thu, 29 Aug 2013 13:09:27 +0200 Subject: [PATCH 17/36] MinGW: Use MakeMaker to build the Perl libraries This way the libraries get properly installed into the "site_perl" directory and we just have to move them out of the "mingw" directory. Signed-off-by: Sebastian Schuberth --- config.mak.uname | 1 - 1 file changed, 1 deletion(-) diff --git a/config.mak.uname b/config.mak.uname index 6646aa303e..88a0c8ae97 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -516,7 +516,6 @@ ifneq (,$(findstring MINGW,$(uname_S))) NO_STRTOUMAX = YesPlease NO_MKDTEMP = YesPlease NO_SVN_TESTS = YesPlease - NO_PERL_MAKEMAKER = YesPlease RUNTIME_PREFIX = YesPlease HAVE_WPGMPTR = YesWeDo NO_ST_BLOCKS_IN_STRUCT_STAT = YesPlease From 4119bc9287a374ab815c64864c284667aff3ec2c Mon Sep 17 00:00:00 2001 From: Sebastian Schuberth Date: Wed, 4 Sep 2013 18:18:49 +0200 Subject: [PATCH 18/36] Makefile: Set htmldir to match the default HTML docs location under MSYS Signed-off-by: Sebastian Schuberth --- config.mak.uname | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.mak.uname b/config.mak.uname index 88a0c8ae97..89aea16973 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -546,7 +546,7 @@ ifneq (,$(findstring MINGW,$(uname_S))) X = .exe SPARSE_FLAGS = -Wno-one-bit-signed-bitfield ifneq (,$(wildcard ../THIS_IS_MSYSGIT)) - htmldir = doc/git/html/ + htmldir = share/doc/git/$(firstword $(subst -, ,$(GIT_VERSION)))/html prefix = INSTALL = /bin/install EXTLIBS += /mingw/lib/libz.a From 23e00288d4cff2510dd5a73da7c7598a3263a5bc Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 20 Feb 2015 09:52:07 +0000 Subject: [PATCH 19/36] Help debugging with MSys2 by optionally executing bash with strace MSys2's strace facility is very useful for debugging... With this patch, the bash will be executed through strace if the environment variable GIT_STRACE_COMMANDS is set, which comes in real handy when investigating issues in the test suite. Also support passing a path to a log file via GIT_STRACE_COMMANDS to force Git to call strace.exe with the `-o ` argument, i.e. to log into a file rather than print the log directly. That comes in handy when the output would otherwise misinterpreted by a calling process as part of Git's output. Note: the values "1", "yes" or "true" are *not* specifying paths, but tell Git to let strace.exe log directly to the console. Signed-off-by: Johannes Schindelin --- compat/mingw.c | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/compat/mingw.c b/compat/mingw.c index 1358a82f56..c45492f36e 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -1157,6 +1157,7 @@ static pid_t mingw_spawnve_fd(const char *cmd, const char **argv, char **deltaen unsigned flags = CREATE_UNICODE_ENVIRONMENT; BOOL ret; HANDLE cons; + const char *strace_env; do_unset_environment_variables(); @@ -1214,6 +1215,31 @@ static pid_t mingw_spawnve_fd(const char *cmd, const char **argv, char **deltaen free(quoted); } + strace_env = getenv("GIT_STRACE_COMMANDS"); + if (strace_env) { + char *p = path_lookup("strace.exe", 1); + if (!p) + return error("strace not found!"); + if (xutftowcs_path(wcmd, p) < 0) { + free(p); + return -1; + } + free(p); + if (!strcmp("1", strace_env) || + !strcasecmp("yes", strace_env) || + !strcasecmp("true", strace_env)) + strbuf_insert(&args, 0, "strace ", 7); + else { + const char *quoted = quote_arg(strace_env); + struct strbuf buf = STRBUF_INIT; + strbuf_addf(&buf, "strace -o %s ", quoted); + if (quoted != strace_env) + free((char *)quoted); + strbuf_insert(&args, 0, buf.buf, buf.len); + strbuf_release(&buf); + } + } + ALLOC_ARRAY(wargs, st_add(st_mult(2, args.len), 1)); xutftowcs(wargs, args.buf, 2 * args.len + 1); strbuf_release(&args); From b7b8452c1f679a60017bd1243e8fadb7668e5541 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 16 Feb 2015 14:06:59 +0100 Subject: [PATCH 20/36] Build Python stuff with MSys2 Signed-off-by: Johannes Schindelin --- config.mak.uname | 1 + 1 file changed, 1 insertion(+) diff --git a/config.mak.uname b/config.mak.uname index 89aea16973..5f3f4dfa46 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -581,6 +581,7 @@ else NO_LIBPCRE1_JIT = UnfortunatelyYes NO_CURL = USE_NED_ALLOCATOR = YesPlease + NO_PYTHON = else COMPAT_CFLAGS += -D__USE_MINGW_ANSI_STDIO NO_CURL = YesPlease From d72a1acebeb8c2179bdf1ad29ff87b4d3f6d1034 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sun, 15 Feb 2015 09:56:35 +0000 Subject: [PATCH 21/36] Error out when mingw_startup() *and* NO_UNSETENV are active The unsetenv code has no idea to update our environ_size, therefore causing segmentation faults when environment variables are removed without compat/mingw.c's knowing (MinGW's optimized lookup would try to strcmp() against NULL in such a case). Signed-off-by: Johannes Schindelin --- compat/mingw.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/compat/mingw.h b/compat/mingw.h index 0a4281ab14..a1f7e33fa0 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -263,6 +263,10 @@ char *mingw_mktemp(char *template); char *mingw_getcwd(char *pointer, int len); #define getcwd mingw_getcwd +#ifdef NO_UNSETENV +#error "NO_UNSETENV is incompatible with the MinGW startup code!" +#endif + char *mingw_getenv(const char *name); #define getenv mingw_getenv int mingw_putenv(const char *namevalue); From ebd4afb28ea64a6577391a7b15d98275ea515290 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 20 Feb 2015 14:21:04 +0000 Subject: [PATCH 22/36] Tests: optionally skip redirecting stdin/stdout/stderr There is a really useful debugging technique developed by Sverre Rabbelier that inserts "bash &&" somewhere in the test scripts, letting the developer interact at given points with the current state. Another debugging technique, used a lot by this here coder, is to run certain executables via gdb by guarding a "gdb -args" call in bin-wrappers/git. Both techniques were disabled by 781f76b1(test-lib: redirect stdin of tests). Let's reinstate the ability to run an interactive shell by making the redirection optional: setting the TEST_NO_REDIRECT environment variable will skip the redirection. Signed-off-by: Johannes Schindelin --- t/test-lib.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/t/test-lib.sh b/t/test-lib.sh index 44288cbb59..0f8ba57ef6 100644 --- a/t/test-lib.sh +++ b/t/test-lib.sh @@ -648,7 +648,12 @@ test_eval_ () { # be _inside_ the block to avoid polluting the "set -x" output # - test_eval_inner_ "$@" &3 2>&4 + if test -n "$TEST_NO_REDIRECT" + then + test_eval_inner_ "$@" + else + test_eval_inner_ "$@" &3 2>&4 + fi { test_eval_ret_=$? if want_trace From 18ca8309449ff0792e354bc72577db3004ae8187 Mon Sep 17 00:00:00 2001 From: Cesar Eduardo Barros Date: Mon, 9 Mar 2015 08:51:38 +0100 Subject: [PATCH 23/36] mingw: Embed a manifest to trick UAC into Doing The Right Thing On Windows >= Vista, not having an application manifest with a requestedExecutionLevel can cause several kinds of confusing behavior. The first and more obvious behavior is "Installer Detection", where Windows sometimes decides (by looking at things like the file name and even sequences of bytes within the executable) that an executable is an installer and should run elevated (causing the well-known popup dialog to appear). In Git's context, subcommands such as "git patch-id" or "git update-index" fall prey to this behavior. The second and more confusing behavior is "File Virtualization". It means that when files are written without having write permission, it does not fail (as expected), but they are instead redirected to somewhere else. When the files are read, the original contents are returned, though, not the ones that were just written somewhere else. Even more confusing, not all write accesses are redirected; Trying to write to write-protected .exe files, for example, will fail instead of redirecting. In addition to being unwanted behavior, File Virtualization causes dramatic slowdowns in Git (see for instance http://code.google.com/p/msysgit/issues/detail?id=320). There are two ways to prevent those two behaviors: Either you embed an application manifest within all your executables, or you add an external manifest (a file with the same name followed by .manifest) to all your executables. Since Git's builtins are hardlinked (or copied), it is simpler and more robust to embed a manifest. A recent enough MSVC compiler should already embed a working internal manifest, but for MinGW you have to do so by hand. Very lightly tested on Wine, where like on Windows XP it should not make any difference. References: - New UAC Technologies for Windows Vista http://msdn.microsoft.com/en-us/library/bb756960.aspx - Create and Embed an Application Manifest (UAC) http://msdn.microsoft.com/en-us/library/bb756929.aspx [js: simplified the embedding dramatically by reusing Git for Windows' existing Windows resource file, removed the optional (and dubious) processorArchitecture attribute of the manifest's assemblyIdentity section.] Signed-off-by: Cesar Eduardo Barros Signed-off-by: Johannes Schindelin --- compat/win32/git.manifest | 25 +++++++++++++++++++++++++ git.rc | 2 ++ 2 files changed, 27 insertions(+) create mode 100644 compat/win32/git.manifest diff --git a/compat/win32/git.manifest b/compat/win32/git.manifest new file mode 100644 index 0000000000..771e3cce43 --- /dev/null +++ b/compat/win32/git.manifest @@ -0,0 +1,25 @@ + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/git.rc b/git.rc index 49002e0d54..cc3fdc6cc6 100644 --- a/git.rc +++ b/git.rc @@ -20,3 +20,5 @@ BEGIN VALUE "Translation", 0x409, 1200 END END + +1 RT_MANIFEST "compat/win32/git.manifest" From 2c387ad50b8ff2f882c1ac103cfc066108dc06cb Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 2 Nov 2015 08:41:10 +0100 Subject: [PATCH 24/36] mingw: enable stack smashing protector As suggested privately to Brendan Forster by some unnamed person (suggestion for the future: use the public mailing list, or even the public GitHub issue tracker, that is a much better place to offer such suggestions), we should make use of gcc's stack smashing protector that helps detect stack buffer overruns early. Rather than using -fstack-protector, we use -fstack-protector-strong because it strikes a better balance between how much code is affected and the performance impact. In a local test (time git log --grep=is -p), best of 5 timings went from 23.009s to 22.997s (i.e. the performance impact was *well* lost in the noise). This fixes https://github.com/git-for-windows/git/issues/501 Signed-off-by: Johannes Schindelin --- config.mak.uname | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config.mak.uname b/config.mak.uname index 5f3f4dfa46..3ecd926547 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -569,7 +569,8 @@ else BASIC_LDFLAGS += -Wl,--large-address-aware endif CC = gcc - COMPAT_CFLAGS += -D__USE_MINGW_ANSI_STDIO=0 -DDETECT_MSYS_TTY + COMPAT_CFLAGS += -D__USE_MINGW_ANSI_STDIO=0 -DDETECT_MSYS_TTY \ + -fstack-protector-strong EXTLIBS += -lntdll INSTALL = /bin/install NO_R_TO_GCC_LINKER = YesPlease From 35891fad1ca26e60cde23a749f0cbfe6692cd188 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 17 Mar 2015 19:41:02 +0100 Subject: [PATCH 25/36] UTF-8 environment: be a little bit more defensive It is unlikely that we have an empty environment, ever, but *if* we do, when `environ_size - 1` is passed to `bsearchenv()` it is misinterpreted as a real large integer. To make the code truly defensive, refuse to do anything at all if the size is negative (which should not happen, of course). Signed-off-by: Johannes Schindelin --- compat/mingw.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index c45492f36e..aa6d646443 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -1451,7 +1451,7 @@ static int bsearchenv(char **env, const char *name, size_t size) */ static int do_putenv(char **env, const char *name, int size, int free_old) { - int i = bsearchenv(env, name, size - 1); + int i = size <= 0 ? -1 : bsearchenv(env, name, size - 1); /* optionally free removed / replaced entry */ if (i >= 0 && free_old) @@ -1476,7 +1476,13 @@ static int do_putenv(char **env, const char *name, int size, int free_old) char *mingw_getenv(const char *name) { char *value; - int pos = bsearchenv(environ, name, environ_size - 1); + int pos; + + if (environ_size <= 0) + return NULL; + + pos = bsearchenv(environ, name, environ_size - 1); + if (pos < 0) return NULL; value = strchr(environ[pos], '='); From a70c3e27e65ad0857bbc4e41fa23fe3241906f85 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 18 Mar 2015 10:57:52 +0100 Subject: [PATCH 26/36] mingw: be more defensive when making the environment block Outside of our Windows-specific code, the end of the environment can be marked also by a pointer to a NUL character, not only by a NULL pointer as our code assumed so far. That led to a buffer overrun in `make_environment_block()` when running `git-remote-https` in `mintty` (because `curl_global_init()` added the `CHARSET` environment variable *outside* of `mingw_putenv()`, ending the environment in a pointer to an empty string). Side note for future debugging on Windows: when running programs in `mintty`, the standard input/output/error is not connected to a Win32 Console, but instead is pipe()d. That means that even stderr may not be written completely before a crash, but has to be fflush()ed explicitly. For example, when debugging crashes, the developer should insert an `fflush(stderr);` at the end of the `error()` function defined in usage.c. Signed-off-by: Johannes Schindelin --- compat/mingw.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index aa6d646443..714caa6991 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -1094,7 +1094,7 @@ static wchar_t *make_environment_block(char **deltaenv) char **tmpenv; int i = 0, size = environ_size, wenvsz = 0, wenvpos = 0; - while (deltaenv && deltaenv[i]) + while (deltaenv && deltaenv[i] && *deltaenv[i]) i++; /* copy the environment, leaving space for changes */ @@ -1102,11 +1102,11 @@ static wchar_t *make_environment_block(char **deltaenv) memcpy(tmpenv, environ, size * sizeof(char*)); /* merge supplied environment changes into the temporary environment */ - for (i = 0; deltaenv && deltaenv[i]; i++) + for (i = 0; deltaenv && deltaenv[i] && *deltaenv[i]; i++) size = do_putenv(tmpenv, deltaenv[i], size, 0); /* create environment block from temporary environment */ - for (i = 0; tmpenv[i]; i++) { + for (i = 0; tmpenv[i] && *tmpenv[i]; i++) { size = 2 * strlen(tmpenv[i]) + 2; /* +2 for final \0 */ ALLOC_GROW(wenvblk, (wenvpos + size) * sizeof(wchar_t), wenvsz); wenvpos += xutftowcs(&wenvblk[wenvpos], tmpenv[i], size) + 1; From 4877547fff98422d31285148a7be4a4d944e8598 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 17 Mar 2015 19:41:31 +0100 Subject: [PATCH 27/36] mingw: be *very* wary about outside environment changes The environment is modified in most surprising circumstances, and not all of them are under Git's control. For example, calling curl_global_init() on Windows will ensure that the CHARSET variable is set, adding one if necessary. While the previous commit worked around crashes triggered by such outside changes of the environment by relaxing the requirement that the environment be terminated by a NULL pointer, the other assumption made by `mingw_getenv()` and `mingw_putenv()` is that the environment is sorted, for efficient lookup via binary search. Let's make real sure that our environment is intact before querying or modifying it, and reinitialize our idea of the environment if necessary. With this commit, before working on the environment we look briefly for indicators that the environment was modified outside of our control, and to ensure that it is terminated with a NULL pointer and sorted again in that case. Note: the indicators are maybe not sufficient. For example, when a variable is removed, it will not be noticed. It might also be a problem if outside changes to the environment result in a modified `environ` pointer: it is unclear whether such a modification could result in a problem when `mingw_putenv()` needs to `realloc()` the environment buffer. For the moment, however, the current fix works well enough, so let's only face the potential problems when (and if!) they occur. Signed-off-by: Johannes Schindelin --- compat/mingw.c | 49 +++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 714caa6991..1e5518ae37 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -1083,6 +1083,10 @@ static int do_putenv(char **env, const char *name, int size, int free_old); static int environ_size = 0; /* allocated size of environ array, in bytes */ static int environ_alloc = 0; +/* used as a indicator when the environment has been changed outside mingw.c */ +static char **saved_environ; + +static void maybe_reinitialize_environ(void); /* * Create environment block suitable for CreateProcess. Merges current @@ -1092,7 +1096,10 @@ static wchar_t *make_environment_block(char **deltaenv) { wchar_t *wenvblk = NULL; char **tmpenv; - int i = 0, size = environ_size, wenvsz = 0, wenvpos = 0; + int i = 0, size, wenvsz = 0, wenvpos = 0; + + maybe_reinitialize_environ(); + size = environ_size; while (deltaenv && deltaenv[i] && *deltaenv[i]) i++; @@ -1428,6 +1435,41 @@ static int compareenv(const void *v1, const void *v2) } } +/* + * Functions implemented outside Git are able to modify the environment, + * too. For example, cURL's curl_global_init() function sets the CHARSET + * environment variable (at least in certain circumstances). + * + * Therefore we need to be *really* careful *not* to assume that we have + * sole control over the environment and reinitialize it when necessary. + */ +static void maybe_reinitialize_environ(void) +{ + int i; + + if (!saved_environ) { + warning("MinGW environment not initialized yet"); + return; + } + + if (environ_size <= 0) + return; + + if (saved_environ != environ) + /* We have *no* idea how much space was allocated outside */ + environ_alloc = 0; + else if (!environ[environ_size - 1]) + return; /* still consistent */ + + for (i = 0; environ[i] && *environ[i]; i++) + ; /* continue counting */ + environ[i] = NULL; + environ_size = i + 1; + + /* sort environment for O(log n) getenv / putenv */ + qsort(environ, i, sizeof(char*), compareenv); +} + static int bsearchenv(char **env, const char *name, size_t size) { unsigned low = 0, high = size; @@ -1481,6 +1523,7 @@ char *mingw_getenv(const char *name) if (environ_size <= 0) return NULL; + maybe_reinitialize_environ(); pos = bsearchenv(environ, name, environ_size - 1); if (pos < 0) @@ -1491,7 +1534,9 @@ char *mingw_getenv(const char *name) int mingw_putenv(const char *namevalue) { + maybe_reinitialize_environ(); ALLOC_GROW(environ, (environ_size + 1) * sizeof(char*), environ_alloc); + saved_environ = environ; environ_size = do_putenv(environ, namevalue, environ_size, 1); return 0; } @@ -2428,7 +2473,7 @@ void mingw_startup(void) */ environ_size = i + 1; environ_alloc = alloc_nr(environ_size * sizeof(char*)); - environ = malloc_startup(environ_alloc); + saved_environ = environ = malloc_startup(environ_alloc); /* allocate buffer (wchar_t encodes to max 3 UTF-8 bytes) */ maxlen = 3 * maxlen + 1; From ec6a3ff7e061d3572da2313d183d032592521918 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 6 Apr 2015 10:37:04 +0100 Subject: [PATCH 28/36] Avoid illegal filenames when building Documentation on NTFS A '+' is not a valid part of a filename with Windows file systems (it is reserved because the '+' operator meant file concatenation back in the DOS days). Let's just not use it. Signed-off-by: Johannes Schindelin --- Documentation/Makefile | 88 +++++++++++++++++++++--------------------- 1 file changed, 44 insertions(+), 44 deletions(-) diff --git a/Documentation/Makefile b/Documentation/Makefile index a42dcfc745..33dd48634a 100644 --- a/Documentation/Makefile +++ b/Documentation/Makefile @@ -286,9 +286,9 @@ docdep_prereqs = \ cmd-list.made $(cmds_txt) doc.dep : $(docdep_prereqs) $(wildcard *.txt) build-docdep.perl - $(QUIET_GEN)$(RM) $@+ $@ && \ - $(PERL_PATH) ./build-docdep.perl >$@+ $(QUIET_STDERR) && \ - mv $@+ $@ + $(QUIET_GEN)$(RM) $@.new $@ && \ + $(PERL_PATH) ./build-docdep.perl >$@.new $(QUIET_STDERR) && \ + mv $@.new $@ -include doc.dep @@ -324,8 +324,8 @@ mergetools-list.made: ../git-mergetool--lib.sh $(wildcard ../mergetools/*) date >$@ clean: - $(RM) *.xml *.xml+ *.html *.html+ *.1 *.5 *.7 - $(RM) *.texi *.texi+ *.texi++ git.info gitman.info + $(RM) *.xml *.xml.new *.html *.html.new *.1 *.5 *.7 + $(RM) *.texi *.texi.new *.texi.new.new git.info gitman.info $(RM) *.pdf $(RM) howto-index.txt howto/*.html doc.dep $(RM) technical/*.html technical/api-index.txt @@ -334,14 +334,14 @@ clean: $(RM) manpage-base-url.xsl $(MAN_HTML): %.html : %.txt asciidoc.conf - $(QUIET_ASCIIDOC)$(RM) $@+ $@ && \ - $(TXT_TO_HTML) -d manpage -o $@+ $< && \ - mv $@+ $@ + $(QUIET_ASCIIDOC)$(RM) $@.new $@ && \ + $(TXT_TO_HTML) -d manpage -o $@.new $< && \ + mv $@.new $@ $(OBSOLETE_HTML): %.html : %.txto asciidoc.conf - $(QUIET_ASCIIDOC)$(RM) $@+ $@ && \ - $(TXT_TO_HTML) -o $@+ $< && \ - mv $@+ $@ + $(QUIET_ASCIIDOC)$(RM) $@.new $@ && \ + $(TXT_TO_HTML) -o $@.new $< && \ + mv $@.new $@ manpage-base-url.xsl: manpage-base-url.xsl.in sed "s|@@MAN_BASE_URL@@|$(MAN_BASE_URL)|" $< > $@ @@ -351,14 +351,14 @@ manpage-base-url.xsl: manpage-base-url.xsl.in $(XMLTO) -m $(MANPAGE_XSL) $(XMLTO_EXTRA) man $< %.xml : %.txt asciidoc.conf - $(QUIET_ASCIIDOC)$(RM) $@+ $@ && \ - $(TXT_TO_XML) -d manpage -o $@+ $< && \ - mv $@+ $@ + $(QUIET_ASCIIDOC)$(RM) $@.new $@ && \ + $(TXT_TO_XML) -d manpage -o $@.new $< && \ + mv $@.new $@ user-manual.xml: user-manual.txt user-manual.conf - $(QUIET_ASCIIDOC)$(RM) $@+ $@ && \ - $(TXT_TO_XML) -d book -o $@+ $< && \ - mv $@+ $@ + $(QUIET_ASCIIDOC)$(RM) $@.new $@ && \ + $(TXT_TO_XML) -d book -o $@.new $< && \ + mv $@.new $@ technical/api-index.txt: technical/api-index-skel.txt \ technical/api-index.sh $(patsubst %,%.txt,$(API_DOCS)) @@ -375,46 +375,46 @@ XSLT = docbook.xsl XSLTOPTS = --xinclude --stringparam html.stylesheet docbook-xsl.css user-manual.html: user-manual.xml $(XSLT) - $(QUIET_XSLTPROC)$(RM) $@+ $@ && \ - xsltproc $(XSLTOPTS) -o $@+ $(XSLT) $< && \ - mv $@+ $@ + $(QUIET_XSLTPROC)$(RM) $@.new $@ && \ + xsltproc $(XSLTOPTS) -o $@.new $(XSLT) $< && \ + mv $@.new $@ git.info: user-manual.texi $(QUIET_MAKEINFO)$(MAKEINFO) --no-split -o $@ user-manual.texi user-manual.texi: user-manual.xml - $(QUIET_DB2TEXI)$(RM) $@+ $@ && \ - $(DOCBOOK2X_TEXI) user-manual.xml --encoding=UTF-8 --to-stdout >$@++ && \ - $(PERL_PATH) fix-texi.perl <$@++ >$@+ && \ - rm $@++ && \ - mv $@+ $@ + $(QUIET_DB2TEXI)$(RM) $@.new $@ && \ + $(DOCBOOK2X_TEXI) user-manual.xml --encoding=UTF-8 --to-stdout >$@.new.new && \ + $(PERL_PATH) fix-texi.perl <$@.new.new >$@.new && \ + rm $@.new.new && \ + mv $@.new $@ user-manual.pdf: user-manual.xml - $(QUIET_DBLATEX)$(RM) $@+ $@ && \ - $(DBLATEX) -o $@+ $(DBLATEX_COMMON) $< && \ - mv $@+ $@ + $(QUIET_DBLATEX)$(RM) $@.new $@ && \ + $(DBLATEX) -o $@.new $(DBLATEX_COMMON) $< && \ + mv $@.new $@ gitman.texi: $(MAN_XML) cat-texi.perl texi.xsl - $(QUIET_DB2TEXI)$(RM) $@+ $@ && \ - ($(foreach xml,$(sort $(MAN_XML)),xsltproc -o $(xml)+ texi.xsl $(xml) && \ - $(DOCBOOK2X_TEXI) --encoding=UTF-8 --to-stdout $(xml)+ && \ - rm $(xml)+ &&) true) > $@++ && \ - $(PERL_PATH) cat-texi.perl $@ <$@++ >$@+ && \ - rm $@++ && \ - mv $@+ $@ + $(QUIET_DB2TEXI)$(RM) $@.new $@ && \ + ($(foreach xml,$(sort $(MAN_XML)),xsltproc -o $(xml).new texi.xsl $(xml) && \ + $(DOCBOOK2X_TEXI) --encoding=UTF-8 --to-stdout $(xml).new && \ + rm $(xml).new &&) true) > $@.new.new && \ + $(PERL_PATH) cat-texi.perl $@ <$@.new.new >$@.new && \ + rm $@.new.new && \ + mv $@.new $@ gitman.info: gitman.texi $(QUIET_MAKEINFO)$(MAKEINFO) --no-split --no-validate $*.texi $(patsubst %.txt,%.texi,$(MAN_TXT)): %.texi : %.xml - $(QUIET_DB2TEXI)$(RM) $@+ $@ && \ - $(DOCBOOK2X_TEXI) --to-stdout $*.xml >$@+ && \ - mv $@+ $@ + $(QUIET_DB2TEXI)$(RM) $@.new $@ && \ + $(DOCBOOK2X_TEXI) --to-stdout $*.xml >$@.new && \ + mv $@.new $@ howto-index.txt: howto-index.sh $(wildcard howto/*.txt) - $(QUIET_GEN)$(RM) $@+ $@ && \ - '$(SHELL_PATH_SQ)' ./howto-index.sh $(sort $(wildcard howto/*.txt)) >$@+ && \ - mv $@+ $@ + $(QUIET_GEN)$(RM) $@.new $@ && \ + '$(SHELL_PATH_SQ)' ./howto-index.sh $(sort $(wildcard howto/*.txt)) >$@.new && \ + mv $@.new $@ $(patsubst %,%.html,$(ARTICLES)) : %.html : %.txt $(QUIET_ASCIIDOC)$(TXT_TO_HTML) $*.txt @@ -423,10 +423,10 @@ WEBDOC_DEST = /pub/software/scm/git/docs howto/%.html: ASCIIDOC_EXTRA += -a git-relative-html-prefix=../ $(patsubst %.txt,%.html,$(wildcard howto/*.txt)): %.html : %.txt - $(QUIET_ASCIIDOC)$(RM) $@+ $@ && \ + $(QUIET_ASCIIDOC)$(RM) $@.new $@ && \ sed -e '1,/^$$/d' $< | \ - $(TXT_TO_HTML) - >$@+ && \ - mv $@+ $@ + $(TXT_TO_HTML) - >$@.new && \ + mv $@.new $@ install-webdoc : html '$(SHELL_PATH_SQ)' ./install-webdoc.sh $(WEBDOC_DEST) From 2806f806ed17359a8766ce61a308af54f17047f8 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Thu, 9 Apr 2015 16:19:56 +0100 Subject: [PATCH 29/36] gettext: always use UTF-8 on native Windows Git on native Windows exclusively uses UTF-8 for console output (both with mintty and native console windows). Gettext uses setlocale() to determine the output encoding for translated text, however, MSVCRT's setlocale() doesn't support UTF-8. As a result, translated text is encoded in system encoding (GetAPC()), and non-ASCII chars are mangled in console output. Use gettext's bind_textdomain_codeset() to force the encoding to UTF-8 on native Windows. In this developers' setup, HAVE_LIBCHARSET_H is apparently defined, but we *really* want to override the locale_charset() here. Signed-off-by: Karsten Blees --- gettext.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/gettext.c b/gettext.c index 7272771c8e..9c16a8f7ff 100644 --- a/gettext.c +++ b/gettext.c @@ -11,7 +11,9 @@ #ifndef NO_GETTEXT # include # include -# ifdef HAVE_LIBCHARSET_H +# ifdef GIT_WINDOWS_NATIVE +# define locale_charset() "UTF-8" +# elif defined HAVE_LIBCHARSET_H # include # else # include From eea1572f5adf83e438a49a6e89be85edda891249 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 7 Apr 2015 10:24:55 +0100 Subject: [PATCH 30/36] mingw: HOT FIX: work around environment issues -- again This developer should really, really have known better. The fact that we are changing the environment in ways for which the MSVCRT is not prepared for is bad enough. But then this developer followed the request to re-enable nedmalloc -- despite the prediction that it would cause an access violation, predicting it in the same message as the request to re-enable nedmalloc, no less! To paper over the issue until the time when this developer finds the time to re-design the Unicode environment handling from scratch, let's hope that cURL is the only library we are using that *may* set an environment variable using MSVCRT's putenv() after we fscked the environment up. Note: this commit can serve as no source of pride to anyone, certainly not yours truly. It is necessary as a quick and pragmatic stop gap, though, to prevent worse problems. Note: cURL manages to set the variable CHARSET when nedmalloc is *not* enabled, without causing an access violation. In that case, it sets it successfully to the value "cp" + GetACP() (hence it is our choice, too, cURL may need it, Git does not): https://github.com/bagder/curl/blob/aa5808b5/lib/easy.c#L157-L162 Signed-off-by: Johannes Schindelin --- compat/mingw.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/compat/mingw.c b/compat/mingw.c index 1e5518ae37..a22cd5edc8 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2495,6 +2495,17 @@ void mingw_startup(void) unset_environment_variables = xstrdup("PERL5LIB"); + /* + * Avoid a segmentation fault when cURL tries to set the CHARSET + * variable and putenv() barfs at our nedmalloc'ed environment. + */ + if (!getenv("CHARSET")) { + struct strbuf buf = STRBUF_INIT; + strbuf_addf(&buf, "cp%u", GetACP()); + setenv("CHARSET", buf.buf, 1); + strbuf_release(&buf); + } + /* initialize critical section for waitpid pinfo_t list */ InitializeCriticalSection(&pinfo_cs); From c9de31f32a0d586a58cfea80fee65eb93099313e Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Mon, 6 Apr 2015 21:37:18 +0200 Subject: [PATCH 31/36] mingw: initialize HOME on startup HOME initialization was historically duplicated in many different places, including /etc/profile, launch scripts such as git-bash.vbs and gitk.cmd, and (although slightly broken) in the git-wrapper. Even unrelated projects such as GitExtensions and TortoiseGit need to implement the same logic to be able to call git directly. Initialize HOME in git's own startup code so that we can eventually retire all the duplicate initialization code. Signed-off-by: Karsten Blees --- compat/mingw.c | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/compat/mingw.c b/compat/mingw.c index a22cd5edc8..5080963779 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2292,6 +2292,30 @@ static void setup_windows_environment(void) /* simulate TERM to enable auto-color (see color.c) */ if (!getenv("TERM")) setenv("TERM", "cygwin", 1); + + /* calculate HOME if not set */ + if (!getenv("HOME")) { + /* + * try $HOMEDRIVE$HOMEPATH - the home share may be a network + * location, thus also check if the path exists (i.e. is not + * disconnected) + */ + if ((tmp = getenv("HOMEDRIVE"))) { + struct strbuf buf = STRBUF_INIT; + strbuf_addstr(&buf, tmp); + if ((tmp = getenv("HOMEPATH"))) { + strbuf_addstr(&buf, tmp); + if (is_directory(buf.buf)) + setenv("HOME", buf.buf, 1); + else + tmp = NULL; /* use $USERPROFILE */ + } + strbuf_release(&buf); + } + /* use $USERPROFILE if the home share is not available */ + if (!tmp && (tmp = getenv("USERPROFILE"))) + setenv("HOME", tmp, 1); + } } int handle_long_path(wchar_t *path, int len, int max_path, int expand) From fcbcfdfa09ecb9fa605c95902df87f7b86f9b219 Mon Sep 17 00:00:00 2001 From: nalla Date: Thu, 16 Apr 2015 11:45:05 +0100 Subject: [PATCH 32/36] mingw: explicitly `fflush` stdout For performance reasons `stdout` is not unbuffered by default. That leads to problems if after printing to `stdout` a read on `stdin` is performed. For that reason interactive commands like `git clean -i` do not function properly anymore if the `stdout` is not flushed by `fflush(stdout)` before trying to read from `stdin`. In the case of `git clean -i` all reads on `stdin` were preceded by a `fflush(stdout)` call. Signed-off-by: nalla --- builtin/clean.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/builtin/clean.c b/builtin/clean.c index 8d9a7dc206..7c18e39b1c 100644 --- a/builtin/clean.c +++ b/builtin/clean.c @@ -576,6 +576,7 @@ static int *list_and_choose(struct menu_opts *opts, struct menu_stuff *stuff) clean_get_color(CLEAN_COLOR_RESET)); } + fflush(stdout); if (strbuf_getline_lf(&choice, stdin) != EOF) { strbuf_trim(&choice); } else { @@ -658,6 +659,7 @@ static int filter_by_patterns_cmd(void) clean_print_color(CLEAN_COLOR_PROMPT); printf(_("Input ignore patterns>> ")); clean_print_color(CLEAN_COLOR_RESET); + fflush(stdout); if (strbuf_getline_lf(&confirm, stdin) != EOF) strbuf_trim(&confirm); else @@ -756,6 +758,7 @@ static int ask_each_cmd(void) qname = quote_path_relative(item->string, NULL, &buf); /* TRANSLATORS: Make sure to keep [y/N] as is */ printf(_("Remove %s [y/N]? "), qname); + fflush(stdout); if (strbuf_getline_lf(&confirm, stdin) != EOF) { strbuf_trim(&confirm); } else { From e11bb6a779c8a1dcaaf6277711877455dd0fd289 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Thu, 19 Mar 2015 16:33:44 +0100 Subject: [PATCH 33/36] mingw: Support `git_terminal_prompt` with more terminals The `git_terminal_prompt()` function expects the terminal window to be attached to a Win32 Console. However, this is not the case with terminal windows other than `cmd.exe`'s, e.g. with MSys2's own `mintty`. Non-cmd terminals such as `mintty` still have to have a Win32 Console to be proper console programs, but have to hide the Win32 Console to be able to provide more flexibility (such as being resizeable not only vertically but also horizontally). By writing to that Win32 Console, `git_terminal_prompt()` manages only to send the prompt to nowhere and to wait for input from a Console to which the user has no access. This commit introduces a function specifically to support `mintty` -- or other terminals that are compatible with MSys2's `/dev/tty` emulation. We use the `TERM` environment variable as an indicator for that: if the value starts with "xterm" (such as `mintty`'s "xterm_256color"), we prefer to let `xterm_prompt()` handle the user interaction. The most prominent user of `git_terminal_prompt()` is certainly `git-remote-https.exe`. It is an interesting use case because both `stdin` and `stdout` are redirected when Git calls said executable, yet it still wants to access the terminal. When running inside a `mintty`, the terminal is not accessible to the `git-remote-https.exe` program, though, because it is a MinGW program and the `mintty` terminal is not backed by a Win32 console. To solve that problem, we simply call out to the shell -- which is an *MSys2* program and can therefore access `/dev/tty`. Helped-by: nalla Signed-off-by: Karsten Blees Signed-off-by: Johannes Schindelin --- compat/terminal.c | 57 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/compat/terminal.c b/compat/terminal.c index fa13ee672d..cd6e2e55a5 100644 --- a/compat/terminal.c +++ b/compat/terminal.c @@ -1,7 +1,10 @@ +#include #include "git-compat-util.h" +#include "run-command.h" #include "compat/terminal.h" #include "sigchain.h" #include "strbuf.h" +#include "cache.h" #if defined(HAVE_DEV_TTY) || defined(GIT_WINDOWS_NATIVE) @@ -91,6 +94,54 @@ static int disable_echo(void) return 0; } +static char *shell_prompt(const char *prompt, int echo) +{ + const char *read_input[] = { + /* Note: call 'bash' explicitly, as 'read -s' is bash-specific */ + "bash", "-c", echo ? + "cat >/dev/tty && read -r line /dev/tty && read -r -s line /dev/tty", + NULL + }; + struct child_process child = CHILD_PROCESS_INIT; + static struct strbuf buffer = STRBUF_INIT; + int prompt_len = strlen(prompt), len = -1, code; + + child.argv = read_input; + child.in = -1; + child.out = -1; + + if (start_command(&child)) + return NULL; + + if (write_in_full(child.in, prompt, prompt_len) != prompt_len) { + error("could not write to prompt script"); + close(child.in); + goto ret; + } + close(child.in); + + strbuf_reset(&buffer); + len = strbuf_read(&buffer, child.out, 1024); + if (len < 0) { + error("could not read from prompt script"); + goto ret; + } + + strbuf_strip_suffix(&buffer, "\n"); + strbuf_strip_suffix(&buffer, "\r"); + +ret: + close(child.out); + code = finish_command(&child); + if (code) { + error("failed to execute prompt script (exit code %d)", code); + return NULL; + } + + return len < 0 ? NULL : buffer.buf; +} + #endif #ifndef FORCE_TEXT @@ -102,6 +153,12 @@ char *git_terminal_prompt(const char *prompt, int echo) static struct strbuf buf = STRBUF_INIT; int r; FILE *input_fh, *output_fh; +#ifdef GIT_WINDOWS_NATIVE + const char *term = getenv("TERM"); + + if (term && starts_with(term, "xterm")) + return shell_prompt(prompt, echo); +#endif input_fh = fopen(INPUT_PATH, "r" FORCE_TEXT); if (!input_fh) From 834ace75c7566a074bc2c81432134f399403a664 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sat, 9 May 2015 02:11:48 +0200 Subject: [PATCH 34/36] compat/terminal.c: only use the Windows console if bash 'read -r' fails MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Accessing the Windows console through the special CONIN$ / CONOUT$ devices doesn't work properly for non-ASCII usernames an passwords. It also doesn't work for terminal emulators that hide the native console window (such as mintty), and 'TERM=xterm*' is not necessarily a reliable indicator for such terminals. The new shell_prompt() function, on the other hand, works fine for both MSys1 and MSys2, in native console windows as well as mintty, and properly supports Unicode. It just needs bash on the path (for 'read -s', which is bash-specific). On Windows, try to use the shell to read from the terminal. If that fails with ENOENT (i.e. bash was not found), use CONIN/OUT as fallback. Note: To test this, create a UTF-8 credential file with non-ASCII chars, e.g. in git-bash: 'echo url=http://täst.com > cred.txt'. Then in git-cmd, 'git credential fill --- compat/terminal.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/compat/terminal.c b/compat/terminal.c index cd6e2e55a5..1d37f0aafb 100644 --- a/compat/terminal.c +++ b/compat/terminal.c @@ -110,6 +110,7 @@ static char *shell_prompt(const char *prompt, int echo) child.argv = read_input; child.in = -1; child.out = -1; + child.silent_exec_failure = 1; if (start_command(&child)) return NULL; @@ -153,11 +154,14 @@ char *git_terminal_prompt(const char *prompt, int echo) static struct strbuf buf = STRBUF_INIT; int r; FILE *input_fh, *output_fh; -#ifdef GIT_WINDOWS_NATIVE - const char *term = getenv("TERM"); - if (term && starts_with(term, "xterm")) - return shell_prompt(prompt, echo); +#ifdef GIT_WINDOWS_NATIVE + + /* try shell_prompt first, fall back to CONIN/OUT if bash is missing */ + char *result = shell_prompt(prompt, echo); + if (result || errno != ENOENT) + return result; + #endif input_fh = fopen(INPUT_PATH, "r" FORCE_TEXT); From e2986e3126d8c94802f8ef6131cfbb638e77a796 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 10 Jan 2017 21:42:08 +0100 Subject: [PATCH 35/36] Windows: force-recompile git.res for differing architectures When git.rc is compiled into git.res, the result is actually dependent on the architecture. That is, you cannot simply link a 32-bit git.res into a 64-bit git.exe. Therefore, to allow 32-bit and 64-bit builds in the same directory, we let git.res depend on GIT-PREFIX so that it gets recompiled when compiling for a different architecture (this works because the exec path changes based on the architecture: /mingw32/libexec/git-core for 32-bit and /mingw64/libexec/git-core for 64-bit). Signed-off-by: Johannes Schindelin --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 5a969f5830..e1eb474acc 100644 --- a/Makefile +++ b/Makefile @@ -2087,7 +2087,7 @@ $(SCRIPT_LIB) : % : %.sh GIT-SCRIPT-DEFINES $(QUIET_GEN)$(cmd_munge_script) && \ mv $@+ $@ -git.res: git.rc GIT-VERSION-FILE +git.res: git.rc GIT-VERSION-FILE GIT-PREFIX $(QUIET_RC)$(RC) \ $(join -DMAJOR= -DMINOR= -DMICRO= -DPATCHLEVEL=, $(wordlist 1, 4, \ $(shell echo $(GIT_VERSION) 0 0 0 0 | tr '.a-zA-Z-' ' '))) \ From 6f745cc4f114be026139efd0bd95a9303ea5373f Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 6 Jan 2017 17:39:50 +0100 Subject: [PATCH 36/36] mingw: load system libraries the recommended way When we access IPv6-related functions, we load the corresponding system library using the `LoadLibrary()` function, which is not the recommended way to load system libraries. In practice, it does not make a difference: the `ws2_32.dll` library containing the IPv6 functions is already loaded into memory, so LoadLibrary() simply reuses the already-loaded library. Still, recommended way is recommended way, so let's use that instead. While at it, also adjust the code in contrib/ that loads system libraries. Signed-off-by: Johannes Schindelin --- compat/mingw.c | 3 ++- contrib/credential/wincred/git-credential-wincred.c | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 5080963779..abd6604b85 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -1686,7 +1686,8 @@ static void ensure_socket_initialization(void) WSAGetLastError()); for (name = libraries; *name; name++) { - ipv6_dll = LoadLibrary(*name); + ipv6_dll = LoadLibraryExA(*name, NULL, + LOAD_LIBRARY_SEARCH_SYSTEM32); if (!ipv6_dll) continue; diff --git a/contrib/credential/wincred/git-credential-wincred.c b/contrib/credential/wincred/git-credential-wincred.c index 86518cd93d..5bdad41de1 100644 --- a/contrib/credential/wincred/git-credential-wincred.c +++ b/contrib/credential/wincred/git-credential-wincred.c @@ -75,7 +75,8 @@ static CredDeleteWT CredDeleteW; static void load_cred_funcs(void) { /* load DLLs */ - advapi = LoadLibrary("advapi32.dll"); + advapi = LoadLibraryExA("advapi32.dll", NULL, + LOAD_LIBRARY_SEARCH_SYSTEM32); if (!advapi) die("failed to load advapi32.dll");