From a44a8043de8f49f9850c2519fce536d87c259075 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 20 Oct 2016 06:31:47 -0700 Subject: [PATCH 01/26] msvc: fix dependencies of compat/msvc.c The file compat/msvc.c includes compat/mingw.c, which means that we have to recompile compat/msvc.o if compat/mingw.c changes. Signed-off-by: Johannes Schindelin --- config.mak.uname | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config.mak.uname b/config.mak.uname index 6244c6cc01..17619ef594 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -380,6 +380,8 @@ else BASIC_CFLAGS += -Zi -MDd endif X = .exe + +compat/msvc.o: compat/msvc.c compat/mingw.c GIT-CFLAGS endif ifeq ($(uname_S),Interix) NO_INITGROUPS = YesPlease From 0341e61ef07083519e2357854470699b7296f172 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 3 Jun 2016 08:38:17 -0400 Subject: [PATCH 02/26] msvc: ignore VS2015 trash files Signed-off-by: Jeff Hostetler --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index b1020b875f..a49a6da7f1 100644 --- a/.gitignore +++ b/.gitignore @@ -215,5 +215,7 @@ *.user *.idb *.pdb +*.ilk +.vs/ /Debug/ /Release/ From 458b3bc84156722632601e896967fad76da83fa5 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 3 Jun 2016 11:32:01 -0400 Subject: [PATCH 03/26] msvc: add NuGet scripts for building with VS2015 This commit contains a GNU Makefile and NuGet configuration scripts to download and install the various third-party libraries that we will need to build/link with when using VS2015 to build Git. The file "compat/vcbuild/README_VS2015.txt" contains instructions for using this. In this commit, "compat/vcbuild/Makefile" contains hard-coded version numbers of the packages we require. These are set to the current versions as of the time of this commit. We use "nuget restore" to install them explicitly using a "package.config". A future improvement would try to use some of the automatic package management functions and eliminate the need to specify exact versions. I tried, but could not get this to work. NuGet was happy dowload "minimum requirements" rather than "lastest" for dependencies -- and only look at one package at a time. For example, both curl and openssl depend upon zlib and have different minimums. It was unclear which version of zlib would be installed and seemed to be dependent on the order of the top-level packages. So, I'm skipping that for now. We need to be very precise when specifying NuGet package versions: while nuget.exe auto-completes a version, say, 1.0.2 to 1.0.2.0, we will want to parse packages.config ourselves, to generate the Visual Studio solution, and there we need the exact version number to be able to generate the exact path to the correct .targets file. Signed-off-by: Jeff Hostetler --- compat/vcbuild/.gitignore | 1 + compat/vcbuild/Makefile | 122 +++++++++++++++++++++++++++++++ compat/vcbuild/README_VS2015.txt | 58 +++++++++++++++ compat/vcbuild/nuget.config | 27 +++++++ compat/vcbuild/packages.config | 23 ++++++ 5 files changed, 231 insertions(+) create mode 100644 compat/vcbuild/.gitignore create mode 100644 compat/vcbuild/Makefile create mode 100644 compat/vcbuild/README_VS2015.txt create mode 100644 compat/vcbuild/nuget.config create mode 100644 compat/vcbuild/packages.config diff --git a/compat/vcbuild/.gitignore b/compat/vcbuild/.gitignore new file mode 100644 index 0000000000..e14691508e --- /dev/null +++ b/compat/vcbuild/.gitignore @@ -0,0 +1 @@ +GEN.* diff --git a/compat/vcbuild/Makefile b/compat/vcbuild/Makefile new file mode 100644 index 0000000000..b94bcd119a --- /dev/null +++ b/compat/vcbuild/Makefile @@ -0,0 +1,122 @@ +## Makefile to install nuget package dependencies. +################################################################## + +INST=GEN.DEPS +INST_INC=$(INST)/include +INST_LIB=$(INST)/lib +INST_BIN=$(INST)/bin + +PKGDIR=GEN.PKGS + +################################################################## +all: unpack expat libssh libssh_redist curl curl_redist openssl zlib + +unpack: + [ -d $(PKGDIR) ] || mkdir $(PKGDIR) + nuget.exe restore packages.config -ConfigFile nuget.config \ + -NonInteractive -OutputDirectory $(PKGDIR) + +insdir: + [ -d $(INST) ] || mkdir $(INST) + [ -d $(INST_INC) ] || mkdir $(INST_INC) + [ -d $(INST_BIN) ] || mkdir $(INST_BIN) + [ -d $(INST_LIB) ] || mkdir $(INST_LIB) + +################################################################## +## We place the expat headers in their own subdirectory. +## The custom is to reference , so compile with: +## -I$(INST_INC)/expat + +EXPAT_VER=2.1.0.11 +EXPAT_ROOT=$(PKGDIR)/expat.$(EXPAT_VER)/build/native +EXPAT_INC=$(EXPAT_ROOT)/include +EXPAT_LIB=$(EXPAT_ROOT)/lib/v110/x64/Release/dynamic/utf8 + +expat: insdir + [ -d $(INST_INC)/expat ] || mkdir $(INST_INC)/expat + cp -r $(EXPAT_INC)/* $(INST_INC)/expat/ + cp -r $(EXPAT_LIB)/* $(INST_LIB)/ + + +################################################################## + +LIBSSH_VER=1.4.3.3 +LIBSSH_ROOT=$(PKGDIR)/libssh2.$(LIBSSH_VER)/build/native +LIBSSH_INC=$(LIBSSH_ROOT)/include +LIBSSH_LIB=$(LIBSSH_ROOT)/lib/v110/x64/Release/dynamic/cdecl + +libssh: insdir + [ -d $(INST_INC)/libssh2 ] || mkdir $(INST_INC)/libssh2 + cp -r $(LIBSSH_INC)/* $(INST_INC)/libssh2 + cp -r $(LIBSSH_LIB)/* $(INST_LIB)/ + + +LIBSSH_REDIST_ROOT=$(PKGDIR)/libssh2.redist.$(LIBSSH_VER)/build/native +LIBSSH_REDIST_BIN=$(LIBSSH_REDIST_ROOT)/bin/v110/x64/Release/dynamic/cdecl + +libssh_redist: insdir + cp -r $(LIBSSH_REDIST_BIN)/* $(INST_BIN)/ + + +################################################################## +## We place the curl headers in their own subdirectory. +## The custom is to reference , so compile with: +## -I$(INST_INC) + +CURL_VER=7.30.0.2 +CURL_ROOT=$(PKGDIR)/curl.$(CURL_VER)/build/native +CURL_INC=$(CURL_ROOT)/include/curl +CURL_LIB=$(CURL_ROOT)/lib/v110/x64/Release/dynamic + +curl: insdir + [ -d $(INST_INC)/curl ] || mkdir $(INST_INC)/curl + cp -r $(CURL_INC)/* $(INST_INC)/curl + cp -r $(CURL_LIB)/* $(INST_LIB)/ + + +CURL_REDIST_ROOT=$(PKGDIR)/curl.redist.$(CURL_VER)/build/native +CURL_REDIST_BIN=$(CURL_REDIST_ROOT)/bin/v110/x64/Release/dynamic + +curl_redist: insdir + cp -r $(CURL_REDIST_BIN)/* $(INST_BIN)/ + + +################################################################## +## We place the openssl headers in their own subdirectory. +## The custom is to reference , so compile with: +## -I$(INST_INC) + +OPENSSL_VER=1.0.2.1 +OPENSSL_ROOT=$(PKGDIR)/openssl.v140.windesktop.msvcstl.dyn.rt-dyn.x64.$(OPENSSL_VER) +OPENSSL_INC=$(OPENSSL_ROOT)/build/native/include/openssl +OPENSSL_LIB=$(OPENSSL_ROOT)/lib/native/v140/windesktop/msvcstl/dyn/rt-dyn/x64/release + +openssl: insdir + [ -d $(INST_INC)/openssl ] || mkdir $(INST_INC)/openssl + cp -r $(OPENSSL_INC)/* $(INST_INC)/openssl + cp -r $(OPENSSL_LIB)/*.lib $(INST_LIB)/ + cp -r $(OPENSSL_LIB)/*.dll $(INST_BIN)/ + cp -r $(OPENSSL_LIB)/*.pdb $(INST_BIN)/ + +################################################################## +## We place the zlib headers in their own subdirectory. +## The custom is to reference , so compile with: +## -I$(INST_INC)/zlib + +ZLIB_VER=1.2.8.8 +ZLIB_ROOT=$(PKGDIR)/zlib.v140.windesktop.msvcstl.dyn.rt-dyn.$(ZLIB_VER) +ZLIB_INC=$(ZLIB_ROOT)/build/native/include +ZLIB_LIB=$(ZLIB_ROOT)/lib/native/v140/windesktop/msvcstl/dyn/rt-dyn/x64/Release + +zlib: insdir + [ -d $(INST_INC)/zlib ] || mkdir $(INST_INC)/zlib + cp -r $(ZLIB_INC)/* $(INST_INC)/zlib + cp -r $(ZLIB_LIB)/*.lib $(INST_LIB)/ + cp -r $(ZLIB_LIB)/*.dll $(INST_BIN)/ + +################################################################## +clean: + rm -rf $(INST) + +clobber: clean + rm -rf $(PKGDIR) diff --git a/compat/vcbuild/README_VS2015.txt b/compat/vcbuild/README_VS2015.txt new file mode 100644 index 0000000000..763523fbb4 --- /dev/null +++ b/compat/vcbuild/README_VS2015.txt @@ -0,0 +1,58 @@ +Instructions for building Git for Windows using VS2015. +================================================================ + +Installing third-party dependencies: +==================================== + +[1] Install nuget.exe somewhere on your system and add it to your PATH. + https://docs.nuget.org/consume/command-line-reference + https://dist.nuget.org/index.html + +[2] Download required nuget packages for third-party libraries. + Using a terminal window, type: + + make -C compat/vcbuild + + This will download the packages, unpack them into GEN.PKGS, + and populate the {include, lib, bin} directories in GEN.DEPS. + + +Building Git for Windows using VS2015: +====================================== + +[3] Build 64-bit version of Git for Windows. + Using a terminal window: + + make MSVC=1 DEBUG=1 + + +[4] Add compat/vcbuild/GEN.DEPS/bin to your PATH. + +[5] You should then be able to run the test suite and any interactive + commands. + +[6] To debug/profile in VS, open the git.exe in VS and run/debug + it. (Be sure to add GEN.DEPS/bin to the PATH in the debug + dialog.) + + +TODO List: +========== + +[A] config.mak.uname currently contains hard-coded paths + to the various MSVC and SDK libraries for the 64-bit + version of the compilers and libaries. + + See: SANE_TOOL_PATH, MSVC_DEPS, MSVC_SDK*, MSVC_VCDIR. + + Long term, we need to figure out how to properly import + values for %VCINSTALLDIR%, %LIB%, %LIBPATH%, and the + other values normally set by "vsvars32.bat" when a + developer command prompt is started. This would also + allow us to switch between 32- and 64-bit tool chains. + +[B] Currently, we leave the third-party DLLs we reference in + "compat/vcbuild/GEN.DEPS/bin". We need an installer + step to move them next to git.exe (or into libexec/git-core). + +[C] We need to build SLN or VCPROJ files. diff --git a/compat/vcbuild/nuget.config b/compat/vcbuild/nuget.config new file mode 100644 index 0000000000..b5e7349504 --- /dev/null +++ b/compat/vcbuild/nuget.config @@ -0,0 +1,27 @@ + + + + + + + + + + + + + + + + + + + + + + + diff --git a/compat/vcbuild/packages.config b/compat/vcbuild/packages.config new file mode 100644 index 0000000000..e6e6d2f63f --- /dev/null +++ b/compat/vcbuild/packages.config @@ -0,0 +1,23 @@ + + + + + + + + + + + + + + + + + + + + + + + From e1ce04f23ef2df4c3feff713168d1fd420e4347c Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 3 Jun 2016 13:34:28 -0400 Subject: [PATCH 04/26] msvc: update compile helper for VS2015 Support -Z flags ("specify PDB options"), only include -l args on link commands, and force PDBs to be created. Signed-off-by: Jeff Hostetler --- compat/vcbuild/scripts/clink.pl | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/compat/vcbuild/scripts/clink.pl b/compat/vcbuild/scripts/clink.pl index a87d0da512..3bbfa47bf4 100755 --- a/compat/vcbuild/scripts/clink.pl +++ b/compat/vcbuild/scripts/clink.pl @@ -12,10 +12,11 @@ use strict; my @args = (); my @cflags = (); +my @lflags = (); my $is_linking = 0; while (@ARGV) { my $arg = shift @ARGV; - if ("$arg" =~ /^-[DIMGO]/) { + if ("$arg" =~ /^-[DIMGOZ]/) { push(@cflags, $arg); } elsif ("$arg" eq "-o") { my $file_out = shift @ARGV; @@ -35,9 +36,11 @@ while (@ARGV) { push(@args, "ssleay32.lib"); } elsif ("$arg" eq "-lcurl") { push(@args, "libcurl.lib"); + } elsif ("$arg" eq "-lexpat") { + push(@args, "libexpat.lib"); } elsif ("$arg" =~ /^-L/ && "$arg" ne "-LTCG") { $arg =~ s/^-L/-LIBPATH:/; - push(@args, $arg); + push(@lflags, $arg); } elsif ("$arg" =~ /^-R/) { # eat } else { @@ -45,6 +48,9 @@ while (@ARGV) { } } if ($is_linking) { + push(@args, @lflags); + # force PDB to be created. + push(@args, "-debug"); unshift(@args, "link.exe"); } else { unshift(@args, "cl.exe"); From 49b2f7dc5193a742e7e2068f581f8516d3771240 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 3 Jun 2016 14:15:02 -0400 Subject: [PATCH 05/26] msvc: update Makefile and compiler settings for VS2015 Signed-off-by: Jeff Hostetler --- Makefile | 2 +- config.mak.uname | 47 ++++++++++++++++++++++++++++++++++++++++++----- git-compat-util.h | 9 +++++++++ 3 files changed, 52 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 88181bd595..aa63363ae4 100644 --- a/Makefile +++ b/Makefile @@ -1012,7 +1012,7 @@ endif ifdef SANE_TOOL_PATH SANE_TOOL_PATH_SQ = $(subst ','\'',$(SANE_TOOL_PATH)) -BROKEN_PATH_FIX = 's|^\# @@BROKEN_PATH_FIX@@$$|git_broken_path_fix $(SANE_TOOL_PATH_SQ)|' +BROKEN_PATH_FIX = 's|^\# @@BROKEN_PATH_FIX@@$$|git_broken_path_fix "$(SANE_TOOL_PATH_SQ)"|' PATH := $(SANE_TOOL_PATH):${PATH} else BROKEN_PATH_FIX = '/^\# @@BROKEN_PATH_FIX@@$$/d' diff --git a/config.mak.uname b/config.mak.uname index 17619ef594..e780fecff9 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -324,6 +324,13 @@ endif ifeq ($(uname_S),Windows) GIT_VERSION := $(GIT_VERSION).MSVC pathsep = ; + # Prepend MSVC 64-bit tool-chain to PATH. + # + # A regular Git Bash *does not* have cl.exe in its $PATH. As there is a + # link.exe next to, and required by, cl.exe, we have to prepend this + # onto the existing $PATH. + # + SANE_TOOL_PATH ?= /c/Program Files (x86)/Microsoft Visual Studio 14.0/VC/bin/amd64 HAVE_ALLOCA_H = YesPlease NO_PREAD = YesPlease NEEDS_CRYPTO_WITH_SSL = YesPlease @@ -340,7 +347,13 @@ ifeq ($(uname_S),Windows) NO_ICONV = YesPlease NO_STRTOUMAX = YesPlease NO_MKDTEMP = YesPlease - SNPRINTF_RETURNS_BOGUS = YesPlease + # VS2015 with UCRT claims that snprintf and friends are C99 compliant, + # so we don't need this. + # + # TODO If we want to support older compilers, we need to make this + # TODO conditional on the compiler version. + # + # SNPRINTF_RETURNS_BOGUS = YesPlease NO_SVN_TESTS = YesPlease RUNTIME_PREFIX = YesPlease NO_ST_BLOCKS_IN_STRUCT_STAT = YesPlease @@ -362,22 +375,46 @@ ifeq ($(uname_S),Windows) CC = compat/vcbuild/scripts/clink.pl AR = compat/vcbuild/scripts/lib.pl CFLAGS = - BASIC_CFLAGS = -nologo -I. -I../zlib -Icompat/vcbuild -Icompat/vcbuild/include -DWIN32 -D_CONSOLE -DHAVE_STRING_H -D_CRT_SECURE_NO_WARNINGS -D_CRT_NONSTDC_NO_DEPRECATE + BASIC_CFLAGS = -nologo -I. -Icompat/vcbuild/include -DWIN32 -D_CONSOLE -DHAVE_STRING_H -D_CRT_SECURE_NO_WARNINGS -D_CRT_NONSTDC_NO_DEPRECATE COMPAT_OBJS = compat/msvc.o compat/winansi.o \ compat/win32/pthread.o compat/win32/syslog.o \ compat/win32/dirent.o compat/win32/fscache.o - COMPAT_CFLAGS = -D__USE_MINGW_ACCESS -DNOGDI -DHAVE_STRING_H -Icompat -Icompat/regex -Icompat/win32 -DSTRIP_EXTENSION=\".exe\" + COMPAT_CFLAGS = -D__USE_MINGW_ACCESS -DDETECT_MSYS_TTY -DNOGDI -DHAVE_STRING_H -Icompat -Icompat/regex -Icompat/win32 -DSTRIP_EXTENSION=\".exe\" BASIC_LDFLAGS = -IGNORE:4217 -IGNORE:4049 -NOLOGO -SUBSYSTEM:CONSOLE - EXTLIBS = user32.lib advapi32.lib shell32.lib wininet.lib ws2_32.lib invalidcontinue.obj + EXTLIBS = user32.lib advapi32.lib shell32.lib wininet.lib ws2_32.lib invalidcontinue.obj kernel32.lib ntdll.lib PTHREAD_LIBS = lib = + # Path to the unpacked third-party libraries + MSVC_DEPS = compat/vcbuild/GEN.DEPS + # Compensate for lack of %VCINSTALLDIR%, %LIB%, %LIBPATH%, and etc. + # since vcvars*.bat did not get a chance to setup the environment of + # the user's shell window. + # + # TODO If we ask the user to launch a "x64 Native" command prompt + # TODO and then have it start a git-bash window, these could be + # TODO inherited. So we wouldn't need to add these lines here. + # + MSVC_SDK81 = "c:/Program Files (x86)/Windows Kits/8.1" + MSVC_SDK10 = "c:/Program Files (x86)/Windows Kits/10" + MSVC_VCDIR = "c:/Program Files (x86)/Microsoft Visual Studio 14.0/VC" + BASIC_CFLAGS += \ + -I$(MSVC_DEPS)/include -I$(MSVC_DEPS)/include/expat -I$(MSVC_DEPS)/include/zlib \ + -L$(MSVC_DEPS)/lib \ + -I$(MSVC_SDK81)/Include/um -I$(MSVC_SDK81)/Include/shared \ + -L$(MSVC_SDK81)/lib/winv6.3/um/x64 \ + -I$(MSVC_SDK10)/Include/10.0.10240.0/ucrt \ + -L$(MSVC_SDK10)/lib/10.0.10240.0/ucrt/x64 \ + -I$(MSVC_VCDIR)/INCLUDE \ + -L$(MSVC_VCDIR)/lib/amd64 + # Optionally enable memory leak reporting. + # BASIC_CLFAGS += -DUSE_MSVC_CRTDBG BASIC_CFLAGS += -DPROTECT_NTFS_DEFAULT=1 ifndef DEBUG BASIC_CFLAGS += -GL -Os -MD BASIC_LDFLAGS += -LTCG AR += -LTCG else - BASIC_CFLAGS += -Zi -MDd + BASIC_CFLAGS += -Zi -MDd -DDEBUG -D_DEBUG endif X = .exe diff --git a/git-compat-util.h b/git-compat-util.h index e2ad123562..b63b298b3b 100644 --- a/git-compat-util.h +++ b/git-compat-util.h @@ -1,6 +1,15 @@ #ifndef GIT_COMPAT_UTIL_H #define GIT_COMPAT_UTIL_H +#ifdef USE_MSVC_CRTDBG +/* + * For these to work they must appear very early in each + * file -- before most of the standard header files. + */ +#include +#include +#endif + #define _FILE_OFFSET_BITS 64 From c8d295250896bcc615d18d61e7122d58f4fd48ec Mon Sep 17 00:00:00 2001 From: Philip Oakley Date: Thu, 21 Apr 2016 14:07:20 +0100 Subject: [PATCH 06/26] msvc: include sigset_t definition On MSVC (VS2008) sigset_t is not defined. Signed-off-by: Philip Oakley --- compat/msvc.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compat/msvc.h b/compat/msvc.h index 580bb55bf4..ec4518eebf 100644 --- a/compat/msvc.h +++ b/compat/msvc.h @@ -24,6 +24,8 @@ static __inline int strcasecmp (const char *s1, const char *s2) #undef ERROR +typedef int sigset_t; + #include "compat/mingw.h" #endif From 478fe0de737140d4509f2ee1736451e5bd3faba2 Mon Sep 17 00:00:00 2001 From: Philip Oakley Date: Thu, 21 Apr 2016 14:52:05 +0100 Subject: [PATCH 07/26] msvc: define O_ACCMODE This constant is not defined in MSVC's headers. In UCRT's fcntl.h, _O_RDONLY, _O_WRONLY and _O_RDWR are defined as 0, 1 and 2, respectively. Yes, that means that UCRT breaks with the tradition that O_RDWR == O_RDONLY | O_WRONLY. It is a perfectly legal way to define those constants, though, therefore we need to take care of defining O_ACCMODE accordingly. This is particularly important in order to keep our "open() can set errno to EISDIR" emulation working: it tests that (flags & O_ACCMODE) is not identical to O_RDONLY before going on to test specifically whether the file for which open() reported EACCES is, in fact, a directory. Signed-off-by: Philip Oakley Signed-off-by: Johannes Schindelin --- compat/msvc.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compat/msvc.h b/compat/msvc.h index ec4518eebf..ae1c58ec13 100644 --- a/compat/msvc.h +++ b/compat/msvc.h @@ -25,6 +25,8 @@ static __inline int strcasecmp (const char *s1, const char *s2) #undef ERROR typedef int sigset_t; +/* open for reading, writing, or both (not in fcntl.h) */ +#define O_ACCMODE (_O_RDONLY | _O_WRONLY | _O_RDWR) #include "compat/mingw.h" From c2248374551010fab9be2c4f65db3c8068e1bb81 Mon Sep 17 00:00:00 2001 From: Philip Oakley Date: Fri, 22 Apr 2016 10:48:13 +0100 Subject: [PATCH 08/26] msvc: fix the declaration of the _REPARSE_DATA_BUFFER structure GCC and MSVC disagree about using the GCC extension _ANONYMOUS_UNION. Simply skip that offending keyword when compiling with MSVC. Signed-off-by: Philip Oakley Signed-off-by: Johannes Schindelin --- compat/mingw.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/compat/mingw.c b/compat/mingw.c index 7b7b0d44a7..ced2d4855f 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2316,7 +2316,10 @@ typedef struct _REPARSE_DATA_BUFFER { DWORD ReparseTag; WORD ReparseDataLength; WORD Reserved; - _ANONYMOUS_UNION union { +#ifndef _MSC_VER + _ANONYMOUS_UNION +#endif + union { struct { WORD SubstituteNameOffset; WORD SubstituteNameLength; From 1c86acc559a92d8694efcb78f1880aa378232b67 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 3 Jun 2016 15:38:25 -0400 Subject: [PATCH 09/26] msvc: convert environment from/to UTF-16 on the fly This adds MSVC versions of getenv() and friends. These take UTF-8 arguments and return UTF-8 values, but use the UNICODE versions of the CRT routines. This avoids the need to write to __environ (which is only visible if you statically link to the CRT). This also avoids the CP_ACP conversions performed inside the CRT. It also avoids various memory leaks and problems. Signed-off-by: Jeff Hostetler --- compat/mingw.c | 238 +++++++++++++++++++++++++++++++++++++++++++++++++ compat/mingw.h | 41 +++++++++ 2 files changed, 279 insertions(+) diff --git a/compat/mingw.c b/compat/mingw.c index ced2d4855f..5a8cf8641c 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -1198,6 +1198,177 @@ static char *path_lookup(const char *cmd, char **path, int exe_only) return prog; } +#if defined(_MSC_VER) +/* + * Build an environment block combining the inherited environment + * merged with the given list of settings. + * + * Values of the form "KEY=VALUE" in deltaenv override inherited values. + * Values of the form "KEY" in deltaenv delete inherited values. + * + * We return a contiguous block of UNICODE strings with a final trailing + * zero word. + */ +static wchar_t *make_environment_block(char **deltaenv) +{ + /* + * The CRT (at least as of UCRT) secretly declares "_wenviron" + * as a function that returns a pointer to a mostly static table. + * Grab the pointer and cache it for the duration of our loop. + */ + extern wchar_t **_wenviron; + const wchar_t **my_wenviron = _wenviron; + + /* + * Internally, we create normal 'C' arrays of strings (pointing + * into the blocks) to help with some of the de-dup work. + */ + wchar_t **wptrs_ins = NULL; + wchar_t **wptrs_del = NULL; + wchar_t *wblock_ins = NULL; + wchar_t *wblock_del = NULL; + wchar_t *wend_ins; + wchar_t *wend_del; + wchar_t *w_ins; + wchar_t *w_del; + + int maxlen = 0; + int nr_delta = 0; + int nr_delta_ins = 0; + int nr_delta_del = 0; + int nr_wenv = 0; + int j, k, k_ins, k_del; + + /* + * Count the number of inserts and deletes in the deltaenv list. + * Allocate 'C' arrays for inserts and deletes. + * Also estimate the block size of our results. + */ + if (deltaenv && deltaenv[0] && *deltaenv[0]) { + for (k = 0; deltaenv && deltaenv[k] && *deltaenv[k]; k++) { + if (strchr(deltaenv[k], '=') == NULL) + nr_delta_del++; + else { + maxlen += strlen(deltaenv[k]) + 1; + nr_delta_ins++; + } + } + + if (nr_delta_ins) + wptrs_ins = (wchar_t**)calloc(nr_delta_ins + 1, sizeof(wchar_t*)); + if (nr_delta_del) + wptrs_del = (wchar_t**)calloc(nr_delta_del + 1, sizeof(wchar_t*)); + + nr_delta = nr_delta_ins + nr_delta_del; + } + while (my_wenviron && my_wenviron[nr_wenv] && *my_wenviron[nr_wenv]) + maxlen += wcslen(my_wenviron[nr_wenv++]) + 1; + maxlen++; + + /* + * Allocate blocks for inserted and deleted items. + * The individual pointers in the 'C' arrays will point into here. + * We will use the wblock_ins as the final result. + */ + if (nr_delta_del) { + wblock_del = (wchar_t*)calloc(maxlen, sizeof(wchar_t)); + wend_del = wblock_del + maxlen; + w_del = wblock_del; + } + wblock_ins = (wchar_t*)calloc(maxlen, sizeof(wchar_t)); + wend_ins = wblock_ins + maxlen; + w_ins = wblock_ins; + + /* + * deltaenv values override inherited environment, so put them + * in the result list first (so that we can de-dup using the + * wide versions of them. + * + * Items in the deltaenv list that DO NOT contain an "=" are + * treated as unsetenv. + * + * Care needs to be taken to handle entries that are added first, and + * then deleted. + */ + k_ins = 0; + k_del = 0; + for (k = 0; k < nr_delta; k++) { + if (strchr(deltaenv[k], '=') == NULL) { + wchar_t *save = w_del; + wptrs_del[k_del++] = w_del; + w_del += xutftowcs(w_del, deltaenv[k], (wend_del - w_del)); + *w_del++ = L'='; /* append '=' to make lookup easier in next step. */ + *w_del++ = 0; + + /* If we added this key, we have to remove it again */ + for (j = 0; j < k_ins; j++) + if (!wcsnicmp(wptrs_ins[j], save, w_del - save - 1)) { + if (j + 1 < k_ins) { + int delta = sizeof(wchar_t) * (wptrs_ins[j + 1] - wptrs_ins[j]), i; + memmove(wptrs_ins[j], wptrs_ins[j + 1], sizeof(wchar_t) * (w_ins - wptrs_ins[j + 1])); + for (i = j; i < --k_ins; i++) + wptrs_ins[i] = wptrs_ins[i + 1] - delta; + w_ins -= delta; + } else + w_ins = wptrs_ins[j]; + k_ins--; + j--; + } + } else { + wptrs_ins[k_ins++] = w_ins; + w_ins += xutftowcs(w_ins, deltaenv[k], (wend_ins - w_ins)) + 1; + } + } + assert(k_ins <= nr_delta_ins); + assert(k_del == nr_delta_del); + + /* + * Walk the inherited environment and copy over unique, non-deleted + * ones into the result set. Note that we only have to de-dup WRT + * the values from deltaenv, because the inherited set should be unique. + */ + for (j = 0; j < nr_wenv; j++) { + const wchar_t *v_j = my_wenviron[j]; + wchar_t *v_j_eq = wcschr(v_j, L'='); + int len_j_eq, len_j; + + if (!v_j_eq) + continue; /* should not happen */ + len_j_eq = v_j_eq + 1 - v_j; /* length(v_j) including '=' */ + + /* lookup v_j in list of to-delete vars */ + for (k_del = 0; k_del < nr_delta_del; k_del++) { + if (wcsnicmp(v_j, wptrs_del[k_del], len_j_eq) == 0) + goto skip_it; + } + + /* lookup v_j in deltaenv portion of result set */ + for (k_ins = 0; k_ins < nr_delta_ins; k_ins++) { + if (wcsnicmp(v_j, wptrs_ins[k_ins], len_j_eq) == 0) + goto skip_it; + } + + /* item is unique, add it to results. */ + len_j = wcslen(v_j); + memcpy(w_ins, v_j, len_j * sizeof(wchar_t)); + w_ins += len_j + 1; + +skip_it: + ; + } + + if (wptrs_ins) + free(wptrs_ins); + if (wptrs_del) + free(wptrs_del); + if (wblock_del) + free(wblock_del); + + return wblock_ins; +} + +#else + static int do_putenv(char **env, const char *name, int size, int free_old); /* used number of elements of environ array, including terminating NULL */ @@ -1244,6 +1415,7 @@ static wchar_t *make_environment_block(char **deltaenv) free(tmpenv); return wenvblk; } +#endif static void do_unset_environment_variables(void) { @@ -1531,6 +1703,70 @@ int mingw_kill(pid_t pid, int sig) return -1; } +#if defined(_MSC_VER) + +/* UTF8 versions of getenv and putenv (and unsetenv). + * Internally, they use the CRT's stock UNICODE routines + * to avoid data loss. + * + * Unlike the mingw version, we DO NOT directly write to + * the CRT variables. We also DO NOT try to manage/replace + * the CRT storage. + */ +char *msc_getenv(const char *name) +{ + int len_key, len_value; + wchar_t *w_key; + char *value; + const wchar_t *w_value; + + if (!name || !*name) + return NULL; + + len_key = strlen(name) + 1; + w_key = calloc(len_key, sizeof(wchar_t)); + xutftowcs(w_key, name, len_key); + w_value = _wgetenv(w_key); + free(w_key); + + if (!w_value) + return NULL; + + len_value = wcslen(w_value) * 3 + 1; + value = calloc(len_value, sizeof(char)); + xwcstoutf(value, w_value, len_value); + + /* TODO Warning: We return "value" which is an allocated + * value and the caller is NOT expecting to have to free + * it, so we leak memory. + */ + return value; +} + +int msc_putenv(const char *name) +{ + int len, result; + char *equal; + wchar_t *wide; + + if (!name || !*name) + return 0; + + len = strlen(name); + equal = strchr(name, '='); + wide = calloc(len+1+!equal, sizeof(wchar_t)); + xutftowcs(wide, name, len+1); + if (!equal) + wcscat(wide, L"="); + + result = _wputenv(wide); + + free(wide); + return result; +} + +#else + /* * Compare environment entries by key (i.e. stopping at '=' or '\0'). */ @@ -1659,6 +1895,8 @@ int mingw_putenv(const char *namevalue) return 0; } +#endif + /* * Note, this isn't a complete replacement for getaddrinfo. It assumes * that service contains a numerical port, or that it is null. It diff --git a/compat/mingw.h b/compat/mingw.h index 8ba86497ab..0c3d4600c0 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -265,12 +265,53 @@ char *mingw_getcwd(char *pointer, int len); #error "NO_UNSETENV is incompatible with the MinGW startup code!" #endif +#if defined(_MSC_VER) +/* + * We bind *env() routines (even the mingw_ ones) to private msc_ versions. + * These talk to the CRT using UNICODE/wchar_t, but maintain the original + * narrow-char API. + * + * Note that the MSCRT maintains both ANSI (getenv()) and UNICODE (_wgetenv()) + * routines and stores both versions of each environment variable in parallel + * (and secretly updates both when you set one or the other), but it uses CP_ACP + * to do the conversion rather than CP_UTF8. + * + * Since everything in the git code base is UTF8, we define the msc_ routines + * to access the CRT using the UNICODE routines and manually convert them to + * UTF8. This also avoids round-trip problems. + * + * This also helps with our linkage, since "_wenviron" is publicly exported + * from the CRT. But to access "_environ" we would have to statically link + * to the CRT (/MT). + * + * We also use "wmain(argc,argv,env)" and get the initial UNICODE setup for us. + * This avoids the need for the msc_startup() to import and convert the + * inherited environment. + * + * We require NO_SETENV (and let gitsetenv() call our msc_putenv). + */ +#define getenv msc_getenv +#define putenv msc_putenv +#define unsetenv msc_putenv +#define mingw_getenv msc_getenv +#define mingw_putenv msc_putenv +char *msc_getenv(const char *name); +int msc_putenv(const char *name); + +#ifndef NO_SETENV +#error "NO_SETENV is required for MSC startup code!" +#endif + +#else + char *mingw_getenv(const char *name); #define getenv mingw_getenv int mingw_putenv(const char *namevalue); #define putenv mingw_putenv #define unsetenv mingw_putenv +#endif + int mingw_gethostname(char *host, int namelen); #define gethostname mingw_gethostname From 7f0390abcb558c48592d522524dca189edc23953 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 3 Jun 2016 15:38:25 -0400 Subject: [PATCH 10/26] msvc: mark a variable as non-const VS2015 complains when using a const pointer in memcpy()/free(). Signed-off-by: Jeff Hostetler --- compat/mingw.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/compat/mingw.c b/compat/mingw.c index 5a8cf8641c..21bf05df99 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -1627,7 +1627,10 @@ static int try_shell_exec(const char *cmd, char *const *argv) prog = path_lookup(interpr, path, 1); if (prog) { int argc = 0; - const char **argv2; +#ifndef _MSC_VER + const +#endif + char **argv2; while (argv[argc]) argc++; ALLOC_ARRAY(argv2, argc + 1); argv2[0] = (char *)cmd; /* full path to the script file */ From 9c449c87e15ffb40025734c67a4fdcd0f81a6181 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 3 Jun 2016 15:38:25 -0400 Subject: [PATCH 11/26] msvc: do not pretend to support all signals This special-cases various signals that are not supported on Windows, such as SIGPIPE. These cause the UCRT to throw asserts (at least in debug mode). Signed-off-by: Jeff Hostetler --- compat/mingw.c | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/compat/mingw.c b/compat/mingw.c index 21bf05df99..a11d95d52a 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2458,8 +2458,34 @@ int mingw_raise(int sig) sigint_fn(SIGINT); return 0; +#if defined(_MSC_VER) + /* + * in the CRT defines 8 signals as being + * supported on the platform. Anything else causes + * an "Invalid signal or error" (which in DEBUG builds + * causes the Abort/Retry/Ignore dialog). We by-pass + * the CRT for things we already know will fail. + */ + /*case SIGINT:*/ + case SIGILL: + case SIGFPE: + case SIGSEGV: + case SIGTERM: + case SIGBREAK: + case SIGABRT: + case SIGABRT_COMPAT: + return raise(sig); + default: + errno = EINVAL; + return -1; + +#else + default: return raise(sig); + +#endif + } } From 819c9d9a9fbc0231a6479c39d23abc758f7b713c Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 10 Jan 2017 23:22:48 +0100 Subject: [PATCH 12/26] msvc: provide a main() wrapper similar to mingw_main() The MINGW version of the main() wrapper gets away with declaring symbols that were intentionally not exported. However, some of these symbols do not actually exist in MSVC's UCRT. So let's add an MSVC version of the main() wrapper that uses wmain() and imports the UNICODE argv and environment. While at it, we pass our UTF-8 version of ARGV to the real main -- rather than overwriting __argv as is done in the MINGW Version. Signed-off-by: Jeff Hostetler --- compat/mingw.c | 93 ++++++++++++++++++++++++++++++++++++++++++++++++++ compat/mingw.h | 21 ++++++++++++ 2 files changed, 114 insertions(+) diff --git a/compat/mingw.c b/compat/mingw.c index a11d95d52a..b16f5fe09b 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2964,6 +2964,7 @@ int handle_long_path(wchar_t *path, int len, int max_path, int expand) } } +#if !defined(_MSC_VER) /* * Disable MSVCRT command line wildcard expansion (__getmainargs called from * mingw startup code, see init.c in mingw runtime). @@ -2976,6 +2977,7 @@ typedef struct { extern int __wgetmainargs(int *argc, wchar_t ***argv, wchar_t ***env, int glob, _startupinfo *si); +#endif static NORETURN void die_startup(void) { @@ -3053,6 +3055,95 @@ static void maybe_redirect_std_handles(void) GENERIC_WRITE, FILE_FLAG_NO_BUFFERING); } +#if defined(_MSC_VER) + +/* + * This routine sits between wmain() and "main" in git.exe. + * We receive UNICODE (wchar_t) values for argv and env. + * + * To be more compatible with the core git code, we convert + * argv into UTF8 and pass them directly to the "main" routine. + * + * We don't bother converting the given UNICODE env vector, + * but rather leave them in the CRT. We replaced the various + * getenv/putenv routines to pull them directly from the CRT. + * + * This is unlike the MINGW version: + * [] It does the UNICODE-2-UTF8 conversion on both sets and + * stuffs the values back into the CRT using exported symbols. + * [] It also maintains a private copy of the environment and + * tries to track external changes to it. + */ +int msc_startup(int argc, wchar_t **w_argv, wchar_t **w_env) +{ + char **my_utf8_argv = NULL, **save = NULL; + char *buffer = NULL; + int maxlen; + int k, exit_status; + +#ifdef USE_MSVC_CRTDBG + _CrtSetDbgFlag(_CRTDBG_ALLOC_MEM_DF | _CRTDBG_LEAK_CHECK_DF); +#endif + + maybe_redirect_std_handles(); + + /* determine size of argv conversion buffer */ + maxlen = wcslen(_wpgmptr); + for (k = 1; k < argc; k++) + maxlen = max(maxlen, wcslen(w_argv[k])); + + /* allocate buffer (wchar_t encodes to max 3 UTF-8 bytes) */ + maxlen = 3 * maxlen + 1; + buffer = malloc_startup(maxlen); + + /* + * Create a UTF-8 version of w_argv. Also create a "save" copy + * to remember all the string pointers because parse_options() + * will remove claimed items from the argv that we pass down. + */ + ALLOC_ARRAY(my_utf8_argv, argc + 1); + ALLOC_ARRAY(save, argc + 1); + save[0] = my_utf8_argv[0] = wcstoutfdup_startup(buffer, _wpgmptr, maxlen); + for (k = 1; k < argc; k++) + save[k] = my_utf8_argv[k] = wcstoutfdup_startup(buffer, w_argv[k], maxlen); + save[k] = my_utf8_argv[k] = NULL; + + free(buffer); + + /* fix Windows specific environment settings */ + setup_windows_environment(); + + unset_environment_variables = xstrdup("PERL5LIB"); + + /* initialize critical section for waitpid pinfo_t list */ + InitializeCriticalSection(&pinfo_cs); + InitializeCriticalSection(&phantom_symlinks_cs); + + /* set up default file mode and file modes for stdin/out/err */ + _fmode = _O_BINARY; + _setmode(_fileno(stdin), _O_BINARY); + _setmode(_fileno(stdout), _O_BINARY); + _setmode(_fileno(stderr), _O_BINARY); + + /* initialize Unicode console */ + winansi_init(); + + /* init length of current directory for handle_long_path */ + current_directory_len = GetCurrentDirectoryW(0, NULL); + + /* invoke the real main() using our utf8 version of argv. */ + exit_status = msc_main(argc, my_utf8_argv); + + for (k = 0; k < argc; k++) + free(save[k]); + free(save); + free(my_utf8_argv); + + return exit_status; +} + +#else + void mingw_startup(void) { int i, maxlen, argc; @@ -3132,6 +3223,8 @@ void mingw_startup(void) current_directory_len = GetCurrentDirectoryW(0, NULL); } +#endif + int uname(struct utsname *buf) { unsigned v = (unsigned)GetVersion(); diff --git a/compat/mingw.h b/compat/mingw.h index 0c3d4600c0..d669277b68 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -686,7 +686,26 @@ extern CRITICAL_SECTION pinfo_cs; /* * A replacement of main() that adds win32 specific initialization. + * + * Note that the end of these macros are unterminated so that the + * brace group following the use of the macro is the body of the + * function. */ +#if defined(_MSC_VER) + +int msc_startup(int argc, wchar_t **w_argv, wchar_t **w_env); +extern int msc_main(int argc, const char **argv); + +#define main(c,v) dummy_decl_msc_main(void); \ +int wmain(int my_argc, \ + wchar_t **my_w_argv, \ + wchar_t **my_w_env) \ +{ \ + return msc_startup(my_argc, my_w_argv, my_w_env); \ +} \ +int msc_main(c, v) + +#else void mingw_startup(void); #define main(c,v) dummy_decl_mingw_main(void); \ @@ -698,6 +717,8 @@ int main(int argc, const char **argv) \ } \ static int mingw_main(c,v) +#endif + /* * Used by Pthread API implementation for Windows */ From f00a31dd279801806bc8ff3aaa179c18d021a3a6 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 3 Jun 2016 15:38:25 -0400 Subject: [PATCH 13/26] msvc: do not re-declare the timespec struct VS2015's headers already declare that struct. Signed-off-by: Jeff Hostetler --- compat/mingw.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compat/mingw.h b/compat/mingw.h index d669277b68..78ba2c9b56 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -394,11 +394,13 @@ static inline long long filetime_to_hnsec(const FILETIME *ft) #ifndef __MINGW64_VERSION_MAJOR #define off_t off64_t #define lseek _lseeki64 +#ifndef _MSC_VER struct timespec { time_t tv_sec; long tv_nsec; }; #endif +#endif static inline void filetime_to_timespec(const FILETIME *ft, struct timespec *ts) { From 8826c4bad92a608d808bedf9483f95c042adaa58 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 3 Jun 2016 15:38:25 -0400 Subject: [PATCH 14/26] msvc: define ftello() It is just called different in MSVC's headers. Signed-off-by: Jeff Hostetler --- compat/msvc.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compat/msvc.h b/compat/msvc.h index ae1c58ec13..b3ac4cc3af 100644 --- a/compat/msvc.h +++ b/compat/msvc.h @@ -24,6 +24,8 @@ static __inline int strcasecmp (const char *s1, const char *s2) #undef ERROR +#define ftello _ftelli64 + typedef int sigset_t; /* open for reading, writing, or both (not in fcntl.h) */ #define O_ACCMODE (_O_RDONLY | _O_WRONLY | _O_RDWR) From 637db05d4647225336d704f45dc58908cc9fe1be Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 10 Jan 2017 22:53:36 +0100 Subject: [PATCH 15/26] msvc: fix detect_msys_tty() The ntstatus.h header is only available in MINGW. Signed-off-by: Jeff Hostetler --- compat/winansi.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/compat/winansi.c b/compat/winansi.c index 8d5e82d386..986bfb30cf 100644 --- a/compat/winansi.c +++ b/compat/winansi.c @@ -515,7 +515,20 @@ static HANDLE swap_osfhnd(int fd, HANDLE new_handle) #ifdef DETECT_MSYS_TTY #include + +#if defined(_MSC_VER) + +typedef struct _OBJECT_NAME_INFORMATION +{ + UNICODE_STRING Name; + WCHAR NameBuffer[0]; +} OBJECT_NAME_INFORMATION, *POBJECT_NAME_INFORMATION; + +#define ObjectNameInformation 1 + +#else #include +#endif static void detect_msys_tty(int fd) { From 6fac7b84138a58e2c286a701af7887900e83736e Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 3 Jun 2016 16:01:35 -0400 Subject: [PATCH 16/26] cache-tree.c: avoid reusing the DEBUG constant In MSVC, the DEBUG constant is set automatically whenever compiling with debug information. This is clearly not what was intended in cache-tree.c, so let's use a less ambiguous constant there. Signed-off-by: Jeff Hostetler --- cache-tree.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/cache-tree.c b/cache-tree.c index 345ea35963..3e9da3f9bf 100644 --- a/cache-tree.c +++ b/cache-tree.c @@ -4,8 +4,8 @@ #include "tree-walk.h" #include "cache-tree.h" -#ifndef DEBUG -#define DEBUG 0 +#ifndef DEBUG_CACHE_TREE +#define DEBUG_CACHE_TREE 0 #endif struct cache_tree *cache_tree(void) @@ -110,7 +110,7 @@ static int do_invalidate_path(struct cache_tree *it, const char *path) int namelen; struct cache_tree_sub *down; -#if DEBUG +#if DEBUG_CACHE_TREE fprintf(stderr, "cache-tree invalidate <%s>\n", path); #endif @@ -392,7 +392,7 @@ static int update_one(struct cache_tree *it, strbuf_addf(&buffer, "%o %.*s%c", mode, entlen, path + baselen, '\0'); strbuf_add(&buffer, sha1, 20); -#if DEBUG +#if DEBUG_CACHE_TREE fprintf(stderr, "cache-tree update-one %o %.*s\n", mode, entlen, path + baselen); #endif @@ -414,7 +414,7 @@ static int update_one(struct cache_tree *it, strbuf_release(&buffer); it->entry_count = to_invalidate ? -1 : i - *skip_count; -#if DEBUG +#if DEBUG_CACHE_TREE fprintf(stderr, "cache-tree update-one (%d ent, %d subtree) %s\n", it->entry_count, it->subtree_nr, sha1_to_hex(it->sha1)); @@ -453,7 +453,7 @@ static void write_one(struct strbuf *buffer, struct cache_tree *it, strbuf_add(buffer, path, pathlen); strbuf_addf(buffer, "%c%d %d\n", 0, it->entry_count, it->subtree_nr); -#if DEBUG +#if DEBUG_CACHE_TREE if (0 <= it->entry_count) fprintf(stderr, "cache-tree <%.*s> (%d ent, %d subtree) %s\n", pathlen, path, it->entry_count, it->subtree_nr, @@ -526,7 +526,7 @@ static struct cache_tree *read_one(const char **buffer, unsigned long *size_p) size -= 20; } -#if DEBUG +#if DEBUG_CACHE_TREE if (0 <= it->entry_count) fprintf(stderr, "cache-tree <%s> (%d ent, %d subtree) %s\n", *buffer, it->entry_count, subtree_nr, From 866abd9e314952065f744eb6974497276b7b36aa Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Thu, 9 Jun 2016 09:40:13 -0400 Subject: [PATCH 17/26] msvc: release mode PDBs and library DLLs Install required third-party DLLs next to EXEs. Build and install release mode PDBs for git executables allowing detailed stack traces in the event of crash. Signed-off-by: Jeff Hostetler --- Makefile | 22 ++++++++++++++++++++++ compat/vcbuild/Makefile | 3 ++- compat/vcbuild/scripts/clink.pl | 7 ++++--- config.mak.uname | 10 +++++++--- 4 files changed, 35 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index aa63363ae4..6fc8eee606 100644 --- a/Makefile +++ b/Makefile @@ -2381,6 +2381,28 @@ install: all $(INSTALL) $(ALL_PROGRAMS) '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' $(INSTALL) -m 644 $(SCRIPT_LIB) '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' $(INSTALL) $(install_bindir_programs) '$(DESTDIR_SQ)$(bindir_SQ)' +ifdef MSVC + $(INSTALL) compat/vcbuild/GEN.DEPS/bin/*.dll '$(DESTDIR_SQ)$(bindir_SQ)' + $(INSTALL) compat/vcbuild/GEN.DEPS/bin/*.pdb '$(DESTDIR_SQ)$(bindir_SQ)' + # We DO NOT install the individual foo.o.pdb files because they + # have already been rolled up into the exe's pdb file. + # We DO NOT have pdb files for the builtin commands (like git-status.exe) + # because it is just a copy/hardlink of git.exe, rather than a unique binary. + $(INSTALL) git.pdb '$(DESTDIR_SQ)$(bindir_SQ)' + $(INSTALL) git-shell.pdb '$(DESTDIR_SQ)$(bindir_SQ)' + $(INSTALL) git-upload-pack.pdb '$(DESTDIR_SQ)$(bindir_SQ)' + $(INSTALL) git-credential-store.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' + $(INSTALL) git-daemon.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' + $(INSTALL) git-fast-import.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' + $(INSTALL) git-http-backend.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' + $(INSTALL) git-http-fetch.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' + $(INSTALL) git-http-push.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' + $(INSTALL) git-imap-send.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' + $(INSTALL) git-remote-http.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' + $(INSTALL) git-remote-testsvn.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' + $(INSTALL) git-sh-i18n--envsubst.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' + $(INSTALL) git-show-index.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' +endif $(MAKE) -C templates DESTDIR='$(DESTDIR_SQ)' install $(INSTALL) -d -m 755 '$(DESTDIR_SQ)$(mergetools_instdir_SQ)' $(INSTALL) -m 644 mergetools/* '$(DESTDIR_SQ)$(mergetools_instdir_SQ)' diff --git a/compat/vcbuild/Makefile b/compat/vcbuild/Makefile index b94bcd119a..cfca611dfc 100644 --- a/compat/vcbuild/Makefile +++ b/compat/vcbuild/Makefile @@ -106,13 +106,14 @@ openssl: insdir ZLIB_VER=1.2.8.8 ZLIB_ROOT=$(PKGDIR)/zlib.v140.windesktop.msvcstl.dyn.rt-dyn.$(ZLIB_VER) ZLIB_INC=$(ZLIB_ROOT)/build/native/include -ZLIB_LIB=$(ZLIB_ROOT)/lib/native/v140/windesktop/msvcstl/dyn/rt-dyn/x64/Release +ZLIB_LIB=$(ZLIB_ROOT)/lib/native/v140/windesktop/msvcstl/dyn/rt-dyn/x64/RelWithDebInfo zlib: insdir [ -d $(INST_INC)/zlib ] || mkdir $(INST_INC)/zlib cp -r $(ZLIB_INC)/* $(INST_INC)/zlib cp -r $(ZLIB_LIB)/*.lib $(INST_LIB)/ cp -r $(ZLIB_LIB)/*.dll $(INST_BIN)/ + cp -r $(ZLIB_LIB)/*.pdb $(INST_BIN)/ ################################################################## clean: diff --git a/compat/vcbuild/scripts/clink.pl b/compat/vcbuild/scripts/clink.pl index 3bbfa47bf4..a8df6bb63b 100755 --- a/compat/vcbuild/scripts/clink.pl +++ b/compat/vcbuild/scripts/clink.pl @@ -22,9 +22,12 @@ while (@ARGV) { my $file_out = shift @ARGV; if ("$file_out" =~ /exe$/) { $is_linking = 1; + # Create foo.exe and foo.pdb push(@args, "-OUT:$file_out"); } else { + # Create foo.o and foo.o.pdb push(@args, "-Fo$file_out"); + push(@args, "-Fd$file_out.pdb"); } } elsif ("$arg" eq "-lz") { push(@args, "zlib.lib"); @@ -49,12 +52,10 @@ while (@ARGV) { } if ($is_linking) { push(@args, @lflags); - # force PDB to be created. - push(@args, "-debug"); unshift(@args, "link.exe"); } else { unshift(@args, "cl.exe"); push(@args, @cflags); } -#printf("**** @args\n"); +printf("**** @args\n\n\n"); exit (system(@args) != 0); diff --git a/config.mak.uname b/config.mak.uname index e780fecff9..63a0ccc990 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -409,12 +409,16 @@ ifeq ($(uname_S),Windows) # Optionally enable memory leak reporting. # BASIC_CLFAGS += -DUSE_MSVC_CRTDBG BASIC_CFLAGS += -DPROTECT_NTFS_DEFAULT=1 + # Always give "-Zi" to the compiler and "-debug" to linker (even in + # release mode) to force a PDB to be generated (like RelWithDebInfo). + BASIC_CFLAGS += -Zi + BASIC_LDFLAGS += -debug ifndef DEBUG - BASIC_CFLAGS += -GL -Os -MD - BASIC_LDFLAGS += -LTCG + BASIC_CFLAGS += -GL -Gy -Os -Oy- -MD -DNDEBUG + BASIC_LDFLAGS += -release -LTCG /OPT:REF /OPT:ICF /INCREMENTAL:NO /DEBUGTYPE:CV,FIXUP AR += -LTCG else - BASIC_CFLAGS += -Zi -MDd -DDEBUG -D_DEBUG + BASIC_CFLAGS += -MDd -DDEBUG -D_DEBUG endif X = .exe From e1a913e3967dcd34a257a1842d51fa3930c23bb2 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 14 Oct 2016 06:01:39 -0700 Subject: [PATCH 18/26] msvc: respect the quiet-by-default output Signed-off-by: Johannes Schindelin --- compat/vcbuild/scripts/clink.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compat/vcbuild/scripts/clink.pl b/compat/vcbuild/scripts/clink.pl index a8df6bb63b..b67b831304 100755 --- a/compat/vcbuild/scripts/clink.pl +++ b/compat/vcbuild/scripts/clink.pl @@ -57,5 +57,5 @@ if ($is_linking) { unshift(@args, "cl.exe"); push(@args, @cflags); } -printf("**** @args\n\n\n"); +printf(STDERR "**** @args\n\n\n") if (!defined($ENV{'QUIET_GEN'})); exit (system(@args) != 0); From 3029a1627bc1144f8045b0776c67ea6bf6d8aa71 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 12 Oct 2016 14:21:42 -0400 Subject: [PATCH 19/26] msvc: use OpenSSL's SHA-1 routines Just like 1e2ce1d (sha1: Use OpenSSL SHA1 routines on MINGW, 2016-10-12), we now use OpenSSL's SHA-1 routines instead of Git's own because OpenSSL is substantially faster as of version 1.0.2: it now uses hardware acceleration on Intel processors much more effectively. Signed-off-by: Johannes Schindelin --- config.mak.uname | 1 - 1 file changed, 1 deletion(-) diff --git a/config.mak.uname b/config.mak.uname index 63a0ccc990..b3f0df6e3f 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -364,7 +364,6 @@ ifeq ($(uname_S),Windows) NO_REGEX = YesPlease NO_GETTEXT = YesPlease NO_PYTHON = YesPlease - BLK_SHA1 = YesPlease ETAGS_TARGET = ETAGS NO_INET_PTON = YesPlease NO_INET_NTOP = YesPlease From e68f061b57b560ec4943dff349556e8114ebe947 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 20 Oct 2016 08:21:05 -0700 Subject: [PATCH 20/26] msvc: use libiconv Signed-off-by: Johannes Schindelin --- compat/vcbuild/Makefile | 22 +++++++++++++++++++++- compat/vcbuild/packages.config | 3 +++ config.mak.uname | 3 +-- 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/compat/vcbuild/Makefile b/compat/vcbuild/Makefile index cfca611dfc..a9d1a6b92b 100644 --- a/compat/vcbuild/Makefile +++ b/compat/vcbuild/Makefile @@ -9,7 +9,8 @@ INST_BIN=$(INST)/bin PKGDIR=GEN.PKGS ################################################################## -all: unpack expat libssh libssh_redist curl curl_redist openssl zlib +all: unpack expat libssh libssh_redist curl curl_redist openssl zlib \ + libiconv libiconv_redist unpack: [ -d $(PKGDIR) ] || mkdir $(PKGDIR) @@ -38,6 +39,25 @@ expat: insdir cp -r $(EXPAT_LIB)/* $(INST_LIB)/ +################################################################## + +LIBICONV_VER=1.14.0.11 +LIBICONV_ROOT=$(PKGDIR)/libiconv.$(LIBICONV_VER)/build/native +LIBICONV_INC=$(LIBICONV_ROOT)/include +LIBICONV_LIB=$(LIBICONV_ROOT)/lib/v110/x64/Release/dynamic/cdecl + +libiconv: insdir + cp -r $(LIBICONV_INC)/* $(INST_INC)/ + cp -r $(LIBICONV_LIB)/libiconv.lib $(INST_LIB)/iconv.lib + + +LIBICONV_REDIST_ROOT=$(PKGDIR)/libiconv.redist.$(LIBICONV_VER)/build/native +LIBICONV_REDIST_BIN=$(LIBICONV_REDIST_ROOT)/bin/v110/x64/Release/dynamic/cdecl + +libiconv_redist: insdir + cp -r $(LIBICONV_REDIST_BIN)/* $(INST_BIN)/ + + ################################################################## LIBSSH_VER=1.4.3.3 diff --git a/compat/vcbuild/packages.config b/compat/vcbuild/packages.config index e6e6d2f63f..ee01be5edb 100644 --- a/compat/vcbuild/packages.config +++ b/compat/vcbuild/packages.config @@ -5,6 +5,9 @@ + + + diff --git a/config.mak.uname b/config.mak.uname index b3f0df6e3f..e42ffc3ce8 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -343,8 +343,7 @@ ifeq ($(uname_S),Windows) NO_STRCASESTR = YesPlease NO_STRLCPY = YesPlease NO_MEMMEM = YesPlease - # NEEDS_LIBICONV = YesPlease - NO_ICONV = YesPlease + NEEDS_LIBICONV = YesPlease NO_STRTOUMAX = YesPlease NO_MKDTEMP = YesPlease # VS2015 with UCRT claims that snprintf and friends are C99 compliant, From bc58d2cdf00aced25175e14524b76b97038805ad Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 21 Oct 2016 02:46:59 -0700 Subject: [PATCH 21/26] msvc: work around iconv() not setting errno When compiling with MSVC, we rely on NuPkgs to provide the binaries of dependencies such as libiconv. The libiconv 1.14.0.11 package available from https://www.nuget.org/packages/libiconv seems to have a bug where it does not set errno (when we would expect it to be E2BIG). Let's simulate the error condition by taking less than 16 bytes remaining in the out buffer as an indicator that we ran out of space. While 16 might seem a bit excessive (when converting from, say, any encoding to UTF-8, 8 bytes should be fine), it is designed to be a safe margin. Signed-off-by: Johannes Schindelin --- git-compat-util.h | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/git-compat-util.h b/git-compat-util.h index b63b298b3b..7c9e36ea6b 100644 --- a/git-compat-util.h +++ b/git-compat-util.h @@ -278,6 +278,33 @@ extern char *gitdirname(char *); #ifndef NO_ICONV #include +#ifdef _MSC_VER +/* + * At least version 1.14.0.11 of the libiconv NuPkg at + * https://www.nuget.org/packages/libiconv/ does not set errno at all. + * + * Let's simulate it by testing whether we might have possibly run out of + * space. + */ +static inline size_t msvc_iconv(iconv_t conv, + const char **inpos, size_t *insize, + char **outpos, size_t *outsize) +{ + int saved_errno = errno; + size_t res; + + errno = ENOENT; + res = iconv(conv, inpos, insize, outpos, outsize); + if (!res) + errno = saved_errno; + else if (errno == ENOENT) + errno = *outsize < 16 ? E2BIG : 0; + + return res; +} +#undef iconv +#define iconv msvc_iconv +#endif #endif #ifndef NO_OPENSSL From a8da631c0623713b0fa62e4c185828d00dd842ba Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sun, 23 Oct 2016 01:22:20 -0700 Subject: [PATCH 22/26] t7800: fix quoting When passing a command-line to call an external diff command to the difftool, we must be prepared for paths containing special characters, e.g. backslashes in the temporary directory's path on Windows. This has been caught by running the test suite with an MSVC-built Git: in contrast to the MINGW one, it does not rewrite `$TMP` to use forward slashes instead of backslashes. Signed-off-by: Johannes Schindelin --- t/t7800-difftool.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/t7800-difftool.sh b/t/t7800-difftool.sh index 25241f4096..6499e8d3c2 100755 --- a/t/t7800-difftool.sh +++ b/t/t7800-difftool.sh @@ -332,7 +332,7 @@ test_expect_success 'difftool --extcmd cat arg1' ' test_expect_success 'difftool --extcmd cat arg2' ' echo branch >expect && git difftool --no-prompt \ - --extcmd sh\ -c\ \"cat\ \$2\" branch >actual && + --extcmd sh\ -c\ \"cat\ \\\"\$2\\\"\" branch >actual && test_cmp expect actual ' From 410a523a42b6e391ef0643a8bb37939906b1362f Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Tue, 8 Nov 2016 11:18:43 -0500 Subject: [PATCH 23/26] vs2015: turn on optimize-for-speed in release build Set -O2 (maximize speed) rather than -Os (favor small code) for non-debug builds. Signed-off-by: Jeff Hostetler --- config.mak.uname | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.mak.uname b/config.mak.uname index e42ffc3ce8..0544660586 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -412,7 +412,7 @@ ifeq ($(uname_S),Windows) BASIC_CFLAGS += -Zi BASIC_LDFLAGS += -debug ifndef DEBUG - BASIC_CFLAGS += -GL -Gy -Os -Oy- -MD -DNDEBUG + BASIC_CFLAGS += -GL -Gy -O2 -Oy- -MD -DNDEBUG BASIC_LDFLAGS += -release -LTCG /OPT:REF /OPT:ICF /INCREMENTAL:NO /DEBUGTYPE:CV,FIXUP AR += -LTCG else From 4de9f2c359bd078721e888e455c1f6f8281603b5 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Tue, 8 Nov 2016 11:39:35 -0500 Subject: [PATCH 24/26] vs2015: teach 'make clean' to delete PDBs Signed-off-by: Jeff Hostetler --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index 6fc8eee606..0f02823a18 100644 --- a/Makefile +++ b/Makefile @@ -2570,6 +2570,9 @@ endif $(RM) GIT-VERSION-FILE GIT-CFLAGS GIT-LDFLAGS GIT-BUILD-OPTIONS $(RM) GIT-USER-AGENT GIT-PREFIX $(RM) GIT-SCRIPT-DEFINES GIT-PERL-DEFINES GIT-PYTHON-VARS +ifdef MSVC + $(RM) $(patsubst %.o,%.o.pdb,$(OBJECTS)) +endif .PHONY: all install profile-clean clean strip .PHONY: shell_compatibility_test please_set_SHELL_PATH_to_a_more_modern_shell From 90a2e94ccdcc4abeca589bf81f11a5c382a3dd02 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Tue, 8 Nov 2016 11:41:45 -0500 Subject: [PATCH 25/26] vs2015: remove todo list item from README_vs2015.txt Remove todo list item for vs2015 build notes regarding the third party DLLs. Signed-off-by: Jeff Hostetler --- compat/vcbuild/README_VS2015.txt | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/compat/vcbuild/README_VS2015.txt b/compat/vcbuild/README_VS2015.txt index 763523fbb4..a9c6f59dc6 100644 --- a/compat/vcbuild/README_VS2015.txt +++ b/compat/vcbuild/README_VS2015.txt @@ -51,8 +51,4 @@ TODO List: developer command prompt is started. This would also allow us to switch between 32- and 64-bit tool chains. -[B] Currently, we leave the third-party DLLs we reference in - "compat/vcbuild/GEN.DEPS/bin". We need an installer - step to move them next to git.exe (or into libexec/git-core). - -[C] We need to build SLN or VCPROJ files. +[B] We need to build SLN or VCPROJ files. From b01acc2bb8e0ed05333b027f27e9d6ddc648541a Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 9 Nov 2016 15:04:22 +0100 Subject: [PATCH 26/26] compat/vcbuild: possibly reuse Git for Windows' SDK's NuGet In Git for Windows' SDK, there is already a script to package Git for Windows as a NuGet package, downloading nuget.exe if needed. Let's just fall back to using that executable (if it is there) if nuget.exe was not found in the PATH. Signed-off-by: Johannes Schindelin --- compat/vcbuild/Makefile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/compat/vcbuild/Makefile b/compat/vcbuild/Makefile index a9d1a6b92b..255dd43084 100644 --- a/compat/vcbuild/Makefile +++ b/compat/vcbuild/Makefile @@ -7,6 +7,10 @@ INST_LIB=$(INST)/lib INST_BIN=$(INST)/bin PKGDIR=GEN.PKGS +NUGET ?= nuget.exe +ifneq ($(shell $(NUGET) help 2>/dev/null; echo $$?),0) + NUGET := /usr/src/build-extra/nuget/nuget.exe +endif ################################################################## all: unpack expat libssh libssh_redist curl curl_redist openssl zlib \ @@ -14,7 +18,7 @@ all: unpack expat libssh libssh_redist curl curl_redist openssl zlib \ unpack: [ -d $(PKGDIR) ] || mkdir $(PKGDIR) - nuget.exe restore packages.config -ConfigFile nuget.config \ + "$(NUGET)" restore packages.config -ConfigFile nuget.config \ -NonInteractive -OutputDirectory $(PKGDIR) insdir: