From 046e4c29d4a9c21c5ff7663d1b2cce593027edf9 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 20 Oct 2016 06:31:47 -0700 Subject: [PATCH 01/26] msvc: fix dependencies of compat/msvc.c The file compat/msvc.c includes compat/mingw.c, which means that we have to recompile compat/msvc.o if compat/mingw.c changes. Signed-off-by: Johannes Schindelin --- config.mak.uname | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config.mak.uname b/config.mak.uname index 268656741c..349113618e 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -387,6 +387,8 @@ else BASIC_CFLAGS += -Zi -MDd endif X = .exe + +compat/msvc.o: compat/msvc.c compat/mingw.c GIT-CFLAGS endif ifeq ($(uname_S),Interix) NO_INITGROUPS = YesPlease From 032affca29e9da2eee49f0cf323273037791e8bc Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 3 Jun 2016 08:38:17 -0400 Subject: [PATCH 02/26] msvc: ignore VS2015 trash files Signed-off-by: Jeff Hostetler --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 833ef3b0b7..92f6405a0f 100644 --- a/.gitignore +++ b/.gitignore @@ -216,5 +216,7 @@ *.user *.idb *.pdb +*.ilk +.vs/ /Debug/ /Release/ From 664483be1f53bc7ffae450214656ecaee297d2dd Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 3 Jun 2016 11:32:01 -0400 Subject: [PATCH 03/26] msvc: add NuGet scripts for building with VS2015 This commit contains a GNU Makefile and NuGet configuration scripts to download and install the various third-party libraries that we will need to build/link with when using VS2015 to build Git. The file "compat/vcbuild/README_VS2015.txt" contains instructions for using this. In this commit, "compat/vcbuild/Makefile" contains hard-coded version numbers of the packages we require. These are set to the current versions as of the time of this commit. We use "nuget restore" to install them explicitly using a "package.config". A future improvement would try to use some of the automatic package management functions and eliminate the need to specify exact versions. I tried, but could not get this to work. NuGet was happy dowload "minimum requirements" rather than "lastest" for dependencies -- and only look at one package at a time. For example, both curl and openssl depend upon zlib and have different minimums. It was unclear which version of zlib would be installed and seemed to be dependent on the order of the top-level packages. So, I'm skipping that for now. We need to be very precise when specifying NuGet package versions: while nuget.exe auto-completes a version, say, 1.0.2 to 1.0.2.0, we will want to parse packages.config ourselves, to generate the Visual Studio solution, and there we need the exact version number to be able to generate the exact path to the correct .targets file. Signed-off-by: Jeff Hostetler --- compat/vcbuild/.gitignore | 1 + compat/vcbuild/Makefile | 122 +++++++++++++++++++++++++++++++ compat/vcbuild/README_VS2015.txt | 58 +++++++++++++++ compat/vcbuild/nuget.config | 27 +++++++ compat/vcbuild/packages.config | 23 ++++++ 5 files changed, 231 insertions(+) create mode 100644 compat/vcbuild/.gitignore create mode 100644 compat/vcbuild/Makefile create mode 100644 compat/vcbuild/README_VS2015.txt create mode 100644 compat/vcbuild/nuget.config create mode 100644 compat/vcbuild/packages.config diff --git a/compat/vcbuild/.gitignore b/compat/vcbuild/.gitignore new file mode 100644 index 0000000000..e14691508e --- /dev/null +++ b/compat/vcbuild/.gitignore @@ -0,0 +1 @@ +GEN.* diff --git a/compat/vcbuild/Makefile b/compat/vcbuild/Makefile new file mode 100644 index 0000000000..b94bcd119a --- /dev/null +++ b/compat/vcbuild/Makefile @@ -0,0 +1,122 @@ +## Makefile to install nuget package dependencies. +################################################################## + +INST=GEN.DEPS +INST_INC=$(INST)/include +INST_LIB=$(INST)/lib +INST_BIN=$(INST)/bin + +PKGDIR=GEN.PKGS + +################################################################## +all: unpack expat libssh libssh_redist curl curl_redist openssl zlib + +unpack: + [ -d $(PKGDIR) ] || mkdir $(PKGDIR) + nuget.exe restore packages.config -ConfigFile nuget.config \ + -NonInteractive -OutputDirectory $(PKGDIR) + +insdir: + [ -d $(INST) ] || mkdir $(INST) + [ -d $(INST_INC) ] || mkdir $(INST_INC) + [ -d $(INST_BIN) ] || mkdir $(INST_BIN) + [ -d $(INST_LIB) ] || mkdir $(INST_LIB) + +################################################################## +## We place the expat headers in their own subdirectory. +## The custom is to reference , so compile with: +## -I$(INST_INC)/expat + +EXPAT_VER=2.1.0.11 +EXPAT_ROOT=$(PKGDIR)/expat.$(EXPAT_VER)/build/native +EXPAT_INC=$(EXPAT_ROOT)/include +EXPAT_LIB=$(EXPAT_ROOT)/lib/v110/x64/Release/dynamic/utf8 + +expat: insdir + [ -d $(INST_INC)/expat ] || mkdir $(INST_INC)/expat + cp -r $(EXPAT_INC)/* $(INST_INC)/expat/ + cp -r $(EXPAT_LIB)/* $(INST_LIB)/ + + +################################################################## + +LIBSSH_VER=1.4.3.3 +LIBSSH_ROOT=$(PKGDIR)/libssh2.$(LIBSSH_VER)/build/native +LIBSSH_INC=$(LIBSSH_ROOT)/include +LIBSSH_LIB=$(LIBSSH_ROOT)/lib/v110/x64/Release/dynamic/cdecl + +libssh: insdir + [ -d $(INST_INC)/libssh2 ] || mkdir $(INST_INC)/libssh2 + cp -r $(LIBSSH_INC)/* $(INST_INC)/libssh2 + cp -r $(LIBSSH_LIB)/* $(INST_LIB)/ + + +LIBSSH_REDIST_ROOT=$(PKGDIR)/libssh2.redist.$(LIBSSH_VER)/build/native +LIBSSH_REDIST_BIN=$(LIBSSH_REDIST_ROOT)/bin/v110/x64/Release/dynamic/cdecl + +libssh_redist: insdir + cp -r $(LIBSSH_REDIST_BIN)/* $(INST_BIN)/ + + +################################################################## +## We place the curl headers in their own subdirectory. +## The custom is to reference , so compile with: +## -I$(INST_INC) + +CURL_VER=7.30.0.2 +CURL_ROOT=$(PKGDIR)/curl.$(CURL_VER)/build/native +CURL_INC=$(CURL_ROOT)/include/curl +CURL_LIB=$(CURL_ROOT)/lib/v110/x64/Release/dynamic + +curl: insdir + [ -d $(INST_INC)/curl ] || mkdir $(INST_INC)/curl + cp -r $(CURL_INC)/* $(INST_INC)/curl + cp -r $(CURL_LIB)/* $(INST_LIB)/ + + +CURL_REDIST_ROOT=$(PKGDIR)/curl.redist.$(CURL_VER)/build/native +CURL_REDIST_BIN=$(CURL_REDIST_ROOT)/bin/v110/x64/Release/dynamic + +curl_redist: insdir + cp -r $(CURL_REDIST_BIN)/* $(INST_BIN)/ + + +################################################################## +## We place the openssl headers in their own subdirectory. +## The custom is to reference , so compile with: +## -I$(INST_INC) + +OPENSSL_VER=1.0.2.1 +OPENSSL_ROOT=$(PKGDIR)/openssl.v140.windesktop.msvcstl.dyn.rt-dyn.x64.$(OPENSSL_VER) +OPENSSL_INC=$(OPENSSL_ROOT)/build/native/include/openssl +OPENSSL_LIB=$(OPENSSL_ROOT)/lib/native/v140/windesktop/msvcstl/dyn/rt-dyn/x64/release + +openssl: insdir + [ -d $(INST_INC)/openssl ] || mkdir $(INST_INC)/openssl + cp -r $(OPENSSL_INC)/* $(INST_INC)/openssl + cp -r $(OPENSSL_LIB)/*.lib $(INST_LIB)/ + cp -r $(OPENSSL_LIB)/*.dll $(INST_BIN)/ + cp -r $(OPENSSL_LIB)/*.pdb $(INST_BIN)/ + +################################################################## +## We place the zlib headers in their own subdirectory. +## The custom is to reference , so compile with: +## -I$(INST_INC)/zlib + +ZLIB_VER=1.2.8.8 +ZLIB_ROOT=$(PKGDIR)/zlib.v140.windesktop.msvcstl.dyn.rt-dyn.$(ZLIB_VER) +ZLIB_INC=$(ZLIB_ROOT)/build/native/include +ZLIB_LIB=$(ZLIB_ROOT)/lib/native/v140/windesktop/msvcstl/dyn/rt-dyn/x64/Release + +zlib: insdir + [ -d $(INST_INC)/zlib ] || mkdir $(INST_INC)/zlib + cp -r $(ZLIB_INC)/* $(INST_INC)/zlib + cp -r $(ZLIB_LIB)/*.lib $(INST_LIB)/ + cp -r $(ZLIB_LIB)/*.dll $(INST_BIN)/ + +################################################################## +clean: + rm -rf $(INST) + +clobber: clean + rm -rf $(PKGDIR) diff --git a/compat/vcbuild/README_VS2015.txt b/compat/vcbuild/README_VS2015.txt new file mode 100644 index 0000000000..763523fbb4 --- /dev/null +++ b/compat/vcbuild/README_VS2015.txt @@ -0,0 +1,58 @@ +Instructions for building Git for Windows using VS2015. +================================================================ + +Installing third-party dependencies: +==================================== + +[1] Install nuget.exe somewhere on your system and add it to your PATH. + https://docs.nuget.org/consume/command-line-reference + https://dist.nuget.org/index.html + +[2] Download required nuget packages for third-party libraries. + Using a terminal window, type: + + make -C compat/vcbuild + + This will download the packages, unpack them into GEN.PKGS, + and populate the {include, lib, bin} directories in GEN.DEPS. + + +Building Git for Windows using VS2015: +====================================== + +[3] Build 64-bit version of Git for Windows. + Using a terminal window: + + make MSVC=1 DEBUG=1 + + +[4] Add compat/vcbuild/GEN.DEPS/bin to your PATH. + +[5] You should then be able to run the test suite and any interactive + commands. + +[6] To debug/profile in VS, open the git.exe in VS and run/debug + it. (Be sure to add GEN.DEPS/bin to the PATH in the debug + dialog.) + + +TODO List: +========== + +[A] config.mak.uname currently contains hard-coded paths + to the various MSVC and SDK libraries for the 64-bit + version of the compilers and libaries. + + See: SANE_TOOL_PATH, MSVC_DEPS, MSVC_SDK*, MSVC_VCDIR. + + Long term, we need to figure out how to properly import + values for %VCINSTALLDIR%, %LIB%, %LIBPATH%, and the + other values normally set by "vsvars32.bat" when a + developer command prompt is started. This would also + allow us to switch between 32- and 64-bit tool chains. + +[B] Currently, we leave the third-party DLLs we reference in + "compat/vcbuild/GEN.DEPS/bin". We need an installer + step to move them next to git.exe (or into libexec/git-core). + +[C] We need to build SLN or VCPROJ files. diff --git a/compat/vcbuild/nuget.config b/compat/vcbuild/nuget.config new file mode 100644 index 0000000000..b5e7349504 --- /dev/null +++ b/compat/vcbuild/nuget.config @@ -0,0 +1,27 @@ + + + + + + + + + + + + + + + + + + + + + + + diff --git a/compat/vcbuild/packages.config b/compat/vcbuild/packages.config new file mode 100644 index 0000000000..e6e6d2f63f --- /dev/null +++ b/compat/vcbuild/packages.config @@ -0,0 +1,23 @@ + + + + + + + + + + + + + + + + + + + + + + + From 0738a47698b48e0b469105205210a07daf7e44c4 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 3 Jun 2016 13:34:28 -0400 Subject: [PATCH 04/26] msvc: update compile helper for VS2015 Support -Z flags ("specify PDB options"), only include -l args on link commands, and force PDBs to be created. Signed-off-by: Jeff Hostetler --- compat/vcbuild/scripts/clink.pl | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/compat/vcbuild/scripts/clink.pl b/compat/vcbuild/scripts/clink.pl index a87d0da512..3bbfa47bf4 100755 --- a/compat/vcbuild/scripts/clink.pl +++ b/compat/vcbuild/scripts/clink.pl @@ -12,10 +12,11 @@ use strict; my @args = (); my @cflags = (); +my @lflags = (); my $is_linking = 0; while (@ARGV) { my $arg = shift @ARGV; - if ("$arg" =~ /^-[DIMGO]/) { + if ("$arg" =~ /^-[DIMGOZ]/) { push(@cflags, $arg); } elsif ("$arg" eq "-o") { my $file_out = shift @ARGV; @@ -35,9 +36,11 @@ while (@ARGV) { push(@args, "ssleay32.lib"); } elsif ("$arg" eq "-lcurl") { push(@args, "libcurl.lib"); + } elsif ("$arg" eq "-lexpat") { + push(@args, "libexpat.lib"); } elsif ("$arg" =~ /^-L/ && "$arg" ne "-LTCG") { $arg =~ s/^-L/-LIBPATH:/; - push(@args, $arg); + push(@lflags, $arg); } elsif ("$arg" =~ /^-R/) { # eat } else { @@ -45,6 +48,9 @@ while (@ARGV) { } } if ($is_linking) { + push(@args, @lflags); + # force PDB to be created. + push(@args, "-debug"); unshift(@args, "link.exe"); } else { unshift(@args, "cl.exe"); From ad3ea41866779cfe8526c47472125ee823103cb9 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 3 Jun 2016 14:15:02 -0400 Subject: [PATCH 05/26] msvc: update Makefile and compiler settings for VS2015 Signed-off-by: Jeff Hostetler --- Makefile | 2 +- config.mak.uname | 47 ++++++++++++++++++++++++++++++++++++++++++----- git-compat-util.h | 9 +++++++++ 3 files changed, 52 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index a53b842b5d..f056ff9bfb 100644 --- a/Makefile +++ b/Makefile @@ -1070,7 +1070,7 @@ endif ifdef SANE_TOOL_PATH SANE_TOOL_PATH_SQ = $(subst ','\'',$(SANE_TOOL_PATH)) -BROKEN_PATH_FIX = 's|^\# @@BROKEN_PATH_FIX@@$$|git_broken_path_fix $(SANE_TOOL_PATH_SQ)|' +BROKEN_PATH_FIX = 's|^\# @@BROKEN_PATH_FIX@@$$|git_broken_path_fix "$(SANE_TOOL_PATH_SQ)"|' PATH := $(SANE_TOOL_PATH):${PATH} else BROKEN_PATH_FIX = '/^\# @@BROKEN_PATH_FIX@@$$/d' diff --git a/config.mak.uname b/config.mak.uname index 349113618e..3c3dd6826b 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -331,6 +331,13 @@ endif ifeq ($(uname_S),Windows) GIT_VERSION := $(GIT_VERSION).MSVC pathsep = ; + # Prepend MSVC 64-bit tool-chain to PATH. + # + # A regular Git Bash *does not* have cl.exe in its $PATH. As there is a + # link.exe next to, and required by, cl.exe, we have to prepend this + # onto the existing $PATH. + # + SANE_TOOL_PATH ?= /c/Program Files (x86)/Microsoft Visual Studio 14.0/VC/bin/amd64 HAVE_ALLOCA_H = YesPlease NO_PREAD = YesPlease NEEDS_CRYPTO_WITH_SSL = YesPlease @@ -347,7 +354,13 @@ ifeq ($(uname_S),Windows) NO_ICONV = YesPlease NO_STRTOUMAX = YesPlease NO_MKDTEMP = YesPlease - SNPRINTF_RETURNS_BOGUS = YesPlease + # VS2015 with UCRT claims that snprintf and friends are C99 compliant, + # so we don't need this. + # + # TODO If we want to support older compilers, we need to make this + # TODO conditional on the compiler version. + # + # SNPRINTF_RETURNS_BOGUS = YesPlease NO_SVN_TESTS = YesPlease RUNTIME_PREFIX = YesPlease NO_ST_BLOCKS_IN_STRUCT_STAT = YesPlease @@ -369,22 +382,46 @@ ifeq ($(uname_S),Windows) CC = compat/vcbuild/scripts/clink.pl AR = compat/vcbuild/scripts/lib.pl CFLAGS = - BASIC_CFLAGS = -nologo -I. -I../zlib -Icompat/vcbuild -Icompat/vcbuild/include -DWIN32 -D_CONSOLE -DHAVE_STRING_H -D_CRT_SECURE_NO_WARNINGS -D_CRT_NONSTDC_NO_DEPRECATE + BASIC_CFLAGS = -nologo -I. -Icompat/vcbuild/include -DWIN32 -D_CONSOLE -DHAVE_STRING_H -D_CRT_SECURE_NO_WARNINGS -D_CRT_NONSTDC_NO_DEPRECATE COMPAT_OBJS = compat/msvc.o compat/winansi.o \ compat/win32/pthread.o compat/win32/syslog.o \ compat/win32/dirent.o compat/win32/fscache.o - COMPAT_CFLAGS = -D__USE_MINGW_ACCESS -DNOGDI -DHAVE_STRING_H -Icompat -Icompat/regex -Icompat/win32 -DSTRIP_EXTENSION=\".exe\" + COMPAT_CFLAGS = -D__USE_MINGW_ACCESS -DDETECT_MSYS_TTY -DNOGDI -DHAVE_STRING_H -Icompat -Icompat/regex -Icompat/win32 -DSTRIP_EXTENSION=\".exe\" BASIC_LDFLAGS = -IGNORE:4217 -IGNORE:4049 -NOLOGO -SUBSYSTEM:CONSOLE - EXTLIBS = user32.lib advapi32.lib shell32.lib wininet.lib ws2_32.lib invalidcontinue.obj + EXTLIBS = user32.lib advapi32.lib shell32.lib wininet.lib ws2_32.lib invalidcontinue.obj kernel32.lib ntdll.lib PTHREAD_LIBS = lib = + # Path to the unpacked third-party libraries + MSVC_DEPS = compat/vcbuild/GEN.DEPS + # Compensate for lack of %VCINSTALLDIR%, %LIB%, %LIBPATH%, and etc. + # since vcvars*.bat did not get a chance to setup the environment of + # the user's shell window. + # + # TODO If we ask the user to launch a "x64 Native" command prompt + # TODO and then have it start a git-bash window, these could be + # TODO inherited. So we wouldn't need to add these lines here. + # + MSVC_SDK81 = "c:/Program Files (x86)/Windows Kits/8.1" + MSVC_SDK10 = "c:/Program Files (x86)/Windows Kits/10" + MSVC_VCDIR = "c:/Program Files (x86)/Microsoft Visual Studio 14.0/VC" + BASIC_CFLAGS += \ + -I$(MSVC_DEPS)/include -I$(MSVC_DEPS)/include/expat -I$(MSVC_DEPS)/include/zlib \ + -L$(MSVC_DEPS)/lib \ + -I$(MSVC_SDK81)/Include/um -I$(MSVC_SDK81)/Include/shared \ + -L$(MSVC_SDK81)/lib/winv6.3/um/x64 \ + -I$(MSVC_SDK10)/Include/10.0.10240.0/ucrt \ + -L$(MSVC_SDK10)/lib/10.0.10240.0/ucrt/x64 \ + -I$(MSVC_VCDIR)/INCLUDE \ + -L$(MSVC_VCDIR)/lib/amd64 + # Optionally enable memory leak reporting. + # BASIC_CLFAGS += -DUSE_MSVC_CRTDBG BASIC_CFLAGS += -DPROTECT_NTFS_DEFAULT=1 ifndef DEBUG BASIC_CFLAGS += -GL -Os -MD BASIC_LDFLAGS += -LTCG AR += -LTCG else - BASIC_CFLAGS += -Zi -MDd + BASIC_CFLAGS += -Zi -MDd -DDEBUG -D_DEBUG endif X = .exe diff --git a/git-compat-util.h b/git-compat-util.h index 445d6b9cc9..4179c8799d 100644 --- a/git-compat-util.h +++ b/git-compat-util.h @@ -1,6 +1,15 @@ #ifndef GIT_COMPAT_UTIL_H #define GIT_COMPAT_UTIL_H +#ifdef USE_MSVC_CRTDBG +/* + * For these to work they must appear very early in each + * file -- before most of the standard header files. + */ +#include +#include +#endif + #define _FILE_OFFSET_BITS 64 From 6834ace62a009202b99e87376def0f0b0ff0e9d3 Mon Sep 17 00:00:00 2001 From: Philip Oakley Date: Thu, 21 Apr 2016 14:07:20 +0100 Subject: [PATCH 06/26] msvc: include sigset_t definition On MSVC (VS2008) sigset_t is not defined. Signed-off-by: Philip Oakley --- compat/msvc.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compat/msvc.h b/compat/msvc.h index 580bb55bf4..ec4518eebf 100644 --- a/compat/msvc.h +++ b/compat/msvc.h @@ -24,6 +24,8 @@ static __inline int strcasecmp (const char *s1, const char *s2) #undef ERROR +typedef int sigset_t; + #include "compat/mingw.h" #endif From 74d1c52e08630df9b13fa770bf06e806455be48c Mon Sep 17 00:00:00 2001 From: Philip Oakley Date: Thu, 21 Apr 2016 14:52:05 +0100 Subject: [PATCH 07/26] msvc: define O_ACCMODE This constant is not defined in MSVC's headers. In UCRT's fcntl.h, _O_RDONLY, _O_WRONLY and _O_RDWR are defined as 0, 1 and 2, respectively. Yes, that means that UCRT breaks with the tradition that O_RDWR == O_RDONLY | O_WRONLY. It is a perfectly legal way to define those constants, though, therefore we need to take care of defining O_ACCMODE accordingly. This is particularly important in order to keep our "open() can set errno to EISDIR" emulation working: it tests that (flags & O_ACCMODE) is not identical to O_RDONLY before going on to test specifically whether the file for which open() reported EACCES is, in fact, a directory. Signed-off-by: Philip Oakley Signed-off-by: Johannes Schindelin --- compat/msvc.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compat/msvc.h b/compat/msvc.h index ec4518eebf..ae1c58ec13 100644 --- a/compat/msvc.h +++ b/compat/msvc.h @@ -25,6 +25,8 @@ static __inline int strcasecmp (const char *s1, const char *s2) #undef ERROR typedef int sigset_t; +/* open for reading, writing, or both (not in fcntl.h) */ +#define O_ACCMODE (_O_RDONLY | _O_WRONLY | _O_RDWR) #include "compat/mingw.h" From 868e87bbb389029c13fc0cc205ecb11cd2f10215 Mon Sep 17 00:00:00 2001 From: Philip Oakley Date: Fri, 22 Apr 2016 10:48:13 +0100 Subject: [PATCH 08/26] msvc: fix the declaration of the _REPARSE_DATA_BUFFER structure GCC and MSVC disagree about using the GCC extension _ANONYMOUS_UNION. Simply skip that offending keyword when compiling with MSVC. Signed-off-by: Philip Oakley Signed-off-by: Johannes Schindelin --- compat/mingw.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/compat/mingw.c b/compat/mingw.c index c28376377b..2e7c968b67 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2283,7 +2283,10 @@ typedef struct _REPARSE_DATA_BUFFER { DWORD ReparseTag; WORD ReparseDataLength; WORD Reserved; - _ANONYMOUS_UNION union { +#ifndef _MSC_VER + _ANONYMOUS_UNION +#endif + union { struct { WORD SubstituteNameOffset; WORD SubstituteNameLength; From b6c7c1153ae9a67e705469d5677a35d3da75c511 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 3 Jun 2016 15:38:25 -0400 Subject: [PATCH 09/26] msvc: convert environment from/to UTF-16 on the fly This adds MSVC versions of getenv() and friends. These take UTF-8 arguments and return UTF-8 values, but use the UNICODE versions of the CRT routines. This avoids the need to write to __environ (which is only visible if you statically link to the CRT). This also avoids the CP_ACP conversions performed inside the CRT. It also avoids various memory leaks and problems. Signed-off-by: Jeff Hostetler --- compat/mingw.c | 238 +++++++++++++++++++++++++++++++++++++++++++++++++ compat/mingw.h | 41 +++++++++ 2 files changed, 279 insertions(+) diff --git a/compat/mingw.c b/compat/mingw.c index 2e7c968b67..1dc196a7cc 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -1162,6 +1162,177 @@ static char *path_lookup(const char *cmd, int exe_only) return prog; } +#if defined(_MSC_VER) +/* + * Build an environment block combining the inherited environment + * merged with the given list of settings. + * + * Values of the form "KEY=VALUE" in deltaenv override inherited values. + * Values of the form "KEY" in deltaenv delete inherited values. + * + * We return a contiguous block of UNICODE strings with a final trailing + * zero word. + */ +static wchar_t *make_environment_block(char **deltaenv) +{ + /* + * The CRT (at least as of UCRT) secretly declares "_wenviron" + * as a function that returns a pointer to a mostly static table. + * Grab the pointer and cache it for the duration of our loop. + */ + extern wchar_t **_wenviron; + const wchar_t **my_wenviron = _wenviron; + + /* + * Internally, we create normal 'C' arrays of strings (pointing + * into the blocks) to help with some of the de-dup work. + */ + wchar_t **wptrs_ins = NULL; + wchar_t **wptrs_del = NULL; + wchar_t *wblock_ins = NULL; + wchar_t *wblock_del = NULL; + wchar_t *wend_ins; + wchar_t *wend_del; + wchar_t *w_ins; + wchar_t *w_del; + + int maxlen = 0; + int nr_delta = 0; + int nr_delta_ins = 0; + int nr_delta_del = 0; + int nr_wenv = 0; + int j, k, k_ins, k_del; + + /* + * Count the number of inserts and deletes in the deltaenv list. + * Allocate 'C' arrays for inserts and deletes. + * Also estimate the block size of our results. + */ + if (deltaenv && deltaenv[0] && *deltaenv[0]) { + for (k = 0; deltaenv && deltaenv[k] && *deltaenv[k]; k++) { + if (strchr(deltaenv[k], '=') == NULL) + nr_delta_del++; + else { + maxlen += strlen(deltaenv[k]) + 1; + nr_delta_ins++; + } + } + + if (nr_delta_ins) + wptrs_ins = (wchar_t**)calloc(nr_delta_ins + 1, sizeof(wchar_t*)); + if (nr_delta_del) + wptrs_del = (wchar_t**)calloc(nr_delta_del + 1, sizeof(wchar_t*)); + + nr_delta = nr_delta_ins + nr_delta_del; + } + while (my_wenviron && my_wenviron[nr_wenv] && *my_wenviron[nr_wenv]) + maxlen += wcslen(my_wenviron[nr_wenv++]) + 1; + maxlen++; + + /* + * Allocate blocks for inserted and deleted items. + * The individual pointers in the 'C' arrays will point into here. + * We will use the wblock_ins as the final result. + */ + if (nr_delta_del) { + wblock_del = (wchar_t*)calloc(maxlen, sizeof(wchar_t)); + wend_del = wblock_del + maxlen; + w_del = wblock_del; + } + wblock_ins = (wchar_t*)calloc(maxlen, sizeof(wchar_t)); + wend_ins = wblock_ins + maxlen; + w_ins = wblock_ins; + + /* + * deltaenv values override inherited environment, so put them + * in the result list first (so that we can de-dup using the + * wide versions of them. + * + * Items in the deltaenv list that DO NOT contain an "=" are + * treated as unsetenv. + * + * Care needs to be taken to handle entries that are added first, and + * then deleted. + */ + k_ins = 0; + k_del = 0; + for (k = 0; k < nr_delta; k++) { + if (strchr(deltaenv[k], '=') == NULL) { + wchar_t *save = w_del; + wptrs_del[k_del++] = w_del; + w_del += xutftowcs(w_del, deltaenv[k], (wend_del - w_del)); + *w_del++ = L'='; /* append '=' to make lookup easier in next step. */ + *w_del++ = 0; + + /* If we added this key, we have to remove it again */ + for (j = 0; j < k_ins; j++) + if (!wcsnicmp(wptrs_ins[j], save, w_del - save - 1)) { + if (j + 1 < k_ins) { + int delta = sizeof(wchar_t) * (wptrs_ins[j + 1] - wptrs_ins[j]), i; + memmove(wptrs_ins[j], wptrs_ins[j + 1], sizeof(wchar_t) * (w_ins - wptrs_ins[j + 1])); + for (i = j; i < --k_ins; i++) + wptrs_ins[i] = wptrs_ins[i + 1] - delta; + w_ins -= delta; + } else + w_ins = wptrs_ins[j]; + k_ins--; + j--; + } + } else { + wptrs_ins[k_ins++] = w_ins; + w_ins += xutftowcs(w_ins, deltaenv[k], (wend_ins - w_ins)) + 1; + } + } + assert(k_ins <= nr_delta_ins); + assert(k_del == nr_delta_del); + + /* + * Walk the inherited environment and copy over unique, non-deleted + * ones into the result set. Note that we only have to de-dup WRT + * the values from deltaenv, because the inherited set should be unique. + */ + for (j = 0; j < nr_wenv; j++) { + const wchar_t *v_j = my_wenviron[j]; + wchar_t *v_j_eq = wcschr(v_j, L'='); + int len_j_eq, len_j; + + if (!v_j_eq) + continue; /* should not happen */ + len_j_eq = v_j_eq + 1 - v_j; /* length(v_j) including '=' */ + + /* lookup v_j in list of to-delete vars */ + for (k_del = 0; k_del < nr_delta_del; k_del++) { + if (wcsnicmp(v_j, wptrs_del[k_del], len_j_eq) == 0) + goto skip_it; + } + + /* lookup v_j in deltaenv portion of result set */ + for (k_ins = 0; k_ins < nr_delta_ins; k_ins++) { + if (wcsnicmp(v_j, wptrs_ins[k_ins], len_j_eq) == 0) + goto skip_it; + } + + /* item is unique, add it to results. */ + len_j = wcslen(v_j); + memcpy(w_ins, v_j, len_j * sizeof(wchar_t)); + w_ins += len_j + 1; + +skip_it: + ; + } + + if (wptrs_ins) + free(wptrs_ins); + if (wptrs_del) + free(wptrs_del); + if (wblock_del) + free(wblock_del); + + return wblock_ins; +} + +#else + static int do_putenv(char **env, const char *name, int size, int free_old); /* used number of elements of environ array, including terminating NULL */ @@ -1208,6 +1379,7 @@ static wchar_t *make_environment_block(char **deltaenv) free(tmpenv); return wenvblk; } +#endif static void do_unset_environment_variables(void) { @@ -1498,6 +1670,70 @@ int mingw_kill(pid_t pid, int sig) return -1; } +#if defined(_MSC_VER) + +/* UTF8 versions of getenv and putenv (and unsetenv). + * Internally, they use the CRT's stock UNICODE routines + * to avoid data loss. + * + * Unlike the mingw version, we DO NOT directly write to + * the CRT variables. We also DO NOT try to manage/replace + * the CRT storage. + */ +char *msc_getenv(const char *name) +{ + int len_key, len_value; + wchar_t *w_key; + char *value; + const wchar_t *w_value; + + if (!name || !*name) + return NULL; + + len_key = strlen(name) + 1; + w_key = calloc(len_key, sizeof(wchar_t)); + xutftowcs(w_key, name, len_key); + w_value = _wgetenv(w_key); + free(w_key); + + if (!w_value) + return NULL; + + len_value = wcslen(w_value) * 3 + 1; + value = calloc(len_value, sizeof(char)); + xwcstoutf(value, w_value, len_value); + + /* TODO Warning: We return "value" which is an allocated + * value and the caller is NOT expecting to have to free + * it, so we leak memory. + */ + return value; +} + +int msc_putenv(const char *name) +{ + int len, result; + char *equal; + wchar_t *wide; + + if (!name || !*name) + return 0; + + len = strlen(name); + equal = strchr(name, '='); + wide = calloc(len+1+!equal, sizeof(wchar_t)); + xutftowcs(wide, name, len+1); + if (!equal) + wcscat(wide, L"="); + + result = _wputenv(wide); + + free(wide); + return result; +} + +#else + /* * Compare environment entries by key (i.e. stopping at '=' or '\0'). */ @@ -1626,6 +1862,8 @@ int mingw_putenv(const char *namevalue) return 0; } +#endif + /* * Note, this isn't a complete replacement for getaddrinfo. It assumes * that service contains a numerical port, or that it is null. It diff --git a/compat/mingw.h b/compat/mingw.h index c20f23d4f0..5f4fa0b2f7 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -265,12 +265,53 @@ char *mingw_getcwd(char *pointer, int len); #error "NO_UNSETENV is incompatible with the MinGW startup code!" #endif +#if defined(_MSC_VER) +/* + * We bind *env() routines (even the mingw_ ones) to private msc_ versions. + * These talk to the CRT using UNICODE/wchar_t, but maintain the original + * narrow-char API. + * + * Note that the MSCRT maintains both ANSI (getenv()) and UNICODE (_wgetenv()) + * routines and stores both versions of each environment variable in parallel + * (and secretly updates both when you set one or the other), but it uses CP_ACP + * to do the conversion rather than CP_UTF8. + * + * Since everything in the git code base is UTF8, we define the msc_ routines + * to access the CRT using the UNICODE routines and manually convert them to + * UTF8. This also avoids round-trip problems. + * + * This also helps with our linkage, since "_wenviron" is publicly exported + * from the CRT. But to access "_environ" we would have to statically link + * to the CRT (/MT). + * + * We also use "wmain(argc,argv,env)" and get the initial UNICODE setup for us. + * This avoids the need for the msc_startup() to import and convert the + * inherited environment. + * + * We require NO_SETENV (and let gitsetenv() call our msc_putenv). + */ +#define getenv msc_getenv +#define putenv msc_putenv +#define unsetenv msc_putenv +#define mingw_getenv msc_getenv +#define mingw_putenv msc_putenv +char *msc_getenv(const char *name); +int msc_putenv(const char *name); + +#ifndef NO_SETENV +#error "NO_SETENV is required for MSC startup code!" +#endif + +#else + char *mingw_getenv(const char *name); #define getenv mingw_getenv int mingw_putenv(const char *namevalue); #define putenv mingw_putenv #define unsetenv mingw_putenv +#endif + int mingw_gethostname(char *host, int namelen); #define gethostname mingw_gethostname From 43394b26c577dfe12f5a9aeadeb42e548c6712df Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 3 Jun 2016 15:38:25 -0400 Subject: [PATCH 10/26] msvc: mark a variable as non-const VS2015 complains when using a const pointer in memcpy()/free(). Signed-off-by: Jeff Hostetler --- compat/mingw.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/compat/mingw.c b/compat/mingw.c index 1dc196a7cc..3ded32d7f8 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -1597,7 +1597,10 @@ static int try_shell_exec(const char *cmd, char *const *argv) prog = path_lookup(interpr, 1); if (prog) { int argc = 0; - const char **argv2; +#ifndef _MSC_VER + const +#endif + char **argv2; while (argv[argc]) argc++; ALLOC_ARRAY(argv2, argc + 1); argv2[0] = (char *)cmd; /* full path to the script file */ From 1a6b496e1b5fbc1ddf1fe2a51bd4782e68f5d718 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 3 Jun 2016 15:38:25 -0400 Subject: [PATCH 11/26] msvc: do not pretend to support all signals This special-cases various signals that are not supported on Windows, such as SIGPIPE. These cause the UCRT to throw asserts (at least in debug mode). Signed-off-by: Jeff Hostetler --- compat/mingw.c | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/compat/mingw.c b/compat/mingw.c index 3ded32d7f8..2d45ada45b 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2425,8 +2425,34 @@ int mingw_raise(int sig) sigint_fn(SIGINT); return 0; +#if defined(_MSC_VER) + /* + * in the CRT defines 8 signals as being + * supported on the platform. Anything else causes + * an "Invalid signal or error" (which in DEBUG builds + * causes the Abort/Retry/Ignore dialog). We by-pass + * the CRT for things we already know will fail. + */ + /*case SIGINT:*/ + case SIGILL: + case SIGFPE: + case SIGSEGV: + case SIGTERM: + case SIGBREAK: + case SIGABRT: + case SIGABRT_COMPAT: + return raise(sig); + default: + errno = EINVAL; + return -1; + +#else + default: return raise(sig); + +#endif + } } From 4e4ed61aef58e357002f61bc60a8033c1e235fd9 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 10 Jan 2017 23:22:48 +0100 Subject: [PATCH 12/26] msvc: provide a main() wrapper similar to mingw_main() The MINGW version of the main() wrapper gets away with declaring symbols that were intentionally not exported. However, some of these symbols do not actually exist in MSVC's UCRT. So let's add an MSVC version of the main() wrapper that uses wmain() and imports the UNICODE argv and environment. While at it, we pass our UTF-8 version of ARGV to the real main -- rather than overwriting __argv as is done in the MINGW Version. Signed-off-by: Jeff Hostetler --- compat/mingw.c | 93 ++++++++++++++++++++++++++++++++++++++++++++++++++ compat/mingw.h | 21 ++++++++++++ 2 files changed, 114 insertions(+) diff --git a/compat/mingw.c b/compat/mingw.c index 2d45ada45b..1d967c0e0e 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2931,6 +2931,7 @@ int handle_long_path(wchar_t *path, int len, int max_path, int expand) } } +#if !defined(_MSC_VER) /* * Disable MSVCRT command line wildcard expansion (__getmainargs called from * mingw startup code, see init.c in mingw runtime). @@ -2943,6 +2944,7 @@ typedef struct { extern int __wgetmainargs(int *argc, wchar_t ***argv, wchar_t ***env, int glob, _startupinfo *si); +#endif static NORETURN void die_startup(void) { @@ -3020,6 +3022,95 @@ static void maybe_redirect_std_handles(void) GENERIC_WRITE, FILE_FLAG_NO_BUFFERING); } +#if defined(_MSC_VER) + +/* + * This routine sits between wmain() and "main" in git.exe. + * We receive UNICODE (wchar_t) values for argv and env. + * + * To be more compatible with the core git code, we convert + * argv into UTF8 and pass them directly to the "main" routine. + * + * We don't bother converting the given UNICODE env vector, + * but rather leave them in the CRT. We replaced the various + * getenv/putenv routines to pull them directly from the CRT. + * + * This is unlike the MINGW version: + * [] It does the UNICODE-2-UTF8 conversion on both sets and + * stuffs the values back into the CRT using exported symbols. + * [] It also maintains a private copy of the environment and + * tries to track external changes to it. + */ +int msc_startup(int argc, wchar_t **w_argv, wchar_t **w_env) +{ + char **my_utf8_argv = NULL, **save = NULL; + char *buffer = NULL; + int maxlen; + int k, exit_status; + +#ifdef USE_MSVC_CRTDBG + _CrtSetDbgFlag(_CRTDBG_ALLOC_MEM_DF | _CRTDBG_LEAK_CHECK_DF); +#endif + + maybe_redirect_std_handles(); + + /* determine size of argv conversion buffer */ + maxlen = wcslen(_wpgmptr); + for (k = 1; k < argc; k++) + maxlen = max(maxlen, wcslen(w_argv[k])); + + /* allocate buffer (wchar_t encodes to max 3 UTF-8 bytes) */ + maxlen = 3 * maxlen + 1; + buffer = malloc_startup(maxlen); + + /* + * Create a UTF-8 version of w_argv. Also create a "save" copy + * to remember all the string pointers because parse_options() + * will remove claimed items from the argv that we pass down. + */ + ALLOC_ARRAY(my_utf8_argv, argc + 1); + ALLOC_ARRAY(save, argc + 1); + save[0] = my_utf8_argv[0] = wcstoutfdup_startup(buffer, _wpgmptr, maxlen); + for (k = 1; k < argc; k++) + save[k] = my_utf8_argv[k] = wcstoutfdup_startup(buffer, w_argv[k], maxlen); + save[k] = my_utf8_argv[k] = NULL; + + free(buffer); + + /* fix Windows specific environment settings */ + setup_windows_environment(); + + unset_environment_variables = xstrdup("PERL5LIB"); + + /* initialize critical section for waitpid pinfo_t list */ + InitializeCriticalSection(&pinfo_cs); + InitializeCriticalSection(&phantom_symlinks_cs); + + /* set up default file mode and file modes for stdin/out/err */ + _fmode = _O_BINARY; + _setmode(_fileno(stdin), _O_BINARY); + _setmode(_fileno(stdout), _O_BINARY); + _setmode(_fileno(stderr), _O_BINARY); + + /* initialize Unicode console */ + winansi_init(); + + /* init length of current directory for handle_long_path */ + current_directory_len = GetCurrentDirectoryW(0, NULL); + + /* invoke the real main() using our utf8 version of argv. */ + exit_status = msc_main(argc, my_utf8_argv); + + for (k = 0; k < argc; k++) + free(save[k]); + free(save); + free(my_utf8_argv); + + return exit_status; +} + +#else + void mingw_startup(void) { int i, maxlen, argc; @@ -3099,6 +3190,8 @@ void mingw_startup(void) current_directory_len = GetCurrentDirectoryW(0, NULL); } +#endif + int uname(struct utsname *buf) { unsigned v = (unsigned)GetVersion(); diff --git a/compat/mingw.h b/compat/mingw.h index 5f4fa0b2f7..a85cec2d94 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -690,7 +690,26 @@ extern CRITICAL_SECTION pinfo_cs; /* * A replacement of main() that adds win32 specific initialization. + * + * Note that the end of these macros are unterminated so that the + * brace group following the use of the macro is the body of the + * function. */ +#if defined(_MSC_VER) + +int msc_startup(int argc, wchar_t **w_argv, wchar_t **w_env); +extern int msc_main(int argc, const char **argv); + +#define main(c,v) dummy_decl_msc_main(void); \ +int wmain(int my_argc, \ + wchar_t **my_w_argv, \ + wchar_t **my_w_env) \ +{ \ + return msc_startup(my_argc, my_w_argv, my_w_env); \ +} \ +int msc_main(c, v) + +#else void mingw_startup(void); #define main(c,v) dummy_decl_mingw_main(void); \ @@ -702,6 +721,8 @@ int main(int argc, const char **argv) \ } \ static int mingw_main(c,v) +#endif + /* * Used by Pthread API implementation for Windows */ From 210f5899b26e7ccdb1a115e10f83a9133faf9497 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 3 Jun 2016 15:38:25 -0400 Subject: [PATCH 13/26] msvc: do not re-declare the timespec struct VS2015's headers already declare that struct. Signed-off-by: Jeff Hostetler --- compat/mingw.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compat/mingw.h b/compat/mingw.h index a85cec2d94..1fa83f5a25 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -394,11 +394,13 @@ static inline long long filetime_to_hnsec(const FILETIME *ft) #ifndef __MINGW64_VERSION_MAJOR #define off_t off64_t #define lseek _lseeki64 +#ifndef _MSC_VER struct timespec { time_t tv_sec; long tv_nsec; }; #endif +#endif static inline void filetime_to_timespec(const FILETIME *ft, struct timespec *ts) { From 11ebbc317b13d149dc80775fdc266954aa199fc5 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 3 Jun 2016 15:38:25 -0400 Subject: [PATCH 14/26] msvc: define ftello() It is just called different in MSVC's headers. Signed-off-by: Jeff Hostetler --- compat/msvc.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compat/msvc.h b/compat/msvc.h index ae1c58ec13..b3ac4cc3af 100644 --- a/compat/msvc.h +++ b/compat/msvc.h @@ -24,6 +24,8 @@ static __inline int strcasecmp (const char *s1, const char *s2) #undef ERROR +#define ftello _ftelli64 + typedef int sigset_t; /* open for reading, writing, or both (not in fcntl.h) */ #define O_ACCMODE (_O_RDONLY | _O_WRONLY | _O_RDWR) From 4d98215ff715420e41bb661b2b01796b39df6774 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 10 Jan 2017 22:53:36 +0100 Subject: [PATCH 15/26] msvc: fix detect_msys_tty() The ntstatus.h header is only available in MINGW. Signed-off-by: Jeff Hostetler --- compat/winansi.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/compat/winansi.c b/compat/winansi.c index 7144abd71c..41221cafe5 100644 --- a/compat/winansi.c +++ b/compat/winansi.c @@ -527,7 +527,20 @@ static HANDLE swap_osfhnd(int fd, HANDLE new_handle) #ifdef DETECT_MSYS_TTY #include + +#if defined(_MSC_VER) + +typedef struct _OBJECT_NAME_INFORMATION +{ + UNICODE_STRING Name; + WCHAR NameBuffer[0]; +} OBJECT_NAME_INFORMATION, *POBJECT_NAME_INFORMATION; + +#define ObjectNameInformation 1 + +#else #include +#endif static void detect_msys_tty(int fd) { From 9aa9478b89f77d0df0c8172826d9fb695531df41 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 3 Jun 2016 16:01:35 -0400 Subject: [PATCH 16/26] cache-tree.c: avoid reusing the DEBUG constant In MSVC, the DEBUG constant is set automatically whenever compiling with debug information. This is clearly not what was intended in cache-tree.c, so let's use a less ambiguous constant there. Signed-off-by: Jeff Hostetler --- cache-tree.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/cache-tree.c b/cache-tree.c index ec23d8c03d..191dbf4b14 100644 --- a/cache-tree.c +++ b/cache-tree.c @@ -4,8 +4,8 @@ #include "tree-walk.h" #include "cache-tree.h" -#ifndef DEBUG -#define DEBUG 0 +#ifndef DEBUG_CACHE_TREE +#define DEBUG_CACHE_TREE 0 #endif struct cache_tree *cache_tree(void) @@ -110,7 +110,7 @@ static int do_invalidate_path(struct cache_tree *it, const char *path) int namelen; struct cache_tree_sub *down; -#if DEBUG +#if DEBUG_CACHE_TREE fprintf(stderr, "cache-tree invalidate <%s>\n", path); #endif @@ -394,7 +394,7 @@ static int update_one(struct cache_tree *it, strbuf_addf(&buffer, "%o %.*s%c", mode, entlen, path + baselen, '\0'); strbuf_add(&buffer, sha1, 20); -#if DEBUG +#if DEBUG_CACHE_TREE fprintf(stderr, "cache-tree update-one %o %.*s\n", mode, entlen, path + baselen); #endif @@ -417,7 +417,7 @@ static int update_one(struct cache_tree *it, strbuf_release(&buffer); it->entry_count = to_invalidate ? -1 : i - *skip_count; -#if DEBUG +#if DEBUG_CACHE_TREE fprintf(stderr, "cache-tree update-one (%d ent, %d subtree) %s\n", it->entry_count, it->subtree_nr, oid_to_hex(&it->oid)); @@ -456,7 +456,7 @@ static void write_one(struct strbuf *buffer, struct cache_tree *it, strbuf_add(buffer, path, pathlen); strbuf_addf(buffer, "%c%d %d\n", 0, it->entry_count, it->subtree_nr); -#if DEBUG +#if DEBUG_CACHE_TREE if (0 <= it->entry_count) fprintf(stderr, "cache-tree <%.*s> (%d ent, %d subtree) %s\n", pathlen, path, it->entry_count, it->subtree_nr, @@ -529,7 +529,7 @@ static struct cache_tree *read_one(const char **buffer, unsigned long *size_p) size -= 20; } -#if DEBUG +#if DEBUG_CACHE_TREE if (0 <= it->entry_count) fprintf(stderr, "cache-tree <%s> (%d ent, %d subtree) %s\n", *buffer, it->entry_count, subtree_nr, From 79f64827c00c002e5a1c8d0b036148e8f01f1907 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Thu, 9 Jun 2016 09:40:13 -0400 Subject: [PATCH 17/26] msvc: release mode PDBs and library DLLs Install required third-party DLLs next to EXEs. Build and install release mode PDBs for git executables allowing detailed stack traces in the event of crash. Signed-off-by: Jeff Hostetler --- Makefile | 22 ++++++++++++++++++++++ compat/vcbuild/Makefile | 3 ++- compat/vcbuild/scripts/clink.pl | 7 ++++--- config.mak.uname | 10 +++++++--- 4 files changed, 35 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index f056ff9bfb..a2fd540e09 100644 --- a/Makefile +++ b/Makefile @@ -2510,6 +2510,28 @@ install: all $(INSTALL) $(ALL_PROGRAMS) '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' $(INSTALL) -m 644 $(SCRIPT_LIB) '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' $(INSTALL) $(install_bindir_programs) '$(DESTDIR_SQ)$(bindir_SQ)' +ifdef MSVC + $(INSTALL) compat/vcbuild/GEN.DEPS/bin/*.dll '$(DESTDIR_SQ)$(bindir_SQ)' + $(INSTALL) compat/vcbuild/GEN.DEPS/bin/*.pdb '$(DESTDIR_SQ)$(bindir_SQ)' + # We DO NOT install the individual foo.o.pdb files because they + # have already been rolled up into the exe's pdb file. + # We DO NOT have pdb files for the builtin commands (like git-status.exe) + # because it is just a copy/hardlink of git.exe, rather than a unique binary. + $(INSTALL) git.pdb '$(DESTDIR_SQ)$(bindir_SQ)' + $(INSTALL) git-shell.pdb '$(DESTDIR_SQ)$(bindir_SQ)' + $(INSTALL) git-upload-pack.pdb '$(DESTDIR_SQ)$(bindir_SQ)' + $(INSTALL) git-credential-store.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' + $(INSTALL) git-daemon.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' + $(INSTALL) git-fast-import.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' + $(INSTALL) git-http-backend.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' + $(INSTALL) git-http-fetch.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' + $(INSTALL) git-http-push.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' + $(INSTALL) git-imap-send.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' + $(INSTALL) git-remote-http.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' + $(INSTALL) git-remote-testsvn.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' + $(INSTALL) git-sh-i18n--envsubst.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' + $(INSTALL) git-show-index.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' +endif $(MAKE) -C templates DESTDIR='$(DESTDIR_SQ)' install $(INSTALL) -d -m 755 '$(DESTDIR_SQ)$(mergetools_instdir_SQ)' $(INSTALL) -m 644 mergetools/* '$(DESTDIR_SQ)$(mergetools_instdir_SQ)' diff --git a/compat/vcbuild/Makefile b/compat/vcbuild/Makefile index b94bcd119a..cfca611dfc 100644 --- a/compat/vcbuild/Makefile +++ b/compat/vcbuild/Makefile @@ -106,13 +106,14 @@ openssl: insdir ZLIB_VER=1.2.8.8 ZLIB_ROOT=$(PKGDIR)/zlib.v140.windesktop.msvcstl.dyn.rt-dyn.$(ZLIB_VER) ZLIB_INC=$(ZLIB_ROOT)/build/native/include -ZLIB_LIB=$(ZLIB_ROOT)/lib/native/v140/windesktop/msvcstl/dyn/rt-dyn/x64/Release +ZLIB_LIB=$(ZLIB_ROOT)/lib/native/v140/windesktop/msvcstl/dyn/rt-dyn/x64/RelWithDebInfo zlib: insdir [ -d $(INST_INC)/zlib ] || mkdir $(INST_INC)/zlib cp -r $(ZLIB_INC)/* $(INST_INC)/zlib cp -r $(ZLIB_LIB)/*.lib $(INST_LIB)/ cp -r $(ZLIB_LIB)/*.dll $(INST_BIN)/ + cp -r $(ZLIB_LIB)/*.pdb $(INST_BIN)/ ################################################################## clean: diff --git a/compat/vcbuild/scripts/clink.pl b/compat/vcbuild/scripts/clink.pl index 3bbfa47bf4..a8df6bb63b 100755 --- a/compat/vcbuild/scripts/clink.pl +++ b/compat/vcbuild/scripts/clink.pl @@ -22,9 +22,12 @@ while (@ARGV) { my $file_out = shift @ARGV; if ("$file_out" =~ /exe$/) { $is_linking = 1; + # Create foo.exe and foo.pdb push(@args, "-OUT:$file_out"); } else { + # Create foo.o and foo.o.pdb push(@args, "-Fo$file_out"); + push(@args, "-Fd$file_out.pdb"); } } elsif ("$arg" eq "-lz") { push(@args, "zlib.lib"); @@ -49,12 +52,10 @@ while (@ARGV) { } if ($is_linking) { push(@args, @lflags); - # force PDB to be created. - push(@args, "-debug"); unshift(@args, "link.exe"); } else { unshift(@args, "cl.exe"); push(@args, @cflags); } -#printf("**** @args\n"); +printf("**** @args\n\n\n"); exit (system(@args) != 0); diff --git a/config.mak.uname b/config.mak.uname index 3c3dd6826b..5e086ce1a9 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -416,12 +416,16 @@ ifeq ($(uname_S),Windows) # Optionally enable memory leak reporting. # BASIC_CLFAGS += -DUSE_MSVC_CRTDBG BASIC_CFLAGS += -DPROTECT_NTFS_DEFAULT=1 + # Always give "-Zi" to the compiler and "-debug" to linker (even in + # release mode) to force a PDB to be generated (like RelWithDebInfo). + BASIC_CFLAGS += -Zi + BASIC_LDFLAGS += -debug ifndef DEBUG - BASIC_CFLAGS += -GL -Os -MD - BASIC_LDFLAGS += -LTCG + BASIC_CFLAGS += -GL -Gy -Os -Oy- -MD -DNDEBUG + BASIC_LDFLAGS += -release -LTCG /OPT:REF /OPT:ICF /INCREMENTAL:NO /DEBUGTYPE:CV,FIXUP AR += -LTCG else - BASIC_CFLAGS += -Zi -MDd -DDEBUG -D_DEBUG + BASIC_CFLAGS += -MDd -DDEBUG -D_DEBUG endif X = .exe From 13e54b56335c25f5fe2e49e133e2b3202d7cd60f Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 14 Oct 2016 06:01:39 -0700 Subject: [PATCH 18/26] msvc: respect the quiet-by-default output Signed-off-by: Johannes Schindelin --- compat/vcbuild/scripts/clink.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compat/vcbuild/scripts/clink.pl b/compat/vcbuild/scripts/clink.pl index a8df6bb63b..b67b831304 100755 --- a/compat/vcbuild/scripts/clink.pl +++ b/compat/vcbuild/scripts/clink.pl @@ -57,5 +57,5 @@ if ($is_linking) { unshift(@args, "cl.exe"); push(@args, @cflags); } -printf("**** @args\n\n\n"); +printf(STDERR "**** @args\n\n\n") if (!defined($ENV{'QUIET_GEN'})); exit (system(@args) != 0); From 699f733b12404d2c8b086648c54a9d91ebb65a88 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 12 Oct 2016 14:21:42 -0400 Subject: [PATCH 19/26] msvc: use OpenSSL's SHA-1 routines Just like 1e2ce1d (sha1: Use OpenSSL SHA1 routines on MINGW, 2016-10-12), we now use OpenSSL's SHA-1 routines instead of Git's own because OpenSSL is substantially faster as of version 1.0.2: it now uses hardware acceleration on Intel processors much more effectively. Signed-off-by: Johannes Schindelin --- config.mak.uname | 1 - 1 file changed, 1 deletion(-) diff --git a/config.mak.uname b/config.mak.uname index 5e086ce1a9..5c4e4934c8 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -371,7 +371,6 @@ ifeq ($(uname_S),Windows) NO_REGEX = YesPlease NO_GETTEXT = YesPlease NO_PYTHON = YesPlease - BLK_SHA1 = YesPlease ETAGS_TARGET = ETAGS NO_INET_PTON = YesPlease NO_INET_NTOP = YesPlease From ba43a338912dcb4b50298c0168cb44643456ae2a Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 20 Oct 2016 08:21:05 -0700 Subject: [PATCH 20/26] msvc: use libiconv Signed-off-by: Johannes Schindelin --- compat/vcbuild/Makefile | 22 +++++++++++++++++++++- compat/vcbuild/packages.config | 3 +++ config.mak.uname | 3 +-- 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/compat/vcbuild/Makefile b/compat/vcbuild/Makefile index cfca611dfc..a9d1a6b92b 100644 --- a/compat/vcbuild/Makefile +++ b/compat/vcbuild/Makefile @@ -9,7 +9,8 @@ INST_BIN=$(INST)/bin PKGDIR=GEN.PKGS ################################################################## -all: unpack expat libssh libssh_redist curl curl_redist openssl zlib +all: unpack expat libssh libssh_redist curl curl_redist openssl zlib \ + libiconv libiconv_redist unpack: [ -d $(PKGDIR) ] || mkdir $(PKGDIR) @@ -38,6 +39,25 @@ expat: insdir cp -r $(EXPAT_LIB)/* $(INST_LIB)/ +################################################################## + +LIBICONV_VER=1.14.0.11 +LIBICONV_ROOT=$(PKGDIR)/libiconv.$(LIBICONV_VER)/build/native +LIBICONV_INC=$(LIBICONV_ROOT)/include +LIBICONV_LIB=$(LIBICONV_ROOT)/lib/v110/x64/Release/dynamic/cdecl + +libiconv: insdir + cp -r $(LIBICONV_INC)/* $(INST_INC)/ + cp -r $(LIBICONV_LIB)/libiconv.lib $(INST_LIB)/iconv.lib + + +LIBICONV_REDIST_ROOT=$(PKGDIR)/libiconv.redist.$(LIBICONV_VER)/build/native +LIBICONV_REDIST_BIN=$(LIBICONV_REDIST_ROOT)/bin/v110/x64/Release/dynamic/cdecl + +libiconv_redist: insdir + cp -r $(LIBICONV_REDIST_BIN)/* $(INST_BIN)/ + + ################################################################## LIBSSH_VER=1.4.3.3 diff --git a/compat/vcbuild/packages.config b/compat/vcbuild/packages.config index e6e6d2f63f..ee01be5edb 100644 --- a/compat/vcbuild/packages.config +++ b/compat/vcbuild/packages.config @@ -5,6 +5,9 @@ + + + diff --git a/config.mak.uname b/config.mak.uname index 5c4e4934c8..31e327a419 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -350,8 +350,7 @@ ifeq ($(uname_S),Windows) NO_STRCASESTR = YesPlease NO_STRLCPY = YesPlease NO_MEMMEM = YesPlease - # NEEDS_LIBICONV = YesPlease - NO_ICONV = YesPlease + NEEDS_LIBICONV = YesPlease NO_STRTOUMAX = YesPlease NO_MKDTEMP = YesPlease # VS2015 with UCRT claims that snprintf and friends are C99 compliant, From 272edf4968b0012fa364d01030a9845d7d8a22c0 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 21 Oct 2016 02:46:59 -0700 Subject: [PATCH 21/26] msvc: work around iconv() not setting errno When compiling with MSVC, we rely on NuPkgs to provide the binaries of dependencies such as libiconv. The libiconv 1.14.0.11 package available from https://www.nuget.org/packages/libiconv seems to have a bug where it does not set errno (when we would expect it to be E2BIG). Let's simulate the error condition by taking less than 16 bytes remaining in the out buffer as an indicator that we ran out of space. While 16 might seem a bit excessive (when converting from, say, any encoding to UTF-8, 8 bytes should be fine), it is designed to be a safe margin. Signed-off-by: Johannes Schindelin --- git-compat-util.h | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/git-compat-util.h b/git-compat-util.h index 4179c8799d..3215e96fa1 100644 --- a/git-compat-util.h +++ b/git-compat-util.h @@ -281,6 +281,33 @@ extern char *gitdirname(char *); #ifndef NO_ICONV #include +#ifdef _MSC_VER +/* + * At least version 1.14.0.11 of the libiconv NuPkg at + * https://www.nuget.org/packages/libiconv/ does not set errno at all. + * + * Let's simulate it by testing whether we might have possibly run out of + * space. + */ +static inline size_t msvc_iconv(iconv_t conv, + const char **inpos, size_t *insize, + char **outpos, size_t *outsize) +{ + int saved_errno = errno; + size_t res; + + errno = ENOENT; + res = iconv(conv, inpos, insize, outpos, outsize); + if (!res) + errno = saved_errno; + else if (errno == ENOENT) + errno = *outsize < 16 ? E2BIG : 0; + + return res; +} +#undef iconv +#define iconv msvc_iconv +#endif #endif #ifndef NO_OPENSSL From 33b646c89906b8177fbadd800a406b1df23b14b5 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sun, 23 Oct 2016 01:22:20 -0700 Subject: [PATCH 22/26] t7800: fix quoting When passing a command-line to call an external diff command to the difftool, we must be prepared for paths containing special characters, e.g. backslashes in the temporary directory's path on Windows. This has been caught by running the test suite with an MSVC-built Git: in contrast to the MINGW one, it does not rewrite `$TMP` to use forward slashes instead of backslashes. Signed-off-by: Johannes Schindelin --- t/t7800-difftool.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/t7800-difftool.sh b/t/t7800-difftool.sh index 668bbee73c..f29aa8f344 100755 --- a/t/t7800-difftool.sh +++ b/t/t7800-difftool.sh @@ -332,7 +332,7 @@ test_expect_success 'difftool --extcmd cat arg1' ' test_expect_success 'difftool --extcmd cat arg2' ' echo branch >expect && git difftool --no-prompt \ - --extcmd sh\ -c\ \"cat\ \$2\" branch >actual && + --extcmd sh\ -c\ \"cat\ \\\"\$2\\\"\" branch >actual && test_cmp expect actual ' From 36356f52b14ba67a5d22c49ddbea57b2ee6e9ff3 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Tue, 8 Nov 2016 11:18:43 -0500 Subject: [PATCH 23/26] vs2015: turn on optimize-for-speed in release build Set -O2 (maximize speed) rather than -Os (favor small code) for non-debug builds. Signed-off-by: Jeff Hostetler --- config.mak.uname | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.mak.uname b/config.mak.uname index 31e327a419..f118009109 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -419,7 +419,7 @@ ifeq ($(uname_S),Windows) BASIC_CFLAGS += -Zi BASIC_LDFLAGS += -debug ifndef DEBUG - BASIC_CFLAGS += -GL -Gy -Os -Oy- -MD -DNDEBUG + BASIC_CFLAGS += -GL -Gy -O2 -Oy- -MD -DNDEBUG BASIC_LDFLAGS += -release -LTCG /OPT:REF /OPT:ICF /INCREMENTAL:NO /DEBUGTYPE:CV,FIXUP AR += -LTCG else From d53f35cec67145c1374b01e75e853b8f09b94245 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Tue, 8 Nov 2016 11:39:35 -0500 Subject: [PATCH 24/26] vs2015: teach 'make clean' to delete PDBs Teach main Makefile to also delete the generated PDB files as well as the PDB files for the various EXE files during "make MSVC=1 clean". Signed-off-by: Jeff Hostetler Signed-off-by: Jeff Hostetler --- Makefile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Makefile b/Makefile index a2fd540e09..cbf269bff1 100644 --- a/Makefile +++ b/Makefile @@ -2699,6 +2699,12 @@ endif $(RM) GIT-VERSION-FILE GIT-CFLAGS GIT-LDFLAGS GIT-BUILD-OPTIONS $(RM) GIT-USER-AGENT GIT-PREFIX $(RM) GIT-SCRIPT-DEFINES GIT-PERL-DEFINES GIT-PYTHON-VARS +ifdef MSVC + $(RM) $(patsubst %.o,%.o.pdb,$(OBJECTS)) + $(RM) $(patsubst %.exe,%.pdb,$(OTHER_PROGRAMS)) + $(RM) $(patsubst %.exe,%.pdb,$(PROGRAMS)) + $(RM) $(patsubst %.exe,%.pdb,$(TEST_PROGRAMS)) +endif .PHONY: all install profile-clean clean strip .PHONY: shell_compatibility_test please_set_SHELL_PATH_to_a_more_modern_shell From b14b8b18915cd19026d18b9cf5d9f3c6a62ea231 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Tue, 8 Nov 2016 11:41:45 -0500 Subject: [PATCH 25/26] vs2015: remove todo list item from README_vs2015.txt Remove todo list item for vs2015 build notes regarding the third party DLLs. Signed-off-by: Jeff Hostetler --- compat/vcbuild/README_VS2015.txt | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/compat/vcbuild/README_VS2015.txt b/compat/vcbuild/README_VS2015.txt index 763523fbb4..a9c6f59dc6 100644 --- a/compat/vcbuild/README_VS2015.txt +++ b/compat/vcbuild/README_VS2015.txt @@ -51,8 +51,4 @@ TODO List: developer command prompt is started. This would also allow us to switch between 32- and 64-bit tool chains. -[B] Currently, we leave the third-party DLLs we reference in - "compat/vcbuild/GEN.DEPS/bin". We need an installer - step to move them next to git.exe (or into libexec/git-core). - -[C] We need to build SLN or VCPROJ files. +[B] We need to build SLN or VCPROJ files. From 793ff1ac467508c2ceecf42042d37cf1ebb88bc2 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 9 Nov 2016 15:04:22 +0100 Subject: [PATCH 26/26] compat/vcbuild: possibly reuse Git for Windows' SDK's NuGet In Git for Windows' SDK, there is already a script to package Git for Windows as a NuGet package, downloading nuget.exe if needed. Let's just fall back to using that executable (if it is there) if nuget.exe was not found in the PATH. Signed-off-by: Johannes Schindelin --- compat/vcbuild/Makefile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/compat/vcbuild/Makefile b/compat/vcbuild/Makefile index a9d1a6b92b..255dd43084 100644 --- a/compat/vcbuild/Makefile +++ b/compat/vcbuild/Makefile @@ -7,6 +7,10 @@ INST_LIB=$(INST)/lib INST_BIN=$(INST)/bin PKGDIR=GEN.PKGS +NUGET ?= nuget.exe +ifneq ($(shell $(NUGET) help 2>/dev/null; echo $$?),0) + NUGET := /usr/src/build-extra/nuget/nuget.exe +endif ################################################################## all: unpack expat libssh libssh_redist curl curl_redist openssl zlib \ @@ -14,7 +18,7 @@ all: unpack expat libssh libssh_redist curl curl_redist openssl zlib \ unpack: [ -d $(PKGDIR) ] || mkdir $(PKGDIR) - nuget.exe restore packages.config -ConfigFile nuget.config \ + "$(NUGET)" restore packages.config -ConfigFile nuget.config \ -NonInteractive -OutputDirectory $(PKGDIR) insdir: