vcpkg: get MSVC dependencies with vcpkg rather than nuget

Dependencies such as cURL and OpenSSL are necessary to build and run
Git. Previously, we obtained those dependencies by fetching NuGet
packages.

However, it is notoriously hard to keep NuGet packages of C/C++
libraries up-to-date, as the toolsets for different Visual Studio
versions are different, and the NuGet packages would have to ship them
all.

That is the reason why the NuGet packages we use are quite old, and even
insecure in the case of cURL and OpenSSL (the versions contain known
security flaws that have been addressed by later versions for which no
NuGet packages are available).

The better way to handle this situation is to use the vcpkg system:
https://github.com/Microsoft/vcpkg

The idea is that a single Git repository contains enough supporting
files to build up-to-date versions of a large number of Open Source
libraries on demand, including cURL and OpenSSL.

We integrate this system via four new .bat files to

1) initialize the vcpkg system,
2) build the packages,
4) set up Git's Makefile system to find the build artifacts, and
3) copy the artifacts into the top-level directory

We now also completely convert the pathname exported in the
%msvc_bin_dir_msys% variable to MSYS format with forward slashes rather
than a mixture of forward and back slashes.

This solves an obscure problem observed by some developers:

    [...]
    http-push.c
        CC remote-curl.o
    remote-curl.c
        * new script parameters
        GEN git-instaweb
    sed: -e expression #7, char 155: invalid reference \2 on `s' command's RHS
    make: *** [Makefile:2023: git-instaweb] Error 1

Signed-off-by: Jeff Hostetler <jeffhost@microsoft.com>
Signed-off-by: Johannes Schindelin <johannes.schindelin@gmx.de>

.gitignore

@@ -1,7 +1,6 @@
 /GIT-BUILD-OPTIONS
 /GIT-CFLAGS
 /GIT-LDFLAGS
-/GIT-MSVC-GEN
 /GIT-PREFIX
 /GIT-PERL-DEFINES
 /GIT-PERL-HEADER

Makefile

@@ -2602,9 +2602,6 @@ ifdef GIT_INTEROP_MAKE_OPTS
 endif
 ifdef TEST_GIT_INDEX_VERSION
 	@echo TEST_GIT_INDEX_VERSION=\''$(subst ','\'',$(subst ','\'',$(TEST_GIT_INDEX_VERSION)))'\' >>$@+
-endif
-ifdef MSVC_DEPS
-	@echo MSVC_DEPS=\''$(subst ','\'',$(subst ','\'',$(MSVC_DEPS)))'\' >>$@+
 endif
 	@if cmp $@+ $@ >/dev/null 2>&1; then $(RM) $@+; else mv $@+ $@; fi
 
@@ -2754,8 +2751,6 @@ install: all
 	$(INSTALL) -m 644 $(SCRIPT_LIB) '$(DESTDIR_SQ)$(gitexec_instdir_SQ)'
 	$(INSTALL) $(install_bindir_programs) '$(DESTDIR_SQ)$(bindir_SQ)'
 ifdef MSVC
-	$(INSTALL) compat/vcbuild/GEN.DEPS/bin/*.dll '$(DESTDIR_SQ)$(bindir_SQ)'
-	$(INSTALL) compat/vcbuild/GEN.DEPS/bin/*.pdb '$(DESTDIR_SQ)$(bindir_SQ)'
 	# We DO NOT install the individual foo.o.pdb files because they
 	# have already been rolled up into the exe's pdb file.
 	# We DO NOT have pdb files for the builtin commands (like git-status.exe)
@@ -2978,9 +2973,15 @@ endif
 ifdef MSVC
 	$(RM) $(patsubst %.o,%.o.pdb,$(OBJECTS))
 	$(RM) $(patsubst %.exe,%.pdb,$(OTHER_PROGRAMS))
+	$(RM) $(patsubst %.exe,%.iobj,$(OTHER_PROGRAMS))
+	$(RM) $(patsubst %.exe,%.ipdb,$(OTHER_PROGRAMS))
 	$(RM) $(patsubst %.exe,%.pdb,$(PROGRAMS))
+	$(RM) $(patsubst %.exe,%.iobj,$(PROGRAMS))
+	$(RM) $(patsubst %.exe,%.ipdb,$(PROGRAMS))
 	$(RM) $(patsubst %.exe,%.pdb,$(TEST_PROGRAMS))
-	$(RM) GIT-MSVC-GEN
+	$(RM) $(patsubst %.exe,%.iobj,$(TEST_PROGRAMS))
+	$(RM) $(patsubst %.exe,%.ipdb,$(TEST_PROGRAMS))
+	$(RM) compat/vcbuild/MSVC-DEFS-GEN
 endif
 
 .PHONY: all install profile-clean cocciclean clean strip

compat/vcbuild/.gitignore

@@ -1 +1,3 @@
-GEN.*
+/vcpkg/
+/MSVC-DEFS-GEN
+/VCPKG-DEFS

compat/vcbuild/README

@@ -1,3 +1,54 @@
+The Steps to Build Git with VS2015 or VS2017 from the command line.
+
+1. Install the "vcpkg" open source package manager and build essential
+   third-party libaries.  The steps for this have been captured in a
+   set of convenience scripts.  These can be run from a stock Command
+   Prompt or from an SDK bash window:
+
+   $ cd <repo_root>
+   $ ./compat/vcbuild/vcpkg_install.bat
+
+   The vcpkg tools and all of the third-party sources will be installed
+   in this folder:
+      <repo_root>/compat/vcbuild/vcpkg/
+
+   A file will be created with a set of Makefile macros pointing to a
+   unified "include", "lib", and "bin" directory (release and debug) for
+   all of the required packages.  This file will be included by the main
+   Makefile:
+      <repo_root>/compat/vcbuild/MSVC-DEFS-GEN
+
+2. OPTIONALLY copy the third-party *.dll and *.pdb files into the repo
+   root to make it easier to run and debug git.exe without having to
+   manipulate your PATH.  This is especially true for debug sessions in
+   Visual Studio.
+
+   Use ONE of the following forms which should match how you want to
+   compile git.exe.
+
+   $ ./compat/vcbuild/vcpkg_copy_packages.bat debug
+   $ ./compat/vcbuild/vcpkg_copy_packages.bat release
+
+3. Build git using MSVC from an SDK bash window using one of the
+   following commands:
+
+   $ make MSVC=1
+   $ make MSVC=1 DEBUG=1
+
+================================================================
+
+Alternatively, run `make MSVC=1 vcxproj` and then load the generated
+git.sln in Visual Studio. The initial build will install the vcpkg
+system and build the dependencies automatically. This will take a while.
+
+Note that this will automatically add and commit the generated
+.sln and .vcxproj files to the repo.  You may want to drop this
+commit before submitting a Pull Request....
+
+Or maybe we should put the .sln/.vcxproj files in the .gitignores
+and not do this.  I'm not sure.
+
+================================================================
 The Steps of Build Git with VS2008
 
 1. You need the build environment, which contains the Git dependencies

compat/vcbuild/find_vs_env.bat

@@ -25,7 +25,7 @@ REM
 REM The output of this script should be written to a make "include
 REM file" and referenced by the top-level Makefile.
 REM
-REM See "config.mak.uname" (look for GIT-MSVC-GEN).
+REM See "config.mak.uname" (look for compat/vcbuild/MSVC-DEFS-GEN).
 REM ================================================================
 REM The provided command prompts are custom to each VS release and
 REM filled with lots of internal knowledge (such as Registry settings);
@@ -154,7 +154,9 @@ REM ================================================================
    REM Include DOS-style and BASH-style path for bin dir.
 
    echo msvc_bin_dir=%msvc_bin_dir%
-   echo msvc_bin_dir_msys=%msvc_bin_dir:C:=/C%
+   SET X1=%msvc_bin_dir:C:=/C%
+   SET X2=%X1:\=/%
+   echo msvc_bin_dir_msys=%X2%
 
    echo msvc_includes=%msvc_includes%
    echo msvc_libs=%msvc_libs%

compat/vcbuild/scripts/clink.pl

@@ -14,8 +14,15 @@ my @args = ();
 my @cflags = ();
 my @lflags = ();
 my $is_linking = 0;
+my $is_debug = 0;
 while (@ARGV) {
 	my $arg = shift @ARGV;
+	if ("$arg" eq "-DDEBUG") {
+	    # Some vcpkg-based libraries have different names for release
+	    # and debug versions.  This hack assumes that -DDEBUG comes
+	    # before any "-l*" flags.
+	    $is_debug = 1;
+	}
 	if ("$arg" =~ /^-[DIMGOZ]/) {
 		push(@cflags, $arg);
 	} elsif ("$arg" eq "-o") {
@@ -30,9 +37,13 @@ while (@ARGV) {
 			push(@args, "-Fd$file_out.pdb");
 		}
 	} elsif ("$arg" eq "-lz") {
+	    if ($is_debug) {
+		push(@args, "zlibd.lib");
+	    } else{
 		push(@args, "zlib.lib");
+	    }
 	} elsif ("$arg" eq "-liconv") {
-		push(@args, "iconv.lib");
+		push(@args, "libiconv.lib");
 	} elsif ("$arg" eq "-lcrypto") {
 		push(@args, "libeay32.lib");
 	} elsif ("$arg" eq "-lssl") {
@@ -40,7 +51,7 @@ while (@ARGV) {
 	} elsif ("$arg" eq "-lcurl") {
 		push(@args, "libcurl.lib");
 	} elsif ("$arg" eq "-lexpat") {
-		push(@args, "libexpat.lib");
+		push(@args, "expat.lib");
 	} elsif ("$arg" =~ /^-L/ && "$arg" ne "-LTCG") {
 		$arg =~ s/^-L/-LIBPATH:/;
 		push(@lflags, $arg);

compat/vcbuild/vcpkg_copy_dlls.bat

@@ -0,0 +1,39 @@
+@ECHO OFF
+REM ================================================================
+REM This script is an optional step. It copies the *.dll and *.pdb
+REM files (created by vcpkg_install.bat) into the top-level directory
+REM of the repo so that you can type "./git.exe" and find them without
+REM having to fixup your PATH.
+REM
+REM NOTE: Because the names of some DLL files change between DEBUG and
+REM NOTE: RELEASE builds when built using "vcpkg.exe", you will need
+REM NOTE: to copy up the corresponding version.
+REM ================================================================
+
+	SETLOCAL EnableDelayedExpansion
+
+	@FOR /F "delims=" %%D IN ("%~dp0") DO @SET cwd=%%~fD
+	cd %cwd%
+
+	SET arch=x64-windows
+	SET inst=%cwd%vcpkg\installed\%arch%
+
+	IF [%1]==[release] (
+		echo Copying RELEASE mode DLLs to repo root...
+	) ELSE IF [%1]==[debug] (
+		SET inst=%inst%\debug
+		echo Copying DEBUG mode DLLs to repo root...
+	) ELSE (
+		echo ERROR: Invalid argument.
+		echo Usage: %~0 release
+		echo Usage: %~0 debug
+		EXIT /B 1
+	)
+
+	xcopy /e/s/v/y %inst%\bin\*.dll ..\..\
+	xcopy /e/s/v/y %inst%\bin\*.pdb ..\..\
+
+	xcopy /e/s/v/y %inst%\bin\*.dll ..\..\t\helper\
+	xcopy /e/s/v/y %inst%\bin\*.pdb ..\..\t\helper\
+
+	EXIT /B 0

compat/vcbuild/vcpkg_install.bat

@@ -0,0 +1,81 @@
+@ECHO OFF
+REM ================================================================
+REM This script installs the "vcpkg" source package manager and uses
+REM it to build the third-party libraries that git requires when it
+REM is built using MSVC.
+REM
+REM [1] Install VCPKG.
+REM     [a] Create <root>/compat/vcbuild/vcpkg/
+REM     [b] Download "vcpkg".
+REM     [c] Compile using the currently installed version of VS.
+REM     [d] Create <root>/compat/vcbuild/vcpkg/vcpkg.exe
+REM
+REM [2] Install third-party libraries.
+REM     [a] Download each (which may also install CMAKE).
+REM     [b] Compile in RELEASE mode and install in:
+REM         vcpkg/installed/<arch>/{bin,lib}
+REM     [c] Compile in DEBUG mode and install in:
+REM         vcpkg/installed/<arch>/debug/{bin,lib}
+REM     [d] Install headers in:
+REM         vcpkg/installed/<arch>/include
+REM
+REM [3] Create a set of MAKE definitions for the top-level
+REM     Makefile to allow "make MSVC=1" to find the above
+REM     third-party libraries.
+REM     [a] Write vcpkg/VCPGK-DEFS
+REM
+REM https://blogs.msdn.microsoft.com/vcblog/2016/09/19/vcpkg-a-tool-to-acquire-and-build-c-open-source-libraries-on-windows/
+REM https://github.com/Microsoft/vcpkg
+REM https://vcpkg.readthedocs.io/en/latest/
+REM ================================================================
+
+	SETLOCAL EnableDelayedExpansion
+
+	@FOR /F "delims=" %%D IN ("%~dp0") DO @SET cwd=%%~fD
+	cd %cwd%
+
+	dir vcpkg\vcpkg.exe >nul 2>nul && GOTO :install_libraries
+
+	echo Fetching vcpkg in %cwd%vcpkg
+	git.exe clone https://github.com/Microsoft/vcpkg vcpkg
+	IF ERRORLEVEL 1 ( EXIT /B 1 )
+
+	cd vcpkg
+	echo Building vcpkg
+	powershell -exec bypass scripts\bootstrap.ps1
+	IF ERRORLEVEL 1 ( EXIT /B 1 )
+
+	echo Successfully installed %cwd%vcpkg\vcpkg.exe
+
+:install_libraries
+	SET arch=x64-windows
+
+	echo Installing third-party libraries...
+	FOR %%i IN (zlib expat libiconv openssl libssh2 curl) DO (
+	    cd %cwd%vcpkg
+	    SET p="packages\%%i_%arch%"
+	    IF NOT EXIST "%p%" CALL :sub__install_one %%i
+	    IF ERRORLEVEL 1 ( EXIT /B 1 )
+	)
+
+:install_defines
+	cd %cwd%
+	SET inst=%cwd%vcpkg\installed\%arch%
+
+	echo vcpkg_inc=-I"%inst%\include">VCPKG-DEFS
+	echo vcpkg_rel_lib=-L"%inst%\lib">>VCPKG-DEFS
+	echo vcpkg_rel_bin="%inst%\bin">>VCPKG-DEFS
+	echo vcpkg_dbg_lib=-L"%inst%\debug\lib">>VCPKG-DEFS
+	echo vcpkg_dbg_bin="%inst%\debug\bin">>VCPKG-DEFS
+
+	EXIT /B 0
+
+
+:sub__install_one
+	echo     Installing package %1...
+
+	.\vcpkg.exe install %1:%arch%
+	IF ERRORLEVEL 1 ( EXIT /B 1 )
+
+	echo     Finished %1
+	goto :EOF

config.mak.uname

@@ -18,9 +18,16 @@ ifdef MSVC
 
 	# Generate and include makefile variables that point to the
 	# currently installed set of MSVC command line tools.
-GIT-MSVC-GEN: ./compat/vcbuild/find_vs_env.bat
-	@./compat/vcbuild/find_vs_env.bat | sed 's|\\|/|g' >GIT-MSVC-GEN
--include GIT-MSVC-GEN
+compat/vcbuild/MSVC-DEFS-GEN: compat/vcbuild/find_vs_env.bat
+	@"$<" | tr '\\' / >"$@"
+include compat/vcbuild/MSVC-DEFS-GEN
+
+	# See if vcpkg and the vcpkg-build versions of the third-party
+	# libraries that we use are installed.  We include the result
+	# to get $(vcpkg_*) variables defined for the Makefile.
+compat/vcbuild/VCPKG-DEFS: compat/vcbuild/vcpkg_install.bat
+	@"$<"
+include compat/vcbuild/VCPKG-DEFS
 endif
 
 # We choose to avoid "if .. else if .. else .. endif endif"
@@ -427,13 +434,13 @@ ifeq ($(uname_S),Windows)
 	EXTLIBS = user32.lib advapi32.lib shell32.lib wininet.lib ws2_32.lib invalidcontinue.obj kernel32.lib ntdll.lib
 	PTHREAD_LIBS =
 	lib =
-	# Path to the unpacked third-party libraries
-	MSVC_DEPS = compat/vcbuild/GEN.DEPS
-	BASIC_CFLAGS += \
-		     -I$(MSVC_DEPS)/include -I$(MSVC_DEPS)/include/expat -I$(MSVC_DEPS)/include/zlib \
-		     -L$(MSVC_DEPS)/lib \
-		     $(sdk_includes) $(sdk_libs) \
-		     $(msvc_includes) $(msvc_libs)
+	BASIC_CFLAGS += $(vcpkg_inc) $(sdk_includes) $(msvc_includes)
+ifndef DEBUG
+	BASIC_CFLAGS += $(vcpkg_rel_lib)
+else
+	BASIC_CFLAGS += $(vcpkg_dbg_lib)
+endif
+	BASIC_CFLAGS += $(sdk_libs) $(msvc_libs)
 
 	# Optionally enable memory leak reporting.
 	# BASIC_CLFAGS += -DUSE_MSVC_CRTDBG

t/test-lib.sh

@@ -57,9 +57,6 @@ fi
 . "$GIT_BUILD_DIR"/GIT-BUILD-OPTIONS
 export PERL_PATH SHELL_PATH
 
-test -z "$MSVC_DEPS" ||
-PATH="$GIT_BUILD_DIR/$MSVC_DEPS/bin:$PATH"
-
 ################################################################
 # It appears that people try to run tests without building...
 test -n "$GIT_TEST_INSTALLED" || "$GIT_BUILD_DIR/git$X" >/dev/null ||