From 33fba9fc809c16cfb79791592fd5bf2919bb4f00 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 10 Jan 2017 21:53:00 +0100 Subject: [PATCH 001/288] mingw: add experimental feature to redirect standard handles On Windows, file handles need to be marked inheritable when they need to be used as standard input/output/error handles for a newly spawned process. The problem with that, of course, is that the "inheritable" flag is global and therefore can wreak havoc with highly multi-threaded applications: other spawned processes will *also* inherit those file handles, despite having *other* input/output/error handles, and never close the former handles because they do not know about them. Let's introduce a set of environment variables (GIT_REDIRECT_STDIN and friends) that point to files, or even better, named pipes and that are used by the spawned Git process. This helps work around above-mentioned issue: those named pipes will be opened in a non-inheritable way upon startup, and no handles are passed around. Signed-off-by: Johannes Schindelin --- compat/mingw.c | 43 +++++++++++++++++++++++++++++++++++++++++++ t/t0001-init.sh | 6 ++++++ 2 files changed, 49 insertions(+) diff --git a/compat/mingw.c b/compat/mingw.c index 3fbfda5978..d2389d50ba 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2182,6 +2182,47 @@ static char *wcstoutfdup_startup(char *buffer, const wchar_t *wcs, size_t len) return memcpy(malloc_startup(len), buffer, len); } +static void maybe_redirect_std_handle(const wchar_t *key, DWORD std_id, int fd, + DWORD desired_access, DWORD flags) +{ + DWORD create_flag = fd ? OPEN_ALWAYS : OPEN_EXISTING; + wchar_t buf[MAX_LONG_PATH]; + DWORD max = ARRAY_SIZE(buf); + HANDLE handle; + DWORD ret = GetEnvironmentVariableW(key, buf, max); + + if (!ret || ret >= max) + return; + + /* make sure this does not leak into child processes */ + SetEnvironmentVariableW(key, NULL); + if (!wcscmp(buf, L"off")) { + close(fd); + handle = GetStdHandle(std_id); + if (handle != INVALID_HANDLE_VALUE) + CloseHandle(handle); + return; + } + handle = CreateFileW(buf, desired_access, 0, NULL, create_flag, + flags, NULL); + if (handle != INVALID_HANDLE_VALUE) { + int new_fd = _open_osfhandle((intptr_t)handle, O_BINARY); + SetStdHandle(std_id, handle); + dup2(new_fd, fd); + close(new_fd); + } +} + +static void maybe_redirect_std_handles(void) +{ + maybe_redirect_std_handle(L"GIT_REDIRECT_STDIN", STD_INPUT_HANDLE, 0, + GENERIC_READ, FILE_ATTRIBUTE_NORMAL); + maybe_redirect_std_handle(L"GIT_REDIRECT_STDOUT", STD_OUTPUT_HANDLE, 1, + GENERIC_WRITE, FILE_ATTRIBUTE_NORMAL); + maybe_redirect_std_handle(L"GIT_REDIRECT_STDERR", STD_ERROR_HANDLE, 2, + GENERIC_WRITE, FILE_FLAG_NO_BUFFERING); +} + void mingw_startup(void) { int i, maxlen, argc; @@ -2189,6 +2230,8 @@ void mingw_startup(void) wchar_t **wenv, **wargv; _startupinfo si; + maybe_redirect_std_handles(); + /* get wide char arguments and environment */ si.newmode = 0; if (__wgetmainargs(&argc, &wargv, &wenv, _CRT_glob, &si) < 0) diff --git a/t/t0001-init.sh b/t/t0001-init.sh index b8fc588b19..06fbefdb1e 100755 --- a/t/t0001-init.sh +++ b/t/t0001-init.sh @@ -410,4 +410,10 @@ test_expect_success 're-init from a linked worktree' ' ) ' +test_expect_success MINGW 'redirect std handles' ' + GIT_REDIRECT_STDOUT=output.txt git rev-parse --git-dir && + test .git = "$(cat output.txt)" && + test -z "$(GIT_REDIRECT_STDOUT=off git rev-parse --git-dir)" +' + test_done From 6a96dc257ee8773bd24dc33eb6511fae43acb922 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 9 Jan 2017 11:12:49 +0100 Subject: [PATCH 002/288] mingw: special-case GIT_REDIRECT_STDERR=2>&1 The "2>&1" notation in POSIX shells implies that stderr is redirected to stdout. Let's special-case this value for the environment variable GIT_REDIRECT_STDERR to allow writing to the same destination as stdout. The functionality was suggested by Jeff Hostetler. Signed-off-by: Johannes Schindelin --- compat/mingw.c | 15 +++++++++++++++ t/t0001-init.sh | 8 +++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/compat/mingw.c b/compat/mingw.c index d2389d50ba..40fd6e00c0 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2203,6 +2203,21 @@ static void maybe_redirect_std_handle(const wchar_t *key, DWORD std_id, int fd, CloseHandle(handle); return; } + if (std_id == STD_ERROR_HANDLE && !wcscmp(buf, L"2>&1")) { + handle = GetStdHandle(STD_OUTPUT_HANDLE); + if (handle == INVALID_HANDLE_VALUE) { + close(fd); + handle = GetStdHandle(std_id); + if (handle != INVALID_HANDLE_VALUE) + CloseHandle(handle); + } else { + int new_fd = _open_osfhandle((intptr_t)handle, O_BINARY); + SetStdHandle(std_id, handle); + dup2(new_fd, fd); + /* do *not* close the new_fd: that would close stdout */ + } + return; + } handle = CreateFileW(buf, desired_access, 0, NULL, create_flag, flags, NULL); if (handle != INVALID_HANDLE_VALUE) { diff --git a/t/t0001-init.sh b/t/t0001-init.sh index 06fbefdb1e..63ff011fc0 100755 --- a/t/t0001-init.sh +++ b/t/t0001-init.sh @@ -413,7 +413,13 @@ test_expect_success 're-init from a linked worktree' ' test_expect_success MINGW 'redirect std handles' ' GIT_REDIRECT_STDOUT=output.txt git rev-parse --git-dir && test .git = "$(cat output.txt)" && - test -z "$(GIT_REDIRECT_STDOUT=off git rev-parse --git-dir)" + test -z "$(GIT_REDIRECT_STDOUT=off git rev-parse --git-dir)" && + test_must_fail env \ + GIT_REDIRECT_STDOUT=output.txt \ + GIT_REDIRECT_STDERR="2>&1" \ + git rev-parse --git-dir --verify refs/invalid && + printf ".git\nfatal: Needed a single revision\n" >expect && + test_cmp expect output.txt ' test_done From 2250570d8f933d0e49359dfc2deaf63258405eb2 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 10 Jan 2017 12:12:50 +0100 Subject: [PATCH 003/288] mingw: document the experimental standard handle redirection This feature is still highly experimental and has not even been contributed to the Git mailing list yet: the feature still needs to be battle-tested more. Signed-off-by: Johannes Schindelin --- Documentation/git.txt | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/Documentation/git.txt b/Documentation/git.txt index bdab261515..dfc52aad24 100644 --- a/Documentation/git.txt +++ b/Documentation/git.txt @@ -1186,6 +1186,23 @@ of clones and fetches. - any external helpers are named by their protocol (e.g., use `hg` to allow the `git-remote-hg` helper) +`GIT_REDIRECT_STDIN`:: +`GIT_REDIRECT_STDOUT`:: +`GIT_REDIRECT_STDERR`:: + (EXPERIMENTAL) Windows-only: allow redirecting the standard + input/output/error handles. This is particularly useful in + multi-threaded applications where the canonical way to pass + standard handles via `CreateProcess()` is not an option because + it would require the handles to be marked inheritable (and + consequently *every* spawned process would inherit them, possibly + blocking regular Git operations). The primary intended use case + is to use named pipes for communication. ++ +Two special values are supported: `off` will simply close the +corresponding standard handle, and if `GIT_REDIRECT_STDERR` is +`2>&1`, standard error will be redirected to the same handle as +standard output. + Discussion[[Discussion]] ------------------------ From da49ec540f7f61d1b6b909b77a12f9e8e9dbb042 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 24 Jul 2015 16:01:09 +0200 Subject: [PATCH 004/288] Add a Code of Conduct It is better to state clearly expectations and intentions than to assume quietly that everybody agrees. This Code of Conduct is the Open Code of Conduct as per http://todogroup.org/opencodeofconduct/ (the only modifications are the adjustments to reflect that there is no "response team" in addition to the Git for Windows maintainer, and the addition of the link to the Open Code of Conduct itself). [Completely revamped, based on the Covenant 1.4 by Brendan Forster] Signed-off-by: Johannes Schindelin --- CONTRIBUTING.md | 76 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000000..590c642cfb --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,76 @@ +# Git for Windows Code of Conduct + +This code of conduct outlines our expectations for participants within the **Git for Windows** community, as well as steps to reporting unacceptable behavior. We are committed to providing a welcoming and inspiring community for all and expect our code of conduct to be honored. Anyone who violates this code of conduct may be banned from the community. + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, gender identity and expression, level of experience, +nationality, personal appearance, race, religion, or sexual identity and +orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or +advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting the project team at **johannes.schindelin@gmx.de**. All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. The project team is +obligated to maintain confidentiality with regard to the reporter of an incident. +Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, +available at [http://contributor-covenant.org/version/1/4][version] + +[homepage]: http://contributor-covenant.org +[version]: http://contributor-covenant.org/version/1/4/ From b49754f0e4888d509e76556a5c226f631900b3db Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 10 Jan 2014 16:16:03 -0600 Subject: [PATCH 005/288] README.md: Add a Windows-specific preamble Signed-off-by: Johannes Schindelin --- README.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/README.md b/README.md index c0cd5580ea..b226e5d5a3 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,19 @@ +Git for Windows +=============== + +[![Join the chat at https://gitter.im/git-for-windows/git](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/git-for-windows/git?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) + +This is [Git for Windows](http://git-for-windows.github.io/), the Windows port +of [Git](http://git-scm.com/). + +The Git for Windows project is run using a [governance +model](http://git-for-windows.github.io/governance-model.html). If you +encounter problems, you can report them as [GitHub +issues](https://github.com/git-for-windows/git/issues), discuss them on Git +for Windows' [Google Group](http://groups.google.com/group/git-for-windows), +and [contribute bug +fixes](https://github.com/git-for-windows/git/wiki/How-to-participate). + Git - fast, scalable, distributed revision control system ========================================================= From b7af94da974eedb5ca6119c7e588806c0714d24c Mon Sep 17 00:00:00 2001 From: Heiko Voigt Date: Thu, 18 Feb 2010 18:27:27 +0100 Subject: [PATCH 006/288] Revert "git-gui: set GIT_DIR and GIT_WORK_TREE after setup" This reverts commit a9fa11fe5bd5978bb175b3b5663f6477a345d428. --- git-gui/git-gui.sh | 43 +++++++++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/git-gui/git-gui.sh b/git-gui/git-gui.sh index 5bc21b878d..ffd29daf53 100755 --- a/git-gui/git-gui.sh +++ b/git-gui/git-gui.sh @@ -1326,9 +1326,6 @@ if {[lindex $_reponame end] eq {.git}} { set _reponame [lindex $_reponame end] } -set env(GIT_DIR) $_gitdir -set env(GIT_WORK_TREE) $_gitworktree - ###################################################################### ## ## global init @@ -2153,7 +2150,7 @@ set starting_gitk_msg [mc "Starting gitk... please wait..."] proc do_gitk {revs {is_submodule false}} { global current_diff_path file_states current_diff_side ui_index - global _gitdir _gitworktree + global _gitworktree # -- Always start gitk through whatever we were loaded with. This # lets us bypass using shell process on Windows systems. @@ -2165,12 +2162,19 @@ proc do_gitk {revs {is_submodule false}} { } else { global env + if {[info exists env(GIT_DIR)]} { + set old_GIT_DIR $env(GIT_DIR) + } else { + set old_GIT_DIR {} + } + set pwd [pwd] if {!$is_submodule} { if {![is_bare]} { cd $_gitworktree } + set env(GIT_DIR) [file normalize [gitdir]] } else { cd $current_diff_path if {$revs eq {--}} { @@ -2191,18 +2195,15 @@ proc do_gitk {revs {is_submodule false}} { } set revs $old_sha1...$new_sha1 } - # GIT_DIR and GIT_WORK_TREE for the submodule are not the ones - # we've been using for the main repository, so unset them. - # TODO we could make life easier (start up faster?) for gitk - # by setting these to the appropriate values to allow gitk - # to skip the heuristics to find their proper value - unset env(GIT_DIR) - unset env(GIT_WORK_TREE) + if {[info exists env(GIT_DIR)]} { + unset env(GIT_DIR) + } } eval exec $cmd $revs "--" "--" & - set env(GIT_DIR) $_gitdir - set env(GIT_WORK_TREE) $_gitworktree + if {$old_GIT_DIR ne {}} { + set env(GIT_DIR) $old_GIT_DIR + } cd $pwd ui_status $::starting_gitk_msg @@ -2223,20 +2224,22 @@ proc do_git_gui {} { error_popup [mc "Couldn't find git gui in PATH"] } else { global env - global _gitdir _gitworktree - # see note in do_gitk about unsetting these vars when - # running tools in a submodule - unset env(GIT_DIR) - unset env(GIT_WORK_TREE) + if {[info exists env(GIT_DIR)]} { + set old_GIT_DIR $env(GIT_DIR) + unset env(GIT_DIR) + } else { + set old_GIT_DIR {} + } set pwd [pwd] cd $current_diff_path eval exec $exe gui & - set env(GIT_DIR) $_gitdir - set env(GIT_WORK_TREE) $_gitworktree + if {$old_GIT_DIR ne {}} { + set env(GIT_DIR) $old_GIT_DIR + } cd $pwd ui_status $::starting_gitk_msg From bf82f39c53ffa0e38acc44b90c09690cdb3aae7b Mon Sep 17 00:00:00 2001 From: Brendan Forster Date: Thu, 18 Feb 2016 21:29:50 +1100 Subject: [PATCH 007/288] Add an issue template With improvements by Clive Chan, Adric Norris, Ben Bodenmiller and Philip Oakley. Signed-off-by: Clive Chan Signed-off-by: Adric Norris Signed-off-by: Ben Bodenmiller Signed-off-by: Philip Oakley Signed-off-by: Brendan Forster Signed-off-by: Johannes Schindelin --- .github/ISSUE_TEMPLATE.md | 63 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE.md diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md new file mode 100644 index 0000000000..75edc4d5b4 --- /dev/null +++ b/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,63 @@ + - [ ] I was not able to find an [open](https://github.com/git-for-windows/git/issues?q=is%3Aopen) or [closed](https://github.com/git-for-windows/git/issues?q=is%3Aclosed) issue matching what I'm seeing + +### Setup + + - Which version of Git for Windows are you using? Is it 32-bit or 64-bit? + +``` +$ git --version --build-options + +** insert your machine's response here ** +``` + + - Which version of Windows are you running? Vista, 7, 8, 10? Is it 32-bit or 64-bit? + +``` +$ cmd.exe /c ver + +** insert your machine's response here ** +``` + + - What options did you set as part of the installation? Or did you choose the + defaults? + +``` +# One of the following: +> type "C:\Program Files\Git\etc\install-options.txt" +> type "C:\Program Files (x86)\Git\etc\install-options.txt" +> type "%USERPROFILE%\AppData\Local\Programs\Git\etc\install-options.txt" +$ cat /etc/install-options.txt + +** insert your machine's response here ** +``` + + - Any other interesting things about your environment that might be related + to the issue you're seeing? + +** insert your response here ** + +### Details + + - Which terminal/shell are you running Git from? e.g Bash/CMD/PowerShell/other + +** insert your response here ** + + - What commands did you run to trigger this issue? If you can provide a + [Minimal, Complete, and Verifiable example](http://stackoverflow.com/help/mcve) + this will help us understand the issue. + +``` +** insert your commands here ** +``` + - What did you expect to occur after running these commands? + +** insert here ** + + - What actually happened instead? + +** insert here ** + + - If the problem was occurring with a specific repository, can you provide the + URL to that repository to help us with testing? + +** insert URL here ** From 4a86ad851f71e3e5f1175d737d315ee777862c72 Mon Sep 17 00:00:00 2001 From: Heiko Voigt Date: Sun, 21 Feb 2010 21:05:04 +0100 Subject: [PATCH 008/288] git-gui: provide question helper for retry fallback on Windows Make use of the new environment variable GIT_ASK_YESNO to support the recently implemented fallback in case unlink, rename or rmdir fail for files in use on Windows. The added dialog will present a yes/no question to the the user which will currently be used by the windows compat layer to let the user retry a failed file operation. Signed-off-by: Heiko Voigt --- git-gui/Makefile | 2 ++ git-gui/git-gui--askyesno | 51 +++++++++++++++++++++++++++++++++++++++ git-gui/git-gui.sh | 3 +++ 3 files changed, 56 insertions(+) create mode 100755 git-gui/git-gui--askyesno diff --git a/git-gui/Makefile b/git-gui/Makefile index fe30be38dc..85633b73df 100644 --- a/git-gui/Makefile +++ b/git-gui/Makefile @@ -291,6 +291,7 @@ install: all $(QUIET)$(INSTALL_D0)'$(DESTDIR_SQ)$(gitexecdir_SQ)' $(INSTALL_D1) $(QUIET)$(INSTALL_X0)git-gui $(INSTALL_X1) '$(DESTDIR_SQ)$(gitexecdir_SQ)' $(QUIET)$(INSTALL_X0)git-gui--askpass $(INSTALL_X1) '$(DESTDIR_SQ)$(gitexecdir_SQ)' + $(QUIET)$(INSTALL_X0)git-gui--askyesno $(INSTALL_X1) '$(DESTDIR_SQ)$(gitexecdir_SQ)' $(QUIET)$(foreach p,$(GITGUI_BUILT_INS), $(INSTALL_L0)'$(DESTDIR_SQ)$(gitexecdir_SQ)/$p' $(INSTALL_L1)'$(DESTDIR_SQ)$(gitexecdir_SQ)/git-gui' $(INSTALL_L2)'$(DESTDIR_SQ)$(gitexecdir_SQ)/$p' $(INSTALL_L3) &&) true ifdef GITGUI_WINDOWS_WRAPPER $(QUIET)$(INSTALL_R0)git-gui.tcl $(INSTALL_R1) '$(DESTDIR_SQ)$(gitexecdir_SQ)' @@ -309,6 +310,7 @@ uninstall: $(QUIET)$(CLEAN_DST) '$(DESTDIR_SQ)$(gitexecdir_SQ)' $(QUIET)$(REMOVE_F0)'$(DESTDIR_SQ)$(gitexecdir_SQ)'/git-gui $(REMOVE_F1) $(QUIET)$(REMOVE_F0)'$(DESTDIR_SQ)$(gitexecdir_SQ)'/git-gui--askpass $(REMOVE_F1) + $(QUIET)$(REMOVE_F0)'$(DESTDIR_SQ)$(gitexecdir_SQ)'/git-gui--askyesno $(REMOVE_F1) $(QUIET)$(foreach p,$(GITGUI_BUILT_INS), $(REMOVE_F0)'$(DESTDIR_SQ)$(gitexecdir_SQ)'/$p $(REMOVE_F1) &&) true ifdef GITGUI_WINDOWS_WRAPPER $(QUIET)$(REMOVE_F0)'$(DESTDIR_SQ)$(gitexecdir_SQ)'/git-gui.tcl $(REMOVE_F1) diff --git a/git-gui/git-gui--askyesno b/git-gui/git-gui--askyesno new file mode 100755 index 0000000000..2a6e6fd111 --- /dev/null +++ b/git-gui/git-gui--askyesno @@ -0,0 +1,51 @@ +#!/bin/sh +# Tcl ignores the next line -*- tcl -*- \ +exec wish "$0" -- "$@" + +# This is an implementation of a simple yes no dialog +# which is injected into the git commandline by git gui +# in case a yesno question needs to be answered. + +set NS {} +set use_ttk [package vsatisfies [package provide Tk] 8.5] +if {$use_ttk} { + set NS ttk +} + +if {$argc < 1} { + puts stderr "Usage: $argv0 " + exit 1 +} else { + set prompt [join $argv " "] +} + +${NS}::frame .t +${NS}::label .t.m -text $prompt -justify center -width 40 +.t.m configure -wraplength 400 +pack .t.m -side top -fill x -padx 20 -pady 20 -expand 1 +pack .t -side top -fill x -ipadx 20 -ipady 20 -expand 1 + +${NS}::frame .b +${NS}::frame .b.left -width 200 +${NS}::button .b.yes -text Yes -command yes +${NS}::button .b.no -text No -command no + + +pack .b.left -side left -expand 1 -fill x +pack .b.yes -side left -expand 1 +pack .b.no -side right -expand 1 -ipadx 5 +pack .b -side bottom -fill x -ipadx 20 -ipady 15 + +bind . {exit 0} +bind . {exit 1} + +proc no {} { + exit 1 +} + +proc yes {} { + exit 0 +} + +wm title . "Question?" +tk::PlaceWindow . diff --git a/git-gui/git-gui.sh b/git-gui/git-gui.sh index ffd29daf53..21dc3cd0b8 100755 --- a/git-gui/git-gui.sh +++ b/git-gui/git-gui.sh @@ -1230,6 +1230,9 @@ set have_tk85 [expr {[package vcompare $tk_version "8.5"] >= 0}] if {![info exists env(SSH_ASKPASS)]} { set env(SSH_ASKPASS) [gitexec git-gui--askpass] } +if {![info exists env(GIT_ASK_YESNO)]} { + set env(GIT_ASK_YESNO) [gitexec git-gui--askyesno] +} ###################################################################### ## From beaa0c35891716af828452fe2be764c2caebd9c4 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 23 Jul 2010 18:06:05 +0200 Subject: [PATCH 009/288] git gui: set GIT_ASKPASS=git-gui--askpass if not set yet Signed-off-by: Johannes Schindelin --- git-gui/git-gui.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/git-gui/git-gui.sh b/git-gui/git-gui.sh index 21dc3cd0b8..0787c10fcd 100755 --- a/git-gui/git-gui.sh +++ b/git-gui/git-gui.sh @@ -1230,6 +1230,9 @@ set have_tk85 [expr {[package vcompare $tk_version "8.5"] >= 0}] if {![info exists env(SSH_ASKPASS)]} { set env(SSH_ASKPASS) [gitexec git-gui--askpass] } +if {![info exists env(GIT_ASKPASS)]} { + set env(GIT_ASKPASS) [gitexec git-gui--askpass] +} if {![info exists env(GIT_ASK_YESNO)]} { set env(GIT_ASK_YESNO) [gitexec git-gui--askyesno] } From 2169f2ba28b8320e889893d1cbf4fcb6f150f613 Mon Sep 17 00:00:00 2001 From: Thomas Klaeger Date: Sun, 18 Oct 2015 22:31:36 +0200 Subject: [PATCH 010/288] git-gui (Windows): use git-bash.exe if it is available MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Git for Windows 2.x ships with an executable that starts the Git Bash with all the environment variables and what not properly set up. It is also adjusted according to the Terminal emulator option chosen when installing Git for Windows (while `bash.exe --login -i` would always launch with Windows' default console). So let's use that executable (usually C:\Program Files\Git\git-bash.exe) instead of `bash.exe --login -i` if its presence was detected. This fixes https://github.com/git-for-windows/git/issues/490 Signed-off-by: Thomas Kläger Signed-off-by: Johannes Schindelin --- git-gui/git-gui.sh | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/git-gui/git-gui.sh b/git-gui/git-gui.sh index 0787c10fcd..494ad612fe 100755 --- a/git-gui/git-gui.sh +++ b/git-gui/git-gui.sh @@ -2691,10 +2691,18 @@ if {![is_bare]} { } if {[is_Windows]} { + # Use /git-bash.exe if available + set normalized [file normalize $::argv0] + regsub "/mingw../libexec/git-core/git-gui$" \ + $normalized "/git-bash.exe" cmdLine + if {$cmdLine != $normalized && [file exists $cmdLine]} { + set cmdLine [list "Git Bash" $cmdLine &] + } else { + set cmdLine [list "Git Bash" bash --login -l &] + } .mbar.repository add command \ -label [mc "Git Bash"] \ - -command {eval exec [auto_execok start] \ - [list "Git Bash" bash --login -l &]} + -command {eval exec [auto_execok start] $cmdLine} } if {[is_Windows] || ![is_bare]} { From 6f7b120c4fe754d5ec5a12d28d6bc31eb342fc15 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sat, 4 Feb 2012 21:54:36 +0100 Subject: [PATCH 011/288] gitk: Unicode file name support Assumes file names in git tree objects are UTF-8 encoded. On most unix systems, the system encoding (and thus the TCL system encoding) will be UTF-8, so file names will be displayed correctly. On Windows, it is impossible to set the system encoding to UTF-8. Changing the TCL system encoding (via 'encoding system ...', e.g. in the startup code) is explicitly discouraged by the TCL docs. Change gitk functions dealing with file names to always convert from and to UTF-8. Signed-off-by: Karsten Blees Signed-off-by: Johannes Schindelin --- gitk-git/gitk | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/gitk-git/gitk b/gitk-git/gitk index 805a1c7030..fbd36a76e8 100755 --- a/gitk-git/gitk +++ b/gitk-git/gitk @@ -7629,7 +7629,7 @@ proc gettreeline {gtf id} { if {[string index $fname 0] eq "\""} { set fname [lindex $fname 0] } - set fname [encoding convertfrom $fname] + set fname [encoding convertfrom utf-8 $fname] lappend treefilelist($id) $fname } if {![eof $gtf]} { @@ -7891,7 +7891,7 @@ proc gettreediffline {gdtf ids} { if {[string index $file 0] eq "\""} { set file [lindex $file 0] } - set file [encoding convertfrom $file] + set file [encoding convertfrom utf-8 $file] if {$file ne [lindex $treediff end]} { lappend treediff $file lappend sublist $file @@ -8036,7 +8036,7 @@ proc makediffhdr {fname ids} { global ctext curdiffstart treediffs diffencoding global ctext_file_names jump_to_here targetline diffline - set fname [encoding convertfrom $fname] + set fname [encoding convertfrom utf-8 $fname] set diffencoding [get_path_encoding $fname] set i [lsearch -exact $treediffs($ids) $fname] if {$i >= 0} { @@ -8093,7 +8093,7 @@ proc parseblobdiffline {ids line} { if {![string compare -length 5 "diff " $line]} { if {![regexp {^diff (--cc|--git) } $line m type]} { - set line [encoding convertfrom $line] + set line [encoding convertfrom utf-8 $line] $ctext insert end "$line\n" hunksep continue } @@ -8140,7 +8140,7 @@ proc parseblobdiffline {ids line} { makediffhdr $fname $ids } elseif {![string compare -length 16 "* Unmerged path " $line]} { - set fname [encoding convertfrom [string range $line 16 end]] + set fname [encoding convertfrom utf-8 [string range $line 16 end]] $ctext insert end "\n" set curdiffstart [$ctext index "end - 1c"] lappend ctext_file_names $fname @@ -8195,7 +8195,7 @@ proc parseblobdiffline {ids line} { if {[string index $fname 0] eq "\""} { set fname [lindex $fname 0] } - set fname [encoding convertfrom $fname] + set fname [encoding convertfrom utf-8 $fname] set i [lsearch -exact $treediffs($ids) $fname] if {$i >= 0} { setinlist difffilestart $i $curdiffstart @@ -8214,6 +8214,7 @@ proc parseblobdiffline {ids line} { set diffinhdr 0 return } + set line [encoding convertfrom utf-8 $line] $ctext insert end "$line\n" filesep } else { @@ -12047,7 +12048,7 @@ proc cache_gitattr {attr pathlist} { foreach row [split $rlist "\n"] { if {[regexp "(.*): $attr: (.*)" $row m path value]} { if {[string index $path 0] eq "\""} { - set path [encoding convertfrom [lindex $path 0]] + set path [encoding convertfrom utf-8 [lindex $path 0]] } set path_attr_cache($attr,$path) $value } From a43b2aeb9c6415e606aefe8981b085db022e00a7 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 11 Aug 2009 02:22:33 +0200 Subject: [PATCH 012/288] gitk: work around the command line limit on Windows On Windows, there are dramatic problems when a command line grows beyond PATH_MAX, which is restricted to 8191 characters on XP and later (according to http://support.microsoft.com/kb/830473). Work around this by just cutting off the command line at that length (actually, at a space boundary) in the hope that only negative refs are chucked: gitk will then do unnecessary work, but that is still better than flashing the gitk window and exiting with exit status 5 (which no Windows user is able to make sense of). The first fix caused Tcl to fail to compile the regexp, see msysGit issue 427. Here is another fix without using regexp, and using a more relaxed command line length limit to fix the original issue 387. Signed-off-by: Sebastian Schuberth Signed-off-by: Pat Thoyts Signed-off-by: Johannes Schindelin --- gitk-git/gitk | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/gitk-git/gitk b/gitk-git/gitk index fbd36a76e8..5f91604135 100755 --- a/gitk-git/gitk +++ b/gitk-git/gitk @@ -10058,7 +10058,19 @@ proc getallcommits {} { } } if {$ids ne {}} { - set fd [open [concat $cmd $ids] r] + set cmd [concat $cmd $ids] + # The maximum command line length for the CreateProcess function is 32767 characters, see + # http://blogs.msdn.com/oldnewthing/archive/2003/12/10/56028.aspx + # Be a little conservative in case Tcl adds some more stuff to the command line we do not + # know about and truncate the command line at a SHA1-boundary below 32000 characters. + if {[tk windowingsystem] == "win32" && + [string length $cmd] > 32000} { + set ndx [string last " " $cmd 32000] + if {$ndx != -1} { + set cmd [string range $cmd 0 $ndx] + } + } + set fd [open $cmd r] fconfigure $fd -blocking 0 incr allcommits nowbusy allcommits From 27eebdd132f45aa75d0d0b661073f589a6527a6d Mon Sep 17 00:00:00 2001 From: "Chris West (Faux)" Date: Mon, 26 Jul 2010 00:36:19 +0100 Subject: [PATCH 013/288] gitk: fix another invocation with an overly long command-line Signed-off-by: Sebastian Schuberth --- gitk-git/gitk | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/gitk-git/gitk b/gitk-git/gitk index 5f91604135..33509e2ba5 100755 --- a/gitk-git/gitk +++ b/gitk-git/gitk @@ -406,7 +406,7 @@ proc start_rev_list {view} { if {$revs eq {}} { return 0 } - set args [concat $vflags($view) $revs] + set args [limit_arg_length [concat $vflags($view) $revs]] } else { set args $vorigargs($view) } @@ -10058,18 +10058,7 @@ proc getallcommits {} { } } if {$ids ne {}} { - set cmd [concat $cmd $ids] - # The maximum command line length for the CreateProcess function is 32767 characters, see - # http://blogs.msdn.com/oldnewthing/archive/2003/12/10/56028.aspx - # Be a little conservative in case Tcl adds some more stuff to the command line we do not - # know about and truncate the command line at a SHA1-boundary below 32000 characters. - if {[tk windowingsystem] == "win32" && - [string length $cmd] > 32000} { - set ndx [string last " " $cmd 32000] - if {$ndx != -1} { - set cmd [string range $cmd 0 $ndx] - } - } + set cmd [limit_arg_length [concat $cmd $ids]] set fd [open $cmd r] fconfigure $fd -blocking 0 incr allcommits @@ -10080,6 +10069,21 @@ proc getallcommits {} { } } +# The maximum command line length for the CreateProcess function is 32767 characters, see +# http://blogs.msdn.com/oldnewthing/archive/2003/12/10/56028.aspx +# Be a little conservative in case Tcl adds some more stuff to the command line we do not +# know about and truncate the command line at a SHA1-boundary below 32000 characters. +proc limit_arg_length {cmd} { + if {[tk windowingsystem] == "win32" && + [string length $cmd] > 32000} { + set ndx [string last " " $cmd 32000] + if {$ndx != -1} { + return [string range $cmd 0 $ndx] + } + } + return $cmd +} + # Since most commits have 1 parent and 1 child, we group strings of # such commits into "arcs" joining branch/merge points (BMPs), which # are commits that either don't have 1 parent or don't have 1 child. From a86d032c15aae3cdc5422dc608de4aee202065a0 Mon Sep 17 00:00:00 2001 From: Sebastian Schuberth Date: Sun, 22 Jul 2012 23:19:24 +0200 Subject: [PATCH 014/288] gitk: Use an external icon file on Windows Git for Windows now ships with the new Git icon from git-scm.com. Use that icon file if it exists instead of the old procedurally drawn one. This patch was sent upstream but so far no decision on its inclusion was made, so commit it to our fork. Signed-off-by: Sebastian Schuberth --- gitk-git/gitk | 41 ++++++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/gitk-git/gitk b/gitk-git/gitk index 33509e2ba5..be93011414 100755 --- a/gitk-git/gitk +++ b/gitk-git/gitk @@ -12094,7 +12094,6 @@ if { [info exists ::env(GITK_MSGSDIR)] } { set gitk_prefix [file dirname [file dirname [file normalize $argv0]]] set gitk_libdir [file join $gitk_prefix share gitk lib] set gitk_msgsdir [file join $gitk_libdir msgs] - unset gitk_prefix } ## Internationalization (i18n) through msgcat and gettext. See @@ -12449,28 +12448,32 @@ if {[expr {[exec git rev-parse --is-inside-work-tree] == "true"}]} { set worktree [exec git rev-parse --show-toplevel] setcoords makewindow -catch { - image create photo gitlogo -width 16 -height 16 +if {$::tcl_platform(platform) eq {windows} && [file exists $gitk_prefix/etc/git.ico]} { + wm iconbitmap . -default $gitk_prefix/etc/git.ico +} else { + catch { + image create photo gitlogo -width 16 -height 16 - image create photo gitlogominus -width 4 -height 2 - gitlogominus put #C00000 -to 0 0 4 2 - gitlogo copy gitlogominus -to 1 5 - gitlogo copy gitlogominus -to 6 5 - gitlogo copy gitlogominus -to 11 5 - image delete gitlogominus + image create photo gitlogominus -width 4 -height 2 + gitlogominus put #C00000 -to 0 0 4 2 + gitlogo copy gitlogominus -to 1 5 + gitlogo copy gitlogominus -to 6 5 + gitlogo copy gitlogominus -to 11 5 + image delete gitlogominus - image create photo gitlogoplus -width 4 -height 4 - gitlogoplus put #008000 -to 1 0 3 4 - gitlogoplus put #008000 -to 0 1 4 3 - gitlogo copy gitlogoplus -to 1 9 - gitlogo copy gitlogoplus -to 6 9 - gitlogo copy gitlogoplus -to 11 9 - image delete gitlogoplus + image create photo gitlogoplus -width 4 -height 4 + gitlogoplus put #008000 -to 1 0 3 4 + gitlogoplus put #008000 -to 0 1 4 3 + gitlogo copy gitlogoplus -to 1 9 + gitlogo copy gitlogoplus -to 6 9 + gitlogo copy gitlogoplus -to 11 9 + image delete gitlogoplus - image create photo gitlogo32 -width 32 -height 32 - gitlogo32 copy gitlogo -zoom 2 2 + image create photo gitlogo32 -width 32 -height 32 + gitlogo32 copy gitlogo -zoom 2 2 - wm iconphoto . -default gitlogo gitlogo32 + wm iconphoto . -default gitlogo gitlogo32 + } } # wait for the window to become visible tkwait visibility . From 4280c32fc64a8fdeedacd5c9314b7f3c42a3ff9b Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 16 Feb 2016 16:42:06 +0100 Subject: [PATCH 015/288] gitk: fix arrow keys in input fields with Tcl/Tk >= 8.6 Tcl/Tk 8.6 introduced new events for the cursor left/right keys and apparently changed the behavior of the previous event. Let's work around that by using the new events when we are running with Tcl/Tk 8.6 or later. This fixes https://github.com/git-for-windows/git/issues/495 Signed-off-by: Johannes Schindelin --- gitk-git/gitk | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/gitk-git/gitk b/gitk-git/gitk index be93011414..c4ef62c33f 100755 --- a/gitk-git/gitk +++ b/gitk-git/gitk @@ -2072,7 +2072,7 @@ proc makewindow {} { global headctxmenu progresscanv progressitem progresscoords statusw global fprogitem fprogcoord lastprogupdate progupdatepending global rprogitem rprogcoord rownumsel numcommits - global have_tk85 use_ttk NS + global have_tk85 have_tk86 use_ttk NS global git_version global worddiff @@ -2562,8 +2562,13 @@ proc makewindow {} { bind . "selnextline 1" bind . "dofind -1 0" bind . "dofind 1 0" - bindkey "goforw" - bindkey "goback" + if {$have_tk86} { + bindkey <> "goforw" + bindkey <> "goback" + } else { + bindkey "goforw" + bindkey "goback" + } bind . "selnextpage -1" bind . "selnextpage 1" bind . <$M1B-Home> "allcanvs yview moveto 0.0" @@ -12384,6 +12389,7 @@ set nullid2 "0000000000000000000000000000000000000001" set nullfile "/dev/null" set have_tk85 [expr {[package vcompare $tk_version "8.5"] >= 0}] +set have_tk86 [expr {[package vcompare $tk_version "8.6"] >= 0}] if {![info exists have_ttk]} { set have_ttk [llength [info commands ::ttk::style]] } From 9abdf4347a4481048bc561bb45a94e943ac4acf0 Mon Sep 17 00:00:00 2001 From: "James J. Raden" Date: Thu, 21 Jan 2016 12:07:47 -0500 Subject: [PATCH 016/288] gitk: make the "list references" default window width wider When using remotes (with git-flow especially), the remote reference names are almost always wordwrapped in the "list references" window because it's somewhat narrow by default. It's possible to resize it with a mouse, but it's annoying to have to do this every time, especially on Windows 10, where the window border seems to be only one (1) pixel wide, thus making the grabbing of the window border tricky. Signed-off-by: James J. Raden --- gitk-git/gitk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitk-git/gitk b/gitk-git/gitk index c4ef62c33f..a897a06f98 100755 --- a/gitk-git/gitk +++ b/gitk-git/gitk @@ -9874,7 +9874,7 @@ proc showrefs {} { text $top.list -background $bgcolor -foreground $fgcolor \ -selectbackground $selectbgcolor -font mainfont \ -xscrollcommand "$top.xsb set" -yscrollcommand "$top.ysb set" \ - -width 30 -height 20 -cursor $maincursor \ + -width 60 -height 20 -cursor $maincursor \ -spacing1 1 -spacing3 1 -state disabled $top.list tag configure highlight -background $selectbgcolor if {![lsearch -exact $bglist $top.list]} { From c552d5bef0865664139a14e36667c272b0944ecd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Bere=C5=BCa=C5=84ski?= Date: Wed, 26 Jun 2013 11:14:44 +0200 Subject: [PATCH 017/288] t0302: check helper can handle empty credentials MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make sure the helper does not crash when blank username and password is provided. If the helper can save such credentials, it should be able to read them back. Signed-off-by: Jakub Bereżański --- t/lib-credential.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/t/lib-credential.sh b/t/lib-credential.sh index d8e41f7ddd..937b831ea6 100755 --- a/t/lib-credential.sh +++ b/t/lib-credential.sh @@ -44,6 +44,7 @@ helper_test_clean() { reject $1 https example.com user2 reject $1 http path.tld user reject $1 https timeout.tld user + reject $1 https sso.tld } reject() { @@ -250,6 +251,24 @@ helper_test() { password=pass2 EOF ' + + test_expect_success "helper ($HELPER) can store empty username" ' + check approve $HELPER <<-\EOF && + protocol=https + host=sso.tld + username= + password= + EOF + check fill $HELPER <<-\EOF + protocol=https + host=sso.tld + -- + protocol=https + host=sso.tld + username= + password= + EOF + ' } helper_test_timeout() { From e518adcc4f5a4e16318dba6cbe665a04a0cac43a Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 10 Jan 2017 23:12:16 +0100 Subject: [PATCH 018/288] Makefile: POSIX windres When environment variable POSIXLY_CORRECT is set, the "input -o output" syntax is not supported. http://cygwin.com/ml/cygwin/2017-01/msg00036.html Signed-off-by: Steven Penny Signed-off-by: Johannes Schindelin --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 76267262c1..bf663ff911 100644 --- a/Makefile +++ b/Makefile @@ -1825,7 +1825,7 @@ $(SCRIPT_LIB) : % : %.sh GIT-SCRIPT-DEFINES git.res: git.rc GIT-VERSION-FILE $(QUIET_RC)$(RC) \ $(join -DMAJOR= -DMINOR=, $(wordlist 1,2,$(subst -, ,$(subst ., ,$(GIT_VERSION))))) \ - -DGIT_VERSION="\\\"$(GIT_VERSION)\\\"" $< -o $@ + -DGIT_VERSION="\\\"$(GIT_VERSION)\\\"" -i $< -o $@ # This makes sure we depend on the NO_PERL setting itself. $(SCRIPT_PERL_GEN): GIT-BUILD-OPTIONS From e4dc91d2160871de39b12f657a89d26a6f7db093 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Bere=C5=BCa=C5=84ski?= Date: Wed, 26 Jun 2013 11:07:55 +0200 Subject: [PATCH 019/288] wincred: handle empty username/password correctly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Empty (length 0) usernames and/or passwords, when saved in the Windows Credential Manager, come back as null when reading the credential. One use case for such empty credentials is with NTLM authentication, where empty username and password instruct libcurl to authenticate using the credentials of the currently logged-on user (single sign-on). When locating the relevant credentials, make empty username match null. When outputting the credentials, handle nulls correctly. Signed-off-by: Jakub Bereżański --- contrib/credential/wincred/git-credential-wincred.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/contrib/credential/wincred/git-credential-wincred.c b/contrib/credential/wincred/git-credential-wincred.c index 006134043a..86518cd93d 100644 --- a/contrib/credential/wincred/git-credential-wincred.c +++ b/contrib/credential/wincred/git-credential-wincred.c @@ -94,6 +94,12 @@ static WCHAR *wusername, *password, *protocol, *host, *path, target[1024]; static void write_item(const char *what, LPCWSTR wbuf, int wlen) { char *buf; + + if (!wbuf || !wlen) { + printf("%s=\n", what); + return; + } + int len = WideCharToMultiByte(CP_UTF8, 0, wbuf, wlen, NULL, 0, NULL, FALSE); buf = xmalloc(len); @@ -160,7 +166,7 @@ static int match_part_last(LPCWSTR *ptarget, LPCWSTR want, LPCWSTR delim) static int match_cred(const CREDENTIALW *cred) { LPCWSTR target = cred->TargetName; - if (wusername && wcscmp(wusername, cred->UserName)) + if (wusername && wcscmp(wusername, cred->UserName ? cred->UserName : L"")) return 0; return match_part(&target, L"git", L":") && @@ -183,7 +189,7 @@ static void get_credential(void) for (i = 0; i < num_creds; ++i) if (match_cred(creds[i])) { write_item("username", creds[i]->UserName, - wcslen(creds[i]->UserName)); + creds[i]->UserName ? wcslen(creds[i]->UserName) : 0); write_item("password", (LPCWSTR)creds[i]->CredentialBlob, creds[i]->CredentialBlobSize / sizeof(WCHAR)); From 84dcb14afaba169c958b856b86ce545d55a8b5fb Mon Sep 17 00:00:00 2001 From: Steven Penny Date: Sun, 8 Jan 2017 00:12:38 -0600 Subject: [PATCH 020/288] Makefile: put LIBS after LDFLAGS for imap-send This matches up with the targets git-%, git-http-fetch, git-http-push and git-remote-testsvn. It must be done this way in Cygwin else lcrypto cannot find lgdi32 and lws2_32. Signed-off-by: Steven Penny Signed-off-by: Johannes Schindelin --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index bf663ff911..75e192d9a0 100644 --- a/Makefile +++ b/Makefile @@ -2055,7 +2055,7 @@ git-%$X: %.o GIT-LDFLAGS $(GITLIBS) git-imap-send$X: imap-send.o $(IMAP_SEND_BUILDDEPS) GIT-LDFLAGS $(GITLIBS) $(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $(filter %.o,$^) \ - $(LIBS) $(IMAP_SEND_LDFLAGS) + $(IMAP_SEND_LDFLAGS) $(LIBS) git-http-fetch$X: http.o http-walker.o http-fetch.o GIT-LDFLAGS $(GITLIBS) $(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $(filter %.o,$^) \ From 06c6407e8dc434679299a2c5dfb227ef898f48e6 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 17 Nov 2016 17:24:10 +0100 Subject: [PATCH 021/288] Allow for platform-specific core.* config settings In the Git for Windows project, we have ample precendent for config settings that apply to Windows, and to Windows only. Let's formalize this concept by introducing a platform_core_config() function that can be #define'd in a platform-specific manner. This will allow us to contain platform-specific code better, as the corresponding variables no longer need to be exported so that they can be defined in environment.c and be set in config.c Signed-off-by: Johannes Schindelin --- compat/mingw.c | 5 +++++ compat/mingw.h | 3 +++ config.c | 2 +- git-compat-util.h | 8 ++++++++ 4 files changed, 17 insertions(+), 1 deletion(-) diff --git a/compat/mingw.c b/compat/mingw.c index 3fbfda5978..97e953465e 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -202,6 +202,11 @@ static int ask_yes_no_if_possible(const char *format, ...) } } +int mingw_core_config(const char *var, const char *value) +{ + return 0; +} + int mingw_unlink(const char *pathname) { int ret, tries = 0; diff --git a/compat/mingw.h b/compat/mingw.h index 3350169555..a386db1bb2 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -11,6 +11,9 @@ typedef _sigset_t sigset_t; #undef _POSIX_THREAD_SAFE_FUNCTIONS #endif +extern int mingw_core_config(const char *var, const char *value); +#define platform_core_config mingw_core_config + /* * things that are not available in header files */ diff --git a/config.c b/config.c index 617b2e3cf4..8fdb30cbed 100644 --- a/config.c +++ b/config.c @@ -1018,7 +1018,7 @@ static int git_default_core_config(const char *var, const char *value) } /* Add other config variables here and to Documentation/config.txt. */ - return 0; + return platform_core_config(var, value); } static int git_default_i18n_config(const char *var, const char *value) diff --git a/git-compat-util.h b/git-compat-util.h index 87237b092b..9355207be2 100644 --- a/git-compat-util.h +++ b/git-compat-util.h @@ -330,6 +330,14 @@ extern char *gitdirname(char *); #define _PATH_DEFPATH "/usr/local/bin:/usr/bin:/bin" #endif +#ifndef platform_core_config +static inline int noop_core_config(const char *var, const char *value) +{ + return 0; +} +#define platform_core_config noop_core_config +#endif + #ifndef has_dos_drive_prefix static inline int git_has_dos_drive_prefix(const char *path) { From 9d81e2b1a49ef65c913e94106d9a01264223c092 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 10 Jan 2017 23:20:54 +0100 Subject: [PATCH 022/288] mingw: include the full version information in the resources This fixes https://github.com/git-for-windows/git/issues/723 Signed-off-by: Johannes Schindelin --- Makefile | 3 ++- git.rc | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 75e192d9a0..3bca483fd3 100644 --- a/Makefile +++ b/Makefile @@ -1824,7 +1824,8 @@ $(SCRIPT_LIB) : % : %.sh GIT-SCRIPT-DEFINES git.res: git.rc GIT-VERSION-FILE $(QUIET_RC)$(RC) \ - $(join -DMAJOR= -DMINOR=, $(wordlist 1,2,$(subst -, ,$(subst ., ,$(GIT_VERSION))))) \ + $(join -DMAJOR= -DMINOR= -DMICRO= -DPATCHLEVEL=, $(wordlist 1, 4, \ + $(shell echo $(GIT_VERSION) 0 0 0 0 | tr '.a-zA-Z-' ' '))) \ -DGIT_VERSION="\\\"$(GIT_VERSION)\\\"" -i $< -o $@ # This makes sure we depend on the NO_PERL setting itself. diff --git a/git.rc b/git.rc index 33aafb786c..49002e0d54 100644 --- a/git.rc +++ b/git.rc @@ -1,6 +1,6 @@ 1 VERSIONINFO -FILEVERSION MAJOR,MINOR,0,0 -PRODUCTVERSION MAJOR,MINOR,0,0 +FILEVERSION MAJOR,MINOR,MICRO,PATCHLEVEL +PRODUCTVERSION MAJOR,MINOR,MICRO,PATCHLEVEL BEGIN BLOCK "StringFileInfo" BEGIN From 7190e447adbddfdbed7fe5f280439ba64e34b0fe Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 17 Nov 2016 17:33:06 +0100 Subject: [PATCH 023/288] Move Windows-specific config settings into compat/mingw.c Signed-off-by: Johannes Schindelin --- cache.h | 8 -------- compat/mingw.c | 17 +++++++++++++++++ config.c | 8 -------- environment.c | 1 - 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/cache.h b/cache.h index 5c035dae57..8821d54dd5 100644 --- a/cache.h +++ b/cache.h @@ -753,14 +753,6 @@ extern int ref_paranoia; extern char comment_line_char; extern int auto_comment_line_char; -/* Windows only */ -enum hide_dotfiles_type { - HIDE_DOTFILES_FALSE = 0, - HIDE_DOTFILES_TRUE, - HIDE_DOTFILES_DOTGITONLY -}; -extern enum hide_dotfiles_type hide_dotfiles; - enum branch_track { BRANCH_TRACK_UNSPECIFIED = -1, BRANCH_TRACK_NEVER = 0, diff --git a/compat/mingw.c b/compat/mingw.c index 97e953465e..27b6aaa907 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -202,8 +202,25 @@ static int ask_yes_no_if_possible(const char *format, ...) } } +/* Windows only */ +enum hide_dotfiles_type { + HIDE_DOTFILES_FALSE = 0, + HIDE_DOTFILES_TRUE, + HIDE_DOTFILES_DOTGITONLY +}; + +static enum hide_dotfiles_type hide_dotfiles = HIDE_DOTFILES_DOTGITONLY; + int mingw_core_config(const char *var, const char *value) { + if (!strcmp(var, "core.hidedotfiles")) { + if (value && !strcasecmp(value, "dotgitonly")) + hide_dotfiles = HIDE_DOTFILES_DOTGITONLY; + else + hide_dotfiles = git_config_bool(var, value); + return 0; + } + return 0; } diff --git a/config.c b/config.c index 8fdb30cbed..b805534902 100644 --- a/config.c +++ b/config.c @@ -1009,14 +1009,6 @@ static int git_default_core_config(const char *var, const char *value) return 0; } - if (!strcmp(var, "core.hidedotfiles")) { - if (value && !strcasecmp(value, "dotgitonly")) - hide_dotfiles = HIDE_DOTFILES_DOTGITONLY; - else - hide_dotfiles = git_config_bool(var, value); - return 0; - } - /* Add other config variables here and to Documentation/config.txt. */ return platform_core_config(var, value); } diff --git a/environment.c b/environment.c index 4bce3eebfa..3487c3aed4 100644 --- a/environment.c +++ b/environment.c @@ -63,7 +63,6 @@ int core_apply_sparse_checkout; int merge_log_config = -1; int precomposed_unicode = -1; /* see probe_utf8_pathname_composition() */ unsigned long pack_size_limit_cfg; -enum hide_dotfiles_type hide_dotfiles = HIDE_DOTFILES_DOTGITONLY; #ifndef PROTECT_HFS_DEFAULT #define PROTECT_HFS_DEFAULT 0 From 8a22dfe3d865b43e41a2105aabfb2ae1d0938a80 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 17 Nov 2016 18:05:04 +0100 Subject: [PATCH 024/288] mingw: unset PERL5LIB by default Git for Windows ships with its own Perl interpreter, and insists on using it, so it will most likely wreak havoc if PERL5LIB is set before launching Git. Let's just unset that environment variables when spawning processes. To make this feature extensible (and overrideable), there is a new config setting `core.unsetenvvars` that allows specifying a comma-separated list of names to unset before spawning processes. Reported by Gabriel Fuhrmann. Signed-off-by: Johannes Schindelin --- Documentation/config.txt | 6 ++++++ compat/mingw.c | 35 ++++++++++++++++++++++++++++++++++- t/t0028-core-unsetenvvars.sh | 30 ++++++++++++++++++++++++++++++ 3 files changed, 70 insertions(+), 1 deletion(-) create mode 100755 t/t0028-core-unsetenvvars.sh diff --git a/Documentation/config.txt b/Documentation/config.txt index 1fee83ca42..e1b0a3d7d9 100644 --- a/Documentation/config.txt +++ b/Documentation/config.txt @@ -760,6 +760,12 @@ relatively high IO latencies. When enabled, Git will do the index comparison to the filesystem data in parallel, allowing overlapping IO's. Defaults to true. +core.unsetenvvars:: + EXPERIMENTAL, Windows-only: comma-separated list of environment + variables' names that need to be unset before spawning any other + process. Defaults to `PERL5LIB` to account for the fact that Git + for Windows insists on using its own Perl interpreter. + core.createObject:: You can set this to 'link', in which case a hardlink followed by a delete of the source are used to make sure that object creation diff --git a/compat/mingw.c b/compat/mingw.c index 27b6aaa907..b833179cf3 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -210,6 +210,7 @@ enum hide_dotfiles_type { }; static enum hide_dotfiles_type hide_dotfiles = HIDE_DOTFILES_DOTGITONLY; +static char *unset_environment_variables; int mingw_core_config(const char *var, const char *value) { @@ -221,6 +222,12 @@ int mingw_core_config(const char *var, const char *value) return 0; } + if (!strcmp(var, "core.unsetenvvars")) { + free(unset_environment_variables); + unset_environment_variables = xstrdup(value); + return 0; + } + return 0; } @@ -1090,6 +1097,27 @@ static wchar_t *make_environment_block(char **deltaenv) return wenvblk; } +static void do_unset_environment_variables(void) +{ + static int done; + char *p = unset_environment_variables; + + if (done || !p) + return; + done = 1; + + for (;;) { + char *comma = strchr(p, ','); + + if (comma) + *comma = '\0'; + unsetenv(p); + if (!comma) + break; + p = comma + 1; + } +} + struct pinfo_t { struct pinfo_t *next; pid_t pid; @@ -1108,9 +1136,12 @@ static pid_t mingw_spawnve_fd(const char *cmd, const char **argv, char **deltaen wchar_t wcmd[MAX_PATH], wdir[MAX_PATH], *wargs, *wenvblk = NULL; unsigned flags = CREATE_UNICODE_ENVIRONMENT; BOOL ret; + HANDLE cons; + + do_unset_environment_variables(); /* Determine whether or not we are associated to a console */ - HANDLE cons = CreateFile("CONOUT$", GENERIC_WRITE, + cons = CreateFile("CONOUT$", GENERIC_WRITE, FILE_SHARE_WRITE, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); if (cons == INVALID_HANDLE_VALUE) { @@ -2251,6 +2282,8 @@ void mingw_startup(void) /* fix Windows specific environment settings */ setup_windows_environment(); + unset_environment_variables = xstrdup("PERL5LIB"); + /* initialize critical section for waitpid pinfo_t list */ InitializeCriticalSection(&pinfo_cs); diff --git a/t/t0028-core-unsetenvvars.sh b/t/t0028-core-unsetenvvars.sh new file mode 100755 index 0000000000..24ce46a6ea --- /dev/null +++ b/t/t0028-core-unsetenvvars.sh @@ -0,0 +1,30 @@ +#!/bin/sh + +test_description='test the Windows-only core.unsetenvvars setting' + +. ./test-lib.sh + +if ! test_have_prereq MINGW +then + skip_all='skipping Windows-specific tests' + test_done +fi + +test_expect_success 'setup' ' + mkdir -p "$TRASH_DIRECTORY/.git/hooks" && + write_script "$TRASH_DIRECTORY/.git/hooks/pre-commit" <<-\EOF + echo $HOBBES >&2 + EOF +' + +test_expect_success 'core.unsetenvvars works' ' + HOBBES=Calvin && + export HOBBES && + git commit --allow-empty -m with 2>err && + grep Calvin err && + git -c core.unsetenvvars=FINDUS,HOBBES,CALVIN \ + commit --allow-empty -m without 2>err && + ! grep Calvin err +' + +test_done From 45f2e46e120c8fa7266a21c1db4cddba04c0ddb8 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sat, 6 Jul 2013 02:09:35 +0200 Subject: [PATCH 025/288] Win32: make FILETIME conversion functions public Signed-off-by: Karsten Blees --- compat/mingw.c | 16 ---------------- compat/mingw.h | 16 ++++++++++++++++ 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index b833179cf3..0ef28ff3e3 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -541,22 +541,6 @@ int mingw_chmod(const char *filename, int mode) return _wchmod(wfilename, mode); } -/* - * The unit of FILETIME is 100-nanoseconds since January 1, 1601, UTC. - * Returns the 100-nanoseconds ("hekto nanoseconds") since the epoch. - */ -static inline long long filetime_to_hnsec(const FILETIME *ft) -{ - long long winTime = ((long long)ft->dwHighDateTime << 32) + ft->dwLowDateTime; - /* Windows to Unix Epoch conversion */ - return winTime - 116444736000000000LL; -} - -static inline time_t filetime_to_time_t(const FILETIME *ft) -{ - return (time_t)(filetime_to_hnsec(ft) / 10000000); -} - /** * Verifies that safe_create_leading_directories() would succeed. */ diff --git a/compat/mingw.h b/compat/mingw.h index a386db1bb2..623d85eabd 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -329,6 +329,22 @@ static inline int getrlimit(int resource, struct rlimit *rlp) return 0; } +/* + * The unit of FILETIME is 100-nanoseconds since January 1, 1601, UTC. + * Returns the 100-nanoseconds ("hekto nanoseconds") since the epoch. + */ +static inline long long filetime_to_hnsec(const FILETIME *ft) +{ + long long winTime = ((long long)ft->dwHighDateTime << 32) + ft->dwLowDateTime; + /* Windows to Unix Epoch conversion */ + return winTime - 116444736000000000LL; +} + +static inline time_t filetime_to_time_t(const FILETIME *ft) +{ + return (time_t)(filetime_to_hnsec(ft) / 10000000); +} + /* * Use mingw specific stat()/lstat()/fstat() implementations on Windows. */ From 2cd384fe7aec2be2a8c5b51e3d41f29892aee7de Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sun, 8 Sep 2013 14:17:31 +0200 Subject: [PATCH 026/288] Win32: dirent.c: Move opendir down Move opendir down in preparation for the next patch. Signed-off-by: Karsten Blees --- compat/win32/dirent.c | 68 +++++++++++++++++++++---------------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/compat/win32/dirent.c b/compat/win32/dirent.c index 52420ec7d4..2603a0fa39 100644 --- a/compat/win32/dirent.c +++ b/compat/win32/dirent.c @@ -18,40 +18,6 @@ static inline void finddata2dirent(struct dirent *ent, WIN32_FIND_DATAW *fdata) ent->d_type = DT_REG; } -DIR *opendir(const char *name) -{ - wchar_t pattern[MAX_PATH + 2]; /* + 2 for '/' '*' */ - WIN32_FIND_DATAW fdata; - HANDLE h; - int len; - DIR *dir; - - /* convert name to UTF-16 and check length < MAX_PATH */ - if ((len = xutftowcs_path(pattern, name)) < 0) - return NULL; - - /* append optional '/' and wildcard '*' */ - if (len && !is_dir_sep(pattern[len - 1])) - pattern[len++] = '/'; - pattern[len++] = '*'; - pattern[len] = 0; - - /* open find handle */ - h = FindFirstFileW(pattern, &fdata); - if (h == INVALID_HANDLE_VALUE) { - DWORD err = GetLastError(); - errno = (err == ERROR_DIRECTORY) ? ENOTDIR : err_win_to_posix(err); - return NULL; - } - - /* initialize DIR structure and copy first dir entry */ - dir = xmalloc(sizeof(DIR)); - dir->dd_handle = h; - dir->dd_stat = 0; - finddata2dirent(&dir->dd_dir, &fdata); - return dir; -} - struct dirent *readdir(DIR *dir) { if (!dir) { @@ -90,3 +56,37 @@ int closedir(DIR *dir) free(dir); return 0; } + +DIR *opendir(const char *name) +{ + wchar_t pattern[MAX_PATH + 2]; /* + 2 for '/' '*' */ + WIN32_FIND_DATAW fdata; + HANDLE h; + int len; + DIR *dir; + + /* convert name to UTF-16 and check length < MAX_PATH */ + if ((len = xutftowcs_path(pattern, name)) < 0) + return NULL; + + /* append optional '/' and wildcard '*' */ + if (len && !is_dir_sep(pattern[len - 1])) + pattern[len++] = '/'; + pattern[len++] = '*'; + pattern[len] = 0; + + /* open find handle */ + h = FindFirstFileW(pattern, &fdata); + if (h == INVALID_HANDLE_VALUE) { + DWORD err = GetLastError(); + errno = (err == ERROR_DIRECTORY) ? ENOTDIR : err_win_to_posix(err); + return NULL; + } + + /* initialize DIR structure and copy first dir entry */ + dir = xmalloc(sizeof(DIR)); + dir->dd_handle = h; + dir->dd_stat = 0; + finddata2dirent(&dir->dd_dir, &fdata); + return dir; +} From e617caa12801fbab9eb32560dd810296708acadb Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sun, 8 Sep 2013 14:18:40 +0200 Subject: [PATCH 027/288] Win32: Make the dirent implementation pluggable Emulating the POSIX dirent API on Windows via FindFirstFile/FindNextFile is pretty staightforward, however, most of the information provided in the WIN32_FIND_DATA structure is thrown away in the process. A more sophisticated implementation may cache this data, e.g. for later reuse in calls to lstat. Make the dirent implementation pluggable so that it can be switched at runtime, e.g. based on a config option. Define a base DIR structure with pointers to readdir/closedir that match the opendir implementation (i.e. similar to vtable pointers in OOP). Define readdir/closedir so that they call the function pointers in the DIR structure. This allows to choose the opendir implementation on a call-by-call basis. Move the fixed sized dirent.d_name buffer to the dirent-specific DIR structure, as d_name may be implementation specific (e.g. a caching implementation may just set d_name to point into the cache instead of copying the entire file name string). Signed-off-by: Karsten Blees --- compat/win32/dirent.c | 27 +++++++++++++++++---------- compat/win32/dirent.h | 26 +++++++++++++++++++------- 2 files changed, 36 insertions(+), 17 deletions(-) diff --git a/compat/win32/dirent.c b/compat/win32/dirent.c index 2603a0fa39..6b87042182 100644 --- a/compat/win32/dirent.c +++ b/compat/win32/dirent.c @@ -1,15 +1,19 @@ #include "../../git-compat-util.h" -struct DIR { +typedef struct dirent_DIR { + struct DIR base_dir; /* extend base struct DIR */ struct dirent dd_dir; /* includes d_type */ HANDLE dd_handle; /* FindFirstFile handle */ int dd_stat; /* 0-based index */ -}; + char dd_name[MAX_PATH * 3]; /* file name (* 3 for UTF-8 conversion) */ +} dirent_DIR; + +DIR *(*opendir)(const char *dirname) = dirent_opendir; static inline void finddata2dirent(struct dirent *ent, WIN32_FIND_DATAW *fdata) { - /* convert UTF-16 name to UTF-8 */ - xwcstoutf(ent->d_name, fdata->cFileName, sizeof(ent->d_name)); + /* convert UTF-16 name to UTF-8 (d_name points to dirent_DIR.dd_name) */ + xwcstoutf(ent->d_name, fdata->cFileName, MAX_PATH * 3); /* Set file type, based on WIN32_FIND_DATA */ if (fdata->dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) @@ -18,7 +22,7 @@ static inline void finddata2dirent(struct dirent *ent, WIN32_FIND_DATAW *fdata) ent->d_type = DT_REG; } -struct dirent *readdir(DIR *dir) +static struct dirent *dirent_readdir(dirent_DIR *dir) { if (!dir) { errno = EBADF; /* No set_errno for mingw */ @@ -45,7 +49,7 @@ struct dirent *readdir(DIR *dir) return &dir->dd_dir; } -int closedir(DIR *dir) +static int dirent_closedir(dirent_DIR *dir) { if (!dir) { errno = EBADF; @@ -57,13 +61,13 @@ int closedir(DIR *dir) return 0; } -DIR *opendir(const char *name) +DIR *dirent_opendir(const char *name) { wchar_t pattern[MAX_PATH + 2]; /* + 2 for '/' '*' */ WIN32_FIND_DATAW fdata; HANDLE h; int len; - DIR *dir; + dirent_DIR *dir; /* convert name to UTF-16 and check length < MAX_PATH */ if ((len = xutftowcs_path(pattern, name)) < 0) @@ -84,9 +88,12 @@ DIR *opendir(const char *name) } /* initialize DIR structure and copy first dir entry */ - dir = xmalloc(sizeof(DIR)); + dir = xmalloc(sizeof(dirent_DIR)); + dir->base_dir.preaddir = (struct dirent *(*)(DIR *dir)) dirent_readdir; + dir->base_dir.pclosedir = (int (*)(DIR *dir)) dirent_closedir; + dir->dd_dir.d_name = dir->dd_name; dir->dd_handle = h; dir->dd_stat = 0; finddata2dirent(&dir->dd_dir, &fdata); - return dir; + return (DIR*) dir; } diff --git a/compat/win32/dirent.h b/compat/win32/dirent.h index 058207e4bf..6b3ddee51b 100644 --- a/compat/win32/dirent.h +++ b/compat/win32/dirent.h @@ -1,20 +1,32 @@ #ifndef DIRENT_H #define DIRENT_H -typedef struct DIR DIR; - #define DT_UNKNOWN 0 #define DT_DIR 1 #define DT_REG 2 #define DT_LNK 3 struct dirent { - unsigned char d_type; /* file type to prevent lstat after readdir */ - char d_name[MAX_PATH * 3]; /* file name (* 3 for UTF-8 conversion) */ + unsigned char d_type; /* file type to prevent lstat after readdir */ + char *d_name; /* file name */ }; -DIR *opendir(const char *dirname); -struct dirent *readdir(DIR *dir); -int closedir(DIR *dir); +/* + * Base DIR structure, contains pointers to readdir/closedir implementations so + * that opendir may choose a concrete implementation on a call-by-call basis. + */ +typedef struct DIR { + struct dirent *(*preaddir)(struct DIR *dir); + int (*pclosedir)(struct DIR *dir); +} DIR; + +/* default dirent implementation */ +extern DIR *dirent_opendir(const char *dirname); + +/* current dirent implementation */ +extern DIR *(*opendir)(const char *dirname); + +#define readdir(dir) (dir->preaddir(dir)) +#define closedir(dir) (dir->pclosedir(dir)) #endif /* DIRENT_H */ From efd508fb6f6370d8e2bac779e0d480081982e000 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sun, 8 Sep 2013 14:21:30 +0200 Subject: [PATCH 028/288] Win32: make the lstat implementation pluggable Emulating the POSIX lstat API on Windows via GetFileAttributes[Ex] is quite slow. Windows operating system APIs seem to be much better at scanning the status of entire directories than checking single files. A caching implementation may improve performance by bulk-reading entire directories or reusing data obtained via opendir / readdir. Make the lstat implementation pluggable so that it can be switched at runtime, e.g. based on a config option. Signed-off-by: Karsten Blees --- compat/mingw.c | 2 ++ compat/mingw.h | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/compat/mingw.c b/compat/mingw.c index 0ef28ff3e3..e9bc3f6752 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -680,6 +680,8 @@ static int do_stat_internal(int follow, const char *file_name, struct stat *buf) return do_lstat(follow, alt_name, buf); } +int (*lstat)(const char *file_name, struct stat *buf) = mingw_lstat; + int mingw_lstat(const char *file_name, struct stat *buf) { return do_stat_internal(0, file_name, buf); diff --git a/compat/mingw.h b/compat/mingw.h index 623d85eabd..2c12f45bff 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -368,7 +368,7 @@ int mingw_fstat(int fd, struct stat *buf); #ifdef lstat #undef lstat #endif -#define lstat mingw_lstat +extern int (*lstat)(const char *file_name, struct stat *buf); #ifndef _stati64 # define _stati64(x,y) mingw_stat(x,y) From d35becd44adc9bcf42e9bf202682b685ddb622e0 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sun, 8 Sep 2013 14:23:27 +0200 Subject: [PATCH 029/288] add infrastructure for read-only file system level caches Add a macro to mark code sections that only read from the file system, along with a config option and documentation. This facilitates implementation of relatively simple file system level caches without the need to synchronize with the file system. Enable read-only sections for 'git status' and preload_index. Signed-off-by: Karsten Blees --- Documentation/config.txt | 6 ++++++ builtin/commit.c | 1 + compat/mingw.c | 6 ++++++ compat/mingw.h | 2 ++ git-compat-util.h | 15 +++++++++++++++ preload-index.c | 2 ++ 6 files changed, 32 insertions(+) diff --git a/Documentation/config.txt b/Documentation/config.txt index e1b0a3d7d9..b87029e79e 100644 --- a/Documentation/config.txt +++ b/Documentation/config.txt @@ -760,6 +760,12 @@ relatively high IO latencies. When enabled, Git will do the index comparison to the filesystem data in parallel, allowing overlapping IO's. Defaults to true. +core.fscache:: + Enable additional caching of file system data for some operations. ++ +Git for Windows uses this to bulk-read and cache lstat data of entire +directories (instead of doing lstat file by file). + core.unsetenvvars:: EXPERIMENTAL, Windows-only: comma-separated list of environment variables' names that need to be unset before spawning any other diff --git a/builtin/commit.c b/builtin/commit.c index 276c74034e..5688620f5e 100644 --- a/builtin/commit.c +++ b/builtin/commit.c @@ -1377,6 +1377,7 @@ int cmd_status(int argc, const char **argv, const char *prefix) PATHSPEC_PREFER_FULL, prefix, argv); + enable_fscache(1); read_cache_preload(&s.pathspec); refresh_index(&the_index, REFRESH_QUIET|REFRESH_UNMERGED, &s.pathspec, NULL, NULL); diff --git a/compat/mingw.c b/compat/mingw.c index e9bc3f6752..918420964f 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -211,6 +211,7 @@ enum hide_dotfiles_type { static enum hide_dotfiles_type hide_dotfiles = HIDE_DOTFILES_DOTGITONLY; static char *unset_environment_variables; +int core_fscache; int mingw_core_config(const char *var, const char *value) { @@ -222,6 +223,11 @@ int mingw_core_config(const char *var, const char *value) return 0; } + if (!strcmp(var, "core.fscache")) { + core_fscache = git_config_bool(var, value); + return 0; + } + if (!strcmp(var, "core.unsetenvvars")) { free(unset_environment_variables); unset_environment_variables = xstrdup(value); diff --git a/compat/mingw.h b/compat/mingw.h index 2c12f45bff..cd20e0b449 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -11,6 +11,8 @@ typedef _sigset_t sigset_t; #undef _POSIX_THREAD_SAFE_FUNCTIONS #endif +extern int core_fscache; + extern int mingw_core_config(const char *var, const char *value); #define platform_core_config mingw_core_config diff --git a/git-compat-util.h b/git-compat-util.h index 9355207be2..7400068f37 100644 --- a/git-compat-util.h +++ b/git-compat-util.h @@ -1109,6 +1109,21 @@ struct tm *git_gmtime_r(const time_t *, struct tm *); #define getc_unlocked(fh) getc(fh) #endif +/* + * Enable/disable a read-only cache for file system data on platforms that + * support it. + * + * Implementing a live-cache is complicated and requires special platform + * support (inotify, ReadDirectoryChangesW...). enable_fscache shall be used + * to mark sections of git code that extensively read from the file system + * without modifying anything. Implementations can use this to cache e.g. stat + * data or even file content without the need to synchronize with the file + * system. + */ +#ifndef enable_fscache +#define enable_fscache(x) /* noop */ +#endif + extern int cmd_main(int, const char **); #endif diff --git a/preload-index.c b/preload-index.c index c1fe3a3ef9..c7970d7cb9 100644 --- a/preload-index.c +++ b/preload-index.c @@ -84,6 +84,7 @@ static void preload_index(struct index_state *index, offset = 0; work = DIV_ROUND_UP(index->cache_nr, threads); memset(&data, 0, sizeof(data)); + enable_fscache(1); for (i = 0; i < threads; i++) { struct thread_data *p = data+i; p->index = index; @@ -100,6 +101,7 @@ static void preload_index(struct index_state *index, if (pthread_join(p->pthread, NULL)) die("unable to join threaded lstat"); } + enable_fscache(0); } #endif From 1b7fc1856ab656ed8362e96f35561d8a4f6499f1 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Tue, 1 Oct 2013 12:51:54 +0200 Subject: [PATCH 030/288] Win32: add a cache below mingw's lstat and dirent implementations Checking the work tree status is quite slow on Windows, due to slow lstat emulation (git calls lstat once for each file in the index). Windows operating system APIs seem to be much better at scanning the status of entire directories than checking single files. Add an lstat implementation that uses a cache for lstat data. Cache misses read the entire parent directory and add it to the cache. Subsequent lstat calls for the same directory are served directly from the cache. Also implement opendir / readdir / closedir so that they create and use directory listings in the cache. The cache doesn't track file system changes and doesn't plug into any modifying file APIs, so it has to be explicitly enabled for git functions that don't modify the working copy. Note: in an earlier version of this patch, the cache was always active and tracked file system changes via ReadDirectoryChangesW. However, this was much more complex and had negative impact on the performance of modifying git commands such as 'git checkout'. Signed-off-by: Karsten Blees --- compat/win32/fscache.c | 458 +++++++++++++++++++++++++++++++++++++++++ compat/win32/fscache.h | 10 + config.mak.uname | 4 +- git-compat-util.h | 2 + 4 files changed, 472 insertions(+), 2 deletions(-) create mode 100644 compat/win32/fscache.c create mode 100644 compat/win32/fscache.h diff --git a/compat/win32/fscache.c b/compat/win32/fscache.c new file mode 100644 index 0000000000..28d22791bd --- /dev/null +++ b/compat/win32/fscache.c @@ -0,0 +1,458 @@ +#include "../../cache.h" +#include "../../hashmap.h" +#include "../win32.h" +#include "fscache.h" + +static int initialized; +static volatile long enabled; +static struct hashmap map; +static CRITICAL_SECTION mutex; + +/* + * An entry in the file system cache. Used for both entire directory listings + * and file entries. + */ +struct fsentry { + struct hashmap_entry ent; + mode_t st_mode; + /* Length of name. */ + unsigned short len; + /* + * Name of the entry. For directory listings: relative path of the + * directory, without trailing '/' (empty for cwd()). For file entries: + * name of the file. Typically points to the end of the structure if + * the fsentry is allocated on the heap (see fsentry_alloc), or to a + * local variable if on the stack (see fsentry_init). + */ + const char *name; + /* Pointer to the directory listing, or NULL for the listing itself. */ + struct fsentry *list; + /* Pointer to the next file entry of the list. */ + struct fsentry *next; + + union { + /* Reference count of the directory listing. */ + volatile long refcnt; + struct { + /* More stat members (only used for file entries). */ + off64_t st_size; + time_t st_atime; + time_t st_mtime; + time_t st_ctime; + }; + }; +}; + +/* + * Compares the paths of two fsentry structures for equality. + */ +static int fsentry_cmp(const struct fsentry *fse1, const struct fsentry *fse2) +{ + int res; + if (fse1 == fse2) + return 0; + + /* compare the list parts first */ + if (fse1->list != fse2->list && (res = fsentry_cmp( + fse1->list ? fse1->list : fse1, + fse2->list ? fse2->list : fse2))) + return res; + + /* if list parts are equal, compare len and name */ + if (fse1->len != fse2->len) + return fse1->len - fse2->len; + return strnicmp(fse1->name, fse2->name, fse1->len); +} + +/* + * Calculates the hash code of an fsentry structure's path. + */ +static unsigned int fsentry_hash(const struct fsentry *fse) +{ + unsigned int hash = fse->list ? fse->list->ent.hash : 0; + return hash ^ memihash(fse->name, fse->len); +} + +/* + * Initialize an fsentry structure for use by fsentry_hash and fsentry_cmp. + */ +static void fsentry_init(struct fsentry *fse, struct fsentry *list, + const char *name, size_t len) +{ + fse->list = list; + fse->name = name; + fse->len = len; + hashmap_entry_init(fse, fsentry_hash(fse)); +} + +/* + * Allocate an fsentry structure on the heap. + */ +static struct fsentry *fsentry_alloc(struct fsentry *list, const char *name, + size_t len) +{ + /* overallocate fsentry and copy the name to the end */ + struct fsentry *fse = xmalloc(sizeof(struct fsentry) + len + 1); + char *nm = ((char*) fse) + sizeof(struct fsentry); + memcpy(nm, name, len); + nm[len] = 0; + /* init the rest of the structure */ + fsentry_init(fse, list, nm, len); + fse->next = NULL; + fse->refcnt = 1; + return fse; +} + +/* + * Add a reference to an fsentry. + */ +inline static void fsentry_addref(struct fsentry *fse) +{ + if (fse->list) + fse = fse->list; + + InterlockedIncrement(&(fse->refcnt)); +} + +/* + * Release the reference to an fsentry, frees the memory if its the last ref. + */ +static void fsentry_release(struct fsentry *fse) +{ + if (fse->list) + fse = fse->list; + + if (InterlockedDecrement(&(fse->refcnt))) + return; + + while (fse) { + struct fsentry *next = fse->next; + free(fse); + fse = next; + } +} + +/* + * Allocate and initialize an fsentry from a WIN32_FIND_DATA structure. + */ +static struct fsentry *fseentry_create_entry(struct fsentry *list, + const WIN32_FIND_DATAW *fdata) +{ + char buf[MAX_PATH * 3]; + int len; + struct fsentry *fse; + len = xwcstoutf(buf, fdata->cFileName, ARRAY_SIZE(buf)); + + fse = fsentry_alloc(list, buf, len); + + fse->st_mode = file_attr_to_st_mode(fdata->dwFileAttributes); + fse->st_size = (((off64_t) (fdata->nFileSizeHigh)) << 32) + | fdata->nFileSizeLow; + fse->st_atime = filetime_to_time_t(&(fdata->ftLastAccessTime)); + fse->st_mtime = filetime_to_time_t(&(fdata->ftLastWriteTime)); + fse->st_ctime = filetime_to_time_t(&(fdata->ftCreationTime)); + + return fse; +} + +/* + * Create an fsentry-based directory listing (similar to opendir / readdir). + * Dir should not contain trailing '/'. Use an empty string for the current + * directory (not "."!). + */ +static struct fsentry *fsentry_create_list(const struct fsentry *dir) +{ + wchar_t pattern[MAX_PATH + 2]; /* + 2 for '/' '*' */ + WIN32_FIND_DATAW fdata; + HANDLE h; + int wlen; + struct fsentry *list, **phead; + DWORD err; + + /* convert name to UTF-16 and check length < MAX_PATH */ + if ((wlen = xutftowcsn(pattern, dir->name, MAX_PATH, dir->len)) < 0) { + if (errno == ERANGE) + errno = ENAMETOOLONG; + return NULL; + } + + /* append optional '/' and wildcard '*' */ + if (wlen) + pattern[wlen++] = '/'; + pattern[wlen++] = '*'; + pattern[wlen] = 0; + + /* open find handle */ + h = FindFirstFileW(pattern, &fdata); + if (h == INVALID_HANDLE_VALUE) { + err = GetLastError(); + errno = (err == ERROR_DIRECTORY) ? ENOTDIR : err_win_to_posix(err); + return NULL; + } + + /* allocate object to hold directory listing */ + list = fsentry_alloc(NULL, dir->name, dir->len); + + /* walk directory and build linked list of fsentry structures */ + phead = &list->next; + do { + *phead = fseentry_create_entry(list, &fdata); + phead = &(*phead)->next; + } while (FindNextFileW(h, &fdata)); + + /* remember result of last FindNextFile, then close find handle */ + err = GetLastError(); + FindClose(h); + + /* return the list if we've got all the files */ + if (err == ERROR_NO_MORE_FILES) + return list; + + /* otherwise free the list and return error */ + fsentry_release(list); + errno = err_win_to_posix(err); + return NULL; +} + +/* + * Adds a directory listing to the cache. + */ +static void fscache_add(struct fsentry *fse) +{ + if (fse->list) + fse = fse->list; + + for (; fse; fse = fse->next) + hashmap_add(&map, fse); +} + +/* + * Removes a directory listing from the cache. + */ +static void fscache_remove(struct fsentry *fse) +{ + if (fse->list) + fse = fse->list; + + for (; fse; fse = fse->next) + hashmap_remove(&map, fse, NULL); +} + +/* + * Clears the cache. + */ +static void fscache_clear(void) +{ + struct hashmap_iter iter; + struct fsentry *fse; + while ((fse = hashmap_iter_first(&map, &iter))) { + fscache_remove(fse); + fsentry_release(fse); + } +} + +/* + * Checks if the cache is enabled for the given path. + */ +static inline int fscache_enabled(const char *path) +{ + return enabled > 0 && !is_absolute_path(path); +} + +/* + * Looks up or creates a cache entry for the specified key. + */ +static struct fsentry *fscache_get(struct fsentry *key) +{ + struct fsentry *fse; + + EnterCriticalSection(&mutex); + /* check if entry is in cache */ + fse = hashmap_get(&map, key, NULL); + if (fse) { + fsentry_addref(fse); + LeaveCriticalSection(&mutex); + return fse; + } + /* if looking for a file, check if directory listing is in cache */ + if (!fse && key->list) { + fse = hashmap_get(&map, key->list, NULL); + if (fse) { + LeaveCriticalSection(&mutex); + /* dir entry without file entry -> file doesn't exist */ + errno = ENOENT; + return NULL; + } + } + + /* create the directory listing (outside mutex!) */ + LeaveCriticalSection(&mutex); + fse = fsentry_create_list(key->list ? key->list : key); + if (!fse) + return NULL; + + EnterCriticalSection(&mutex); + /* add directory listing if it hasn't been added by some other thread */ + if (!hashmap_get(&map, key, NULL)) + fscache_add(fse); + + /* lookup file entry if requested (fse already points to directory) */ + if (key->list) + fse = hashmap_get(&map, key, NULL); + + /* return entry or ENOENT */ + if (fse) + fsentry_addref(fse); + else + errno = ENOENT; + + LeaveCriticalSection(&mutex); + return fse; +} + +/* + * Enables or disables the cache. Note that the cache is read-only, changes to + * the working directory are NOT reflected in the cache while enabled. + */ +int fscache_enable(int enable) +{ + int result; + + if (!initialized) { + /* allow the cache to be disabled entirely */ + if (!core_fscache) + return 0; + + InitializeCriticalSection(&mutex); + hashmap_init(&map, (hashmap_cmp_fn) fsentry_cmp, 0); + initialized = 1; + } + + result = enable ? InterlockedIncrement(&enabled) + : InterlockedDecrement(&enabled); + + if (enable && result == 1) { + /* redirect opendir and lstat to the fscache implementations */ + opendir = fscache_opendir; + lstat = fscache_lstat; + } else if (!enable && !result) { + /* reset opendir and lstat to the original implementations */ + opendir = dirent_opendir; + lstat = mingw_lstat; + EnterCriticalSection(&mutex); + fscache_clear(); + LeaveCriticalSection(&mutex); + } + return result; +} + +/* + * Lstat replacement, uses the cache if enabled, otherwise redirects to + * mingw_lstat. + */ +int fscache_lstat(const char *filename, struct stat *st) +{ + int dirlen, base, len; + struct fsentry key[2], *fse; + + if (!fscache_enabled(filename)) + return mingw_lstat(filename, st); + + /* split filename into path + name */ + len = strlen(filename); + if (len && is_dir_sep(filename[len - 1])) + len--; + base = len; + while (base && !is_dir_sep(filename[base - 1])) + base--; + dirlen = base ? base - 1 : 0; + + /* lookup entry for path + name in cache */ + fsentry_init(key, NULL, filename, dirlen); + fsentry_init(key + 1, key, filename + base, len - base); + fse = fscache_get(key + 1); + if (!fse) + return -1; + + /* copy stat data */ + st->st_ino = 0; + st->st_gid = 0; + st->st_uid = 0; + st->st_dev = 0; + st->st_rdev = 0; + st->st_nlink = 1; + st->st_mode = fse->st_mode; + st->st_size = fse->st_size; + st->st_atime = fse->st_atime; + st->st_mtime = fse->st_mtime; + st->st_ctime = fse->st_ctime; + + /* don't forget to release fsentry */ + fsentry_release(fse); + return 0; +} + +typedef struct fscache_DIR { + struct DIR base_dir; /* extend base struct DIR */ + struct fsentry *pfsentry; + struct dirent dirent; +} fscache_DIR; + +/* + * Readdir replacement. + */ +static struct dirent *fscache_readdir(DIR *base_dir) +{ + fscache_DIR *dir = (fscache_DIR*) base_dir; + struct fsentry *next = dir->pfsentry->next; + if (!next) + return NULL; + dir->pfsentry = next; + dir->dirent.d_type = S_ISDIR(next->st_mode) ? DT_DIR : DT_REG; + dir->dirent.d_name = (char*) next->name; + return &(dir->dirent); +} + +/* + * Closedir replacement. + */ +static int fscache_closedir(DIR *base_dir) +{ + fscache_DIR *dir = (fscache_DIR*) base_dir; + fsentry_release(dir->pfsentry); + free(dir); + return 0; +} + +/* + * Opendir replacement, uses a directory listing from the cache if enabled, + * otherwise calls original dirent implementation. + */ +DIR *fscache_opendir(const char *dirname) +{ + struct fsentry key, *list; + fscache_DIR *dir; + int len; + + if (!fscache_enabled(dirname)) + return dirent_opendir(dirname); + + /* prepare name (strip trailing '/', replace '.') */ + len = strlen(dirname); + if ((len == 1 && dirname[0] == '.') || + (len && is_dir_sep(dirname[len - 1]))) + len--; + + /* get directory listing from cache */ + fsentry_init(&key, NULL, dirname, len); + list = fscache_get(&key); + if (!list) + return NULL; + + /* alloc and return DIR structure */ + dir = (fscache_DIR*) xmalloc(sizeof(fscache_DIR)); + dir->base_dir.preaddir = fscache_readdir; + dir->base_dir.pclosedir = fscache_closedir; + dir->pfsentry = list; + return (DIR*) dir; +} diff --git a/compat/win32/fscache.h b/compat/win32/fscache.h new file mode 100644 index 0000000000..ed518b422d --- /dev/null +++ b/compat/win32/fscache.h @@ -0,0 +1,10 @@ +#ifndef FSCACHE_H +#define FSCACHE_H + +int fscache_enable(int enable); +#define enable_fscache(x) fscache_enable(x) + +DIR *fscache_opendir(const char *dir); +int fscache_lstat(const char *file_name, struct stat *buf); + +#endif diff --git a/config.mak.uname b/config.mak.uname index b232908f8c..c517b378cf 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -382,7 +382,7 @@ ifeq ($(uname_S),Windows) BASIC_CFLAGS = -nologo -I. -I../zlib -Icompat/vcbuild -Icompat/vcbuild/include -DWIN32 -D_CONSOLE -DHAVE_STRING_H -D_CRT_SECURE_NO_WARNINGS -D_CRT_NONSTDC_NO_DEPRECATE COMPAT_OBJS = compat/msvc.o compat/winansi.o \ compat/win32/pthread.o compat/win32/syslog.o \ - compat/win32/dirent.o + compat/win32/dirent.o compat/win32/fscache.o COMPAT_CFLAGS = -D__USE_MINGW_ACCESS -DNOGDI -DHAVE_STRING_H -Icompat -Icompat/regex -Icompat/win32 -DSTRIP_EXTENSION=\".exe\" BASIC_LDFLAGS = -IGNORE:4217 -IGNORE:4049 -NOLOGO -SUBSYSTEM:CONSOLE EXTLIBS = user32.lib advapi32.lib shell32.lib wininet.lib ws2_32.lib invalidcontinue.obj @@ -528,7 +528,7 @@ ifneq (,$(findstring MINGW,$(uname_S))) COMPAT_CFLAGS += -DSTRIP_EXTENSION=\".exe\" COMPAT_OBJS += compat/mingw.o compat/winansi.o \ compat/win32/pthread.o compat/win32/syslog.o \ - compat/win32/dirent.o + compat/win32/dirent.o compat/win32/fscache.o BASIC_CFLAGS += -DPROTECT_NTFS_DEFAULT=1 EXTLIBS += -lws2_32 GITLIBS += git.res diff --git a/git-compat-util.h b/git-compat-util.h index 7400068f37..1287ae5045 100644 --- a/git-compat-util.h +++ b/git-compat-util.h @@ -192,8 +192,10 @@ #if defined(__MINGW32__) /* pull in Windows compatibility stuff */ #include "compat/mingw.h" +#include "compat/win32/fscache.h" #elif defined(_MSC_VER) #include "compat/msvc.h" +#include "compat/win32/fscache.h" #else #include #include From 57c71a5c27085e3b4eb49fbd33c68055416a1529 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Tue, 24 Jun 2014 13:22:35 +0200 Subject: [PATCH 031/288] fscache: load directories only once If multiple threads access a directory that is not yet in the cache, the directory will be loaded by each thread. Only one of the results is added to the cache, all others are leaked. This wastes performance and memory. On cache miss, add a future object to the cache to indicate that the directory is currently being loaded. Subsequent threads register themselves with the future object and wait. When the first thread has loaded the directory, it replaces the future object with the result and notifies waiting threads. Signed-off-by: Karsten Blees --- compat/win32/fscache.c | 67 +++++++++++++++++++++++++++++++++++------- 1 file changed, 57 insertions(+), 10 deletions(-) diff --git a/compat/win32/fscache.c b/compat/win32/fscache.c index 28d22791bd..cbfcb18474 100644 --- a/compat/win32/fscache.c +++ b/compat/win32/fscache.c @@ -33,6 +33,8 @@ struct fsentry { union { /* Reference count of the directory listing. */ volatile long refcnt; + /* Handle to wait on the loading thread. */ + HANDLE hwait; struct { /* More stat members (only used for file entries). */ off64_t st_size; @@ -259,16 +261,43 @@ static inline int fscache_enabled(const char *path) return enabled > 0 && !is_absolute_path(path); } +/* + * Looks up a cache entry, waits if its being loaded by another thread. + * The mutex must be owned by the calling thread. + */ +static struct fsentry *fscache_get_wait(struct fsentry *key) +{ + struct fsentry *fse = hashmap_get(&map, key, NULL); + + /* return if its a 'real' entry (future entries have refcnt == 0) */ + if (!fse || fse->list || fse->refcnt) + return fse; + + /* create an event and link our key to the future entry */ + key->hwait = CreateEvent(NULL, TRUE, FALSE, NULL); + key->next = fse->next; + fse->next = key; + + /* wait for the loading thread to signal us */ + LeaveCriticalSection(&mutex); + WaitForSingleObject(key->hwait, INFINITE); + CloseHandle(key->hwait); + EnterCriticalSection(&mutex); + + /* repeat cache lookup */ + return hashmap_get(&map, key, NULL); +} + /* * Looks up or creates a cache entry for the specified key. */ static struct fsentry *fscache_get(struct fsentry *key) { - struct fsentry *fse; + struct fsentry *fse, *future, *waiter; EnterCriticalSection(&mutex); /* check if entry is in cache */ - fse = hashmap_get(&map, key, NULL); + fse = fscache_get_wait(key); if (fse) { fsentry_addref(fse); LeaveCriticalSection(&mutex); @@ -276,7 +305,7 @@ static struct fsentry *fscache_get(struct fsentry *key) } /* if looking for a file, check if directory listing is in cache */ if (!fse && key->list) { - fse = hashmap_get(&map, key->list, NULL); + fse = fscache_get_wait(key->list); if (fse) { LeaveCriticalSection(&mutex); /* dir entry without file entry -> file doesn't exist */ @@ -285,16 +314,34 @@ static struct fsentry *fscache_get(struct fsentry *key) } } + /* add future entry to indicate that we're loading it */ + future = key->list ? key->list : key; + future->next = NULL; + future->refcnt = 0; + hashmap_add(&map, future); + /* create the directory listing (outside mutex!) */ LeaveCriticalSection(&mutex); - fse = fsentry_create_list(key->list ? key->list : key); - if (!fse) - return NULL; - + fse = fsentry_create_list(future); EnterCriticalSection(&mutex); - /* add directory listing if it hasn't been added by some other thread */ - if (!hashmap_get(&map, key, NULL)) - fscache_add(fse); + + /* remove future entry and signal waiting threads */ + hashmap_remove(&map, future, NULL); + waiter = future->next; + while (waiter) { + HANDLE h = waiter->hwait; + waiter = waiter->next; + SetEvent(h); + } + + /* leave on error (errno set by fsentry_create_list) */ + if (!fse) { + LeaveCriticalSection(&mutex); + return NULL; + } + + /* add directory listing to the cache */ + fscache_add(fse); /* lookup file entry if requested (fse already points to directory) */ if (key->list) From 1ca846d5687f8c454950662ac00919fc5d402f9a Mon Sep 17 00:00:00 2001 From: Doug Kelly Date: Wed, 8 Jan 2014 20:28:15 -0600 Subject: [PATCH 032/288] Add a test demonstrating a problem with long submodule paths [jes: adusted test number to avoid conflicts, fixed non-portable use of the 'export' statement] Signed-off-by: Johannes Schindelin --- t/t7413-submodule-long-path.sh | 101 +++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100755 t/t7413-submodule-long-path.sh diff --git a/t/t7413-submodule-long-path.sh b/t/t7413-submodule-long-path.sh new file mode 100755 index 0000000000..6bb641cd4b --- /dev/null +++ b/t/t7413-submodule-long-path.sh @@ -0,0 +1,101 @@ +#!/bin/sh +# +# Copyright (c) 2013 Doug Kelly +# + +test_description='Test submodules with a path near PATH_MAX + +This test verifies that "git submodule" initialization, update and clones work, including with recursive submodules and paths approaching PATH_MAX (260 characters on Windows) +' + +TEST_NO_CREATE_REPO=1 +. ./test-lib.sh + +longpath="" +for (( i=0; i<4; i++ )); do + longpath="0123456789abcdefghijklmnopqrstuvwxyz$longpath" +done +# Pick a substring maximum of 90 characters +# This should be good, since we'll add on a lot for temp directories +longpath=${longpath:0:90}; export longpath + +test_expect_failure 'submodule with a long path' ' + git init --bare remote && + test_create_repo bundle1 && + ( + cd bundle1 && + test_commit "shoot" && + git rev-parse --verify HEAD >../expect + ) && + mkdir home && + ( + cd home && + git clone ../remote test && + cd test && + git submodule add ../bundle1 $longpath && + test_commit "sogood" && + ( + cd $longpath && + git rev-parse --verify HEAD >actual && + test_cmp ../../../expect actual + ) && + git push origin master + ) && + mkdir home2 && + ( + cd home2 && + git clone ../remote test && + cd test && + git checkout master && + git submodule update --init && + ( + cd $longpath && + git rev-parse --verify HEAD >actual && + test_cmp ../../../expect actual + ) + ) +' + +test_expect_failure 'recursive submodule with a long path' ' + git init --bare super && + test_create_repo child && + ( + cd child && + test_commit "shoot" && + git rev-parse --verify HEAD >../expect + ) && + test_create_repo parent && + ( + cd parent && + git submodule add ../child $longpath && + test_commit "aim" + ) && + mkdir home3 && + ( + cd home3 && + git clone ../super test && + cd test && + git submodule add ../parent foo && + git submodule update --init --recursive + test_commit "sogood" && + ( + cd foo/$longpath && + git rev-parse --verify HEAD >actual && + test_cmp ../../../../expect actual + ) && + git push origin master + ) && + mkdir home4 && + ( + cd home4 && + git clone ../super test --recursive && + ( + cd test/foo/$longpath && + git rev-parse --verify HEAD >actual && + test_cmp ../../../../expect actual + ) + ) +' +unset longpath + +test_done From aca04c405fccec6dbed07cc73cf5dce68f621c05 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 28 Jul 2015 21:07:41 +0200 Subject: [PATCH 033/288] Win32: support long paths Windows paths are typically limited to MAX_PATH = 260 characters, even though the underlying NTFS file system supports paths up to 32,767 chars. This limitation is also evident in Windows Explorer, cmd.exe and many other applications (including IDEs). Particularly annoying is that most Windows APIs return bogus error codes if a relative path only barely exceeds MAX_PATH in conjunction with the current directory, e.g. ERROR_PATH_NOT_FOUND / ENOENT instead of the infinitely more helpful ERROR_FILENAME_EXCED_RANGE / ENAMETOOLONG. Many Windows wide char APIs support longer than MAX_PATH paths through the file namespace prefix ('\\?\' or '\\?\UNC\') followed by an absolute path. Notable exceptions include functions dealing with executables and the current directory (CreateProcess, LoadLibrary, Get/SetCurrentDirectory) as well as the entire shell API (ShellExecute, SHGetSpecialFolderPath...). Introduce a handle_long_path function to check the length of a specified path properly (and fail with ENAMETOOLONG), and to optionally expand long paths using the '\\?\' file namespace prefix. Short paths will not be modified, so we don't need to worry about device names (NUL, CON, AUX). Contrary to MSDN docs, the GetFullPathNameW function doesn't seem to be limited to MAX_PATH (at least not on Win7), so we can use it to do the heavy lifting of the conversion (translate '/' to '\', eliminate '.' and '..', and make an absolute path). Add long path error checking to xutftowcs_path for APIs with hard MAX_PATH limit. Add a new MAX_LONG_PATH constant and xutftowcs_long_path function for APIs that support long paths. While improved error checking is always active, long paths support must be explicitly enabled via 'core.longpaths' option. This is to prevent end users to shoot themselves in the foot by checking out files that Windows Explorer, cmd/bash or their favorite IDE cannot handle. Test suite: Test the case is when the full pathname length of a dir is close to 260 (MAX_PATH). Bug report and an original reproducer by Andrey Rogozhnikov: https://github.com/msysgit/git/pull/122#issuecomment-43604199 [jes: adjusted test number to avoid conflicts] Thanks-to: Martin W. Kirst Thanks-to: Doug Kelly Signed-off-by: Karsten Blees Original-test-by: Andrey Rogozhnikov Signed-off-by: Stepan Kasal Signed-off-by: Johannes Schindelin --- Documentation/config.txt | 7 ++ compat/mingw.c | 138 ++++++++++++++++++++++++++------- compat/mingw.h | 75 ++++++++++++++++-- compat/win32/dirent.c | 14 ++-- compat/win32/fscache.c | 17 ++-- t/t2027-checkout-long-paths.sh | 99 +++++++++++++++++++++++ t/t7413-submodule-long-path.sh | 24 +++--- 7 files changed, 319 insertions(+), 55 deletions(-) create mode 100755 t/t2027-checkout-long-paths.sh diff --git a/Documentation/config.txt b/Documentation/config.txt index b87029e79e..667c4bfc83 100644 --- a/Documentation/config.txt +++ b/Documentation/config.txt @@ -766,6 +766,13 @@ core.fscache:: Git for Windows uses this to bulk-read and cache lstat data of entire directories (instead of doing lstat file by file). +core.longpaths:: + Enable long path (> 260) support for builtin commands in Git for + Windows. This is disabled by default, as long paths are not supported + by Windows Explorer, cmd.exe and the Git for Windows tool chain + (msys, bash, tcl, perl...). Only enable this if you know what you're + doing and are prepared to live with a few quirks. + core.unsetenvvars:: EXPERIMENTAL, Windows-only: comma-separated list of environment variables' names that need to be unset before spawning any other diff --git a/compat/mingw.c b/compat/mingw.c index 918420964f..219b601b10 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -212,6 +212,7 @@ enum hide_dotfiles_type { static enum hide_dotfiles_type hide_dotfiles = HIDE_DOTFILES_DOTGITONLY; static char *unset_environment_variables; int core_fscache; +int core_long_paths; int mingw_core_config(const char *var, const char *value) { @@ -228,6 +229,11 @@ int mingw_core_config(const char *var, const char *value) return 0; } + if (!strcmp(var, "core.longpaths")) { + core_long_paths = git_config_bool(var, value); + return 0; + } + if (!strcmp(var, "core.unsetenvvars")) { free(unset_environment_variables); unset_environment_variables = xstrdup(value); @@ -240,8 +246,8 @@ int mingw_core_config(const char *var, const char *value) int mingw_unlink(const char *pathname) { int ret, tries = 0; - wchar_t wpathname[MAX_PATH]; - if (xutftowcs_path(wpathname, pathname) < 0) + wchar_t wpathname[MAX_LONG_PATH]; + if (xutftowcs_long_path(wpathname, pathname) < 0) return -1; /* read-only files cannot be removed */ @@ -270,7 +276,7 @@ static int is_dir_empty(const wchar_t *wpath) { WIN32_FIND_DATAW findbuf; HANDLE handle; - wchar_t wbuf[MAX_PATH + 2]; + wchar_t wbuf[MAX_LONG_PATH + 2]; wcscpy(wbuf, wpath); wcscat(wbuf, L"\\*"); handle = FindFirstFileW(wbuf, &findbuf); @@ -291,8 +297,8 @@ static int is_dir_empty(const wchar_t *wpath) int mingw_rmdir(const char *pathname) { int ret, tries = 0; - wchar_t wpathname[MAX_PATH]; - if (xutftowcs_path(wpathname, pathname) < 0) + wchar_t wpathname[MAX_LONG_PATH]; + if (xutftowcs_long_path(wpathname, pathname) < 0) return -1; while ((ret = _wrmdir(wpathname)) == -1 && tries < ARRAY_SIZE(delay)) { @@ -367,9 +373,12 @@ static int set_hidden_flag(const wchar_t *path, int set) int mingw_mkdir(const char *path, int mode) { int ret; - wchar_t wpath[MAX_PATH]; - if (xutftowcs_path(wpath, path) < 0) + wchar_t wpath[MAX_LONG_PATH]; + /* CreateDirectoryW path limit is 248 (MAX_PATH - 8.3 file name) */ + if (xutftowcs_path_ex(wpath, path, MAX_LONG_PATH, -1, 248, + core_long_paths) < 0) return -1; + ret = _wmkdir(wpath); if (!ret && needs_hiding(path)) return set_hidden_flag(wpath, 1); @@ -381,7 +390,7 @@ int mingw_open (const char *filename, int oflags, ...) va_list args; unsigned mode; int fd; - wchar_t wfilename[MAX_PATH]; + wchar_t wfilename[MAX_LONG_PATH]; va_start(args, oflags); mode = va_arg(args, int); @@ -390,7 +399,7 @@ int mingw_open (const char *filename, int oflags, ...) if (filename && !strcmp(filename, "/dev/null")) filename = "nul"; - if (xutftowcs_path(wfilename, filename) < 0) + if (xutftowcs_long_path(wfilename, filename) < 0) return -1; fd = _wopen(wfilename, oflags, mode); @@ -447,10 +456,10 @@ FILE *mingw_fopen (const char *filename, const char *otype) { int hide = needs_hiding(filename); FILE *file; - wchar_t wfilename[MAX_PATH], wotype[4]; + wchar_t wfilename[MAX_LONG_PATH], wotype[4]; if (filename && !strcmp(filename, "/dev/null")) filename = "nul"; - if (xutftowcs_path(wfilename, filename) < 0 || + if (xutftowcs_long_path(wfilename, filename) < 0 || xutftowcs(wotype, otype, ARRAY_SIZE(wotype)) < 0) return NULL; if (hide && !access(filename, F_OK) && set_hidden_flag(wfilename, 0)) { @@ -467,10 +476,10 @@ FILE *mingw_freopen (const char *filename, const char *otype, FILE *stream) { int hide = needs_hiding(filename); FILE *file; - wchar_t wfilename[MAX_PATH], wotype[4]; + wchar_t wfilename[MAX_LONG_PATH], wotype[4]; if (filename && !strcmp(filename, "/dev/null")) filename = "nul"; - if (xutftowcs_path(wfilename, filename) < 0 || + if (xutftowcs_long_path(wfilename, filename) < 0 || xutftowcs(wotype, otype, ARRAY_SIZE(wotype)) < 0) return NULL; if (hide && !access(filename, F_OK) && set_hidden_flag(wfilename, 0)) { @@ -524,25 +533,32 @@ ssize_t mingw_write(int fd, const void *buf, size_t len) int mingw_access(const char *filename, int mode) { - wchar_t wfilename[MAX_PATH]; - if (xutftowcs_path(wfilename, filename) < 0) + wchar_t wfilename[MAX_LONG_PATH]; + if (xutftowcs_long_path(wfilename, filename) < 0) return -1; /* X_OK is not supported by the MSVCRT version */ return _waccess(wfilename, mode & ~X_OK); } +/* cached length of current directory for handle_long_path */ +static int current_directory_len = 0; + int mingw_chdir(const char *dirname) { + int result; wchar_t wdirname[MAX_PATH]; + /* SetCurrentDirectoryW doesn't support long paths */ if (xutftowcs_path(wdirname, dirname) < 0) return -1; - return _wchdir(wdirname); + result = _wchdir(wdirname); + current_directory_len = GetCurrentDirectoryW(0, NULL); + return result; } int mingw_chmod(const char *filename, int mode) { - wchar_t wfilename[MAX_PATH]; - if (xutftowcs_path(wfilename, filename) < 0) + wchar_t wfilename[MAX_LONG_PATH]; + if (xutftowcs_long_path(wfilename, filename) < 0) return -1; return _wchmod(wfilename, mode); } @@ -590,8 +606,8 @@ static int has_valid_directory_prefix(wchar_t *wfilename) static int do_lstat(int follow, const char *file_name, struct stat *buf) { WIN32_FILE_ATTRIBUTE_DATA fdata; - wchar_t wfilename[MAX_PATH]; - if (xutftowcs_path(wfilename, file_name) < 0) + wchar_t wfilename[MAX_LONG_PATH]; + if (xutftowcs_long_path(wfilename, file_name) < 0) return -1; if (GetFileAttributesExW(wfilename, GetFileExInfoStandard, &fdata)) { @@ -740,8 +756,8 @@ int mingw_utime (const char *file_name, const struct utimbuf *times) FILETIME mft, aft; int fh, rc; DWORD attrs; - wchar_t wfilename[MAX_PATH]; - if (xutftowcs_path(wfilename, file_name) < 0) + wchar_t wfilename[MAX_LONG_PATH]; + if (xutftowcs_long_path(wfilename, file_name) < 0) return -1; /* must have write permission */ @@ -789,6 +805,7 @@ unsigned int sleep (unsigned int seconds) char *mingw_mktemp(char *template) { wchar_t wtemplate[MAX_PATH]; + /* we need to return the path, thus no long paths here! */ if (xutftowcs_path(wtemplate, template) < 0) return NULL; if (!_wmktemp(wtemplate)) @@ -1163,6 +1180,7 @@ static pid_t mingw_spawnve_fd(const char *cmd, const char **argv, char **deltaen si.hStdOutput = winansi_get_osfhandle(fhout); si.hStdError = winansi_get_osfhandle(fherr); + /* executables and the current directory don't support long paths */ if (xutftowcs_path(wcmd, cmd) < 0) return -1; if (dir && xutftowcs_path(wdir, dir) < 0) @@ -1738,8 +1756,9 @@ int mingw_rename(const char *pold, const char *pnew) { DWORD attrs, gle; int tries = 0; - wchar_t wpold[MAX_PATH], wpnew[MAX_PATH]; - if (xutftowcs_path(wpold, pold) < 0 || xutftowcs_path(wpnew, pnew) < 0) + wchar_t wpold[MAX_LONG_PATH], wpnew[MAX_LONG_PATH]; + if (xutftowcs_long_path(wpold, pold) < 0 || + xutftowcs_long_path(wpnew, pnew) < 0) return -1; /* @@ -1979,9 +1998,9 @@ int link(const char *oldpath, const char *newpath) { typedef BOOL (WINAPI *T)(LPCWSTR, LPCWSTR, LPSECURITY_ATTRIBUTES); static T create_hard_link = NULL; - wchar_t woldpath[MAX_PATH], wnewpath[MAX_PATH]; - if (xutftowcs_path(woldpath, oldpath) < 0 || - xutftowcs_path(wnewpath, newpath) < 0) + wchar_t woldpath[MAX_LONG_PATH], wnewpath[MAX_LONG_PATH]; + if (xutftowcs_long_path(woldpath, oldpath) < 0 || + xutftowcs_long_path(wnewpath, newpath) < 0) return -1; if (!create_hard_link) { @@ -2194,6 +2213,68 @@ static void setup_windows_environment(void) setenv("TERM", "cygwin", 1); } +int handle_long_path(wchar_t *path, int len, int max_path, int expand) +{ + int result; + wchar_t buf[MAX_LONG_PATH]; + + /* + * we don't need special handling if path is relative to the current + * directory, and current directory + path don't exceed the desired + * max_path limit. This should cover > 99 % of cases with minimal + * performance impact (git almost always uses relative paths). + */ + if ((len < 2 || (!is_dir_sep(path[0]) && path[1] != ':')) && + (current_directory_len + len < max_path)) + return len; + + /* + * handle everything else: + * - absolute paths: "C:\dir\file" + * - absolute UNC paths: "\\server\share\dir\file" + * - absolute paths on current drive: "\dir\file" + * - relative paths on other drive: "X:file" + * - prefixed paths: "\\?\...", "\\.\..." + */ + + /* convert to absolute path using GetFullPathNameW */ + result = GetFullPathNameW(path, MAX_LONG_PATH, buf, NULL); + if (!result) { + errno = err_win_to_posix(GetLastError()); + return -1; + } + + /* + * return absolute path if it fits within max_path (even if + * "cwd + path" doesn't due to '..' components) + */ + if (result < max_path) { + wcscpy(path, buf); + return result; + } + + /* error out if we shouldn't expand the path or buf is too small */ + if (!expand || result >= MAX_LONG_PATH - 6) { + errno = ENAMETOOLONG; + return -1; + } + + /* prefix full path with "\\?\" or "\\?\UNC\" */ + if (buf[0] == '\\') { + /* ...unless already prefixed */ + if (buf[1] == '\\' && (buf[2] == '?' || buf[2] == '.')) + return len; + + wcscpy(path, L"\\\\?\\UNC\\"); + wcscpy(path + 8, buf + 2); + return result + 6; + } else { + wcscpy(path, L"\\\\?\\"); + wcscpy(path + 4, buf); + return result + 4; + } +} + /* * Disable MSVCRT command line wildcard expansion (__getmainargs called from * mingw startup code, see init.c in mingw runtime). @@ -2287,6 +2368,9 @@ void mingw_startup(void) /* initialize Unicode console */ winansi_init(); + + /* init length of current directory for handle_long_path */ + current_directory_len = GetCurrentDirectoryW(0, NULL); } int uname(struct utsname *buf) diff --git a/compat/mingw.h b/compat/mingw.h index cd20e0b449..018bb38eb2 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -12,6 +12,7 @@ typedef _sigset_t sigset_t; #endif extern int core_fscache; +extern int core_long_paths; extern int mingw_core_config(const char *var, const char *value); #define platform_core_config mingw_core_config @@ -445,6 +446,42 @@ int mingw_offset_1st_component(const char *path); #include #endif +/** + * Max length of long paths (exceeding MAX_PATH). The actual maximum supported + * by NTFS is 32,767 (* sizeof(wchar_t)), but we choose an arbitrary smaller + * value to limit required stack memory. + */ +#define MAX_LONG_PATH 4096 + +/** + * Handles paths that would exceed the MAX_PATH limit of Windows Unicode APIs. + * + * With expand == false, the function checks for over-long paths and fails + * with ENAMETOOLONG. The path parameter is not modified, except if cwd + path + * exceeds max_path, but the resulting absolute path doesn't (e.g. due to + * eliminating '..' components). The path parameter must point to a buffer + * of max_path wide characters. + * + * With expand == true, an over-long path is automatically converted in place + * to an absolute path prefixed with '\\?\', and the new length is returned. + * The path parameter must point to a buffer of MAX_LONG_PATH wide characters. + * + * Parameters: + * path: path to check and / or convert + * len: size of path on input (number of wide chars without \0) + * max_path: max short path length to check (usually MAX_PATH = 260, but just + * 248 for CreateDirectoryW) + * expand: false to only check the length, true to expand the path to a + * '\\?\'-prefixed absolute path + * + * Return: + * length of the resulting path, or -1 on failure + * + * Errors: + * ENAMETOOLONG if path is too long + */ +int handle_long_path(wchar_t *path, int len, int max_path, int expand); + /** * Converts UTF-8 encoded string to UTF-16LE. * @@ -502,17 +539,45 @@ static inline int xutftowcs(wchar_t *wcs, const char *utf, size_t wcslen) return xutftowcsn(wcs, utf, wcslen, -1); } +/** + * Simplified file system specific wrapper of xutftowcsn and handle_long_path. + * Converts ERANGE to ENAMETOOLONG. If expand is true, wcs must be at least + * MAX_LONG_PATH wide chars (see handle_long_path). + */ +static inline int xutftowcs_path_ex(wchar_t *wcs, const char *utf, + size_t wcslen, int utflen, int max_path, int expand) +{ + int result = xutftowcsn(wcs, utf, wcslen, utflen); + if (result < 0 && errno == ERANGE) + errno = ENAMETOOLONG; + if (result >= 0) + result = handle_long_path(wcs, result, max_path, expand); + return result; +} + /** * Simplified file system specific variant of xutftowcsn, assumes output * buffer size is MAX_PATH wide chars and input string is \0-terminated, - * fails with ENAMETOOLONG if input string is too long. + * fails with ENAMETOOLONG if input string is too long. Typically used for + * Windows APIs that don't support long paths, e.g. SetCurrentDirectory, + * LoadLibrary, CreateProcess... */ static inline int xutftowcs_path(wchar_t *wcs, const char *utf) { - int result = xutftowcsn(wcs, utf, MAX_PATH, -1); - if (result < 0 && errno == ERANGE) - errno = ENAMETOOLONG; - return result; + return xutftowcs_path_ex(wcs, utf, MAX_PATH, -1, MAX_PATH, 0); +} + +/** + * Simplified file system specific variant of xutftowcsn for Windows APIs + * that support long paths via '\\?\'-prefix, assumes output buffer size is + * MAX_LONG_PATH wide chars, fails with ENAMETOOLONG if input string is too + * long. The 'core.longpaths' git-config option controls whether the path + * is only checked or expanded to a long path. + */ +static inline int xutftowcs_long_path(wchar_t *wcs, const char *utf) +{ + return xutftowcs_path_ex(wcs, utf, MAX_LONG_PATH, -1, MAX_PATH, + core_long_paths); } /** diff --git a/compat/win32/dirent.c b/compat/win32/dirent.c index 6b87042182..b3bd8d7af7 100644 --- a/compat/win32/dirent.c +++ b/compat/win32/dirent.c @@ -63,19 +63,23 @@ static int dirent_closedir(dirent_DIR *dir) DIR *dirent_opendir(const char *name) { - wchar_t pattern[MAX_PATH + 2]; /* + 2 for '/' '*' */ + wchar_t pattern[MAX_LONG_PATH + 2]; /* + 2 for "\*" */ WIN32_FIND_DATAW fdata; HANDLE h; int len; dirent_DIR *dir; - /* convert name to UTF-16 and check length < MAX_PATH */ - if ((len = xutftowcs_path(pattern, name)) < 0) + /* convert name to UTF-16 and check length */ + if ((len = xutftowcs_path_ex(pattern, name, MAX_LONG_PATH, -1, + MAX_PATH - 2, core_long_paths)) < 0) return NULL; - /* append optional '/' and wildcard '*' */ + /* + * append optional '\' and wildcard '*'. Note: we need to use '\' as + * Windows doesn't translate '/' to '\' for "\\?\"-prefixed paths. + */ if (len && !is_dir_sep(pattern[len - 1])) - pattern[len++] = '/'; + pattern[len++] = '\\'; pattern[len++] = '*'; pattern[len] = 0; diff --git a/compat/win32/fscache.c b/compat/win32/fscache.c index cbfcb18474..63d522f49a 100644 --- a/compat/win32/fscache.c +++ b/compat/win32/fscache.c @@ -164,23 +164,24 @@ static struct fsentry *fseentry_create_entry(struct fsentry *list, */ static struct fsentry *fsentry_create_list(const struct fsentry *dir) { - wchar_t pattern[MAX_PATH + 2]; /* + 2 for '/' '*' */ + wchar_t pattern[MAX_LONG_PATH + 2]; /* + 2 for "\*" */ WIN32_FIND_DATAW fdata; HANDLE h; int wlen; struct fsentry *list, **phead; DWORD err; - /* convert name to UTF-16 and check length < MAX_PATH */ - if ((wlen = xutftowcsn(pattern, dir->name, MAX_PATH, dir->len)) < 0) { - if (errno == ERANGE) - errno = ENAMETOOLONG; + /* convert name to UTF-16 and check length */ + if ((wlen = xutftowcs_path_ex(pattern, dir->name, MAX_LONG_PATH, + dir->len, MAX_PATH - 2, core_long_paths)) < 0) return NULL; - } - /* append optional '/' and wildcard '*' */ + /* + * append optional '\' and wildcard '*'. Note: we need to use '\' as + * Windows doesn't translate '/' to '\' for "\\?\"-prefixed paths. + */ if (wlen) - pattern[wlen++] = '/'; + pattern[wlen++] = '\\'; pattern[wlen++] = '*'; pattern[wlen] = 0; diff --git a/t/t2027-checkout-long-paths.sh b/t/t2027-checkout-long-paths.sh new file mode 100755 index 0000000000..e65a38da4d --- /dev/null +++ b/t/t2027-checkout-long-paths.sh @@ -0,0 +1,99 @@ +#!/bin/sh + +test_description='checkout long paths on Windows + +Ensures that Git for Windows can deal with long paths (>260) enabled via core.longpaths' + +. ./test-lib.sh + +if test_have_prereq !MINGW +then + skip_all='skipping MINGW specific long paths test' + test_done +fi + +test_expect_success setup ' + p=longpathxx && # -> 10 + p=$p$p$p$p$p && # -> 50 + p=$p$p$p$p$p && # -> 250 + + path=${p}/longtestfile && # -> 263 (MAX_PATH = 260) + + blob=$(echo foobar | git hash-object -w --stdin) && + + printf "100644 %s 0\t%s\n" "$blob" "$path" | + git update-index --add --index-info && + git commit -m initial -q +' + +test_expect_success 'checkout of long paths without core.longpaths fails' ' + git config core.longpaths false && + test_must_fail git checkout -f 2>error && + grep -q "Filename too long" error && + test_path_is_missing longpa~1/longtestfile +' + +test_expect_success 'checkout of long paths with core.longpaths works' ' + git config core.longpaths true && + git checkout -f && + test_path_is_file longpa~1/longtestfile +' + +test_expect_success 'update of long paths' ' + echo frotz >> longpa~1/longtestfile && + echo $path > expect && + git ls-files -m > actual && + test_cmp expect actual && + git add $path && + git commit -m second && + git grep "frotz" HEAD -- $path +' + +test_expect_success cleanup ' + # bash cannot delete the trash dir if it contains a long path + # lets help cleaning up (unless in debug mode) + test ! -z "$debug" || rm -rf longpa~1 +' + +# check that the template used in the test won't be too long: +abspath="$(pwd -W)"/testdir +test ${#abspath} -gt 230 || +test_set_prereq SHORTABSPATH + +test_expect_success SHORTABSPATH 'clean up path close to MAX_PATH' ' + p=/123456789abcdef/123456789abcdef/123456789abcdef/123456789abc/ef && + p=y$p$p$p$p && + subdir="x$(echo "$p" | tail -c $((253 - ${#abspath})) - )" && + # Now, $abspath/$subdir has exactly 254 characters, and is inside CWD + p2="$abspath/$subdir" && + test 254 = ${#p2} && + + # Be careful to overcome path limitations of the MSys tools and split + # the $subdir into two parts. ($subdir2 has to contain 16 chars and a + # slash somewhere following; that is why we asked for abspath <= 230 and + # why we placed a slash near the end of the $subdir template.) + subdir2=${subdir#????????????????*/} && + subdir1=testdir/${subdir%/$subdir2} && + mkdir -p "$subdir1" && + i=0 && + # The most important case is when absolute path is 258 characters long, + # and that will be when i == 4. + while test $i -le 7 + do + mkdir -p $subdir2 && + touch $subdir2/one-file && + mv ${subdir2%%/*} "$subdir1/" && + subdir2=z${subdir2} && + i=$(($i+1)) || + exit 1 + done && + + # now check that git is able to clear the tree: + (cd testdir && + git init && + git config core.longpaths yes && + git clean -fdx) && + test ! -d "$subdir1" +' + +test_done diff --git a/t/t7413-submodule-long-path.sh b/t/t7413-submodule-long-path.sh index 6bb641cd4b..59b3d347db 100755 --- a/t/t7413-submodule-long-path.sh +++ b/t/t7413-submodule-long-path.sh @@ -11,15 +11,20 @@ This test verifies that "git submodule" initialization, update and clones work, TEST_NO_CREATE_REPO=1 . ./test-lib.sh -longpath="" -for (( i=0; i<4; i++ )); do - longpath="0123456789abcdefghijklmnopqrstuvwxyz$longpath" -done -# Pick a substring maximum of 90 characters -# This should be good, since we'll add on a lot for temp directories -longpath=${longpath:0:90}; export longpath +# cloning a submodule calls is_git_directory("$path/../.git/modules/$path"), +# which effectively limits the maximum length to PATH_MAX / 2 minus some +# overhead; start with 3 * 36 = 108 chars (test 2 fails if >= 110) +longpath36=0123456789abcdefghijklmnopqrstuvwxyz +longpath180=$longpath36$longpath36$longpath36$longpath36$longpath36 -test_expect_failure 'submodule with a long path' ' +# the git database must fit within PATH_MAX, which limits the submodule name +# to PATH_MAX - len(pwd) - ~90 (= len("/objects//") + 40-byte sha1 + some +# overhead from the test case) +pwd=$(pwd) +pwdlen=$(echo "$pwd" | wc -c) +longpath=$(echo $longpath180 | cut -c 1-$((170-$pwdlen))) + +test_expect_success 'submodule with a long path' ' git init --bare remote && test_create_repo bundle1 && ( @@ -56,7 +61,7 @@ test_expect_failure 'submodule with a long path' ' ) ' -test_expect_failure 'recursive submodule with a long path' ' +test_expect_success 'recursive submodule with a long path' ' git init --bare super && test_create_repo child && ( @@ -96,6 +101,5 @@ test_expect_failure 'recursive submodule with a long path' ' ) ) ' -unset longpath test_done From b47616d2099012dfbf4ef92d6e648bb91326a3dc Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sun, 30 Aug 2015 18:49:05 +0200 Subject: [PATCH 034/288] Win32: support long paths Windows paths are typically limited to MAX_PATH = 260 characters, even though the underlying NTFS file system supports paths up to 32,767 chars. This limitation is also evident in Windows Explorer, cmd.exe and many other applications (including IDEs). Particularly annoying is that most Windows APIs return bogus error codes if a relative path only barely exceeds MAX_PATH in conjunction with the current directory, e.g. ERROR_PATH_NOT_FOUND / ENOENT instead of the infinitely more helpful ERROR_FILENAME_EXCED_RANGE / ENAMETOOLONG. Many Windows wide char APIs support longer than MAX_PATH paths through the file namespace prefix ('\\?\' or '\\?\UNC\') followed by an absolute path. Notable exceptions include functions dealing with executables and the current directory (CreateProcess, LoadLibrary, Get/SetCurrentDirectory) as well as the entire shell API (ShellExecute, SHGetSpecialFolderPath...). Introduce a handle_long_path function to check the length of a specified path properly (and fail with ENAMETOOLONG), and to optionally expand long paths using the '\\?\' file namespace prefix. Short paths will not be modified, so we don't need to worry about device names (NUL, CON, AUX). Contrary to MSDN docs, the GetFullPathNameW function doesn't seem to be limited to MAX_PATH (at least not on Win7), so we can use it to do the heavy lifting of the conversion (translate '/' to '\', eliminate '.' and '..', and make an absolute path). Add long path error checking to xutftowcs_path for APIs with hard MAX_PATH limit. Add a new MAX_LONG_PATH constant and xutftowcs_long_path function for APIs that support long paths. While improved error checking is always active, long paths support must be explicitly enabled via 'core.longpaths' option. This is to prevent end users to shoot themselves in the foot by checking out files that Windows Explorer, cmd/bash or their favorite IDE cannot handle. Test suite: Test the case is when the full pathname length of a dir is close to 260 (MAX_PATH). Bug report and an original reproducer by Andrey Rogozhnikov: https://github.com/msysgit/git/pull/122#issuecomment-43604199 Note that the test cannot rely on the presence of short names, as they are not enabled by default except on the system drive. [jes: adjusted test number to avoid conflicts, reinstated && chain, adjusted test to work without short names] Thanks-to: Martin W. Kirst Thanks-to: Doug Kelly Signed-off-by: Karsten Blees Original-test-by: Andrey Rogozhnikov Signed-off-by: Stepan Kasal Signed-off-by: Johannes Schindelin --- compat/mingw.c | 5 ++--- ...t-long-paths.sh => t2029-checkout-long-paths.sh} | 13 +++++++++---- 2 files changed, 11 insertions(+), 7 deletions(-) rename t/{t2027-checkout-long-paths.sh => t2029-checkout-long-paths.sh} (93%) diff --git a/compat/mingw.c b/compat/mingw.c index 219b601b10..93197f741c 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -546,9 +546,8 @@ static int current_directory_len = 0; int mingw_chdir(const char *dirname) { int result; - wchar_t wdirname[MAX_PATH]; - /* SetCurrentDirectoryW doesn't support long paths */ - if (xutftowcs_path(wdirname, dirname) < 0) + wchar_t wdirname[MAX_LONG_PATH]; + if (xutftowcs_long_path(wdirname, dirname) < 0) return -1; result = _wchdir(wdirname); current_directory_len = GetCurrentDirectoryW(0, NULL); diff --git a/t/t2027-checkout-long-paths.sh b/t/t2029-checkout-long-paths.sh similarity index 93% rename from t/t2027-checkout-long-paths.sh rename to t/t2029-checkout-long-paths.sh index e65a38da4d..18e5eabe15 100755 --- a/t/t2027-checkout-long-paths.sh +++ b/t/t2029-checkout-long-paths.sh @@ -30,17 +30,19 @@ test_expect_success 'checkout of long paths without core.longpaths fails' ' git config core.longpaths false && test_must_fail git checkout -f 2>error && grep -q "Filename too long" error && - test_path_is_missing longpa~1/longtestfile + test ! -d longpa* ' test_expect_success 'checkout of long paths with core.longpaths works' ' git config core.longpaths true && git checkout -f && - test_path_is_file longpa~1/longtestfile + (cd longpa* && + test_path_is_file longtestfile) ' test_expect_success 'update of long paths' ' - echo frotz >> longpa~1/longtestfile && + (cd longpa* && + echo frotz >> longtestfile) && echo $path > expect && git ls-files -m > actual && test_cmp expect actual && @@ -52,7 +54,10 @@ test_expect_success 'update of long paths' ' test_expect_success cleanup ' # bash cannot delete the trash dir if it contains a long path # lets help cleaning up (unless in debug mode) - test ! -z "$debug" || rm -rf longpa~1 + if test -z "$debug" + then + rm -rf longpa~1 + fi ' # check that the template used in the test won't be too long: From 15bb26c5035c9a0e07563f7903dffed2b31b1374 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sat, 5 Jul 2014 00:00:36 +0200 Subject: [PATCH 035/288] Win32: fix 'lstat("dir/")' with long paths Use a suffciently large buffer to strip the trailing slash. Signed-off-by: Karsten Blees --- compat/mingw.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 93197f741c..b5bc7c91ab 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -677,7 +677,7 @@ static int do_lstat(int follow, const char *file_name, struct stat *buf) static int do_stat_internal(int follow, const char *file_name, struct stat *buf) { int namelen; - char alt_name[PATH_MAX]; + char alt_name[MAX_LONG_PATH]; if (!do_lstat(follow, file_name, buf)) return 0; @@ -693,7 +693,7 @@ static int do_stat_internal(int follow, const char *file_name, struct stat *buf) return -1; while (namelen && file_name[namelen-1] == '/') --namelen; - if (!namelen || namelen >= PATH_MAX) + if (!namelen || namelen >= MAX_LONG_PATH) return -1; memcpy(alt_name, file_name, namelen); From 2b35ed673b172dc3b45418ff35eed4a3e03832fb Mon Sep 17 00:00:00 2001 From: nalla Date: Wed, 15 Apr 2015 07:54:39 +0100 Subject: [PATCH 036/288] asciidoctor: fix user-manual to be built by `asciidoctor` The `user-manual.txt` is designed as a `book` but the `Makefile` wants to build it as an `article`. This seems to be a problem when building the documentation with `asciidoctor`. Furthermore the parts *Git Glossary* and *Appendix B* had no subsections which is not allowed when building with `asciidoctor`. So lets add a *dummy* section. Signed-off-by: nalla --- Documentation/Makefile | 2 +- Documentation/user-manual.txt | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/Documentation/Makefile b/Documentation/Makefile index b43d66eae6..a9fb497b83 100644 --- a/Documentation/Makefile +++ b/Documentation/Makefile @@ -337,7 +337,7 @@ manpage-base-url.xsl: manpage-base-url.xsl.in user-manual.xml: user-manual.txt user-manual.conf $(QUIET_ASCIIDOC)$(RM) $@+ $@ && \ - $(TXT_TO_XML) -d article -o $@+ $< && \ + $(TXT_TO_XML) -d book -o $@+ $< && \ mv $@+ $@ technical/api-index.txt: technical/api-index-skel.txt \ diff --git a/Documentation/user-manual.txt b/Documentation/user-manual.txt index 5e07454572..bc29298678 100644 --- a/Documentation/user-manual.txt +++ b/Documentation/user-manual.txt @@ -4395,6 +4395,10 @@ itself! Git Glossary ============ +[[git-explained]] +Git explained +------------- + include::glossary-content.txt[] [[git-quick-start]] @@ -4636,6 +4640,10 @@ $ git gc Appendix B: Notes and todo list for this manual =============================================== +[[todo-list]] +Todo list +--------- + This is a work in progress. The basic requirements: From 009750a0ca2d78de74bc32698706861edb3163f7 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 9 Nov 2016 12:45:00 +0100 Subject: [PATCH 037/288] giteveryday: unbreak rendering with AsciiDoctor The "giteveryday" document has a callout list that contains a code block. This is not a problem for AsciiDoc, but AsciiDoctor sadly was explicitly designed *not* to render this correctly [*1*]. The symptom is an unhelpful line 322: callout list item index: expected 1 got 12 line 325: no callouts refer to list item 1 line 325: callout list item index: expected 2 got 13 line 327: no callouts refer to list item 2 In Git for Windows, we rely on the speed improvement of AsciiDoctor (on this developer's machine, `make -j15 html` takes roughly 30 seconds with AsciiDoctor, 70 seconds with AsciiDoc), therefore we need a way to render this correctly. The easiest way out is to simplify the callout list, as suggested by AsciiDoctor's author, even while one may very well disagree with him that a code block hath no place in a callout list. *1*: https://github.com/asciidoctor/asciidoctor/issues/1478 Signed-off-by: Johannes Schindelin --- Documentation/giteveryday.txt | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/Documentation/giteveryday.txt b/Documentation/giteveryday.txt index 35473ad02f..10c8ff93c0 100644 --- a/Documentation/giteveryday.txt +++ b/Documentation/giteveryday.txt @@ -307,9 +307,16 @@ master or exposed as a part of a stable branch. <9> backport a critical fix. <10> create a signed tag. <11> make sure master was not accidentally rewound beyond that -already pushed out. `ko` shorthand points at the Git maintainer's +already pushed out. +<12> In the output from `git show-branch`, `master` should have +everything `ko/master` has, and `next` should have +everything `ko/next` has, etc. +<13> push out the bleeding edge, together with new tags that point +into the pushed history. + +In this example, the `ko` shorthand points at the Git maintainer's repository at kernel.org, and looks like this: -+ + ------------ (in .git/config) [remote "ko"] @@ -320,12 +327,6 @@ repository at kernel.org, and looks like this: push = +refs/heads/pu push = refs/heads/maint ------------ -+ -<12> In the output from `git show-branch`, `master` should have -everything `ko/master` has, and `next` should have -everything `ko/next` has, etc. -<13> push out the bleeding edge, together with new tags that point -into the pushed history. Repository Administration[[ADMINISTRATION]] From d24a9f4588617adbba4cbb53ae9ad15c484f0b7e Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 12 Oct 2016 13:45:37 -0400 Subject: [PATCH 038/288] mingw: use OpenSSL's SHA-1 routines Use OpenSSL's SHA-1 routines rather than builtin block-sha1 routines. This improves performance on SHA1 operations on Intel processors. OpenSSL 1.0.2 has made considerable performance improvements and support the Intel hardware acceleration features. See: https://software.intel.com/en-us/articles/improving-openssl-performance https://software.intel.com/en-us/articles/intel-sha-extensions To test this I added/staged a single file in a gigantic repository having a 450MB index file. The code in read-cache.c verifies the header SHA as it reads the index and computes a new header SHA as it writes out the new index. Therefore, in this test the SHA code must process 900MB of data. Testing was done on an Intel I7-4770 CPU @ 3.40GHz (Intel64, Family 6, Model 60) CPU. The block-sha1 version averaged 5.27 seconds. The OpenSSL version averaged 4.50 seconds. ================================================================ $ echo xxx >> project.mk $ time /e/blk_sha/bin/git.exe add project.mk real 0m5.207s user 0m0.000s sys 0m0.250s $ echo xxx >> project.mk $ time /e/blk_sha/bin/git.exe add project.mk real 0m5.362s user 0m0.015s sys 0m0.234s $ echo xxx >> project.mk $ time /e/blk_sha/bin/git.exe add project.mk real 0m5.300s user 0m0.016s sys 0m0.250s $ echo xxx >> project.mk $ time /e/blk_sha/bin/git.exe add project.mk real 0m5.216s user 0m0.000s sys 0m0.250s ================================================================ $ echo xxx >> project.mk $ time /e/openssl/bin/git.exe add project.mk real 0m4.431s user 0m0.000s sys 0m0.250s $ echo xxx >> project.mk $ time /e/openssl/bin/git.exe add project.mk real 0m4.478s user 0m0.000s sys 0m0.265s $ echo xxx >> project.mk $ time /e/openssl/bin/git.exe add project.mk real 0m4.690s user 0m0.000s sys 0m0.250s $ echo xxx >> project.mk $ time /e/openssl/bin/git.exe add project.mk real 0m4.420s user 0m0.000s sys 0m0.234s ================================================================ Signed-off-by: Jeff Hostetler Signed-off-by: Johannes Schindelin --- config.mak.uname | 1 - 1 file changed, 1 deletion(-) diff --git a/config.mak.uname b/config.mak.uname index c517b378cf..b105a2cf86 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -518,7 +518,6 @@ ifneq (,$(findstring MINGW,$(uname_S))) OBJECT_CREATION_USES_RENAMES = UnfortunatelyNeedsTo NO_REGEX = YesPlease NO_PYTHON = YesPlease - BLK_SHA1 = YesPlease ETAGS_TARGET = ETAGS NO_INET_PTON = YesPlease NO_INET_NTOP = YesPlease From 3ce736ff820c9916b3aa6dc274921c2cea479e1c Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 11 Dec 2015 06:59:13 +0100 Subject: [PATCH 039/288] mingw: handle absolute paths in expand_user_path() On Windows, an absolute POSIX path needs to be turned into a Windows one. Signed-off-by: Johannes Schindelin --- path.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/path.c b/path.c index efcedafba6..fafbc47db4 100644 --- a/path.c +++ b/path.c @@ -7,6 +7,7 @@ #include "dir.h" #include "worktree.h" #include "submodule-config.h" +#include "exec_cmd.h" static int get_st_mode_bits(const char *path, int *mode) { @@ -646,6 +647,10 @@ char *expand_user_path(const char *path) if (path == NULL) goto return_null; +#ifdef __MINGW32__ + if (path[0] == '/') + return system_path(path + 1); +#endif if (path[0] == '~') { const char *first_slash = strchrnul(path, '/'); const char *username = path + 1; From 15e2f32fb0eaf685a193169acdde073f3f7c3b4f Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 11 Dec 2015 07:00:48 +0100 Subject: [PATCH 040/288] http: also treat config options sslCert and sslKey as paths This is a companion patch to bf9acba (http: treat config options sslCAPath and sslCAInfo as paths, 2015-11-23). Signed-off-by: Johannes Schindelin --- http.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/http.c b/http.c index 051fe6e5ab..96171ebaf0 100644 --- a/http.c +++ b/http.c @@ -263,10 +263,10 @@ static int http_options(const char *var, const char *value, void *cb) if (!strcmp("http.sslversion", var)) return git_config_string(&ssl_version, var, value); if (!strcmp("http.sslcert", var)) - return git_config_string(&ssl_cert, var, value); + return git_config_pathname(&ssl_cert, var, value); #if LIBCURL_VERSION_NUM >= 0x070903 if (!strcmp("http.sslkey", var)) - return git_config_string(&ssl_key, var, value); + return git_config_pathname(&ssl_key, var, value); #endif #if LIBCURL_VERSION_NUM >= 0x070908 if (!strcmp("http.sslcapath", var)) From f5dab1a048e55b0e158f8ff2e29b133c4e679a28 Mon Sep 17 00:00:00 2001 From: Sebastian Schuberth Date: Thu, 29 Aug 2013 13:09:27 +0200 Subject: [PATCH 041/288] MinGW: Use MakeMaker to build the Perl libraries This way the libraries get properly installed into the "site_perl" directory and we just have to move them out of the "mingw" directory. Signed-off-by: Sebastian Schuberth --- config.mak.uname | 1 - 1 file changed, 1 deletion(-) diff --git a/config.mak.uname b/config.mak.uname index b105a2cf86..649f97af60 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -508,7 +508,6 @@ ifneq (,$(findstring MINGW,$(uname_S))) NO_MKDTEMP = YesPlease NO_MKSTEMPS = YesPlease NO_SVN_TESTS = YesPlease - NO_PERL_MAKEMAKER = YesPlease RUNTIME_PREFIX = YesPlease NO_ST_BLOCKS_IN_STRUCT_STAT = YesPlease NO_NSEC = YesPlease From e60ced34e2623e30b4e02c264dfcb69a525236ee Mon Sep 17 00:00:00 2001 From: Sebastian Schuberth Date: Wed, 4 Sep 2013 18:18:49 +0200 Subject: [PATCH 042/288] Makefile: Set htmldir to match the default HTML docs location under MSYS Signed-off-by: Sebastian Schuberth --- config.mak.uname | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.mak.uname b/config.mak.uname index 649f97af60..e21d690a67 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -536,7 +536,7 @@ ifneq (,$(findstring MINGW,$(uname_S))) X = .exe SPARSE_FLAGS = -Wno-one-bit-signed-bitfield ifneq (,$(wildcard ../THIS_IS_MSYSGIT)) - htmldir = doc/git/html/ + htmldir = share/doc/git/$(firstword $(subst -, ,$(GIT_VERSION)))/html prefix = INSTALL = /bin/install EXTLIBS += /mingw/lib/libz.a From 66600d191798e9e847fbdc53496d3bb050cfdcab Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 20 Feb 2015 09:52:07 +0000 Subject: [PATCH 043/288] Help debugging with MSys2 by optionally executing bash with strace MSys2's strace facility is very useful for debugging... With this patch, the bash will be executed through strace if the environment variable GIT_STRACE_COMMANDS is set, which comes in real handy when investigating issues in the test suite. Signed-off-by: Johannes Schindelin --- compat/mingw.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/compat/mingw.c b/compat/mingw.c index b5bc7c91ab..22574f5cd7 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -1202,6 +1202,16 @@ static pid_t mingw_spawnve_fd(const char *cmd, const char **argv, char **deltaen free(quoted); } + if (getenv("GIT_STRACE_COMMANDS")) { + char **path = get_path_split(); + cmd = path_lookup("strace.exe", path, 1); + if (!cmd) + return error("strace not found!"); + if (xutftowcs_path(wcmd, cmd) < 0) + return -1; + strbuf_insert(&args, 0, "strace ", 7); + } + ALLOC_ARRAY(wargs, st_add(st_mult(2, args.len), 1)); xutftowcs(wargs, args.buf, 2 * args.len + 1); strbuf_release(&args); From 114a2847a492cd4bc0493b919b94d63803af3727 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 16 Feb 2015 14:06:59 +0100 Subject: [PATCH 044/288] Build Python stuff with MSys2 Signed-off-by: Johannes Schindelin --- config.mak.uname | 1 + 1 file changed, 1 insertion(+) diff --git a/config.mak.uname b/config.mak.uname index e21d690a67..45866cb3ce 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -570,6 +570,7 @@ else USE_LIBPCRE= YesPlease NO_CURL = USE_NED_ALLOCATOR = YesPlease + NO_PYTHON = else COMPAT_CFLAGS += -D__USE_MINGW_ANSI_STDIO NO_CURL = YesPlease From 503c4a9cfac72e4631c736d9b78aeb42941f8eec Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sun, 15 Feb 2015 09:56:35 +0000 Subject: [PATCH 045/288] Error out when mingw_startup() *and* NO_UNSETENV are active The unsetenv code has no idea to update our environ_size, therefore causing segmentation faults when environment variables are removed without compat/mingw.c's knowing (MinGW's optimized lookup would try to strcmp() against NULL in such a case). Signed-off-by: Johannes Schindelin --- compat/mingw.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/compat/mingw.h b/compat/mingw.h index 018bb38eb2..22dcb3a826 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -263,6 +263,10 @@ char *mingw_mktemp(char *template); char *mingw_getcwd(char *pointer, int len); #define getcwd mingw_getcwd +#ifdef NO_UNSETENV +#error "NO_UNSETENV is incompatible with the MinGW startup code!" +#endif + char *mingw_getenv(const char *name); #define getenv mingw_getenv int mingw_putenv(const char *namevalue); From 886bd86b1620edc5849dc5888a1fef003833d823 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 20 Feb 2015 14:21:04 +0000 Subject: [PATCH 046/288] Tests: optionally skip redirecting stdin/stdout/stderr There is a really useful debugging technique developed by Sverre Rabbelier that inserts "bash &&" somewhere in the test scripts, letting the developer interact at given points with the current state. Another debugging technique, used a lot by this here coder, is to run certain executables via gdb by guarding a "gdb -args" call in bin-wrappers/git. Both techniques were disabled by 781f76b1(test-lib: redirect stdin of tests). Let's reinstate the ability to run an interactive shell by making the redirection optional: setting the TEST_NO_REDIRECT environment variable will skip the redirection. Signed-off-by: Johannes Schindelin --- t/test-lib.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/t/test-lib.sh b/t/test-lib.sh index cde7fc7fcf..9980a46133 100644 --- a/t/test-lib.sh +++ b/t/test-lib.sh @@ -598,6 +598,20 @@ test_eval_ () { # # The test itself is run with stderr put back to &4 (so either to # /dev/null, or to the original stderr if --verbose was used). + if test -n "$TEST_NO_REDIRECT" + then + test_eval_inner_ "$@" + test_eval_ret_=$? + if test "$trace" = t + then + set +x + if test "$test_eval_ret_" != 0 + then + say_color error >&4 "error: last command exited with \$?=$test_eval_ret_" + fi + fi + return $test_eval_ret_ + fi { test_eval_inner_ "$@" &3 2>&4 test_eval_ret_=$? From 55972920b4a7db619f4c5a79b7774ede3a743724 Mon Sep 17 00:00:00 2001 From: Cesar Eduardo Barros Date: Mon, 9 Mar 2015 08:51:38 +0100 Subject: [PATCH 047/288] mingw: Embed a manifest to trick UAC into Doing The Right Thing On Windows >= Vista, not having an application manifest with a requestedExecutionLevel can cause several kinds of confusing behavior. The first and more obvious behavior is "Installer Detection", where Windows sometimes decides (by looking at things like the file name and even sequences of bytes within the executable) that an executable is an installer and should run elevated (causing the well-known popup dialog to appear). In Git's context, subcommands such as "git patch-id" or "git update-index" fall prey to this behavior. The second and more confusing behavior is "File Virtualization". It means that when files are written without having write permission, it does not fail (as expected), but they are instead redirected to somewhere else. When the files are read, the original contents are returned, though, not the ones that were just written somewhere else. Even more confusing, not all write accesses are redirected; Trying to write to write-protected .exe files, for example, will fail instead of redirecting. In addition to being unwanted behavior, File Virtualization causes dramatic slowdowns in Git (see for instance http://code.google.com/p/msysgit/issues/detail?id=320). There are two ways to prevent those two behaviors: Either you embed an application manifest within all your executables, or you add an external manifest (a file with the same name followed by .manifest) to all your executables. Since Git's builtins are hardlinked (or copied), it is simpler and more robust to embed a manifest. A recent enough MSVC compiler should already embed a working internal manifest, but for MinGW you have to do so by hand. Very lightly tested on Wine, where like on Windows XP it should not make any difference. References: - New UAC Technologies for Windows Vista http://msdn.microsoft.com/en-us/library/bb756960.aspx - Create and Embed an Application Manifest (UAC) http://msdn.microsoft.com/en-us/library/bb756929.aspx [js: simplified the embedding dramatically by reusing Git for Windows' existing Windows resource file, removed the optional (and dubious) processorArchitecture attribute of the manifest's assemblyIdentity section.] Signed-off-by: Cesar Eduardo Barros Signed-off-by: Johannes Schindelin --- compat/win32/git.manifest | 25 +++++++++++++++++++++++++ git.rc | 2 ++ 2 files changed, 27 insertions(+) create mode 100644 compat/win32/git.manifest diff --git a/compat/win32/git.manifest b/compat/win32/git.manifest new file mode 100644 index 0000000000..771e3cce43 --- /dev/null +++ b/compat/win32/git.manifest @@ -0,0 +1,25 @@ + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/git.rc b/git.rc index 49002e0d54..cc3fdc6cc6 100644 --- a/git.rc +++ b/git.rc @@ -20,3 +20,5 @@ BEGIN VALUE "Translation", 0x409, 1200 END END + +1 RT_MANIFEST "compat/win32/git.manifest" From e2275c38d660665633e5039624b0d67fa2ffb95a Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 2 Nov 2015 08:41:10 +0100 Subject: [PATCH 048/288] mingw: enable stack smashing protector As suggested privately to Brendan Forster by some unnamed person (suggestion for the future: use the public mailing list, or even the public GitHub issue tracker, that is a much better place to offer such suggestions), we should make use of gcc's stack smashing protector that helps detect stack buffer overruns early. Rather than using -fstack-protector, we use -fstack-protector-strong because it strikes a better balance between how much code is affected and the performance impact. In a local test (time git log --grep=is -p), best of 5 timings went from 23.009s to 22.997s (i.e. the performance impact was *well* lost in the noise). This fixes https://github.com/git-for-windows/git/issues/501 Signed-off-by: Johannes Schindelin --- config.mak.uname | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config.mak.uname b/config.mak.uname index 45866cb3ce..ed2cf5b6dd 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -559,7 +559,8 @@ else BASIC_LDFLAGS += -Wl,--large-address-aware endif CC = gcc - COMPAT_CFLAGS += -D__USE_MINGW_ANSI_STDIO=0 -DDETECT_MSYS_TTY + COMPAT_CFLAGS += -D__USE_MINGW_ANSI_STDIO=0 -DDETECT_MSYS_TTY \ + -fstack-protector-strong EXTLIBS += -lntdll INSTALL = /bin/install NO_R_TO_GCC_LINKER = YesPlease From f9687cc6209d7b4c1843222f0a0ef07cbad44bb6 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 17 Mar 2015 19:41:02 +0100 Subject: [PATCH 049/288] UTF-8 environment: be a little bit more defensive It is unlikely that we have an empty environment, ever, but *if* we do, when `environ_size - 1` is passed to `bsearchenv()` it is misinterpreted as a real large integer. To make the code truly defensive, refuse to do anything at all if the size is negative (which should not happen, of course). Signed-off-by: Johannes Schindelin --- compat/mingw.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 22574f5cd7..a1ffd54bd7 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -1430,7 +1430,7 @@ static int bsearchenv(char **env, const char *name, size_t size) */ static int do_putenv(char **env, const char *name, int size, int free_old) { - int i = bsearchenv(env, name, size - 1); + int i = size <= 0 ? -1 : bsearchenv(env, name, size - 1); /* optionally free removed / replaced entry */ if (i >= 0 && free_old) @@ -1455,7 +1455,13 @@ static int do_putenv(char **env, const char *name, int size, int free_old) char *mingw_getenv(const char *name) { char *value; - int pos = bsearchenv(environ, name, environ_size - 1); + int pos; + + if (environ_size <= 0) + return NULL; + + pos = bsearchenv(environ, name, environ_size - 1); + if (pos < 0) return NULL; value = strchr(environ[pos], '='); From 9e3e370328ae57abf739506ca46f25cdc2aeb961 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 18 Mar 2015 10:57:52 +0100 Subject: [PATCH 050/288] mingw: be more defensive when making the environment block Outside of our Windows-specific code, the end of the environment can be marked also by a pointer to a NUL character, not only by a NULL pointer as our code assumed so far. That led to a buffer overrun in `make_environment_block()` when running `git-remote-https` in `mintty` (because `curl_global_init()` added the `CHARSET` environment variable *outside* of `mingw_putenv()`, ending the environment in a pointer to an empty string). Side note for future debugging on Windows: when running programs in `mintty`, the standard input/output/error is not connected to a Win32 Console, but instead is pipe()d. That means that even stderr may not be written completely before a crash, but has to be fflush()ed explicitly. For example, when debugging crashes, the developer should insert an `fflush(stderr);` at the end of the `error()` function defined in usage.c. Signed-off-by: Johannes Schindelin --- compat/mingw.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index a1ffd54bd7..d5779c12e3 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -1082,7 +1082,7 @@ static wchar_t *make_environment_block(char **deltaenv) char **tmpenv; int i = 0, size = environ_size, wenvsz = 0, wenvpos = 0; - while (deltaenv && deltaenv[i]) + while (deltaenv && deltaenv[i] && *deltaenv[i]) i++; /* copy the environment, leaving space for changes */ @@ -1090,11 +1090,11 @@ static wchar_t *make_environment_block(char **deltaenv) memcpy(tmpenv, environ, size * sizeof(char*)); /* merge supplied environment changes into the temporary environment */ - for (i = 0; deltaenv && deltaenv[i]; i++) + for (i = 0; deltaenv && deltaenv[i] && *deltaenv[i]; i++) size = do_putenv(tmpenv, deltaenv[i], size, 0); /* create environment block from temporary environment */ - for (i = 0; tmpenv[i]; i++) { + for (i = 0; tmpenv[i] && *tmpenv[i]; i++) { size = 2 * strlen(tmpenv[i]) + 2; /* +2 for final \0 */ ALLOC_GROW(wenvblk, (wenvpos + size) * sizeof(wchar_t), wenvsz); wenvpos += xutftowcs(&wenvblk[wenvpos], tmpenv[i], size) + 1; From fe67db8d540fb00afb0b57807d9e1d15fe0954df Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 17 Mar 2015 19:41:31 +0100 Subject: [PATCH 051/288] mingw: be *very* wary about outside environment changes The environment is modified in most surprising circumstances, and not all of them are under Git's control. For example, calling curl_global_init() on Windows will ensure that the CHARSET variable is set, adding one if necessary. While the previous commit worked around crashes triggered by such outside changes of the environment by relaxing the requirement that the environment be terminated by a NULL pointer, the other assumption made by `mingw_getenv()` and `mingw_putenv()` is that the environment is sorted, for efficient lookup via binary search. Let's make real sure that our environment is intact before querying or modifying it, and reinitialize our idea of the environment if necessary. With this commit, before working on the environment we look briefly for indicators that the environment was modified outside of our control, and to ensure that it is terminated with a NULL pointer and sorted again in that case. Note: the indicators are maybe not sufficient. For example, when a variable is removed, it will not be noticed. It might also be a problem if outside changes to the environment result in a modified `environ` pointer: it is unclear whether such a modification could result in a problem when `mingw_putenv()` needs to `realloc()` the environment buffer. For the moment, however, the current fix works well enough, so let's only face the potential problems when (and if!) they occur. Signed-off-by: Johannes Schindelin --- compat/mingw.c | 49 +++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index d5779c12e3..db4c61f4e2 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -1071,6 +1071,10 @@ static int do_putenv(char **env, const char *name, int size, int free_old); static int environ_size = 0; /* allocated size of environ array, in bytes */ static int environ_alloc = 0; +/* used as a indicator when the environment has been changed outside mingw.c */ +static char **saved_environ; + +static void maybe_reinitialize_environ(void); /* * Create environment block suitable for CreateProcess. Merges current @@ -1080,7 +1084,10 @@ static wchar_t *make_environment_block(char **deltaenv) { wchar_t *wenvblk = NULL; char **tmpenv; - int i = 0, size = environ_size, wenvsz = 0, wenvpos = 0; + int i = 0, size, wenvsz = 0, wenvpos = 0; + + maybe_reinitialize_environ(); + size = environ_size; while (deltaenv && deltaenv[i] && *deltaenv[i]) i++; @@ -1407,6 +1414,41 @@ static int compareenv(const void *v1, const void *v2) } } +/* + * Functions implemented outside Git are able to modify the environment, + * too. For example, cURL's curl_global_init() function sets the CHARSET + * environment variable (at least in certain circumstances). + * + * Therefore we need to be *really* careful *not* to assume that we have + * sole control over the environment and reinitalize it when necessary. + */ +static void maybe_reinitialize_environ(void) +{ + int i; + + if (!saved_environ) { + warning("MinGW environment not initialized yet"); + return; + } + + if (environ_size <= 0) + return; + + if (saved_environ != environ) + /* We have *no* idea how much space was allocated outside */ + environ_alloc = 0; + else if (!environ[environ_size - 1]) + return; /* still consistent */ + + for (i = 0; environ[i] && *environ[i]; i++) + ; /* continue counting */ + environ[i] = NULL; + environ_size = i + 1; + + /* sort environment for O(log n) getenv / putenv */ + qsort(environ, i, sizeof(char*), compareenv); +} + static int bsearchenv(char **env, const char *name, size_t size) { unsigned low = 0, high = size; @@ -1460,6 +1502,7 @@ char *mingw_getenv(const char *name) if (environ_size <= 0) return NULL; + maybe_reinitialize_environ(); pos = bsearchenv(environ, name, environ_size - 1); if (pos < 0) @@ -1470,7 +1513,9 @@ char *mingw_getenv(const char *name) int mingw_putenv(const char *namevalue) { + maybe_reinitialize_environ(); ALLOC_GROW(environ, (environ_size + 1) * sizeof(char*), environ_alloc); + saved_environ = environ; environ_size = do_putenv(environ, namevalue, environ_size, 1); return 0; } @@ -2349,7 +2394,7 @@ void mingw_startup(void) */ environ_size = i + 1; environ_alloc = alloc_nr(environ_size * sizeof(char*)); - environ = malloc_startup(environ_alloc); + saved_environ = environ = malloc_startup(environ_alloc); /* allocate buffer (wchar_t encodes to max 3 UTF-8 bytes) */ maxlen = 3 * maxlen + 1; From bfb9bfc7996ea217adb9a66610479f0a6d96cbba Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 6 Apr 2015 14:34:59 +0100 Subject: [PATCH 052/288] git-gui (MinGW): make use of MSys2's msgfmt When Git for Windows was still based on MSys1, we had no gettext, ergo no msgfmt, either. Therefore, we introduced a small and simple Tcl script to perform the same task. However, with MSys2, we no longer need that because we have a proper msgfmt executable. Plus, the po2msg.sh script somehow manages to hang when run in parallel. Two reasons to use real msgfmt.exe instead. Signed-off-by: Johannes Schindelin --- git-gui/Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/git-gui/Makefile b/git-gui/Makefile index fe30be38dc..918a8de369 100644 --- a/git-gui/Makefile +++ b/git-gui/Makefile @@ -161,7 +161,9 @@ ifeq ($(uname_S),Darwin) endif endif ifneq (,$(findstring MINGW,$(uname_S))) +ifeq ($(shell expr "$(uname_R)" : '1\.'),2) NO_MSGFMT=1 +endif GITGUI_WINDOWS_WRAPPER := YesPlease GITGUI_RELATIVE := 1 endif From c47fe1a64698f517bb78d03b39177e7706442feb Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 6 Apr 2015 10:37:04 +0100 Subject: [PATCH 053/288] Avoid illegal filenames when building Documentation on NTFS A '+' is not a valid part of a filename with Windows file systems (it is reserved because the '+' operator meant file concatenation back in the DOS days). Let's just not use it. Signed-off-by: Johannes Schindelin --- Documentation/Makefile | 84 +++++++++++++++++++++--------------------- 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/Documentation/Makefile b/Documentation/Makefile index a9fb497b83..6517b59c6b 100644 --- a/Documentation/Makefile +++ b/Documentation/Makefile @@ -267,9 +267,9 @@ docdep_prereqs = \ cmd-list.made $(cmds_txt) doc.dep : $(docdep_prereqs) $(wildcard *.txt) build-docdep.perl - $(QUIET_GEN)$(RM) $@+ $@ && \ - $(PERL_PATH) ./build-docdep.perl >$@+ $(QUIET_STDERR) && \ - mv $@+ $@ + $(QUIET_GEN)$(RM) $@.new $@ && \ + $(PERL_PATH) ./build-docdep.perl >$@.new $(QUIET_STDERR) && \ + mv $@.new $@ -include doc.dep @@ -305,8 +305,8 @@ mergetools-list.made: ../git-mergetool--lib.sh $(wildcard ../mergetools/*) date >$@ clean: - $(RM) *.xml *.xml+ *.html *.html+ *.1 *.5 *.7 - $(RM) *.texi *.texi+ *.texi++ git.info gitman.info + $(RM) *.xml *.xml.new *.html *.html.new *.1 *.5 *.7 + $(RM) *.texi *.texi.new *.texi.new.new git.info gitman.info $(RM) *.pdf $(RM) howto-index.txt howto/*.html doc.dep $(RM) technical/*.html technical/api-index.txt @@ -314,14 +314,14 @@ clean: $(RM) manpage-base-url.xsl $(MAN_HTML): %.html : %.txt asciidoc.conf - $(QUIET_ASCIIDOC)$(RM) $@+ $@ && \ - $(TXT_TO_HTML) -d manpage -o $@+ $< && \ - mv $@+ $@ + $(QUIET_ASCIIDOC)$(RM) $@.new $@ && \ + $(TXT_TO_HTML) -d manpage -o $@.new $< && \ + mv $@.new $@ $(OBSOLETE_HTML): %.html : %.txto asciidoc.conf - $(QUIET_ASCIIDOC)$(RM) $@+ $@ && \ - $(TXT_TO_HTML) -o $@+ $< && \ - mv $@+ $@ + $(QUIET_ASCIIDOC)$(RM) $@.new $@ && \ + $(TXT_TO_HTML) -o $@.new $< && \ + mv $@.new $@ manpage-base-url.xsl: manpage-base-url.xsl.in sed "s|@@MAN_BASE_URL@@|$(MAN_BASE_URL)|" $< > $@ @@ -331,14 +331,14 @@ manpage-base-url.xsl: manpage-base-url.xsl.in $(XMLTO) -m $(MANPAGE_XSL) $(XMLTO_EXTRA) man $< %.xml : %.txt asciidoc.conf - $(QUIET_ASCIIDOC)$(RM) $@+ $@ && \ - $(TXT_TO_XML) -d manpage -o $@+ $< && \ - mv $@+ $@ + $(QUIET_ASCIIDOC)$(RM) $@.new $@ && \ + $(TXT_TO_XML) -d manpage -o $@.new $< && \ + mv $@.new $@ user-manual.xml: user-manual.txt user-manual.conf - $(QUIET_ASCIIDOC)$(RM) $@+ $@ && \ - $(TXT_TO_XML) -d book -o $@+ $< && \ - mv $@+ $@ + $(QUIET_ASCIIDOC)$(RM) $@.new $@ && \ + $(TXT_TO_XML) -d book -o $@.new $< && \ + mv $@.new $@ technical/api-index.txt: technical/api-index-skel.txt \ technical/api-index.sh $(patsubst %,%.txt,$(API_DOCS)) @@ -352,45 +352,45 @@ XSLT = docbook.xsl XSLTOPTS = --xinclude --stringparam html.stylesheet docbook-xsl.css user-manual.html: user-manual.xml $(XSLT) - $(QUIET_XSLTPROC)$(RM) $@+ $@ && \ - xsltproc $(XSLTOPTS) -o $@+ $(XSLT) $< && \ - mv $@+ $@ + $(QUIET_XSLTPROC)$(RM) $@.new $@ && \ + xsltproc $(XSLTOPTS) -o $@.new $(XSLT) $< && \ + mv $@.new $@ git.info: user-manual.texi $(QUIET_MAKEINFO)$(MAKEINFO) --no-split -o $@ user-manual.texi user-manual.texi: user-manual.xml - $(QUIET_DB2TEXI)$(RM) $@+ $@ && \ - $(DOCBOOK2X_TEXI) user-manual.xml --encoding=UTF-8 --to-stdout >$@++ && \ - $(PERL_PATH) fix-texi.perl <$@++ >$@+ && \ - rm $@++ && \ - mv $@+ $@ + $(QUIET_DB2TEXI)$(RM) $@.new $@ && \ + $(DOCBOOK2X_TEXI) user-manual.xml --encoding=UTF-8 --to-stdout >$@.new.new && \ + $(PERL_PATH) fix-texi.perl <$@.new.new >$@.new && \ + rm $@.new.new && \ + mv $@.new $@ user-manual.pdf: user-manual.xml - $(QUIET_DBLATEX)$(RM) $@+ $@ && \ - $(DBLATEX) -o $@+ -p $(ASCIIDOC_DBLATEX_DIR)/asciidoc-dblatex.xsl -s $(ASCIIDOC_DBLATEX_DIR)/asciidoc-dblatex.sty $< && \ - mv $@+ $@ + $(QUIET_DBLATEX)$(RM) $@.new $@ && \ + $(DBLATEX) -o $@.new -p $(ASCIIDOC_DBLATEX_DIR)/asciidoc-dblatex.xsl -s $(ASCIIDOC_DBLATEX_DIR)/asciidoc-dblatex.sty $< && \ + mv $@.new $@ gitman.texi: $(MAN_XML) cat-texi.perl - $(QUIET_DB2TEXI)$(RM) $@+ $@ && \ + $(QUIET_DB2TEXI)$(RM) $@.new $@ && \ ($(foreach xml,$(MAN_XML),$(DOCBOOK2X_TEXI) --encoding=UTF-8 \ - --to-stdout $(xml) &&) true) > $@++ && \ - $(PERL_PATH) cat-texi.perl $@ <$@++ >$@+ && \ - rm $@++ && \ - mv $@+ $@ + --to-stdout $(xml) &&) true) > $@.new.new && \ + $(PERL_PATH) cat-texi.perl $@ <$@.new.new >$@.new && \ + rm $@.new.new && \ + mv $@.new $@ gitman.info: gitman.texi $(QUIET_MAKEINFO)$(MAKEINFO) --no-split --no-validate $*.texi $(patsubst %.txt,%.texi,$(MAN_TXT)): %.texi : %.xml - $(QUIET_DB2TEXI)$(RM) $@+ $@ && \ - $(DOCBOOK2X_TEXI) --to-stdout $*.xml >$@+ && \ - mv $@+ $@ + $(QUIET_DB2TEXI)$(RM) $@.new $@ && \ + $(DOCBOOK2X_TEXI) --to-stdout $*.xml >$@.new && \ + mv $@.new $@ howto-index.txt: howto-index.sh $(wildcard howto/*.txt) - $(QUIET_GEN)$(RM) $@+ $@ && \ - '$(SHELL_PATH_SQ)' ./howto-index.sh $(sort $(wildcard howto/*.txt)) >$@+ && \ - mv $@+ $@ + $(QUIET_GEN)$(RM) $@.new $@ && \ + '$(SHELL_PATH_SQ)' ./howto-index.sh $(sort $(wildcard howto/*.txt)) >$@.new && \ + mv $@.new $@ $(patsubst %,%.html,$(ARTICLES)) : %.html : %.txt $(QUIET_ASCIIDOC)$(TXT_TO_HTML) $*.txt @@ -399,10 +399,10 @@ WEBDOC_DEST = /pub/software/scm/git/docs howto/%.html: ASCIIDOC_EXTRA += -a git-relative-html-prefix=../ $(patsubst %.txt,%.html,$(wildcard howto/*.txt)): %.html : %.txt - $(QUIET_ASCIIDOC)$(RM) $@+ $@ && \ + $(QUIET_ASCIIDOC)$(RM) $@.new $@ && \ sed -e '1,/^$$/d' $< | \ - $(TXT_TO_HTML) - >$@+ && \ - mv $@+ $@ + $(TXT_TO_HTML) - >$@.new && \ + mv $@.new $@ install-webdoc : html '$(SHELL_PATH_SQ)' ./install-webdoc.sh $(WEBDOC_DEST) From 9a54c388877f37d3b72657540fddcbfd234b6284 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Thu, 9 Apr 2015 16:19:56 +0100 Subject: [PATCH 054/288] gettext: always use UTF-8 on native Windows Git on native Windows exclusively uses UTF-8 for console output (both with mintty and native console windows). Gettext uses setlocale() to determine the output encoding for translated text, however, MSVCRT's setlocale() doesn't support UTF-8. As a result, translated text is encoded in system encoding (GetAPC()), and non-ASCII chars are mangled in console output. Use gettext's bind_textdomain_codeset() to force the encoding to UTF-8 on native Windows. In this developers' setup, HAVE_LIBCHARSET_H is apparently defined, but we *really* want to override the locale_charset() here. Signed-off-by: Karsten Blees --- gettext.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/gettext.c b/gettext.c index db727ea020..7d9e2c27ef 100644 --- a/gettext.c +++ b/gettext.c @@ -10,7 +10,9 @@ #ifndef NO_GETTEXT # include # include -# ifdef HAVE_LIBCHARSET_H +# ifdef GIT_WINDOWS_NATIVE +# define locale_charset() "UTF-8" +# elif defined HAVE_LIBCHARSET_H # include # else # include From 880785bd8170ddabdc893d5a2c2954d55c3bf996 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 7 Apr 2015 10:24:55 +0100 Subject: [PATCH 055/288] mingw: HOT FIX: work around environment issues -- again This developer should really, really have known better. The fact that we are changing the environment in ways for which the MSVCRT is not prepared for is bad enough. But then this developer followed the request to re-enable nedmalloc -- despite the prediction that it would cause an access violation, predicting it in the same message as the request to re-enable nedmalloc, no less! To paper over the issue until the time when this developer finds the time to re-design the Unicode environment handling from scratch, let's hope that cURL is the only library we are using that *may* set an environment variable using MSVCRT's putenv() after we fscked the environment up. Note: this commit can serve as no source of pride to anyone, certainly not yours truly. It is necessary as a quick and pragmatic stop gap, though, to prevent worse problems. Note: cURL manages to set the variable CHARSET when nedmalloc is *not* enabled, without causing an access violation. In that case, it sets it successfully to the value "cp" + GetACP() (hence it is our choice, too, cURL may need it, Git does not): https://github.com/bagder/curl/blob/aa5808b5/lib/easy.c#L157-L162 Signed-off-by: Johannes Schindelin --- compat/mingw.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/compat/mingw.c b/compat/mingw.c index db4c61f4e2..b4af360d13 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2417,6 +2417,17 @@ void mingw_startup(void) unset_environment_variables = xstrdup("PERL5LIB"); + /* + * Avoid a segmentation fault when cURL tries to set the CHARSET + * variable and putenv() barfs at our nedmalloc'ed environment. + */ + if (!getenv("CHARSET")) { + struct strbuf buf = STRBUF_INIT; + strbuf_addf(&buf, "cp%u", GetACP()); + setenv("CHARSET", buf.buf, 1); + strbuf_release(&buf); + } + /* initialize critical section for waitpid pinfo_t list */ InitializeCriticalSection(&pinfo_cs); From 4990bfea8e93b6edbfca517a4442c8909d90ffd5 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Mon, 6 Apr 2015 21:37:18 +0200 Subject: [PATCH 056/288] mingw: initialize HOME on startup HOME initialization was historically duplicated in many different places, including /etc/profile, launch scripts such as git-bash.vbs and gitk.cmd, and (although slightly broken) in the git-wrapper. Even unrelated projects such as GitExtensions and TortoiseGit need to implement the same logic to be able to call git directly. Initialize HOME in git's own startup code so that we can eventually retire all the duplicate initialization code. Signed-off-by: Karsten Blees --- compat/mingw.c | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/compat/mingw.c b/compat/mingw.c index b4af360d13..d51984523e 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2271,6 +2271,30 @@ static void setup_windows_environment(void) /* simulate TERM to enable auto-color (see color.c) */ if (!getenv("TERM")) setenv("TERM", "cygwin", 1); + + /* calculate HOME if not set */ + if (!getenv("HOME")) { + /* + * try $HOMEDRIVE$HOMEPATH - the home share may be a network + * location, thus also check if the path exists (i.e. is not + * disconnected) + */ + if ((tmp = getenv("HOMEDRIVE"))) { + struct strbuf buf = STRBUF_INIT; + strbuf_addstr(&buf, tmp); + if ((tmp = getenv("HOMEPATH"))) { + strbuf_addstr(&buf, tmp); + if (is_directory(buf.buf)) + setenv("HOME", buf.buf, 1); + else + tmp = NULL; /* use $USERPROFILE */ + } + strbuf_release(&buf); + } + /* use $USERPROFILE if the home share is not available */ + if (!tmp && (tmp = getenv("USERPROFILE"))) + setenv("HOME", tmp, 1); + } } int handle_long_path(wchar_t *path, int len, int max_path, int expand) From 2b56aeed44838627577b04235f977d75553814bc Mon Sep 17 00:00:00 2001 From: nalla Date: Thu, 16 Apr 2015 11:45:05 +0100 Subject: [PATCH 057/288] mingw: explicitly `fflush` stdout For performance reasons `stdout` is not unbuffered by default. That leads to problems if after printing to `stdout` a read on `stdin` is performed. For that reason interactive commands like `git clean -i` do not function properly anymore if the `stdout` is not flushed by `fflush(stdout)` before trying to read from `stdin`. In the case of `git clean -i` all reads on `stdin` were preceded by a `fflush(stdout)` call. Signed-off-by: nalla --- builtin/clean.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/builtin/clean.c b/builtin/clean.c index 0371010afb..f54bcba497 100644 --- a/builtin/clean.c +++ b/builtin/clean.c @@ -570,6 +570,7 @@ static int *list_and_choose(struct menu_opts *opts, struct menu_stuff *stuff) clean_get_color(CLEAN_COLOR_RESET)); } + fflush(stdout); if (strbuf_getline_lf(&choice, stdin) != EOF) { strbuf_trim(&choice); } else { @@ -652,6 +653,7 @@ static int filter_by_patterns_cmd(void) clean_print_color(CLEAN_COLOR_PROMPT); printf(_("Input ignore patterns>> ")); clean_print_color(CLEAN_COLOR_RESET); + fflush(stdout); if (strbuf_getline_lf(&confirm, stdin) != EOF) strbuf_trim(&confirm); else @@ -750,6 +752,7 @@ static int ask_each_cmd(void) qname = quote_path_relative(item->string, NULL, &buf); /* TRANSLATORS: Make sure to keep [y/N] as is */ printf(_("Remove %s [y/N]? "), qname); + fflush(stdout); if (strbuf_getline_lf(&confirm, stdin) != EOF) { strbuf_trim(&confirm); } else { From 044981d0a96c8e08859ed8b7ba21a5e1c736c20e Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Thu, 19 Mar 2015 16:33:44 +0100 Subject: [PATCH 058/288] mingw: Support `git_terminal_prompt` with more terminals The `git_terminal_prompt()` function expects the terminal window to be attached to a Win32 Console. However, this is not the case with terminal windows other than `cmd.exe`'s, e.g. with MSys2's own `mintty`. Non-cmd terminals such as `mintty` still have to have a Win32 Console to be proper console programs, but have to hide the Win32 Console to be able to provide more flexibility (such as being resizeable not only vertically but also horizontally). By writing to that Win32 Console, `git_terminal_prompt()` manages only to send the prompt to nowhere and to wait for input from a Console to which the user has no access. This commit introduces a function specifically to support `mintty` -- or other terminals that are compatible with MSys2's `/dev/tty` emulation. We use the `TERM` environment variable as an indicator for that: if the value starts with "xterm" (such as `mintty`'s "xterm_256color"), we prefer to let `xterm_prompt()` handle the user interaction. The most prominent user of `git_terminal_prompt()` is certainly `git-remote-https.exe`. It is an interesting use case because both `stdin` and `stdout` are redirected when Git calls said executable, yet it still wants to access the terminal. When running inside a `mintty`, the terminal is not accessible to the `git-remote-https.exe` program, though, because it is a MinGW program and the `mintty` terminal is not backed by a Win32 console. To solve that problem, we simply call out to the shell -- which is an *MSys2* program and can therefore access `/dev/tty`. Helped-by: nalla Signed-off-by: Karsten Blees Signed-off-by: Johannes Schindelin --- compat/terminal.c | 57 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/compat/terminal.c b/compat/terminal.c index fa13ee672d..cd6e2e55a5 100644 --- a/compat/terminal.c +++ b/compat/terminal.c @@ -1,7 +1,10 @@ +#include #include "git-compat-util.h" +#include "run-command.h" #include "compat/terminal.h" #include "sigchain.h" #include "strbuf.h" +#include "cache.h" #if defined(HAVE_DEV_TTY) || defined(GIT_WINDOWS_NATIVE) @@ -91,6 +94,54 @@ static int disable_echo(void) return 0; } +static char *shell_prompt(const char *prompt, int echo) +{ + const char *read_input[] = { + /* Note: call 'bash' explicitly, as 'read -s' is bash-specific */ + "bash", "-c", echo ? + "cat >/dev/tty && read -r line /dev/tty && read -r -s line /dev/tty", + NULL + }; + struct child_process child = CHILD_PROCESS_INIT; + static struct strbuf buffer = STRBUF_INIT; + int prompt_len = strlen(prompt), len = -1, code; + + child.argv = read_input; + child.in = -1; + child.out = -1; + + if (start_command(&child)) + return NULL; + + if (write_in_full(child.in, prompt, prompt_len) != prompt_len) { + error("could not write to prompt script"); + close(child.in); + goto ret; + } + close(child.in); + + strbuf_reset(&buffer); + len = strbuf_read(&buffer, child.out, 1024); + if (len < 0) { + error("could not read from prompt script"); + goto ret; + } + + strbuf_strip_suffix(&buffer, "\n"); + strbuf_strip_suffix(&buffer, "\r"); + +ret: + close(child.out); + code = finish_command(&child); + if (code) { + error("failed to execute prompt script (exit code %d)", code); + return NULL; + } + + return len < 0 ? NULL : buffer.buf; +} + #endif #ifndef FORCE_TEXT @@ -102,6 +153,12 @@ char *git_terminal_prompt(const char *prompt, int echo) static struct strbuf buf = STRBUF_INIT; int r; FILE *input_fh, *output_fh; +#ifdef GIT_WINDOWS_NATIVE + const char *term = getenv("TERM"); + + if (term && starts_with(term, "xterm")) + return shell_prompt(prompt, echo); +#endif input_fh = fopen(INPUT_PATH, "r" FORCE_TEXT); if (!input_fh) From 8d7ff77e2d994570cc13aec8263ef0b9bdd42657 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sat, 9 May 2015 02:11:48 +0200 Subject: [PATCH 059/288] compat/terminal.c: only use the Windows console if bash 'read -r' fails MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Accessing the Windows console through the special CONIN$ / CONOUT$ devices doesn't work properly for non-ASCII usernames an passwords. It also doesn't work for terminal emulators that hide the native console window (such as mintty), and 'TERM=xterm*' is not necessarily a reliable indicator for such terminals. The new shell_prompt() function, on the other hand, works fine for both MSys1 and MSys2, in native console windows as well as mintty, and properly supports Unicode. It just needs bash on the path (for 'read -s', which is bash-specific). On Windows, try to use the shell to read from the terminal. If that fails with ENOENT (i.e. bash was not found), use CONIN/OUT as fallback. Note: To test this, create a UTF-8 credential file with non-ASCII chars, e.g. in git-bash: 'echo url=http://täst.com > cred.txt'. Then in git-cmd, 'git credential fill --- compat/terminal.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/compat/terminal.c b/compat/terminal.c index cd6e2e55a5..1d37f0aafb 100644 --- a/compat/terminal.c +++ b/compat/terminal.c @@ -110,6 +110,7 @@ static char *shell_prompt(const char *prompt, int echo) child.argv = read_input; child.in = -1; child.out = -1; + child.silent_exec_failure = 1; if (start_command(&child)) return NULL; @@ -153,11 +154,14 @@ char *git_terminal_prompt(const char *prompt, int echo) static struct strbuf buf = STRBUF_INIT; int r; FILE *input_fh, *output_fh; -#ifdef GIT_WINDOWS_NATIVE - const char *term = getenv("TERM"); - if (term && starts_with(term, "xterm")) - return shell_prompt(prompt, echo); +#ifdef GIT_WINDOWS_NATIVE + + /* try shell_prompt first, fall back to CONIN/OUT if bash is missing */ + char *result = shell_prompt(prompt, echo); + if (result || errno != ENOENT) + return result; + #endif input_fh = fopen(INPUT_PATH, "r" FORCE_TEXT); From 318fdb4279d7000b2f5a369e4fe42c04925ee65a Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 13 Aug 2015 15:25:59 +0000 Subject: [PATCH 060/288] mingw: Suppress warning that :.gitattributes does not exist On Windows, a file name containing a colon is illegal. We should therefore expect the corresponding errno when `fopen()` is called for a path of the form :.gitattributes. This fixes https://github.com/git-for-windows/git/issues/255. Signed-off-by: Johannes Schindelin --- attr.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/attr.c b/attr.c index 1fcf042b87..2850104719 100644 --- a/attr.c +++ b/attr.c @@ -373,7 +373,7 @@ static struct attr_stack *read_attr_from_file(const char *path, int macro_ok) int lineno = 0; if (!fp) { - if (errno != ENOENT && errno != ENOTDIR) + if (errno != ENOENT && errno != ENOTDIR && errno != EINVAL) warn_on_inaccessible(path); return NULL; } From 84e61a9ea7818f16a74fb71322502789d0c57d53 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 10 Jan 2017 21:42:08 +0100 Subject: [PATCH 061/288] Windows: force-recompile git.res for differing architectures When git.rc is compiled into git.res, the result is actually dependent on the architecture. That is, you cannot simply link a 32-bit git.res into a 64-bit git.exe. Therefore, to allow 32-bit and 64-bit builds in the same directory, we let git.res depend on GIT-PREFIX so that it gets recompiled when compiling for a different architecture (this works because the exec path changes based on the architecture: /mingw32/libexec/git-core for 32-bit and /mingw64/libexec/git-core for 64-bit). Signed-off-by: Johannes Schindelin --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 3bca483fd3..7c7a21b054 100644 --- a/Makefile +++ b/Makefile @@ -1822,7 +1822,7 @@ $(SCRIPT_LIB) : % : %.sh GIT-SCRIPT-DEFINES $(QUIET_GEN)$(cmd_munge_script) && \ mv $@+ $@ -git.res: git.rc GIT-VERSION-FILE +git.res: git.rc GIT-VERSION-FILE GIT-PREFIX $(QUIET_RC)$(RC) \ $(join -DMAJOR= -DMINOR= -DMICRO= -DPATCHLEVEL=, $(wordlist 1, 4, \ $(shell echo $(GIT_VERSION) 0 0 0 0 | tr '.a-zA-Z-' ' '))) \ From 630888f152de96244cfc021bc40880a4153a305b Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 6 Jan 2017 17:39:50 +0100 Subject: [PATCH 062/288] mingw: load system libraries the recommended way When we access IPv6-related functions, we load the corresponding system library using the `LoadLibrary()` function, which is not the recommended way to load system libraries. In practice, it does not make a difference: the `ws2_32.dll` library containing the IPv6 functions is already loaded into memory, so LoadLibrary() simply reuses the already-loaded library. Still, recommended way is recommended way, so let's use that instead. While at it, also adjust the code in contrib/ that loads system libraries. Signed-off-by: Johannes Schindelin --- compat/mingw.c | 3 ++- contrib/credential/wincred/git-credential-wincred.c | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index d51984523e..ba62cea707 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -1665,7 +1665,8 @@ static void ensure_socket_initialization(void) WSAGetLastError()); for (name = libraries; *name; name++) { - ipv6_dll = LoadLibrary(*name); + ipv6_dll = LoadLibraryExA(*name, NULL, + LOAD_LIBRARY_SEARCH_SYSTEM32); if (!ipv6_dll) continue; diff --git a/contrib/credential/wincred/git-credential-wincred.c b/contrib/credential/wincred/git-credential-wincred.c index 006134043a..1af0cbbdc4 100644 --- a/contrib/credential/wincred/git-credential-wincred.c +++ b/contrib/credential/wincred/git-credential-wincred.c @@ -75,7 +75,8 @@ static CredDeleteWT CredDeleteW; static void load_cred_funcs(void) { /* load DLLs */ - advapi = LoadLibrary("advapi32.dll"); + advapi = LoadLibraryExA("advapi32.dll", NULL, + LOAD_LIBRARY_SEARCH_SYSTEM32); if (!advapi) die("failed to load advapi32.dll"); From 8dcf2d9fdc88b3a4ddcdb77446310d6d73768c92 Mon Sep 17 00:00:00 2001 From: Thomas Braun Date: Thu, 2 Oct 2014 14:43:05 +0200 Subject: [PATCH 063/288] t0027: Disable test on MINGW We can't mmap 2GB of RAM on our 32bit platform, so just disable the test. Signed-off-by: Thomas Braun --- t/t0021-conversion.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/t0021-conversion.sh b/t/t0021-conversion.sh index 161f560446..8b66e649e4 100755 --- a/t/t0021-conversion.sh +++ b/t/t0021-conversion.sh @@ -276,7 +276,7 @@ test_expect_success 'filter that does not read is fine' ' test_cmp expect actual ' -test_expect_success EXPENSIVE 'filter large file' ' +test_expect_success EXPENSIVE,!MINGW 'filter large file' ' test_config filter.largefile.smudge cat && test_config filter.largefile.clean cat && for i in $(test_seq 1 2048); do printf "%1048576d" 1; done >2GB && From ab9f1164c194fef2ebd3d6b77eb72a509725d14e Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 8 Oct 2014 11:15:59 +0200 Subject: [PATCH 064/288] Mark t0027-auto-crlf as cheap enough for MinGW t0027 is marked expensive, but really, for MinGW we want to run these tests always. Suggested by Thomas Braun. Signed-off-by: Johannes Schindelin --- t/t0027-auto-crlf.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/t/t0027-auto-crlf.sh b/t/t0027-auto-crlf.sh index 90db54c9f9..6e52195ff7 100755 --- a/t/t0027-auto-crlf.sh +++ b/t/t0027-auto-crlf.sh @@ -4,9 +4,9 @@ test_description='CRLF conversion all combinations' . ./test-lib.sh -if ! test_have_prereq EXPENSIVE +if ! test_have_prereq EXPENSIVE && ! test_have_prereq MINGW then - skip_all="EXPENSIVE not set" + skip_all="Neither EXPENSIVE nor MINGW set" test_done fi From 8eace805300cc6c1b18b3f0073344e9e0a2c7ab0 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 23 Feb 2015 15:55:47 +0000 Subject: [PATCH 065/288] Skip t9020 with MSys2 POSIX-to-Windows path mangling would make it fail. Signed-off-by: Johannes Schindelin --- t/t9020-remote-svn.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/t/t9020-remote-svn.sh b/t/t9020-remote-svn.sh index 4d81ba1c2c..7e54d26ced 100755 --- a/t/t9020-remote-svn.sh +++ b/t/t9020-remote-svn.sh @@ -12,6 +12,12 @@ then test_done fi +if test_have_prereq MINGW +then + skip_all='skipping remote-svn tests for lack of POSIX' + test_done +fi + # Override svnrdump with our simulator PATH="$HOME:$PATH" export PATH PYTHON_PATH GIT_BUILD_DIR From d2e581eb29074b201222f175fedc81d9f7a92456 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 18 Sep 2015 17:32:26 +0200 Subject: [PATCH 066/288] t0302: add forgotten quotes This was probably missed because nobody had a left-over `trash/` directory and the `-f` flag made sure that no error message was produced when the file was not found that *actually* wanted to be removed. Signed-off-by: Johannes Schindelin --- t/t0302-credential-store.sh | 2 +- t/t3900-i18n-commit.sh | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/t/t0302-credential-store.sh b/t/t0302-credential-store.sh index 1d8d1f210b..d6b54e8c65 100755 --- a/t/t0302-credential-store.sh +++ b/t/t0302-credential-store.sh @@ -37,7 +37,7 @@ helper_test store unset XDG_CONFIG_HOME test_expect_success 'if custom xdg file exists, home and xdg files not created' ' - test_when_finished "rm -f $HOME/xdg/git/credentials" && + test_when_finished "rm -f \"$HOME/xdg/git/credentials\"" && test -s "$HOME/xdg/git/credentials" && test_path_is_missing "$HOME/.git-credentials" && test_path_is_missing "$HOME/.config/git/credentials" diff --git a/t/t3900-i18n-commit.sh b/t/t3900-i18n-commit.sh index 3b94283e35..9e4e694d93 100755 --- a/t/t3900-i18n-commit.sh +++ b/t/t3900-i18n-commit.sh @@ -40,7 +40,7 @@ test_expect_success 'UTF-16 refused because of NULs' ' ' test_expect_success 'UTF-8 invalid characters refused' ' - test_when_finished "rm -f $HOME/stderr $HOME/invalid" && + test_when_finished "rm -f \"$HOME/stderr $HOME/invalid\"" && echo "UTF-8 characters" >F && printf "Commit message\n\nInvalid surrogate:\355\240\200\n" \ >"$HOME/invalid" && @@ -49,7 +49,7 @@ test_expect_success 'UTF-8 invalid characters refused' ' ' test_expect_success 'UTF-8 overlong sequences rejected' ' - test_when_finished "rm -f $HOME/stderr $HOME/invalid" && + test_when_finished "rm -f \"$HOME/stderr $HOME/invalid\"" && rm -f "$HOME/stderr" "$HOME/invalid" && echo "UTF-8 overlong" >F && printf "\340\202\251ommit message\n\nThis is not a space:\300\240\n" \ @@ -59,7 +59,7 @@ test_expect_success 'UTF-8 overlong sequences rejected' ' ' test_expect_success 'UTF-8 non-characters refused' ' - test_when_finished "rm -f $HOME/stderr $HOME/invalid" && + test_when_finished "rm -f \"$HOME/stderr $HOME/invalid\"" && echo "UTF-8 non-character 1" >F && printf "Commit message\n\nNon-character:\364\217\277\276\n" \ >"$HOME/invalid" && @@ -68,7 +68,7 @@ test_expect_success 'UTF-8 non-characters refused' ' ' test_expect_success 'UTF-8 non-characters refused' ' - test_when_finished "rm -f $HOME/stderr $HOME/invalid" && + test_when_finished "rm -f \"$HOME/stderr $HOME/invalid\"" && echo "UTF-8 non-character 2." >F && printf "Commit message\n\nNon-character:\357\267\220\n" \ >"$HOME/invalid" && From 6b99960cce29cf16d63d4334afad67953fb58a10 Mon Sep 17 00:00:00 2001 From: Sverre Rabbelier Date: Sun, 24 Jul 2011 15:54:04 +0200 Subject: [PATCH 067/288] t9350: point out that refs are not updated correctly This happens only when the corresponding commits are not exported in the current fast-export run. This can happen either when the relevant commit is already marked, or when the commit is explicitly marked as UNINTERESTING with a negative ref by another argument. This breaks fast-export basec remote helpers. Signed-off-by: Sverre Rabbelier --- t/t9350-fast-export.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/t/t9350-fast-export.sh b/t/t9350-fast-export.sh index b5149fde6e..d5fab265cb 100755 --- a/t/t9350-fast-export.sh +++ b/t/t9350-fast-export.sh @@ -522,4 +522,15 @@ test_expect_success 'delete refspec' ' test_cmp expected actual ' +cat > expected << EOF +reset refs/heads/master +from $(git rev-parse master) + +EOF + +test_expect_failure 'refs are updated even if no commits need to be exported' ' + git fast-export master..master > actual && + test_cmp expected actual +' + test_done From a349d71288c05b4e5b89c03c7e2fb228f74939ba Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sat, 23 Jul 2011 15:55:26 +0200 Subject: [PATCH 068/288] fast-export: do not refer to non-existing marks When calling `git fast-export a..a b` when a and b refer to the same commit, nothing would be exported, and an incorrect reset line would be printed for b ('from :0'). Signed-off-by: Johannes Schindelin Signed-off-by: Sverre Rabbelier --- builtin/fast-export.c | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/builtin/fast-export.c b/builtin/fast-export.c index 1e815b5577..bfa10f5124 100644 --- a/builtin/fast-export.c +++ b/builtin/fast-export.c @@ -847,9 +847,20 @@ static void get_tags_and_duplicates(struct rev_cmdline_info *info) } } +static void handle_reset(const char *name, struct object *object) +{ + int mark = get_object_mark(object); + + if (mark) + printf("reset %s\nfrom :%d\n\n", name, + get_object_mark(object)); + else + printf("reset %s\nfrom %s\n\n", name, + oid_to_hex(&object->oid)); +} + static void handle_tags_and_duplicates(void) { - struct commit *commit; int i; for (i = extra_refs.nr - 1; i >= 0; i--) { @@ -863,9 +874,7 @@ static void handle_tags_and_duplicates(void) if (anonymize) name = anonymize_refname(name); /* create refs pointing to already seen commits */ - commit = (struct commit *)object; - printf("reset %s\nfrom :%d\n\n", name, - get_object_mark(&commit->object)); + handle_reset(name, object); show_progress(); break; } From 8df5c1504a61b319af0fdb73bf47f1b5a9e0dc9b Mon Sep 17 00:00:00 2001 From: Sverre Rabbelier Date: Sat, 28 Aug 2010 20:49:01 -0500 Subject: [PATCH 069/288] transport-helper: add trailing -- [PT: ensure we add an additional element to the argv array] --- transport-helper.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/transport-helper.c b/transport-helper.c index 91aed35ebb..ec96eb4aee 100644 --- a/transport-helper.c +++ b/transport-helper.c @@ -472,6 +472,8 @@ static int get_exporter(struct transport *transport, for (i = 0; i < revlist_args->nr; i++) argv_array_push(&fastexport->args, revlist_args->items[i].string); + argv_array_push(&fastexport->args, "--"); + fastexport->git_cmd = 1; return start_command(fastexport); } From 2fff40f35a0b1f08607553fe45e1bafc48760eef Mon Sep 17 00:00:00 2001 From: Sverre Rabbelier Date: Sun, 24 Jul 2011 00:06:00 +0200 Subject: [PATCH 070/288] remote-helper: check helper status after import/export Signed-off-by: Johannes Schindelin Signed-off-by: Sverre Rabbelier --- builtin/clone.c | 4 +++- t/t5801-remote-helpers.sh | 2 +- transport-helper.c | 15 +++++++++++++++ 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/builtin/clone.c b/builtin/clone.c index 6c76a6ed66..f361fe700d 100644 --- a/builtin/clone.c +++ b/builtin/clone.c @@ -1076,7 +1076,9 @@ int cmd_clone(int argc, const char **argv, const char *prefix) } if (!is_local && !complete_refs_before_fetch) - transport_fetch_refs(transport, mapped_refs); + if (transport_fetch_refs(transport, mapped_refs)) + die(_("could not fetch refs from %s"), + transport->url); remote_head = find_ref_by_name(refs, "HEAD"); remote_head_points_at = diff --git a/t/t5801-remote-helpers.sh b/t/t5801-remote-helpers.sh index 362b1581e0..8176883c97 100755 --- a/t/t5801-remote-helpers.sh +++ b/t/t5801-remote-helpers.sh @@ -228,7 +228,7 @@ test_expect_success 'push update refs failure' ' echo "update fail" >>file && git commit -a -m "update fail" && git rev-parse --verify testgit/origin/heads/update >expect && - test_expect_code 1 env GIT_REMOTE_TESTGIT_FAILURE="non-fast forward" \ + test_must_fail env GIT_REMOTE_TESTGIT_FAILURE="non-fast forward" \ git push origin update && git rev-parse --verify testgit/origin/heads/update >actual && test_cmp expect actual diff --git a/transport-helper.c b/transport-helper.c index ec96eb4aee..5c7e167f2b 100644 --- a/transport-helper.c +++ b/transport-helper.c @@ -478,6 +478,19 @@ static int get_exporter(struct transport *transport, return start_command(fastexport); } +static void check_helper_status(struct helper_data *data) +{ + int pid, status; + + pid = waitpid(data->helper->pid, &status, WNOHANG); + if (pid < 0) + die("Could not retrieve status of remote helper '%s'", + data->name); + if (pid > 0 && WIFEXITED(status)) + die("Remote helper '%s' died with %d", + data->name, WEXITSTATUS(status)); +} + static int fetch_with_import(struct transport *transport, int nr_heads, struct ref **to_fetch) { @@ -514,6 +527,7 @@ static int fetch_with_import(struct transport *transport, if (finish_command(&fastimport)) die("Error while running fast-import"); + check_helper_status(data); /* * The fast-import stream of a remote helper that advertises @@ -974,6 +988,7 @@ static int push_refs_with_export(struct transport *transport, if (finish_command(&exporter)) die("Error while running fast-export"); + check_helper_status(data); if (push_update_refs_status(data, remote_refs, flags)) return 1; From 6b15fb134b864f8099abf9198ec99002bbee12a5 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 9 Apr 2012 13:04:35 -0500 Subject: [PATCH 071/288] Always auto-gc after calling a fast-import transport After importing anything with fast-import, we should always let the garbage collector do its job, since the objects are written to disk inefficiently. This brings down an initial import of http://selenic.com/hg from about 230 megabytes to about 14. In the future, we may want to make this configurable on a per-remote basis, or maybe teach fast-import about it in the first place. Signed-off-by: Johannes Schindelin --- transport-helper.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/transport-helper.c b/transport-helper.c index 5c7e167f2b..be516c8021 100644 --- a/transport-helper.c +++ b/transport-helper.c @@ -13,6 +13,8 @@ #include "refs.h" static int debug; +/* TODO: put somewhere sensible, e.g. git_transport_options? */ +static int auto_gc = 1; struct helper_data { const char *name; @@ -561,6 +563,12 @@ static int fetch_with_import(struct transport *transport, } } strbuf_release(&buf); + if (auto_gc) { + const char *argv_gc_auto[] = { + "gc", "--auto", "--quiet", NULL, + }; + run_command_v_opt(argv_gc_auto, RUN_GIT_CMD); + } return 0; } From dd1f6fedb2bdd04abb917ac91685ddaee8416c4b Mon Sep 17 00:00:00 2001 From: Evgeny Pashkin Date: Thu, 29 Sep 2011 22:32:37 +0400 Subject: [PATCH 072/288] Fixed wrong path delimiter in exe finding On Windows XP3 in git bash git clone git@github.com:octocat/Spoon-Knife.git cd Spoon-Knife git gui menu Remote\Fetch from\origin error: cannot spawn git: No such file or directory error: could not run rev-list if u run git fetch --all it worked normal in git bash or gitgui tools In second version CreateProcess get 'C:\Git\libexec\git-core/git.exe' in first version - C:/Git/libexec/git-core/git.exe and not executes (unix slashes) after fixing C:\Git\libexec\git-core\git.exe or C:/Git/libexec/git-core\git.exe it works normal Signed-off-by: Johannes Schindelin --- compat/mingw.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compat/mingw.c b/compat/mingw.c index 3fbfda5978..481e7c6df6 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -998,7 +998,7 @@ static void free_path_split(char **path) static char *lookup_prog(const char *dir, const char *cmd, int isexe, int exe_only) { char path[MAX_PATH]; - snprintf(path, sizeof(path), "%s/%s.exe", dir, cmd); + snprintf(path, sizeof(path), "%s\\%s.exe", dir, cmd); if (!isexe && access(path, F_OK) == 0) return xstrdup(path); From abd1e07630f76367d060f97c58e05ab1c3f89752 Mon Sep 17 00:00:00 2001 From: Adam Roben Date: Tue, 5 Jun 2012 10:24:11 -0400 Subject: [PATCH 073/288] Fix launching of externals from Unicode paths If Git were installed in a path containing non-ASCII characters, commands such as git-am and git-submodule, which are implemented as externals, would fail to launch with the following error: > fatal: 'am' appears to be a git command, but we were not > able to execute it. Maybe git-am is broken? This was due to lookup_prog not being Unicode-aware. It was somehow missed in 2ee5a1a14ad17ff35f0ad52390a27fbbc41258f3. Note that the only problem in this function was calling GetFileAttributes instead of GetFileAttributesW. The calls to access() were fine because access() is a macro which resolves to mingw_access, which already handles Unicode correctly. But I changed lookup_prog to use _waccess directly so that we only convert the path to UTF-16 once. Signed-off-by: Adam Roben --- compat/mingw.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 481e7c6df6..7197e66769 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -998,14 +998,20 @@ static void free_path_split(char **path) static char *lookup_prog(const char *dir, const char *cmd, int isexe, int exe_only) { char path[MAX_PATH]; + wchar_t wpath[MAX_PATH]; snprintf(path, sizeof(path), "%s\\%s.exe", dir, cmd); - if (!isexe && access(path, F_OK) == 0) + if (xutftowcs_path(wpath, path) < 0) + return NULL; + + if (!isexe && _waccess(wpath, F_OK) == 0) return xstrdup(path); path[strlen(path)-4] = '\0'; - if ((!exe_only || isexe) && access(path, F_OK) == 0) - if (!(GetFileAttributes(path) & FILE_ATTRIBUTE_DIRECTORY)) + if ((!exe_only || isexe) && _waccess(wpath, F_OK) == 0) { + + if (!(GetFileAttributesW(wpath) & FILE_ATTRIBUTE_DIRECTORY)) return xstrdup(path); + } return NULL; } From bfd25b7dd44205d6597b8f76b7acd42abf35caba Mon Sep 17 00:00:00 2001 From: Adam Roben Date: Tue, 5 Jun 2012 15:40:33 -0400 Subject: [PATCH 074/288] Make non-.exe externals work again 7ebac8cb94f3a06d3fbdde469414a1443ca45510 made launching of .exe externals work when installed in Unicode paths. But it broke launching of non-.exe externals, no matter where they were installed. We now correctly maintain the UTF-8 and UTF-16 paths in tandem in lookup_prog. This fixes t5526, among others. Signed-off-by: Adam Roben --- compat/mingw.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 7197e66769..b931c07126 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -1006,11 +1006,12 @@ static char *lookup_prog(const char *dir, const char *cmd, int isexe, int exe_on if (!isexe && _waccess(wpath, F_OK) == 0) return xstrdup(path); - path[strlen(path)-4] = '\0'; + wpath[wcslen(wpath)-4] = '\0'; if ((!exe_only || isexe) && _waccess(wpath, F_OK) == 0) { - - if (!(GetFileAttributesW(wpath) & FILE_ATTRIBUTE_DIRECTORY)) + if (!(GetFileAttributesW(wpath) & FILE_ATTRIBUTE_DIRECTORY)) { + path[strlen(path)-4] = '\0'; return xstrdup(path); + } } return NULL; } From 18f36bb9b71b5157f354577453d999b150b205c0 Mon Sep 17 00:00:00 2001 From: Heiko Voigt Date: Fri, 22 Jun 2012 19:51:20 +0200 Subject: [PATCH 075/288] help: correct behavior for is_executable on Windows The previous implementation said that the filesystem information on Windows is not reliable to determine whether a file is executable. To find gather this information it was peeking into the first two bytes of a file to see whether it looks executable. Apart from the fact that on Windows executables are usually defined as such by their extension it lead to slow opening of help file in some situations. When you have virus scanner running calling open on an executable file is a potentially expensive operation. See the following measurements (in seconds) for example. With virus scanner running (coldcache): $ ./a.exe /libexec/git-core/ before open (git-add.exe): 0.000000 after open (git-add.exe): 0.412873 before open (git-annotate.exe): 0.000175 after open (git-annotate.exe): 0.397925 before open (git-apply.exe): 0.000243 after open (git-apply.exe): 0.399996 before open (git-archive.exe): 0.000147 after open (git-archive.exe): 0.397783 before open (git-bisect--helper.exe): 0.000160 after open (git-bisect--helper.exe): 0.397700 before open (git-blame.exe): 0.000160 after open (git-blame.exe): 0.399136 ... With virus scanner running (hotcache): $ ./a.exe /libexec/git-core/ before open (git-add.exe): 0.000000 after open (git-add.exe): 0.000325 before open (git-annotate.exe): 0.000229 after open (git-annotate.exe): 0.000177 before open (git-apply.exe): 0.000167 after open (git-apply.exe): 0.000150 before open (git-archive.exe): 0.000154 after open (git-archive.exe): 0.000156 before open (git-bisect--helper.exe): 0.000132 after open (git-bisect--helper.exe): 0.000180 before open (git-blame.exe): 0.000718 after open (git-blame.exe): 0.000724 ... This test did just list the given directory and open() each file in it. With this patch I get: $ time git help git Launching default browser to display HTML ... real 0m8.723s user 0m0.000s sys 0m0.000s and without $ time git help git Launching default browser to display HTML ... real 1m37.734s user 0m0.000s sys 0m0.031s both tests with cold cache and giving the machine some time to settle down after restart. Signed-off-by: Heiko Voigt --- help.c | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/help.c b/help.c index 53e2a67e00..704e467aa9 100644 --- a/help.c +++ b/help.c @@ -105,7 +105,22 @@ static int is_executable(const char *name) return 0; #if defined(GIT_WINDOWS_NATIVE) -{ /* cannot trust the executable bit, peek into the file instead */ + /* + * On Windows there is no executable bit. The file extension + * indicates whether it can be run as an executable, and Git + * has special-handling to detect scripts and launch them + * through the indicated script interpreter. We test for the + * file extension first because virus scanners may make + * opening an executable for reading expensive. + */ + if (ends_with(name, ".exe")) + return S_IXUSR; + +{ + /* + * Now that we know it does not have an executable extension, + * peek into the file instead. + */ char buf[3] = { 0 }; int n; int fd = open(name, O_RDONLY); @@ -113,8 +128,8 @@ static int is_executable(const char *name) if (fd >= 0) { n = read(fd, buf, 2); if (n == 2) - /* DOS executables start with "MZ" */ - if (!strcmp(buf, "#!") || !strcmp(buf, "MZ")) + /* look for a she-bang */ + if (!strcmp(buf, "#!")) st.st_mode |= S_IXUSR; close(fd); } From bf6b9a3434cae4a4d354ee05ea5088a2f6683930 Mon Sep 17 00:00:00 2001 From: Thomas Braun Date: Thu, 8 May 2014 21:43:24 +0200 Subject: [PATCH 076/288] Config option to disable side-band-64k for transport Since commit 0c499ea60f the send-pack builtin uses the side-band-64k capability if advertised by the server. Unfortunately this breaks pushing over the dump git protocol if used over a network connection. The detailed reasons for this breakage are (by courtesy of Jeff Preshing, quoted from ttps://groups.google.com/d/msg/msysgit/at8D7J-h7mw/eaLujILGUWoJ): ---------------------------------------------------------------------------- MinGW wraps Windows sockets in CRT file descriptors in order to mimic the functionality of POSIX sockets. This causes msvcrt.dll to treat sockets as Installable File System (IFS) handles, calling ReadFile, WriteFile, DuplicateHandle and CloseHandle on them. This approach works well in simple cases on recent versions of Windows, but does not support all usage patterns. In particular, using this approach, any attempt to read & write concurrently on the same socket (from one or more processes) will deadlock in a scenario where the read waits for a response from the server which is only invoked after the write. This is what send_pack currently attempts to do in the use_sideband codepath. ---------------------------------------------------------------------------- The new config option "sendpack.sideband" allows to override the side-band-64k capability of the server, and thus makes the dump git protocol work. Other transportation methods like ssh and http/https still benefit from the sideband channel, therefore the default value of "sendpack.sideband" is still true. Signed-off-by: Thomas Braun --- Documentation/config.txt | 6 ++++++ send-pack.c | 14 +++++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/Documentation/config.txt b/Documentation/config.txt index 1fee83ca42..5c7486af8f 100644 --- a/Documentation/config.txt +++ b/Documentation/config.txt @@ -3083,3 +3083,9 @@ web.browser:: Specify a web browser that may be used by some commands. Currently only linkgit:git-instaweb[1] and linkgit:git-help[1] may use it. + +sendpack.sideband:: + Allows to disable the side-band-64k capability for send-pack even + when it is advertised by the server. Makes it possible to work + around a limitation in the git for windows implementation together + with the dump git protocol. Defaults to true. diff --git a/send-pack.c b/send-pack.c index 6195b43e9a..18bc1df6f5 100644 --- a/send-pack.c +++ b/send-pack.c @@ -36,6 +36,16 @@ int option_parse_push_signed(const struct option *opt, die("bad %s argument: %s", opt->long_name, arg); } +static int config_use_sideband = 1; + +static int send_pack_config(const char *var, const char *value, void *unused) +{ + if (!strcmp("sendpack.sideband", var)) + config_use_sideband = git_config_bool(var, value); + + return 0; +} + static void feed_object(const unsigned char *sha1, FILE *fh, int negative) { if (negative && !has_sha1_file(sha1)) @@ -380,6 +390,8 @@ int send_pack(struct send_pack_args *args, struct async demux; const char *push_cert_nonce = NULL; + git_config(send_pack_config, NULL); + /* Does the other end support the reporting? */ if (server_supports("report-status")) status_report = 1; @@ -387,7 +399,7 @@ int send_pack(struct send_pack_args *args, allow_deleting_refs = 1; if (server_supports("ofs-delta")) args->use_ofs_delta = 1; - if (server_supports("side-band-64k")) + if (config_use_sideband && server_supports("side-band-64k")) use_sideband = 1; if (server_supports("quiet")) quiet_supported = 1; From 9bbba2714ac51f24c8dc49d7e40d05020824acb4 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 10 Jan 2017 23:14:20 +0100 Subject: [PATCH 077/288] Win32: simplify loading of DLL functions Dynamic loading of DLL functions is duplicated in several places. Add a set of macros to simplify the process. Signed-off-by: Karsten Blees Signed-off-by: Johannes Schindelin --- compat/win32.h | 2 ++ compat/win32/lazyload.h | 43 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 compat/win32/lazyload.h diff --git a/compat/win32.h b/compat/win32.h index a97e880757..4a0ccb8075 100644 --- a/compat/win32.h +++ b/compat/win32.h @@ -6,6 +6,8 @@ #include #endif +#include "compat/win32/lazyload.h" + static inline int file_attr_to_st_mode (DWORD attr) { int fMode = S_IREAD; diff --git a/compat/win32/lazyload.h b/compat/win32/lazyload.h new file mode 100644 index 0000000000..9d1a055502 --- /dev/null +++ b/compat/win32/lazyload.h @@ -0,0 +1,43 @@ +#ifndef LAZYLOAD_H +#define LAZYLOAD_H + +/* simplify loading of DLL functions */ + +struct proc_addr { + const char *const dll; + const char *const function; + FARPROC pfunction; + unsigned initialized : 1; +}; + +/* Declares a function to be loaded dynamically from a DLL. */ +#define DECLARE_PROC_ADDR(dll, rettype, function, ...) \ + static struct proc_addr proc_addr_##function = \ + { #dll, #function, NULL, 0 }; \ + static rettype (WINAPI *function)(__VA_ARGS__) + +/* + * Loads a function from a DLL (once-only). + * Returns non-NULL function pointer on success. + * Returns NULL + errno == ENOSYS on failure. + */ +#define INIT_PROC_ADDR(function) (function = get_proc_addr(&proc_addr_##function)) + +static inline void *get_proc_addr(struct proc_addr *proc) +{ + /* only do this once */ + if (!proc->initialized) { + HANDLE hnd; + proc->initialized = 1; + hnd = LoadLibraryExA(proc->dll, NULL, + LOAD_LIBRARY_SEARCH_SYSTEM32); + if (hnd) + proc->pfunction = GetProcAddress(hnd, proc->function); + } + /* set ENOSYS if DLL or function was not found */ + if (!proc->pfunction) + errno = ENOSYS; + return proc->pfunction; +} + +#endif From 95951f6dd11ea3aac1c39fa6f6876344fbcada2d Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 12 Oct 2015 19:24:05 +0000 Subject: [PATCH 078/288] getpwuid(mingw): initialize the structure only once Signed-off-by: Johannes Schindelin --- compat/mingw.c | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 3fbfda5978..9cd1fc800c 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -1765,16 +1765,27 @@ int mingw_getpagesize(void) struct passwd *getpwuid(int uid) { + static unsigned initialized; static char user_name[100]; - static struct passwd p; + static struct passwd *p; + DWORD len; - DWORD len = sizeof(user_name); - if (!GetUserName(user_name, &len)) + if (initialized) + return p; + + len = sizeof(user_name); + if (!GetUserName(user_name, &len)) { + initialized = 1; return NULL; - p.pw_name = user_name; - p.pw_gecos = "unknown"; - p.pw_dir = NULL; - return &p; + } + + p = xmalloc(sizeof(*p)); + p->pw_name = user_name; + p->pw_gecos = "unknown"; + p->pw_dir = NULL; + + initialized = 1; + return p; } static HANDLE timer_event; From 114de8441eef9f583c6d1582658647324148f7a2 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 13 Oct 2015 10:05:53 +0000 Subject: [PATCH 079/288] getpwuid(mingw): provide a better default for the user name We do have the excellent GetUserInfoEx() function to obtain more detailed information of the current user (if the user is part of a Windows domain); Let's use it. Suggested by Lutz Roeder. To avoid the cost of loading Secur32.dll (even lazily, loading DLLs takes a non-neglibile amount of time), we use the established technique to load DLLs only when, and if, needed. Signed-off-by: Johannes Schindelin --- compat/mingw.c | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/compat/mingw.c b/compat/mingw.c index 9cd1fc800c..afe6e522fd 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -1763,6 +1763,33 @@ int mingw_getpagesize(void) return si.dwAllocationGranularity; } +/* See https://msdn.microsoft.com/en-us/library/windows/desktop/ms724435.aspx */ +enum EXTENDED_NAME_FORMAT { + NameDisplay = 3, + NameUserPrincipal = 8 +}; + +static char *get_extended_user_info(enum EXTENDED_NAME_FORMAT type) +{ + DECLARE_PROC_ADDR(secur32.dll, BOOL, GetUserNameExW, + enum EXTENDED_NAME_FORMAT, LPCWSTR, PULONG); + static wchar_t wbuffer[1024]; + DWORD len; + + if (!INIT_PROC_ADDR(GetUserNameExW)) + return NULL; + + len = ARRAY_SIZE(wbuffer); + if (GetUserNameExW(type, wbuffer, &len)) { + char *converted = xmalloc((len *= 3)); + if (xwcstoutf(converted, wbuffer, len) >= 0) + return converted; + free(converted); + } + + return NULL; +} + struct passwd *getpwuid(int uid) { static unsigned initialized; @@ -1781,7 +1808,9 @@ struct passwd *getpwuid(int uid) p = xmalloc(sizeof(*p)); p->pw_name = user_name; - p->pw_gecos = "unknown"; + p->pw_gecos = get_extended_user_info(NameDisplay); + if (!p->pw_gecos) + p->pw_gecos = "unknown"; p->pw_dir = NULL; initialized = 1; From 5f01f03e4053d0e4dcbf99718b8648cfc7a4e951 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 13 Oct 2015 10:36:52 +0000 Subject: [PATCH 080/288] mingw: use domain information for default email When a user is registered in a Windows domain, it is really easy to obtain the email address. So let's do that. Suggested by Lutz Roeder. Signed-off-by: Johannes Schindelin --- compat/mingw.c | 5 +++++ compat/mingw.h | 2 ++ git-compat-util.h | 4 ++++ ident.c | 4 +++- 4 files changed, 14 insertions(+), 1 deletion(-) diff --git a/compat/mingw.c b/compat/mingw.c index afe6e522fd..ef790b1036 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -1790,6 +1790,11 @@ static char *get_extended_user_info(enum EXTENDED_NAME_FORMAT type) return NULL; } +char *mingw_query_user_email(void) +{ + return get_extended_user_info(NameUserPrincipal); +} + struct passwd *getpwuid(int uid) { static unsigned initialized; diff --git a/compat/mingw.h b/compat/mingw.h index 3350169555..dabc43d49a 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -417,6 +417,8 @@ static inline void convert_slashes(char *path) int mingw_offset_1st_component(const char *path); #define offset_1st_component mingw_offset_1st_component #define PATH_SEP ';' +extern char *mingw_query_user_email(void); +#define query_user_email mingw_query_user_email #if !defined(__MINGW64_VERSION_MAJOR) && (!defined(_MSC_VER) || _MSC_VER < 1800) #define PRIuMAX "I64u" #define PRId64 "I64d" diff --git a/git-compat-util.h b/git-compat-util.h index 87237b092b..03fb901231 100644 --- a/git-compat-util.h +++ b/git-compat-util.h @@ -370,6 +370,10 @@ static inline char *git_find_last_dir_sep(const char *path) #define find_last_dir_sep git_find_last_dir_sep #endif +#ifndef query_user_email +#define query_user_email() NULL +#endif + #if defined(__HP_cc) && (__HP_cc >= 61000) #define NORETURN __attribute__((noreturn)) #define NORETURN_PTR diff --git a/ident.c b/ident.c index ac4ae02b48..dc0d905881 100644 --- a/ident.c +++ b/ident.c @@ -169,7 +169,9 @@ const char *ident_default_email(void) strbuf_addstr(&git_default_email, email); committer_ident_explicitly_given |= IDENT_MAIL_GIVEN; author_ident_explicitly_given |= IDENT_MAIL_GIVEN; - } else + } else if ((email = query_user_email()) && email[0]) + strbuf_addstr(&git_default_email, email); + else copy_email(xgetpwuid_self(&default_email_is_bogus), &git_default_email, &default_email_is_bogus); strbuf_trim(&git_default_email); From 737a66a65991413f9a8f4ba339b1a4266e3f0a55 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 10 Jan 2017 23:14:20 +0100 Subject: [PATCH 081/288] winansi: simplify loading the GetCurrentConsoleFontEx() function We introduced helper macros to simplify loading functions dynamically. Might just as well use them. Signed-off-by: Karsten Blees Signed-off-by: Johannes Schindelin --- compat/winansi.c | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/compat/winansi.c b/compat/winansi.c index 82b89ab137..c7ab58361b 100644 --- a/compat/winansi.c +++ b/compat/winansi.c @@ -41,26 +41,21 @@ typedef struct _CONSOLE_FONT_INFOEX { #endif #endif -typedef BOOL (WINAPI *PGETCURRENTCONSOLEFONTEX)(HANDLE, BOOL, - PCONSOLE_FONT_INFOEX); - static void warn_if_raster_font(void) { DWORD fontFamily = 0; - PGETCURRENTCONSOLEFONTEX pGetCurrentConsoleFontEx; + DECLARE_PROC_ADDR(kernel32.dll, BOOL, GetCurrentConsoleFontEx, + HANDLE, BOOL, PCONSOLE_FONT_INFOEX); /* don't bother if output was ascii only */ if (!non_ascii_used) return; /* GetCurrentConsoleFontEx is available since Vista */ - pGetCurrentConsoleFontEx = (PGETCURRENTCONSOLEFONTEX) GetProcAddress( - GetModuleHandle("kernel32.dll"), - "GetCurrentConsoleFontEx"); - if (pGetCurrentConsoleFontEx) { + if (INIT_PROC_ADDR(GetCurrentConsoleFontEx)) { CONSOLE_FONT_INFOEX cfi; cfi.cbSize = sizeof(cfi); - if (pGetCurrentConsoleFontEx(console, 0, &cfi)) + if (GetCurrentConsoleFontEx(console, 0, &cfi)) fontFamily = cfi.FontFamily; } else { /* pre-Vista: check default console font in registry */ From 916b0e7a67f98952d1f9bbbcc06a2aeba86176aa Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 10 Jan 2017 23:14:20 +0100 Subject: [PATCH 082/288] mingw: simplify loading the CreadHardLinkW() function We introduced helper macros to simplify loading functions dynamically. Might just as well use them. Signed-off-by: Karsten Blees Signed-off-by: Johannes Schindelin --- compat/mingw.c | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 6b2fbd4904..c78f05d749 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2102,24 +2102,18 @@ int mingw_raise(int sig) int link(const char *oldpath, const char *newpath) { - typedef BOOL (WINAPI *T)(LPCWSTR, LPCWSTR, LPSECURITY_ATTRIBUTES); - static T create_hard_link = NULL; + DECLARE_PROC_ADDR(kernel32.dll, BOOL, CreateHardLinkW, + LPCWSTR, LPCWSTR, LPSECURITY_ATTRIBUTES); wchar_t woldpath[MAX_LONG_PATH], wnewpath[MAX_LONG_PATH]; + + if (!INIT_PROC_ADDR(CreateHardLinkW)) + return -1; + if (xutftowcs_long_path(woldpath, oldpath) < 0 || xutftowcs_long_path(wnewpath, newpath) < 0) return -1; - if (!create_hard_link) { - create_hard_link = (T) GetProcAddress( - GetModuleHandle("kernel32.dll"), "CreateHardLinkW"); - if (!create_hard_link) - create_hard_link = (T)-1; - } - if (create_hard_link == (T)-1) { - errno = ENOSYS; - return -1; - } - if (!create_hard_link(wnewpath, woldpath, NULL)) { + if (!CreateHardLinkW(wnewpath, woldpath, NULL)) { errno = err_win_to_posix(GetLastError()); return -1; } From cebdae48f58e3171467c2f0bd3cd6f824cd98988 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Mon, 11 May 2015 19:54:23 +0200 Subject: [PATCH 083/288] strbuf_readlink: don't call readlink twice if hint is the exact link size strbuf_readlink() calls readlink() twice if the hint argument specifies the exact size of the link target (e.g. by passing stat.st_size as returned by lstat()). This is necessary because 'readlink(..., hint) == hint' could mean that the buffer was too small. Use hint + 1 as buffer size to prevent this. Signed-off-by: Karsten Blees --- strbuf.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/strbuf.c b/strbuf.c index 8fec6579f7..7e802755ec 100644 --- a/strbuf.c +++ b/strbuf.c @@ -420,12 +420,12 @@ int strbuf_readlink(struct strbuf *sb, const char *path, size_t hint) while (hint < STRBUF_MAXLINK) { int len; - strbuf_grow(sb, hint); - len = readlink(path, sb->buf, hint); + strbuf_grow(sb, hint + 1); + len = readlink(path, sb->buf, hint + 1); if (len < 0) { if (errno != ERANGE) break; - } else if (len < hint) { + } else if (len <= hint) { strbuf_setlen(sb, len); return 0; } From 76adf1f7898c5e39bbfab83c1d7453c3d8cd8bb4 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Mon, 11 May 2015 22:15:40 +0200 Subject: [PATCH 084/288] strbuf_readlink: support link targets that exceed PATH_MAX strbuf_readlink() refuses to read link targets that exceed PATH_MAX (even if a sufficient size was specified by the caller). As some platforms support longer paths, remove this restriction (similar to strbuf_getcwd()). Signed-off-by: Karsten Blees --- strbuf.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/strbuf.c b/strbuf.c index 7e802755ec..51535003a8 100644 --- a/strbuf.c +++ b/strbuf.c @@ -408,8 +408,6 @@ ssize_t strbuf_write(struct strbuf *sb, FILE *f) } -#define STRBUF_MAXLINK (2*PATH_MAX) - int strbuf_readlink(struct strbuf *sb, const char *path, size_t hint) { size_t oldalloc = sb->alloc; @@ -417,7 +415,7 @@ int strbuf_readlink(struct strbuf *sb, const char *path, size_t hint) if (hint < 32) hint = 32; - while (hint < STRBUF_MAXLINK) { + for (;; hint *= 2) { int len; strbuf_grow(sb, hint + 1); @@ -429,9 +427,6 @@ int strbuf_readlink(struct strbuf *sb, const char *path, size_t hint) strbuf_setlen(sb, len); return 0; } - - /* .. the buffer was too small - try again */ - hint *= 2; } if (oldalloc == 0) strbuf_release(sb); From ae4f10fa751559765216ac78b625ea425ef703e1 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Mon, 11 May 2015 19:58:14 +0200 Subject: [PATCH 085/288] lockfile.c: use is_dir_sep() instead of hardcoded '/' checks Signed-off-by: Karsten Blees --- lockfile.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lockfile.c b/lockfile.c index 9268cdf325..65d6d8cd64 100644 --- a/lockfile.c +++ b/lockfile.c @@ -17,14 +17,14 @@ static void trim_last_path_component(struct strbuf *path) int i = path->len; /* back up past trailing slashes, if any */ - while (i && path->buf[i - 1] == '/') + while (i && is_dir_sep(path->buf[i - 1])) i--; /* * then go backwards until a slash, or the beginning of the * string */ - while (i && path->buf[i - 1] != '/') + while (i && !is_dir_sep(path->buf[i - 1])) i--; strbuf_setlen(path, i); From 0beb909476a097888873b33ee30a4bf4e1ad2311 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Tue, 12 May 2015 11:09:01 +0200 Subject: [PATCH 086/288] Win32: don't call GetFileAttributes twice in mingw_lstat() GetFileAttributes cannot handle paths with trailing dir separator. The current [l]stat implementation calls GetFileAttributes twice if the path has trailing slashes (first with the original path passed to [l]stat, and and a second time with a path copy with trailing '/' removed). With Unicode conversion, we get the length of the path for free and also have a (wide char) buffer that can be modified. Remove trailing directory separators before calling the Win32 API. Signed-off-by: Karsten Blees --- compat/mingw.c | 48 ++++++++++++------------------------------------ 1 file changed, 12 insertions(+), 36 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index c78f05d749..bdd593248d 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -606,9 +606,18 @@ static int do_lstat(int follow, const char *file_name, struct stat *buf) { WIN32_FILE_ATTRIBUTE_DATA fdata; wchar_t wfilename[MAX_LONG_PATH]; - if (xutftowcs_long_path(wfilename, file_name) < 0) + int wlen = xutftowcs_long_path(wfilename, file_name); + if (wlen < 0) return -1; + /* strip trailing '/', or GetFileAttributes will fail */ + while (wlen && is_dir_sep(wfilename[wlen - 1])) + wfilename[--wlen] = 0; + if (!wlen) { + errno = ENOENT; + return -1; + } + if (GetFileAttributesExW(wfilename, GetFileExInfoStandard, &fdata)) { buf->st_ino = 0; buf->st_gid = 0; @@ -668,48 +677,15 @@ static int do_lstat(int follow, const char *file_name, struct stat *buf) return -1; } -/* We provide our own lstat/fstat functions, since the provided - * lstat/fstat functions are so slow. These stat functions are - * tailored for Git's usage (read: fast), and are not meant to be - * complete. Note that Git stat()s are redirected to mingw_lstat() - * too, since Windows doesn't really handle symlinks that well. - */ -static int do_stat_internal(int follow, const char *file_name, struct stat *buf) -{ - int namelen; - char alt_name[MAX_LONG_PATH]; - - if (!do_lstat(follow, file_name, buf)) - return 0; - - /* if file_name ended in a '/', Windows returned ENOENT; - * try again without trailing slashes - */ - if (errno != ENOENT) - return -1; - - namelen = strlen(file_name); - if (namelen && file_name[namelen-1] != '/') - return -1; - while (namelen && file_name[namelen-1] == '/') - --namelen; - if (!namelen || namelen >= MAX_LONG_PATH) - return -1; - - memcpy(alt_name, file_name, namelen); - alt_name[namelen] = 0; - return do_lstat(follow, alt_name, buf); -} - int (*lstat)(const char *file_name, struct stat *buf) = mingw_lstat; int mingw_lstat(const char *file_name, struct stat *buf) { - return do_stat_internal(0, file_name, buf); + return do_lstat(0, file_name, buf); } int mingw_stat(const char *file_name, struct stat *buf) { - return do_stat_internal(1, file_name, buf); + return do_lstat(1, file_name, buf); } int mingw_fstat(int fd, struct stat *buf) From 1564fb9028ddb8e2db861f8c41d95d16eeb69289 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sat, 16 May 2015 01:18:14 +0200 Subject: [PATCH 087/288] Win32: implement stat() with symlink support With respect to symlinks, the current stat() implementation is almost the same as lstat(): except for the file type (st_mode & S_IFMT), it returns information about the link rather than the target. Implement stat by opening the file with as little permissions as possible and calling GetFileInformationByHandle on it. This way, all link resoltion is handled by the Windows file system layer. If symlinks are disabled, use lstat() as before, but fail with ELOOP if a symlink would have to be resolved. Signed-off-by: Karsten Blees --- compat/mingw.c | 64 +++++++++++++++++++++++++++++++++++++------------- 1 file changed, 48 insertions(+), 16 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index bdd593248d..c2be165ec7 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -683,16 +683,60 @@ int mingw_lstat(const char *file_name, struct stat *buf) { return do_lstat(0, file_name, buf); } + +static int get_file_info_by_handle(HANDLE hnd, struct stat *buf) +{ + BY_HANDLE_FILE_INFORMATION fdata; + if (!GetFileInformationByHandle(hnd, &fdata)) { + errno = err_win_to_posix(GetLastError()); + return -1; + } + buf->st_ino = 0; + buf->st_dev = buf->st_rdev = 0; /* not used by Git */ + buf->st_gid = buf->st_uid = 0; + buf->st_nlink = 1; + buf->st_mode = file_attr_to_st_mode(fdata.dwFileAttributes); + buf->st_size = fdata.nFileSizeLow | (((off_t) fdata.nFileSizeHigh) << 32); + buf->st_atime = filetime_to_time_t(&(fdata.ftLastAccessTime)); + buf->st_mtime = filetime_to_time_t(&(fdata.ftLastWriteTime)); + buf->st_ctime = filetime_to_time_t(&(fdata.ftCreationTime)); + return 0; +} + int mingw_stat(const char *file_name, struct stat *buf) { - return do_lstat(1, file_name, buf); + wchar_t wfile_name[MAX_LONG_PATH]; + HANDLE hnd; + int result; + + /* if symlinks are disabled, use lstat() (without following links) */ + if (!has_symlinks) { + result = lstat(file_name, buf); + if (!result && S_ISLNK(buf->st_mode)) { + errno = ELOOP; + return -1; + } + return result; + } + + /* otherwise just open the file and let Windows resolve the links */ + if (xutftowcs_long_path(wfile_name, file_name) < 0) + return -1; + hnd = CreateFileW(wfile_name, 0, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, NULL, + OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); + if (hnd == INVALID_HANDLE_VALUE) { + errno = err_win_to_posix(GetLastError()); + return -1; + } + result = get_file_info_by_handle(hnd, buf); + CloseHandle(hnd); + return result; } int mingw_fstat(int fd, struct stat *buf) { HANDLE fh = (HANDLE)_get_osfhandle(fd); - BY_HANDLE_FILE_INFORMATION fdata; - if (fh == INVALID_HANDLE_VALUE) { errno = EBADF; return -1; @@ -701,20 +745,8 @@ int mingw_fstat(int fd, struct stat *buf) if (GetFileType(fh) != FILE_TYPE_DISK) return _fstati64(fd, buf); - if (GetFileInformationByHandle(fh, &fdata)) { - buf->st_ino = 0; - buf->st_gid = 0; - buf->st_uid = 0; - buf->st_nlink = 1; - buf->st_mode = file_attr_to_st_mode(fdata.dwFileAttributes); - buf->st_size = fdata.nFileSizeLow | - (((off_t)fdata.nFileSizeHigh)<<32); - buf->st_dev = buf->st_rdev = 0; /* not used by Git */ - buf->st_atime = filetime_to_time_t(&(fdata.ftLastAccessTime)); - buf->st_mtime = filetime_to_time_t(&(fdata.ftLastWriteTime)); - buf->st_ctime = filetime_to_time_t(&(fdata.ftCreationTime)); + if (!get_file_info_by_handle(fh, buf)) return 0; - } errno = EBADF; return -1; } From 6643acf85cd908e8fbae26dfcd3c636394b5355d Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Tue, 12 May 2015 00:58:39 +0200 Subject: [PATCH 088/288] Win32: remove separate do_lstat() function With the new mingw_stat() implementation, do_lstat() is only called from mingw_lstat() (with follow == 0). Remove the extra function and the old mingw_stat()-specific (follow == 1) logic. Signed-off-by: Karsten Blees --- compat/mingw.c | 22 ++-------------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index c2be165ec7..11236ca505 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -595,14 +595,7 @@ static int has_valid_directory_prefix(wchar_t *wfilename) return 1; } -/* We keep the do_lstat code in a separate function to avoid recursion. - * When a path ends with a slash, the stat will fail with ENOENT. In - * this case, we strip the trailing slashes and stat again. - * - * If follow is true then act like stat() and report on the link - * target. Otherwise report on the link itself. - */ -static int do_lstat(int follow, const char *file_name, struct stat *buf) +int mingw_lstat(const char *file_name, struct stat *buf) { WIN32_FILE_ATTRIBUTE_DATA fdata; wchar_t wfilename[MAX_LONG_PATH]; @@ -636,13 +629,7 @@ static int do_lstat(int follow, const char *file_name, struct stat *buf) if (handle != INVALID_HANDLE_VALUE) { if ((findbuf.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) && (findbuf.dwReserved0 == IO_REPARSE_TAG_SYMLINK)) { - if (follow) { - char buffer[MAXIMUM_REPARSE_DATA_BUFFER_SIZE]; - buf->st_size = readlink(file_name, buffer, MAXIMUM_REPARSE_DATA_BUFFER_SIZE); - } else { - buf->st_mode = S_IFLNK; - } - buf->st_mode |= S_IREAD; + buf->st_mode = S_IFLNK | S_IREAD; if (!(findbuf.dwFileAttributes & FILE_ATTRIBUTE_READONLY)) buf->st_mode |= S_IWRITE; } @@ -679,11 +666,6 @@ static int do_lstat(int follow, const char *file_name, struct stat *buf) int (*lstat)(const char *file_name, struct stat *buf) = mingw_lstat; -int mingw_lstat(const char *file_name, struct stat *buf) -{ - return do_lstat(0, file_name, buf); -} - static int get_file_info_by_handle(HANDLE hnd, struct stat *buf) { BY_HANDLE_FILE_INFORMATION fdata; From dd6ba5d40134ad3f17e52e929fb251815bbe1ac2 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sun, 24 May 2015 00:17:56 +0200 Subject: [PATCH 089/288] Win32: let mingw_lstat() error early upon problems with reparse points When obtaining lstat information for reparse points, we need to call FindFirstFile() in addition to GetFileInformationEx() to obtain the type of the reparse point (symlink, mount point etc.). However, currently there is no error handling whatsoever if FindFirstFile() fails. Call FindFirstFile() before modifying the stat *buf output parameter and error out if the call fails. Note: The FindFirstFile() return value includes all the data that we get from GetFileAttributesEx(), so we could replace GetFileAttributesEx() with FindFirstFile(). We don't do that because GetFileAttributesEx() is about twice as fast for single files. I.e. we only pay the extra cost of calling FindFirstFile() in the rare case that we encounter a reparse point. Note: The indentation of the remaining reparse point code will be fixed in the next patch. Signed-off-by: Karsten Blees --- compat/mingw.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 11236ca505..7c38d7ed06 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -598,6 +598,7 @@ static int has_valid_directory_prefix(wchar_t *wfilename) int mingw_lstat(const char *file_name, struct stat *buf) { WIN32_FILE_ATTRIBUTE_DATA fdata; + WIN32_FIND_DATAW findbuf; wchar_t wfilename[MAX_LONG_PATH]; int wlen = xutftowcs_long_path(wfilename, file_name); if (wlen < 0) @@ -612,6 +613,13 @@ int mingw_lstat(const char *file_name, struct stat *buf) } if (GetFileAttributesExW(wfilename, GetFileExInfoStandard, &fdata)) { + /* for reparse points, use FindFirstFile to get the reparse tag */ + if (fdata.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) { + HANDLE handle = FindFirstFileW(wfilename, &findbuf); + if (handle == INVALID_HANDLE_VALUE) + goto error; + FindClose(handle); + } buf->st_ino = 0; buf->st_gid = 0; buf->st_uid = 0; @@ -624,20 +632,16 @@ int mingw_lstat(const char *file_name, struct stat *buf) buf->st_mtime = filetime_to_time_t(&(fdata.ftLastWriteTime)); buf->st_ctime = filetime_to_time_t(&(fdata.ftCreationTime)); if (fdata.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) { - WIN32_FIND_DATAW findbuf; - HANDLE handle = FindFirstFileW(wfilename, &findbuf); - if (handle != INVALID_HANDLE_VALUE) { if ((findbuf.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) && (findbuf.dwReserved0 == IO_REPARSE_TAG_SYMLINK)) { buf->st_mode = S_IFLNK | S_IREAD; if (!(findbuf.dwFileAttributes & FILE_ATTRIBUTE_READONLY)) buf->st_mode |= S_IWRITE; } - FindClose(handle); - } } return 0; } +error: switch (GetLastError()) { case ERROR_ACCESS_DENIED: case ERROR_SHARING_VIOLATION: From 91c1f39f1172c8630ef166d344b1a50bbe05c905 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 10 Jan 2017 23:21:56 +0100 Subject: [PATCH 090/288] Win32: teach fscache and dirent about symlinks Move S_IFLNK detection to file_attr_to_st_mode() and reuse it in fscache. Implement DT_LNK detection in dirent.c and the fscache readdir version. Signed-off-by: Karsten Blees --- compat/mingw.c | 13 +++---------- compat/win32.h | 6 ++++-- compat/win32/dirent.c | 5 ++++- compat/win32/fscache.c | 6 ++++-- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 7c38d7ed06..26feb255a9 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -624,21 +624,14 @@ int mingw_lstat(const char *file_name, struct stat *buf) buf->st_gid = 0; buf->st_uid = 0; buf->st_nlink = 1; - buf->st_mode = file_attr_to_st_mode(fdata.dwFileAttributes); + buf->st_mode = file_attr_to_st_mode(fdata.dwFileAttributes, + findbuf.dwReserved0); buf->st_size = fdata.nFileSizeLow | (((off_t)fdata.nFileSizeHigh)<<32); buf->st_dev = buf->st_rdev = 0; /* not used by Git */ buf->st_atime = filetime_to_time_t(&(fdata.ftLastAccessTime)); buf->st_mtime = filetime_to_time_t(&(fdata.ftLastWriteTime)); buf->st_ctime = filetime_to_time_t(&(fdata.ftCreationTime)); - if (fdata.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) { - if ((findbuf.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) && - (findbuf.dwReserved0 == IO_REPARSE_TAG_SYMLINK)) { - buf->st_mode = S_IFLNK | S_IREAD; - if (!(findbuf.dwFileAttributes & FILE_ATTRIBUTE_READONLY)) - buf->st_mode |= S_IWRITE; - } - } return 0; } error: @@ -681,7 +674,7 @@ static int get_file_info_by_handle(HANDLE hnd, struct stat *buf) buf->st_dev = buf->st_rdev = 0; /* not used by Git */ buf->st_gid = buf->st_uid = 0; buf->st_nlink = 1; - buf->st_mode = file_attr_to_st_mode(fdata.dwFileAttributes); + buf->st_mode = file_attr_to_st_mode(fdata.dwFileAttributes, 0); buf->st_size = fdata.nFileSizeLow | (((off_t) fdata.nFileSizeHigh) << 32); buf->st_atime = filetime_to_time_t(&(fdata.ftLastAccessTime)); buf->st_mtime = filetime_to_time_t(&(fdata.ftLastWriteTime)); diff --git a/compat/win32.h b/compat/win32.h index 4a0ccb8075..add794bdc7 100644 --- a/compat/win32.h +++ b/compat/win32.h @@ -8,10 +8,12 @@ #include "compat/win32/lazyload.h" -static inline int file_attr_to_st_mode (DWORD attr) +static inline int file_attr_to_st_mode (DWORD attr, DWORD tag) { int fMode = S_IREAD; - if (attr & FILE_ATTRIBUTE_DIRECTORY) + if ((attr & FILE_ATTRIBUTE_REPARSE_POINT) && tag == IO_REPARSE_TAG_SYMLINK) + fMode |= S_IFLNK; + else if (attr & FILE_ATTRIBUTE_DIRECTORY) fMode |= S_IFDIR; else fMode |= S_IFREG; diff --git a/compat/win32/dirent.c b/compat/win32/dirent.c index b3bd8d7af7..8c654d722b 100644 --- a/compat/win32/dirent.c +++ b/compat/win32/dirent.c @@ -16,7 +16,10 @@ static inline void finddata2dirent(struct dirent *ent, WIN32_FIND_DATAW *fdata) xwcstoutf(ent->d_name, fdata->cFileName, MAX_PATH * 3); /* Set file type, based on WIN32_FIND_DATA */ - if (fdata->dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) + if ((fdata->dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) + && fdata->dwReserved0 == IO_REPARSE_TAG_SYMLINK) + ent->d_type = DT_LNK; + else if (fdata->dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) ent->d_type = DT_DIR; else ent->d_type = DT_REG; diff --git a/compat/win32/fscache.c b/compat/win32/fscache.c index 63d522f49a..cda7b835aa 100644 --- a/compat/win32/fscache.c +++ b/compat/win32/fscache.c @@ -147,7 +147,8 @@ static struct fsentry *fseentry_create_entry(struct fsentry *list, fse = fsentry_alloc(list, buf, len); - fse->st_mode = file_attr_to_st_mode(fdata->dwFileAttributes); + fse->st_mode = file_attr_to_st_mode(fdata->dwFileAttributes, + fdata->dwReserved0); fse->st_size = (((off64_t) (fdata->nFileSizeHigh)) << 32) | fdata->nFileSizeLow; fse->st_atime = filetime_to_time_t(&(fdata->ftLastAccessTime)); @@ -456,7 +457,8 @@ static struct dirent *fscache_readdir(DIR *base_dir) if (!next) return NULL; dir->pfsentry = next; - dir->dirent.d_type = S_ISDIR(next->st_mode) ? DT_DIR : DT_REG; + dir->dirent.d_type = S_ISREG(next->st_mode) ? DT_REG : + S_ISDIR(next->st_mode) ? DT_DIR : DT_LNK; dir->dirent.d_name = (char*) next->name; return &(dir->dirent); } From d2e4ec9e39e8c3b574045e3814f91c4f3316964d Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sat, 16 May 2015 01:11:37 +0200 Subject: [PATCH 091/288] Win32: lstat(): return adequate stat.st_size for symlinks Git typically doesn't trust the stat.st_size member of symlinks (e.g. see strbuf_readlink()). However, some functions take shortcuts if st_size is 0 (e.g. diff_populate_filespec()). In mingw_lstat() and fscache_lstat(), make sure to return an adequate size. The extra overhead of opening and reading the reparse point to calculate the exact size is not necessary, as git doesn't rely on the value anyway. Signed-off-by: Karsten Blees --- compat/mingw.c | 4 ++-- compat/win32/fscache.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 26feb255a9..47b4bbbbc1 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -626,8 +626,8 @@ int mingw_lstat(const char *file_name, struct stat *buf) buf->st_nlink = 1; buf->st_mode = file_attr_to_st_mode(fdata.dwFileAttributes, findbuf.dwReserved0); - buf->st_size = fdata.nFileSizeLow | - (((off_t)fdata.nFileSizeHigh)<<32); + buf->st_size = S_ISLNK(buf->st_mode) ? MAX_LONG_PATH : + fdata.nFileSizeLow | (((off_t) fdata.nFileSizeHigh) << 32); buf->st_dev = buf->st_rdev = 0; /* not used by Git */ buf->st_atime = filetime_to_time_t(&(fdata.ftLastAccessTime)); buf->st_mtime = filetime_to_time_t(&(fdata.ftLastWriteTime)); diff --git a/compat/win32/fscache.c b/compat/win32/fscache.c index cda7b835aa..d23d3a4a27 100644 --- a/compat/win32/fscache.c +++ b/compat/win32/fscache.c @@ -149,8 +149,8 @@ static struct fsentry *fseentry_create_entry(struct fsentry *list, fse->st_mode = file_attr_to_st_mode(fdata->dwFileAttributes, fdata->dwReserved0); - fse->st_size = (((off64_t) (fdata->nFileSizeHigh)) << 32) - | fdata->nFileSizeLow; + fse->st_size = S_ISLNK(fse->st_mode) ? MAX_LONG_PATH : + fdata->nFileSizeLow | (((off_t) fdata->nFileSizeHigh) << 32); fse->st_atime = filetime_to_time_t(&(fdata->ftLastAccessTime)); fse->st_mtime = filetime_to_time_t(&(fdata->ftLastWriteTime)); fse->st_ctime = filetime_to_time_t(&(fdata->ftCreationTime)); From 903b51f4003c0a8e7d034be02d51ab29c7f891de Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Tue, 19 May 2015 21:48:55 +0200 Subject: [PATCH 092/288] Win32: factor out retry logic The retry pattern is duplicated in three places. It also seems to be too hard to use: mingw_unlink() and mingw_rmdir() duplicate the code to retry, and both of them do so incompletely. They also do not restore errno if the user answers 'no'. Introduce a retry_ask_yes_no() helper function that handles retry with small delay, asking the user, and restoring errno. mingw_unlink: include _wchmod in the retry loop (which may fail if the file is locked exclusively). mingw_rmdir: include special error handling in the retry loop. Signed-off-by: Karsten Blees --- compat/mingw.c | 98 ++++++++++++++++++++++---------------------------- 1 file changed, 43 insertions(+), 55 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 47b4bbbbc1..4f8d5e906c 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -8,8 +8,6 @@ #define HCAST(type, handle) ((type)(intptr_t)handle) -static const int delay[] = { 0, 1, 10, 20, 40 }; - int err_win_to_posix(DWORD winerr) { int error = ENOSYS; @@ -172,15 +170,12 @@ static int read_yes_no_answer(void) return -1; } -static int ask_yes_no_if_possible(const char *format, ...) +static int ask_yes_no_if_possible(const char *format, va_list args) { char question[4096]; const char *retry_hook[] = { NULL, NULL, NULL }; - va_list args; - va_start(args, format); vsnprintf(question, sizeof(question), format, args); - va_end(args); if ((retry_hook[0] = mingw_getenv("GIT_ASK_YESNO"))) { retry_hook[1] = question; @@ -202,6 +197,31 @@ static int ask_yes_no_if_possible(const char *format, ...) } } +static int retry_ask_yes_no(int *tries, const char *format, ...) +{ + static const int delay[] = { 0, 1, 10, 20, 40 }; + va_list args; + int result, saved_errno = errno; + + if ((*tries) < ARRAY_SIZE(delay)) { + /* + * We assume that some other process had the file open at the wrong + * moment and retry. In order to give the other process a higher + * chance to complete its operation, we give up our time slice now. + * If we have to retry again, we do sleep a bit. + */ + Sleep(delay[*tries]); + (*tries)++; + return 1; + } + + va_start(args, format); + result = ask_yes_no_if_possible(format, args); + va_end(args); + errno = saved_errno; + return result; +} + /* Windows only */ enum hide_dotfiles_type { HIDE_DOTFILES_FALSE = 0, @@ -245,31 +265,21 @@ int mingw_core_config(const char *var, const char *value) int mingw_unlink(const char *pathname) { - int ret, tries = 0; + int tries = 0; wchar_t wpathname[MAX_LONG_PATH]; if (xutftowcs_long_path(wpathname, pathname) < 0) return -1; - /* read-only files cannot be removed */ - _wchmod(wpathname, 0666); - while ((ret = _wunlink(wpathname)) == -1 && tries < ARRAY_SIZE(delay)) { + do { + /* read-only files cannot be removed */ + _wchmod(wpathname, 0666); + if (!_wunlink(wpathname)) + return 0; if (!is_file_in_use_error(GetLastError())) break; - /* - * We assume that some other process had the source or - * destination file open at the wrong moment and retry. - * In order to give the other process a higher chance to - * complete its operation, we give up our time slice now. - * If we have to retry again, we do sleep a bit. - */ - Sleep(delay[tries]); - tries++; - } - while (ret == -1 && is_file_in_use_error(GetLastError()) && - ask_yes_no_if_possible("Unlink of file '%s' failed. " - "Should I try again?", pathname)) - ret = _wunlink(wpathname); - return ret; + } while (retry_ask_yes_no(&tries, "Unlink of file '%s' failed. " + "Should I try again?", pathname)); + return -1; } static int is_dir_empty(const wchar_t *wpath) @@ -296,12 +306,14 @@ static int is_dir_empty(const wchar_t *wpath) int mingw_rmdir(const char *pathname) { - int ret, tries = 0; + int tries = 0; wchar_t wpathname[MAX_LONG_PATH]; if (xutftowcs_long_path(wpathname, pathname) < 0) return -1; - while ((ret = _wrmdir(wpathname)) == -1 && tries < ARRAY_SIZE(delay)) { + do { + if (!_wrmdir(wpathname)) + return 0; if (!is_file_in_use_error(GetLastError())) errno = err_win_to_posix(GetLastError()); if (errno != EACCES) @@ -310,21 +322,9 @@ int mingw_rmdir(const char *pathname) errno = ENOTEMPTY; break; } - /* - * We assume that some other process had the source or - * destination file open at the wrong moment and retry. - * In order to give the other process a higher chance to - * complete its operation, we give up our time slice now. - * If we have to retry again, we do sleep a bit. - */ - Sleep(delay[tries]); - tries++; - } - while (ret == -1 && errno == EACCES && is_file_in_use_error(GetLastError()) && - ask_yes_no_if_possible("Deletion of directory '%s' failed. " - "Should I try again?", pathname)) - ret = _wrmdir(wpathname); - return ret; + } while (retry_ask_yes_no(&tries, "Deletion of directory '%s' failed. " + "Should I try again?", pathname)); + return -1; } static inline int needs_hiding(const char *path) @@ -1842,20 +1842,8 @@ repeat: SetFileAttributesW(wpnew, attrs); } } - if (tries < ARRAY_SIZE(delay) && gle == ERROR_ACCESS_DENIED) { - /* - * We assume that some other process had the source or - * destination file open at the wrong moment and retry. - * In order to give the other process a higher chance to - * complete its operation, we give up our time slice now. - * If we have to retry again, we do sleep a bit. - */ - Sleep(delay[tries]); - tries++; - goto repeat; - } if (gle == ERROR_ACCESS_DENIED && - ask_yes_no_if_possible("Rename from '%s' to '%s' failed. " + retry_ask_yes_no(&tries, "Rename from '%s' to '%s' failed. " "Should I try again?", pold, pnew)) goto repeat; From 9d5e21e2932f6a7fbe9a5618bd5c6f8048c9e8a8 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sun, 24 May 2015 01:55:05 +0200 Subject: [PATCH 093/288] Win32: change default of 'core.symlinks' to false Symlinks on Windows don't work the same way as on Unix systems. E.g. there are different types of symlinks for directories and files, creating symlinks requires administrative privileges etc. By default, disable symlink support on Windows. I.e. users explicitly have to enable it with 'git config [--system|--global] core.symlinks true'. The test suite ignores system / global config files. Allow testing *with* symlink support by checking if native symlinks are enabled in MSys2 (via 'MSYS=winsymlinks:nativestrict'). Reminder: This would need to be changed if / when we find a way to run the test suite in a non-MSys-based shell (e.g. dash). Signed-off-by: Karsten Blees --- compat/mingw.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/compat/mingw.c b/compat/mingw.c index 4f8d5e906c..7bfab44935 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2310,6 +2310,15 @@ static void setup_windows_environment(void) if (!tmp && (tmp = getenv("USERPROFILE"))) setenv("HOME", tmp, 1); } + + /* + * Change 'core.symlinks' default to false, unless native symlinks are + * enabled in MSys2 (via 'MSYS=winsymlinks:nativestrict'). Thus we can + * run the test suite (which doesn't obey config files) with or without + * symlink support. + */ + if (!(tmp = getenv("MSYS")) || !strstr(tmp, "winsymlinks:nativestrict")) + has_symlinks = 0; } int handle_long_path(wchar_t *path, int len, int max_path, int expand) From 6d9dcf91ac1f193696bc0e5fe41bbcc857a24f8a Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sat, 16 May 2015 00:32:03 +0200 Subject: [PATCH 094/288] Win32: add symlink-specific error codes Signed-off-by: Karsten Blees --- compat/mingw.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/compat/mingw.c b/compat/mingw.c index 7bfab44935..c6508b8f13 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -70,6 +70,7 @@ int err_win_to_posix(DWORD winerr) case ERROR_INVALID_PARAMETER: error = EINVAL; break; case ERROR_INVALID_PASSWORD: error = EPERM; break; case ERROR_INVALID_PRIMARY_GROUP: error = EINVAL; break; + case ERROR_INVALID_REPARSE_DATA: error = EINVAL; break; case ERROR_INVALID_SIGNAL_NUMBER: error = EINVAL; break; case ERROR_INVALID_TARGET_HANDLE: error = EIO; break; case ERROR_INVALID_WORKSTATION: error = EACCES; break; @@ -84,6 +85,7 @@ int err_win_to_posix(DWORD winerr) case ERROR_NEGATIVE_SEEK: error = ESPIPE; break; case ERROR_NOACCESS: error = EFAULT; break; case ERROR_NONE_MAPPED: error = EINVAL; break; + case ERROR_NOT_A_REPARSE_POINT: error = EINVAL; break; case ERROR_NOT_ENOUGH_MEMORY: error = ENOMEM; break; case ERROR_NOT_READY: error = EAGAIN; break; case ERROR_NOT_SAME_DEVICE: error = EXDEV; break; @@ -104,6 +106,9 @@ int err_win_to_posix(DWORD winerr) case ERROR_PIPE_NOT_CONNECTED: error = EPIPE; break; case ERROR_PRIVILEGE_NOT_HELD: error = EACCES; break; case ERROR_READ_FAULT: error = EIO; break; + case ERROR_REPARSE_ATTRIBUTE_CONFLICT: error = EINVAL; break; + case ERROR_REPARSE_TAG_INVALID: error = EINVAL; break; + case ERROR_REPARSE_TAG_MISMATCH: error = EINVAL; break; case ERROR_SEEK: error = EIO; break; case ERROR_SEEK_ON_DEVICE: error = ESPIPE; break; case ERROR_SHARING_BUFFER_EXCEEDED: error = ENFILE; break; From dfaf0875a264d1238e920d5a3a2a69d275aebbeb Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sun, 24 May 2015 01:06:10 +0200 Subject: [PATCH 095/288] Win32: mingw_unlink: support symlinks to directories _wunlink() / DeleteFileW() refuses to delete symlinks to directories. If _wunlink() fails with ERROR_ACCESS_DENIED, try _wrmdir() as well. Signed-off-by: Karsten Blees --- compat/mingw.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/compat/mingw.c b/compat/mingw.c index c6508b8f13..3c179f211c 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -282,6 +282,13 @@ int mingw_unlink(const char *pathname) return 0; if (!is_file_in_use_error(GetLastError())) break; + /* + * _wunlink() / DeleteFileW() for directory symlinks fails with + * ERROR_ACCESS_DENIED (EACCES), so try _wrmdir() as well. This is the + * same error we get if a file is in use (already checked above). + */ + if (!_wrmdir(wpathname)) + return 0; } while (retry_ask_yes_no(&tries, "Unlink of file '%s' failed. " "Should I try again?", pathname)); return -1; From d9f31d1182aa62920c4645cb3f06100bdfb2cc5f Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Tue, 19 May 2015 22:42:48 +0200 Subject: [PATCH 096/288] Win32: mingw_rename: support renaming symlinks MSVCRT's _wrename() cannot rename symlinks over existing files: it returns success without doing anything. Newer MSVCR*.dll versions probably do not have this problem: according to CRT sources, they just call MoveFileEx() with the MOVEFILE_COPY_ALLOWED flag. Get rid of _wrename() and call MoveFileEx() with proper error handling. Signed-off-by: Karsten Blees --- compat/mingw.c | 39 +++++++++++++++++---------------------- 1 file changed, 17 insertions(+), 22 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 3c179f211c..e6453e541e 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -1814,28 +1814,29 @@ int mingw_accept(int sockfd1, struct sockaddr *sa, socklen_t *sz) #undef rename int mingw_rename(const char *pold, const char *pnew) { - DWORD attrs, gle; + DWORD attrs = INVALID_FILE_ATTRIBUTES, gle; int tries = 0; wchar_t wpold[MAX_LONG_PATH], wpnew[MAX_LONG_PATH]; if (xutftowcs_long_path(wpold, pold) < 0 || xutftowcs_long_path(wpnew, pnew) < 0) return -1; - /* - * Try native rename() first to get errno right. - * It is based on MoveFile(), which cannot overwrite existing files. - */ - if (!_wrename(wpold, wpnew)) - return 0; - if (errno != EEXIST) - return -1; repeat: - if (MoveFileExW(wpold, wpnew, MOVEFILE_REPLACE_EXISTING)) + if (MoveFileExW(wpold, wpnew, + MOVEFILE_REPLACE_EXISTING | MOVEFILE_COPY_ALLOWED)) return 0; - /* TODO: translate more errors */ gle = GetLastError(); - if (gle == ERROR_ACCESS_DENIED && - (attrs = GetFileAttributesW(wpnew)) != INVALID_FILE_ATTRIBUTES) { + + /* revert file attributes on failure */ + if (attrs != INVALID_FILE_ATTRIBUTES) + SetFileAttributesW(wpnew, attrs); + + if (!is_file_in_use_error(gle)) { + errno = err_win_to_posix(gle); + return -1; + } + + if ((attrs = GetFileAttributesW(wpnew)) != INVALID_FILE_ATTRIBUTES) { if (attrs & FILE_ATTRIBUTE_DIRECTORY) { DWORD attrsold = GetFileAttributesW(wpold); if (attrsold == INVALID_FILE_ATTRIBUTES || @@ -1846,16 +1847,10 @@ repeat: return -1; } if ((attrs & FILE_ATTRIBUTE_READONLY) && - SetFileAttributesW(wpnew, attrs & ~FILE_ATTRIBUTE_READONLY)) { - if (MoveFileExW(wpold, wpnew, MOVEFILE_REPLACE_EXISTING)) - return 0; - gle = GetLastError(); - /* revert file attributes on failure */ - SetFileAttributesW(wpnew, attrs); - } + SetFileAttributesW(wpnew, attrs & ~FILE_ATTRIBUTE_READONLY)) + goto repeat; } - if (gle == ERROR_ACCESS_DENIED && - retry_ask_yes_no(&tries, "Rename from '%s' to '%s' failed. " + if (retry_ask_yes_no(&tries, "Rename from '%s' to '%s' failed. " "Should I try again?", pold, pnew)) goto repeat; From 83196db12fd74e8c0deb1a32e8cf58fc54c5028a Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sun, 24 May 2015 01:17:31 +0200 Subject: [PATCH 097/288] Win32: mingw_chdir: change to symlink-resolved directory If symlinks are enabled, resolve all symlinks when changing directories, as required by POSIX. Note: Git's real_path() function bases its link resolution algorithm on this property of chdir(). Unfortunately, the current directory on Windows is limited to only MAX_PATH (260) characters. Therefore using symlinks and long paths in combination may be problematic. Note: GetFinalPathNameByHandleW() was introduced with symlink support in Windows Vista. Thus, for compatibility with Windows XP, we need to load it dynamically and behave gracefully if it isnt's available. Signed-off-by: Karsten Blees --- compat/mingw.c | 49 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/compat/mingw.c b/compat/mingw.c index e6453e541e..9cae0f0911 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -268,6 +268,34 @@ int mingw_core_config(const char *var, const char *value) return 0; } +/* Normalizes NT paths as returned by some low-level APIs. */ +static wchar_t *normalize_ntpath(wchar_t *wbuf) +{ + int i; + /* fix absolute path prefixes */ + if (wbuf[0] == '\\') { + /* strip NT namespace prefixes */ + if (!wcsncmp(wbuf, L"\\??\\", 4) || + !wcsncmp(wbuf, L"\\\\?\\", 4)) + wbuf += 4; + else if (!wcsnicmp(wbuf, L"\\DosDevices\\", 12)) + wbuf += 12; + /* replace remaining '...UNC\' with '\\' */ + if (!wcsnicmp(wbuf, L"UNC\\", 4)) { + wbuf += 2; + *wbuf = '\\'; + } + } + /* convert backslashes to slashes */ + for (i = 0; wbuf[i]; i++) + if (wbuf[i] == '\\') + wbuf[i] = '/'; + /* remove potential trailing slashes */ + while (i && wbuf[i - 1] == '/') + wbuf[--i] = 0; + return wbuf; +} + int mingw_unlink(const char *pathname) { int tries = 0; @@ -558,10 +586,29 @@ static int current_directory_len = 0; int mingw_chdir(const char *dirname) { int result; + DECLARE_PROC_ADDR(kernel32.dll, DWORD, GetFinalPathNameByHandleW, + HANDLE, LPWSTR, DWORD, DWORD); wchar_t wdirname[MAX_LONG_PATH]; if (xutftowcs_long_path(wdirname, dirname) < 0) return -1; - result = _wchdir(wdirname); + + if (has_symlinks && INIT_PROC_ADDR(GetFinalPathNameByHandleW)) { + HANDLE hnd = CreateFileW(wdirname, 0, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, NULL, + OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); + if (hnd == INVALID_HANDLE_VALUE) { + errno = err_win_to_posix(GetLastError()); + return -1; + } + if (!GetFinalPathNameByHandleW(hnd, wdirname, ARRAY_SIZE(wdirname), 0)) { + errno = err_win_to_posix(GetLastError()); + CloseHandle(hnd); + return -1; + } + CloseHandle(hnd); + } + + result = _wchdir(normalize_ntpath(wdirname)); current_directory_len = GetCurrentDirectoryW(0, NULL); return result; } From cabf433238c3459e6478e34afec19f977099fee3 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sun, 24 May 2015 01:24:41 +0200 Subject: [PATCH 098/288] Win32: implement readlink() Implement readlink() by reading NTFS reparse points. Works for symlinks and directory junctions. If symlinks are disabled, fail with ENOSYS. Signed-off-by: Karsten Blees --- compat/mingw.c | 101 +++++++++++++++++++++++++++++++++++++++++++++++++ compat/mingw.h | 3 +- 2 files changed, 102 insertions(+), 2 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 9cae0f0911..88a6054444 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2,6 +2,7 @@ #include "win32.h" #include #include +#include #include "../strbuf.h" #include "../run-command.h" #include "../cache.h" @@ -2149,6 +2150,106 @@ int link(const char *oldpath, const char *newpath) return 0; } +#ifndef _WINNT_H +/* + * The REPARSE_DATA_BUFFER structure is defined in the Windows DDK (in + * ntifs.h) and in MSYS1's winnt.h (which defines _WINNT_H). So define + * it ourselves if we are on MSYS2 (whose winnt.h defines _WINNT_). + */ +typedef struct _REPARSE_DATA_BUFFER { + DWORD ReparseTag; + WORD ReparseDataLength; + WORD Reserved; + _ANONYMOUS_UNION union { + struct { + WORD SubstituteNameOffset; + WORD SubstituteNameLength; + WORD PrintNameOffset; + WORD PrintNameLength; + ULONG Flags; + WCHAR PathBuffer[1]; + } SymbolicLinkReparseBuffer; + struct { + WORD SubstituteNameOffset; + WORD SubstituteNameLength; + WORD PrintNameOffset; + WORD PrintNameLength; + WCHAR PathBuffer[1]; + } MountPointReparseBuffer; + struct { + BYTE DataBuffer[1]; + } GenericReparseBuffer; + } DUMMYUNIONNAME; +} REPARSE_DATA_BUFFER, *PREPARSE_DATA_BUFFER; +#endif + +int readlink(const char *path, char *buf, size_t bufsiz) +{ + HANDLE handle; + WCHAR wpath[MAX_LONG_PATH], *wbuf; + REPARSE_DATA_BUFFER *b = alloca(MAXIMUM_REPARSE_DATA_BUFFER_SIZE); + DWORD dummy; + char tmpbuf[MAX_LONG_PATH]; + int len; + + /* fail if symlinks are disabled */ + if (!has_symlinks) { + errno = ENOSYS; + return -1; + } + + if (xutftowcs_long_path(wpath, path) < 0) + return -1; + + /* read reparse point data */ + handle = CreateFileW(wpath, 0, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, NULL, + OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT, NULL); + if (handle == INVALID_HANDLE_VALUE) { + errno = err_win_to_posix(GetLastError()); + return -1; + } + if (!DeviceIoControl(handle, FSCTL_GET_REPARSE_POINT, NULL, 0, b, + MAXIMUM_REPARSE_DATA_BUFFER_SIZE, &dummy, NULL)) { + errno = err_win_to_posix(GetLastError()); + CloseHandle(handle); + return -1; + } + CloseHandle(handle); + + /* get target path for symlinks or mount points (aka 'junctions') */ + switch (b->ReparseTag) { + case IO_REPARSE_TAG_SYMLINK: + wbuf = (WCHAR*) (((char*) b->SymbolicLinkReparseBuffer.PathBuffer) + + b->SymbolicLinkReparseBuffer.SubstituteNameOffset); + *(WCHAR*) (((char*) wbuf) + + b->SymbolicLinkReparseBuffer.SubstituteNameLength) = 0; + break; + case IO_REPARSE_TAG_MOUNT_POINT: + wbuf = (WCHAR*) (((char*) b->MountPointReparseBuffer.PathBuffer) + + b->MountPointReparseBuffer.SubstituteNameOffset); + *(WCHAR*) (((char*) wbuf) + + b->MountPointReparseBuffer.SubstituteNameLength) = 0; + break; + default: + errno = EINVAL; + return -1; + } + + /* + * Adapt to strange readlink() API: Copy up to bufsiz *bytes*, potentially + * cutting off a UTF-8 sequence. Insufficient bufsize is *not* a failure + * condition. There is no conversion function that produces invalid UTF-8, + * so convert to a (hopefully large enough) temporary buffer, then memcpy + * the requested number of bytes (including '\0' for robustness). + */ + if ((len = xwcstoutf(tmpbuf, normalize_ntpath(wbuf), MAX_LONG_PATH)) < 0) + return -1; + memcpy(buf, tmpbuf, min(bufsiz, len + 1)); + return min(bufsiz, len); +} + pid_t waitpid(pid_t pid, int *status, int options) { HANDLE h = OpenProcess(SYNCHRONIZE | PROCESS_QUERY_INFORMATION, diff --git a/compat/mingw.h b/compat/mingw.h index a94df53584..7e1d1f6c04 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -123,8 +123,6 @@ struct utsname { * trivial stubs */ -static inline int readlink(const char *path, char *buf, size_t bufsiz) -{ errno = ENOSYS; return -1; } static inline int symlink(const char *oldpath, const char *newpath) { errno = ENOSYS; return -1; } static inline int fchmod(int fildes, mode_t mode) @@ -218,6 +216,7 @@ int setitimer(int type, struct itimerval *in, struct itimerval *out); int sigaction(int sig, struct sigaction *in, struct sigaction *out); int link(const char *oldpath, const char *newpath); int uname(struct utsname *buf); +int readlink(const char *path, char *buf, size_t bufsiz); /* * replacements of existing functions From dcb881fed7f4b3ca3d0570eb5bca621aa9879e58 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sun, 24 May 2015 01:32:03 +0200 Subject: [PATCH 099/288] Win32: implement basic symlink() functionality (file symlinks only) Implement symlink() that always creates file symlinks. Fails with ENOSYS if symlinks are disabled or unsupported. Note: CreateSymbolicLinkW() was introduced with symlink support in Windows Vista. For compatibility with Windows XP, we need to load it dynamically and fail gracefully if it isnt's available. Signed-off-by: Karsten Blees --- compat/mingw.c | 30 ++++++++++++++++++++++++++++++ compat/mingw.h | 3 +-- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 88a6054444..dff47f48a4 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -269,6 +269,8 @@ int mingw_core_config(const char *var, const char *value) return 0; } +DECLARE_PROC_ADDR(kernel32.dll, BOOL, CreateSymbolicLinkW, LPCWSTR, LPCWSTR, DWORD); + /* Normalizes NT paths as returned by some low-level APIs. */ static wchar_t *normalize_ntpath(wchar_t *wbuf) { @@ -2150,6 +2152,34 @@ int link(const char *oldpath, const char *newpath) return 0; } +int symlink(const char *target, const char *link) +{ + wchar_t wtarget[MAX_LONG_PATH], wlink[MAX_LONG_PATH]; + int len; + + /* fail if symlinks are disabled or API is not supported (WinXP) */ + if (!has_symlinks || !INIT_PROC_ADDR(CreateSymbolicLinkW)) { + errno = ENOSYS; + return -1; + } + + if ((len = xutftowcs_long_path(wtarget, target)) < 0 + || xutftowcs_long_path(wlink, link) < 0) + return -1; + + /* convert target dir separators to backslashes */ + while (len--) + if (wtarget[len] == '/') + wtarget[len] = '\\'; + + /* create file symlink */ + if (!CreateSymbolicLinkW(wlink, wtarget, 0)) { + errno = err_win_to_posix(GetLastError()); + return -1; + } + return 0; +} + #ifndef _WINNT_H /* * The REPARSE_DATA_BUFFER structure is defined in the Windows DDK (in diff --git a/compat/mingw.h b/compat/mingw.h index 7e1d1f6c04..e585263ee4 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -123,8 +123,6 @@ struct utsname { * trivial stubs */ -static inline int symlink(const char *oldpath, const char *newpath) -{ errno = ENOSYS; return -1; } static inline int fchmod(int fildes, mode_t mode) { errno = ENOSYS; return -1; } #ifndef __MINGW64_VERSION_MAJOR @@ -216,6 +214,7 @@ int setitimer(int type, struct itimerval *in, struct itimerval *out); int sigaction(int sig, struct sigaction *in, struct sigaction *out); int link(const char *oldpath, const char *newpath); int uname(struct utsname *buf); +int symlink(const char *target, const char *link); int readlink(const char *path, char *buf, size_t bufsiz); /* From 3e3d1610de689bd3baee2ea8af16bf926d83974a Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sun, 24 May 2015 01:48:35 +0200 Subject: [PATCH 100/288] Win32: symlink: add support for symlinks to directories Symlinks on Windows have a flag that indicates whether the target is a file or a directory. Symlinks of wrong type simply don't work. This even affects core Win32 APIs (e.g. DeleteFile() refuses to delete directory symlinks). However, CreateFile() with FILE_FLAG_BACKUP_SEMANTICS doesn't seem to care. Check the target type by first creating a tentative file symlink, opening it, and checking the type of the resulting handle. If it is a directory, recreate the symlink with the directory flag set. It is possible to create symlinks before the target exists (or in case of symlinks to symlinks: before the target type is known). If this happens, create a tentative file symlink and postpone the directory decision: keep a list of phantom symlinks to be processed whenever a new directory is created in mingw_mkdir(). Limitations: This algorithm may fail if a link target changes from file to directory or vice versa, or if the target directory is created in another process. Signed-off-by: Karsten Blees --- compat/mingw.c | 121 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 121 insertions(+) diff --git a/compat/mingw.c b/compat/mingw.c index dff47f48a4..277576b01c 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -271,6 +271,88 @@ int mingw_core_config(const char *var, const char *value) DECLARE_PROC_ADDR(kernel32.dll, BOOL, CreateSymbolicLinkW, LPCWSTR, LPCWSTR, DWORD); +enum phantom_symlink_result { + PHANTOM_SYMLINK_RETRY, + PHANTOM_SYMLINK_DONE, + PHANTOM_SYMLINK_DIRECTORY +}; + +/* + * Changes a file symlink to a directory symlink if the target exists and is a + * directory. + */ +static enum phantom_symlink_result process_phantom_symlink( + const wchar_t *wtarget, const wchar_t *wlink) { + HANDLE hnd; + BY_HANDLE_FILE_INFORMATION fdata; + + /* check that wlink is still a file symlink */ + if ((GetFileAttributesW(wlink) + & (FILE_ATTRIBUTE_REPARSE_POINT | FILE_ATTRIBUTE_DIRECTORY)) + != FILE_ATTRIBUTE_REPARSE_POINT) + return PHANTOM_SYMLINK_DONE; + + /* let Windows resolve the link by opening it */ + hnd = CreateFileW(wlink, 0, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, NULL, + OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); + if (hnd == INVALID_HANDLE_VALUE) { + errno = err_win_to_posix(GetLastError()); + return PHANTOM_SYMLINK_RETRY; + } + + if (!GetFileInformationByHandle(hnd, &fdata)) { + errno = err_win_to_posix(GetLastError()); + CloseHandle(hnd); + return PHANTOM_SYMLINK_RETRY; + } + CloseHandle(hnd); + + /* if target exists and is a file, we're done */ + if (!(fdata.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)) + return PHANTOM_SYMLINK_DONE; + + /* otherwise recreate the symlink with directory flag */ + if (DeleteFileW(wlink) && CreateSymbolicLinkW(wlink, wtarget, 1)) + return PHANTOM_SYMLINK_DIRECTORY; + + errno = err_win_to_posix(GetLastError()); + return PHANTOM_SYMLINK_RETRY; +} + +/* keep track of newly created symlinks to non-existing targets */ +struct phantom_symlink_info { + struct phantom_symlink_info *next; + wchar_t *wlink; + wchar_t *wtarget; +}; + +static struct phantom_symlink_info *phantom_symlinks = NULL; +static CRITICAL_SECTION phantom_symlinks_cs; + +static void process_phantom_symlinks(void) +{ + struct phantom_symlink_info *current, **psi; + EnterCriticalSection(&phantom_symlinks_cs); + /* process phantom symlinks list */ + psi = &phantom_symlinks; + while ((current = *psi)) { + enum phantom_symlink_result result = process_phantom_symlink( + current->wtarget, current->wlink); + if (result == PHANTOM_SYMLINK_RETRY) { + psi = ¤t->next; + } else { + /* symlink was processed, remove from list */ + *psi = current->next; + free(current); + /* if symlink was a directory, start over */ + if (result == PHANTOM_SYMLINK_DIRECTORY) + psi = &phantom_symlinks; + } + } + LeaveCriticalSection(&phantom_symlinks_cs); +} + /* Normalizes NT paths as returned by some low-level APIs. */ static wchar_t *normalize_ntpath(wchar_t *wbuf) { @@ -423,6 +505,8 @@ int mingw_mkdir(const char *path, int mode) return -1; ret = _wmkdir(wpath); + if (!ret) + process_phantom_symlinks(); if (!ret && needs_hiding(path)) return set_hidden_flag(wpath, 1); return ret; @@ -2177,6 +2261,42 @@ int symlink(const char *target, const char *link) errno = err_win_to_posix(GetLastError()); return -1; } + + /* convert to directory symlink if target exists */ + switch (process_phantom_symlink(wtarget, wlink)) { + case PHANTOM_SYMLINK_RETRY: { + /* if target doesn't exist, add to phantom symlinks list */ + wchar_t wfullpath[MAX_LONG_PATH]; + struct phantom_symlink_info *psi; + + /* convert to absolute path to be independent of cwd */ + len = GetFullPathNameW(wlink, MAX_LONG_PATH, wfullpath, NULL); + if (!len || len >= MAX_LONG_PATH) { + errno = err_win_to_posix(GetLastError()); + return -1; + } + + /* over-allocate and fill phantom_smlink_info structure */ + psi = xmalloc(sizeof(struct phantom_symlink_info) + + sizeof(wchar_t) * (len + wcslen(wtarget) + 2)); + psi->wlink = (wchar_t *)(psi + 1); + wcscpy(psi->wlink, wfullpath); + psi->wtarget = psi->wlink + len + 1; + wcscpy(psi->wtarget, wtarget); + + EnterCriticalSection(&phantom_symlinks_cs); + psi->next = phantom_symlinks; + phantom_symlinks = psi; + LeaveCriticalSection(&phantom_symlinks_cs); + break; + } + case PHANTOM_SYMLINK_DIRECTORY: + /* if we created a dir symlink, process other phantom symlinks */ + process_phantom_symlinks(); + break; + default: + break; + } return 0; } @@ -2663,6 +2783,7 @@ void mingw_startup(void) /* initialize critical section for waitpid pinfo_t list */ InitializeCriticalSection(&pinfo_cs); + InitializeCriticalSection(&phantom_symlinks_cs); /* set up default file mode and file modes for stdin/out/err */ _fmode = _O_BINARY; From 951cc588b03a4f510c2d5f0b1561873f59ca6675 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 19 Jun 2015 21:00:46 +0000 Subject: [PATCH 101/288] mingw: keep trailing slashes for _wchdir() and readlink() This is needed so that `_wchdir()` can be used with drive root directories, e.g. C:\ (`_wchdir("C:")` fails to switch the directory to the root directory). This fixes https://github.com/msysgit/git/issues/359 (in Git for Windows 2.x only, though). Likewise, `readlink()`'s semantics require a trailing slash for symbolic links pointing to directories. Otherwise all checked out symbolic links pointing to directories would be marked as modified even directly after a fresh clone. This fixes https://github.com/git-for-windows/git/issues/210 Signed-off-by: Johannes Schindelin --- compat/mingw.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 277576b01c..9bd6582e68 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -375,9 +375,6 @@ static wchar_t *normalize_ntpath(wchar_t *wbuf) for (i = 0; wbuf[i]; i++) if (wbuf[i] == '\\') wbuf[i] = '/'; - /* remove potential trailing slashes */ - while (i && wbuf[i - 1] == '/') - wbuf[--i] = 0; return wbuf; } From fc555cc2defc487a3de62e886287c13c01f299df Mon Sep 17 00:00:00 2001 From: lchiocca Date: Mon, 22 Jun 2015 10:29:43 +0200 Subject: [PATCH 102/288] The stat() function should be independent of core.symlinks The contract for the stat() and lstat() function is: > stat(): stats the file pointed to by path and fills in buf. > lstat(): is identical to stat(), except that if path is a symbolic link, > then the link itself is stat-ed, not the file that it refers to. stat() should always return the statistics of the file or directory a symbolic link is pointing to. The lstat() function is used to get the stats for the symlink. Hence the check should not be there. Signed-off-by: Loris Chiocca --- compat/mingw.c | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 9bd6582e68..b5b39614bc 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -831,17 +831,7 @@ int mingw_stat(const char *file_name, struct stat *buf) HANDLE hnd; int result; - /* if symlinks are disabled, use lstat() (without following links) */ - if (!has_symlinks) { - result = lstat(file_name, buf); - if (!result && S_ISLNK(buf->st_mode)) { - errno = ELOOP; - return -1; - } - return result; - } - - /* otherwise just open the file and let Windows resolve the links */ + /* open the file and let Windows resolve the links */ if (xutftowcs_long_path(wfile_name, file_name) < 0) return -1; hnd = CreateFileW(wfile_name, 0, From 4e62380815f06a5ad861f4376a5bb7838fee920a Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Fri, 25 Sep 2015 18:02:55 +0200 Subject: [PATCH 103/288] Win32: replace MSVCRT's fstat() with a Win32-based implementation fstat() is the only stat-related CRT function for which we don't have a full replacement yet (and thus the only reason to stick with MSVCRT's 'struct stat' definition). Fully implement fstat(), in preparation of implementing a POSIX 2013 compatible 'struct stat' with nanosecond-precision file times. Signed-off-by: Karsten Blees --- compat/mingw.c | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index b5b39614bc..b7fd5931b9 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -849,18 +849,31 @@ int mingw_stat(const char *file_name, struct stat *buf) int mingw_fstat(int fd, struct stat *buf) { HANDLE fh = (HANDLE)_get_osfhandle(fd); - if (fh == INVALID_HANDLE_VALUE) { + DWORD avail, type = GetFileType(fh) & ~FILE_TYPE_REMOTE; + + switch (type) { + case FILE_TYPE_DISK: + return get_file_info_by_handle(fh, buf); + + case FILE_TYPE_CHAR: + case FILE_TYPE_PIPE: + /* initialize stat fields */ + memset(buf, 0, sizeof(*buf)); + buf->st_nlink = 1; + + if (type == FILE_TYPE_CHAR) { + buf->st_mode = _S_IFCHR; + } else { + buf->st_mode = _S_IFIFO; + if (PeekNamedPipe(fh, NULL, 0, NULL, &avail, NULL)) + buf->st_size = avail; + } + return 0; + + default: errno = EBADF; return -1; } - /* direct non-file handles to MS's fstat() */ - if (GetFileType(fh) != FILE_TYPE_DISK) - return _fstati64(fd, buf); - - if (!get_file_info_by_handle(fh, buf)) - return 0; - errno = EBADF; - return -1; } static inline void time_t_to_filetime(time_t t, FILETIME *ft) From 6e3de84f69945d9c5655c70148a111917edf45ad Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sat, 26 Sep 2015 02:03:28 +0200 Subject: [PATCH 104/288] Win32: implement nanosecond-precision file times We no longer use any of MSVCRT's stat-functions, so there's no need to stick to a CRT-compatible 'struct stat' either. Define and use our own POSIX-2013-compatible 'struct stat' with nanosecond- precision file times. Note: Due to performance issues when using git variants with different file time resolutions, this patch does *not* yet enable nanosecond precision in the Makefile (use 'make USE_NSEC=1'). Signed-off-by: Karsten Blees --- compat/mingw.c | 12 +++++------ compat/mingw.h | 48 +++++++++++++++++++++++++++++------------- compat/win32/fscache.c | 18 ++++++++-------- config.mak.uname | 2 -- 4 files changed, 48 insertions(+), 32 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index b7fd5931b9..af959561f5 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -772,9 +772,9 @@ int mingw_lstat(const char *file_name, struct stat *buf) buf->st_size = S_ISLNK(buf->st_mode) ? MAX_LONG_PATH : fdata.nFileSizeLow | (((off_t) fdata.nFileSizeHigh) << 32); buf->st_dev = buf->st_rdev = 0; /* not used by Git */ - buf->st_atime = filetime_to_time_t(&(fdata.ftLastAccessTime)); - buf->st_mtime = filetime_to_time_t(&(fdata.ftLastWriteTime)); - buf->st_ctime = filetime_to_time_t(&(fdata.ftCreationTime)); + filetime_to_timespec(&(fdata.ftLastAccessTime), &(buf->st_atim)); + filetime_to_timespec(&(fdata.ftLastWriteTime), &(buf->st_mtim)); + filetime_to_timespec(&(fdata.ftCreationTime), &(buf->st_ctim)); return 0; } error: @@ -819,9 +819,9 @@ static int get_file_info_by_handle(HANDLE hnd, struct stat *buf) buf->st_nlink = 1; buf->st_mode = file_attr_to_st_mode(fdata.dwFileAttributes, 0); buf->st_size = fdata.nFileSizeLow | (((off_t) fdata.nFileSizeHigh) << 32); - buf->st_atime = filetime_to_time_t(&(fdata.ftLastAccessTime)); - buf->st_mtime = filetime_to_time_t(&(fdata.ftLastWriteTime)); - buf->st_ctime = filetime_to_time_t(&(fdata.ftCreationTime)); + filetime_to_timespec(&(fdata.ftLastAccessTime), &(buf->st_atim)); + filetime_to_timespec(&(fdata.ftLastWriteTime), &(buf->st_mtim)); + filetime_to_timespec(&(fdata.ftCreationTime), &(buf->st_ctim)); return 0; } diff --git a/compat/mingw.h b/compat/mingw.h index e585263ee4..8ba86497ab 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -345,24 +345,49 @@ static inline long long filetime_to_hnsec(const FILETIME *ft) return winTime - 116444736000000000LL; } -static inline time_t filetime_to_time_t(const FILETIME *ft) -{ - return (time_t)(filetime_to_hnsec(ft) / 10000000); -} - /* - * Use mingw specific stat()/lstat()/fstat() implementations on Windows. + * Use mingw specific stat()/lstat()/fstat() implementations on Windows, + * including our own struct stat with 64 bit st_size and nanosecond-precision + * file times. */ #ifndef __MINGW64_VERSION_MAJOR #define off_t off64_t #define lseek _lseeki64 +struct timespec { + time_t tv_sec; + long tv_nsec; +}; #endif -/* use struct stat with 64 bit st_size */ +static inline void filetime_to_timespec(const FILETIME *ft, struct timespec *ts) +{ + long long hnsec = filetime_to_hnsec(ft); + ts->tv_sec = (time_t)(hnsec / 10000000); + ts->tv_nsec = (hnsec % 10000000) * 100; +} + +struct mingw_stat { + _dev_t st_dev; + _ino_t st_ino; + _mode_t st_mode; + short st_nlink; + short st_uid; + short st_gid; + _dev_t st_rdev; + off64_t st_size; + struct timespec st_atim; + struct timespec st_mtim; + struct timespec st_ctim; +}; + +#define st_atime st_atim.tv_sec +#define st_mtime st_mtim.tv_sec +#define st_ctime st_ctim.tv_sec + #ifdef stat #undef stat #endif -#define stat _stati64 +#define stat mingw_stat int mingw_lstat(const char *file_name, struct stat *buf); int mingw_stat(const char *file_name, struct stat *buf); int mingw_fstat(int fd, struct stat *buf); @@ -375,13 +400,6 @@ int mingw_fstat(int fd, struct stat *buf); #endif extern int (*lstat)(const char *file_name, struct stat *buf); -#ifndef _stati64 -# define _stati64(x,y) mingw_stat(x,y) -#elif defined (_USE_32BIT_TIME_T) -# define _stat32i64(x,y) mingw_stat(x,y) -#else -# define _stat64(x,y) mingw_stat(x,y) -#endif int mingw_utime(const char *file_name, const struct utimbuf *times); #define utime mingw_utime diff --git a/compat/win32/fscache.c b/compat/win32/fscache.c index d23d3a4a27..990d7dc8dd 100644 --- a/compat/win32/fscache.c +++ b/compat/win32/fscache.c @@ -38,9 +38,9 @@ struct fsentry { struct { /* More stat members (only used for file entries). */ off64_t st_size; - time_t st_atime; - time_t st_mtime; - time_t st_ctime; + struct timespec st_atim; + struct timespec st_mtim; + struct timespec st_ctim; }; }; }; @@ -151,9 +151,9 @@ static struct fsentry *fseentry_create_entry(struct fsentry *list, fdata->dwReserved0); fse->st_size = S_ISLNK(fse->st_mode) ? MAX_LONG_PATH : fdata->nFileSizeLow | (((off_t) fdata->nFileSizeHigh) << 32); - fse->st_atime = filetime_to_time_t(&(fdata->ftLastAccessTime)); - fse->st_mtime = filetime_to_time_t(&(fdata->ftLastWriteTime)); - fse->st_ctime = filetime_to_time_t(&(fdata->ftCreationTime)); + filetime_to_timespec(&(fdata->ftLastAccessTime), &(fse->st_atim)); + filetime_to_timespec(&(fdata->ftLastWriteTime), &(fse->st_mtim)); + filetime_to_timespec(&(fdata->ftCreationTime), &(fse->st_ctim)); return fse; } @@ -432,9 +432,9 @@ int fscache_lstat(const char *filename, struct stat *st) st->st_nlink = 1; st->st_mode = fse->st_mode; st->st_size = fse->st_size; - st->st_atime = fse->st_atime; - st->st_mtime = fse->st_mtime; - st->st_ctime = fse->st_ctime; + st->st_atim = fse->st_atim; + st->st_mtim = fse->st_mtim; + st->st_ctim = fse->st_ctim; /* don't forget to release fsentry */ fsentry_release(fse); diff --git a/config.mak.uname b/config.mak.uname index ed2cf5b6dd..c13d5cb441 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -360,7 +360,6 @@ ifeq ($(uname_S),Windows) NO_SVN_TESTS = YesPlease RUNTIME_PREFIX = YesPlease NO_ST_BLOCKS_IN_STRUCT_STAT = YesPlease - NO_NSEC = YesPlease USE_WIN32_MMAP = YesPlease # USE_NED_ALLOCATOR = YesPlease UNRELIABLE_FSTAT = UnfortunatelyYes @@ -510,7 +509,6 @@ ifneq (,$(findstring MINGW,$(uname_S))) NO_SVN_TESTS = YesPlease RUNTIME_PREFIX = YesPlease NO_ST_BLOCKS_IN_STRUCT_STAT = YesPlease - NO_NSEC = YesPlease USE_WIN32_MMAP = YesPlease USE_NED_ALLOCATOR = YesPlease UNRELIABLE_FSTAT = UnfortunatelyYes From 12b6576cf6e6acdb89aea2d794d00bc8a89b1043 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 18 Sep 2015 16:57:33 +0200 Subject: [PATCH 105/288] mingw: ensure `getcwd()` reports the correct case When switching the current working directory, say, in PowerShell, it is quite possible to use a different capitalization than the one that is recorded on disk. While doing the same in `cmd.exe` adjusts the capitalization magically, that does not happen in PowerShell so that `getcwd()` returns the current directory in a different way than is recorded on disk. Typically this creates no problems except when you call git log . in a subdirectory called, say, "GIT/" but you switched to "Git/" and your `getcwd()` reports the latter, then Git won't understand that you wanted to see the history as per the `GIT/` subdirectory but it thinks you wanted to see the history of some directory that may have existed in the past (but actually never did). So let's be extra careful to adjust the capitalization of the current directory before working with it. Reported by a few PowerShell power users ;-) Signed-off-by: Johannes Schindelin --- compat/mingw.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 3fbfda5978..560a4f9b48 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -837,8 +837,13 @@ struct tm *localtime_r(const time_t *timep, struct tm *result) char *mingw_getcwd(char *pointer, int len) { - wchar_t wpointer[MAX_PATH]; - if (!_wgetcwd(wpointer, ARRAY_SIZE(wpointer))) + wchar_t cwd[MAX_PATH], wpointer[MAX_PATH]; + DWORD ret = GetCurrentDirectoryW(ARRAY_SIZE(cwd), cwd); + + if (ret < 0 || ret >= ARRAY_SIZE(cwd)) + return NULL; + ret = GetLongPathNameW(cwd, wpointer, ARRAY_SIZE(wpointer)); + if (!ret || ret >= ARRAY_SIZE(wpointer)) return NULL; if (xwcstoutf(pointer, wpointer, len) < 0) return NULL; From e74e4e9f0b895dd4dc7a47959bab4c917eb6a004 Mon Sep 17 00:00:00 2001 From: Anton Serbulov Date: Fri, 15 Jan 2016 12:25:02 +0600 Subject: [PATCH 106/288] mingw: fix getcwd when the parent directory cannot be queried `GetLongPathName()` function may fail when it is unable to query the parent directory of a path component to determine the long name for that component. It happens, because of it uses `FindFirstFile()` function for each next short part of path. The `FindFirstFile()` requires `List Directory` and `Synchronize` desired access for a calling process. In case of lacking such permission for some part of path, the `GetLongPathName()` returns 0 as result and `GetLastError()` returns ERROR_ACCESS_DENIED. `GetFinalPathNameByHandle()` function can help in such cases, because it requires `Read Attributes` and `Synchronize` desired access to the target path only. The `GetFinalPathNameByHandle()` function was introduced on `Windows Server 2008/Windows Vista`. So we need to load it dynamically. `CreateFile()` parameters: `lpFileName` = path to the current directory `dwDesiredAccess` = 0 (it means `Read Attributes` and `Synchronize`) `dwShareMode` = FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE (it prevents `Sharing Violation`) `lpSecurityAttributes` = NULL (default security attributes) `dwCreationDisposition` = OPEN_EXISTING (required to obtain a directory handle) `dwFlagsAndAttributes` = FILE_FLAG_BACKUP_SEMANTICS (required to obtain a directory handle) `hTemplateFile` = NULL (when opening an existing file or directory, `CreateFile` ignores this parameter) The string that is returned by `GetFinalPathNameByHandle()` function uses the \\?\ syntax. To skip the prefix and convert backslashes to slashes, the `normalize_ntpath()` mingw function will be used. Note: `GetFinalPathNameByHandle()` function returns a final path. It is the path that is returned when a path is fully resolved. For example, for a symbolic link named "C:\tmp\mydir" that points to "D:\yourdir", the final path would be "D:\yourdir". Signed-off-by: Anton Serbulov --- compat/mingw.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/compat/mingw.c b/compat/mingw.c index 560a4f9b48..2bcb893389 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -838,11 +838,28 @@ struct tm *localtime_r(const time_t *timep, struct tm *result) char *mingw_getcwd(char *pointer, int len) { wchar_t cwd[MAX_PATH], wpointer[MAX_PATH]; + DECLARE_PROC_ADDR(kernel32.dll, DWORD, GetFinalPathNameByHandleW, + HANDLE, LPWSTR, DWORD, DWORD); DWORD ret = GetCurrentDirectoryW(ARRAY_SIZE(cwd), cwd); if (ret < 0 || ret >= ARRAY_SIZE(cwd)) return NULL; ret = GetLongPathNameW(cwd, wpointer, ARRAY_SIZE(wpointer)); + if (!ret && GetLastError() == ERROR_ACCESS_DENIED && + INIT_PROC_ADDR(GetFinalPathNameByHandleW)) { + HANDLE hnd = CreateFileW(cwd, 0, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, NULL, + OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); + if (hnd == INVALID_HANDLE_VALUE) + return NULL; + ret = GetFinalPathNameByHandleW(hnd, wpointer, ARRAY_SIZE(wpointer), 0); + CloseHandle(hnd); + if (!ret || ret >= ARRAY_SIZE(wpointer)) + return NULL; + if (xwcstoutf(pointer, normalize_ntpath(wpointer), len) < 0) + return NULL; + return pointer; + } if (!ret || ret >= ARRAY_SIZE(wpointer)) return NULL; if (xwcstoutf(pointer, wpointer, len) < 0) From 57f3c5799eccebc000361bc4ee2cd4c3dd7fd22f Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Fri, 22 May 2015 15:20:21 +0200 Subject: [PATCH 107/288] config.c: create missing parent directories when modifying config files 'git config' (--add / --unset etc.) automatically creates missing config files. However, it fails with a misleading error message "could not lock config file" if the parent directory doesn't exist. Also create missing parent directories. This is particularly important when calling git config -f /non/existing/directory/config ... We *must not* create the leading directories when locking a config file. It is the caller's responsibility to ensure that the directory exists, just like it is the caller's responsibility to call `git init` before running repository operations. Point in case: if we simply create all leading directories, calling `git config user.name me` *outside* of a Git worktree will *create* .git/! This fixes https://github.com/git-for-windows/git/issues/643 and https://groups.google.com/forum/#!topic/git-for-windows/fVRdnDIKVuw [jes: prevented bogus .git/ directories from being created] Signed-off-by: Karsten Blees Signed-off-by: Johannes Schindelin --- config.c | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/config.c b/config.c index b805534902..87606f8968 100644 --- a/config.c +++ b/config.c @@ -2063,6 +2063,20 @@ int git_config_key_is_valid(const char *key) return !git_config_parse_key_1(key, NULL, NULL, 1); } +static int lock_config_file(const char *config_filename, + struct lock_file **result) +{ + int fd; + + *result = xcalloc(1, sizeof(struct lock_file)); + fd = hold_lock_file_for_update(*result, config_filename, 0); + if (fd < 0) + error("could not lock config file %s: %s", config_filename, + strerror(errno)); + + return fd; +} + /* * If value==NULL, unset in (remove from) config, * if value_regex!=NULL, disregard key/value pairs where value does not match. @@ -2114,8 +2128,7 @@ int git_config_set_multivar_in_file_gently(const char *config_filename, * The lock serves a purpose in addition to locking: the new * contents of .git/config will be written into it. */ - lock = xcalloc(1, sizeof(struct lock_file)); - fd = hold_lock_file_for_update(lock, config_filename, 0); + fd = lock_config_file(config_filename, &lock); if (fd < 0) { error_errno("could not lock config file %s", config_filename); free(store.key); @@ -2421,12 +2434,9 @@ int git_config_rename_section_in_file(const char *config_filename, if (!config_filename) config_filename = filename_buf = git_pathdup("config"); - lock = xcalloc(1, sizeof(struct lock_file)); - out_fd = hold_lock_file_for_update(lock, config_filename, 0); - if (out_fd < 0) { - ret = error("could not lock config file %s", config_filename); + out_fd = lock_config_file(config_filename, &lock); + if (out_fd < 0) goto out; - } if (!(config_file = fopen(config_filename, "rb"))) { /* no config file means nothing to rename, no error */ From c76f6d2d25d5e2f96d7b8ac966aebf2898926f42 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 22 Apr 2015 14:47:27 +0100 Subject: [PATCH 108/288] Windows: add support for a Windows-wide configuration Between the libgit2 and the Git for Windows project, there has been a discussion how we could share Git configuration to avoid duplication (or worse: skew). Earlier, libgit2 was nice enough to just re-use Git for Windows' C:\Program Files (x86)\Git\etc\gitconfig but with the upcoming Git for Windows 2.x, there would be more paths to search, as we will have 64-bit and 32-bit versions, and the corresponding config files will be in %PROGRAMFILES%\Git\mingw64\etc and ...\mingw32\etc, respectively. Worse: there are portable Git for Windows versions out there which live in totally unrelated directories, still. Therefore we came to a consensus to use `%PROGRAMDATA%\Git\config` as the location for shared Git settings that are of wider interest than just Git for Windows. On XP, there is no %PROGRAMDATA%, therefore we need to use "%ALLUSERSPROFILE%\Application Data\Git\config" in those setups. Of course, the configuration in `%PROGRAMDATA%\Git\config` has the widest reach, therefore it must take the lowest precedence, i.e. Git for Windows can still override settings in its `etc/gitconfig` file. Signed-off-by: Johannes Schindelin --- Documentation/config.txt | 4 +++- Documentation/git-config.txt | 5 +++++ Documentation/git.txt | 3 ++- compat/mingw.c | 19 +++++++++++++++++++ compat/mingw.h | 2 ++ config.c | 13 ++++++++++--- git-compat-util.h | 4 ++++ 7 files changed, 45 insertions(+), 5 deletions(-) diff --git a/Documentation/config.txt b/Documentation/config.txt index 667c4bfc83..94045363ef 100644 --- a/Documentation/config.txt +++ b/Documentation/config.txt @@ -6,7 +6,9 @@ the Git commands' behavior. The `.git/config` file in each repository is used to store the configuration for that repository, and `$HOME/.gitconfig` is used to store a per-user configuration as fallback values for the `.git/config` file. The file `/etc/gitconfig` -can be used to store a system-wide default configuration. +can be used to store a system-wide default configuration. On Windows, +configuration can also be stored in `C:\ProgramData\Git\config`; This +file will be used also by libgit2-based software. The configuration variables are used by both the Git plumbing and the porcelains. The variables are divided into sections, wherein diff --git a/Documentation/git-config.txt b/Documentation/git-config.txt index 83f86b9231..de004f7c8e 100644 --- a/Documentation/git-config.txt +++ b/Documentation/git-config.txt @@ -253,6 +253,11 @@ $XDG_CONFIG_HOME/git/config:: $GIT_DIR/config:: Repository specific configuration file. +On Windows, as there is no central `/etc/` directory, there is yet another +config file, intended to contain settings for *all* Git-related software +running on the machine. Consequently, this config file takes an even lower +precedence than the `$(prefix)/etc/gitconfig` file. + If no further options are given, all reading options will read all of these files that are available. If the global or the system-wide configuration file are not available they will be ignored. If the repository configuration diff --git a/Documentation/git.txt b/Documentation/git.txt index bdab261515..2873c1cb35 100644 --- a/Documentation/git.txt +++ b/Documentation/git.txt @@ -1033,7 +1033,8 @@ for further details. `GIT_CONFIG_NOSYSTEM`:: Whether to skip reading settings from the system-wide - `$(prefix)/etc/gitconfig` file. This environment variable can + `$(prefix)/etc/gitconfig` file (and on Windows, also from the + `%PROGRAMDATA%\Git\config` file). This environment variable can be used along with `$HOME` and `$XDG_CONFIG_HOME` to create a predictable environment for a picky script, or you can set it temporarily to avoid using a buggy `/etc/gitconfig` file while diff --git a/compat/mingw.c b/compat/mingw.c index 6b2fbd4904..a5c292592b 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2526,3 +2526,22 @@ int uname(struct utsname *buf) "%u", (v >> 16) & 0x7fff); return 0; } + +const char *program_data_config(void) +{ + static struct strbuf path = STRBUF_INIT; + static unsigned initialized; + + if (!initialized) { + const char *env = mingw_getenv("PROGRAMDATA"); + const char *extra = ""; + if (!env) { + env = mingw_getenv("ALLUSERSPROFILE"); + extra = "/Application Data"; + } + if (env) + strbuf_addf(&path, "%s%s/Git/config", env, extra); + initialized = 1; + } + return *path.buf ? path.buf : NULL; +} diff --git a/compat/mingw.h b/compat/mingw.h index a94df53584..ccbfdf6914 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -445,6 +445,8 @@ int mingw_offset_1st_component(const char *path); #define PATH_SEP ';' extern char *mingw_query_user_email(void); #define query_user_email mingw_query_user_email +extern const char *program_data_config(void); +#define git_program_data_config program_data_config #if !defined(__MINGW64_VERSION_MAJOR) && (!defined(_MSC_VER) || _MSC_VER < 1800) #define PRIuMAX "I64u" #define PRId64 "I64d" diff --git a/config.c b/config.c index 87606f8968..55dd56f026 100644 --- a/config.c +++ b/config.c @@ -1306,9 +1306,16 @@ static int do_git_config_sequence(config_fn_t fn, void *data) char *repo_config = have_git_dir() ? git_pathdup("config") : NULL; current_parsing_scope = CONFIG_SCOPE_SYSTEM; - if (git_config_system() && !access_or_die(git_etc_gitconfig(), R_OK, 0)) - ret += git_config_from_file(fn, git_etc_gitconfig(), - data); + if (git_config_system()) { + if (git_program_data_config() && + !access_or_die(git_program_data_config(), R_OK, 0)) + ret += git_config_from_file(fn, + git_program_data_config(), + data); + if (!access_or_die(git_etc_gitconfig(), R_OK, 0)) + ret += git_config_from_file(fn, git_etc_gitconfig(), + data); + } current_parsing_scope = CONFIG_SCOPE_GLOBAL; if (xdg_config && !access_or_die(xdg_config, R_OK, ACCESS_EACCES_OK)) diff --git a/git-compat-util.h b/git-compat-util.h index 1377cfa7b8..a9e832694d 100644 --- a/git-compat-util.h +++ b/git-compat-util.h @@ -384,6 +384,10 @@ static inline char *git_find_last_dir_sep(const char *path) #define query_user_email() NULL #endif +#ifndef git_program_data_config +#define git_program_data_config() NULL +#endif + #if defined(__HP_cc) && (__HP_cc >= 61000) #define NORETURN __attribute__((noreturn)) #define NORETURN_PTR From 382bbb3c2cce6956efdbd20bdb27c92a0fee051b Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 16 Feb 2016 15:21:45 +0100 Subject: [PATCH 109/288] Clarify the location of the Windows-specific ProgramData config On Windows, there is no (single) `/etc/` directory. To address that, in conjunction with the libgit2 project, Git for Windows introduced yet another level of system-wide config files, located in C:\ProgramData (and the equivalent on Windows XP). Let's spell this out in the documentation. This closes https://github.com/git-for-windows/git/pull/470 (because there was no reaction in three months in that Pull Request). Signed-off-by: Johannes Schindelin --- Documentation/git-config.txt | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Documentation/git-config.txt b/Documentation/git-config.txt index de004f7c8e..273b6d0186 100644 --- a/Documentation/git-config.txt +++ b/Documentation/git-config.txt @@ -254,9 +254,12 @@ $GIT_DIR/config:: Repository specific configuration file. On Windows, as there is no central `/etc/` directory, there is yet another -config file, intended to contain settings for *all* Git-related software -running on the machine. Consequently, this config file takes an even lower -precedence than the `$(prefix)/etc/gitconfig` file. +config file (located at `$PROGRAMDATA/Git/config`), intended to contain +settings for *all* Git-related software running on the machine. Consequently, +this config file takes an even lower precedence than the +`$(prefix)/etc/gitconfig` file. Typically `$PROGRAMDATA` points to +`C:\ProgramData` (on Windows XP the equivalent in `$ALLUSERSPROFILE` is used, +i.e. `C:\Documents and Settings\All Users\Application Data\Git\config`). If no further options are given, all reading options will read all of these files that are available. If the global or the system-wide configuration From 8384098838eb5103661ef7e4699caeb87d8962df Mon Sep 17 00:00:00 2001 From: Kelly Heller Date: Wed, 27 May 2015 14:51:43 -0700 Subject: [PATCH 110/288] Allow `add -p` and `add -i` with a large number of files This fixes https://github.com/msysgit/git/issues/182. Inspired by Pull Request 218 using code from @PhilipDavis. [jes: simplified code quite a bit] Signed-off-by: Kelly Heller Signed-off-by: Johannes Schindelin --- git-add--interactive.perl | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/git-add--interactive.perl b/git-add--interactive.perl index ee3d812695..4ed09694bd 100755 --- a/git-add--interactive.perl +++ b/git-add--interactive.perl @@ -180,6 +180,24 @@ sub run_cmd_pipe { die "$^O does not support: @invalid\n" if @invalid; my @args = map { m/ /o ? "\"$_\"": $_ } @_; return qx{@args}; + } elsif (($^O eq 'MSWin32' || $^O eq 'msys') && (scalar @_ > 200) && + grep $_ eq '--', @_) { + use File::Temp qw(tempfile); + my ($fhargs, $filename) = + tempfile('git-args-XXXXXX', UNLINK => 1); + + my $cmd = 'cat '.$filename.' | xargs -0 -s 20000 '; + while ($_[0] ne '--') { + $cmd = $cmd . shift(@_) . ' '; + } + + shift(@_); + print $fhargs join("\0", @_); + close($fhargs); + + my $fh = undef; + open($fh, '-|', $cmd) or die; + return <$fh>; } else { my $fh = undef; open($fh, '-|', @_) or die; From f2622c9a075a670a2188187f657987758b2c0571 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sat, 12 Sep 2015 12:25:47 +0200 Subject: [PATCH 111/288] t3701: verify that we can add *lots* of files interactively Signed-off-by: Johannes Schindelin --- t/t3701-add-interactive.sh | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/t/t3701-add-interactive.sh b/t/t3701-add-interactive.sh index deae948c76..314b38e533 100755 --- a/t/t3701-add-interactive.sh +++ b/t/t3701-add-interactive.sh @@ -380,4 +380,25 @@ test_expect_success 'patch mode ignores unmerged entries' ' test_cmp expected diff ' +test_expect_success EXPENSIVE 'add -i with a lot of files' ' + git reset --hard && + x160=0123456789012345678901234567890123456789 && + x160=$x160$x160$x160$x160 && + y= && + i=0 && + while test $i -le 200 + do + name=$(printf "%s%03d" $x160 $i) && + echo $name >$name && + git add -N $name && + y="${y}y$LF" && + i=$(($i+1)) || + break + done && + echo "$y" | git add -p -- . && + git diff --cached >staged && + test_line_count = 1407 staged && + git reset --hard +' + test_done From dc85b893f899e028811d86b0e9f5f34570ee4203 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 20 Oct 2016 06:31:47 -0700 Subject: [PATCH 112/288] msvc: fix dependencies of compat/msvc.c The file compat/msvc.c includes compat/mingw.c, which means that we have to recompile compat/msvc.o if compat/mingw.c changes. Signed-off-by: Johannes Schindelin --- config.mak.uname | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config.mak.uname b/config.mak.uname index c13d5cb441..18a76dcb25 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -396,6 +396,8 @@ else BASIC_CFLAGS += -Zi -MDd endif X = .exe + +compat/msvc.o: compat/msvc.c compat/mingw.c GIT-CFLAGS endif ifeq ($(uname_S),Interix) NO_INITGROUPS = YesPlease From 2f06dc75f67e6a2638b3f6a7abc0c83a60020a2f Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 3 Jun 2016 08:38:17 -0400 Subject: [PATCH 113/288] msvc: ignore VS2015 trash files Signed-off-by: Jeff Hostetler --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 6722f78f9a..426bd70d1a 100644 --- a/.gitignore +++ b/.gitignore @@ -216,5 +216,7 @@ *.user *.idb *.pdb +*.ilk +.vs/ /Debug/ /Release/ From 4e6e309779c7bd706efd705b825d38ff49e54d2b Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 3 Jun 2016 11:32:01 -0400 Subject: [PATCH 114/288] msvc: add NuGet scripts for building with VS2015 This commit contains a GNU Makefile and NuGet configuration scripts to download and install the various third-party libraries that we will need to build/link with when using VS2015 to build Git. The file "compat/vcbuild/README_VS2015.txt" contains instructions for using this. In this commit, "compat/vcbuild/Makefile" contains hard-coded version numbers of the packages we require. These are set to the current versions as of the time of this commit. We use "nuget restore" to install them explicitly using a "package.config". A future improvement would try to use some of the automatic package management functions and eliminate the need to specify exact versions. I tried, but could not get this to work. NuGet was happy dowload "minimum requirements" rather than "lastest" for dependencies -- and only look at one package at a time. For example, both curl and openssl depend upon zlib and have different minimums. It was unclear which version of zlib would be installed and seemed to be dependent on the order of the top-level packages. So, I'm skipping that for now. We need to be very precise when specifying NuGet package versions: while nuget.exe auto-completes a version, say, 1.0.2 to 1.0.2.0, we will want to parse packages.config ourselves, to generate the Visual Studio solution, and there we need the exact version number to be able to generate the exact path to the correct .targets file. Signed-off-by: Jeff Hostetler --- compat/vcbuild/.gitignore | 1 + compat/vcbuild/Makefile | 122 +++++++++++++++++++++++++++++++ compat/vcbuild/README_VS2015.txt | 58 +++++++++++++++ compat/vcbuild/nuget.config | 27 +++++++ compat/vcbuild/packages.config | 23 ++++++ 5 files changed, 231 insertions(+) create mode 100644 compat/vcbuild/.gitignore create mode 100644 compat/vcbuild/Makefile create mode 100644 compat/vcbuild/README_VS2015.txt create mode 100644 compat/vcbuild/nuget.config create mode 100644 compat/vcbuild/packages.config diff --git a/compat/vcbuild/.gitignore b/compat/vcbuild/.gitignore new file mode 100644 index 0000000000..e14691508e --- /dev/null +++ b/compat/vcbuild/.gitignore @@ -0,0 +1 @@ +GEN.* diff --git a/compat/vcbuild/Makefile b/compat/vcbuild/Makefile new file mode 100644 index 0000000000..b94bcd119a --- /dev/null +++ b/compat/vcbuild/Makefile @@ -0,0 +1,122 @@ +## Makefile to install nuget package dependencies. +################################################################## + +INST=GEN.DEPS +INST_INC=$(INST)/include +INST_LIB=$(INST)/lib +INST_BIN=$(INST)/bin + +PKGDIR=GEN.PKGS + +################################################################## +all: unpack expat libssh libssh_redist curl curl_redist openssl zlib + +unpack: + [ -d $(PKGDIR) ] || mkdir $(PKGDIR) + nuget.exe restore packages.config -ConfigFile nuget.config \ + -NonInteractive -OutputDirectory $(PKGDIR) + +insdir: + [ -d $(INST) ] || mkdir $(INST) + [ -d $(INST_INC) ] || mkdir $(INST_INC) + [ -d $(INST_BIN) ] || mkdir $(INST_BIN) + [ -d $(INST_LIB) ] || mkdir $(INST_LIB) + +################################################################## +## We place the expat headers in their own subdirectory. +## The custom is to reference , so compile with: +## -I$(INST_INC)/expat + +EXPAT_VER=2.1.0.11 +EXPAT_ROOT=$(PKGDIR)/expat.$(EXPAT_VER)/build/native +EXPAT_INC=$(EXPAT_ROOT)/include +EXPAT_LIB=$(EXPAT_ROOT)/lib/v110/x64/Release/dynamic/utf8 + +expat: insdir + [ -d $(INST_INC)/expat ] || mkdir $(INST_INC)/expat + cp -r $(EXPAT_INC)/* $(INST_INC)/expat/ + cp -r $(EXPAT_LIB)/* $(INST_LIB)/ + + +################################################################## + +LIBSSH_VER=1.4.3.3 +LIBSSH_ROOT=$(PKGDIR)/libssh2.$(LIBSSH_VER)/build/native +LIBSSH_INC=$(LIBSSH_ROOT)/include +LIBSSH_LIB=$(LIBSSH_ROOT)/lib/v110/x64/Release/dynamic/cdecl + +libssh: insdir + [ -d $(INST_INC)/libssh2 ] || mkdir $(INST_INC)/libssh2 + cp -r $(LIBSSH_INC)/* $(INST_INC)/libssh2 + cp -r $(LIBSSH_LIB)/* $(INST_LIB)/ + + +LIBSSH_REDIST_ROOT=$(PKGDIR)/libssh2.redist.$(LIBSSH_VER)/build/native +LIBSSH_REDIST_BIN=$(LIBSSH_REDIST_ROOT)/bin/v110/x64/Release/dynamic/cdecl + +libssh_redist: insdir + cp -r $(LIBSSH_REDIST_BIN)/* $(INST_BIN)/ + + +################################################################## +## We place the curl headers in their own subdirectory. +## The custom is to reference , so compile with: +## -I$(INST_INC) + +CURL_VER=7.30.0.2 +CURL_ROOT=$(PKGDIR)/curl.$(CURL_VER)/build/native +CURL_INC=$(CURL_ROOT)/include/curl +CURL_LIB=$(CURL_ROOT)/lib/v110/x64/Release/dynamic + +curl: insdir + [ -d $(INST_INC)/curl ] || mkdir $(INST_INC)/curl + cp -r $(CURL_INC)/* $(INST_INC)/curl + cp -r $(CURL_LIB)/* $(INST_LIB)/ + + +CURL_REDIST_ROOT=$(PKGDIR)/curl.redist.$(CURL_VER)/build/native +CURL_REDIST_BIN=$(CURL_REDIST_ROOT)/bin/v110/x64/Release/dynamic + +curl_redist: insdir + cp -r $(CURL_REDIST_BIN)/* $(INST_BIN)/ + + +################################################################## +## We place the openssl headers in their own subdirectory. +## The custom is to reference , so compile with: +## -I$(INST_INC) + +OPENSSL_VER=1.0.2.1 +OPENSSL_ROOT=$(PKGDIR)/openssl.v140.windesktop.msvcstl.dyn.rt-dyn.x64.$(OPENSSL_VER) +OPENSSL_INC=$(OPENSSL_ROOT)/build/native/include/openssl +OPENSSL_LIB=$(OPENSSL_ROOT)/lib/native/v140/windesktop/msvcstl/dyn/rt-dyn/x64/release + +openssl: insdir + [ -d $(INST_INC)/openssl ] || mkdir $(INST_INC)/openssl + cp -r $(OPENSSL_INC)/* $(INST_INC)/openssl + cp -r $(OPENSSL_LIB)/*.lib $(INST_LIB)/ + cp -r $(OPENSSL_LIB)/*.dll $(INST_BIN)/ + cp -r $(OPENSSL_LIB)/*.pdb $(INST_BIN)/ + +################################################################## +## We place the zlib headers in their own subdirectory. +## The custom is to reference , so compile with: +## -I$(INST_INC)/zlib + +ZLIB_VER=1.2.8.8 +ZLIB_ROOT=$(PKGDIR)/zlib.v140.windesktop.msvcstl.dyn.rt-dyn.$(ZLIB_VER) +ZLIB_INC=$(ZLIB_ROOT)/build/native/include +ZLIB_LIB=$(ZLIB_ROOT)/lib/native/v140/windesktop/msvcstl/dyn/rt-dyn/x64/Release + +zlib: insdir + [ -d $(INST_INC)/zlib ] || mkdir $(INST_INC)/zlib + cp -r $(ZLIB_INC)/* $(INST_INC)/zlib + cp -r $(ZLIB_LIB)/*.lib $(INST_LIB)/ + cp -r $(ZLIB_LIB)/*.dll $(INST_BIN)/ + +################################################################## +clean: + rm -rf $(INST) + +clobber: clean + rm -rf $(PKGDIR) diff --git a/compat/vcbuild/README_VS2015.txt b/compat/vcbuild/README_VS2015.txt new file mode 100644 index 0000000000..763523fbb4 --- /dev/null +++ b/compat/vcbuild/README_VS2015.txt @@ -0,0 +1,58 @@ +Instructions for building Git for Windows using VS2015. +================================================================ + +Installing third-party dependencies: +==================================== + +[1] Install nuget.exe somewhere on your system and add it to your PATH. + https://docs.nuget.org/consume/command-line-reference + https://dist.nuget.org/index.html + +[2] Download required nuget packages for third-party libraries. + Using a terminal window, type: + + make -C compat/vcbuild + + This will download the packages, unpack them into GEN.PKGS, + and populate the {include, lib, bin} directories in GEN.DEPS. + + +Building Git for Windows using VS2015: +====================================== + +[3] Build 64-bit version of Git for Windows. + Using a terminal window: + + make MSVC=1 DEBUG=1 + + +[4] Add compat/vcbuild/GEN.DEPS/bin to your PATH. + +[5] You should then be able to run the test suite and any interactive + commands. + +[6] To debug/profile in VS, open the git.exe in VS and run/debug + it. (Be sure to add GEN.DEPS/bin to the PATH in the debug + dialog.) + + +TODO List: +========== + +[A] config.mak.uname currently contains hard-coded paths + to the various MSVC and SDK libraries for the 64-bit + version of the compilers and libaries. + + See: SANE_TOOL_PATH, MSVC_DEPS, MSVC_SDK*, MSVC_VCDIR. + + Long term, we need to figure out how to properly import + values for %VCINSTALLDIR%, %LIB%, %LIBPATH%, and the + other values normally set by "vsvars32.bat" when a + developer command prompt is started. This would also + allow us to switch between 32- and 64-bit tool chains. + +[B] Currently, we leave the third-party DLLs we reference in + "compat/vcbuild/GEN.DEPS/bin". We need an installer + step to move them next to git.exe (or into libexec/git-core). + +[C] We need to build SLN or VCPROJ files. diff --git a/compat/vcbuild/nuget.config b/compat/vcbuild/nuget.config new file mode 100644 index 0000000000..b5e7349504 --- /dev/null +++ b/compat/vcbuild/nuget.config @@ -0,0 +1,27 @@ + + + + + + + + + + + + + + + + + + + + + + + diff --git a/compat/vcbuild/packages.config b/compat/vcbuild/packages.config new file mode 100644 index 0000000000..e6e6d2f63f --- /dev/null +++ b/compat/vcbuild/packages.config @@ -0,0 +1,23 @@ + + + + + + + + + + + + + + + + + + + + + + + From f30228d55bb492a868c31c8dd6e30fc1f8247c72 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 3 Jun 2016 13:34:28 -0400 Subject: [PATCH 115/288] msvc: update compile helper for VS2015 Support -Z flags ("specify PDB options"), only include -l args on link commands, and force PDBs to be created. Signed-off-by: Jeff Hostetler --- compat/vcbuild/scripts/clink.pl | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/compat/vcbuild/scripts/clink.pl b/compat/vcbuild/scripts/clink.pl index a87d0da512..3bbfa47bf4 100755 --- a/compat/vcbuild/scripts/clink.pl +++ b/compat/vcbuild/scripts/clink.pl @@ -12,10 +12,11 @@ use strict; my @args = (); my @cflags = (); +my @lflags = (); my $is_linking = 0; while (@ARGV) { my $arg = shift @ARGV; - if ("$arg" =~ /^-[DIMGO]/) { + if ("$arg" =~ /^-[DIMGOZ]/) { push(@cflags, $arg); } elsif ("$arg" eq "-o") { my $file_out = shift @ARGV; @@ -35,9 +36,11 @@ while (@ARGV) { push(@args, "ssleay32.lib"); } elsif ("$arg" eq "-lcurl") { push(@args, "libcurl.lib"); + } elsif ("$arg" eq "-lexpat") { + push(@args, "libexpat.lib"); } elsif ("$arg" =~ /^-L/ && "$arg" ne "-LTCG") { $arg =~ s/^-L/-LIBPATH:/; - push(@args, $arg); + push(@lflags, $arg); } elsif ("$arg" =~ /^-R/) { # eat } else { @@ -45,6 +48,9 @@ while (@ARGV) { } } if ($is_linking) { + push(@args, @lflags); + # force PDB to be created. + push(@args, "-debug"); unshift(@args, "link.exe"); } else { unshift(@args, "cl.exe"); From 24bdd6dbdbd73c7e806b5a37bef1ca1558985aff Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 3 Jun 2016 14:15:02 -0400 Subject: [PATCH 116/288] msvc: update Makefile and compiler settings for VS2015 Signed-off-by: Jeff Hostetler --- Makefile | 2 +- config.mak.uname | 47 ++++++++++++++++++++++++++++++++++++++++++----- git-compat-util.h | 9 +++++++++ 3 files changed, 52 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 7c7a21b054..2a8bd86766 100644 --- a/Makefile +++ b/Makefile @@ -1011,7 +1011,7 @@ endif ifdef SANE_TOOL_PATH SANE_TOOL_PATH_SQ = $(subst ','\'',$(SANE_TOOL_PATH)) -BROKEN_PATH_FIX = 's|^\# @@BROKEN_PATH_FIX@@$$|git_broken_path_fix $(SANE_TOOL_PATH_SQ)|' +BROKEN_PATH_FIX = 's|^\# @@BROKEN_PATH_FIX@@$$|git_broken_path_fix "$(SANE_TOOL_PATH_SQ)"|' PATH := $(SANE_TOOL_PATH):${PATH} else BROKEN_PATH_FIX = '/^\# @@BROKEN_PATH_FIX@@$$/d' diff --git a/config.mak.uname b/config.mak.uname index 18a76dcb25..a476fa9b87 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -339,6 +339,13 @@ endif ifeq ($(uname_S),Windows) GIT_VERSION := $(GIT_VERSION).MSVC pathsep = ; + # Prepend MSVC 64-bit tool-chain to PATH. + # + # A regular Git Bash *does not* have cl.exe in its $PATH. As there is a + # link.exe next to, and required by, cl.exe, we have to prepend this + # onto the existing $PATH. + # + SANE_TOOL_PATH ?= /c/Program Files (x86)/Microsoft Visual Studio 14.0/VC/bin/amd64 HAVE_ALLOCA_H = YesPlease NO_PREAD = YesPlease NEEDS_CRYPTO_WITH_SSL = YesPlease @@ -356,7 +363,13 @@ ifeq ($(uname_S),Windows) NO_STRTOUMAX = YesPlease NO_MKDTEMP = YesPlease NO_MKSTEMPS = YesPlease - SNPRINTF_RETURNS_BOGUS = YesPlease + # VS2015 with UCRT claims that snprintf and friends are C99 compliant, + # so we don't need this. + # + # TODO If we want to support older compilers, we need to make this + # TODO conditional on the compiler version. + # + # SNPRINTF_RETURNS_BOGUS = YesPlease NO_SVN_TESTS = YesPlease RUNTIME_PREFIX = YesPlease NO_ST_BLOCKS_IN_STRUCT_STAT = YesPlease @@ -378,22 +391,46 @@ ifeq ($(uname_S),Windows) CC = compat/vcbuild/scripts/clink.pl AR = compat/vcbuild/scripts/lib.pl CFLAGS = - BASIC_CFLAGS = -nologo -I. -I../zlib -Icompat/vcbuild -Icompat/vcbuild/include -DWIN32 -D_CONSOLE -DHAVE_STRING_H -D_CRT_SECURE_NO_WARNINGS -D_CRT_NONSTDC_NO_DEPRECATE + BASIC_CFLAGS = -nologo -I. -Icompat/vcbuild/include -DWIN32 -D_CONSOLE -DHAVE_STRING_H -D_CRT_SECURE_NO_WARNINGS -D_CRT_NONSTDC_NO_DEPRECATE COMPAT_OBJS = compat/msvc.o compat/winansi.o \ compat/win32/pthread.o compat/win32/syslog.o \ compat/win32/dirent.o compat/win32/fscache.o - COMPAT_CFLAGS = -D__USE_MINGW_ACCESS -DNOGDI -DHAVE_STRING_H -Icompat -Icompat/regex -Icompat/win32 -DSTRIP_EXTENSION=\".exe\" + COMPAT_CFLAGS = -D__USE_MINGW_ACCESS -DDETECT_MSYS_TTY -DNOGDI -DHAVE_STRING_H -Icompat -Icompat/regex -Icompat/win32 -DSTRIP_EXTENSION=\".exe\" BASIC_LDFLAGS = -IGNORE:4217 -IGNORE:4049 -NOLOGO -SUBSYSTEM:CONSOLE - EXTLIBS = user32.lib advapi32.lib shell32.lib wininet.lib ws2_32.lib invalidcontinue.obj + EXTLIBS = user32.lib advapi32.lib shell32.lib wininet.lib ws2_32.lib invalidcontinue.obj kernel32.lib ntdll.lib PTHREAD_LIBS = lib = + # Path to the unpacked third-party libraries + MSVC_DEPS = compat/vcbuild/GEN.DEPS + # Compensate for lack of %VCINSTALLDIR%, %LIB%, %LIBPATH%, and etc. + # since vcvars*.bat did not get a chance to setup the environment of + # the user's shell window. + # + # TODO If we ask the user to launch a "x64 Native" command prompt + # TODO and then have it start a git-bash window, these could be + # TODO inherited. So we wouldn't need to add these lines here. + # + MSVC_SDK81 = "c:/Program Files (x86)/Windows Kits/8.1" + MSVC_SDK10 = "c:/Program Files (x86)/Windows Kits/10" + MSVC_VCDIR = "c:/Program Files (x86)/Microsoft Visual Studio 14.0/VC" + BASIC_CFLAGS += \ + -I$(MSVC_DEPS)/include -I$(MSVC_DEPS)/include/expat -I$(MSVC_DEPS)/include/zlib \ + -L$(MSVC_DEPS)/lib \ + -I$(MSVC_SDK81)/Include/um -I$(MSVC_SDK81)/Include/shared \ + -L$(MSVC_SDK81)/lib/winv6.3/um/x64 \ + -I$(MSVC_SDK10)/Include/10.0.10240.0/ucrt \ + -L$(MSVC_SDK10)/lib/10.0.10240.0/ucrt/x64 \ + -I$(MSVC_VCDIR)/INCLUDE \ + -L$(MSVC_VCDIR)/lib/amd64 + # Optionally enable memory leak reporting. + # BASIC_CLFAGS += -DUSE_MSVC_CRTDBG BASIC_CFLAGS += -DPROTECT_NTFS_DEFAULT=1 ifndef DEBUG BASIC_CFLAGS += -GL -Os -MD BASIC_LDFLAGS += -LTCG AR += -LTCG else - BASIC_CFLAGS += -Zi -MDd + BASIC_CFLAGS += -Zi -MDd -DDEBUG -D_DEBUG endif X = .exe diff --git a/git-compat-util.h b/git-compat-util.h index 1377cfa7b8..ca3775b73f 100644 --- a/git-compat-util.h +++ b/git-compat-util.h @@ -1,6 +1,15 @@ #ifndef GIT_COMPAT_UTIL_H #define GIT_COMPAT_UTIL_H +#ifdef USE_MSVC_CRTDBG +/* + * For these to work they must appear very early in each + * file -- before most of the standard header files. + */ +#include +#include +#endif + #define _FILE_OFFSET_BITS 64 From d0cc84fd3c59cf620d43d48358a538615f8c9166 Mon Sep 17 00:00:00 2001 From: Philip Oakley Date: Thu, 21 Apr 2016 14:07:20 +0100 Subject: [PATCH 117/288] msvc: include sigset_t definition On MSVC (VS2008) sigset_t is not defined. Signed-off-by: Philip Oakley --- compat/msvc.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compat/msvc.h b/compat/msvc.h index 580bb55bf4..ec4518eebf 100644 --- a/compat/msvc.h +++ b/compat/msvc.h @@ -24,6 +24,8 @@ static __inline int strcasecmp (const char *s1, const char *s2) #undef ERROR +typedef int sigset_t; + #include "compat/mingw.h" #endif From 6305d9f7475cbac7abf294082c287706dbd98d03 Mon Sep 17 00:00:00 2001 From: Philip Oakley Date: Thu, 21 Apr 2016 14:52:05 +0100 Subject: [PATCH 118/288] msvc: define O_ACCMODE This constant is not defined in MSVC's headers. In UCRT's fcntl.h, _O_RDONLY, _O_WRONLY and _O_RDWR are defined as 0, 1 and 2, respectively. Yes, that means that UCRT breaks with the tradition that O_RDWR == O_RDONLY | O_WRONLY. It is a perfectly legal way to define those constants, though, therefore we need to take care of defining O_ACCMODE accordingly. This is particularly important in order to keep our "open() can set errno to EISDIR" emulation working: it tests that (flags & O_ACCMODE) is not identical to O_RDONLY before going on to test specifically whether the file for which open() reported EACCES is, in fact, a directory. Signed-off-by: Philip Oakley Signed-off-by: Johannes Schindelin --- compat/msvc.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compat/msvc.h b/compat/msvc.h index ec4518eebf..ae1c58ec13 100644 --- a/compat/msvc.h +++ b/compat/msvc.h @@ -25,6 +25,8 @@ static __inline int strcasecmp (const char *s1, const char *s2) #undef ERROR typedef int sigset_t; +/* open for reading, writing, or both (not in fcntl.h) */ +#define O_ACCMODE (_O_RDONLY | _O_WRONLY | _O_RDWR) #include "compat/mingw.h" From e0ba88b534f5ae7be7d51e7c00fe0cbf3daa83b6 Mon Sep 17 00:00:00 2001 From: Philip Oakley Date: Fri, 22 Apr 2016 10:48:13 +0100 Subject: [PATCH 119/288] msvc: fix the declaration of the _REPARSE_DATA_BUFFER structure GCC and MSVC disagree about using the GCC extension _ANONYMOUS_UNION. Simply skip that offending keyword when compiling with MSVC. Signed-off-by: Philip Oakley Signed-off-by: Johannes Schindelin --- compat/mingw.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/compat/mingw.c b/compat/mingw.c index eae34e5130..df4f972840 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2310,7 +2310,10 @@ typedef struct _REPARSE_DATA_BUFFER { DWORD ReparseTag; WORD ReparseDataLength; WORD Reserved; - _ANONYMOUS_UNION union { +#ifndef _MSC_VER + _ANONYMOUS_UNION +#endif + union { struct { WORD SubstituteNameOffset; WORD SubstituteNameLength; From 73dc3f76aa8df390c404bc22ed7643feace1d281 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 3 Jun 2016 15:38:25 -0400 Subject: [PATCH 120/288] msvc: convert environment from/to UTF-16 on the fly This adds MSVC versions of getenv() and friends. These take UTF-8 arguments and return UTF-8 values, but use the UNICODE versions of the CRT routines. This avoids the need to write to __environ (which is only visible if you statically link to the CRT). This also avoids the CP_ACP conversions performed inside the CRT. It also avoids various memory leaks and problems. Signed-off-by: Jeff Hostetler --- compat/mingw.c | 221 +++++++++++++++++++++++++++++++++++++++++++++++++ compat/mingw.h | 41 +++++++++ 2 files changed, 262 insertions(+) diff --git a/compat/mingw.c b/compat/mingw.c index df4f972840..4dffb47e4b 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -1198,6 +1198,160 @@ static char *path_lookup(const char *cmd, char **path, int exe_only) return prog; } +#if defined(_MSC_VER) +/* + * Build an environment block combining the inherited environment + * merged with the given list of settings. + * + * Values of the form "KEY=VALUE" in deltaenv override inherited values. + * Values of the form "KEY" in deltaenv delete inherited values. + * + * We return a contiguous block of UNICODE strings with a final trailing + * zero word. + */ +static wchar_t *make_environment_block(char **deltaenv) +{ + /* + * The CRT (at least as of UCRT) secretly declares "_wenviron" + * as a function that returns a pointer to a mostly static table. + * Grab the pointer and cache it for the duration of our loop. + */ + extern wchar_t **_wenviron; + const wchar_t **my_wenviron = _wenviron; + + /* + * Internally, we create normal 'C' arrays of strings (pointing + * into the blocks) to help with some of the de-dup work. + */ + wchar_t **wptrs_ins = NULL; + wchar_t **wptrs_del = NULL; + wchar_t *wblock_ins = NULL; + wchar_t *wblock_del = NULL; + wchar_t *wend_ins; + wchar_t *wend_del; + wchar_t *w_ins; + wchar_t *w_del; + + int maxlen = 0; + int nr_delta = 0; + int nr_delta_ins = 0; + int nr_delta_del = 0; + int nr_wenv = 0; + int j, k, k_ins, k_del; + + /* + * Count the number of inserts and deletes in the deltaenv list. + * Allocate 'C' arrays for inserts and deletes. + * Also estimate the block size of our results. + */ + if (deltaenv && deltaenv[0] && *deltaenv[0]) { + for (k = 0; deltaenv && deltaenv[k] && *deltaenv[k]; k++) { + if (strchr(deltaenv[k], '=') == NULL) + nr_delta_del++; + else { + maxlen += strlen(deltaenv[k]) + 1; + nr_delta_ins++; + } + } + + if (nr_delta_ins) + wptrs_ins = (wchar_t**)calloc(nr_delta_ins + 1, sizeof(wchar_t*)); + if (nr_delta_del) + wptrs_del = (wchar_t**)calloc(nr_delta_del + 1, sizeof(wchar_t*)); + + nr_delta = nr_delta_ins + nr_delta_del; + } + while (my_wenviron && my_wenviron[nr_wenv] && *my_wenviron[nr_wenv]) + maxlen += wcslen(my_wenviron[nr_wenv++]) + 1; + maxlen++; + + /* + * Allocate blocks for inserted and deleted items. + * The individual pointers in the 'C' arrays will point into here. + * We will use the wblock_ins as the final result. + */ + if (nr_delta_del) { + wblock_del = (wchar_t*)calloc(maxlen, sizeof(wchar_t)); + wend_del = wblock_del + maxlen; + w_del = wblock_del; + } + wblock_ins = (wchar_t*)calloc(maxlen, sizeof(wchar_t)); + wend_ins = wblock_ins + maxlen; + w_ins = wblock_ins; + + /* + * deltaenv values override inherited environment, so put them + * in the result list first (so that we can de-dup using the + * wide versions of them. + * + * Items in the deltaenv list that DO NOT contain an "=" are + * treated as unsetenv. + * + * I'm going assume that there are no duplicates in deltaenv itself. + */ + k_ins = 0; + k_del = 0; + for (k = 0; k < nr_delta; k++) { + if (strchr(deltaenv[k], '=') == NULL) { + wptrs_del[k_del++] = w_del; + w_del += xutftowcs(w_del, deltaenv[k], (wend_del - w_del)); + *w_del++ = L'='; /* append '=' to make lookup easier in next step. */ + *w_del++ = 0; + } else { + wptrs_ins[k_ins++] = w_ins; + w_ins += xutftowcs(w_ins, deltaenv[k], (wend_ins - w_ins)) + 1; + } + } + assert(k_ins == nr_delta_ins); + assert(k_del == nr_delta_del); + + /* + * Walk the inherited environment and copy over unique, non-deleted + * ones into the result set. Note that we only have to de-dup WRT + * the values from deltaenv, because the inherited set should be unique. + */ + for (j = 0; j < nr_wenv; j++) { + const wchar_t *v_j = my_wenviron[j]; + wchar_t *v_j_eq = wcschr(v_j, L'='); + int len_j_eq, len_j; + + if (!v_j_eq) + continue; /* should not happen */ + len_j_eq = v_j_eq + 1 - v_j; /* length(v_j) including '=' */ + + /* lookup v_j in list of to-delete vars */ + for (k_del = 0; k_del < nr_delta_del; k_del++) { + if (wcsnicmp(v_j, wptrs_del[k_del], len_j_eq) == 0) + goto skip_it; + } + + /* lookup v_j in deltaenv portion of result set */ + for (k_ins = 0; k_ins < nr_delta_ins; k_ins++) { + if (wcsnicmp(v_j, wptrs_ins[k_ins], len_j_eq) == 0) + goto skip_it; + } + + /* item is unique, add it to results. */ + len_j = wcslen(v_j); + memcpy(w_ins, v_j, len_j * sizeof(wchar_t)); + w_ins += len_j + 1; + +skip_it: + ; + } + + if (wptrs_ins) + free(wptrs_ins); + if (wptrs_del) + free(wptrs_del); + if (wblock_del) + free(wblock_del); + + return wblock_ins; +} + +#else + static int do_putenv(char **env, const char *name, int size, int free_old); /* used number of elements of environ array, including terminating NULL */ @@ -1244,6 +1398,7 @@ static wchar_t *make_environment_block(char **deltaenv) free(tmpenv); return wenvblk; } +#endif static void do_unset_environment_variables(void) { @@ -1525,6 +1680,70 @@ int mingw_kill(pid_t pid, int sig) return -1; } +#if defined(_MSC_VER) + +/* UTF8 versions of getenv and putenv (and unsetenv). + * Internally, they use the CRT's stock UNICODE routines + * to avoid data loss. + * + * Unlike the mingw version, we DO NOT directly write to + * the CRT variables. We also DO NOT try to manage/replace + * the CRT storage. + */ +char *msc_getenv(const char *name) +{ + int len_key, len_value; + wchar_t *w_key; + char *value; + const wchar_t *w_value; + + if (!name || !*name) + return NULL; + + len_key = strlen(name) + 1; + w_key = calloc(len_key, sizeof(wchar_t)); + xutftowcs(w_key, name, len_key); + w_value = _wgetenv(w_key); + free(w_key); + + if (!w_value) + return NULL; + + len_value = wcslen(w_value) * 3 + 1; + value = calloc(len_value, sizeof(char)); + xwcstoutf(value, w_value, len_value); + + /* TODO Warning: We return "value" which is an allocated + * value and the caller is NOT expecting to have to free + * it, so we leak memory. + */ + return value; +} + +int msc_putenv(const char *name) +{ + int len, result; + char *equal; + wchar_t *wide; + + if (!name || !*name) + return 0; + + len = strlen(name); + equal = strchr(name, '='); + wide = calloc(len+1+!equal, sizeof(wchar_t)); + xutftowcs(wide, name, len+1); + if (!equal) + wcscat(wide, L"="); + + result = _wputenv(wide); + + free(wide); + return result; +} + +#else + /* * Compare environment entries by key (i.e. stopping at '=' or '\0'). */ @@ -1653,6 +1872,8 @@ int mingw_putenv(const char *namevalue) return 0; } +#endif + /* * Note, this isn't a complete replacement for getaddrinfo. It assumes * that service contains a numerical port, or that it is null. It diff --git a/compat/mingw.h b/compat/mingw.h index 8ba86497ab..0c3d4600c0 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -265,12 +265,53 @@ char *mingw_getcwd(char *pointer, int len); #error "NO_UNSETENV is incompatible with the MinGW startup code!" #endif +#if defined(_MSC_VER) +/* + * We bind *env() routines (even the mingw_ ones) to private msc_ versions. + * These talk to the CRT using UNICODE/wchar_t, but maintain the original + * narrow-char API. + * + * Note that the MSCRT maintains both ANSI (getenv()) and UNICODE (_wgetenv()) + * routines and stores both versions of each environment variable in parallel + * (and secretly updates both when you set one or the other), but it uses CP_ACP + * to do the conversion rather than CP_UTF8. + * + * Since everything in the git code base is UTF8, we define the msc_ routines + * to access the CRT using the UNICODE routines and manually convert them to + * UTF8. This also avoids round-trip problems. + * + * This also helps with our linkage, since "_wenviron" is publicly exported + * from the CRT. But to access "_environ" we would have to statically link + * to the CRT (/MT). + * + * We also use "wmain(argc,argv,env)" and get the initial UNICODE setup for us. + * This avoids the need for the msc_startup() to import and convert the + * inherited environment. + * + * We require NO_SETENV (and let gitsetenv() call our msc_putenv). + */ +#define getenv msc_getenv +#define putenv msc_putenv +#define unsetenv msc_putenv +#define mingw_getenv msc_getenv +#define mingw_putenv msc_putenv +char *msc_getenv(const char *name); +int msc_putenv(const char *name); + +#ifndef NO_SETENV +#error "NO_SETENV is required for MSC startup code!" +#endif + +#else + char *mingw_getenv(const char *name); #define getenv mingw_getenv int mingw_putenv(const char *namevalue); #define putenv mingw_putenv #define unsetenv mingw_putenv +#endif + int mingw_gethostname(char *host, int namelen); #define gethostname mingw_gethostname From 907871306e59f1e603b6c5c02cb79d0aeebfe69f Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 3 Jun 2016 15:38:25 -0400 Subject: [PATCH 121/288] msvc: mark a variable as non-const VS2015 complains when using a const pointer in memcpy()/free(). Signed-off-by: Jeff Hostetler --- compat/mingw.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/compat/mingw.c b/compat/mingw.c index 4dffb47e4b..b48685429c 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -1604,7 +1604,10 @@ static int try_shell_exec(const char *cmd, char *const *argv) prog = path_lookup(interpr, path, 1); if (prog) { int argc = 0; - const char **argv2; +#ifndef _MSC_VER + const +#endif + char **argv2; while (argv[argc]) argc++; ALLOC_ARRAY(argv2, argc + 1); argv2[0] = (char *)cmd; /* full path to the script file */ From 1d11ebae28b9b2c155333e650e7eae6d5411722f Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 3 Jun 2016 15:38:25 -0400 Subject: [PATCH 122/288] msvc: do not pretend to support all signals This special-cases various signals that are not supported on Windows, such as SIGPIPE. These cause the UCRT to throw asserts (at least in debug mode). Signed-off-by: Jeff Hostetler --- compat/mingw.c | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/compat/mingw.c b/compat/mingw.c index b48685429c..a5d19c4f76 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2435,8 +2435,34 @@ int mingw_raise(int sig) sigint_fn(SIGINT); return 0; +#if defined(_MSC_VER) + /* + * in the CRT defines 8 signals as being + * supported on the platform. Anything else causes + * an "Invalid signal or error" (which in DEBUG builds + * causes the Abort/Retry/Ignore dialog). We by-pass + * the CRT for things we already know will fail. + */ + /*case SIGINT:*/ + case SIGILL: + case SIGFPE: + case SIGSEGV: + case SIGTERM: + case SIGBREAK: + case SIGABRT: + case SIGABRT_COMPAT: + return raise(sig); + default: + errno = EINVAL; + return -1; + +#else + default: return raise(sig); + +#endif + } } From dcc450084d5303a141b383ac9ee41751cb9cf89d Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 10 Jan 2017 23:22:48 +0100 Subject: [PATCH 123/288] msvc: provide a main() wrapper similar to mingw_main() The MINGW version of the main() wrapper gets away with declaring symbols that were intentionally not exported. However, some of these symbols do not actually exist in MSVC's UCRT. So let's add an MSVC version of the main() wrapper that uses wmain() and imports the UNICODE argv and environment. While at it, we pass our UTF-8 version of ARGV to the real main -- rather than overwriting __argv as is done in the MINGW Version. Signed-off-by: Jeff Hostetler --- compat/mingw.c | 93 ++++++++++++++++++++++++++++++++++++++++++++++++++ compat/mingw.h | 21 ++++++++++++ 2 files changed, 114 insertions(+) diff --git a/compat/mingw.c b/compat/mingw.c index a5d19c4f76..8d93b181e4 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2941,6 +2941,7 @@ int handle_long_path(wchar_t *path, int len, int max_path, int expand) } } +#if !defined(_MSC_VER) /* * Disable MSVCRT command line wildcard expansion (__getmainargs called from * mingw startup code, see init.c in mingw runtime). @@ -2953,6 +2954,7 @@ typedef struct { extern int __wgetmainargs(int *argc, wchar_t ***argv, wchar_t ***env, int glob, _startupinfo *si); +#endif static NORETURN void die_startup(void) { @@ -3030,6 +3032,95 @@ static void maybe_redirect_std_handles(void) GENERIC_WRITE, FILE_FLAG_NO_BUFFERING); } +#if defined(_MSC_VER) + +/* + * This routine sits between wmain() and "main" in git.exe. + * We receive UNICODE (wchar_t) values for argv and env. + * + * To be more compatible with the core git code, we convert + * argv into UTF8 and pass them directly to the "main" routine. + * + * We don't bother converting the given UNICODE env vector, + * but rather leave them in the CRT. We replaced the various + * getenv/putenv routines to pull them directly from the CRT. + * + * This is unlike the MINGW version: + * [] It does the UNICODE-2-UTF8 conversion on both sets and + * stuffs the values back into the CRT using exported symbols. + * [] It also maintains a private copy of the environment and + * tries to track external changes to it. + */ +int msc_startup(int argc, wchar_t **w_argv, wchar_t **w_env) +{ + char **my_utf8_argv = NULL, **save = NULL; + char *buffer = NULL; + int maxlen; + int k, exit_status; + +#ifdef USE_MSVC_CRTDBG + _CrtSetDbgFlag(_CRTDBG_ALLOC_MEM_DF | _CRTDBG_LEAK_CHECK_DF); +#endif + + maybe_redirect_std_handles(); + + /* determine size of argv conversion buffer */ + maxlen = wcslen(_wpgmptr); + for (k = 1; k < argc; k++) + maxlen = max(maxlen, wcslen(w_argv[k])); + + /* allocate buffer (wchar_t encodes to max 3 UTF-8 bytes) */ + maxlen = 3 * maxlen + 1; + buffer = malloc_startup(maxlen); + + /* + * Create a UTF-8 version of w_argv. Also create a "save" copy + * to remember all the string pointers because parse_options() + * will remove claimed items from the argv that we pass down. + */ + ALLOC_ARRAY(my_utf8_argv, argc + 1); + ALLOC_ARRAY(save, argc + 1); + save[0] = my_utf8_argv[0] = wcstoutfdup_startup(buffer, _wpgmptr, maxlen); + for (k = 1; k < argc; k++) + save[k] = my_utf8_argv[k] = wcstoutfdup_startup(buffer, w_argv[k], maxlen); + save[k] = my_utf8_argv[k] = NULL; + + free(buffer); + + /* fix Windows specific environment settings */ + setup_windows_environment(); + + unset_environment_variables = xstrdup("PERL5LIB"); + + /* initialize critical section for waitpid pinfo_t list */ + InitializeCriticalSection(&pinfo_cs); + InitializeCriticalSection(&phantom_symlinks_cs); + + /* set up default file mode and file modes for stdin/out/err */ + _fmode = _O_BINARY; + _setmode(_fileno(stdin), _O_BINARY); + _setmode(_fileno(stdout), _O_BINARY); + _setmode(_fileno(stderr), _O_BINARY); + + /* initialize Unicode console */ + winansi_init(); + + /* init length of current directory for handle_long_path */ + current_directory_len = GetCurrentDirectoryW(0, NULL); + + /* invoke the real main() using our utf8 version of argv. */ + exit_status = msc_main(argc, my_utf8_argv); + + for (k = 0; k < argc; k++) + free(save[k]); + free(save); + free(my_utf8_argv); + + return exit_status; +} + +#else + void mingw_startup(void) { int i, maxlen, argc; @@ -3109,6 +3200,8 @@ void mingw_startup(void) current_directory_len = GetCurrentDirectoryW(0, NULL); } +#endif + int uname(struct utsname *buf) { unsigned v = (unsigned)GetVersion(); diff --git a/compat/mingw.h b/compat/mingw.h index 0c3d4600c0..d669277b68 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -686,7 +686,26 @@ extern CRITICAL_SECTION pinfo_cs; /* * A replacement of main() that adds win32 specific initialization. + * + * Note that the end of these macros are unterminated so that the + * brace group following the use of the macro is the body of the + * function. */ +#if defined(_MSC_VER) + +int msc_startup(int argc, wchar_t **w_argv, wchar_t **w_env); +extern int msc_main(int argc, const char **argv); + +#define main(c,v) dummy_decl_msc_main(void); \ +int wmain(int my_argc, \ + wchar_t **my_w_argv, \ + wchar_t **my_w_env) \ +{ \ + return msc_startup(my_argc, my_w_argv, my_w_env); \ +} \ +int msc_main(c, v) + +#else void mingw_startup(void); #define main(c,v) dummy_decl_mingw_main(void); \ @@ -698,6 +717,8 @@ int main(int argc, const char **argv) \ } \ static int mingw_main(c,v) +#endif + /* * Used by Pthread API implementation for Windows */ From a76209e83fb053e5194a26818090d54081205af3 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 3 Jun 2016 15:38:25 -0400 Subject: [PATCH 124/288] msvc: do not re-declare the timespec struct VS2015's headers already declare that struct. Signed-off-by: Jeff Hostetler --- compat/mingw.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compat/mingw.h b/compat/mingw.h index d669277b68..78ba2c9b56 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -394,11 +394,13 @@ static inline long long filetime_to_hnsec(const FILETIME *ft) #ifndef __MINGW64_VERSION_MAJOR #define off_t off64_t #define lseek _lseeki64 +#ifndef _MSC_VER struct timespec { time_t tv_sec; long tv_nsec; }; #endif +#endif static inline void filetime_to_timespec(const FILETIME *ft, struct timespec *ts) { From ce933b58debd463c6573878f794f7f8d85233372 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 3 Jun 2016 15:38:25 -0400 Subject: [PATCH 125/288] msvc: define ftello() It is just called different in MSVC's headers. Signed-off-by: Jeff Hostetler --- compat/msvc.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compat/msvc.h b/compat/msvc.h index ae1c58ec13..b3ac4cc3af 100644 --- a/compat/msvc.h +++ b/compat/msvc.h @@ -24,6 +24,8 @@ static __inline int strcasecmp (const char *s1, const char *s2) #undef ERROR +#define ftello _ftelli64 + typedef int sigset_t; /* open for reading, writing, or both (not in fcntl.h) */ #define O_ACCMODE (_O_RDONLY | _O_WRONLY | _O_RDWR) From 3e1a83a5d493321be48bde75539280960bd06a85 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 10 Jan 2017 22:53:36 +0100 Subject: [PATCH 126/288] msvc: fix detect_msys_tty() The ntstatus.h header is only available in MINGW. Signed-off-by: Jeff Hostetler --- compat/winansi.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/compat/winansi.c b/compat/winansi.c index c7ab58361b..664b672d40 100644 --- a/compat/winansi.c +++ b/compat/winansi.c @@ -513,7 +513,20 @@ static HANDLE swap_osfhnd(int fd, HANDLE new_handle) #ifdef DETECT_MSYS_TTY #include + +#if defined(_MSC_VER) + +typedef struct _OBJECT_NAME_INFORMATION +{ + UNICODE_STRING Name; + WCHAR NameBuffer[0]; +} OBJECT_NAME_INFORMATION, *POBJECT_NAME_INFORMATION; + +#define ObjectNameInformation 1 + +#else #include +#endif static void detect_msys_tty(int fd) { From 7945e2252ebb9083fb37fb1ad5f08bc5214e4d79 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 3 Jun 2016 16:01:35 -0400 Subject: [PATCH 127/288] cache-tree.c: avoid reusing the DEBUG constant In MSVC, the DEBUG constant is set automatically whenever compiling with debug information. This is clearly not what was intended in cache-tree.c, so let's use a less ambiguous constant there. Signed-off-by: Jeff Hostetler --- cache-tree.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/cache-tree.c b/cache-tree.c index 345ea35963..3e9da3f9bf 100644 --- a/cache-tree.c +++ b/cache-tree.c @@ -4,8 +4,8 @@ #include "tree-walk.h" #include "cache-tree.h" -#ifndef DEBUG -#define DEBUG 0 +#ifndef DEBUG_CACHE_TREE +#define DEBUG_CACHE_TREE 0 #endif struct cache_tree *cache_tree(void) @@ -110,7 +110,7 @@ static int do_invalidate_path(struct cache_tree *it, const char *path) int namelen; struct cache_tree_sub *down; -#if DEBUG +#if DEBUG_CACHE_TREE fprintf(stderr, "cache-tree invalidate <%s>\n", path); #endif @@ -392,7 +392,7 @@ static int update_one(struct cache_tree *it, strbuf_addf(&buffer, "%o %.*s%c", mode, entlen, path + baselen, '\0'); strbuf_add(&buffer, sha1, 20); -#if DEBUG +#if DEBUG_CACHE_TREE fprintf(stderr, "cache-tree update-one %o %.*s\n", mode, entlen, path + baselen); #endif @@ -414,7 +414,7 @@ static int update_one(struct cache_tree *it, strbuf_release(&buffer); it->entry_count = to_invalidate ? -1 : i - *skip_count; -#if DEBUG +#if DEBUG_CACHE_TREE fprintf(stderr, "cache-tree update-one (%d ent, %d subtree) %s\n", it->entry_count, it->subtree_nr, sha1_to_hex(it->sha1)); @@ -453,7 +453,7 @@ static void write_one(struct strbuf *buffer, struct cache_tree *it, strbuf_add(buffer, path, pathlen); strbuf_addf(buffer, "%c%d %d\n", 0, it->entry_count, it->subtree_nr); -#if DEBUG +#if DEBUG_CACHE_TREE if (0 <= it->entry_count) fprintf(stderr, "cache-tree <%.*s> (%d ent, %d subtree) %s\n", pathlen, path, it->entry_count, it->subtree_nr, @@ -526,7 +526,7 @@ static struct cache_tree *read_one(const char **buffer, unsigned long *size_p) size -= 20; } -#if DEBUG +#if DEBUG_CACHE_TREE if (0 <= it->entry_count) fprintf(stderr, "cache-tree <%s> (%d ent, %d subtree) %s\n", *buffer, it->entry_count, subtree_nr, From efa84c6908b9bf7c1193f170bc795cc920c661ea Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Mon, 6 Jun 2016 11:06:28 -0400 Subject: [PATCH 128/288] msvc: fix setvbuf() call The VS2015 version of the CRT asserts when you pass a zero buffer length and request line buffering. This fix sets it to the default BUFSIZ. Signed-off-by: Jeff Hostetler --- usage.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/usage.c b/usage.c index 17f52c1b5c..d81bcfe541 100644 --- a/usage.c +++ b/usage.c @@ -15,7 +15,12 @@ void vreportf(const char *prefix, const char *err, va_list params) fflush(fh); if (!tweaked_error_buffering) { +#if defined(_MSC_VER) + /* UCRT doesn't like zero buffer size */ + setvbuf(fh, NULL, _IOLBF, BUFSIZ); +#else setvbuf(fh, NULL, _IOLBF, 0); +#endif tweaked_error_buffering = 1; } From 23eecbfa79ec2931661eb35a3ca43ded4cdee884 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Thu, 9 Jun 2016 09:40:13 -0400 Subject: [PATCH 129/288] msvc: release mode PDBs and library DLLs Install required third-party DLLs next to EXEs. Build and install release mode PDBs for git executables allowing detailed stack traces in the event of crash. Signed-off-by: Jeff Hostetler --- Makefile | 22 ++++++++++++++++++++++ compat/vcbuild/Makefile | 3 ++- compat/vcbuild/scripts/clink.pl | 7 ++++--- config.mak.uname | 10 +++++++--- 4 files changed, 35 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 2a8bd86766..0c23983d7d 100644 --- a/Makefile +++ b/Makefile @@ -2378,6 +2378,28 @@ install: all $(INSTALL) $(ALL_PROGRAMS) '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' $(INSTALL) -m 644 $(SCRIPT_LIB) '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' $(INSTALL) $(install_bindir_programs) '$(DESTDIR_SQ)$(bindir_SQ)' +ifdef MSVC + $(INSTALL) compat/vcbuild/GEN.DEPS/bin/*.dll '$(DESTDIR_SQ)$(bindir_SQ)' + $(INSTALL) compat/vcbuild/GEN.DEPS/bin/*.pdb '$(DESTDIR_SQ)$(bindir_SQ)' + # We DO NOT install the individual foo.o.pdb files because they + # have already been rolled up into the exe's pdb file. + # We DO NOT have pdb files for the builtin commands (like git-status.exe) + # because it is just a copy/hardlink of git.exe, rather than a unique binary. + $(INSTALL) git.pdb '$(DESTDIR_SQ)$(bindir_SQ)' + $(INSTALL) git-shell.pdb '$(DESTDIR_SQ)$(bindir_SQ)' + $(INSTALL) git-upload-pack.pdb '$(DESTDIR_SQ)$(bindir_SQ)' + $(INSTALL) git-credential-store.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' + $(INSTALL) git-daemon.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' + $(INSTALL) git-fast-import.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' + $(INSTALL) git-http-backend.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' + $(INSTALL) git-http-fetch.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' + $(INSTALL) git-http-push.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' + $(INSTALL) git-imap-send.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' + $(INSTALL) git-remote-http.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' + $(INSTALL) git-remote-testsvn.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' + $(INSTALL) git-sh-i18n--envsubst.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' + $(INSTALL) git-show-index.pdb '$(DESTDIR_SQ)$(gitexec_instdir_SQ)' +endif $(MAKE) -C templates DESTDIR='$(DESTDIR_SQ)' install $(INSTALL) -d -m 755 '$(DESTDIR_SQ)$(mergetools_instdir_SQ)' $(INSTALL) -m 644 mergetools/* '$(DESTDIR_SQ)$(mergetools_instdir_SQ)' diff --git a/compat/vcbuild/Makefile b/compat/vcbuild/Makefile index b94bcd119a..cfca611dfc 100644 --- a/compat/vcbuild/Makefile +++ b/compat/vcbuild/Makefile @@ -106,13 +106,14 @@ openssl: insdir ZLIB_VER=1.2.8.8 ZLIB_ROOT=$(PKGDIR)/zlib.v140.windesktop.msvcstl.dyn.rt-dyn.$(ZLIB_VER) ZLIB_INC=$(ZLIB_ROOT)/build/native/include -ZLIB_LIB=$(ZLIB_ROOT)/lib/native/v140/windesktop/msvcstl/dyn/rt-dyn/x64/Release +ZLIB_LIB=$(ZLIB_ROOT)/lib/native/v140/windesktop/msvcstl/dyn/rt-dyn/x64/RelWithDebInfo zlib: insdir [ -d $(INST_INC)/zlib ] || mkdir $(INST_INC)/zlib cp -r $(ZLIB_INC)/* $(INST_INC)/zlib cp -r $(ZLIB_LIB)/*.lib $(INST_LIB)/ cp -r $(ZLIB_LIB)/*.dll $(INST_BIN)/ + cp -r $(ZLIB_LIB)/*.pdb $(INST_BIN)/ ################################################################## clean: diff --git a/compat/vcbuild/scripts/clink.pl b/compat/vcbuild/scripts/clink.pl index 3bbfa47bf4..a8df6bb63b 100755 --- a/compat/vcbuild/scripts/clink.pl +++ b/compat/vcbuild/scripts/clink.pl @@ -22,9 +22,12 @@ while (@ARGV) { my $file_out = shift @ARGV; if ("$file_out" =~ /exe$/) { $is_linking = 1; + # Create foo.exe and foo.pdb push(@args, "-OUT:$file_out"); } else { + # Create foo.o and foo.o.pdb push(@args, "-Fo$file_out"); + push(@args, "-Fd$file_out.pdb"); } } elsif ("$arg" eq "-lz") { push(@args, "zlib.lib"); @@ -49,12 +52,10 @@ while (@ARGV) { } if ($is_linking) { push(@args, @lflags); - # force PDB to be created. - push(@args, "-debug"); unshift(@args, "link.exe"); } else { unshift(@args, "cl.exe"); push(@args, @cflags); } -#printf("**** @args\n"); +printf("**** @args\n\n\n"); exit (system(@args) != 0); diff --git a/config.mak.uname b/config.mak.uname index a476fa9b87..a54b412b2b 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -425,12 +425,16 @@ ifeq ($(uname_S),Windows) # Optionally enable memory leak reporting. # BASIC_CLFAGS += -DUSE_MSVC_CRTDBG BASIC_CFLAGS += -DPROTECT_NTFS_DEFAULT=1 + # Always give "-Zi" to the compiler and "-debug" to linker (even in + # release mode) to force a PDB to be generated (like RelWithDebInfo). + BASIC_CFLAGS += -Zi + BASIC_LDFLAGS += -debug ifndef DEBUG - BASIC_CFLAGS += -GL -Os -MD - BASIC_LDFLAGS += -LTCG + BASIC_CFLAGS += -GL -Gy -Os -Oy- -MD -DNDEBUG + BASIC_LDFLAGS += -release -LTCG /OPT:REF /OPT:ICF /INCREMENTAL:NO /DEBUGTYPE:CV,FIXUP AR += -LTCG else - BASIC_CFLAGS += -Zi -MDd -DDEBUG -D_DEBUG + BASIC_CFLAGS += -MDd -DDEBUG -D_DEBUG endif X = .exe From 1a4bc8f9f8fd32a476d0d654d17b8d1fef2d7dd6 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 14 Oct 2016 06:01:39 -0700 Subject: [PATCH 130/288] msvc: respect the quiet-by-default output Signed-off-by: Johannes Schindelin --- compat/vcbuild/scripts/clink.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compat/vcbuild/scripts/clink.pl b/compat/vcbuild/scripts/clink.pl index a8df6bb63b..b67b831304 100755 --- a/compat/vcbuild/scripts/clink.pl +++ b/compat/vcbuild/scripts/clink.pl @@ -57,5 +57,5 @@ if ($is_linking) { unshift(@args, "cl.exe"); push(@args, @cflags); } -printf("**** @args\n\n\n"); +printf(STDERR "**** @args\n\n\n") if (!defined($ENV{'QUIET_GEN'})); exit (system(@args) != 0); From cc06c6179df672b2471f8630b58d4d175dd5ad99 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 12 Oct 2016 14:21:42 -0400 Subject: [PATCH 131/288] msvc: use OpenSSL's SHA-1 routines Just like 1e2ce1d (sha1: Use OpenSSL SHA1 routines on MINGW, 2016-10-12), we now use OpenSSL's SHA-1 routines instead of Git's own because OpenSSL is substantially faster as of version 1.0.2: it now uses hardware acceleration on Intel processors much more effectively. Signed-off-by: Johannes Schindelin --- config.mak.uname | 1 - 1 file changed, 1 deletion(-) diff --git a/config.mak.uname b/config.mak.uname index a54b412b2b..c25788a97f 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -380,7 +380,6 @@ ifeq ($(uname_S),Windows) NO_REGEX = YesPlease NO_GETTEXT = YesPlease NO_PYTHON = YesPlease - BLK_SHA1 = YesPlease ETAGS_TARGET = ETAGS NO_INET_PTON = YesPlease NO_INET_NTOP = YesPlease From b005fc8da329b05b86d91906d75618942b136999 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 20 Oct 2016 08:21:05 -0700 Subject: [PATCH 132/288] msvc: use libiconv Signed-off-by: Johannes Schindelin --- compat/vcbuild/Makefile | 22 +++++++++++++++++++++- compat/vcbuild/packages.config | 3 +++ config.mak.uname | 3 +-- 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/compat/vcbuild/Makefile b/compat/vcbuild/Makefile index cfca611dfc..a9d1a6b92b 100644 --- a/compat/vcbuild/Makefile +++ b/compat/vcbuild/Makefile @@ -9,7 +9,8 @@ INST_BIN=$(INST)/bin PKGDIR=GEN.PKGS ################################################################## -all: unpack expat libssh libssh_redist curl curl_redist openssl zlib +all: unpack expat libssh libssh_redist curl curl_redist openssl zlib \ + libiconv libiconv_redist unpack: [ -d $(PKGDIR) ] || mkdir $(PKGDIR) @@ -38,6 +39,25 @@ expat: insdir cp -r $(EXPAT_LIB)/* $(INST_LIB)/ +################################################################## + +LIBICONV_VER=1.14.0.11 +LIBICONV_ROOT=$(PKGDIR)/libiconv.$(LIBICONV_VER)/build/native +LIBICONV_INC=$(LIBICONV_ROOT)/include +LIBICONV_LIB=$(LIBICONV_ROOT)/lib/v110/x64/Release/dynamic/cdecl + +libiconv: insdir + cp -r $(LIBICONV_INC)/* $(INST_INC)/ + cp -r $(LIBICONV_LIB)/libiconv.lib $(INST_LIB)/iconv.lib + + +LIBICONV_REDIST_ROOT=$(PKGDIR)/libiconv.redist.$(LIBICONV_VER)/build/native +LIBICONV_REDIST_BIN=$(LIBICONV_REDIST_ROOT)/bin/v110/x64/Release/dynamic/cdecl + +libiconv_redist: insdir + cp -r $(LIBICONV_REDIST_BIN)/* $(INST_BIN)/ + + ################################################################## LIBSSH_VER=1.4.3.3 diff --git a/compat/vcbuild/packages.config b/compat/vcbuild/packages.config index e6e6d2f63f..ee01be5edb 100644 --- a/compat/vcbuild/packages.config +++ b/compat/vcbuild/packages.config @@ -5,6 +5,9 @@ + + + diff --git a/config.mak.uname b/config.mak.uname index c25788a97f..203851bd3e 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -358,8 +358,7 @@ ifeq ($(uname_S),Windows) NO_STRCASESTR = YesPlease NO_STRLCPY = YesPlease NO_MEMMEM = YesPlease - # NEEDS_LIBICONV = YesPlease - NO_ICONV = YesPlease + NEEDS_LIBICONV = YesPlease NO_STRTOUMAX = YesPlease NO_MKDTEMP = YesPlease NO_MKSTEMPS = YesPlease From b877d99854ddaf68ca10805d57905fa5eef058e3 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 21 Oct 2016 02:46:59 -0700 Subject: [PATCH 133/288] msvc: work around iconv() not setting errno When compiling with MSVC, we rely on NuPkgs to provide the binaries of dependencies such as libiconv. The libiconv 1.14.0.11 package available from https://www.nuget.org/packages/libiconv seems to have a bug where it does not set errno (when we would expect it to be E2BIG). Let's simulate the error condition by taking less than 16 bytes remaining in the out buffer as an indicator that we ran out of space. While 16 might seem a bit excessive (when converting from, say, any encoding to UTF-8, 8 bytes should be fine), it is designed to be a safe margin. Signed-off-by: Johannes Schindelin --- git-compat-util.h | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/git-compat-util.h b/git-compat-util.h index ca3775b73f..8a69331241 100644 --- a/git-compat-util.h +++ b/git-compat-util.h @@ -278,6 +278,33 @@ extern char *gitdirname(char *); #ifndef NO_ICONV #include +#ifdef _MSC_VER +/* + * At least version 1.14.0.11 of the libiconv NuPkg at + * https://www.nuget.org/packages/libiconv/ does not set errno at all. + * + * Let's simulate it by testing whether we might have possibly run out of + * space. + */ +static inline size_t msvc_iconv(iconv_t conv, + const char **inpos, size_t *insize, + char **outpos, size_t *outsize) +{ + int saved_errno = errno; + size_t res; + + errno = ENOENT; + res = iconv(conv, inpos, insize, outpos, outsize); + if (!res) + errno = saved_errno; + else if (errno == ENOENT) + errno = *outsize < 16 ? E2BIG : 0; + + return res; +} +#undef iconv +#define iconv msvc_iconv +#endif #endif #ifndef NO_OPENSSL From 73db0c9898000149333cbde402bf2f5598471654 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sun, 23 Oct 2016 01:22:20 -0700 Subject: [PATCH 134/288] t7800: fix quoting When passing a command-line to call an external diff command to the difftool, we must be prepared for paths containing special characters, e.g. backslashes in the temporary directory's path on Windows. This has been caught by running the test suite with an MSVC-built Git: in contrast to the MINGW one, it does not rewrite `$TMP` to use forward slashes instead of backslashes. Signed-off-by: Johannes Schindelin --- t/t7800-difftool.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/t7800-difftool.sh b/t/t7800-difftool.sh index 99d4123461..77d58cf509 100755 --- a/t/t7800-difftool.sh +++ b/t/t7800-difftool.sh @@ -320,7 +320,7 @@ test_expect_success PERL 'difftool --extcmd cat arg1' ' test_expect_success PERL 'difftool --extcmd cat arg2' ' echo branch >expect && git difftool --no-prompt \ - --extcmd sh\ -c\ \"cat\ \$2\" branch >actual && + --extcmd sh\ -c\ \"cat\ \\\"\$2\\\"\" branch >actual && test_cmp expect actual ' From 02242c9c6c459bcaf664dc448a4ca635b88537b2 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Tue, 8 Nov 2016 11:18:43 -0500 Subject: [PATCH 135/288] vs2015: turn on optimize-for-speed in release build Set -O2 (maximize speed) rather than -Os (favor small code) for non-debug builds. Signed-off-by: Jeff Hostetler --- config.mak.uname | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.mak.uname b/config.mak.uname index 203851bd3e..d455929efd 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -428,7 +428,7 @@ ifeq ($(uname_S),Windows) BASIC_CFLAGS += -Zi BASIC_LDFLAGS += -debug ifndef DEBUG - BASIC_CFLAGS += -GL -Gy -Os -Oy- -MD -DNDEBUG + BASIC_CFLAGS += -GL -Gy -O2 -Oy- -MD -DNDEBUG BASIC_LDFLAGS += -release -LTCG /OPT:REF /OPT:ICF /INCREMENTAL:NO /DEBUGTYPE:CV,FIXUP AR += -LTCG else From f91eb61dc6a15f447f1c95df99c1141aed49a308 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Tue, 8 Nov 2016 11:39:35 -0500 Subject: [PATCH 136/288] vs2015: teach 'make clean' to delete PDBs Signed-off-by: Jeff Hostetler --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index 0c23983d7d..3632167c3a 100644 --- a/Makefile +++ b/Makefile @@ -2567,6 +2567,9 @@ endif $(RM) GIT-VERSION-FILE GIT-CFLAGS GIT-LDFLAGS GIT-BUILD-OPTIONS $(RM) GIT-USER-AGENT GIT-PREFIX $(RM) GIT-SCRIPT-DEFINES GIT-PERL-DEFINES GIT-PYTHON-VARS +ifdef MSVC + $(RM) $(patsubst %.o,%.o.pdb,$(OBJECTS)) +endif .PHONY: all install profile-clean clean strip .PHONY: shell_compatibility_test please_set_SHELL_PATH_to_a_more_modern_shell From 9a5db8d63cf3138be18a9b050f32c025de53adf2 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Tue, 8 Nov 2016 11:41:45 -0500 Subject: [PATCH 137/288] vs2015: remove todo list item from README_vs2015.txt Remove todo list item for vs2015 build notes regarding the third party DLLs. Signed-off-by: Jeff Hostetler --- compat/vcbuild/README_VS2015.txt | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/compat/vcbuild/README_VS2015.txt b/compat/vcbuild/README_VS2015.txt index 763523fbb4..a9c6f59dc6 100644 --- a/compat/vcbuild/README_VS2015.txt +++ b/compat/vcbuild/README_VS2015.txt @@ -51,8 +51,4 @@ TODO List: developer command prompt is started. This would also allow us to switch between 32- and 64-bit tool chains. -[B] Currently, we leave the third-party DLLs we reference in - "compat/vcbuild/GEN.DEPS/bin". We need an installer - step to move them next to git.exe (or into libexec/git-core). - -[C] We need to build SLN or VCPROJ files. +[B] We need to build SLN or VCPROJ files. From 70e8a57f9818094d9d5bac104057421410d006af Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 9 Nov 2016 15:04:22 +0100 Subject: [PATCH 138/288] compat/vcbuild: possibly reuse Git for Windows' SDK's NuGet In Git for Windows' SDK, there is already a script to package Git for Windows as a NuGet package, downloading nuget.exe if needed. Let's just fall back to using that executable (if it is there) if nuget.exe was not found in the PATH. Signed-off-by: Johannes Schindelin --- compat/vcbuild/Makefile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/compat/vcbuild/Makefile b/compat/vcbuild/Makefile index a9d1a6b92b..255dd43084 100644 --- a/compat/vcbuild/Makefile +++ b/compat/vcbuild/Makefile @@ -7,6 +7,10 @@ INST_LIB=$(INST)/lib INST_BIN=$(INST)/bin PKGDIR=GEN.PKGS +NUGET ?= nuget.exe +ifneq ($(shell $(NUGET) help 2>/dev/null; echo $$?),0) + NUGET := /usr/src/build-extra/nuget/nuget.exe +endif ################################################################## all: unpack expat libssh libssh_redist curl curl_redist openssl zlib \ @@ -14,7 +18,7 @@ all: unpack expat libssh libssh_redist curl curl_redist openssl zlib \ unpack: [ -d $(PKGDIR) ] || mkdir $(PKGDIR) - nuget.exe restore packages.config -ConfigFile nuget.config \ + "$(NUGET)" restore packages.config -ConfigFile nuget.config \ -NonInteractive -OutputDirectory $(PKGDIR) insdir: From 6ea7e234f050941880f68a65751c65f5b2d6be94 Mon Sep 17 00:00:00 2001 From: Philip Oakley Date: Mon, 20 Jul 2015 16:44:59 +0100 Subject: [PATCH 139/288] engine.pl: ignore invalidcontinue.obj which is known to MSVC Commit 4b623d8 (MSVC: link in invalidcontinue.obj for better POSIX compatibility, 2014-03-29) introduced invalidcontinue.obj into the Makefile output, which was not parsed correctly by the buildsystem. Ignore it, as it is known to Visual Studio and, there is no matching source file. Only substitute filenames ending with .o when generating the source .c filename, otherwise a .cbj file may be expected. Split the .o and .obj processing; 'make' does not produce .obj files. In the future there may be source files that produce .obj files so keep the two issues (.obj files with & without source files) separate. Signed-off-by: Philip Oakley Signed-off-by: Duncan Smart (cherry picked from commit d01d71fe1aed67f4e3a5ab80eeadeaf525ad0846) --- contrib/buildsystems/engine.pl | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/contrib/buildsystems/engine.pl b/contrib/buildsystems/engine.pl index 23da787dc5..53e65d4db7 100755 --- a/contrib/buildsystems/engine.pl +++ b/contrib/buildsystems/engine.pl @@ -282,7 +282,7 @@ sub handleLibLine # exit(1); foreach (@objfiles) { my $sourcefile = $_; - $sourcefile =~ s/\.o/.c/; + $sourcefile =~ s/\.o$/.c/; push(@sources, $sourcefile); push(@cflags, @{$compile_options{"${sourcefile}_CFLAGS"}}); push(@defines, @{$compile_options{"${sourcefile}_DEFINES"}}); @@ -326,8 +326,12 @@ sub handleLinkLine } elsif ($part =~ /\.(a|lib)$/) { $part =~ s/\.a$/.lib/; push(@libs, $part); - } elsif ($part =~ /\.(o|obj)$/) { + } elsif ($part eq 'invalidcontinue.obj') { + # ignore - known to MSVC + } elsif ($part =~ /\.o$/) { push(@objfiles, $part); + } elsif ($part =~ /\.obj$/) { + # do nothing, 'make' should not be producing .obj, only .o files } else { die "Unhandled lib option @ line $lineno: $part"; } @@ -336,7 +340,7 @@ sub handleLinkLine # exit(1); foreach (@objfiles) { my $sourcefile = $_; - $sourcefile =~ s/\.o/.c/; + $sourcefile =~ s/\.o$/.c/; push(@sources, $sourcefile); push(@cflags, @{$compile_options{"${sourcefile}_CFLAGS"}}); push(@defines, @{$compile_options{"${sourcefile}_DEFINES"}}); From b2287c7278bf9b4669a02bd324d985d6c6bd113f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=B0smail=20D=C3=B6nmez?= Date: Sat, 16 Jan 2016 18:59:31 +0200 Subject: [PATCH 140/288] Don't let ld strip relocations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is the first step for enabling ASLR (Address Space Layout Randomization) support. We want to enable ASLR for better protection against exploiting security holes in Git. The problem fixed by this commit is that `ld.exe` seems to be stripping relocations which in turn will break ASLR support. We just make sure it's not stripping the main executable entry. Signed-off-by: İsmail Dönmez Signed-off-by: Johannes Schindelin --- config.mak.uname | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config.mak.uname b/config.mak.uname index b232908f8c..14c6a8d326 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -553,9 +553,11 @@ else prefix = /usr/ ifeq (MINGW32,$(MSYSTEM)) prefix = /mingw32 + BASIC_LDFLAGS += -Wl,--pic-executable,-e,_mainCRTStartup endif ifeq (MINGW64,$(MSYSTEM)) prefix = /mingw64 + BASIC_LDFLAGS += -Wl,--pic-executable,-e,mainCRTStartup else COMPAT_CFLAGS += -D_USE_32BIT_TIME_T BASIC_LDFLAGS += -Wl,--large-address-aware From 038f82022bf05a89496f1ed1c080c38d3cf6a19a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=B0smail=20D=C3=B6nmez?= Date: Sat, 16 Jan 2016 19:09:34 +0200 Subject: [PATCH 141/288] Enable DEP and ASLR MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Enable DEP (Data Execution Prevention) and ASLR (Address Space Layout Randomization) support. This applies to both 32bit and 64bit builds and makes it substantially harder to exploit security holes in Git by offering a much more unpredictable attack surface. ASLR interferes with GDB's ability to set breakpoints. A similar issue holds true when compiling with -O2 (in which case single-stepping is messed up because GDB cannot map the code back to the original source code properly). Therefore we simply enable ASLR only when an optimization flag is present in the CFLAGS, using it as an indicator that the developer does not want to debug in GDB anyway. Signed-off-by: İsmail Dönmez Signed-off-by: Johannes Schindelin --- config.mak.uname | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/config.mak.uname b/config.mak.uname index 14c6a8d326..af119bcdfb 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -551,6 +551,12 @@ else ifeq ($(shell expr "$(uname_R)" : '2\.'),2) # MSys2 prefix = /usr/ + # Enable DEP + BASIC_LDFLAGS += -Wl,--nxcompat + # Enable ASLR (unless debugging) + ifneq (,$(findstring -O,$(CFLAGS))) + BASIC_LDFLAGS += -Wl,--dynamicbase + endif ifeq (MINGW32,$(MSYSTEM)) prefix = /mingw32 BASIC_LDFLAGS += -Wl,--pic-executable,-e,_mainCRTStartup From 24ab046eb8ffb9cd7ad9277624db3e1d66cc88ff Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 16 Feb 2016 16:31:12 +0100 Subject: [PATCH 142/288] remove_dirs: do not swallow error when stat() failed Without an error message when stat() failed, e.g. `git clean` would abort without an error message, leaving the user quite puzzled. This fixes https://github.com/git-for-windows/git/issues/521 Signed-off-by: Johannes Schindelin --- builtin/clean.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/builtin/clean.c b/builtin/clean.c index 0371010afb..179aa3a2af 100644 --- a/builtin/clean.c +++ b/builtin/clean.c @@ -192,7 +192,8 @@ static int remove_dirs(struct strbuf *path, const char *prefix, int force_flag, strbuf_setlen(path, len); strbuf_addstr(path, e->d_name); if (lstat(path->buf, &st)) - ; /* fall thru */ + warning("Could not stat path '%s': %s", + path->buf, strerror(errno)); else if (S_ISDIR(st.st_mode)) { if (remove_dirs(path, prefix, force_flag, dry_run, quiet, &gone)) ret = 1; From 7cb2d6f554802a579c295671b39b89c29faac3bb Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 16 Feb 2016 16:36:10 +0100 Subject: [PATCH 143/288] t7300: `git clean -dfx` must show an error with long paths In particular on Windows, where the default maximum path length is quite small, but there are ways to circumvent that limit in many cases, it is very important that users be given an indication why their command failed because of too long paths when it did. This test case makes sure that a warning is issued that would have helped the user who reported Git for Windows' issue 521: https://github.com/git-for-windows/git/issues/521 Signed-off-by: Johannes Schindelin --- t/t7300-clean.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/t/t7300-clean.sh b/t/t7300-clean.sh index b89fd2a6ad..62cb12affd 100755 --- a/t/t7300-clean.sh +++ b/t/t7300-clean.sh @@ -653,4 +653,15 @@ test_expect_success 'git clean -d respects pathspecs (pathspec is prefix of dir) test_path_is_dir foobar ' +test_expect_success MINGW 'handle clean & core.longpaths = false nicely' ' + git config core.longpaths false && + test_when_finished git config --unset core.longpaths && + a50=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa && + mkdir -p $a50$a50/$a50$a50/$a50$a50 && + touch $a50$a50/test.txt && + touch $a50$a50/$a50$a50/$a50$a50/test.txt && + test_must_fail git clean -xdf 2>.git/err && + grep "too long" .git/err +' + test_done From 231462d4bff5e7b59d3dd2a97f36ed1a8e64726b Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 16 Feb 2016 21:03:02 +0100 Subject: [PATCH 144/288] git gui: fix staging a second line to a 1-line file When a 1-line file is augmented by a second line, and the user tries to stage that single line via the "Stage Line" context menu item, we do not want to see "apply: corrupt patch at line 5". The reason for this error was that the hunk header looks like this: @@ -1 +1,2 @@ but the existing code expects the original range always to contain a comma. This problem is easily fixed by cutting the string "1 +1,2" (that Git GUI formerly mistook for the starting line) at the space. This fixes https://github.com/git-for-windows/git/issues/515 Signed-off-by: Johannes Schindelin --- git-gui/lib/diff.tcl | 1 + 1 file changed, 1 insertion(+) diff --git a/git-gui/lib/diff.tcl b/git-gui/lib/diff.tcl index 4cae10a4c7..68c4a6c736 100644 --- a/git-gui/lib/diff.tcl +++ b/git-gui/lib/diff.tcl @@ -698,6 +698,7 @@ proc apply_range_or_line {x y} { set hh [$ui_diff get $i_l "$i_l + 1 lines"] set hh [lindex [split $hh ,] 0] set hln [lindex [split $hh -] 1] + set hln [lindex [split $hln " "] 0] # There is a special situation to take care of. Consider this # hunk: From f01f0a7193be05d8244fdccfd84e8939ae9bb097 Mon Sep 17 00:00:00 2001 From: yaras Date: Tue, 1 Mar 2016 16:12:23 +0100 Subject: [PATCH 145/288] Do not mask the username when reading credentials When user is asked for credentials there is no need to mask username, so PROMPT_ASKPASS flag on calling credential_ask_one for login is unnecessary. credential_ask_one internally uses git_prompt which in case of given flag PROMPT_ASKPASS uses masked input method instead of git_terminal_prompt, which does not mask user input. This fixes #675 Signed-off-by: yaras --- credential.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/credential.c b/credential.c index aa996669fc..1ba5ce6d02 100644 --- a/credential.c +++ b/credential.c @@ -135,7 +135,9 @@ static void credential_getpass(struct credential *c) { if (!c->username) c->username = credential_ask_one("Username", c, - PROMPT_ASKPASS|PROMPT_ECHO); + (getenv("GIT_ASKPASS") ? + PROMPT_ASKPASS : 0) | + PROMPT_ECHO); if (!c->password) c->password = credential_ask_one("Password", c, PROMPT_ASKPASS); From 13deadc0e13b65b73c68a7267ca1ff454e896009 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sat, 2 Apr 2016 13:05:08 +0200 Subject: [PATCH 146/288] mingw: support spawning programs containing spaces in their names The CreateProcessW() function does not really support spaces in its first argument, lpApplicationName. But it supports passing NULL as lpApplicationName, which makes it figure out the application from the (possibly quoted) first argument of lpCommandLine. Let's use that trick (if we are certain that the first argument matches the executable's path) to support launching programs whose path contains spaces. This fixes https://github.com/git-for-windows/git/issue/692 Signed-off-by: Johannes Schindelin --- compat/mingw.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index f4001bc8cb..6d7189313e 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -1504,7 +1504,9 @@ static pid_t mingw_spawnve_fd(const char *cmd, const char **argv, char **deltaen si.hStdError = winansi_get_osfhandle(fherr); /* executables and the current directory don't support long paths */ - if (xutftowcs_path(wcmd, cmd) < 0) + if (*argv && !strcmp(cmd, *argv)) + wcmd[0] = L'\0'; + else if (xutftowcs_path(wcmd, cmd) < 0) return -1; if (dir && xutftowcs_path(wdir, dir) < 0) return -1; @@ -1543,8 +1545,8 @@ static pid_t mingw_spawnve_fd(const char *cmd, const char **argv, char **deltaen wenvblk = make_environment_block(deltaenv); memset(&pi, 0, sizeof(pi)); - ret = CreateProcessW(wcmd, wargs, NULL, NULL, TRUE, flags, - wenvblk, dir ? wdir : NULL, &si, &pi); + ret = CreateProcessW(*wcmd ? wcmd : NULL, wargs, NULL, NULL, TRUE, + flags, wenvblk, dir ? wdir : NULL, &si, &pi); free(wenvblk); free(wargs); From f1c2f449653d99a37671c204fd55a64229be94e1 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 8 Jun 2016 08:32:21 +0200 Subject: [PATCH 147/288] gc/repack: release packs when needed On Windows, files cannot be removed nor renamed if there are still handles held by a process. To remedy that, we introduced the close_all_packs() function. Earlier, we made sure that the packs are released just before `git gc` is spawned, in case that gc wants to remove no-longer needed packs. But this developer forgot that gc itself also needs to let go of packs, e.g. when consolidating all packs via the --aggressive option. Likewise, `git repack -d` wants to delete obsolete packs and therefore needs to close all pack handles, too. Signed-off-by: Johannes Schindelin --- builtin/gc.c | 4 +++- builtin/repack.c | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/builtin/gc.c b/builtin/gc.c index 331f219260..eed0f4e363 100644 --- a/builtin/gc.c +++ b/builtin/gc.c @@ -441,8 +441,10 @@ int cmd_gc(int argc, const char **argv, const char *prefix) report_garbage = report_pack_garbage; reprepare_packed_git(); - if (pack_garbage.nr > 0) + if (pack_garbage.nr > 0) { + close_all_packs(); clean_pack_garbage(); + } if (auto_gc && too_many_loose_objects()) warning(_("There are too many unreachable loose objects; " diff --git a/builtin/repack.c b/builtin/repack.c index 677bc7c81a..29677664e3 100644 --- a/builtin/repack.c +++ b/builtin/repack.c @@ -298,6 +298,8 @@ int cmd_repack(int argc, const char **argv, const char *prefix) if (!names.nr && !quiet) printf("Nothing new to pack.\n"); + close_all_packs(); + /* * Ok we have prepared all new packfiles. * First see if there are packs of the same name and if so From 44b67a4ae503c1832395cfde5324273082e23732 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABl=20Lhez?= Date: Wed, 22 Jun 2016 23:30:18 +0200 Subject: [PATCH 148/288] `git bundle create ` leaks handle the revlist is empty. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit issue #790: git bundle create does not close handle to *.lock file This problem happens when an user tries to create an empty bundle, using the following command: `git bundle create ` and when resolve to an empty list (for example, like `master..master`), `git bundle` fails and warn the user about how it don't want to create empty bundle. In that case, git tries to delete the `.lock` file, and since there's still an open file handle, fails to do so and ask the user if it should retry (which will fail again). The lock can still be deleted manually by the user (and it is required if the user want to create a bundle after revising his rev-list). Signed-off-by: Gaël Lhez --- bundle.c | 7 ++++--- t/tgfw790-git-bundle.sh | 15 +++++++++++++++ 2 files changed, 19 insertions(+), 3 deletions(-) create mode 100644 t/tgfw790-git-bundle.sh diff --git a/bundle.c b/bundle.c index bbf4efa0a0..da45536224 100644 --- a/bundle.c +++ b/bundle.c @@ -447,10 +447,11 @@ int create_bundle(struct bundle_header *header, const char *path, object_array_remove_duplicates(&revs.pending); ref_count = write_bundle_refs(bundle_fd, &revs); - if (!ref_count) - die(_("Refusing to create empty bundle.")); - else if (ref_count < 0) + if (ref_count <= 0) { + if (!ref_count) + error(_("Refusing to create empty bundle.")); goto err; + } /* write pack */ if (write_pack_data(bundle_fd, &revs)) { diff --git a/t/tgfw790-git-bundle.sh b/t/tgfw790-git-bundle.sh new file mode 100644 index 0000000000..6bfa23c141 --- /dev/null +++ b/t/tgfw790-git-bundle.sh @@ -0,0 +1,15 @@ +#!/bin/sh +test_description=' + test git-bundle under git for Windows + + When we select an empty set of commit (like git bundle create foobar.bundle master..master), + we should not have problem with the foobar.bundle.lock being locked (especially on Windows). +' + +. ./test-lib.sh + +test_expect_failure 'try to create a bundle with empty ref count' ' + git bundle create foobar.bundle master..master +' + +test_done From e905a7d8ffe023968d38057f58a8f705d69fd85c Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 12 Aug 2016 10:54:26 +0200 Subject: [PATCH 149/288] status: offer *not* to lock the index and update it When a third-party tool periodically runs `git status` in order to keep track of the state of the working tree, it is a bad idea to lock the index: it might interfere with interactive commands executed by the user, e.g. when the user wants to commit files. Let's introduce the option `--no-lock-index` to prevent such problems. The idea is that the third-party tool calls `git status` with this option, preventing it from ever updating the index. The downside is that the periodic `git status` calls will be a little bit more wasteful because they may have to refresh the index repeatedly, only to throw away the updates when it exits. This cannot really be helped, though, as tools wanting to get a periodic update of the status have no way to predict when the user may want to lock the index herself. Note that the regression test added in this commit does not *really* verify that no index.lock file was written; that test is not possible in a portable way. Instead, we verify that .git/index is rewritten *only* when `git status` is run without `--no-lock-index`. Signed-off-by: Johannes Schindelin --- Documentation/git-status.txt | 5 +++++ builtin/commit.c | 5 ++++- t/t7508-status.sh | 11 +++++++++++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/Documentation/git-status.txt b/Documentation/git-status.txt index 725065ef2d..5602325d1f 100644 --- a/Documentation/git-status.txt +++ b/Documentation/git-status.txt @@ -108,6 +108,11 @@ configuration variable documented in linkgit:git-config[1]. without options are equivalent to 'always' and 'never' respectively. +--no-lock-index:: +--lock-index:: + Specifies whether `git status` should try to lock the index and + update it afterwards if any changes were detected. Defaults to + `--lock-index`. OUTPUT ------ diff --git a/builtin/commit.c b/builtin/commit.c index 276c74034e..9499c32198 100644 --- a/builtin/commit.c +++ b/builtin/commit.c @@ -1330,6 +1330,7 @@ static int git_status_config(const char *k, const char *v, void *cb) int cmd_status(int argc, const char **argv, const char *prefix) { + static int no_lock_index = 0; static struct wt_status s; int fd; unsigned char sha1[20]; @@ -1357,6 +1358,8 @@ int cmd_status(int argc, const char **argv, const char *prefix) N_("ignore changes to submodules, optional when: all, dirty, untracked. (Default: all)"), PARSE_OPT_OPTARG, NULL, (intptr_t)"all" }, OPT_COLUMN(0, "column", &s.colopts, N_("list untracked files in columns")), + OPT_BOOL(0, "no-lock-index", &no_lock_index, + N_("do not lock the index")), OPT_END(), }; @@ -1380,7 +1383,7 @@ int cmd_status(int argc, const char **argv, const char *prefix) read_cache_preload(&s.pathspec); refresh_index(&the_index, REFRESH_QUIET|REFRESH_UNMERGED, &s.pathspec, NULL, NULL); - fd = hold_locked_index(&index_lock, 0); + fd = no_lock_index ? -1 : hold_locked_index(&index_lock, 0); s.is_initial = get_sha1(s.reference, sha1) ? 1 : 0; if (!s.is_initial) diff --git a/t/t7508-status.sh b/t/t7508-status.sh index fb00e6d9b0..fdf99fa50e 100755 --- a/t/t7508-status.sh +++ b/t/t7508-status.sh @@ -1499,4 +1499,15 @@ test_expect_success 'git commit -m will commit a staged but ignored submodule' ' git config -f .gitmodules --remove-section submodule.subname ' +test_expect_success '--no-lock-index' ' + test_commit some-file && + test-chmtime =1234567890 .git/index && + git status --no-lock-index && + test-chmtime -v +0 .git/index >out && + grep ^1234567890 out && + git status && + test-chmtime -v +0 .git/index >out && + ! grep ^1234567890 out +' + test_done From 17a80dddefa32ee54a067e8bae4c3fe94386beb3 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 10 Jan 2017 23:30:13 +0100 Subject: [PATCH 150/288] mingw: respect core.hidedotfiles = false in git-init again This is a brown paper bag. When adding the tests, we actually failed to verify that the config variable is heeded in git-init at all. And when changing the original patch that marked the .git/ directory as hidden after reading the config, it was lost on this developer that the new code would use the hide_dotfiles variable before the config was read. The fix is obvious: read the (limited, pre-init) config *before* creating the .git/ directory. This fixes https://github.com/git-for-windows/git/issues/789 Signed-off-by: Johannes Schindelin --- builtin/init-db.c | 6 ++++++ t/t0001-init.sh | 12 ++++++++++++ 2 files changed, 18 insertions(+) diff --git a/builtin/init-db.c b/builtin/init-db.c index 2399b97d90..b73bd20d08 100644 --- a/builtin/init-db.c +++ b/builtin/init-db.c @@ -153,6 +153,9 @@ static int git_init_db_config(const char *k, const char *v, void *cb) if (!strcmp(k, "init.templatedir")) return git_config_pathname(&init_db_template_dir, k, v); + if (starts_with(k, "core.")) + return platform_core_config(k, v); + return 0; } @@ -359,6 +362,9 @@ int init_db(const char *git_dir, const char *real_git_dir, } startup_info->have_repository = 1; + /* Just look for `init.templatedir` and `core.hidedotfiles` */ + git_config(git_init_db_config, NULL); + safe_create_dir(git_dir, 0); init_is_bare_repository = is_bare_repository(); diff --git a/t/t0001-init.sh b/t/t0001-init.sh index 63ff011fc0..43c361f006 100755 --- a/t/t0001-init.sh +++ b/t/t0001-init.sh @@ -410,6 +410,18 @@ test_expect_success 're-init from a linked worktree' ' ) ' +test_expect_success MINGW 'core.hidedotfiles = false' ' + git config --global core.hidedotfiles false && + rm -rf newdir && + ( + unset GIT_DIR GIT_WORK_TREE GIT_CONFIG + mkdir newdir && + cd newdir && + git init + ) && + ! is_hidden newdir/.git +' + test_expect_success MINGW 'redirect std handles' ' GIT_REDIRECT_STDOUT=output.txt git rev-parse --git-dir && test .git = "$(cat output.txt)" && From bb0d489ee36fbf0d9e91366b588173f73883194a Mon Sep 17 00:00:00 2001 From: Adric Norris Date: Wed, 31 Aug 2016 16:09:14 -0500 Subject: [PATCH 151/288] Preliminary support for reporting build platform Add preliminary support for detection of the build plaform, and reporting of same with the `git version --build-options' command. This can be useful for bug reporting, to distinguish between 32 and 64-bit builds for example. The current implementation can only distinguish between x86 and x86_64. This will be extended in future patches. In addition, all 32-bit variants (i686, i586, etc.) are collapsed into `x86'. An example of the output is: $ git version --build-options git version 2.9.3.windows.2.826.g06c0f2f sizeof-long: 4 machine: x86_64 The label of `machine' was chosen so the new information will approximate the output of `uname -m'. Signed-off-by: Adric Norris --- help.c | 3 +++ help.h | 13 +++++++++++++ 2 files changed, 16 insertions(+) diff --git a/help.c b/help.c index 53e2a67e00..a17ed4aef7 100644 --- a/help.c +++ b/help.c @@ -409,6 +409,8 @@ const char *help_unknown_cmd(const char *cmd) int cmd_version(int argc, const char **argv, const char *prefix) { + static char build_platform[] = GIT_BUILD_PLATFORM; + /* * The format of this string should be kept stable for compatibility * with external projects that rely on the output of "git version". @@ -417,6 +419,7 @@ int cmd_version(int argc, const char **argv, const char *prefix) while (*++argv) { if (!strcmp(*argv, "--build-options")) { printf("sizeof-long: %d\n", (int)sizeof(long)); + printf("machine: %s\n", build_platform); /* NEEDSWORK: also save and output GIT-BUILD_OPTIONS? */ } } diff --git a/help.h b/help.h index b21d7c94e8..1cf9b6fa4c 100644 --- a/help.h +++ b/help.h @@ -33,3 +33,16 @@ extern void list_commands(unsigned int colopts, struct cmdnames *main_cmds, stru */ extern void help_unknown_ref(const char *ref, const char *cmd, const char *error); #endif /* HELP_H */ + +/* + * identify build platform + */ +#ifndef GIT_BUILD_PLATFORM + #if defined __x86__ || defined __i386__ || defined __i586__ || defined __i686__ + #define GIT_BUILD_PLATFORM "x86"; + #elif defined __x86_64__ + #define GIT_BUILD_PLATFORM "x86_64"; + #else + #define GIT_BUILD_PLATFORM "unknown"; + #endif +#endif From cabed5157ae0e7fb9c61f854012685d220a6c4c5 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 9 Dec 2016 17:48:24 +0100 Subject: [PATCH 152/288] version --build-options: report commit, too, if possible In particular when local tags are used (or tags that are pushed to some fork) to build Git, it is very hard to figure out from which particular revision a particular Git executable was built. Let's just report that in our build options. We need to be careful, though, to report when the current commit cannot be determined, e.g. when building from a tarball without any associated Git repository. Signed-off-by: Johannes Schindelin --- Makefile | 4 +++- help.c | 2 ++ version.c | 1 + version.h | 1 + 4 files changed, 7 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 76267262c1..3f42273d04 100644 --- a/Makefile +++ b/Makefile @@ -1773,7 +1773,9 @@ builtin/help.sp builtin/help.s builtin/help.o: EXTRA_CPPFLAGS = \ version.sp version.s version.o: GIT-VERSION-FILE GIT-USER-AGENT version.sp version.s version.o: EXTRA_CPPFLAGS = \ '-DGIT_VERSION="$(GIT_VERSION)"' \ - '-DGIT_USER_AGENT=$(GIT_USER_AGENT_CQ_SQ)' + '-DGIT_USER_AGENT=$(GIT_USER_AGENT_CQ_SQ)' \ + '-DGIT_BUILT_FROM_COMMIT="$(shell git rev-parse -q --verify HEAD || \ + echo "(unknown)")"' $(BUILT_INS): git$X $(QUIET_BUILT_IN)$(RM) $@ && \ diff --git a/help.c b/help.c index a17ed4aef7..d6bc815003 100644 --- a/help.c +++ b/help.c @@ -418,6 +418,8 @@ int cmd_version(int argc, const char **argv, const char *prefix) printf("git version %s\n", git_version_string); while (*++argv) { if (!strcmp(*argv, "--build-options")) { + printf("built from commit: %s\n", + git_built_from_commit_string); printf("sizeof-long: %d\n", (int)sizeof(long)); printf("machine: %s\n", build_platform); /* NEEDSWORK: also save and output GIT-BUILD_OPTIONS? */ diff --git a/version.c b/version.c index 6106a8098c..41b718c29e 100644 --- a/version.c +++ b/version.c @@ -3,6 +3,7 @@ #include "strbuf.h" const char git_version_string[] = GIT_VERSION; +const char git_built_from_commit_string[] = GIT_BUILT_FROM_COMMIT; const char *git_user_agent(void) { diff --git a/version.h b/version.h index 6911a4f155..7c62e80577 100644 --- a/version.h +++ b/version.h @@ -2,6 +2,7 @@ #define VERSION_H extern const char git_version_string[]; +extern const char git_built_from_commit_string[]; const char *git_user_agent(void); const char *git_user_agent_sanitized(void); From 4132996c331a502169a73482cfb0b4193d787fc6 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 7 Sep 2016 17:06:54 +0200 Subject: [PATCH 153/288] mingw: abort on invalid strftime formats On Windows, strftime() does not silently ignore invalid formats, but warns about them and then returns 0 and sets errno to EINVAL. Unfortunately, Git does not expect such a behavior, as it disagrees with strftime()'s semantics on Linux. As a consequence, Git misinterprets the return value 0 as "I need more space" and grows the buffer. As the larger buffer does not fix the format, the buffer grows and grows and grows until we are out of memory and abort. Ideally, we would switch off the parameter validation just for strftime(), but we cannot even override the invalid parameter handler via _set_thread_local_invalid_parameter_handler() using MINGW because that function is not declared. Even _set_invalid_parameter_handler(), which *is* declared, does not help, as it simply does... nothing. So let's just bite the bullet and override strftime() for MINGW and abort on an invalid format string. While this does not provide the best user experience, it is the best we can do. See https://msdn.microsoft.com/en-us/library/fe06s4ak.aspx for more details. This fixes https://github.com/git-for-windows/git/issues/863 Signed-off-by: Johannes Schindelin --- compat/mingw.c | 11 +++++++++++ compat/mingw.h | 3 +++ 2 files changed, 14 insertions(+) diff --git a/compat/mingw.c b/compat/mingw.c index 3fbfda5978..69ac1e1957 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -759,6 +759,17 @@ revert_attrs: return rc; } +#undef strftime +size_t mingw_strftime(char *s, size_t max, + const char *format, const struct tm *tm) +{ + size_t ret = strftime(s, max, format, tm); + + if (!ret && errno == EINVAL) + die("invalid strftime format: '%s'", format); + return ret; +} + unsigned int sleep (unsigned int seconds) { Sleep(seconds*1000); diff --git a/compat/mingw.h b/compat/mingw.h index 3350169555..d117a58da7 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -361,6 +361,9 @@ int mingw_fstat(int fd, struct stat *buf); int mingw_utime(const char *file_name, const struct utimbuf *times); #define utime mingw_utime +size_t mingw_strftime(char *s, size_t max, + const char *format, const struct tm *tm); +#define strftime mingw_strftime pid_t mingw_spawnvpe(const char *cmd, const char **argv, char **env, const char *dir, From 3474e34aec242a93945cdad3d9b8182966ab3bd6 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 7 Sep 2016 18:07:04 +0200 Subject: [PATCH 154/288] reset: support the experimental --stdin option Just like with other Git commands, this option makes it read the paths from the standard input. It comes in handy when resetting many, many paths at once and wildcards are not an option (e.g. when the paths are generated by a tool). Note: we first parse the entire list and perform the actual reset action only in a second phase. Not only does this make things simpler, it also helps performance, as do_diff_cache() traverses the index and the (sorted) pathspecs in simultaneously to avoid unnecessary lookups. This feature is marked experimental because it is still under review in the upstream Git project. Signed-off-by: Johannes Schindelin --- Documentation/git-reset.txt | 10 +++++++ builtin/reset.c | 54 ++++++++++++++++++++++++++++++++++++- t/t7108-reset-stdin.sh | 32 ++++++++++++++++++++++ 3 files changed, 95 insertions(+), 1 deletion(-) create mode 100755 t/t7108-reset-stdin.sh diff --git a/Documentation/git-reset.txt b/Documentation/git-reset.txt index 25432d9257..b50c9c1c1e 100644 --- a/Documentation/git-reset.txt +++ b/Documentation/git-reset.txt @@ -10,6 +10,7 @@ SYNOPSIS [verse] 'git reset' [-q] [] [--] ... 'git reset' (--patch | -p) [] [--] [...] +EXPERIMENTAL: 'git reset' [-q] [--stdin [-z]] [] 'git reset' [--soft | --mixed [-N] | --hard | --merge | --keep] [-q] [] DESCRIPTION @@ -97,6 +98,15 @@ OPTIONS --quiet:: Be quiet, only report errors. +--stdin:: + EXPERIMENTAL: Instead of taking list of paths from the + command line, read list of paths from the standard input. + Paths are separated by LF (i.e. one path per line) by + default. + +-z:: + EXPERIMENTAL: Only meaningful with `--stdin`; paths are + separated with NUL character instead of LF. EXAMPLES -------- diff --git a/builtin/reset.c b/builtin/reset.c index c04ac076dc..f3f6e74087 100644 --- a/builtin/reset.c +++ b/builtin/reset.c @@ -21,10 +21,13 @@ #include "parse-options.h" #include "unpack-trees.h" #include "cache-tree.h" +#include "strbuf.h" +#include "quote.h" static const char * const git_reset_usage[] = { N_("git reset [--mixed | --soft | --hard | --merge | --keep] [-q] []"), N_("git reset [-q] [] [--] ..."), + N_("EXPERIMENTAL: git reset [-q] [--stdin [-z]] []"), N_("git reset --patch [] [--] [...]"), NULL }; @@ -267,7 +270,9 @@ static int reset_refs(const char *rev, const struct object_id *oid) int cmd_reset(int argc, const char **argv, const char *prefix) { int reset_type = NONE, update_ref_status = 0, quiet = 0; - int patch_mode = 0, unborn; + int patch_mode = 0, nul_term_line = 0, read_from_stdin = 0, unborn; + char **stdin_paths = NULL; + int stdin_nr = 0, stdin_alloc = 0; const char *rev; struct object_id oid; struct pathspec pathspec; @@ -286,6 +291,10 @@ int cmd_reset(int argc, const char **argv, const char *prefix) OPT_BOOL('p', "patch", &patch_mode, N_("select hunks interactively")), OPT_BOOL('N', "intent-to-add", &intent_to_add, N_("record only the fact that removed paths will be added later")), + OPT_BOOL('z', NULL, &nul_term_line, + N_("EXPERIMENTAL: paths are separated with NUL character")), + OPT_BOOL(0, "stdin", &read_from_stdin, + N_("EXPERIMENTAL: read paths from ")), OPT_END() }; @@ -295,6 +304,43 @@ int cmd_reset(int argc, const char **argv, const char *prefix) PARSE_OPT_KEEP_DASHDASH); parse_args(&pathspec, argv, prefix, patch_mode, &rev); + if (read_from_stdin) { + strbuf_getline_fn getline_fn = nul_term_line ? + strbuf_getline_nul : strbuf_getline_lf; + int flags = PATHSPEC_PREFER_FULL | + PATHSPEC_STRIP_SUBMODULE_SLASH_CHEAP; + struct strbuf buf = STRBUF_INIT; + struct strbuf unquoted = STRBUF_INIT; + + if (patch_mode) + die(_("--stdin is incompatible with --patch")); + + if (pathspec.nr) + die(_("--stdin is incompatible with path arguments")); + + while (getline_fn(&buf, stdin) != EOF) { + if (!nul_term_line && buf.buf[0] == '"') { + strbuf_reset(&unquoted); + if (unquote_c_style(&unquoted, buf.buf, NULL)) + die(_("line is badly quoted")); + strbuf_swap(&buf, &unquoted); + } + ALLOC_GROW(stdin_paths, stdin_nr + 1, stdin_alloc); + stdin_paths[stdin_nr++] = xstrdup(buf.buf); + strbuf_reset(&buf); + } + strbuf_release(&unquoted); + strbuf_release(&buf); + + ALLOC_GROW(stdin_paths, stdin_nr + 1, stdin_alloc); + stdin_paths[stdin_nr++] = NULL; + flags |= PATHSPEC_LITERAL_PATH; + parse_pathspec(&pathspec, 0, flags, prefix, + (const char **)stdin_paths); + + } else if (nul_term_line) + die(_("-z requires --stdin")); + unborn = !strcmp(rev, "HEAD") && get_sha1("HEAD", oid.hash); if (unborn) { /* reset on unborn branch: treat as reset to empty tree */ @@ -385,5 +431,11 @@ int cmd_reset(int argc, const char **argv, const char *prefix) if (!pathspec.nr) remove_branch_state(); + if (stdin_paths) { + while (stdin_nr) + free(stdin_paths[--stdin_nr]); + free(stdin_paths); + } + return update_ref_status; } diff --git a/t/t7108-reset-stdin.sh b/t/t7108-reset-stdin.sh new file mode 100755 index 0000000000..b7cbcbf869 --- /dev/null +++ b/t/t7108-reset-stdin.sh @@ -0,0 +1,32 @@ +#!/bin/sh + +test_description='reset --stdin' + +. ./test-lib.sh + +test_expect_success 'reset --stdin' ' + test_commit hello && + git rm hello.t && + test -z "$(git ls-files hello.t)" && + echo hello.t | git reset --stdin && + test hello.t = "$(git ls-files hello.t)" +' + +test_expect_success 'reset --stdin -z' ' + test_commit world && + git rm hello.t world.t && + test -z "$(git ls-files hello.t world.t)" && + printf world.tQworld.tQhello.tQ | q_to_nul | git reset --stdin -z && + printf "hello.t\nworld.t\n" >expect && + git ls-files >actual && + test_cmp expect actual +' + +test_expect_success '--stdin requires --mixed' ' + echo hello.t >list && + test_must_fail git reset --soft --stdin Date: Wed, 2 Nov 2016 13:51:20 -0400 Subject: [PATCH 155/288] cvsexportcommit: force crlf translation When using cvsnt + msys + git, it seems like the output of cvs status had \r\n in it, and caused the command to fail. This fixes that. Signed-off-by: Dustin Spicuzza --- git-cvsexportcommit.perl | 1 + 1 file changed, 1 insertion(+) diff --git a/git-cvsexportcommit.perl b/git-cvsexportcommit.perl index d13f02da95..fc00d5946a 100755 --- a/git-cvsexportcommit.perl +++ b/git-cvsexportcommit.perl @@ -431,6 +431,7 @@ END sub safe_pipe_capture { my @output; if (my $pid = open my $child, '-|') { + binmode($child, ":crlf"); @output = (<$child>); close $child or die join(' ',@_).": $! $?"; } else { From 2451aceeebb835bfda7dab463ac71823f27f25fa Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Mon, 14 Nov 2016 10:28:42 -0500 Subject: [PATCH 156/288] preload-index: avoid lstat for skip-worktree items Teach preload-index to avoid lstat() calls for index-entries with skip-worktree bit set. This is a performance optimization. During a sparse-checkout, the skip-worktree bit is set on items that were not populated and therefore are not present in the worktree. The per-thread preload-index loop performs a series of tests on each index-entry as it attempts to compare the worktree version with the index and mark them up-to-date. This patch short-cuts that work. On a Windows 10 system with a very large repo (450MB index) and various levels of sparseness, performance was improved in the {preloadindex=true, fscache=false} case by 80% and in the {preloadindex=true, fscache=true} case by 20% for various commands. Signed-off-by: Jeff Hostetler --- preload-index.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/preload-index.c b/preload-index.c index c1fe3a3ef9..70a4c80878 100644 --- a/preload-index.c +++ b/preload-index.c @@ -53,6 +53,8 @@ static void *preload_thread(void *_data) continue; if (ce_uptodate(ce)) continue; + if (ce_skip_worktree(ce)) + continue; if (!ce_path_match(ce, &p->pathspec, NULL)) continue; if (threaded_has_symlink_leading_path(&cache, ce->name, ce_namelen(ce))) From efaf5b6916280cfc08ae027f6511010bfcf00da9 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Thu, 17 Nov 2016 09:14:40 -0500 Subject: [PATCH 157/288] name-hash: eliminate duplicate memihash call Remove duplicate memihash() call in hash_dir_entry(). The existing code called memihash() to do the find_dir_entry() and it not found, called memihash() again to do the hashmap_add(). Signed-off-by: Jeff Hostetler --- name-hash.c | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/name-hash.c b/name-hash.c index 6d9f23e932..860b8dd0c7 100644 --- a/name-hash.c +++ b/name-hash.c @@ -23,13 +23,19 @@ static int dir_entry_cmp(const struct dir_entry *e1, name ? name : e2->name, e1->namelen); } +static struct dir_entry *find_dir_entry__hash(struct index_state *istate, + const char *name, unsigned int namelen, unsigned int hash) +{ + struct dir_entry key; + hashmap_entry_init(&key, hash); + key.namelen = namelen; + return hashmap_get(&istate->dir_hash, &key, name); +} + static struct dir_entry *find_dir_entry(struct index_state *istate, const char *name, unsigned int namelen) { - struct dir_entry key; - hashmap_entry_init(&key, memihash(name, namelen)); - key.namelen = namelen; - return hashmap_get(&istate->dir_hash, &key, name); + return find_dir_entry__hash(istate, name, namelen, memihash(name,namelen)); } static struct dir_entry *hash_dir_entry(struct index_state *istate, @@ -43,6 +49,7 @@ static struct dir_entry *hash_dir_entry(struct index_state *istate, * in index_state.name_hash (as ordinary cache_entries). */ struct dir_entry *dir; + unsigned int hash; /* get length of parent directory */ while (namelen > 0 && !is_dir_sep(ce->name[namelen - 1])) @@ -52,11 +59,12 @@ static struct dir_entry *hash_dir_entry(struct index_state *istate, namelen--; /* lookup existing entry for that directory */ - dir = find_dir_entry(istate, ce->name, namelen); + hash = memihash(ce->name, namelen); + dir = find_dir_entry__hash(istate, ce->name, namelen, hash); if (!dir) { /* not found, create it and add to hash table */ FLEX_ALLOC_MEM(dir, name, ce->name, namelen); - hashmap_entry_init(dir, memihash(ce->name, namelen)); + hashmap_entry_init(dir, hash); dir->namelen = namelen; hashmap_add(&istate->dir_hash, dir); From a2768acb1e1eaf5d35a6414e1c1f752944b14bdf Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Thu, 17 Nov 2016 09:23:55 -0500 Subject: [PATCH 158/288] hashmap: allow memihash computation to be continued Add variant of memihash() to allow the hash computation to be continued. There are times when we compute the hash on a full path and then the hash on just the path to the parent directory. This can be expensive on large repositories. With this, we can hash the parent directory first. And then continue the computation to include the "/filename". Signed-off-by: Jeff Hostetler --- hashmap.c | 17 +++++++++++++++++ hashmap.h | 1 + 2 files changed, 18 insertions(+) diff --git a/hashmap.c b/hashmap.c index b10b642229..505e63fd3c 100644 --- a/hashmap.c +++ b/hashmap.c @@ -50,6 +50,23 @@ unsigned int memihash(const void *buf, size_t len) return hash; } +/* + * Incoporate another chunk of data into a memihash + * computation. + */ +unsigned int memihash_cont(unsigned int hash_seed, const void *buf, size_t len) +{ + unsigned int hash = hash_seed; + unsigned char *ucbuf = (unsigned char *) buf; + while (len--) { + unsigned int c = *ucbuf++; + if (c >= 'a' && c <= 'z') + c -= 'a' - 'A'; + hash = (hash * FNV32_PRIME) ^ c; + } + return hash; +} + #define HASHMAP_INITIAL_SIZE 64 /* grow / shrink by 2^2 */ #define HASHMAP_RESIZE_BITS 2 diff --git a/hashmap.h b/hashmap.h index ab7958ae33..45eda693bd 100644 --- a/hashmap.h +++ b/hashmap.h @@ -12,6 +12,7 @@ extern unsigned int strhash(const char *buf); extern unsigned int strihash(const char *buf); extern unsigned int memhash(const void *buf, size_t len); extern unsigned int memihash(const void *buf, size_t len); +extern unsigned int memihash_cont(unsigned int hash_seed, const void *buf, size_t len); static inline unsigned int sha1hash(const unsigned char *sha1) { From c47c3dfdd370b55431bbbc2ce75cc51bf17b87a1 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Thu, 17 Nov 2016 09:59:59 -0500 Subject: [PATCH 159/288] name-hash: precompute hash values during preload-index Precompute the istate.name_hash and istate.dir_hash values for each cache-entry during the preload-index phase. Move the expensive memihash() calculations from lazy_init_name_hash() to the multi-threaded preload-index phase. Signed-off-by: Jeff Hostetler --- cache.h | 16 ++++++++++++ name-hash.c | 66 +++++++++++++++++++++++++++++++++++++++++++++++-- preload-index.c | 2 ++ read-cache.c | 3 +++ 4 files changed, 85 insertions(+), 2 deletions(-) diff --git a/cache.h b/cache.h index 5c035dae57..2a3ee42f7f 100644 --- a/cache.h +++ b/cache.h @@ -173,6 +173,9 @@ struct cache_entry { unsigned int ce_flags; unsigned int ce_namelen; unsigned int index; /* for link extension */ + unsigned int precompute_hash_state; + unsigned int precompute_hash_name; + unsigned int precompute_hash_dir; struct object_id oid; char name[FLEX_ARRAY]; /* more */ }; @@ -229,6 +232,19 @@ struct cache_entry { #error "CE_EXTENDED_FLAGS out of range" #endif +/* + * Bit set if preload-index precomputed the hash value(s) + * for this cache-entry. + */ +#define CE_PRECOMPUTE_HASH_STATE__SET (1 << 0) +/* + * Bit set if precompute-index also precomputed the hash value + * for the parent directory. + */ +#define CE_PRECOMPUTE_HASH_STATE__DIR (1 << 1) + +void precompute_istate_hashes(struct cache_entry *ce); + /* Forward structure decls */ struct pathspec; struct child_process; diff --git a/name-hash.c b/name-hash.c index 860b8dd0c7..7a220187ad 100644 --- a/name-hash.c +++ b/name-hash.c @@ -50,6 +50,17 @@ static struct dir_entry *hash_dir_entry(struct index_state *istate, */ struct dir_entry *dir; unsigned int hash; + int use_precomputed_dir_hash = 0; + + if (ce->precompute_hash_state & CE_PRECOMPUTE_HASH_STATE__SET) { + if (!(ce->precompute_hash_state & CE_PRECOMPUTE_HASH_STATE__DIR)) + return NULL; /* item does not have a parent directory */ + if (namelen == ce_namelen(ce)) { + /* dir hash only valid for outer-most call (not recursive ones) */ + use_precomputed_dir_hash = 1; + hash = ce->precompute_hash_dir; + } + } /* get length of parent directory */ while (namelen > 0 && !is_dir_sep(ce->name[namelen - 1])) @@ -59,7 +70,8 @@ static struct dir_entry *hash_dir_entry(struct index_state *istate, namelen--; /* lookup existing entry for that directory */ - hash = memihash(ce->name, namelen); + if (!use_precomputed_dir_hash) + hash = memihash(ce->name, namelen); dir = find_dir_entry__hash(istate, ce->name, namelen, hash); if (!dir) { /* not found, create it and add to hash table */ @@ -99,10 +111,18 @@ static void remove_dir_entry(struct index_state *istate, struct cache_entry *ce) static void hash_index_entry(struct index_state *istate, struct cache_entry *ce) { + unsigned int h; + if (ce->ce_flags & CE_HASHED) return; ce->ce_flags |= CE_HASHED; - hashmap_entry_init(ce, memihash(ce->name, ce_namelen(ce))); + + if (ce->precompute_hash_state & CE_PRECOMPUTE_HASH_STATE__SET) + h = ce->precompute_hash_name; + else + h = memihash(ce->name, ce_namelen(ce)); + + hashmap_entry_init(ce, h); hashmap_add(&istate->name_hash, ce); if (ignore_case) @@ -244,3 +264,45 @@ void free_name_hash(struct index_state *istate) hashmap_free(&istate->name_hash, 0); hashmap_free(&istate->dir_hash, 1); } + +/* + * Precompute the hash values for this cache_entry + * for use in the istate.name_hash and istate.dir_hash. + * + * If the item is in the root directory, just compute the + * hash value (for istate.name_hash) on the full path. + * + * If the item is in a subdirectory, first compute the + * hash value for the immediate parent directory (for + * istate.dir_hash) and then the hash value for the full + * path by continuing the computation. + * + * Note that these hashes will be used by + * wt_status_collect_untracked() as it scans the worktree + * and maps observed paths back to the index (optionally + * ignoring case). Therefore, we probably only *NEED* to + * precompute this for non-skip-worktree items (since + * status should not observe skipped items), but because + * lazy_init_name_hash() hashes everything, we force it + * here. + */ +void precompute_istate_hashes(struct cache_entry *ce) +{ + int namelen = ce_namelen(ce); + + while (namelen > 0 && !is_dir_sep(ce->name[namelen - 1])) + namelen--; + + if (namelen <= 0) { + ce->precompute_hash_name = memihash(ce->name, ce_namelen(ce)); + ce->precompute_hash_state = CE_PRECOMPUTE_HASH_STATE__SET; + } else { + namelen--; + ce->precompute_hash_dir = memihash(ce->name, namelen); + ce->precompute_hash_name = memihash_cont( + ce->precompute_hash_dir, &ce->name[namelen], + ce_namelen(ce) - namelen); + ce->precompute_hash_state = + CE_PRECOMPUTE_HASH_STATE__SET | CE_PRECOMPUTE_HASH_STATE__DIR; + } +} diff --git a/preload-index.c b/preload-index.c index c1fe3a3ef9..602737f9d0 100644 --- a/preload-index.c +++ b/preload-index.c @@ -47,6 +47,8 @@ static void *preload_thread(void *_data) struct cache_entry *ce = *cep++; struct stat st; + precompute_istate_hashes(ce); + if (ce_stage(ce)) continue; if (S_ISGITLINK(ce->ce_mode)) diff --git a/read-cache.c b/read-cache.c index 585ff02dc3..023e5a9d34 100644 --- a/read-cache.c +++ b/read-cache.c @@ -73,6 +73,7 @@ void rename_index_entry_at(struct index_state *istate, int nr, const char *new_n copy_cache_entry(new, old); new->ce_flags &= ~CE_HASHED; new->ce_namelen = namelen; + new->precompute_hash_state = 0; new->index = 0; memcpy(new->name, new_name, namelen + 1); @@ -621,6 +622,7 @@ static struct cache_entry *create_alias_ce(struct index_state *istate, new = xcalloc(1, cache_entry_size(len)); memcpy(new->name, alias->name, len); copy_cache_entry(new, ce); + new->precompute_hash_state = 0; save_or_free_index_entry(istate, ce); return new; } @@ -1456,6 +1458,7 @@ static struct cache_entry *cache_entry_from_ondisk(struct ondisk_cache_entry *on ce->ce_stat_data.sd_size = get_be32(&ondisk->size); ce->ce_flags = flags & ~CE_NAMEMASK; ce->ce_namelen = len; + ce->precompute_hash_state = 0; ce->index = 0; hashcpy(ce->oid.hash, ondisk->sha1); memcpy(ce->name, name, len); From 4723d1cfe62c6a89179a4a941d273e8c4d51cb26 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Thu, 17 Nov 2016 10:05:26 -0500 Subject: [PATCH 160/288] name-hash: specify initial size for istate.dir_hash table Specify an initial size for the istate.dir_hash HashMap matching the size of the istate.name_hash. Previously hashmap_init() was given 0, causing a 64 bucket hashmap to be created. When working with very large repositories, this would cause numerous rehash() calls to realloc and rebalance the hashmap. This is especially true when the worktree is deep, with many directories containing a few files. Signed-off-by: Jeff Hostetler --- name-hash.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/name-hash.c b/name-hash.c index 7a220187ad..b2f6d2df7d 100644 --- a/name-hash.c +++ b/name-hash.c @@ -148,7 +148,8 @@ static void lazy_init_name_hash(struct index_state *istate) return; hashmap_init(&istate->name_hash, (hashmap_cmp_fn) cache_entry_cmp, istate->cache_nr); - hashmap_init(&istate->dir_hash, (hashmap_cmp_fn) dir_entry_cmp, 0); + hashmap_init(&istate->dir_hash, (hashmap_cmp_fn) dir_entry_cmp, + istate->cache_nr); for (nr = 0; nr < istate->cache_nr; nr++) hash_index_entry(istate, istate->cache[nr]); istate->name_hash_initialized = 1; From aac8f185f3f1cc85a15bb86682eb1967ea7e4897 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Thu, 17 Nov 2016 10:37:01 -0500 Subject: [PATCH 161/288] name-hash: remember previous dir_entry during lazy_init_name_hash Teach hash_dir_entry() to remember the previously found dir_entry during lazy_init_name_hash() iteration. This is a performance optimization. Since items in the index array are sorted by full pathname, adjacent items are likely to be in the same directory. This can save memihash() computations and HashMap lookups. Signed-off-by: Jeff Hostetler --- name-hash.c | 43 +++++++++++++++++++++++++++++++------------ 1 file changed, 31 insertions(+), 12 deletions(-) diff --git a/name-hash.c b/name-hash.c index b2f6d2df7d..fcd2f90cf7 100644 --- a/name-hash.c +++ b/name-hash.c @@ -39,7 +39,7 @@ static struct dir_entry *find_dir_entry(struct index_state *istate, } static struct dir_entry *hash_dir_entry(struct index_state *istate, - struct cache_entry *ce, int namelen) + struct cache_entry *ce, int namelen, struct dir_entry **p_previous_dir) { /* * Throw each directory component in the hash for quick lookup @@ -70,9 +70,21 @@ static struct dir_entry *hash_dir_entry(struct index_state *istate, namelen--; /* lookup existing entry for that directory */ - if (!use_precomputed_dir_hash) - hash = memihash(ce->name, namelen); - dir = find_dir_entry__hash(istate, ce->name, namelen, hash); + if (p_previous_dir && *p_previous_dir + && namelen == (*p_previous_dir)->namelen + && memcmp(ce->name, (*p_previous_dir)->name, namelen) == 0) { + /* + * When our caller is sequentially iterating thru the index, + * items in the same directory will be sequential, and therefore + * refer to the same dir_entry. + */ + dir = *p_previous_dir; + } else { + if (!use_precomputed_dir_hash) + hash = memihash(ce->name, namelen); + dir = find_dir_entry__hash(istate, ce->name, namelen, hash); + } + if (!dir) { /* not found, create it and add to hash table */ FLEX_ALLOC_MEM(dir, name, ce->name, namelen); @@ -81,15 +93,20 @@ static struct dir_entry *hash_dir_entry(struct index_state *istate, hashmap_add(&istate->dir_hash, dir); /* recursively add missing parent directories */ - dir->parent = hash_dir_entry(istate, ce, namelen); + dir->parent = hash_dir_entry(istate, ce, namelen, NULL); } + + if (p_previous_dir) + *p_previous_dir = dir; + return dir; } -static void add_dir_entry(struct index_state *istate, struct cache_entry *ce) +static void add_dir_entry(struct index_state *istate, struct cache_entry *ce, + struct dir_entry **p_previous_dir) { /* Add reference to the directory entry (and parents if 0). */ - struct dir_entry *dir = hash_dir_entry(istate, ce, ce_namelen(ce)); + struct dir_entry *dir = hash_dir_entry(istate, ce, ce_namelen(ce), p_previous_dir); while (dir && !(dir->nr++)) dir = dir->parent; } @@ -100,7 +117,7 @@ static void remove_dir_entry(struct index_state *istate, struct cache_entry *ce) * Release reference to the directory entry. If 0, remove and continue * with parent directory. */ - struct dir_entry *dir = hash_dir_entry(istate, ce, ce_namelen(ce)); + struct dir_entry *dir = hash_dir_entry(istate, ce, ce_namelen(ce), NULL); while (dir && !(--dir->nr)) { struct dir_entry *parent = dir->parent; hashmap_remove(&istate->dir_hash, dir, NULL); @@ -109,7 +126,8 @@ static void remove_dir_entry(struct index_state *istate, struct cache_entry *ce) } } -static void hash_index_entry(struct index_state *istate, struct cache_entry *ce) +static void hash_index_entry(struct index_state *istate, struct cache_entry *ce, + struct dir_entry **p_previous_dir) { unsigned int h; @@ -126,7 +144,7 @@ static void hash_index_entry(struct index_state *istate, struct cache_entry *ce) hashmap_add(&istate->name_hash, ce); if (ignore_case) - add_dir_entry(istate, ce); + add_dir_entry(istate, ce, p_previous_dir); } static int cache_entry_cmp(const struct cache_entry *ce1, @@ -142,6 +160,7 @@ static int cache_entry_cmp(const struct cache_entry *ce1, static void lazy_init_name_hash(struct index_state *istate) { + struct dir_entry *previous_dir = NULL; int nr; if (istate->name_hash_initialized) @@ -151,14 +170,14 @@ static void lazy_init_name_hash(struct index_state *istate) hashmap_init(&istate->dir_hash, (hashmap_cmp_fn) dir_entry_cmp, istate->cache_nr); for (nr = 0; nr < istate->cache_nr; nr++) - hash_index_entry(istate, istate->cache[nr]); + hash_index_entry(istate, istate->cache[nr], &previous_dir); istate->name_hash_initialized = 1; } void add_name_hash(struct index_state *istate, struct cache_entry *ce) { if (istate->name_hash_initialized) - hash_index_entry(istate, ce); + hash_index_entry(istate, ce, NULL); } void remove_name_hash(struct index_state *istate, struct cache_entry *ce) From bc14ef8e8f72c1e20560f401776d78830a91c108 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 23 Nov 2016 20:59:44 +0100 Subject: [PATCH 162/288] Export the preload_index() function The purpose of this function is to stat() the files listed in the index in a multi-threaded fashion. It is called directly after reading the index in the read_index_preloaded() function. However, in some cases we may want to separate the index reading from the preloading step, e.g. in builtin/add.c, where we need to load the index before we parse the pathspecs (which needs to error out if one of the pathspecs refers to a path within a submodule, for which the index must have been read already), and only then will we want to preload, possibly limited by the just-parsed pathspecs. So let's just export that function to allow calling it separately. Signed-off-by: Johannes Schindelin --- cache.h | 1 + preload-index.c | 6 ++---- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/cache.h b/cache.h index 8821d54dd5..7541708552 100644 --- a/cache.h +++ b/cache.h @@ -560,6 +560,7 @@ extern int daemonize(void); /* Initialize and use the cache information */ struct lock_file; extern int read_index(struct index_state *); +extern void preload_index(struct index_state *, const struct pathspec *pathspec); extern int read_index_preload(struct index_state *, const struct pathspec *pathspec); extern int do_read_index(struct index_state *istate, const char *path, int must_exist); /* for testting only! */ diff --git a/preload-index.c b/preload-index.c index c7970d7cb9..2d673d0d07 100644 --- a/preload-index.c +++ b/preload-index.c @@ -6,8 +6,7 @@ #include "dir.h" #ifdef NO_PTHREADS -static void preload_index(struct index_state *index, - const struct pathspec *pathspec) +void preload_index(struct index_state *index, const struct pathspec *pathspec) { ; /* nothing */ } @@ -67,8 +66,7 @@ static void *preload_thread(void *_data) return NULL; } -static void preload_index(struct index_state *index, - const struct pathspec *pathspec) +void preload_index(struct index_state *index, const struct pathspec *pathspec) { int threads, i, work, offset; struct thread_data data[MAX_PARALLEL]; From 95d0d1f771872049ed241cac44e5ad612fea6aab Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Tue, 22 Nov 2016 11:26:38 -0500 Subject: [PATCH 163/288] add: use preload-index and fscache for performance Teach "add" to use preload-index and fscache features to improve performance on very large repositories. During an "add", a call is made to run_diff_files() which calls check_remove() for each index-entry. This calls lstat(). On Windows, the fscache code intercepts the lstat() calls and builds a private cache using the FindFirst/FindNext routines, which are much faster. Somewhat independent of this, is the preload-index code which distributes some of the start-up costs across multiple threads. We need to keep the call to read_cache() before parsing the pathspecs (and hence cannot use the pathspecs to limit any preload) because parse_pathspec() is using the index to determine whether a pathspec is, in fact, in a submodule. If we would not read the index first, parse_pathspec() would not error out on a path that is inside a submodule, and t7400-submodule-basic.sh would fail with not ok 47 - do not add files from a submodule We still want the nice preload performance boost, though, so we simply call read_cache_preload(&pathspecs) after parsing the pathspecs. Signed-off-by: Jeff Hostetler Signed-off-by: Johannes Schindelin --- builtin/add.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/builtin/add.c b/builtin/add.c index e8fb80b36e..c3736aadcd 100644 --- a/builtin/add.c +++ b/builtin/add.c @@ -389,6 +389,10 @@ int cmd_add(int argc, const char **argv, const char *prefix) PATHSPEC_STRIP_SUBMODULE_SLASH_EXPENSIVE, prefix, argv); + enable_fscache(1); + /* We do not really re-read the index, but update the up-to-date flags */ + preload_index(&the_index, &pathspec); + if (add_new_files) { int baselen; From 0f53359bdf5c1ce6b98fbe53a2baa0232f327b9f Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 8 Nov 2016 16:39:44 +0100 Subject: [PATCH 164/288] Really work around "uninitialized value" warning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ever since 457f08a (git-rev-list: add --bisect-vars option., 2007-03-21), Git's source code uses the following trick to fool GCC into *not* warning about uninitialized values: int value = value; We use this trick to silence the "warning: ‘x’ is used uninitialized in this function [-Wuninitialized]" when the variables are not really used uninitialized (but it is hard for the compiler to determine that). This trick works well for GCC, and even Clang seems to appease that workaround. Not so Visual C. It does realize that this is just a trick to fool it, and it simply refuses to be fooled. The only way to silence the warning for Visual C would be to write something like this: #pragma warning(suppress: 4700) int value; Obviously this is not portable, and neither is that trick that fools GCC. So let's just introduce a new macro that continues to fool GCC, but simply initializes the values everywhere else. Signed-off-by: Johannes Schindelin --- builtin/rev-list.c | 3 ++- fast-import.c | 4 ++-- git-compat-util.h | 17 +++++++++++++++++ merge-recursive.c | 2 +- read-cache.c | 2 +- 5 files changed, 23 insertions(+), 5 deletions(-) diff --git a/builtin/rev-list.c b/builtin/rev-list.c index c43decda70..487b76b5ad 100644 --- a/builtin/rev-list.c +++ b/builtin/rev-list.c @@ -388,7 +388,8 @@ int cmd_rev_list(int argc, const char **argv, const char *prefix) mark_edges_uninteresting(&revs, show_edge); if (bisect_list) { - int reaches = reaches, all = all; + FAKE_INIT(int, reaches, 0); + FAKE_INIT(int, all, 0); revs.commits = find_bisection(revs.commits, &reaches, &all, bisect_find_all); diff --git a/fast-import.c b/fast-import.c index 64fe602f0b..da65ba86ac 100644 --- a/fast-import.c +++ b/fast-import.c @@ -2992,7 +2992,7 @@ static void cat_blob(struct object_entry *oe, unsigned char sha1[20]) static void parse_get_mark(const char *p) { - struct object_entry *oe = oe; + FAKE_INIT(struct object_entry *, oe, NULL); char output[42]; /* get-mark SP LF */ @@ -3009,7 +3009,7 @@ static void parse_get_mark(const char *p) static void parse_cat_blob(const char *p) { - struct object_entry *oe = oe; + FAKE_INIT(struct object_entry *, oe, NULL); unsigned char sha1[20]; /* cat-blob SP LF */ diff --git a/git-compat-util.h b/git-compat-util.h index 8a69331241..a1ac7b6e36 100644 --- a/git-compat-util.h +++ b/git-compat-util.h @@ -51,6 +51,23 @@ #endif #endif +/* + * Under certain circumstances Git's source code is cleverer than the C + * compiler when the latter warns about some "uninitialized value", e.g. when + * a value is both initialized and used under the same condition. + * + * GCC can be fooled to not spit out this warning by using the construct: + * "int value = value;". Other C compilers are not that easily fooled and would + * require a #pragma (which is not portable, and would litter the source code). + * + * To keep things simple, we only fool GCC, and initialize such values instead + * when compiling with other C compilers. + */ +#ifdef __GNUC__ +#define FAKE_INIT(a, b, c) a b = b +#else +#define FAKE_INIT(a, b, c) a b = c +#endif /* * BUILD_ASSERT_OR_ZERO - assert a build-time dependency, as an expression. diff --git a/merge-recursive.c b/merge-recursive.c index 23d6992f40..f1dcd9ddf0 100644 --- a/merge-recursive.c +++ b/merge-recursive.c @@ -2008,7 +2008,7 @@ int merge_recursive(struct merge_options *o, { struct commit_list *iter; struct commit *merged_common_ancestors; - struct tree *mrtree = mrtree; + FAKE_INIT(struct tree *, mrtree, NULL); int clean; if (show(o, 4)) { diff --git a/read-cache.c b/read-cache.c index 585ff02dc3..9ec895d10c 100644 --- a/read-cache.c +++ b/read-cache.c @@ -1903,7 +1903,7 @@ static int ce_write_entry(git_SHA_CTX *c, int fd, struct cache_entry *ce, { int size; struct ondisk_cache_entry *ondisk; - int saved_namelen = saved_namelen; /* compiler workaround */ + FAKE_INIT(int, saved_namelen, 0); char *name; int result; From 13fb95e3611f19e3a8a6d3b777cb236f25ecb48c Mon Sep 17 00:00:00 2001 From: Philip Oakley Date: Sun, 19 Jul 2015 15:59:04 +0100 Subject: [PATCH 165/288] perl/Makefile: treat a missing PM.stamp as if empty 'make clean', or a 'git clean -dfx' will delete the PM stamp file, so it cannot be a direct target in such clean conditions, resulting in an error. Normally the PM.stamp is recreated by the git/Makefile, except when a dry-run is requested, for example, as used in the msysgit msvc-build script which implements the compat/vcbuild/README using contrib/buildsystems. The script msvc-build is introduced later in this series. Protect the PM.stamp target when the PM.stamp file does not exist, allowing a Git 'Makefile -n' to succeed on a clean repo. Signed-off-by: Philip Oakley --- This is development of the original "[PATCH 4/17] Makefile: a dry-run can error out if no perl. Document the issue" 2015-06-25, (http://marc.info/?l=git&m=143519054716960&w=2), which simply documented the issue and then used NO_PERL to avoid the problem. See follow on email thread for some discussion. --- perl/Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/perl/Makefile b/perl/Makefile index 15d96fcc7a..5b86aac41c 100644 --- a/perl/Makefile +++ b/perl/Makefile @@ -22,7 +22,9 @@ clean: $(RM) $(makfile).old $(RM) PM.stamp +ifneq (,$(wildcard PM.stamp)) $(makfile): PM.stamp +endif ifdef NO_PERL_MAKEMAKER instdir_SQ = $(subst ','\'',$(prefix)/lib) From bda79d90628099d98ebd77d2ab6ae0b5ea9caf08 Mon Sep 17 00:00:00 2001 From: Philip Oakley Date: Fri, 6 May 2016 11:46:05 +0100 Subject: [PATCH 166/288] Avoid multiple PREFIX definitions The short and sweet PREFIX can be confused when used in many places. Rename both usages to better describe their purpose. EXEC_CMD_PREFIX is used in full to disambiguate it from the nearby GIT_EXEC_PATH. The PREFIX in sideband.c, while nominally independant of the exec_cmd PREFIX, does reside within libgit[1], so the definitions would clash when taken together with a PREFIX given on the command line for use by exec_cmd.c. Noticed when compiling Git for Windows using MSVC/Visual Studio [1] which reports the conflict beteeen the command line definition and the definition in sideband.c within the libgit project. [1] the libgit functions are brought into a single sub-project within the Visual Studio construction script provided in contrib, and hence uses a single command for both exec_cmd.c and sideband.c. Signed-off-by: Philip Oakley --- Makefile | 2 +- exec_cmd.c | 4 ++-- sideband.c | 10 +++++----- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 3632167c3a..7cafc2ae49 100644 --- a/Makefile +++ b/Makefile @@ -2015,7 +2015,7 @@ exec_cmd.sp exec_cmd.s exec_cmd.o: GIT-PREFIX exec_cmd.sp exec_cmd.s exec_cmd.o: EXTRA_CPPFLAGS = \ '-DGIT_EXEC_PATH="$(gitexecdir_SQ)"' \ '-DBINDIR="$(bindir_relative_SQ)"' \ - '-DPREFIX="$(prefix_SQ)"' + '-DFALLBACK_RUNTIME_PREFIX="$(prefix_SQ)"' builtin/init-db.sp builtin/init-db.s builtin/init-db.o: GIT-PREFIX builtin/init-db.sp builtin/init-db.s builtin/init-db.o: EXTRA_CPPFLAGS = \ diff --git a/exec_cmd.c b/exec_cmd.c index fb94aeba9c..4151ffa222 100644 --- a/exec_cmd.c +++ b/exec_cmd.c @@ -12,7 +12,7 @@ char *system_path(const char *path) #ifdef RUNTIME_PREFIX static const char *prefix; #else - static const char *prefix = PREFIX; + static const char *prefix = FALLBACK_RUNTIME_PREFIX; #endif struct strbuf d = STRBUF_INIT; @@ -27,7 +27,7 @@ char *system_path(const char *path) !(prefix = strip_path_suffix(argv0_path, GIT_EXEC_PATH)) && !(prefix = strip_path_suffix(argv0_path, BINDIR)) && !(prefix = strip_path_suffix(argv0_path, "git"))) { - prefix = PREFIX; + prefix = FALLBACK_RUNTIME_PREFIX; trace_printf("RUNTIME_PREFIX requested, " "but prefix computation failed. " "Using static fallback '%s'.\n", prefix); diff --git a/sideband.c b/sideband.c index 1e4d684d6c..82dda5fa15 100644 --- a/sideband.c +++ b/sideband.c @@ -13,7 +13,7 @@ * the remote died unexpectedly. A flush() concludes the stream. */ -#define PREFIX "remote: " +#define DISPLAY_PREFIX "remote: " #define ANSI_SUFFIX "\033[K" #define DUMB_SUFFIX " " @@ -50,7 +50,7 @@ int recv_sideband(const char *me, int in_stream, int out) switch (band) { case 3: strbuf_addf(&outbuf, "%s%s%s", outbuf.len ? "\n" : "", - PREFIX, buf + 1); + DISPLAY_PREFIX, buf + 1); retval = SIDEBAND_REMOTE_ERROR; break; case 2: @@ -68,7 +68,7 @@ int recv_sideband(const char *me, int in_stream, int out) int linelen = brk - b; if (!outbuf.len) - strbuf_addstr(&outbuf, PREFIX); + strbuf_addstr(&outbuf, DISPLAY_PREFIX); if (linelen > 0) { strbuf_addf(&outbuf, "%.*s%s%c", linelen, b, suffix, *brk); @@ -82,8 +82,8 @@ int recv_sideband(const char *me, int in_stream, int out) } if (*b) - strbuf_addf(&outbuf, "%s%s", - outbuf.len ? "" : PREFIX, b); + strbuf_addf(&outbuf, "%s%s", outbuf.len ? + "" : DISPLAY_PREFIX, b); break; case 1: write_or_die(out, buf + 1, len); From ed7a48eb7af1ea7df04f2d2293cccc48ea2090ba Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 27 Oct 2016 06:25:56 -0700 Subject: [PATCH 167/288] obstack: fix compiler warning MS Visual C suggests that the construct condition ? (int) i : (ptrdiff_t) d is incorrect. Let's fix this by casting to ptrdiff_t also for the positive arm of the conditional. Signed-off-by: Johannes Schindelin --- compat/obstack.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compat/obstack.h b/compat/obstack.h index ceb4bdbcdd..892dc28828 100644 --- a/compat/obstack.h +++ b/compat/obstack.h @@ -493,7 +493,7 @@ __extension__ \ ( (h)->temp.tempint = (char *) (obj) - (char *) (h)->chunk, \ ((((h)->temp.tempint > 0 \ && (h)->temp.tempint < (h)->chunk_limit - (char *) (h)->chunk)) \ - ? (int) ((h)->next_free = (h)->object_base \ + ? (ptrdiff_t) ((h)->next_free = (h)->object_base \ = (h)->temp.tempint + (char *) (h)->chunk) \ : (((obstack_free) ((h), (h)->temp.tempint + (char *) (h)->chunk), 0), 0))) From e71debb35b12ee7e1f4fb1466337fdae25f5f1b9 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 27 Oct 2016 08:06:29 -0700 Subject: [PATCH 168/288] terminal.c: guard the inclusion of inttypes.h We do have a lovely Makefile option to state that that header file is not available. Let's use it everywhere... Signed-off-by: Johannes Schindelin --- compat/terminal.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compat/terminal.c b/compat/terminal.c index 1d37f0aafb..d9d3945afa 100644 --- a/compat/terminal.c +++ b/compat/terminal.c @@ -1,4 +1,6 @@ +#ifndef NO_INTTYPES_H #include +#endif #include "git-compat-util.h" #include "run-command.h" #include "compat/terminal.h" From 484f7e0ae6207017565da37f9fb9dabf2d239995 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 25 Nov 2016 18:29:51 +0100 Subject: [PATCH 169/288] Ensure that bin-wrappers/* targets adds `.exe` if necessary When compiling with Visual Studio, the projects' names are identical to the executables modulo the extensions. Which means that the bin-wrappers *need* to target the .exe files lest they try to execute directories. Signed-off-by: Johannes Schindelin --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 7cafc2ae49..a36c7b9e4e 100644 --- a/Makefile +++ b/Makefile @@ -2276,7 +2276,7 @@ bin-wrappers/%: wrap-for-bin.sh @mkdir -p bin-wrappers $(QUIET_GEN)sed -e '1s|#!.*/sh|#!$(SHELL_PATH_SQ)|' \ -e 's|@@BUILD_DIR@@|$(shell pwd)|' \ - -e 's|@@PROG@@|$(patsubst test-%,t/helper/test-%,$(@F))|' < $< > $@ && \ + -e 's|@@PROG@@|$(patsubst test-%,t/helper/test-%$(X),$(@F))$(patsubst git%,$(X),$(filter $(@F),$(BINDIR_PROGRAMS_NEED_X)))|' < $< > $@ && \ chmod +x $@ # GNU make supports exporting all variables by "export" without parameters. From 20f1e1df3c25b798319ccb1e45bd54eebb6ffe72 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 29 Nov 2016 15:37:01 +0100 Subject: [PATCH 170/288] .gitattributes: fix end-of-lines The test suite does actually not pass when files are checked out with CR/LF line endings, so we have to force them to LF-only. For good measure, we simply force all source files to LF, also to make it easier for Git for Windows contributors to contribute to upstream Git's source code. Signed-off-by: Johannes Schindelin --- .gitattributes | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitattributes b/.gitattributes index 320e33c327..9b742a03a1 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,3 +1,4 @@ * whitespace=!indent,trail,space -*.[ch] whitespace=indent,trail,space diff=cpp -*.sh whitespace=indent,trail,space +*.[ch] whitespace=indent,trail,space diff=cpp eol=lf +*.sh whitespace=indent,trail,space eol=lf +*.{perl,pm,txt} eol=lf From 21811292582d0cfa049886d494fc1ed701209e0d Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 29 Nov 2016 23:07:18 +0100 Subject: [PATCH 171/288] windows: clarify the need for invalidcontinue.obj Git's source code wants to be able to close() the same file descriptor multiple times, ignoring the error returned by the second call (and the ones after that), or to access the osfhandle of an already-closed stdout, among other things that the UCRT does not like. Simply linking invalidcontinue.obj allows such usage without resorting to Debug Assertions (or exiting with exit code 9 in Release Mode). Let's add a note so we don't forget, as suggested by Jeff Hostetler. See https://msdn.microsoft.com/en-us/library/ms235330.aspx for more details. Signed-off-by: Johannes Schindelin --- config.mak.uname | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config.mak.uname b/config.mak.uname index d455929efd..68825c87d2 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -395,6 +395,9 @@ ifeq ($(uname_S),Windows) compat/win32/dirent.o compat/win32/fscache.o COMPAT_CFLAGS = -D__USE_MINGW_ACCESS -DDETECT_MSYS_TTY -DNOGDI -DHAVE_STRING_H -Icompat -Icompat/regex -Icompat/win32 -DSTRIP_EXTENSION=\".exe\" BASIC_LDFLAGS = -IGNORE:4217 -IGNORE:4049 -NOLOGO -SUBSYSTEM:CONSOLE + # invalidcontinue.obj allows Git's source code to close the same file + # handle twice, or to access the osfhandle of an already-closed stdout + # See https://msdn.microsoft.com/en-us/library/ms235330.aspx EXTLIBS = user32.lib advapi32.lib shell32.lib wininet.lib ws2_32.lib invalidcontinue.obj kernel32.lib ntdll.lib PTHREAD_LIBS = lib = From 838e406137d18cffef862cc6391a5816f4775c07 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 30 Nov 2016 12:00:59 +0100 Subject: [PATCH 172/288] t5505,t5516: create .git/branches/ when needed It is a real old anachronism from the Cogito days to have a .git/branches/ directory. And to have tests that ensure that Cogito users can migrate away from using that directory. But so be it, let's continue testing it. Let's make sure, however, that git init does not need to create that directory. This bug was noticed when testing with templates that had been pre-committed, skipping the empty branches/ directory of course because Git does not track empty directories. Signed-off-by: Johannes Schindelin --- t/t5505-remote.sh | 2 ++ t/t5516-fetch-push.sh | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/t/t5505-remote.sh b/t/t5505-remote.sh index 8198d8eb05..b2a7d58600 100755 --- a/t/t5505-remote.sh +++ b/t/t5505-remote.sh @@ -804,6 +804,7 @@ test_expect_success 'migrate a remote from named file in $GIT_DIR/branches' ' ( cd six && git remote rm origin && + mkdir -p .git/branches && echo "$origin_url" >.git/branches/origin && git remote rename origin origin && test_path_is_missing .git/branches/origin && @@ -818,6 +819,7 @@ test_expect_success 'migrate a remote from named file in $GIT_DIR/branches (2)' ( cd seven && git remote rm origin && + mkdir -p .git/branches && echo "quux#foom" > .git/branches/origin && git remote rename origin origin && test_path_is_missing .git/branches/origin && diff --git a/t/t5516-fetch-push.sh b/t/t5516-fetch-push.sh index 26b2cafc47..de5747a39a 100755 --- a/t/t5516-fetch-push.sh +++ b/t/t5516-fetch-push.sh @@ -866,6 +866,7 @@ test_expect_success 'fetch with branches' ' mk_empty testrepo && git branch second $the_first_commit && git checkout second && + mkdir -p testrepo/.git/branches && echo ".." > testrepo/.git/branches/branch1 && ( cd testrepo && @@ -879,6 +880,7 @@ test_expect_success 'fetch with branches' ' test_expect_success 'fetch with branches containing #' ' mk_empty testrepo && + mkdir -p testrepo/.git/branches && echo "..#second" > testrepo/.git/branches/branch2 && ( cd testrepo && @@ -893,6 +895,7 @@ test_expect_success 'fetch with branches containing #' ' test_expect_success 'push with branches' ' mk_empty testrepo && git checkout second && + mkdir -p .git/branches && echo "testrepo" > .git/branches/branch1 && git push branch1 && ( @@ -905,6 +908,7 @@ test_expect_success 'push with branches' ' test_expect_success 'push with branches containing #' ' mk_empty testrepo && + mkdir -p .git/branches && echo "testrepo#branch3" > .git/branches/branch2 && git push branch2 && ( From 86e142bbefd52524468b28eb3c6b977928f697e9 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 28 Nov 2016 18:17:49 +0100 Subject: [PATCH 173/288] git: avoid calling aliased builtins via their dashed form This is one of the few places where Git violates its own deprecation of the dashed form. It is not necessary, either. Signed-off-by: Johannes Schindelin --- git.c | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/git.c b/git.c index c8fe6637df..549432d404 100644 --- a/git.c +++ b/git.c @@ -622,6 +622,32 @@ static int run_argv(int *argcp, const char ***argv) */ if (!done_alias) handle_builtin(*argcp, *argv); + else if (get_builtin(**argv)) { + struct argv_array args = ARGV_ARRAY_INIT; + int i; + + argv_array_push(&args, "git"); + for (i = 0; i < *argcp; i++) + argv_array_push(&args, (*argv)[i]); + + if (get_super_prefix()) + die("%s doesn't support --super-prefix", args.argv[0]); + + if (use_pager == -1) + use_pager = check_pager_config(args.argv[0]); + commit_pager_choice(); + + trace_argv_printf(args.argv, "trace: exec:"); + + /* + * if we fail because the command is not found, it is + * OK to return. Otherwise, we just pass along the status code. + */ + i = run_command_v_opt(args.argv, RUN_SILENT_EXEC_FAILURE | RUN_CLEAN_ON_EXIT); + if (i >= 0 || errno != ENOENT) + exit(i); + die("could not execute builtin %s", args.argv[1]); + } /* .. then try the external ones */ execv_dashed_external(*argv); From f5a4b8667f64afc52d9df9479325f701a8b9992c Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 29 Nov 2016 11:40:55 +0100 Subject: [PATCH 174/288] merge-resolve: avoid dashed call We started deprecating the dashed form a long time ago, advertising the fact that dashed invocations of Git commands are deprecated for several major versions. Yet we still used them ourselves. With ashes on our heads, we now start to finally get rid of those calls. Signed-off-by: Johannes Schindelin --- git-merge-resolve.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git-merge-resolve.sh b/git-merge-resolve.sh index c9da747fcf..343fe7bccd 100755 --- a/git-merge-resolve.sh +++ b/git-merge-resolve.sh @@ -45,7 +45,7 @@ then exit 0 else echo "Simple merge failed, trying Automatic merge." - if git-merge-index -o git-merge-one-file -a + if git merge-index -o git-merge-one-file -a then exit 0 else From 4dd97ca3f7a0aee0200a9f7a7fff74b2561b0c09 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 29 Nov 2016 23:34:18 +0100 Subject: [PATCH 175/288] rebase--merge: avoid dashed invocation We deprecated the dashed invocations ages ago. Might just as well hold ourselves to that. Signed-off-by: Johannes Schindelin --- git-rebase--merge.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git-rebase--merge.sh b/git-rebase--merge.sh index 06a4723d4d..9bc88b7054 100644 --- a/git-rebase--merge.sh +++ b/git-rebase--merge.sh @@ -69,7 +69,7 @@ call_merge () { test -z "$strategy" && strategy=recursive # If cmt doesn't have a parent, don't include it as a base base=$(git rev-parse --verify --quiet $cmt^) - eval 'git-merge-$strategy' $strategy_opts $base ' -- "$hd" "$cmt"' + eval 'git merge-$strategy' $strategy_opts $base ' -- "$hd" "$cmt"' rv=$? case "$rv" in 0) From d2f1136d09b1c6f9fdb99cdddce0ddcbfba29843 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 29 Nov 2016 23:34:18 +0100 Subject: [PATCH 176/288] stash: avoid dashed invocation The invocation of dashed Git commands was rightfully deprecated a long time ago. We failed to heed that deprecation ourselves, but it is never too late... Signed-off-by: Johannes Schindelin --- git-stash.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git-stash.sh b/git-stash.sh index 10c284d1aa..ba3ec4f1eb 100755 --- a/git-stash.sh +++ b/git-stash.sh @@ -502,7 +502,7 @@ apply_stash () { if test -n "$u_tree" then - GIT_INDEX_FILE="$TMPindex" git-read-tree "$u_tree" && + GIT_INDEX_FILE="$TMPindex" git read-tree "$u_tree" && GIT_INDEX_FILE="$TMPindex" git checkout-index --all && rm -f "$TMPindex" || die "$(gettext "Could not restore untracked files from stash")" From 384ab2d6b4dae9366c546fdf1429b3213a1d37b6 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 29 Nov 2016 23:34:18 +0100 Subject: [PATCH 177/288] merge-octopus: avoid dashed invocation A long time ago, in turn a long time after introducing builtins and moving Git's scripts to libexec/git-core/, we deprecated the invocation of dashed commands. The merge-octopus script is a holdover from the time before that, as it still tries to invoke a builtin by its dashed form. Let's just not. Signed-off-by: Johannes Schindelin --- git-merge-octopus.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git-merge-octopus.sh b/git-merge-octopus.sh index bcf0d92ec2..6c390d6c22 100755 --- a/git-merge-octopus.sh +++ b/git-merge-octopus.sh @@ -100,7 +100,7 @@ do if test $? -ne 0 then gettextln "Simple merge did not work, trying automatic merge." - git-merge-index -o git-merge-one-file -a || + git merge-index -o git-merge-one-file -a || OCTOPUS_FAILURE=1 next=$(git write-tree 2>/dev/null) fi From 77a54d858fc033734f9f6d3d62dfdd4a9fd5c10e Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 29 Nov 2016 23:34:18 +0100 Subject: [PATCH 178/288] merge-one-file: avoid dashed invocation The original idea of deprecating invocations of Git subcommands via their dashed form was to be able to ship without having to hard-link each and every builtin to its dashed form. We need to follow this plan ourselves, by patching our very own scripts accordingly. Signed-off-by: Johannes Schindelin --- git-merge-one-file.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/git-merge-one-file.sh b/git-merge-one-file.sh index 424b034e34..9879c59395 100755 --- a/git-merge-one-file.sh +++ b/git-merge-one-file.sh @@ -115,16 +115,16 @@ case "${1:-.}${2:-.}${3:-.}" in ;; esac - src1=$(git-unpack-file $2) - src2=$(git-unpack-file $3) + src1=$(git unpack-file $2) + src2=$(git unpack-file $3) case "$1" in '') echo "Added $4 in both, but differently." - orig=$(git-unpack-file e69de29bb2d1d6434b8b29ae775ad8c2e48c5391) + orig=$(git unpack-file e69de29bb2d1d6434b8b29ae775ad8c2e48c5391) ;; *) echo "Auto-merging $4" - orig=$(git-unpack-file $1) + orig=$(git unpack-file $1) ;; esac From 6ae25c4c12eca774ada1bb995bf31fd70161a033 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 29 Nov 2016 23:34:18 +0100 Subject: [PATCH 179/288] submodule: avoid dashed invocation In the years-old effort to clean up the PATH a bit, we deprecated dashed invocations. But we did not hold ourselves to that. Until now. Signed-off-by: Johannes Schindelin --- git-submodule.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/git-submodule.sh b/git-submodule.sh index a024a135d6..4aac71f076 100755 --- a/git-submodule.sh +++ b/git-submodule.sh @@ -843,7 +843,7 @@ cmd_summary() { test $status != A && test $ignore_config = all && continue fi # Also show added or modified modules which are checked out - GIT_DIR="$sm_path/.git" git-rev-parse --git-dir >/dev/null 2>&1 && + GIT_DIR="$sm_path/.git" git rev-parse --git-dir >/dev/null 2>&1 && printf '%s\n' "$sm_path" done ) @@ -877,11 +877,11 @@ cmd_summary() { missing_dst= test $mod_src = 160000 && - ! GIT_DIR="$name/.git" git-rev-parse -q --verify $sha1_src^0 >/dev/null && + ! GIT_DIR="$name/.git" git rev-parse -q --verify $sha1_src^0 >/dev/null && missing_src=t test $mod_dst = 160000 && - ! GIT_DIR="$name/.git" git-rev-parse -q --verify $sha1_dst^0 >/dev/null && + ! GIT_DIR="$name/.git" git rev-parse -q --verify $sha1_dst^0 >/dev/null && missing_dst=t display_name=$(git submodule--helper relative-path "$name" "$wt_prefix") From 0eebda819178bda693b75823eef94153d6880652 Mon Sep 17 00:00:00 2001 From: Philip Oakley Date: Sun, 19 Jul 2015 16:02:40 +0100 Subject: [PATCH 180/288] .gitignore: improve MSVC ignore patterns Add the Microsoft .manifest pattern, and correct the generic 'Debug' and 'Release' directory patterns which were mechanically adjusted way back in c591d5f (gitignore: root most patterns at the top-level directory, 2009-10-26) to allow multi-level projects within the Git suite. Signed-off-by: Philip Oakley --- .gitignore | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 426bd70d1a..64275664fe 100644 --- a/.gitignore +++ b/.gitignore @@ -218,5 +218,6 @@ *.pdb *.ilk .vs/ -/Debug/ -/Release/ +*.manifest +Debug/ +Release/ From a70b0ea44d7fb614f51fca9e248e7c52d041e3fa Mon Sep 17 00:00:00 2001 From: Philip Oakley Date: Mon, 23 Feb 2015 12:50:35 +0000 Subject: [PATCH 181/288] .gitignore: ignore library directories created by MSVC VS2008 buildsystem Signed-off-by: Philip Oakley --- .gitignore | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/.gitignore b/.gitignore index 64275664fe..92c7c1eec4 100644 --- a/.gitignore +++ b/.gitignore @@ -179,7 +179,43 @@ /gitweb/gitweb.cgi /gitweb/static/gitweb.js /gitweb/static/gitweb.min.* +/libgit +/test-chmtime +/test-ctype +/test-config +/test-date +/test-delta +/test-dump-cache-tree +/test-dump-split-index +/test-dump-untracked-cache +/test-fake-ssh +/test-scrap-cache-tree +/test-genrandom +/test-hashmap +/test-index-version +/test-line-buffer +/test-match-trees +/test-mergesort +/test-mktemp +/test-parse-options +/test-path-utils +/test-prio-queue +/test-read-cache +/test-regex +/test-revision-walking +/test-run-command +/test-sha1 +/test-sha1-array +/test-sigchain +/test-string-list +/test-submodule-config +/test-subprocess +/test-svn-fe +/test-urlmatch-normalization +/test-wildmatch /common-cmds.h +/vcs-svn_lib +/xdiff_lib *.tar.gz *.dsc *.deb From 6d485e9d301a19eb36926870dfba429d2b0e5d95 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 25 Oct 2016 06:06:10 -0700 Subject: [PATCH 182/288] .gitignore: ignore Visual Studio's temporary/generated files Signed-off-by: Johannes Schindelin --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 92c7c1eec4..46cb500304 100644 --- a/.gitignore +++ b/.gitignore @@ -257,3 +257,6 @@ *.manifest Debug/ Release/ +/UpgradeLog*.htm +/git.VC.VC.opendb +/git.VC.db From b0e7662d68e1980119ffe38d31e67b1e84182dd4 Mon Sep 17 00:00:00 2001 From: Philip Oakley Date: Sun, 19 Jul 2015 17:41:13 +0100 Subject: [PATCH 183/288] contrib/buildsystems: fix misleading error message The error message talked about a "lib option", but it clearly referred to a link option. Signed-off-by: Philip Oakley Signed-off-by: Johannes Schindelin --- contrib/buildsystems/engine.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/buildsystems/engine.pl b/contrib/buildsystems/engine.pl index 23da787dc5..c8a5258be5 100755 --- a/contrib/buildsystems/engine.pl +++ b/contrib/buildsystems/engine.pl @@ -329,7 +329,7 @@ sub handleLinkLine } elsif ($part =~ /\.(o|obj)$/) { push(@objfiles, $part); } else { - die "Unhandled lib option @ line $lineno: $part"; + die "Unhandled link option @ line $lineno: $part"; } } # print "AppOut: '$appout'\nLFlags: @lflags\nLibs : @libs\nOfiles: @objfiles\n"; From 4346e49e1e17954a63ae506226a74923a262f379 Mon Sep 17 00:00:00 2001 From: Philip Oakley Date: Sun, 19 Jul 2015 16:08:21 +0100 Subject: [PATCH 184/288] contrib/buildsystems: handle quoted spaces in filenames The engine.pl script expects file names not to contain spaces. However, paths with spaces are quite prevalent on Windows. Use shellwords() rather than split() to parse them correctly. Helped-by: Junio C Hamano Signed-off-by: Philip Oakley Signed-off-by: Johannes Schindelin --- contrib/buildsystems/engine.pl | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/contrib/buildsystems/engine.pl b/contrib/buildsystems/engine.pl index c8a5258be5..24b8992baf 100755 --- a/contrib/buildsystems/engine.pl +++ b/contrib/buildsystems/engine.pl @@ -12,6 +12,7 @@ use File::Basename; use File::Spec; use Cwd; use Generators; +use Text::ParseWords; my (%build_structure, %compile_options, @makedry); my $out_dir = getcwd(); @@ -231,7 +232,7 @@ sub removeDuplicates sub handleCompileLine { my ($line, $lineno) = @_; - my @parts = split(' ', $line); + my @parts = shellwords($line); my $sourcefile; shift(@parts); # ignore cmd while (my $part = shift @parts) { @@ -265,7 +266,7 @@ sub handleLibLine my (@objfiles, @lflags, $libout, $part); # kill cmd and rm 'prefix' $line =~ s/^rm -f .* && .* rcs //; - my @parts = split(' ', $line); + my @parts = shellwords($line); while ($part = shift @parts) { if ($part =~ /^-/) { push(@lflags, $part); @@ -306,7 +307,7 @@ sub handleLinkLine { my ($line, $lineno) = @_; my (@objfiles, @lflags, @libs, $appout, $part); - my @parts = split(' ', $line); + my @parts = shellwords($line); shift(@parts); # ignore cmd while ($part = shift @parts) { if ($part =~ /^-IGNORE/) { From 05b9af3ffc92eb48970397ef1618e0df2a41b102 Mon Sep 17 00:00:00 2001 From: Philip Oakley Date: Mon, 9 Feb 2015 14:34:29 +0000 Subject: [PATCH 185/288] contrib/buildsystems: ignore gettext stuff Git's build contains steps to handle internationalization. This caused hiccups in the parser used to generate QMake/Visual Studio project files. As those steps are irrelevant in this context, let's just ignore them. Signed-off-by: Philip Oakley Signed-off-by: Johannes Schindelin --- contrib/buildsystems/engine.pl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/contrib/buildsystems/engine.pl b/contrib/buildsystems/engine.pl index 24b8992baf..60c7a7d76d 100755 --- a/contrib/buildsystems/engine.pl +++ b/contrib/buildsystems/engine.pl @@ -141,6 +141,12 @@ sub parseMakeOutput next; } + if ($text =~ /^(mkdir|msgfmt) /) { + # options to the Portable Object translations + # the line "mkdir ... && msgfmt ..." contains no linker options + next; + } + if($text =~ / -c /) { # compilation handleCompileLine($text, $line); From 08c6ed3eb3ea70893a2b1ac2f0a79199dba34b36 Mon Sep 17 00:00:00 2001 From: Philip Oakley Date: Sun, 19 Jul 2015 16:45:32 +0100 Subject: [PATCH 186/288] contrib/buildsystems: redirect errors of the dry run into a log file Rather than swallowing the errors, it is better to have them in a file. To make it obvious what this is about, use the file name 'msvc-build-makedryerrors.txt'. Further, if the output is empty, simply delete that file. As we target Git for Windows' SDK (which, unlike its predecessor msysGit, offers Perl versions newer than 5.8), we can use the quite readable syntax `if -f -z $ErrsFile` (available in Perl >=5.10). Note that the file will contain the new values of the GIT_VERSION and GITGUI_VERSION if they were generated by the make file. They are omitted if the release is tagged and indentically defined in their respective GIT_VERSION_GEN file DEF_VER variables. Signed-off-by: Philip Oakley Signed-off-by: Johannes Schindelin --- contrib/buildsystems/engine.pl | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/contrib/buildsystems/engine.pl b/contrib/buildsystems/engine.pl index 60c7a7d76d..33298c1f61 100755 --- a/contrib/buildsystems/engine.pl +++ b/contrib/buildsystems/engine.pl @@ -73,7 +73,12 @@ Running GNU Make to figure out build structure... EOM # Pipe a make --dry-run into a variable, if not already loaded from file -@makedry = `cd $git_dir && make -n MSVC=1 V=1 2>/dev/null` if !@makedry; +# Capture the make dry stderr to file for review (will be empty for a release build). + +my $ErrsFile = "msvc-build-makedryerrors.txt"; +@makedry = `cd $git_dir && make -n MSVC=1 V=1 2>$ErrsFile` if !@makedry; +# test for an empty Errors file and remove it +unlink $ErrsFile if -f -z $ErrsFile; # Parse the make output into usable info parseMakeOutput(); From 14212510421f851a067e75de3f5e10867b1ec89d Mon Sep 17 00:00:00 2001 From: Philip Oakley Date: Wed, 4 May 2016 15:53:48 +0100 Subject: [PATCH 187/288] contrib/buildsystems: handle the curl library option Upon seeing the '-lcurl' option, point to the libcurl.lib. While there, fix the elsif indentation. Signed-off-by: Philip Oakley Signed-off-by: Johannes Schindelin --- contrib/buildsystems/engine.pl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/contrib/buildsystems/engine.pl b/contrib/buildsystems/engine.pl index 33298c1f61..4db3a2aa99 100755 --- a/contrib/buildsystems/engine.pl +++ b/contrib/buildsystems/engine.pl @@ -329,10 +329,12 @@ sub handleLinkLine $appout = shift @parts; } elsif ("$part" eq "-lz") { push(@libs, "zlib.lib"); - } elsif ("$part" eq "-lcrypto") { + } elsif ("$part" eq "-lcrypto") { push(@libs, "libeay32.lib"); } elsif ("$part" eq "-lssl") { push(@libs, "ssleay32.lib"); + } elsif ("$part" eq "-lcurl") { + push(@libs, "libcurl.lib"); } elsif ($part =~ /^-/) { push(@lflags, $part); } elsif ($part =~ /\.(a|lib)$/) { From 973ca18ad3704914160822630d1e31ab75105069 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 26 Oct 2016 04:59:06 -0700 Subject: [PATCH 188/288] contrib/buildsystems: ignore irrelevant files in Generators/ The Generators/ directory can contain spurious files such as editors' backup files. Even worse, there could be .swp files which are not even valid Perl scripts. Let's just ignore anything but .pm files in said directory. Signed-off-by: Johannes Schindelin --- contrib/buildsystems/Generators.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/buildsystems/Generators.pm b/contrib/buildsystems/Generators.pm index 408ef714b8..aa4cbaa2ad 100644 --- a/contrib/buildsystems/Generators.pm +++ b/contrib/buildsystems/Generators.pm @@ -17,7 +17,7 @@ BEGIN { $me = dirname($me); if (opendir(D,"$me/Generators")) { foreach my $gen (readdir(D)) { - next if ($gen =~ /^\.\.?$/); + next unless ($gen =~ /\.pm$/); require "${me}/Generators/$gen"; $gen =~ s,\.pm,,; push(@AVAILABLE, $gen); From 646f3e745690c3002b5cbfd40ec49e9e540b6e54 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 27 Oct 2016 01:45:49 -0700 Subject: [PATCH 189/288] contrib/buildsystems: handle libiconv, too Git's test suite shows tons of breakages unless Git is compiled *without* NO_ICONV. That means, in turn, that we need to generate build definitions *with* libiconv, which in turn implies that we have to handle the -liconv option properly. Signed-off-by: Johannes Schindelin --- contrib/buildsystems/engine.pl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/contrib/buildsystems/engine.pl b/contrib/buildsystems/engine.pl index 4db3a2aa99..94100b3e0a 100755 --- a/contrib/buildsystems/engine.pl +++ b/contrib/buildsystems/engine.pl @@ -335,6 +335,8 @@ sub handleLinkLine push(@libs, "ssleay32.lib"); } elsif ("$part" eq "-lcurl") { push(@libs, "libcurl.lib"); + } elsif ("$part" eq "-liconv") { + push(@libs, "libiconv.lib"); } elsif ($part =~ /^-/) { push(@lflags, $part); } elsif ($part =~ /\.(a|lib)$/) { From 62c6220e668746fd981c7613abe77b05a61e7372 Mon Sep 17 00:00:00 2001 From: Philip Oakley Date: Thu, 16 Jul 2015 23:40:13 +0100 Subject: [PATCH 190/288] contrib/buildsystems: optionally capture the dry-run in a file Add an option for capturing the output of the make dry-run used in determining the msvc-build structure for easy debugging. You can use the output of `--make-out ` in subsequent runs via the `--in ` option. Signed-off-by: Philip Oakley Signed-off-by: Johannes Schindelin --- contrib/buildsystems/engine.pl | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/contrib/buildsystems/engine.pl b/contrib/buildsystems/engine.pl index 94100b3e0a..2aac540ded 100755 --- a/contrib/buildsystems/engine.pl +++ b/contrib/buildsystems/engine.pl @@ -32,6 +32,7 @@ generate usage: -g --gen Specify the buildsystem generator (default: $gen) Available: $genlist -o --out Specify output directory generation (default: .) + --make-out Write the output of GNU Make into a file -i --in Specify input file, instead of running GNU Make -h,-? --help This help EOM @@ -39,6 +40,7 @@ EOM } # Parse command-line options +my $make_out; while (@ARGV) { my $arg = shift @ARGV; if ("$arg" eq "-h" || "$arg" eq "--help" || "$arg" eq "-?") { @@ -46,6 +48,8 @@ while (@ARGV) { exit(0); } elsif("$arg" eq "--out" || "$arg" eq "-o") { $out_dir = shift @ARGV; + } elsif("$arg" eq "--make-out") { + $make_out = shift @ARGV; } elsif("$arg" eq "--gen" || "$arg" eq "-g") { $gen = shift @ARGV; } elsif("$arg" eq "--in" || "$arg" eq "-i") { @@ -80,6 +84,12 @@ my $ErrsFile = "msvc-build-makedryerrors.txt"; # test for an empty Errors file and remove it unlink $ErrsFile if -f -z $ErrsFile; +if (defined $make_out) { + open OUT, ">" . $make_out; + print OUT @makedry; + close OUT; +} + # Parse the make output into usable info parseMakeOutput(); From 6515995a43bd30b2b2dbb00a4cbba487599e65c1 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 27 Oct 2016 07:25:00 -0700 Subject: [PATCH 191/288] contrib/buildsystems: handle options starting with a slash With the recent changes to allow building with MSVC=1, we now pass the /OPT:REF option to the compiler. This confuses the parser that wants to turn the output of a dry run into project definitions for QMake and Visual Studio: Unhandled link option @ line 213: /OPT:REF at [...] Let's just extend the code that passes through options that start with a dash, so that it passes through options that start with a slash, too. Signed-off-by: Johannes Schindelin --- contrib/buildsystems/engine.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/buildsystems/engine.pl b/contrib/buildsystems/engine.pl index 2aac540ded..328d3ec39d 100755 --- a/contrib/buildsystems/engine.pl +++ b/contrib/buildsystems/engine.pl @@ -347,7 +347,7 @@ sub handleLinkLine push(@libs, "libcurl.lib"); } elsif ("$part" eq "-liconv") { push(@libs, "libiconv.lib"); - } elsif ($part =~ /^-/) { + } elsif ($part =~ /^[-\/]/) { push(@lflags, $part); } elsif ($part =~ /\.(a|lib)$/) { $part =~ s/\.a$/.lib/; From c00972749e2fcc05f74163892cadc9c58bd639e2 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 26 Oct 2016 02:28:10 -0700 Subject: [PATCH 192/288] contrib/buildsystems: support modern Visual Studio project definitions The .sln/.vcproj files were used to define projects up until Visual Studio 2008, but starting with Visual Studio 2010 the project definitions are stored in .sln/.vcxproj files (which can also be used by the MSBuild system). Let's just copy-edit the generator of the .vcproj files to a new generator that produces .vcxproj files directly, without forcing Visual Studio to upgrade the project definitions. Signed-off-by: Johannes Schindelin --- contrib/buildsystems/Generators/Vcxproj.pm | 399 +++++++++++++++++++++ 1 file changed, 399 insertions(+) create mode 100644 contrib/buildsystems/Generators/Vcxproj.pm diff --git a/contrib/buildsystems/Generators/Vcxproj.pm b/contrib/buildsystems/Generators/Vcxproj.pm new file mode 100644 index 0000000000..be56ed694d --- /dev/null +++ b/contrib/buildsystems/Generators/Vcxproj.pm @@ -0,0 +1,399 @@ +package Generators::Vcxproj; +require Exporter; + +use strict; +use vars qw($VERSION); + +our $VERSION = '1.00'; +our(@ISA, @EXPORT, @EXPORT_OK, @AVAILABLE); +@ISA = qw(Exporter); + +BEGIN { + push @EXPORT_OK, qw(generate); +} + +my $guid_index = 0; +my @GUIDS = ( + "{E07B9989-2BF7-4F21-8918-BE22BA467AC3}", + "{278FFB51-0296-4A44-A81A-22B87B7C3592}", + "{7346A2C4-F0FD-444F-9EBE-1AF23B2B5650}", + "{67F421AC-EB34-4D49-820B-3196807B423F}", + "{385DCFE1-CC8C-4211-A451-80FCFC31CA51}", + "{97CC46C5-D2CC-4D26-B634-E75792B79916}", + "{C7CE21FE-6EF8-4012-A5C7-A22BCEDFBA11}", + "{51575134-3FDF-42D1-BABD-3FB12669C6C9}", + "{0AE195E4-9823-4B87-8E6F-20C5614AF2FF}", + "{4B918255-67CA-43BB-A46C-26704B666E6B}", + "{18CCFEEF-C8EE-4CC1-A265-26F95C9F4649}", + "{5D5D90FA-01B7-4973-AFE5-CA88C53AC197}", + "{1F054320-036D-49E1-B384-FB5DF0BC8AC0}", + "{7CED65EE-F2D9-4171-825B-C7D561FE5786}", + "{8D341679-0F07-4664-9A56-3BA0DE88B9BC}", + "{C189FEDC-2957-4BD7-9FA4-7622241EA145}", + "{66844203-1B9F-4C53-9274-164FFF95B847}", + "{E4FEA145-DECC-440D-AEEA-598CF381FD43}", + "{73300A8E-C8AC-41B0-B555-4F596B681BA7}", + "{873FDEB1-D01D-40BF-A1BF-8BBC58EC0F51}", + "{7922C8BE-76C5-4AC6-8BF7-885C0F93B782}", + "{E245D370-308B-4A49-BFC1-1E527827975F}", + "{F6FA957B-66FC-4ED7-B260-E59BBE4FE813}", + "{E6055070-0198-431A-BC49-8DB6CEE770AE}", + "{54159234-C3EB-43DA-906B-CE5DA5C74654}", + "{594CFC35-0B60-46F6-B8EF-9983ACC1187D}", + "{D93FCAB7-1F01-48D2-B832-F761B83231A5}", + "{DBA5E6AC-E7BE-42D3-8703-4E787141526E}", + "{6171953F-DD26-44C7-A3BE-CC45F86FC11F}", + "{9E19DDBE-F5E4-4A26-A2FE-0616E04879B8}", + "{AE81A615-99E3-4885-9CE0-D9CAA193E867}", + "{FBF4067E-1855-4F6C-8BCD-4D62E801A04D}", + "{17007948-6593-4AEB-8106-F7884B4F2C19}", + "{199D4C8D-8639-4DA6-82EF-08668C35DEE0}", + "{E085E50E-C140-4CF3-BE4B-094B14F0DDD6}", + "{00785268-A9CC-4E40-AC29-BAC0019159CE}", + "{4C06F56A-DCDB-46A6-B67C-02339935CF12}", + "{3A62D3FD-519E-4EC9-8171-D2C1BFEA022F}", + "{9392EB58-D7BA-410B-B1F0-B2FAA6BC89A7}", + "{2ACAB2D5-E0CE-4027-BCA0-D78B2D7A6C66}", + "{86E216C3-43CE-481A-BCB2-BE5E62850635}", + "{FB631291-7923-4B91-9A57-7B18FDBB7A42}", + "{0A176EC9-E934-45B8-B87F-16C7F4C80039}", + "{DF55CA80-46E8-4C53-B65B-4990A23DD444}", + "{3A0F9895-55D2-4710-BE5E-AD7498B5BF44}", + "{294BDC5A-F448-48B6-8110-DD0A81820F8C}", + "{4B9F66E9-FAC9-47AB-B1EF-C16756FBFD06}", + "{72EA49C6-2806-48BD-B81B-D4905102E19C}", + "{5728EB7E-8929-486C-8CD5-3238D060E768}", + "{A3E300FC-5630-4850-A470-E9F2C2EFA7E7}", + "{CEA071D4-D9F3-4250-98F7-44AFDC8ACAA1}", + "{3FD87BB4-2236-4A1B-ADD2-46211A302442}", + "{49B03F41-5157-4079-95A7-64D728BCF74F}", + "{95D5A28B-80E2-40A9-BEA3-C52B9CA488E3}", + "{B85E6545-D523-4323-9F29-45389D090343}", + "{06840CEF-746C-4B71-9442-C395DD6590A5}" +); + +sub generate { + my ($git_dir, $out_dir, $rel_dir, %build_structure) = @_; + my @libs = @{$build_structure{"LIBS"}}; + foreach (@libs) { + createProject($_, $git_dir, $out_dir, $rel_dir, \%build_structure, 1); + } + + my @apps = @{$build_structure{"APPS"}}; + foreach (@apps) { + createProject($_, $git_dir, $out_dir, $rel_dir, \%build_structure, 0); + } + + createGlueProject($git_dir, $out_dir, $rel_dir, %build_structure); + return 0; +} + +sub createProject { + my ($name, $git_dir, $out_dir, $rel_dir, $build_structure, $static_library) = @_; + my $label = $static_library ? "lib" : "app"; + my $prefix = $static_library ? "LIBS_" : "APPS_"; + my $config_type = $static_library ? "StaticLibrary" : "Utility"; + print "Generate $name vcxproj $label project\n"; + my $cdup = $name; + $cdup =~ s/[^\/]+/../g; + $cdup =~ s/\//\\/g; + $rel_dir = $rel_dir eq "." ? $cdup : "$cdup\\$rel_dir"; + $rel_dir =~ s/\//\\/g; + + my $target = $name; + if ($static_library) { + $target =~ s/\.a//; + } else { + $target =~ s/\.exe//; + } + + my $uuid = $GUIDS[$guid_index++]; + $$build_structure{"$prefix${target}_GUID"} = $uuid; + my $vcxproj = $target; + $vcxproj =~ s/(.*\/)?(.*)/$&\/$2.vcxproj/; + $vcxproj =~ s/([^\/]*)(\/lib)\/(lib.vcxproj)/$1$2\/$1_$3/; + $$build_structure{"$prefix${target}_VCXPROJ"} = $vcxproj; + + my @srcs = sort(map("$rel_dir\\$_", @{$$build_structure{"$prefix${name}_SOURCES"}})); + my @sources; + foreach (@srcs) { + $_ =~ s/\//\\/g; + push(@sources, $_); + } + my $defines = join(";", sort(@{$$build_structure{"$prefix${name}_DEFINES"}})); + my $includes= join(";", sort(map { s/^-I//; s/\//\\/g; File::Spec->file_name_is_absolute($_) ? $_ : "$rel_dir\\$_" } @{$$build_structure{"$prefix${name}_INCLUDES"}})); + my $cflags = join(" ", sort(map { s/^-[GLMOZ].*//; s/.* .*/"$&"/; $_; } @{$$build_structure{"$prefix${name}_CFLAGS"}})); + $cflags =~ s/\"/"/g; + $cflags =~ s//>/g; + + my $libs = ''; + if (!$static_library) { + $libs = join(";", sort(grep /^(?!libgit\.lib|xdiff\/lib\.lib)/, @{$$build_structure{"$prefix${name}_LIBS"}})); + } + my $ldirs = join(";", map { /^-L(?!TCG)(.*)/ ? (File::Spec->file_name_is_absolute($1) ? $1 : "$rel_dir\\$1") : () } @{$$build_structure{"$prefix${name}_LFLAGS"}}); + + $defines =~ s/-D//g; + $defines =~ s/\"/"/g; + $defines =~ s//>/g; + $defines =~ s/\'//g; + + die "Could not create the directory $target for $label project!\n" unless (-d "$target" || mkdir "$target"); + open F, ">$vcxproj" or die "Could not open $vcxproj for writing!\n"; + binmode F, ":crlf :utf8"; + print F chr(0xFEFF); + print F << "EOM"; + + + + + Debug + Win32 + + + Release + Win32 + + + Debug + x64 + + + Release + x64 + + + + $uuid + Win32Proj + + + + true + true + + + false + true + + + $config_type + v140 + + ..\\ + + + + + + + + + + + + + false + true + + + + $cflags %(AdditionalOptions) + $includes;%(AdditionalIncludeDirectories) + + true + OnlyExplicitInline + + ProgramDatabase + + + true + + + $libs;\$(AdditionalDependencies) + $ldirs + $cdup\\compat\\win32\\git.manifest + Console + + + + + MachineX86 + + + + + Disabled + WIN32;_DEBUG;$defines;%(PreprocessorDefinitions) + MultiThreadedDebugDLL + + + true + + + + + MaxSpeed + true + WIN32;NDEBUG;$defines;%(PreprocessorDefinitions) + MultiThreadedDLL + true + Speed + + + true + true + true + + + +EOM + foreach(@sources) { + print F << "EOM"; + +EOM + } + print F << "EOM"; + +EOM + if (!$static_library) { + my $uuid_libgit = $$build_structure{"LIBS_libgit_GUID"}; + my $uuid_xdiff_lib = $$build_structure{"LIBS_xdiff/lib_GUID"}; + + print F << "EOM"; + + + $uuid_libgit + false + + + $uuid_xdiff_lib + false + +EOM + if ($name =~ /(test-(line-buffer|svn-fe)|^git-remote-testsvn)\.exe$/) { + my $uuid_vcs_svn_lib = $$build_structure{"LIBS_vcs-svn/lib_GUID"}; + print F << "EOM"; + + $uuid_vcs_svn_lib + false + +EOM + } + print F << "EOM"; + +EOM + } + print F << "EOM"; + + + + +EOM + close F; +} + +sub createGlueProject { + my ($git_dir, $out_dir, $rel_dir, %build_structure) = @_; + print "Generate solutions file\n"; + $rel_dir = "..\\$rel_dir"; + $rel_dir =~ s/\//\\/g; + my $SLN_HEAD = "Microsoft Visual Studio Solution File, Format Version 11.00\n# Visual Studio 2010\n"; + my $SLN_PRE = "Project(\"{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}\") = "; + my $SLN_POST = "\nEndProject\n"; + + my @libs = @{$build_structure{"LIBS"}}; + my @tmp; + foreach (@libs) { + $_ =~ s/\.a//; + push(@tmp, $_); + } + @libs = @tmp; + + my @apps = @{$build_structure{"APPS"}}; + @tmp = (); + foreach (@apps) { + $_ =~ s/\.exe//; + if ($_ eq "git" ) { + unshift(@tmp, $_); + } else { + push(@tmp, $_); + } + } + @apps = @tmp; + + open F, ">git.sln" || die "Could not open git.sln for writing!\n"; + binmode F, ":crlf :utf8"; + print F chr(0xFEFF); + print F "$SLN_HEAD"; + + foreach (@apps) { + my $appname = $_; + my $uuid = $build_structure{"APPS_${appname}_GUID"}; + print F "$SLN_PRE"; + my $vcxproj = $build_structure{"APPS_${appname}_VCXPROJ"}; + $vcxproj =~ s/\//\\/g; + $appname =~ s/.*\///; + print F "\"${appname}\", \"${vcxproj}\", \"${uuid}\""; + print F "$SLN_POST"; + } + foreach (@libs) { + my $libname = $_; + my $uuid = $build_structure{"LIBS_${libname}_GUID"}; + print F "$SLN_PRE"; + my $vcxproj = $build_structure{"LIBS_${libname}_VCXPROJ"}; + $vcxproj =~ s/\//\\/g; + $libname =~ s/\//_/g; + print F "\"${libname}\", \"${vcxproj}\", \"${uuid}\""; + print F "$SLN_POST"; + } + + print F << "EOM"; +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|x64 = Debug|x64 + Debug|x86 = Debug|x86 + Release|x64 = Release|x64 + Release|x86 = Release|x86 + EndGlobalSection +EOM + print F << "EOM"; + GlobalSection(ProjectConfigurationPlatforms) = postSolution +EOM + foreach (@apps) { + my $appname = $_; + my $uuid = $build_structure{"APPS_${appname}_GUID"}; + print F "\t\t${uuid}.Debug|x64.ActiveCfg = Debug|x64\n"; + print F "\t\t${uuid}.Debug|x64.Build.0 = Debug|x64\n"; + print F "\t\t${uuid}.Debug|x86.ActiveCfg = Debug|Win32\n"; + print F "\t\t${uuid}.Debug|x86.Build.0 = Debug|Win32\n"; + print F "\t\t${uuid}.Release|x64.ActiveCfg = Release|x64\n"; + print F "\t\t${uuid}.Release|x64.Build.0 = Release|x64\n"; + print F "\t\t${uuid}.Release|x86.ActiveCfg = Release|Win32\n"; + print F "\t\t${uuid}.Release|x86.Build.0 = Release|Win32\n"; + } + foreach (@libs) { + my $libname = $_; + my $uuid = $build_structure{"LIBS_${libname}_GUID"}; + print F "\t\t${uuid}.Debug|x64.ActiveCfg = Debug|x64\n"; + print F "\t\t${uuid}.Debug|x64.Build.0 = Debug|x64\n"; + print F "\t\t${uuid}.Debug|x86.ActiveCfg = Debug|Win32\n"; + print F "\t\t${uuid}.Debug|x86.Build.0 = Debug|Win32\n"; + print F "\t\t${uuid}.Release|x64.ActiveCfg = Release|x64\n"; + print F "\t\t${uuid}.Release|x64.Build.0 = Release|x64\n"; + print F "\t\t${uuid}.Release|x86.ActiveCfg = Release|Win32\n"; + print F "\t\t${uuid}.Release|x86.Build.0 = Release|Win32\n"; + } + + print F << "EOM"; + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal +EOM + close F; +} + +1; From 3dc805e7428198f546db1792fe6de0a6578e9354 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sat, 5 Nov 2016 15:05:47 +0100 Subject: [PATCH 193/288] contrib/buildsystems: make 'Restore NuGet Packages' work in Visual Studio Visual Studio has this very neat feature that you can get dependencies in the form of NuGet packages, and even further: you can specify in a project what NuGet packages it needs. These dependencies can then be fetched via right-clicking the solution in the Solution Explorer and clicking the "Restore NuGet Packages" entry. This feature is so neat, in fact, that we want to support it in Git for Windows. The idea is that we will be able to provide developers with a checkout of the Git sources that can be built outside of the Git for Windows SDK, using *only* Visual Studio. Signed-off-by: Johannes Schindelin --- contrib/buildsystems/Generators/Vcxproj.pm | 56 ++++++++++++++++++++-- 1 file changed, 53 insertions(+), 3 deletions(-) diff --git a/contrib/buildsystems/Generators/Vcxproj.pm b/contrib/buildsystems/Generators/Vcxproj.pm index be56ed694d..154d74fab3 100644 --- a/contrib/buildsystems/Generators/Vcxproj.pm +++ b/contrib/buildsystems/Generators/Vcxproj.pm @@ -92,7 +92,7 @@ sub createProject { my ($name, $git_dir, $out_dir, $rel_dir, $build_structure, $static_library) = @_; my $label = $static_library ? "lib" : "app"; my $prefix = $static_library ? "LIBS_" : "APPS_"; - my $config_type = $static_library ? "StaticLibrary" : "Utility"; + my $config_type = $static_library ? "StaticLibrary" : "Application"; print "Generate $name vcxproj $label project\n"; my $cdup = $name; $cdup =~ s/[^\/]+/../g; @@ -129,7 +129,7 @@ sub createProject { my $libs = ''; if (!$static_library) { - $libs = join(";", sort(grep /^(?!libgit\.lib|xdiff\/lib\.lib)/, @{$$build_structure{"$prefix${name}_LIBS"}})); + $libs = join(";", sort(grep /^(?!libgit\.lib|xdiff\/lib\.lib|vcs-svn\/lib\.lib|libcurl\.lib|libeay32\.lib|libiconv\.lib|ssleay32\.lib|zlib\.lib)/, @{$$build_structure{"$prefix${name}_LIBS"}})); } my $ldirs = join(";", map { /^-L(?!TCG)(.*)/ ? (File::Spec->file_name_is_absolute($1) ? $1 : "$rel_dir\\$1") : () } @{$$build_structure{"$prefix${name}_LFLAGS"}}); @@ -140,6 +140,39 @@ sub createProject { $defines =~ s/\'//g; die "Could not create the directory $target for $label project!\n" unless (-d "$target" || mkdir "$target"); + + use File::Copy; + copy("$git_dir/compat/vcbuild/packages.config", "$target/packages.config"); + + my $needsCurl = grep(/libcurl.lib/, @{$$build_structure{"$prefix${name}_LIBS"}}); + my $targetsImport = ''; + my $targetsErrors = ''; + my $afterTargets = ''; + open F, "<$git_dir/compat/vcbuild/packages.config"; + while () { + if (/"; + } elsif ($needsCurl && $1 eq 'curl') { + # libcurl is only available targeting v100 and v110 + $libs .= ";$rel_dir\\compat\\vcbuild\\GEN.PKGS\\$1.$2\\build\\native\\lib\\v110\\\$(Platform)\\Release\\dynamic\\libcurl.lib"; + $afterTargets .= "\n "; + } elsif ($needsCurl && $1 eq 'expat') { + # libexpat is only available targeting v100 and v110 + $libs .= ";$rel_dir\\compat\\vcbuild\\GEN.PKGS\\$1.$2\\build\\native\\lib\\v110\\\$(Platform)\\Release\\dynamic\\utf8\\libexpat.lib"; + } + next if ($1 =~ /^(zlib$|openssl(?!.*(x64|x86)$))/); + my $targetsFile = "$rel_dir\\compat\\vcbuild\\GEN.PKGS\\$1.$2\\build\\native\\$1.targets"; + $targetsImport .= "\n "; + $targetsErrors .= "\n "; + } + } + close F; + open F, ">$vcxproj" or die "Could not open $vcxproj for writing!\n"; binmode F, ":crlf :utf8"; print F chr(0xFEFF); @@ -213,6 +246,7 @@ sub createProject { $libs;\$(AdditionalDependencies) $ldirs + invalidcontinue.obj %(AdditionalOptions) $cdup\\compat\\win32\\git.manifest Console @@ -286,9 +320,25 @@ EOM EOM } print F << "EOM"; + + + - + $targetsImport + + + This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. + $targetsErrors + +EOM + if (!$static_library && $afterTargets ne '') { + print F << "EOM"; + $afterTargets + +EOM + } + print F << "EOM"; EOM close F; From dc5ef3e416947462b9e409dac4375f0b8a265e0e Mon Sep 17 00:00:00 2001 From: Philip Oakley Date: Mon, 13 Jul 2015 13:59:55 +0100 Subject: [PATCH 194/288] Vcproj.pm: remove duplicate GUID Delete the duplicated GUID from the generation code for the Visual Studio .sln project file. The duplicate GUID tended to be allocated to test-svn-fe, which was then ignored by Visual Studio / MSVC, and its omission from the build never noticed. Signed-off-by: Philip Oakley Signed-off-by: Johannes Schindelin --- contrib/buildsystems/Generators/Vcproj.pm | 1 - 1 file changed, 1 deletion(-) diff --git a/contrib/buildsystems/Generators/Vcproj.pm b/contrib/buildsystems/Generators/Vcproj.pm index cfa74adcc2..1b01d5821f 100644 --- a/contrib/buildsystems/Generators/Vcproj.pm +++ b/contrib/buildsystems/Generators/Vcproj.pm @@ -52,7 +52,6 @@ my @GUIDS = ( "{00785268-A9CC-4E40-AC29-BAC0019159CE}", "{4C06F56A-DCDB-46A6-B67C-02339935CF12}", "{3A62D3FD-519E-4EC9-8171-D2C1BFEA022F}", - "{3A62D3FD-519E-4EC9-8171-D2C1BFEA022F}", "{9392EB58-D7BA-410B-B1F0-B2FAA6BC89A7}", "{2ACAB2D5-E0CE-4027-BCA0-D78B2D7A6C66}", "{86E216C3-43CE-481A-BCB2-BE5E62850635}", From 478f18e3f6b5b59712a76aa1a1d660d031798c8b Mon Sep 17 00:00:00 2001 From: Philip Oakley Date: Sun, 19 Jul 2015 17:43:29 +0100 Subject: [PATCH 195/288] Vcproj.pm: list git.exe first to be startup project MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Visual Studio takes the first listed application/library as the default startup project [1]. Detect the 'git' project and place it the head of the apps list, rather than the tail. Export the apps list before libs list for both the projects and global structures of the .sln file. [1] http://stackoverflow.com/questions/1238553/ vs2008-where-is-the-startup-project-setting-stored-for-a-solution "In the solution file, there are a list of pseudo-XML "Project" entries. It turns out that whatever is the first one ends up as the Startup Project, unless it’s overridden in the suo file. Argh. I just rearranged the order in the file and it’s good." "just moving the pseudo-xml isn't enough. You also have to move the group of entries in the "GlobalSection(ProjectConfigurationPlatforms) = postSolution" group that has the GUID of the project you moved to the top. So there are two places to move lines." Signed-off-by: Philip Oakley --- contrib/buildsystems/Generators/Vcproj.pm | 33 +++++++++++++---------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/contrib/buildsystems/Generators/Vcproj.pm b/contrib/buildsystems/Generators/Vcproj.pm index 1b01d5821f..cc2fa21533 100644 --- a/contrib/buildsystems/Generators/Vcproj.pm +++ b/contrib/buildsystems/Generators/Vcproj.pm @@ -560,20 +560,18 @@ sub createGlueProject { foreach (@apps) { $_ =~ s/\//_/g; $_ =~ s/\.exe//; - push(@tmp, $_); + if ($_ eq "git" ) { + unshift(@tmp, $_); + } else { + push(@tmp, $_); + } } @apps = @tmp; open F, ">git.sln" || die "Could not open git.sln for writing!\n"; binmode F, ":crlf"; print F "$SLN_HEAD"; - foreach (@libs) { - my $libname = $_; - my $uuid = $build_structure{"LIBS_${libname}_GUID"}; - print F "$SLN_PRE"; - print F "\"${libname}\", \"${libname}\\${libname}.vcproj\", \"${uuid}\""; - print F "$SLN_POST"; - } + my $uuid_libgit = $build_structure{"LIBS_libgit_GUID"}; my $uuid_xdiff_lib = $build_structure{"LIBS_xdiff_lib_GUID"}; foreach (@apps) { @@ -587,6 +585,13 @@ sub createGlueProject { print F " EndProjectSection"; print F "$SLN_POST"; } + foreach (@libs) { + my $libname = $_; + my $uuid = $build_structure{"LIBS_${libname}_GUID"}; + print F "$SLN_PRE"; + print F "\"${libname}\", \"${libname}\\${libname}.vcproj\", \"${uuid}\""; + print F "$SLN_POST"; + } print F << "EOM"; Global @@ -598,17 +603,17 @@ EOM print F << "EOM"; GlobalSection(ProjectConfigurationPlatforms) = postSolution EOM - foreach (@libs) { - my $libname = $_; - my $uuid = $build_structure{"LIBS_${libname}_GUID"}; + foreach (@apps) { + my $appname = $_; + my $uuid = $build_structure{"APPS_${appname}_GUID"}; print F "\t\t${uuid}.Debug|Win32.ActiveCfg = Debug|Win32\n"; print F "\t\t${uuid}.Debug|Win32.Build.0 = Debug|Win32\n"; print F "\t\t${uuid}.Release|Win32.ActiveCfg = Release|Win32\n"; print F "\t\t${uuid}.Release|Win32.Build.0 = Release|Win32\n"; } - foreach (@apps) { - my $appname = $_; - my $uuid = $build_structure{"APPS_${appname}_GUID"}; + foreach (@libs) { + my $libname = $_; + my $uuid = $build_structure{"LIBS_${libname}_GUID"}; print F "\t\t${uuid}.Debug|Win32.ActiveCfg = Debug|Win32\n"; print F "\t\t${uuid}.Debug|Win32.Build.0 = Debug|Win32\n"; print F "\t\t${uuid}.Release|Win32.ActiveCfg = Release|Win32\n"; From 9a76b8200439aff95f21cc1d6f57e6d670173dd4 Mon Sep 17 00:00:00 2001 From: Philip Oakley Date: Wed, 6 Apr 2016 15:24:42 +0100 Subject: [PATCH 196/288] Vcproj.pm: provide more GUIDs Needed for: test-config; t-dump-split-index; t-dump-untracked-cache; t-fake-ssh; t-sha1-array; t-submodule-config. Plus a few spares. Signed-off-by: Philip Oakley --- contrib/buildsystems/Generators/Vcproj.pm | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/contrib/buildsystems/Generators/Vcproj.pm b/contrib/buildsystems/Generators/Vcproj.pm index cc2fa21533..0572ba8609 100644 --- a/contrib/buildsystems/Generators/Vcproj.pm +++ b/contrib/buildsystems/Generators/Vcproj.pm @@ -62,7 +62,14 @@ my @GUIDS = ( "{294BDC5A-F448-48B6-8110-DD0A81820F8C}", "{4B9F66E9-FAC9-47AB-B1EF-C16756FBFD06}", "{72EA49C6-2806-48BD-B81B-D4905102E19C}", - "{5728EB7E-8929-486C-8CD5-3238D060E768}" + "{5728EB7E-8929-486C-8CD5-3238D060E768}", + "{A3E300FC-5630-4850-A470-E9F2C2EFA7E7}", + "{CEA071D4-D9F3-4250-98F7-44AFDC8ACAA1}", + "{3FD87BB4-2236-4A1B-ADD2-46211A302442}", + "{49B03F41-5157-4079-95A7-64D728BCF74F}", + "{95D5A28B-80E2-40A9-BEA3-C52B9CA488E3}", + "{B85E6545-D523-4323-9F29-45389D090343}", + "{06840CEF-746C-4B71-9442-C395DD6590A5}" ); sub generate { From e007575532dad8ed6fec95a9473119786f328b22 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 25 Oct 2016 03:02:36 -0700 Subject: [PATCH 197/288] Vcproj.pm: do not configure VCWebServiceProxyGeneratorTool It is not necessary, and Visual Studio 2015 no longer supports it, anyway. Signed-off-by: Johannes Schindelin --- contrib/buildsystems/Generators/Vcproj.pm | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/contrib/buildsystems/Generators/Vcproj.pm b/contrib/buildsystems/Generators/Vcproj.pm index 0572ba8609..70e7c8d7fc 100644 --- a/contrib/buildsystems/Generators/Vcproj.pm +++ b/contrib/buildsystems/Generators/Vcproj.pm @@ -168,9 +168,6 @@ sub createLibProject { - @@ -234,9 +231,6 @@ sub createLibProject { - @@ -393,9 +387,6 @@ sub createAppProject { - @@ -464,9 +455,6 @@ sub createAppProject { - From 94e088eea5bda2855eb811d680d6153ecbd84cc5 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 25 Oct 2016 03:11:22 -0700 Subject: [PATCH 198/288] Vcproj.pm: urlencode '<' and '>' when generating VC projects Signed-off-by: Johannes Schindelin --- contrib/buildsystems/Generators/Vcproj.pm | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/contrib/buildsystems/Generators/Vcproj.pm b/contrib/buildsystems/Generators/Vcproj.pm index 70e7c8d7fc..1c61ea6b93 100644 --- a/contrib/buildsystems/Generators/Vcproj.pm +++ b/contrib/buildsystems/Generators/Vcproj.pm @@ -112,6 +112,8 @@ sub createLibProject { my $includes= join(";", sort(map(""$rel_dir\\$_"", @{$$build_structure{"LIBS_${libname}_INCLUDES"}}))); my $cflags = join(" ", sort(@{$$build_structure{"LIBS_${libname}_CFLAGS"}})); $cflags =~ s/\"/"/g; + $cflags =~ s//>/g; my $cflags_debug = $cflags; $cflags_debug =~ s/-MT/-MTd/; @@ -133,6 +135,8 @@ sub createLibProject { $defines =~ s/-D//g; $defines =~ s/\"/\\"/g; + $defines =~ s//>/g; $defines =~ s/\'//g; $includes =~ s/-I//g; mkdir "$target" || die "Could not create the directory $target for lib project!\n"; @@ -325,6 +329,8 @@ sub createAppProject { my $includes= join(";", sort(map(""$rel_dir\\$_"", @{$$build_structure{"APPS_${appname}_INCLUDES"}}))); my $cflags = join(" ", sort(@{$$build_structure{"APPS_${appname}_CFLAGS"}})); $cflags =~ s/\"/"/g; + $cflags =~ s//>/g; my $cflags_debug = $cflags; $cflags_debug =~ s/-MT/-MTd/; @@ -351,6 +357,8 @@ sub createAppProject { $defines =~ s/-D//g; $defines =~ s/\"/\\"/g; + $defines =~ s//>/g; $defines =~ s/\'//g; $defines =~ s/\\\\/\\/g; $includes =~ s/-I//g; From e33998ca36a304b3349d84f73d9e9b8a9cd81b89 Mon Sep 17 00:00:00 2001 From: Philip Oakley Date: Sun, 19 Jul 2015 17:03:15 +0100 Subject: [PATCH 199/288] msvc: add No_SafeExceptionHandler option Microsoft flipped the Windows Safe Exception Handling default in VS2013 so that zlib became unacceptable to certain OS versions (Vista and subsequent 32-bit OS's) without the addition of the option -SAFESEH:NO. Provide a switch to disable the Safe Exception Handler when required. The option ImageHasSafeExceptionHandlers for VS2013 is not available in earlier versions, so use the SAFESEH:NO linker flag. See https://msdn.microsoft.com/en-us/library/9a89h429.aspx for further details. This has only had limited testing due to the lack of a suitable system. Helped-by: Yue Lin Ho Signed-off-by: Philip Oakley Signed-off-by: Johannes Schindelin --- config.mak.uname | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/config.mak.uname b/config.mak.uname index d455929efd..eb8aa8bb74 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -1,5 +1,9 @@ # Platform specific Makefile tweaks based on uname detection +# Define NO_SAFESEH if you need MSVC/Visual Studio to ignore the lack of +# Microsoft's Safe Exception Handling in libraries (such as zlib). +# Typically required for VS2013+/32-bit compilation on Vista+ versions. + uname_S := $(shell sh -c 'uname -s 2>/dev/null || echo not') uname_M := $(shell sh -c 'uname -m 2>/dev/null || echo not') uname_O := $(shell sh -c 'uname -o 2>/dev/null || echo not') @@ -427,6 +431,11 @@ ifeq ($(uname_S),Windows) # release mode) to force a PDB to be generated (like RelWithDebInfo). BASIC_CFLAGS += -Zi BASIC_LDFLAGS += -debug + +ifdef NO_SAFESEH + LDFLAGS += -SAFESEH:NO +endif + ifndef DEBUG BASIC_CFLAGS += -GL -Gy -O2 -Oy- -MD -DNDEBUG BASIC_LDFLAGS += -release -LTCG /OPT:REF /OPT:ICF /INCREMENTAL:NO /DEBUGTYPE:CV,FIXUP From d20e3796253a9b362df5b5c668a725ef5e8e35af Mon Sep 17 00:00:00 2001 From: Philip Oakley Date: Fri, 22 Apr 2016 13:33:33 +0100 Subject: [PATCH 200/288] msvc: do not include inttypes.h This file is not available in earlier MSVC versions, and it is not necessary to include it with MSVC, anyway. Signed-off-by: Philip Oakley Signed-off-by: Johannes Schindelin --- config.mak.uname | 1 + 1 file changed, 1 insertion(+) diff --git a/config.mak.uname b/config.mak.uname index eb8aa8bb74..c4bd24c754 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -366,6 +366,7 @@ ifeq ($(uname_S),Windows) NO_STRTOUMAX = YesPlease NO_MKDTEMP = YesPlease NO_MKSTEMPS = YesPlease + NO_INTTYPES_H = YesPlease # VS2015 with UCRT claims that snprintf and friends are C99 compliant, # so we don't need this. # From 578fcb4b5c8c1c1ba33d5560e2e388a154d84f71 Mon Sep 17 00:00:00 2001 From: Philip Oakley Date: Wed, 4 May 2016 16:18:07 +0100 Subject: [PATCH 201/288] msvc: add pragmas for common warnings MSVC can be overzealous about some warnings. Disable them. Signed-off-by: Philip Oakley --- compat/msvc.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/compat/msvc.h b/compat/msvc.h index b3ac4cc3af..d081043f53 100644 --- a/compat/msvc.h +++ b/compat/msvc.h @@ -6,6 +6,10 @@ #include #include +#pragma warning(disable: 4018) /* signed/unsigned comparison */ +#pragma warning(disable: 4244) /* type conversion, possible loss of data */ +#pragma warning(disable: 4090) /* 'function' : different 'const' qualifiers (ALLOC_GROW etc.)*/ + /* porting function */ #define inline __inline #define __inline__ __inline From 82b628e9e1e1b112e4344cc63e8c80951c0362bb Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sat, 5 Nov 2016 15:19:31 +0100 Subject: [PATCH 202/288] msvc: tell Visual Studio where we expect NuGet packages to be unpacked We already unpack the NuGet packages in a certain place, via compat/vcbuild/Makefile. Let's let Visual Studio use the very same place. Signed-off-by: Johannes Schindelin --- .nuget/NuGet.config | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .nuget/NuGet.config diff --git a/.nuget/NuGet.config b/.nuget/NuGet.config new file mode 100644 index 0000000000..93e50042cb --- /dev/null +++ b/.nuget/NuGet.config @@ -0,0 +1,6 @@ + + + + + + From a81f3dd5c33e060e53442db9efe6e6f0ca92006f Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 8 Nov 2016 11:00:01 +0100 Subject: [PATCH 203/288] msvc: ignore .dll files copied into the top-level directory Signed-off-by: Johannes Schindelin --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 46cb500304..59e25d6dc7 100644 --- a/.gitignore +++ b/.gitignore @@ -253,6 +253,7 @@ *.idb *.pdb *.ilk +*.dll .vs/ *.manifest Debug/ From f7bab4a03526632d9e945ab0cfef32506367bb4b Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 8 Nov 2016 13:17:12 +0100 Subject: [PATCH 204/288] msvc: ignore incremental compile output Signed-off-by: Johannes Schindelin --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 59e25d6dc7..2b0bdbd3f6 100644 --- a/.gitignore +++ b/.gitignore @@ -253,6 +253,8 @@ *.idb *.pdb *.ilk +*.iobj +*.ipdb *.dll .vs/ *.manifest From 39406558285591873b29e53638c758be0805373d Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 3 Nov 2016 09:40:12 -0700 Subject: [PATCH 205/288] msvc: fix `make test` without having to play PATH games When building with Microsoft Visual C, we use NuGet to acquire the dependencies (such as OpenSSL, cURL, etc). We even unpack those dependencies. This patch teaches the test suite to add the directory with the unpacked .dll files to the PATH before running the tests. Signed-off-by: Johannes Schindelin --- Makefile | 4 ++++ t/test-lib-functions.sh | 2 +- t/test-lib.sh | 19 ++++++++++++++----- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 3632167c3a..db97e84312 100644 --- a/Makefile +++ b/Makefile @@ -2224,6 +2224,7 @@ GIT-BUILD-OPTIONS: FORCE @echo NO_PYTHON=\''$(subst ','\'',$(subst ','\'',$(NO_PYTHON)))'\' >>$@+ @echo NO_UNIX_SOCKETS=\''$(subst ','\'',$(subst ','\'',$(NO_UNIX_SOCKETS)))'\' >>$@+ @echo PAGER_ENV=\''$(subst ','\'',$(subst ','\'',$(PAGER_ENV)))'\' >>$@+ + @echo X=\'$(X)\' >>$@+ ifdef TEST_OUTPUT_DIRECTORY @echo TEST_OUTPUT_DIRECTORY=\''$(subst ','\'',$(subst ','\'',$(TEST_OUTPUT_DIRECTORY)))'\' >>$@+ endif @@ -2252,6 +2253,9 @@ ifdef GIT_PERF_MAKE_OPTS endif ifdef TEST_GIT_INDEX_VERSION @echo TEST_GIT_INDEX_VERSION=\''$(subst ','\'',$(subst ','\'',$(TEST_GIT_INDEX_VERSION)))'\' >>$@+ +endif +ifdef MSVC_DEPS + @echo MSVC_DEPS=\''$(subst ','\'',$(subst ','\'',$(MSVC_DEPS)))'\' >>$@+ endif @if cmp $@+ $@ >/dev/null 2>&1; then $(RM) $@+; else mv $@+ $@; fi diff --git a/t/test-lib-functions.sh b/t/test-lib-functions.sh index adab7f51f4..25284bdd3e 100644 --- a/t/test-lib-functions.sh +++ b/t/test-lib-functions.sh @@ -786,7 +786,7 @@ test_create_repo () { mkdir -p "$repo" ( cd "$repo" || error "Cannot setup test environment" - "$GIT_EXEC_PATH/git-init" "--template=$GIT_BUILD_DIR/templates/blt/" >&3 2>&4 || + "$GIT_EXEC_PATH/git$X" init "--template=$GIT_BUILD_DIR/templates/blt/" >&3 2>&4 || error "cannot run git init -- have you built things yet?" mv .git/hooks .git/hooks-disabled ) || exit diff --git a/t/test-lib.sh b/t/test-lib.sh index 9980a46133..6e65569d1c 100644 --- a/t/test-lib.sh +++ b/t/test-lib.sh @@ -36,18 +36,27 @@ then fi GIT_BUILD_DIR="$TEST_DIRECTORY"/.. -################################################################ -# It appears that people try to run tests without building... -"$GIT_BUILD_DIR/git" >/dev/null -if test $? != 1 +if test ! -f "$GIT_BUILD_DIR"/GIT-BUILD-OPTIONS then - echo >&2 'error: you do not seem to have built git yet.' + echo >&2 'error: GIT-BUILD-OPTIONS missing (has Git been built?).' exit 1 fi . "$GIT_BUILD_DIR"/GIT-BUILD-OPTIONS export PERL_PATH SHELL_PATH +test -z "$MSVC_DEPS" || +PATH="$GIT_BUILD_DIR/$MSVC_DEPS/bin:$PATH" + +################################################################ +# It appears that people try to run tests without building... +"$GIT_BUILD_DIR/git$X" >/dev/null +if test $? != 1 +then + echo >&2 'error: you do not seem to have built git yet.' + exit 1 +fi + # if --tee was passed, write the output not only to the terminal, but # additionally to the file test-results/$BASENAME.out, too. case "$GIT_TEST_TEE_STARTED, $* " in From 2de14bdbeef112a54e1dad9afa1857aa814226cd Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 28 Nov 2016 15:54:59 +0100 Subject: [PATCH 206/288] msvc: add a Makefile target to pre-generate the VS solution The entire idea of generating the VS solution makes only sense if we generate it via Continuous Integration; otherwise potential users would still have to download the entire Git for Windows SDK. So let's just add a target in the Makefile that can be used to generate said solution; The generated files will then be committed so that they can be pushed to a branch ready to check out by Visual Studio users. To make things even more useful, we also generate and commit other files that are required to run the test suite, such as templates and bin-wrappers: with this, developers can run the test suite in a regular Git Bash (that is part of a regular Git for Windows installation) after building the solution in Visual Studio. Signed-off-by: Johannes Schindelin --- config.mak.uname | 59 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/config.mak.uname b/config.mak.uname index c4bd24c754..b4bb2e520b 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -447,6 +447,65 @@ endif X = .exe compat/msvc.o: compat/msvc.c compat/mingw.c GIT-CFLAGS + +vcxproj: + # Require clean work tree + git update-index -q --refresh && \ + git diff-files --quiet && \ + git diff-index --cached --quiet HEAD -- + + # Make .vcxproj files and add them + unset QUIET_GEN QUIET_BUILT_IN; \ + perl contrib/buildsystems/generate -g Vcxproj + git add -f git.sln {*,*/lib,t/helper/*}/{packages.config,*.vcxproj} + + # Add common-cmds.h + $(MAKE) MSVC=1 prefix=/mingw64 common-cmds.h + git add -f common-cmds.h + + # Add scripts + rm -f perl/perl.mak + $(MAKE) MSVC=1 prefix=/mingw64 \ + $(SCRIPT_LIB) $(SCRIPT_SH_GEN) $(SCRIPT_PERL_GEN) + # Strip out the sane tool path, needed only for building + sed -i '/^git_broken_path_fix ".*/d' git-sh-setup + git add -f $(SCRIPT_LIB) $(SCRIPT_SH_GEN) $(SCRIPT_PERL_GEN) + + # Add Perl module + $(MAKE) -C perl + git add -f perl/blib/lib + + # Add bin-wrappers, for testing + rm -rf bin-wrappers/ + $(MAKE) MSVC=1 prefix=/mingw64 $(test_bindir_programs) + # Ensure that the GIT_EXEC_PATH is a Unix-y one, and that the absolute + # path of the repository is not hard-coded (GIT_EXEC_PATH will be set + # by test-lib.sh according to the current setup) + sed -i -e 's/^\(GIT_EXEC_PATH\)=.*/test -n "$${\1##*:*}" ||\ + \1="$$(cygpath -u "$$\1")"/' \ + -e "s|'$$(pwd)|\"\$$GIT_EXEC_PATH\"'|g" bin-wrappers/* + # Ensure that test-* helpers find the .dll files copied to top-level + sed -i 's|^PATH=.*|&:"$$GIT_EXEC_PATH"|' bin-wrappers/test-* + # We do not want to force hard-linking builtins + sed -i 's|\(git\)-\([-a-z]*\)\.exe"|\1.exe" \2|g' \ + bin-wrappers/git-{receive-pack,upload-archive} + git add -f $(test_bindir_programs) + # remote-ext is a builtin, but invoked as if it were external + sed 's|receive-pack|remote-ext|g' \ + bin-wrappers/git-remote-ext + git add -f bin-wrappers/git-remote-ext + + # Add templates + $(MAKE) -C templates + git add -f templates/boilerplates.made templates/blt/ + + # Add build options + $(MAKE) MSVC=1 prefix=/mingw64 GIT-BUILD-OPTIONS + git add -f GIT-BUILD-OPTIONS + + # Commit the whole shebang + git commit -m "Generate Visual Studio solution" \ + -m "Auto-generated by \`$(MAKE)$(MAKEFLAGS) $@\`" endif ifeq ($(uname_S),Interix) NO_INITGROUPS = YesPlease From 2726539dbf9aab1481d751ffa53c46c0344dac22 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 30 Nov 2016 22:26:20 +0100 Subject: [PATCH 207/288] msvc: avoid debug assertion windows in Debug Mode For regular debugging, it is pretty helpful when a debug assertion in a running application triggers a window that offers to start the debugger. However, when running the test suite, it is not so helpful, in particular when the debug assertions are then suppressed anyway because we disable the invalid parameter checking (via invalidcontinue.obj, see the comment in config.mak.uname about that object for more information). So let's simply disable that window in Debug Mode (it is already disabled in Release Mode). Signed-off-by: Johannes Schindelin --- compat/mingw.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/compat/mingw.c b/compat/mingw.c index 8d93b181e4..08f17425b2 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -3034,6 +3034,10 @@ static void maybe_redirect_std_handles(void) #if defined(_MSC_VER) +#ifdef _DEBUG +#include +#endif + /* * This routine sits between wmain() and "main" in git.exe. * We receive UNICODE (wchar_t) values for argv and env. @@ -3058,6 +3062,10 @@ int msc_startup(int argc, wchar_t **w_argv, wchar_t **w_env) int maxlen; int k, exit_status; +#ifdef _DEBUG + _CrtSetReportMode(_CRT_ASSERT, _CRTDBG_MODE_DEBUG); +#endif + #ifdef USE_MSVC_CRTDBG _CrtSetDbgFlag(_CRTDBG_ALLOC_MEM_DF | _CRTDBG_LEAK_CHECK_DF); #endif From ffd4faec90dd52eae97aeeaad9a7f3fbe4e40c91 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 30 Nov 2016 15:17:50 -0500 Subject: [PATCH 208/288] read-cache: run verify_hdr() in background thread This is a performance optimization. Teach do_read_index() to call verify_hdr() using a thread and allow SHA1 verification to run concurrently with the parsing of index-entries and extensions. For large index files, this cuts the startup time in half. Signed-off-by: Jeff Hostetler --- read-cache.c | 60 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/read-cache.c b/read-cache.c index 585ff02dc3..b885d6cd41 100644 --- a/read-cache.c +++ b/read-cache.c @@ -19,6 +19,10 @@ #include "split-index.h" #include "utf8.h" +#ifndef NO_PTHREADS +#include +#endif + /* Mask for the name length in ce_flags in the on-disk index */ #define CE_NAMEMASK (0x0fff) @@ -1397,6 +1401,34 @@ static int verify_hdr(struct cache_header *hdr, unsigned long size) return 0; } +#ifndef NO_PTHREADS +/* + * Require index file to be larger than this threshold before + * we bother using a thread to verify the SHA. + * This value was arbitrarily chosen. + */ +#define VERIFY_HDR_THRESHOLD 10*1024*1024 + +struct verify_hdr_thread_data +{ + pthread_t thread_id; + struct cache_header *hdr; + size_t size; + int result; +}; + +/* + * A thread proc to run the verify_hdr() computation + * in a background thread. + */ +static void *verify_hdr_thread(void *_data) +{ + struct verify_hdr_thread_data *p = _data; + p->result = verify_hdr(p->hdr, (unsigned long)p->size); + return NULL; +} +#endif + static int read_index_extension(struct index_state *istate, const char *ext, void *data, unsigned long sz) { @@ -1584,6 +1616,9 @@ int do_read_index(struct index_state *istate, const char *path, int must_exist) void *mmap; size_t mmap_size; struct strbuf previous_name_buf = STRBUF_INIT, *previous_name; +#ifndef NO_PTHREADS + struct verify_hdr_thread_data verify_hdr_thread_data; +#endif if (istate->initialized) return istate->cache_nr; @@ -1610,8 +1645,23 @@ int do_read_index(struct index_state *istate, const char *path, int must_exist) close(fd); hdr = mmap; +#ifdef NO_PTHREADS if (verify_hdr(hdr, mmap_size) < 0) goto unmap; +#else + if (mmap_size < VERIFY_HDR_THRESHOLD) { + if (verify_hdr(hdr, mmap_size) < 0) + goto unmap; + } else { + verify_hdr_thread_data.hdr = hdr; + verify_hdr_thread_data.size = mmap_size; + verify_hdr_thread_data.result = -1; + if (pthread_create( + &verify_hdr_thread_data.thread_id, NULL, + verify_hdr_thread, &verify_hdr_thread_data)) + die_errno("unable to start verify_hdr_thread"); + } +#endif hashcpy(istate->sha1, (const unsigned char *)hdr + mmap_size - 20); istate->version = ntohl(hdr->hdr_version); @@ -1659,6 +1709,16 @@ int do_read_index(struct index_state *istate, const char *path, int must_exist) src_offset += 8; src_offset += extsize; } + +#ifndef NO_PTHREADS + if (mmap_size >= VERIFY_HDR_THRESHOLD) { + if (pthread_join(verify_hdr_thread_data.thread_id, NULL)) + die_errno("unable to join verify_hdr_thread"); + if (verify_hdr_thread_data.result < 0) + goto unmap; + } +#endif + munmap(mmap, mmap_size); return istate->cache_nr; From ada2687cc40df42ae854fa1d8f54b7da08e27a6c Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Tue, 6 Dec 2016 16:33:39 -0500 Subject: [PATCH 209/288] read-cache: speed up add_index_entry during checkout Teach add_index_entry_with_check() and has_dir_name() to see if the path of the new item is greater than the last path in the index array before attempting to search for it. This is a performance optimization. During checkout, merge_working_tree() populates the new index in sorted order, so this change saves at least 2 lookups per file. Signed-off-by: Jeff Hostetler --- cache.h | 1 + read-cache.c | 72 +++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 72 insertions(+), 1 deletion(-) diff --git a/cache.h b/cache.h index 5c035dae57..1c2873d107 100644 --- a/cache.h +++ b/cache.h @@ -572,6 +572,7 @@ extern int write_locked_index(struct index_state *, struct lock_file *lock, unsi extern int discard_index(struct index_state *); extern int unmerged_index(const struct index_state *); extern int verify_path(const char *path); +extern int strcmp_offset(const char *s1_in, const char *s2_in, int *first_change); extern int index_dir_exists(struct index_state *istate, const char *name, int namelen); extern void adjust_dirname_case(struct index_state *istate, char *name); extern struct cache_entry *index_file_exists(struct index_state *istate, const char *name, int namelen, int igncase); diff --git a/read-cache.c b/read-cache.c index 585ff02dc3..a9e90d6173 100644 --- a/read-cache.c +++ b/read-cache.c @@ -894,6 +894,34 @@ static int has_file_name(struct index_state *istate, return retval; } +/* + * Like strcmp(), but also return the offset of the first change. + */ +int strcmp_offset(const char *s1_in, const char *s2_in, int *first_change) +{ + const unsigned char *s1 = (const unsigned char *)s1_in; + const unsigned char *s2 = (const unsigned char *)s2_in; + int diff = 0; + int k; + + *first_change = 0; + for (k=0; s1[k]; k++) + if ((diff = (s1[k] - s2[k]))) + goto found_it; + if (!s2[k]) + return 0; + diff = -1; + +found_it: + *first_change = k; + if (diff > 0) + return 1; + else if (diff < 0) + return -1; + else + return 0; +} + /* * Do we have another file with a pathname that is a proper * subset of the name we're trying to add? @@ -905,6 +933,21 @@ static int has_dir_name(struct index_state *istate, int stage = ce_stage(ce); const char *name = ce->name; const char *slash = name + ce_namelen(ce); + int len_eq_last; + int cmp_last = 0; + + if (istate->cache_nr > 0) { + /* + * Compare the entry's full path with the last path in the index. + * If it sorts AFTER the last entry in the index and they have no + * common prefix, then there cannot be any F/D name conflicts. + */ + cmp_last = strcmp_offset(name, + istate->cache[istate->cache_nr-1]->name, + &len_eq_last); + if (cmp_last > 0 && len_eq_last == 0) + return retval; + } for (;;) { int len; @@ -917,6 +960,24 @@ static int has_dir_name(struct index_state *istate, } len = slash - name; + if (cmp_last > 0) { + /* + * If this part of the directory prefix (including the trailing + * slash) already appears in the path of the last entry in the + * index, then we cannot also have a file with this prefix (or + * any parent directory prefix). + */ + if (len+1 <= len_eq_last) + return retval; + /* + * If this part of the directory prefix (excluding the trailing + * slash) is longer than the known equal portions, then this part + * of the prefix cannot collide with a file. Go on to the parent. + */ + if (len > len_eq_last) + continue; + } + pos = index_name_stage_pos(istate, name, len, stage); if (pos >= 0) { /* @@ -1008,7 +1069,16 @@ static int add_index_entry_with_check(struct index_state *istate, struct cache_e if (!(option & ADD_CACHE_KEEP_CACHE_TREE)) cache_tree_invalidate_path(istate, ce->name); - pos = index_name_stage_pos(istate, ce->name, ce_namelen(ce), ce_stage(ce)); + + /* + * If this entry's path sorts after the last entry in the index, + * we can avoid searching for it. + */ + if (istate->cache_nr > 0 && + strcmp(ce->name, istate->cache[istate->cache_nr - 1]->name) > 0) + pos = -istate->cache_nr - 1; + else + pos = index_name_stage_pos(istate, ce->name, ce_namelen(ce), ce_stage(ce)); /* existing match? Just replace it. */ if (pos >= 0) { From 208f89e97b563d7e32c5f697ff45af2466c14406 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Thu, 8 Dec 2016 10:12:15 -0500 Subject: [PATCH 210/288] test-strcmp-offset: created test for strcmp_offset Signed-off-by: Jeff Hostetler --- Makefile | 1 + t/helper/test-strcmp-offset.c | 59 +++++++++++++++++++++++++++++++++++ t/t0065-strcmp-offset.sh | 11 +++++++ 3 files changed, 71 insertions(+) create mode 100644 t/helper/test-strcmp-offset.c create mode 100755 t/t0065-strcmp-offset.sh diff --git a/Makefile b/Makefile index 76267262c1..cd50a936df 100644 --- a/Makefile +++ b/Makefile @@ -629,6 +629,7 @@ TEST_PROGRAMS_NEED_X += test-scrap-cache-tree TEST_PROGRAMS_NEED_X += test-sha1 TEST_PROGRAMS_NEED_X += test-sha1-array TEST_PROGRAMS_NEED_X += test-sigchain +TEST_PROGRAMS_NEED_X += test-strcmp-offset TEST_PROGRAMS_NEED_X += test-string-list TEST_PROGRAMS_NEED_X += test-submodule-config TEST_PROGRAMS_NEED_X += test-subprocess diff --git a/t/helper/test-strcmp-offset.c b/t/helper/test-strcmp-offset.c new file mode 100644 index 0000000000..56fd8c1e41 --- /dev/null +++ b/t/helper/test-strcmp-offset.c @@ -0,0 +1,59 @@ +#include "cache.h" + +struct test_data { + const char *s1; + const char *s2; + int first_change; +}; + +static struct test_data data[] = { + { "abc", "abc", 0 }, + { "abc", "def", 0 }, + + { "abc", "abz", 2 }, + + { "abc", "abcdef", 3 }, + + { "abc\xF0zzz", "abc\xFFzzz", 3 }, + + { NULL, NULL, 0 } +}; + +int try_pair(const char *sa, const char *sb, int first_change) +{ + int failed = 0; + int offset, r_exp, r_tst; + + r_exp = strcmp(sa, sb); + r_tst = strcmp_offset(sa, sb, &offset); + if (r_tst != r_exp) { + if ((r_tst < 0 && r_exp < 0) || (r_tst > 0 && r_exp > 0)) + warning("'%s' vs '%s', imprecise result: %d != %d", + sa, sb, r_exp, r_tst); + else { + error("'%s' vs '%s', result expect %d, observed %d", + sa, sb, r_exp, r_tst); + failed = 1; + } + } + if (offset != first_change) { + error("'%s' vs '%s', offset expect %d, observed %d", + sa, sb, first_change, offset); + failed = 1; + } + + return failed; +} + +int cmd_main(int argc, const char **argv) +{ + int failed = 0; + int k; + + for (k=0; data[k].s1; k++) { + failed += try_pair(data[k].s1, data[k].s2, data[k].first_change); + failed += try_pair(data[k].s2, data[k].s1, data[k].first_change); + } + + return failed; +} diff --git a/t/t0065-strcmp-offset.sh b/t/t0065-strcmp-offset.sh new file mode 100755 index 0000000000..0176c8c924 --- /dev/null +++ b/t/t0065-strcmp-offset.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +test_description='Test strcmp_offset functionality' + +. ./test-lib.sh + +test_expect_success run_helper ' + test-strcmp-offset +' + +test_done From f75a86a4054275a5e4e3b57e91459b0cf9c9e907 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 6 Sep 2016 09:50:33 +0200 Subject: [PATCH 211/288] Unbreak interactive GPG prompt upon signing With the recent update in efee955 (gpg-interface: check gpg signature creation status, 2016-06-17), we ask GPG to send all status updates to stderr, and then catch the stderr in an strbuf. But GPG might fail, and send error messages to stderr. And we simply do not show them to the user. Even worse: this swallows any interactive prompt for a passphrase. And detaches stderr from the tty so that the passphrase cannot be read. So while the first problem could be fixed (by printing the captured stderr upon error), the second problem cannot be easily fixed, and presents a major regression. So let's just revert commit efee9553a4f97b2ecd8f49be19606dd4cf7d9c28. This fixes https://github.com/git-for-windows/git/issues/871 Cc: Michael J Gruber Signed-off-by: Johannes Schindelin --- gpg-interface.c | 8 ++------ t/t7004-tag.sh | 9 +-------- 2 files changed, 3 insertions(+), 14 deletions(-) diff --git a/gpg-interface.c b/gpg-interface.c index e44cc27da1..0c52c649e4 100644 --- a/gpg-interface.c +++ b/gpg-interface.c @@ -160,11 +160,9 @@ int sign_buffer(struct strbuf *buffer, struct strbuf *signature, const char *sig struct child_process gpg = CHILD_PROCESS_INIT; int ret; size_t i, j, bottom; - struct strbuf gpg_status = STRBUF_INIT; argv_array_pushl(&gpg.args, gpg_program, - "--status-fd=2", "-bsau", signing_key, NULL); @@ -176,12 +174,10 @@ int sign_buffer(struct strbuf *buffer, struct strbuf *signature, const char *sig */ sigchain_push(SIGPIPE, SIG_IGN); ret = pipe_command(&gpg, buffer->buf, buffer->len, - signature, 1024, &gpg_status, 0); + signature, 1024, NULL, 0); sigchain_pop(SIGPIPE); - ret |= !strstr(gpg_status.buf, "\n[GNUPG:] SIG_CREATED "); - strbuf_release(&gpg_status); - if (ret) + if (ret || signature->len == bottom) return error(_("gpg failed to sign the data")); /* Strip CR from the line endings, in case we are on Windows. */ diff --git a/t/t7004-tag.sh b/t/t7004-tag.sh index 8b0f71a2ac..f9b7d79af5 100755 --- a/t/t7004-tag.sh +++ b/t/t7004-tag.sh @@ -1202,17 +1202,10 @@ test_expect_success GPG,RFC1991 \ # try to sign with bad user.signingkey git config user.signingkey BobTheMouse test_expect_success GPG \ - 'git tag -s fails if gpg is misconfigured (bad key)' \ + 'git tag -s fails if gpg is misconfigured' \ 'test_must_fail git tag -s -m tail tag-gpg-failure' git config --unset user.signingkey -# try to produce invalid signature -test_expect_success GPG \ - 'git tag -s fails if gpg is misconfigured (bad signature format)' \ - 'test_config gpg.program echo && - test_must_fail git tag -s -m tail tag-gpg-failure' - - # try to verify without gpg: rm -rf gpghome From b2e5f9e904edbbeadc23084f37a576a80625317c Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 16 Nov 2016 17:49:17 +0100 Subject: [PATCH 212/288] mingw: make readlink() independent of core.symlinks Regardless whether we think we are able to create symbolic links, we should always read them. This fixes https://github.com/git-for-windows/git/issues/958 Signed-off-by: Johannes Schindelin --- compat/mingw.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index da84e9209c..7cbe7aed55 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2637,12 +2637,6 @@ int readlink(const char *path, char *buf, size_t bufsiz) char tmpbuf[MAX_LONG_PATH]; int len; - /* fail if symlinks are disabled */ - if (!has_symlinks) { - errno = ENOSYS; - return -1; - } - if (xutftowcs_long_path(wpath, path) < 0) return -1; From 10651388c605bdf42b2620a4b54d32a04a91adef Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Mon, 12 Dec 2016 12:49:46 -0500 Subject: [PATCH 213/288] string-list: use ALLOC_GROW macro when reallocing string_list Use ALLOC_GROW() macro when reallocing a string_list array rather than simply increasing it by 32. This is a performance optimization. During status on a very large repo and there are many changes, a significant percentage of the total run time was spent reallocing the wt_status.changes array. This change decreased the time in wt_status_collect_changes_worktree() from 125 seconds to 45 seconds on my very large repository. Signed-off-by: Jeff Hostetler --- string-list.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/string-list.c b/string-list.c index 8c83cac189..07c888dddd 100644 --- a/string-list.c +++ b/string-list.c @@ -41,10 +41,8 @@ static int add_entry(int insert_at, struct string_list *list, const char *string if (exact_match) return -1 - index; - if (list->nr + 1 >= list->alloc) { - list->alloc += 32; - REALLOC_ARRAY(list->items, list->alloc); - } + if (list->nr + 1 >= list->alloc) + ALLOC_GROW(list->items, list->nr+1, list->alloc); if (index < list->nr) memmove(list->items + index + 1, list->items + index, (list->nr - index) From 6c1e1d8cd6aac94b1d73a398468d8dacf0df7931 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 9 Dec 2016 16:37:49 +0100 Subject: [PATCH 214/288] http(s): automatically try NTLM authentication first It is common in corporate setups to have permissions managed via a domain account. That means that the user does not really have to log in when accessing a central repository via https://, but that the login credentials are used to authenticate with that repository. The common way to do that used to require empty credentials, i.e. hitting Enter twice when being asked for user name and password, or by using the very funny notation https://:@server/repository A recent commit (5275c3081c (http: http.emptyauth should allow empty (not just NULL) usernames, 2016-10-04)) broke that usage, though, all of a sudden requiring users to set http.emptyAuth = true. Which brings us to the bigger question why http.emptyAuth defaults to false, to begin with. It would be one thing if cURL would not let the user specify credentials interactively after attempting NTLM authentication (i.e. login credentials), but that is not the case. It would be another thing if attempting NTLM authentication was not usually what users need to do when trying to authenticate via https://. But that is also not the case. So let's just go ahead and change the default, and unbreak the NTLM authentication. As a bonus, this also makes the "you need to hit Enter twice" (which is hard to explain: why enter empty credentials when you want to authenticate with your login credentials?) and the ":@" hack (which is also pretty, pretty hard to explain to users) obsolete. This fixes https://github.com/git-for-windows/git/issues/987 Signed-off-by: Johannes Schindelin --- http.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/http.c b/http.c index 051fe6e5ab..a80e2e03ab 100644 --- a/http.c +++ b/http.c @@ -109,7 +109,7 @@ static int curl_save_cookies; struct credential http_auth = CREDENTIAL_INIT; static int http_proactive_auth; static const char *user_agent; -static int curl_empty_auth; +static int curl_empty_auth = 1; enum http_follow_config http_follow_config = HTTP_FOLLOW_INITIAL; From def4de6811d1082e302fd5c44bf9c08cd44cacbf Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 25 May 2015 10:36:17 +0200 Subject: [PATCH 215/288] Support Vagrant: quick & easy Linux virtual machine setup When developing Git for Windows, we always have to ensure that we do not break any non-Windows platforms, e.g. by introducing Windows-specific code into the platform-independent source code. At other times, it is necessary to test whether a bug is Windows-specific or not, in order to send the bug report to the correct place. Having access to a Linux-based Git comes in really handy in such a situation. Vagrant offers a painless way to install and use a defined Linux development environment on Windows (and other Operating Systems). We offer a Vagrantfile to that end for two reasons: 1) To allow Windows users to gain the full power of Linux' Git 2) To offer users an easy path to verify that the issue they are about to report is really a Windows-specific issue; otherwise they would need to report it to git@vger.kernel.org instead. Using it is easy: Download and install https://www.virtualbox.org/, then download and install https://www.vagrantup.com/, then direct your command-line window to the Git source directory containing the Vagrantfile and run the commands: vagrant up vagrant ssh See https://github.com/git-for-windows/git/wiki/Vagrant for details. As part of switching Git for Windows' development environment from msysGit to the MSys2-based Git SDK, this Vagrantfile was copy-edited from msysGit: https://github.com/msysgit/msysgit/blob/0be8f2208/Vagrantfile Signed-off-by: Johannes Schindelin --- .gitignore | 3 +++ Vagrantfile | 57 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 Vagrantfile diff --git a/.gitignore b/.gitignore index 2b0bdbd3f6..630eb30e45 100644 --- a/.gitignore +++ b/.gitignore @@ -263,3 +263,6 @@ Release/ /UpgradeLog*.htm /git.VC.VC.opendb /git.VC.db +/.vagrant/ +/vagrant2[0-9][0-9][0-9]*-*-* +/d2[0-9][0-9][0-9]*-*-* diff --git a/Vagrantfile b/Vagrantfile new file mode 100644 index 0000000000..10bfb45353 --- /dev/null +++ b/Vagrantfile @@ -0,0 +1,57 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +# This Vagrantfile defines the requirements of a Linux development environment +# to develop/run Git. This environment can be set up conveniently by installing +# Vagrant and VirtualBox and calling "vagrant up" in the /usr/src/git directory. +# +# See https://github.com/git-for-windows/git/wiki/Vagrant for details. + +# Vagrantfile API/syntax version. Don't touch unless you know what you're doing! +VAGRANTFILE_API_VERSION = "2" + +$provision = < /tmp/.profile + mv /tmp/.profile /home/vagrant/.profile +} + +# add a nice greeting +cat >> /home/vagrant/.profile << \EOF + +cd /vagrant/ +export PATH=/home/vagrant/bin:$PATH +cat << \TOOEOF + +Welcome to the Vagrant setup for Git! +-------------------------------------- + +To build & install Git, just execute + + make -j NO_PERL_MAKEMAKER=t install + +For more information, see https://github.com/git-for-windows/git/wiki/Vagrant +TOOEOF +EOF + +cat << EOF + +Now that everything is set up, connect to the Vagrant machine with the command: + + vagrant ssh + +EOF +PROVISION + +Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| + config.vm.box = "ubuntu/trusty64" + config.vm.box_url = "https://atlas.hashicorp.com/ubuntu/boxes/trusty64" + + config.vm.provision :shell, :inline => $provision +end From 5829412cd5898e362cd275ab28f6b375a2a18e3f Mon Sep 17 00:00:00 2001 From: Steve Hoelzer Date: Mon, 19 Dec 2016 11:10:09 -0600 Subject: [PATCH 216/288] poll: Use GetTickCount64 to avoid wraparound issues From Visual Studio 2015 Code Analysis: Warning C28159 Consider using 'GetTickCount64' instead of 'GetTickCount'. Reason: GetTickCount overflows roughly every 49 days. Code that does not take that into account can loop indefinitely. GetTickCount64 operates on 64 bit values and does not have that problem. Signed-off-by: Steve Hoelzer --- compat/poll/poll.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/compat/poll/poll.c b/compat/poll/poll.c index b10adc780f..9cb267f40a 100644 --- a/compat/poll/poll.c +++ b/compat/poll/poll.c @@ -446,7 +446,8 @@ poll (struct pollfd *pfd, nfds_t nfd, int timeout) static HANDLE hEvent; WSANETWORKEVENTS ev; HANDLE h, handle_array[FD_SETSIZE + 2]; - DWORD ret, wait_timeout, nhandles, start = 0, elapsed, orig_timeout = 0; + DWORD ret, wait_timeout, nhandles, elapsed, orig_timeout = 0; + ULONGLONG start = 0; fd_set rfds, wfds, xfds; BOOL poll_again; MSG msg; @@ -462,7 +463,7 @@ poll (struct pollfd *pfd, nfds_t nfd, int timeout) if (timeout != INFTIM) { orig_timeout = timeout; - start = GetTickCount(); + start = GetTickCount64(); } if (!hEvent) @@ -611,7 +612,7 @@ restart: if (!rc && orig_timeout && timeout != INFTIM) { - elapsed = GetTickCount() - start; + elapsed = (DWORD)(GetTickCount64() - start); timeout = elapsed >= orig_timeout ? 0 : orig_timeout - elapsed; } From 93d0425332783367d5bf752c75d664e5de530e01 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 20 Dec 2016 17:18:05 +0100 Subject: [PATCH 217/288] poll: lazy-load GetTickCount64() This fixes the compilation, actually, as we still did not make the jump to post-Windows XP completely: we still compile with _WIN32_WINNT set to 0x0502 (which corresponds to Windows Server 2003 and is technically greater than Windows XP's 0x0501). However, GetTickCount64() is only available starting with Windows Vista/Windows Server 2008. Let's just lazy-load the function, which should also help Git for Windows contributors who want to reinstate Windows XP support. Signed-off-by: Johannes Schindelin --- compat/poll/poll.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/compat/poll/poll.c b/compat/poll/poll.c index 9cb267f40a..8e6d2ff10a 100644 --- a/compat/poll/poll.c +++ b/compat/poll/poll.c @@ -267,6 +267,20 @@ win32_compute_revents_socket (SOCKET h, int sought, long lNetworkEvents) return happened; } +#include +#include "compat/win32/lazyload.h" + +static ULONGLONG CompatGetTickCount64(void) +{ + DECLARE_PROC_ADDR(kernel32.dll, ULONGLONG, GetTickCount64, void); + + if (!INIT_PROC_ADDR(GetTickCount64)) + return (ULONGLONG)GetTickCount(); + + return GetTickCount64(); +} +#define GetTickCount64 CompatGetTickCount64 + #else /* !MinGW */ /* Convert select(2) returned fd_sets into poll(2) revents values. */ From c95651ae155f787f2b7851a204ce93d57983f839 Mon Sep 17 00:00:00 2001 From: J Wyman Date: Mon, 19 Dec 2016 10:31:37 -0500 Subject: [PATCH 218/288] Carry non-locking status value in the environment. If the user has specified '--no-lock-index' when calling git-status, it only seems reasonable that the user intends that option to be carried through to any child forks/procs as well. Currently, the '--no-lock-status' call is lost when submodules are checked. This change places the desired option into the environment, which is in turn passed down to all subsequent children. With cmd_status checking for '--no-lock--status' first from args then from environment, we're able to keep the option set in all children. Signed-off-by: J Wyman --- builtin/commit.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/builtin/commit.c b/builtin/commit.c index 1bd3e169c3..e2a7abea49 100644 --- a/builtin/commit.c +++ b/builtin/commit.c @@ -1373,6 +1373,11 @@ int cmd_status(int argc, const char **argv, const char *prefix) finalize_colopts(&s.colopts, -1); finalize_deferred_config(&s); + if (no_lock_index) + setenv("GIT_LOCK_INDEX", "false", 1); + else if (!git_parse_maybe_bool(getenv("GIT_LOCK_INDEX"))) + no_lock_index = 1; + handle_untracked_files_arg(&s); if (show_ignored_in_status) s.show_ignored_files = 1; From 8f5025e163580ca0689eade9a63c16a229272062 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Thu, 15 Dec 2016 11:34:39 -0500 Subject: [PATCH 219/288] diffcore-rename: speed up register_rename_src Teach register_rename_src() to see if new file pair can simply be appended to the rename_src[] array before performing the binary search to find the proper insertion point. This is a performance optimization. This routine is called during run_diff_files in status and the caller is iterating over the sorted index, so we should expect to be able to append in the normal case. The existing insert logic is preserved so we don't have to assume that, but simply take advantage of it if possible. Signed-off-by: Jeff Hostetler --- diffcore-rename.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/diffcore-rename.c b/diffcore-rename.c index f7444c86bd..543a409e1c 100644 --- a/diffcore-rename.c +++ b/diffcore-rename.c @@ -81,6 +81,18 @@ static struct diff_rename_src *register_rename_src(struct diff_filepair *p) first = 0; last = rename_src_nr; + + if (last > 0) { + struct diff_rename_src *src = &(rename_src[last-1]); + int cmp = strcmp(one->path, src->p->one->path); + if (!cmp) + return src; + if (cmp > 0) { + first = last; + goto append_it; + } + } + while (last > first) { int next = (last + first) >> 1; struct diff_rename_src *src = &(rename_src[next]); @@ -94,6 +106,7 @@ static struct diff_rename_src *register_rename_src(struct diff_filepair *p) first = next+1; } +append_it: /* insert to make it at "first" */ ALLOC_GROW(rename_src, rename_src_nr + 1, rename_src_alloc); rename_src_nr++; From 863ea05b9f3ba756ee160cffeaae4c66172d905c Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 23 Dec 2016 12:12:50 +0100 Subject: [PATCH 220/288] mingw: demonstrate a regression pushing to UNC paths On Windows, there are "UNC paths" to access network (AKA shared) folders, of the form \\server\sharename\directory. This provides a convenient way for Windows developers to share their Git repositories without having to have a dedicated server. Git for Windows v2.11.0 introduced a regression where pushing to said UNC paths no longer works, although fetching and cloning still does. Demonstrate what is the problem, using so-called "administrative shares": disk volumes are automatically shared under certain circumstances, e.g. the C: drive is shared as \\localhost\c$. The test needs to be skipped if the current directory is inaccessible via said administrative share, of course. Original-report: https://github.com/git-for-windows/git/issues/979 Signed-off-by: Johannes Schindelin --- t/t5580-clone-push-unc.sh | 46 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100755 t/t5580-clone-push-unc.sh diff --git a/t/t5580-clone-push-unc.sh b/t/t5580-clone-push-unc.sh new file mode 100755 index 0000000000..77abeaf4bd --- /dev/null +++ b/t/t5580-clone-push-unc.sh @@ -0,0 +1,46 @@ +#!/bin/sh + +test_description='various UNC path tests (Windows-only)' +. ./test-lib.sh + +if ! test_have_prereq MINGW; then + skip_all='skipping UNC path tests, requires Windows' + test_done +fi + +UNCPATH="$(pwd)" +case "$UNCPATH" in +[A-Z]:*) + # Use administrative share e.g. \\localhost\C$\git-sdk-64\usr\src\git + # (we use forward slashes here because MSYS2 and Git accept them, and + # they are easier on the eyes) + UNCPATH="//localhost/${UNCPATH%%:*}\$/${UNCPATH#?:}" + test -d "$UNCPATH" || { + skip_all='could not access administrative share; skipping' + test_done + } + ;; +*) + skip_all='skipping UNC path tests, cannot determine current path as UNC' + test_done + ;; +esac + +test_expect_success setup ' + test_commit initial +' + +test_expect_success clone ' + git clone "file://$UNCPATH" clone +' + +test_expect_failure push ' + ( + cd clone && + git checkout -b to-push && + test_commit to-push && + git push origin HEAD + ) +' + +test_done From 19ea0346da0f22fd742b8ffcc187a769c091353a Mon Sep 17 00:00:00 2001 From: Johannes Sixt Date: Wed, 14 Dec 2016 20:37:38 +0100 Subject: [PATCH 221/288] fix pushing to //server/share/dir on Windows normalize_path_copy() is not prepared to keep the double-slash of a //server/share/dir kind of path, but treats it like a regular POSIX style path and transforms it to /server/share/dir. The bug manifests when 'git push //server/share/dir master' is run, because tmp_objdir_add_as_alternate() uses the path in normalized form when it registers the quarantine object database via link_alt_odb_entries(). Needless to say that the directory cannot be accessed using the wrongly normalized path. Fix it by skipping all of the root part, not just a potential drive prefix. offset_1st_component takes care of this, see the implementation in compat/mingw.c::mingw_offset_1st_component(). This fixes https://github.com/git-for-windows/git/issues/979 Signed-off-by: Johannes Sixt Signed-off-by: Johannes Schindelin --- t/t5580-clone-push-unc.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/t5580-clone-push-unc.sh b/t/t5580-clone-push-unc.sh index 77abeaf4bd..e06d230724 100755 --- a/t/t5580-clone-push-unc.sh +++ b/t/t5580-clone-push-unc.sh @@ -34,7 +34,7 @@ test_expect_success clone ' git clone "file://$UNCPATH" clone ' -test_expect_failure push ' +test_expect_success push ' ( cd clone && git checkout -b to-push && From d6f88fd031985bef2f405ab70c212ffb526bd6b7 Mon Sep 17 00:00:00 2001 From: Segev Finer Date: Fri, 23 Dec 2016 23:49:02 +0200 Subject: [PATCH 222/288] connect: handle putty/plink also in GIT_SSH_COMMAND Git for Windows has special support for the popular SSH client PuTTY: when using PuTTY's non-interactive version ("plink.exe"), we use the -P option to specify the port rather than OpenSSH's -p option. TortoiseGit ships with its own, forked version of plink.exe, that adds support for the -batch option, and for good measure we special-case that, too. However, this special-casing of PuTTY only covers the case where the user overrides the SSH command via the environment variable GIT_SSH (which allows specifying the name of the executable), not GIT_SSH_COMMAND (which allows specifying a full command, including additional command-line options). When users want to pass any additional arguments to (Tortoise-)Plink, such as setting a private key, they are required to either use a shell script named plink or tortoiseplink or duplicate the logic that is already in Git for passing the correct style of command line arguments, which can be difficult, error prone and annoying to get right. This patch simply reuses the existing logic and expands it to cover GIT_SSH_COMMAND, too. Note: it may look a little heavy-handed to duplicate the entire command-line and then split it, only to extract the name of the executable. However, this is not a performance-critical code path, and the code is much more readable this way. Signed-off-by: Segev Finer Signed-off-by: Johannes Schindelin --- connect.c | 21 +++++++++++++++------ t/t5601-clone.sh | 15 +++++++++++++++ 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/connect.c b/connect.c index 8cb93b0720..c81f77001b 100644 --- a/connect.c +++ b/connect.c @@ -772,6 +772,7 @@ struct child_process *git_connect(int fd[2], const char *url, int putty = 0, tortoiseplink = 0; char *ssh_host = hostandport; const char *port = NULL; + char *ssh_argv0 = NULL; transport_check_allowed("ssh"); get_host_and_port(&ssh_host, &port); @@ -792,10 +793,15 @@ struct child_process *git_connect(int fd[2], const char *url, } ssh = get_ssh_command(); - if (!ssh) { - const char *base; - char *ssh_dup; + if (ssh) { + char *split_ssh = xstrdup(ssh); + const char **ssh_argv; + if (split_cmdline(split_ssh, &ssh_argv)) + ssh_argv0 = xstrdup(ssh_argv[0]); + free(split_ssh); + free((void *)ssh_argv); + } else { /* * GIT_SSH is the no-shell version of * GIT_SSH_COMMAND (and must remain so for @@ -807,8 +813,11 @@ struct child_process *git_connect(int fd[2], const char *url, if (!ssh) ssh = "ssh"; - ssh_dup = xstrdup(ssh); - base = basename(ssh_dup); + ssh_argv0 = xstrdup(ssh); + } + + if (ssh_argv0) { + const char *base = basename(ssh_argv0); tortoiseplink = !strcasecmp(base, "tortoiseplink") || !strcasecmp(base, "tortoiseplink.exe"); @@ -816,7 +825,7 @@ struct child_process *git_connect(int fd[2], const char *url, !strcasecmp(base, "plink") || !strcasecmp(base, "plink.exe"); - free(ssh_dup); + free(ssh_argv0); } argv_array_push(&conn->args, ssh); diff --git a/t/t5601-clone.sh b/t/t5601-clone.sh index a433394200..5b228e2675 100755 --- a/t/t5601-clone.sh +++ b/t/t5601-clone.sh @@ -386,6 +386,21 @@ test_expect_success 'tortoiseplink is like putty, with extra arguments' ' expect_ssh "-batch -P 123" myhost src ' +test_expect_success 'double quoted plink.exe in GIT_SSH_COMMAND' ' + copy_ssh_wrapper_as "$TRASH_DIRECTORY/plink.exe" && + GIT_SSH_COMMAND="\"$TRASH_DIRECTORY/plink.exe\" -v" \ + git clone "[myhost:123]:src" ssh-bracket-clone-plink-3 && + expect_ssh "-v -P 123" myhost src +' + +SQ="'" +test_expect_success 'single quoted plink.exe in GIT_SSH_COMMAND' ' + copy_ssh_wrapper_as "$TRASH_DIRECTORY/plink.exe" && + GIT_SSH_COMMAND="$SQ$TRASH_DIRECTORY/plink.exe$SQ -v" \ + git clone "[myhost:123]:src" ssh-bracket-clone-plink-4 && + expect_ssh "-v -P 123" myhost src +' + # Reset the GIT_SSH environment variable for clone tests. setup_ssh_wrapper From 3dc54d2680eef97ebaf97f2b1726abd1ef1b08e2 Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Wed, 25 Jan 2017 14:37:32 -0800 Subject: [PATCH 223/288] connect: rename tortoiseplink and putty variables One of these two may have originally been named after "what exact SSH implementation do we have" so that we can tweak the command line options, but these days "putty=1" no longer means "We are using the plink SSH implementation that comes with PuTTY". It is set when we guess that either PuTTY plink or Tortoiseplink is in use. Rename them after what effect is desired. The current "putty" option is about using "-P " when OpenSSH would use "-p ", so rename it to port_option whose value is either 'p' or 'P". The other one is about passing an extra command line option "-batch", so rename it needs_batch. [jes: wrapped overly-long line] Signed-off-by: Junio C Hamano Signed-off-by: Johannes Schindelin --- connect.c | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/connect.c b/connect.c index c81f77001b..9f750eacb6 100644 --- a/connect.c +++ b/connect.c @@ -769,7 +769,8 @@ struct child_process *git_connect(int fd[2], const char *url, conn->in = conn->out = -1; if (protocol == PROTO_SSH) { const char *ssh; - int putty = 0, tortoiseplink = 0; + int needs_batch = 0; + int port_option = 'p'; char *ssh_host = hostandport; const char *port = NULL; char *ssh_argv0 = NULL; @@ -819,12 +820,14 @@ struct child_process *git_connect(int fd[2], const char *url, if (ssh_argv0) { const char *base = basename(ssh_argv0); - tortoiseplink = !strcasecmp(base, "tortoiseplink") || - !strcasecmp(base, "tortoiseplink.exe"); - putty = tortoiseplink || - !strcasecmp(base, "plink") || - !strcasecmp(base, "plink.exe"); - + if (!strcasecmp(base, "tortoiseplink") || + !strcasecmp(base, "tortoiseplink.exe")) { + port_option = 'P'; + needs_batch = 1; + } else if (!strcasecmp(base, "plink") || + !strcasecmp(base, "plink.exe")) { + port_option = 'P'; + } free(ssh_argv0); } @@ -833,11 +836,11 @@ struct child_process *git_connect(int fd[2], const char *url, argv_array_push(&conn->args, "-4"); else if (flags & CONNECT_IPV6) argv_array_push(&conn->args, "-6"); - if (tortoiseplink) + if (needs_batch) argv_array_push(&conn->args, "-batch"); if (port) { - /* P is for PuTTY, p is for OpenSSH */ - argv_array_push(&conn->args, putty ? "-P" : "-p"); + argv_array_pushf(&conn->args, + "-%c", port_option); argv_array_push(&conn->args, port); } argv_array_push(&conn->args, ssh_host); From b6a731126c42baf618fcaef9b8df68bd57ab3952 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 1 Feb 2017 11:35:49 +0100 Subject: [PATCH 224/288] git_connect(): factor out SSH variant handling We handle plink and tortoiseplink as OpenSSH replacements, by passing the correct command-line options when detecting that they are used. To let users override that auto-detection (in case Git gets it wrong), we need to introduce new code to that end. In preparation for this code, let's factor out the SSH variant handling into its own function, handle_ssh_variant(). Signed-off-by: Johannes Schindelin --- connect.c | 72 +++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 46 insertions(+), 26 deletions(-) diff --git a/connect.c b/connect.c index 9f750eacb6..2734b9a1ca 100644 --- a/connect.c +++ b/connect.c @@ -691,6 +691,44 @@ static const char *get_ssh_command(void) return NULL; } +static int handle_ssh_variant(const char *ssh_command, int is_cmdline, + int *port_option, int *needs_batch) +{ + const char *variant; + char *p = NULL; + + if (!is_cmdline) { + p = xstrdup(ssh_command); + variant = basename(p); + } else { + const char **ssh_argv; + + p = xstrdup(ssh_command); + if (split_cmdline(p, &ssh_argv)) { + variant = basename((char *)ssh_argv[0]); + /* + * At this point, variant points into the buffer + * referenced by p, hence we do not need ssh_argv + * any longer. + */ + free(ssh_argv); + } else + return 0; + } + + if (!strcasecmp(variant, "plink") || + !strcasecmp(variant, "plink.exe")) + *port_option = 'P'; + else if (!strcasecmp(variant, "tortoiseplink") || + !strcasecmp(variant, "tortoiseplink.exe")) { + *port_option = 'P'; + *needs_batch = 1; + } + free(p); + + return 1; +} + /* * This returns a dummy child_process if the transport protocol does not * need fork(2), or a struct child_process object if it does. Once done, @@ -773,7 +811,6 @@ struct child_process *git_connect(int fd[2], const char *url, int port_option = 'p'; char *ssh_host = hostandport; const char *port = NULL; - char *ssh_argv0 = NULL; transport_check_allowed("ssh"); get_host_and_port(&ssh_host, &port); @@ -794,15 +831,10 @@ struct child_process *git_connect(int fd[2], const char *url, } ssh = get_ssh_command(); - if (ssh) { - char *split_ssh = xstrdup(ssh); - const char **ssh_argv; - - if (split_cmdline(split_ssh, &ssh_argv)) - ssh_argv0 = xstrdup(ssh_argv[0]); - free(split_ssh); - free((void *)ssh_argv); - } else { + if (ssh) + handle_ssh_variant(ssh, 1, &port_option, + &needs_batch); + else { /* * GIT_SSH is the no-shell version of * GIT_SSH_COMMAND (and must remain so for @@ -813,22 +845,10 @@ struct child_process *git_connect(int fd[2], const char *url, ssh = getenv("GIT_SSH"); if (!ssh) ssh = "ssh"; - - ssh_argv0 = xstrdup(ssh); - } - - if (ssh_argv0) { - const char *base = basename(ssh_argv0); - - if (!strcasecmp(base, "tortoiseplink") || - !strcasecmp(base, "tortoiseplink.exe")) { - port_option = 'P'; - needs_batch = 1; - } else if (!strcasecmp(base, "plink") || - !strcasecmp(base, "plink.exe")) { - port_option = 'P'; - } - free(ssh_argv0); + else + handle_ssh_variant(ssh, 0, + &port_option, + &needs_batch); } argv_array_push(&conn->args, ssh); From 21137a55f774d5b3a8a70994632ae6078b5c0bf0 Mon Sep 17 00:00:00 2001 From: Segev Finer Date: Sat, 14 Jan 2017 17:19:04 +0200 Subject: [PATCH 225/288] connect: Add the envvar GIT_SSH_VARIANT and ssh.variant config This environment variable and configuration value allow to override the autodetection of plink/tortoiseplink in case that Git gets it wrong. [jes: wrapped overly-long lines, factored out and changed get_ssh_variant() to handle_ssh_variant() to accomodate the change from the putty/tortoiseplink variables to port_option/needs_batch, adjusted the documentation, free()d value obtained from the config.] Signed-off-by: Segev Finer Signed-off-by: Johannes Schindelin --- Documentation/config.txt | 11 +++++++++++ Documentation/git.txt | 6 ++++++ connect.c | 11 ++++++++--- t/t5601-clone.sh | 26 ++++++++++++++++++++++++++ 4 files changed, 51 insertions(+), 3 deletions(-) diff --git a/Documentation/config.txt b/Documentation/config.txt index 8be0181b82..be88cd1a17 100644 --- a/Documentation/config.txt +++ b/Documentation/config.txt @@ -1970,6 +1970,17 @@ Environment variable settings always override any matches. The URLs that are matched against are those given directly to Git commands. This means any URLs visited as a result of a redirection do not participate in matching. +ssh.variant:: + Depending on the value of the environment variables `GIT_SSH` or + `GIT_SSH_COMMAND`, or the config setting `core.sshCommand`, Git + auto-detects whether to adjust its command-line parameters for use + with plink or tortoiseplink, as opposed to the default (OpenSSH). ++ +The config variable `ssh.variant` can be set to override this auto-detection; +valid values are `ssh`, `plink`, `putty` or `tortoiseplink`. Any other value +will be treated as normal ssh. This setting can be overridden via the +environment variable `GIT_SSH_VARIANT`. + i18n.commitEncoding:: Character encoding the commit messages are stored in; Git itself does not care per se, but this information is necessary e.g. when diff --git a/Documentation/git.txt b/Documentation/git.txt index 97e4ee8083..c9f26be56f 100644 --- a/Documentation/git.txt +++ b/Documentation/git.txt @@ -1020,6 +1020,12 @@ Usually it is easier to configure any desired options through your personal `.ssh/config` file. Please consult your ssh documentation for further details. +`GIT_SSH_VARIANT`:: + If this environment variable is set, it overrides Git's autodetection + whether `GIT_SSH`/`GIT_SSH_COMMAND`/`core.sshCommand` refer to OpenSSH, + plink or tortoiseplink. This variable overrides the config setting + `ssh.variant` that serves the same purpose. + `GIT_ASKPASS`:: If this environment variable is set, then Git commands which need to acquire passwords or passphrases (e.g. for HTTP or IMAP authentication) diff --git a/connect.c b/connect.c index 2734b9a1ca..7f1f802396 100644 --- a/connect.c +++ b/connect.c @@ -694,10 +694,14 @@ static const char *get_ssh_command(void) static int handle_ssh_variant(const char *ssh_command, int is_cmdline, int *port_option, int *needs_batch) { - const char *variant; + const char *variant = getenv("GIT_SSH_VARIANT"); char *p = NULL; - if (!is_cmdline) { + if (variant) + ; /* okay, fall through */ + else if (!git_config_get_string("ssh.variant", &p)) + variant = p; + else if (!is_cmdline) { p = xstrdup(ssh_command); variant = basename(p); } else { @@ -717,7 +721,8 @@ static int handle_ssh_variant(const char *ssh_command, int is_cmdline, } if (!strcasecmp(variant, "plink") || - !strcasecmp(variant, "plink.exe")) + !strcasecmp(variant, "plink.exe") || + !strcasecmp(variant, "putty")) *port_option = 'P'; else if (!strcasecmp(variant, "tortoiseplink") || !strcasecmp(variant, "tortoiseplink.exe")) { diff --git a/t/t5601-clone.sh b/t/t5601-clone.sh index 5b228e2675..ba08b0f7c2 100755 --- a/t/t5601-clone.sh +++ b/t/t5601-clone.sh @@ -401,6 +401,32 @@ test_expect_success 'single quoted plink.exe in GIT_SSH_COMMAND' ' expect_ssh "-v -P 123" myhost src ' +test_expect_success 'GIT_SSH_VARIANT overrides plink detection' ' + copy_ssh_wrapper_as "$TRASH_DIRECTORY/plink" && + GIT_SSH_VARIANT=ssh \ + git clone "[myhost:123]:src" ssh-bracket-clone-variant-1 && + expect_ssh "-p 123" myhost src +' + +test_expect_success 'ssh.variant overrides plink detection' ' + copy_ssh_wrapper_as "$TRASH_DIRECTORY/plink" && + git -c ssh.variant=ssh \ + clone "[myhost:123]:src" ssh-bracket-clone-variant-2 && + expect_ssh "-p 123" myhost src +' + +test_expect_success 'GIT_SSH_VARIANT overrides plink detection to plink' ' + GIT_SSH_VARIANT=plink \ + git clone "[myhost:123]:src" ssh-bracket-clone-variant-3 && + expect_ssh "-P 123" myhost src +' + +test_expect_success 'GIT_SSH_VARIANT overrides plink to tortoiseplink' ' + GIT_SSH_VARIANT=tortoiseplink \ + git clone "[myhost:123]:src" ssh-bracket-clone-variant-4 && + expect_ssh "-batch -P 123" myhost src +' + # Reset the GIT_SSH environment variable for clone tests. setup_ssh_wrapper From 27be8f2c9062d0f1a36cb6ee9f2dd8cfd51690f5 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 14 Nov 2016 13:21:37 +0100 Subject: [PATCH 226/288] difftool: add a skeleton for the upcoming builtin This adds a builtin difftool that still falls back to the legacy Perl version, which has been renamed to `legacy-difftool`. The idea is that the new, experimental, builtin difftool immediately hands off to the legacy difftool for now, unless the config variable difftool.useBuiltin is set to true. This feature flag will be used in the upcoming Git for Windows v2.11.0 release, to allow early testers to opt-in to use the builtin difftool and flesh out any bugs. Signed-off-by: Johannes Schindelin --- .gitignore | 1 + Makefile | 3 +- builtin.h | 1 + builtin/difftool.c | 63 +++++++++++++++++++ git-difftool.perl => git-legacy-difftool.perl | 0 git.c | 6 ++ t/t7800-difftool.sh | 2 + 7 files changed, 75 insertions(+), 1 deletion(-) create mode 100644 builtin/difftool.c rename git-difftool.perl => git-legacy-difftool.perl (100%) diff --git a/.gitignore b/.gitignore index 6722f78f9a..5555ae025b 100644 --- a/.gitignore +++ b/.gitignore @@ -76,6 +76,7 @@ /git-init-db /git-interpret-trailers /git-instaweb +/git-legacy-difftool /git-log /git-ls-files /git-ls-remote diff --git a/Makefile b/Makefile index 76267262c1..4137a9915b 100644 --- a/Makefile +++ b/Makefile @@ -527,7 +527,7 @@ SCRIPT_LIB += git-sh-setup SCRIPT_LIB += git-sh-i18n SCRIPT_PERL += git-add--interactive.perl -SCRIPT_PERL += git-difftool.perl +SCRIPT_PERL += git-legacy-difftool.perl SCRIPT_PERL += git-archimport.perl SCRIPT_PERL += git-cvsexportcommit.perl SCRIPT_PERL += git-cvsimport.perl @@ -888,6 +888,7 @@ BUILTIN_OBJS += builtin/diff-files.o BUILTIN_OBJS += builtin/diff-index.o BUILTIN_OBJS += builtin/diff-tree.o BUILTIN_OBJS += builtin/diff.o +BUILTIN_OBJS += builtin/difftool.o BUILTIN_OBJS += builtin/fast-export.o BUILTIN_OBJS += builtin/fetch-pack.o BUILTIN_OBJS += builtin/fetch.o diff --git a/builtin.h b/builtin.h index b9122bc5f4..67f80519da 100644 --- a/builtin.h +++ b/builtin.h @@ -60,6 +60,7 @@ extern int cmd_diff_files(int argc, const char **argv, const char *prefix); extern int cmd_diff_index(int argc, const char **argv, const char *prefix); extern int cmd_diff(int argc, const char **argv, const char *prefix); extern int cmd_diff_tree(int argc, const char **argv, const char *prefix); +extern int cmd_difftool(int argc, const char **argv, const char *prefix); extern int cmd_fast_export(int argc, const char **argv, const char *prefix); extern int cmd_fetch(int argc, const char **argv, const char *prefix); extern int cmd_fetch_pack(int argc, const char **argv, const char *prefix); diff --git a/builtin/difftool.c b/builtin/difftool.c new file mode 100644 index 0000000000..53870bbaf7 --- /dev/null +++ b/builtin/difftool.c @@ -0,0 +1,63 @@ +/* + * "git difftool" builtin command + * + * This is a wrapper around the GIT_EXTERNAL_DIFF-compatible + * git-difftool--helper script. + * + * This script exports GIT_EXTERNAL_DIFF and GIT_PAGER for use by git. + * The GIT_DIFF* variables are exported for use by git-difftool--helper. + * + * Any arguments that are unknown to this script are forwarded to 'git diff'. + * + * Copyright (C) 2016 Johannes Schindelin + */ +#include "builtin.h" +#include "run-command.h" +#include "exec_cmd.h" + +/* + * NEEDSWORK: this function can go once the legacy-difftool Perl script is + * retired. + * + * We intentionally avoid reading the config directly here, to avoid messing up + * the GIT_* environment variables when we need to fall back to exec()ing the + * Perl script. + */ +static int use_builtin_difftool(void) { + struct child_process cp = CHILD_PROCESS_INIT; + struct strbuf out = STRBUF_INIT; + int ret; + + argv_array_pushl(&cp.args, + "config", "--bool", "difftool.usebuiltin", NULL); + cp.git_cmd = 1; + if (capture_command(&cp, &out, 6)) + return 0; + strbuf_trim(&out); + ret = !strcmp("true", out.buf); + strbuf_release(&out); + return ret; +} + +int cmd_difftool(int argc, const char **argv, const char *prefix) +{ + /* + * NEEDSWORK: Once the builtin difftool has been tested enough + * and git-legacy-difftool.perl is retired to contrib/, this preamble + * can be removed. + */ + if (!use_builtin_difftool()) { + const char *path = mkpath("%s/git-legacy-difftool", + git_exec_path()); + + if (sane_execvp(path, (char **)argv) < 0) + die_errno("could not exec %s", path); + + return 0; + } + prefix = setup_git_directory(); + trace_repo_setup(prefix); + setup_work_tree(); + + die("TODO"); +} diff --git a/git-difftool.perl b/git-legacy-difftool.perl similarity index 100% rename from git-difftool.perl rename to git-legacy-difftool.perl diff --git a/git.c b/git.c index c8fe6637df..519a4c6f82 100644 --- a/git.c +++ b/git.c @@ -424,6 +424,12 @@ static struct cmd_struct commands[] = { { "diff-files", cmd_diff_files, RUN_SETUP | NEED_WORK_TREE }, { "diff-index", cmd_diff_index, RUN_SETUP }, { "diff-tree", cmd_diff_tree, RUN_SETUP }, + /* + * NEEDSWORK: Once the redirection to git-legacy-difftool.perl in + * builtin/difftool.c has been removed, this entry should be changed to + * RUN_SETUP | NEED_WORK_TREE + */ + { "difftool", cmd_difftool }, { "fast-export", cmd_fast_export, RUN_SETUP }, { "fetch", cmd_fetch, RUN_SETUP }, { "fetch-pack", cmd_fetch_pack, RUN_SETUP }, diff --git a/t/t7800-difftool.sh b/t/t7800-difftool.sh index 99d4123461..e94910c563 100755 --- a/t/t7800-difftool.sh +++ b/t/t7800-difftool.sh @@ -23,6 +23,8 @@ prompt_given () test "$prompt" = "Launch 'test-tool' [Y/n]? branch" } +# NEEDSWORK: lose all the PERL prereqs once legacy-difftool is retired. + # Create a file on master and change it on branch test_expect_success PERL 'setup' ' echo master >file && From 5add67cfb99703bcdb07cd12580214df87ee3a50 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 14 Nov 2016 13:21:37 +0100 Subject: [PATCH 227/288] difftool: implement the functionality in the builtin This patch gives life to the skeleton added in the previous patch. The motivation for converting the difftool is that Perl scripts are not at all native on Windows, and that `git difftool` therefore is pretty slow on that platform, when there is no good reason for it to be slow. In addition, Perl does not really have access to Git's internals. That means that any script will always have to jump through unnecessary hoops. The current version of the builtin difftool does not, however, make full use of the internals but instead chooses to spawn a couple of Git processes, still, to make for an easier conversion. There remains a lot of room for improvement, left for a later date. Note: to play it safe, the original difftool is still called unless the config setting difftool.useBuiltin is set to true. The reason: this new, experimental, builtin difftool will be shipped as part of Git for Windows v2.11.0, to allow for easier large-scale testing, but of course as an opt-in feature. Sadly, the speedup is more noticable on Linux than on Windows: a quick test shows that t7800-difftool.sh runs in (2.183s/0.052s/0.108s) (real/user/sys) in a Linux VM, down from (6.529s/3.112s/0.644s), while on Windows, it is (36.064s/2.730s/7.194s), down from (47.637s/2.407s/6.863s). The culprit is most likely the overhead incurred from *still* having to shell out to mergetool-lib.sh and difftool--helper.sh. Signed-off-by: Johannes Schindelin --- builtin/difftool.c | 672 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 671 insertions(+), 1 deletion(-) diff --git a/builtin/difftool.c b/builtin/difftool.c index 53870bbaf7..2115e548a5 100644 --- a/builtin/difftool.c +++ b/builtin/difftool.c @@ -11,9 +11,610 @@ * * Copyright (C) 2016 Johannes Schindelin */ +#include "cache.h" #include "builtin.h" #include "run-command.h" #include "exec_cmd.h" +#include "parse-options.h" +#include "argv-array.h" +#include "strbuf.h" +#include "lockfile.h" +#include "dir.h" + +static char *diff_gui_tool; +static int trust_exit_code; + +static const char *const builtin_difftool_usage[] = { + N_("git difftool [] [ []] [--] [...]"), + NULL +}; + +static int difftool_config(const char *var, const char *value, void *cb) +{ + if (!strcmp(var, "diff.guitool")) { + diff_gui_tool = xstrdup(value); + return 0; + } + + if (!strcmp(var, "difftool.trustexitcode")) { + trust_exit_code = git_config_bool(var, value); + return 0; + } + + return git_default_config(var, value, cb); +} + +static int print_tool_help(void) +{ + const char *argv[] = { "mergetool", "--tool-help=diff", NULL }; + return run_command_v_opt(argv, RUN_GIT_CMD); +} + +static int parse_index_info(char *p, int *mode1, int *mode2, + struct object_id *oid1, struct object_id *oid2, + char *status) +{ + if (*p != ':') + return error("expected ':', got '%c'", *p); + *mode1 = (int)strtol(p + 1, &p, 8); + if (*p != ' ') + return error("expected ' ', got '%c'", *p); + *mode2 = (int)strtol(p + 1, &p, 8); + if (*p != ' ') + return error("expected ' ', got '%c'", *p); + if (get_oid_hex(++p, oid1)) + return error("expected object ID, got '%s'", p + 1); + p += GIT_SHA1_HEXSZ; + if (*p != ' ') + return error("expected ' ', got '%c'", *p); + if (get_oid_hex(++p, oid2)) + return error("expected object ID, got '%s'", p + 1); + p += GIT_SHA1_HEXSZ; + if (*p != ' ') + return error("expected ' ', got '%c'", *p); + *status = *++p; + if (!*status) + return error("missing status"); + if (p[1] && !isdigit(p[1])) + return error("unexpected trailer: '%s'", p + 1); + return 0; +} + +/* + * Remove any trailing slash from $workdir + * before starting to avoid double slashes in symlink targets. + */ +static void add_path(struct strbuf *buf, size_t base_len, const char *path) +{ + strbuf_setlen(buf, base_len); + if (buf->len && buf->buf[buf->len - 1] != '/') + strbuf_addch(buf, '/'); + strbuf_addstr(buf, path); +} + +/* + * Determine whether we can simply reuse the file in the worktree. + */ +static int use_wt_file(const char *workdir, const char *name, + struct object_id *oid) +{ + struct strbuf buf = STRBUF_INIT; + struct stat st; + int use = 0; + + strbuf_addstr(&buf, workdir); + add_path(&buf, buf.len, name); + + if (!lstat(buf.buf, &st) && !S_ISLNK(st.st_mode)) { + struct object_id wt_oid; + int fd = open(buf.buf, O_RDONLY); + + if (fd >= 0 && + !index_fd(wt_oid.hash, fd, &st, OBJ_BLOB, name, 0)) { + if (is_null_oid(oid)) { + oidcpy(oid, &wt_oid); + use = 1; + } else if (!oidcmp(oid, &wt_oid)) + use = 1; + } + } + + strbuf_release(&buf); + + return use; +} + +struct working_tree_entry { + struct hashmap_entry entry; + char path[FLEX_ARRAY]; +}; + +static int working_tree_entry_cmp(struct working_tree_entry *a, + struct working_tree_entry *b, void *keydata) +{ + return strcmp(a->path, b->path); +} + +/* + * The `left` and `right` entries hold paths for the symlinks hashmap, + * and a SHA-1 surrounded by brief text for submodules. + */ +struct pair_entry { + struct hashmap_entry entry; + char left[PATH_MAX], right[PATH_MAX]; + const char path[FLEX_ARRAY]; +}; + +static int pair_cmp(struct pair_entry *a, struct pair_entry *b, void *keydata) +{ + return strcmp(a->path, b->path); +} + +static void add_left_or_right(struct hashmap *map, const char *path, + const char *content, int is_right) +{ + struct pair_entry *e, *existing; + + FLEX_ALLOC_STR(e, path, path); + hashmap_entry_init(e, strhash(path)); + existing = hashmap_get(map, e, NULL); + if (existing) { + free(e); + e = existing; + } else { + e->left[0] = e->right[0] = '\0'; + hashmap_add(map, e); + } + strlcpy(is_right ? e->right : e->left, content, PATH_MAX); +} + +struct path_entry { + struct hashmap_entry entry; + char path[FLEX_ARRAY]; +}; + +static int path_entry_cmp(struct path_entry *a, struct path_entry *b, void *key) +{ + return strcmp(a->path, key ? key : b->path); +} + +static void changed_files(struct hashmap *result, const char *index_path, + const char *workdir) +{ + struct child_process update_index = CHILD_PROCESS_INIT; + struct child_process diff_files = CHILD_PROCESS_INIT; + struct strbuf index_env = STRBUF_INIT, buf = STRBUF_INIT; + const char *git_dir = absolute_path(get_git_dir()), *env[] = { + NULL, NULL + }; + FILE *fp; + + strbuf_addf(&index_env, "GIT_INDEX_FILE=%s", index_path); + env[0] = index_env.buf; + + argv_array_pushl(&update_index.args, + "--git-dir", git_dir, "--work-tree", workdir, + "update-index", "--really-refresh", "-q", + "--unmerged", NULL); + update_index.no_stdin = 1; + update_index.no_stdout = 1; + update_index.no_stderr = 1; + update_index.git_cmd = 1; + update_index.use_shell = 0; + update_index.clean_on_exit = 1; + update_index.dir = workdir; + update_index.env = env; + /* Ignore any errors of update-index */ + run_command(&update_index); + + argv_array_pushl(&diff_files.args, + "--git-dir", git_dir, "--work-tree", workdir, + "diff-files", "--name-only", "-z", NULL); + diff_files.no_stdin = 1; + diff_files.git_cmd = 1; + diff_files.use_shell = 0; + diff_files.clean_on_exit = 1; + diff_files.out = -1; + diff_files.dir = workdir; + diff_files.env = env; + if (start_command(&diff_files)) + die("could not obtain raw diff"); + fp = xfdopen(diff_files.out, "r"); + while (!strbuf_getline_nul(&buf, fp)) { + struct path_entry *entry; + FLEX_ALLOC_STR(entry, path, buf.buf); + hashmap_entry_init(entry, strhash(buf.buf)); + hashmap_add(result, entry); + } + if (finish_command(&diff_files)) + die("diff-files did not exit properly"); + strbuf_release(&index_env); + strbuf_release(&buf); +} + +static NORETURN void exit_cleanup(const char *tmpdir, int exit_code) +{ + struct strbuf buf = STRBUF_INIT; + strbuf_addstr(&buf, tmpdir); + remove_dir_recursively(&buf, 0); + if (exit_code) + warning(_("failed: %d"), exit_code); + exit(exit_code); +} + +static int ensure_leading_directories(char *path) +{ + switch (safe_create_leading_directories(path)) { + case SCLD_OK: + case SCLD_EXISTS: + return 0; + default: + return error(_("could not create leading directories " + "of '%s'"), path); + } +} + +static int run_dir_diff(const char *extcmd, int symlinks, const char *prefix, + int argc, const char **argv) +{ + char tmpdir[PATH_MAX]; + struct strbuf info = STRBUF_INIT, lpath = STRBUF_INIT; + struct strbuf rpath = STRBUF_INIT, buf = STRBUF_INIT; + struct strbuf ldir = STRBUF_INIT, rdir = STRBUF_INIT; + struct strbuf wtdir = STRBUF_INIT; + size_t ldir_len, rdir_len, wtdir_len; + struct cache_entry *ce = xcalloc(1, sizeof(ce) + PATH_MAX + 1); + const char *workdir, *tmp; + int ret = 0, i; + FILE *fp; + struct hashmap working_tree_dups, submodules, symlinks2; + struct hashmap_iter iter; + struct pair_entry *entry; + enum object_type type; + unsigned long size; + struct index_state wtindex; + struct checkout lstate, rstate; + int rc, flags = RUN_GIT_CMD, err = 0; + struct child_process child = CHILD_PROCESS_INIT; + const char *helper_argv[] = { "difftool--helper", NULL, NULL, NULL }; + struct hashmap wt_modified, tmp_modified; + int indices_loaded = 0; + + workdir = get_git_work_tree(); + + /* Setup temp directories */ + tmp = getenv("TMPDIR"); + xsnprintf(tmpdir, sizeof(tmpdir), "%s/git-difftool.XXXXXX", tmp ? tmp : "/tmp"); + if (!mkdtemp(tmpdir)) + return error("could not create '%s'", tmpdir); + strbuf_addf(&ldir, "%s/left/", tmpdir); + strbuf_addf(&rdir, "%s/right/", tmpdir); + strbuf_addstr(&wtdir, workdir); + if (!wtdir.len || !is_dir_sep(wtdir.buf[wtdir.len - 1])) + strbuf_addch(&wtdir, '/'); + mkdir(ldir.buf, 0700); + mkdir(rdir.buf, 0700); + + memset(&wtindex, 0, sizeof(wtindex)); + + memset(&lstate, 0, sizeof(lstate)); + lstate.base_dir = ldir.buf; + lstate.base_dir_len = ldir.len; + lstate.force = 1; + memset(&rstate, 0, sizeof(rstate)); + rstate.base_dir = rdir.buf; + rstate.base_dir_len = rdir.len; + rstate.force = 1; + + ldir_len = ldir.len; + rdir_len = rdir.len; + wtdir_len = wtdir.len; + + hashmap_init(&working_tree_dups, + (hashmap_cmp_fn)working_tree_entry_cmp, 0); + hashmap_init(&submodules, (hashmap_cmp_fn)pair_cmp, 0); + hashmap_init(&symlinks2, (hashmap_cmp_fn)pair_cmp, 0); + + child.no_stdin = 1; + child.git_cmd = 1; + child.use_shell = 0; + child.clean_on_exit = 1; + child.dir = prefix; + child.out = -1; + argv_array_pushl(&child.args, "diff", "--raw", "--no-abbrev", "-z", + NULL); + for (i = 0; i < argc; i++) + argv_array_push(&child.args, argv[i]); + if (start_command(&child)) + die("could not obtain raw diff"); + fp = xfdopen(child.out, "r"); + + /* Build index info for left and right sides of the diff */ + i = 0; + while (!strbuf_getline_nul(&info, fp)) { + int lmode, rmode; + struct object_id loid, roid; + char status; + const char *src_path, *dst_path; + size_t src_path_len, dst_path_len; + + if (starts_with(info.buf, "::")) + die(N_("combined diff formats('-c' and '--cc') are " + "not supported in\n" + "directory diff mode('-d' and '--dir-diff').")); + + if (parse_index_info(info.buf, &lmode, &rmode, &loid, &roid, + &status)) + break; + if (strbuf_getline_nul(&lpath, fp)) + break; + src_path = lpath.buf; + src_path_len = lpath.len; + + i++; + if (status != 'C' && status != 'R') { + dst_path = src_path; + dst_path_len = src_path_len; + } else { + if (strbuf_getline_nul(&rpath, fp)) + break; + dst_path = rpath.buf; + dst_path_len = rpath.len; + } + + if (S_ISGITLINK(lmode) || S_ISGITLINK(rmode)) { + strbuf_reset(&buf); + strbuf_addf(&buf, "Subproject commit %s", + oid_to_hex(&loid)); + add_left_or_right(&submodules, src_path, buf.buf, 0); + strbuf_reset(&buf); + strbuf_addf(&buf, "Subproject commit %s", + oid_to_hex(&roid)); + if (!oidcmp(&loid, &roid)) + strbuf_addstr(&buf, "-dirty"); + add_left_or_right(&submodules, dst_path, buf.buf, 1); + continue; + } + + if (S_ISLNK(lmode)) { + char *content = read_sha1_file(loid.hash, &type, &size); + add_left_or_right(&symlinks2, src_path, content, 0); + free(content); + } + + if (S_ISLNK(rmode)) { + char *content = read_sha1_file(roid.hash, &type, &size); + add_left_or_right(&symlinks2, dst_path, content, 1); + free(content); + } + + if (lmode && status != 'C') { + ce->ce_mode = lmode; + oidcpy(&ce->oid, &loid); + strcpy(ce->name, src_path); + ce->ce_namelen = src_path_len; + if (checkout_entry(ce, &lstate, NULL)) + return error("could not write '%s'", src_path); + } + + if (rmode) { + struct working_tree_entry *entry; + + /* Avoid duplicate working_tree entries */ + FLEX_ALLOC_STR(entry, path, dst_path); + hashmap_entry_init(entry, strhash(dst_path)); + if (hashmap_get(&working_tree_dups, entry, NULL)) { + free(entry); + continue; + } + hashmap_add(&working_tree_dups, entry); + + if (!use_wt_file(workdir, dst_path, &roid)) { + ce->ce_mode = rmode; + oidcpy(&ce->oid, &roid); + strcpy(ce->name, dst_path); + ce->ce_namelen = dst_path_len; + if (checkout_entry(ce, &rstate, NULL)) + return error("could not write '%s'", + dst_path); + } else if (!is_null_oid(&roid)) { + /* + * Changes in the working tree need special + * treatment since they are not part of the + * index. + */ + struct cache_entry *ce2 = + make_cache_entry(rmode, roid.hash, + dst_path, 0, 0); + + add_index_entry(&wtindex, ce2, + ADD_CACHE_JUST_APPEND); + + add_path(&rdir, rdir_len, dst_path); + if (ensure_leading_directories(rdir.buf)) + return error("could not create " + "directory for '%s'", + dst_path); + add_path(&wtdir, wtdir_len, dst_path); + if (symlinks) { + if (symlink(wtdir.buf, rdir.buf)) { + ret = error_errno("could not symlink '%s' to '%s'", wtdir.buf, rdir.buf); + goto finish; + } + } else { + struct stat st; + if (stat(wtdir.buf, &st)) + st.st_mode = 0644; + if (copy_file(rdir.buf, wtdir.buf, + st.st_mode)) { + ret = error("could not copy '%s' to '%s'", wtdir.buf, rdir.buf); + goto finish; + } + } + } + } + } + + if (finish_command(&child)) { + ret = error("error occurred running diff --raw"); + goto finish; + } + + if (!i) + return 0; + + /* + * Changes to submodules require special treatment.This loop writes a + * temporary file to both the left and right directories to show the + * change in the recorded SHA1 for the submodule. + */ + hashmap_iter_init(&submodules, &iter); + while ((entry = hashmap_iter_next(&iter))) { + if (*entry->left) { + add_path(&ldir, ldir_len, entry->path); + ensure_leading_directories(ldir.buf); + write_file(ldir.buf, "%s", entry->left); + } + if (*entry->right) { + add_path(&rdir, rdir_len, entry->path); + ensure_leading_directories(rdir.buf); + write_file(rdir.buf, "%s", entry->right); + } + } + + /* + * Symbolic links require special treatment.The standard "git diff" + * shows only the link itself, not the contents of the link target. + * This loop replicates that behavior. + */ + hashmap_iter_init(&symlinks2, &iter); + while ((entry = hashmap_iter_next(&iter))) { + if (*entry->left) { + add_path(&ldir, ldir_len, entry->path); + ensure_leading_directories(ldir.buf); + write_file(ldir.buf, "%s", entry->left); + } + if (*entry->right) { + add_path(&rdir, rdir_len, entry->path); + ensure_leading_directories(rdir.buf); + write_file(rdir.buf, "%s", entry->right); + } + } + + strbuf_release(&buf); + + strbuf_setlen(&ldir, ldir_len); + helper_argv[1] = ldir.buf; + strbuf_setlen(&rdir, rdir_len); + helper_argv[2] = rdir.buf; + + if (extcmd) { + helper_argv[0] = extcmd; + flags = 0; + } else + setenv("GIT_DIFFTOOL_DIRDIFF", "true", 1); + rc = run_command_v_opt(helper_argv, flags); + + /* + * If the diff includes working copy files and those + * files were modified during the diff, then the changes + * should be copied back to the working tree. + * Do not copy back files when symlinks are used and the + * external tool did not replace the original link with a file. + * + * These hashes are loaded lazily since they aren't needed + * in the common case of --symlinks and the difftool updating + * files through the symlink. + */ + hashmap_init(&wt_modified, (hashmap_cmp_fn)path_entry_cmp, + wtindex.cache_nr); + hashmap_init(&tmp_modified, (hashmap_cmp_fn)path_entry_cmp, + wtindex.cache_nr); + + for (i = 0; i < wtindex.cache_nr; i++) { + struct hashmap_entry dummy; + const char *name = wtindex.cache[i]->name; + struct stat st; + + add_path(&rdir, rdir_len, name); + if (lstat(rdir.buf, &st)) + continue; + + if ((symlinks && S_ISLNK(st.st_mode)) || !S_ISREG(st.st_mode)) + continue; + + if (!indices_loaded) { + static struct lock_file lock; + strbuf_reset(&buf); + strbuf_addf(&buf, "%s/wtindex", tmpdir); + if (hold_lock_file_for_update(&lock, buf.buf, 0) < 0 || + write_locked_index(&wtindex, &lock, COMMIT_LOCK)) { + ret = error("could not write %s", buf.buf); + rollback_lock_file(&lock); + goto finish; + } + changed_files(&wt_modified, buf.buf, workdir); + strbuf_setlen(&rdir, rdir_len); + changed_files(&tmp_modified, buf.buf, rdir.buf); + add_path(&rdir, rdir_len, name); + indices_loaded = 1; + } + + hashmap_entry_init(&dummy, strhash(name)); + if (hashmap_get(&tmp_modified, &dummy, name)) { + add_path(&wtdir, wtdir_len, name); + if (hashmap_get(&wt_modified, &dummy, name)) { + warning(_("both files modified: '%s' and '%s'."), + wtdir.buf, rdir.buf); + warning(_("working tree file has been left.")); + warning(""); + err = 1; + } else if (unlink(wtdir.buf) || + copy_file(wtdir.buf, rdir.buf, st.st_mode)) + warning_errno(_("could not copy '%s' to '%s'"), + rdir.buf, wtdir.buf); + } + } + + if (err) { + warning(_("temporary files exist in '%s'."), tmpdir); + warning(_("you may want to cleanup or recover these.")); + exit(1); + } else + exit_cleanup(tmpdir, rc); + +finish: + free(ce); + strbuf_release(&ldir); + strbuf_release(&rdir); + strbuf_release(&wtdir); + strbuf_release(&buf); + + return ret; +} + +static int run_file_diff(int prompt, const char *prefix, + int argc, const char **argv) +{ + struct argv_array args = ARGV_ARRAY_INIT; + const char *env[] = { + "GIT_PAGER=", "GIT_EXTERNAL_DIFF=git-difftool--helper", NULL, + NULL + }; + int ret = 0, i; + + if (prompt > 0) + env[2] = "GIT_DIFFTOOL_PROMPT=true"; + else if (!prompt) + env[2] = "GIT_DIFFTOOL_NO_PROMPT=true"; + + + argv_array_push(&args, "diff"); + for (i = 0; i < argc; i++) + argv_array_push(&args, argv[i]); + ret = run_command_v_opt_cd_env(args.argv, RUN_GIT_CMD, prefix, env); + exit(ret); +} /* * NEEDSWORK: this function can go once the legacy-difftool Perl script is @@ -41,6 +642,35 @@ static int use_builtin_difftool(void) { int cmd_difftool(int argc, const char **argv, const char *prefix) { + int use_gui_tool = 0, dir_diff = 0, prompt = -1, symlinks = 0, + tool_help = 0; + static char *difftool_cmd = NULL, *extcmd = NULL; + struct option builtin_difftool_options[] = { + OPT_BOOL('g', "gui", &use_gui_tool, + N_("use `diff.guitool` instead of `diff.tool`")), + OPT_BOOL('d', "dir-diff", &dir_diff, + N_("perform a full-directory diff")), + { OPTION_SET_INT, 'y', "no-prompt", &prompt, NULL, + N_("do not prompt before launching a diff tool"), + PARSE_OPT_NOARG | PARSE_OPT_NONEG, NULL, 0}, + { OPTION_SET_INT, 0, "prompt", &prompt, NULL, NULL, + PARSE_OPT_NOARG | PARSE_OPT_NONEG | PARSE_OPT_HIDDEN, + NULL, 1 }, + OPT_BOOL(0, "symlinks", &symlinks, + N_("use symlinks in dir-diff mode")), + OPT_STRING('t', "tool", &difftool_cmd, N_(""), + N_("use the specified diff tool")), + OPT_BOOL(0, "tool-help", &tool_help, + N_("print a list of diff tools that may be used with " + "`--tool`")), + OPT_BOOL(0, "trust-exit-code", &trust_exit_code, + N_("make 'git-difftool' exit when an invoked diff " + "tool returns a non - zero exit code")), + OPT_STRING('x', "extcmd", &extcmd, N_(""), + N_("specify a custom command for viewing diffs")), + OPT_END() + }; + /* * NEEDSWORK: Once the builtin difftool has been tested enough * and git-legacy-difftool.perl is retired to contrib/, this preamble @@ -58,6 +688,46 @@ int cmd_difftool(int argc, const char **argv, const char *prefix) prefix = setup_git_directory(); trace_repo_setup(prefix); setup_work_tree(); + /* NEEDSWORK: once we no longer spawn anything, remove this */ + setenv(GIT_DIR_ENVIRONMENT, absolute_path(get_git_dir()), 1); + setenv(GIT_WORK_TREE_ENVIRONMENT, absolute_path(get_git_work_tree()), 1); - die("TODO"); + git_config(difftool_config, NULL); + symlinks = has_symlinks; + + argc = parse_options(argc, argv, prefix, builtin_difftool_options, + builtin_difftool_usage, PARSE_OPT_KEEP_UNKNOWN | + PARSE_OPT_KEEP_DASHDASH); + + if (tool_help) + return print_tool_help(); + + if (use_gui_tool && diff_gui_tool && *diff_gui_tool) + setenv("GIT_DIFF_TOOL", diff_gui_tool, 1); + else if (difftool_cmd) { + if (*difftool_cmd) + setenv("GIT_DIFF_TOOL", difftool_cmd, 1); + else + die(_("no given for --tool=")); + } + + if (extcmd) { + if (*extcmd) + setenv("GIT_DIFFTOOL_EXTCMD", extcmd, 1); + else + die(_("no given for --extcmd=")); + } + + setenv("GIT_DIFFTOOL_TRUST_EXIT_CODE", + trust_exit_code ? "true" : "false", 1); + + /* + * In directory diff mode, 'git-difftool--helper' is called once + * to compare the a / b directories. In file diff mode, 'git diff' + * will invoke a separate instance of 'git-difftool--helper' for + * each file that changed. + */ + if (dir_diff) + return run_dir_diff(extcmd, symlinks, prefix, argc, argv); + return run_file_diff(prompt, prefix, argc, argv); } From 67a64eee48ca7ed7b48fa5667580847f70e71460 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 30 Nov 2016 17:14:45 +0100 Subject: [PATCH 228/288] t7800: run both builtin and scripted difftool, for now This is uglier than a simple touch "$GIT_EXEC_PATH/use-builtin-difftool" of course. But oh well. Signed-off-by: Johannes Schindelin --- t/t7800-difftool.sh | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/t/t7800-difftool.sh b/t/t7800-difftool.sh index e94910c563..273ab55723 100755 --- a/t/t7800-difftool.sh +++ b/t/t7800-difftool.sh @@ -23,6 +23,20 @@ prompt_given () test "$prompt" = "Launch 'test-tool' [Y/n]? branch" } +for use_builtin_difftool in false true +do + +test_expect_success 'verify we are running the correct difftool' ' + if test true = '$use_builtin_difftool' + then + test_must_fail ok=129 git difftool -h >help && + grep "g, --gui" help + else + git difftool -h >help && + grep "g|--gui" help + fi +' + # NEEDSWORK: lose all the PERL prereqs once legacy-difftool is retired. # Create a file on master and change it on branch @@ -606,4 +620,17 @@ test_expect_success PERL,SYMLINKS 'difftool --dir-diff symlinked directories' ' ) ' +test true != $use_builtin_difftool || break + +test_expect_success 'tear down for re-run' ' + rm -rf * .[a-z]* && + git init +' + +# run as builtin difftool now +GIT_CONFIG_PARAMETERS="'difftool.usebuiltin=true'" +export GIT_CONFIG_PARAMETERS + +done + test_done From 9c8e73622348e3265d189686b65d85f82c392338 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 19 Dec 2016 14:50:30 +0100 Subject: [PATCH 229/288] sequencer: avoid unnecessary curly braces This was noticed while addressing Junio Hamano's concern that some "else" operators were on separate lines than the preceding closing brace. Signed-off-by: Johannes Schindelin --- sequencer.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sequencer.c b/sequencer.c index 0b78f3149f..b2a65547d8 100644 --- a/sequencer.c +++ b/sequencer.c @@ -679,9 +679,8 @@ static int do_pick_commit(enum todo_command command, struct commit *commit, } discard_cache(); - if (!commit->parents) { + if (!commit->parents) parent = NULL; - } else if (commit->parents->next) { /* Reverting or cherry-picking a merge commit */ int cnt; From 8abcaf13988f23950ac9b1608f9aa59c944a3393 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 19 Dec 2016 14:52:04 +0100 Subject: [PATCH 230/288] sequencer: move "else" keyword onto the same line as preceding brace It is the current coding style of the Git project to write if (...) { ... } else { ... } instead of putting the closing brace and the "else" keyword on separate lines. Pointed out by Junio Hamano. Signed-off-by: Johannes Schindelin --- sequencer.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sequencer.c b/sequencer.c index b2a65547d8..654122a445 100644 --- a/sequencer.c +++ b/sequencer.c @@ -1117,8 +1117,7 @@ static int create_seq_dir(void) error(_("a cherry-pick or revert is already in progress")); advise(_("try \"git cherry-pick (--continue | --quit | --abort)\"")); return -1; - } - else if (mkdir(git_path_seq_dir(), 0777) < 0) + } else if (mkdir(git_path_seq_dir(), 0777) < 0) return error_errno(_("could not create sequencer directory '%s'"), git_path_seq_dir()); return 0; From 2e944e09e6c955b63b006e25cc4eb1674ef898c0 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 19 Dec 2016 17:28:27 +0100 Subject: [PATCH 231/288] sequencer: use a helper to find the commit message It is actually not safe to look for a commit message by looking for the first empty line and skipping it. The find_commit_subject() function looks more carefully, so let's use it. Since we are interested in the entire commit message, we re-compute the string length after verifying that the commit subject is not empty (in which case the entire commit message would be empty, something that should not happen but that we want to handle gracefully). Signed-off-by: Johannes Schindelin --- sequencer.c | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/sequencer.c b/sequencer.c index 654122a445..a5463e2c69 100644 --- a/sequencer.c +++ b/sequencer.c @@ -750,14 +750,9 @@ static int do_pick_commit(enum todo_command command, struct commit *commit, next = commit; next_label = msg.label; - /* - * Append the commit log message to msgbuf; it starts - * after the tree, parent, author, committer - * information followed by "\n\n". - */ - p = strstr(msg.message, "\n\n"); - if (p) - strbuf_addstr(&msgbuf, skip_blank_lines(p + 2)); + /* Append the commit log message to msgbuf. */ + if (find_commit_subject(msg.message, &p)) + strbuf_addstr(&msgbuf, p); if (opts->record_origin) { if (!has_conforming_footer(&msgbuf, NULL, 0)) From 796751dce3fe266c460d29504741876d511e3c1a Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 9 Feb 2016 14:10:43 +0100 Subject: [PATCH 232/288] sequencer: support a new action: 'interactive rebase' This patch introduces a new action for the sequencer. It really does not do a whole lot of its own right now, but lays the ground work for patches to come. The intention, of course, is to finally make the sequencer the work horse of the interactive rebase (the original idea behind the "sequencer" concept). Signed-off-by: Johannes Schindelin --- sequencer.c | 36 ++++++++++++++++++++++++++++++++---- sequencer.h | 3 ++- 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/sequencer.c b/sequencer.c index a5463e2c69..ce84c2905f 100644 --- a/sequencer.c +++ b/sequencer.c @@ -29,6 +29,14 @@ static GIT_PATH_FUNC(git_path_opts_file, "sequencer/opts") static GIT_PATH_FUNC(git_path_head_file, "sequencer/head") static GIT_PATH_FUNC(git_path_abort_safety_file, "sequencer/abort-safety") +static GIT_PATH_FUNC(rebase_path, "rebase-merge") +/* + * The file containing rebase commands, comments, and empty lines. + * This file is created by "git rebase -i" then edited by the user. As + * the lines are processed, they are removed from the front of this + * file and written to the tail of 'done'. + */ +static GIT_PATH_FUNC(rebase_path_todo, "rebase-merge/git-rebase-todo") /* * A script to set the GIT_AUTHOR_NAME, GIT_AUTHOR_EMAIL, and * GIT_AUTHOR_DATE that will be used for the commit that is currently @@ -41,19 +49,22 @@ static GIT_PATH_FUNC(rebase_path_author_script, "rebase-merge/author-script") */ static GIT_PATH_FUNC(rebase_path_gpg_sign_opt, "rebase-merge/gpg_sign_opt") -/* We will introduce the 'interactive rebase' mode later */ static inline int is_rebase_i(const struct replay_opts *opts) { - return 0; + return opts->action == REPLAY_INTERACTIVE_REBASE; } static const char *get_dir(const struct replay_opts *opts) { + if (is_rebase_i(opts)) + return rebase_path(); return git_path_seq_dir(); } static const char *get_todo_path(const struct replay_opts *opts) { + if (is_rebase_i(opts)) + return rebase_path_todo(); return git_path_todo_file(); } @@ -169,7 +180,15 @@ int sequencer_remove_state(struct replay_opts *opts) static const char *action_name(const struct replay_opts *opts) { - return opts->action == REPLAY_REVERT ? N_("revert") : N_("cherry-pick"); + switch (opts->action) { + case REPLAY_REVERT: + return N_("revert"); + case REPLAY_PICK: + return N_("cherry-pick"); + case REPLAY_INTERACTIVE_REBASE: + return N_("rebase -i"); + } + die(_("Unknown action: %d"), opts->action); } struct commit_message { @@ -411,7 +430,9 @@ static int do_recursive_merge(struct commit *base, struct commit *next, if (active_cache_changed && write_locked_index(&the_index, &index_lock, COMMIT_LOCK)) - /* TRANSLATORS: %s will be "revert" or "cherry-pick" */ + /* TRANSLATORS: %s will be "revert", "cherry-pick" or + * "rebase -i". + */ return error(_("%s: Unable to write new index file"), _(action_name(opts))); rollback_lock_file(&index_lock); @@ -1245,6 +1266,13 @@ static int save_todo(struct todo_list *todo_list, struct replay_opts *opts) const char *todo_path = get_todo_path(opts); int next = todo_list->current, offset, fd; + /* + * rebase -i writes "git-rebase-todo" without the currently executing + * command, appending it to "done" instead. + */ + if (is_rebase_i(opts)) + next++; + fd = hold_lock_file_for_update(&todo_lock, todo_path, 0); if (fd < 0) return error_errno(_("could not lock '%s'"), todo_path); diff --git a/sequencer.h b/sequencer.h index 7a513c576b..cb21cfddee 100644 --- a/sequencer.h +++ b/sequencer.h @@ -7,7 +7,8 @@ const char *git_path_seq_dir(void); enum replay_action { REPLAY_REVERT, - REPLAY_PICK + REPLAY_PICK, + REPLAY_INTERACTIVE_REBASE }; struct replay_opts { From 7bcd39f5339ebc09d4cbc45ac4aa6a5d4fc0b374 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 13 Apr 2016 14:07:35 +0200 Subject: [PATCH 233/288] sequencer (rebase -i): implement the 'noop' command The 'noop' command is probably the most boring of all rebase -i commands to support in the sequencer. Which makes it an excellent candidate for this first stab to add support for rebase -i's commands to the sequencer. For the moment, let's also treat empty lines and commented-out lines as 'noop'; We will refine that handling later in this patch series. To make it easier to identify "classes" of todo_commands (such as: determine whether a command is pick-like, i.e. handles a single commit), let's enforce a certain order of said commands. Signed-off-by: Johannes Schindelin --- sequencer.c | 39 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/sequencer.c b/sequencer.c index ce84c2905f..7a5c680ac6 100644 --- a/sequencer.c +++ b/sequencer.c @@ -654,14 +654,23 @@ static int allow_empty(struct replay_opts *opts, struct commit *commit) return 1; } +/* + * Note that ordering matters in this enum. Not only must it match the mapping + * below, it is also divided into several sections that matter. When adding + * new commands, make sure you add it in the right section. + */ enum todo_command { + /* commands that handle commits */ TODO_PICK = 0, - TODO_REVERT + TODO_REVERT, + /* commands that do nothing but are counted for reporting progress */ + TODO_NOOP }; static const char *todo_command_strings[] = { "pick", - "revert" + "revert", + "noop" }; static const char *command_to_string(const enum todo_command command) @@ -671,6 +680,10 @@ static const char *command_to_string(const enum todo_command command) die("Unknown command: %d", command); } +static int is_noop(const enum todo_command command) +{ + return TODO_NOOP <= (size_t)command; +} static int do_pick_commit(enum todo_command command, struct commit *commit, struct replay_opts *opts) @@ -926,6 +939,14 @@ static int parse_insn_line(struct todo_item *item, const char *bol, char *eol) /* left-trim */ bol += strspn(bol, " \t"); + if (bol == eol || *bol == '\r' || *bol == comment_line_char) { + item->command = TODO_NOOP; + item->commit = NULL; + item->arg = bol; + item->arg_len = eol - bol; + return 0; + } + for (i = 0; i < ARRAY_SIZE(todo_command_strings); i++) if (skip_prefix(bol, todo_command_strings[i], &bol)) { item->command = i; @@ -934,6 +955,13 @@ static int parse_insn_line(struct todo_item *item, const char *bol, char *eol) if (i >= ARRAY_SIZE(todo_command_strings)) return -1; + if (item->command == TODO_NOOP) { + item->commit = NULL; + item->arg = bol; + item->arg_len = eol - bol; + return 0; + } + /* Eat up extra spaces/ tabs before object name */ padding = strspn(bol, " \t"); if (!padding) @@ -1336,7 +1364,12 @@ static int pick_commits(struct todo_list *todo_list, struct replay_opts *opts) struct todo_item *item = todo_list->items + todo_list->current; if (save_todo(todo_list, opts)) return -1; - res = do_pick_commit(item->command, item->commit, opts); + if (item->command <= TODO_REVERT) + res = do_pick_commit(item->command, item->commit, + opts); + else if (!is_noop(item->command)) + return error(_("unknown command %d"), item->command); + todo_list->current++; if (res) return res; From b5dcf93d84df8f98fbe1d793b466079dd190086a Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 13 Apr 2016 14:07:47 +0200 Subject: [PATCH 234/288] sequencer (rebase -i): implement the 'edit' command This patch is a straight-forward reimplementation of the `edit` operation of the interactive rebase command. Well, not *quite* straight-forward: when stopping, the `edit` command wants to write the `patch` file (which is not only the patch, but includes the commit message and author information). To that end, this patch requires the earlier work that taught the log-tree machinery to respect the `file` setting of rev_info->diffopt to write to a file stream different than stdout. Signed-off-by: Johannes Schindelin --- sequencer.c | 117 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 114 insertions(+), 3 deletions(-) diff --git a/sequencer.c b/sequencer.c index 7a5c680ac6..9cf7d5eb34 100644 --- a/sequencer.c +++ b/sequencer.c @@ -16,6 +16,7 @@ #include "refs.h" #include "argv-array.h" #include "quote.h" +#include "log-tree.h" #define GIT_REFLOG_ACTION "GIT_REFLOG_ACTION" @@ -43,6 +44,20 @@ static GIT_PATH_FUNC(rebase_path_todo, "rebase-merge/git-rebase-todo") * being rebased. */ static GIT_PATH_FUNC(rebase_path_author_script, "rebase-merge/author-script") +/* + * When an "edit" rebase command is being processed, the SHA1 of the + * commit to be edited is recorded in this file. When "git rebase + * --continue" is executed, if there are any staged changes then they + * will be amended to the HEAD commit, but only provided the HEAD + * commit is still the commit to be edited. When any other rebase + * command is processed, this file is deleted. + */ +static GIT_PATH_FUNC(rebase_path_amend, "rebase-merge/amend") +/* + * When we stop at a given patch via the "edit" command, this file contains + * the abbreviated commit name of the corresponding patch. + */ +static GIT_PATH_FUNC(rebase_path_stopped_sha, "rebase-merge/stopped-sha") /* * The following files are written by git-rebase just after parsing the * command-line (and are only consumed, not modified, by the sequencer). @@ -663,6 +678,7 @@ enum todo_command { /* commands that handle commits */ TODO_PICK = 0, TODO_REVERT, + TODO_EDIT, /* commands that do nothing but are counted for reporting progress */ TODO_NOOP }; @@ -670,6 +686,7 @@ enum todo_command { static const char *todo_command_strings[] = { "pick", "revert", + "edit", "noop" }; @@ -1349,9 +1366,87 @@ static int save_opts(struct replay_opts *opts) return res; } +static int make_patch(struct commit *commit, struct replay_opts *opts) +{ + struct strbuf buf = STRBUF_INIT; + struct rev_info log_tree_opt; + const char *subject, *p; + int res = 0; + + p = short_commit_name(commit); + if (write_message(p, strlen(p), rebase_path_stopped_sha(), 1) < 0) + return -1; + + strbuf_addf(&buf, "%s/patch", get_dir(opts)); + memset(&log_tree_opt, 0, sizeof(log_tree_opt)); + init_revisions(&log_tree_opt, NULL); + log_tree_opt.abbrev = 0; + log_tree_opt.diff = 1; + log_tree_opt.diffopt.output_format = DIFF_FORMAT_PATCH; + log_tree_opt.disable_stdin = 1; + log_tree_opt.no_commit_id = 1; + log_tree_opt.diffopt.file = fopen(buf.buf, "w"); + log_tree_opt.diffopt.use_color = GIT_COLOR_NEVER; + if (!log_tree_opt.diffopt.file) + res |= error_errno(_("could not open '%s'"), buf.buf); + else { + res |= log_tree_commit(&log_tree_opt, commit); + fclose(log_tree_opt.diffopt.file); + } + strbuf_reset(&buf); + + strbuf_addf(&buf, "%s/message", get_dir(opts)); + if (!file_exists(buf.buf)) { + const char *commit_buffer = get_commit_buffer(commit, NULL); + find_commit_subject(commit_buffer, &subject); + res |= write_message(subject, strlen(subject), buf.buf, 1); + unuse_commit_buffer(commit, commit_buffer); + } + strbuf_release(&buf); + + return res; +} + +static int intend_to_amend(void) +{ + unsigned char head[20]; + char *p; + + if (get_sha1("HEAD", head)) + return error(_("cannot read HEAD")); + + p = sha1_to_hex(head); + return write_message(p, strlen(p), rebase_path_amend(), 1); +} + +static int error_with_patch(struct commit *commit, + const char *subject, int subject_len, + struct replay_opts *opts, int exit_code, int to_amend) +{ + if (make_patch(commit, opts)) + return -1; + + if (to_amend) { + if (intend_to_amend()) + return -1; + + fprintf(stderr, "You can amend the commit now, with\n" + "\n" + " git commit --amend %s\n" + "\n" + "Once you are satisfied with your changes, run\n" + "\n" + " git rebase --continue\n", gpg_sign_opt_quoted(opts)); + } else if (exit_code) + fprintf(stderr, "Could not apply %s... %.*s\n", + short_commit_name(commit), subject_len, subject); + + return exit_code; +} + static int pick_commits(struct todo_list *todo_list, struct replay_opts *opts) { - int res; + int res = 0; setenv(GIT_REFLOG_ACTION, action_name(opts), 0); if (opts->allow_ff) @@ -1364,10 +1459,20 @@ static int pick_commits(struct todo_list *todo_list, struct replay_opts *opts) struct todo_item *item = todo_list->items + todo_list->current; if (save_todo(todo_list, opts)) return -1; - if (item->command <= TODO_REVERT) + if (item->command <= TODO_EDIT) { res = do_pick_commit(item->command, item->commit, opts); - else if (!is_noop(item->command)) + if (item->command == TODO_EDIT) { + struct commit *commit = item->commit; + if (!res) + warning(_("stopped at %s... %.*s"), + short_commit_name(commit), + item->arg_len, item->arg); + return error_with_patch(commit, + item->arg, item->arg_len, opts, res, + !res); + } + } else if (!is_noop(item->command)) return error(_("unknown command %d"), item->command); todo_list->current++; @@ -1375,6 +1480,12 @@ static int pick_commits(struct todo_list *todo_list, struct replay_opts *opts) return res; } + if (is_rebase_i(opts)) { + /* Stopped in the middle, as planned? */ + if (todo_list->current < todo_list->nr) + return 0; + } + /* * Sequence of picks finished successfully; cleanup by * removing the .git/sequencer directory From d01d00e7fdd6db988b57e5f6985bf2b7352bc79e Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 13 Apr 2016 14:08:01 +0200 Subject: [PATCH 235/288] sequencer (rebase -i): implement the 'exec' command The 'exec' command is a little special among rebase -i's commands, as it does *not* have a SHA-1 as first parameter. Instead, everything after the `exec` command is treated as command-line to execute. Let's reuse the arg/arg_len fields of the todo_item structure (which hold the oneline for pick/edit commands) to point to the command-line. Signed-off-by: Johannes Schindelin --- sequencer.c | 57 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/sequencer.c b/sequencer.c index 9cf7d5eb34..8a4418d655 100644 --- a/sequencer.c +++ b/sequencer.c @@ -17,6 +17,7 @@ #include "argv-array.h" #include "quote.h" #include "log-tree.h" +#include "wt-status.h" #define GIT_REFLOG_ACTION "GIT_REFLOG_ACTION" @@ -679,6 +680,8 @@ enum todo_command { TODO_PICK = 0, TODO_REVERT, TODO_EDIT, + /* commands that do something else than handling a single commit */ + TODO_EXEC, /* commands that do nothing but are counted for reporting progress */ TODO_NOOP }; @@ -687,6 +690,7 @@ static const char *todo_command_strings[] = { "pick", "revert", "edit", + "exec", "noop" }; @@ -985,6 +989,12 @@ static int parse_insn_line(struct todo_item *item, const char *bol, char *eol) return -1; bol += padding; + if (item->command == TODO_EXEC) { + item->arg = bol; + item->arg_len = (int)(eol - bol); + return 0; + } + end_of_object_name = (char *) bol + strcspn(bol, " \t\n"); saved = *end_of_object_name; *end_of_object_name = '\0'; @@ -1444,6 +1454,46 @@ static int error_with_patch(struct commit *commit, return exit_code; } +static int do_exec(const char *command_line) +{ + const char *child_argv[] = { NULL, NULL }; + int dirty, status; + + fprintf(stderr, "Executing: %s\n", command_line); + child_argv[0] = command_line; + status = run_command_v_opt(child_argv, RUN_USING_SHELL); + + /* force re-reading of the cache */ + if (discard_cache() < 0 || read_cache() < 0) + return error(_("could not read index")); + + dirty = require_clean_work_tree("rebase", NULL, 1, 1); + + if (status) { + warning(_("execution failed: %s\n%s" + "You can fix the problem, and then run\n" + "\n" + " git rebase --continue\n" + "\n"), + command_line, + dirty ? N_("and made changes to the index and/or the " + "working tree\n") : ""); + if (status == 127) + /* command not found */ + status = 1; + } else if (dirty) { + warning(_("execution succeeded: %s\nbut " + "left changes to the index and/or the working tree\n" + "Commit or stash your changes, and then run\n" + "\n" + " git rebase --continue\n" + "\n"), command_line); + status = 1; + } + + return status; +} + static int pick_commits(struct todo_list *todo_list, struct replay_opts *opts) { int res = 0; @@ -1472,6 +1522,13 @@ static int pick_commits(struct todo_list *todo_list, struct replay_opts *opts) item->arg, item->arg_len, opts, res, !res); } + } else if (item->command == TODO_EXEC) { + char *end_of_arg = (char *)(item->arg + item->arg_len); + int saved = *end_of_arg; + + *end_of_arg = '\0'; + res = do_exec(item->arg); + *end_of_arg = saved; } else if (!is_noop(item->command)) return error(_("unknown command %d"), item->command); From abc6924c3257feb8192ff4fed921525cb148ee34 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 12 Apr 2016 15:30:26 +0200 Subject: [PATCH 236/288] sequencer (rebase -i): learn about the 'verbose' mode When calling `git rebase -i -v`, the user wants to see some statistics after the commits were rebased. Let's show some. The strbuf we use to perform that task will be used for other things in subsequent commits, hence it is declared and initialized in a wider scope than strictly needed here. Signed-off-by: Johannes Schindelin --- sequencer.c | 28 ++++++++++++++++++++++++++++ sequencer.h | 1 + 2 files changed, 29 insertions(+) diff --git a/sequencer.c b/sequencer.c index 8a4418d655..e6895a76bb 100644 --- a/sequencer.c +++ b/sequencer.c @@ -64,6 +64,8 @@ static GIT_PATH_FUNC(rebase_path_stopped_sha, "rebase-merge/stopped-sha") * command-line (and are only consumed, not modified, by the sequencer). */ static GIT_PATH_FUNC(rebase_path_gpg_sign_opt, "rebase-merge/gpg_sign_opt") +static GIT_PATH_FUNC(rebase_path_orig_head, "rebase-merge/orig-head") +static GIT_PATH_FUNC(rebase_path_verbose, "rebase-merge/verbose") static inline int is_rebase_i(const struct replay_opts *opts) { @@ -1135,6 +1137,9 @@ static int read_populate_opts(struct replay_opts *opts) } strbuf_release(&buf); + if (file_exists(rebase_path_verbose())) + opts->verbose = 1; + return 0; } @@ -1538,9 +1543,32 @@ static int pick_commits(struct todo_list *todo_list, struct replay_opts *opts) } if (is_rebase_i(opts)) { + struct strbuf buf = STRBUF_INIT; + /* Stopped in the middle, as planned? */ if (todo_list->current < todo_list->nr) return 0; + + if (opts->verbose) { + struct rev_info log_tree_opt; + struct object_id orig, head; + + memset(&log_tree_opt, 0, sizeof(log_tree_opt)); + init_revisions(&log_tree_opt, NULL); + log_tree_opt.diff = 1; + log_tree_opt.diffopt.output_format = + DIFF_FORMAT_DIFFSTAT; + log_tree_opt.disable_stdin = 1; + + if (read_oneliner(&buf, rebase_path_orig_head(), 0) && + !get_sha1(buf.buf, orig.hash) && + !get_sha1("HEAD", head.hash)) { + diff_tree_sha1(orig.hash, head.hash, + "", &log_tree_opt.diffopt); + log_tree_diff_flush(&log_tree_opt); + } + } + strbuf_release(&buf); } /* diff --git a/sequencer.h b/sequencer.h index cb21cfddee..f885b68395 100644 --- a/sequencer.h +++ b/sequencer.h @@ -24,6 +24,7 @@ struct replay_opts { int allow_empty; int allow_empty_message; int keep_redundant_commits; + int verbose; int mainline; From a8f17766cb53df7a054aea16dd2f0c777a8ff210 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 4 Mar 2016 16:18:50 +0100 Subject: [PATCH 237/288] sequencer (rebase -i): write the 'done' file In the interactive rebase, commands that were successfully processed are not simply discarded, but appended to the 'done' file instead. This is used e.g. to display the current state to the user in the output of `git status` or the progress. Signed-off-by: Johannes Schindelin --- sequencer.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/sequencer.c b/sequencer.c index e6895a76bb..542dbfaa38 100644 --- a/sequencer.c +++ b/sequencer.c @@ -39,6 +39,12 @@ static GIT_PATH_FUNC(rebase_path, "rebase-merge") * file and written to the tail of 'done'. */ static GIT_PATH_FUNC(rebase_path_todo, "rebase-merge/git-rebase-todo") +/* + * The rebase command lines that have already been processed. A line + * is moved here when it is first handled, before any associated user + * actions. + */ +static GIT_PATH_FUNC(rebase_path_done, "rebase-merge/done") /* * A script to set the GIT_AUTHOR_NAME, GIT_AUTHOR_EMAIL, and * GIT_AUTHOR_DATE that will be used for the commit that is currently @@ -1343,6 +1349,23 @@ static int save_todo(struct todo_list *todo_list, struct replay_opts *opts) return error_errno(_("could not write to '%s'"), todo_path); if (commit_lock_file(&todo_lock) < 0) return error(_("failed to finalize '%s'."), todo_path); + + if (is_rebase_i(opts)) { + const char *done_path = rebase_path_done(); + int fd = open(done_path, O_CREAT | O_WRONLY | O_APPEND, 0666); + int prev_offset = !next ? 0 : + todo_list->items[next - 1].offset_in_buf; + + if (fd >= 0 && offset > prev_offset && + write_in_full(fd, todo_list->buf.buf + prev_offset, + offset - prev_offset) < 0) { + close(fd); + return error_errno(_("could not write to '%s'"), + done_path); + } + if (fd >= 0) + close(fd); + } return 0; } From 4786926366901dc721e531e9e06597c5300f3cae Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 23 May 2016 16:49:51 +0200 Subject: [PATCH 238/288] sequencer (rebase -i): add support for the 'fixup' and 'squash' commands This is a huge patch, and at the same time a huge step forward to execute the performance-critical parts of the interactive rebase in a builtin command. Since 'fixup' and 'squash' are not only similar, but also need to know about each other (we want to reduce a series of fixups/squashes into a single, final commit message edit, from the user's point of view), we really have to implement them both at the same time. Most of the actual work is done by the existing code path that already handles the "pick" and the "edit" commands; We added support for other features (e.g. to amend the commit message) in the patches leading up to this one, yet there are still quite a few bits in this patch that simply would not make sense as individual patches (such as: determining whether there was anything to "fix up" in the "todo" script, etc). In theory, it would be possible to reuse the fast-forward code path also for the fixup and the squash code paths, but in practice this would make the code less readable. The end result cannot be fast-forwarded anyway, therefore let's just extend the cherry-picking code path for now. Since the sequencer parses the entire `git-rebase-todo` script in one go, fixup or squash commands without a preceding pick can be reported early (in git-rebase--interactive, we could only report such errors just before executing the fixup/squash). Signed-off-by: Johannes Schindelin --- sequencer.c | 229 +++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 218 insertions(+), 11 deletions(-) diff --git a/sequencer.c b/sequencer.c index 542dbfaa38..de4065b614 100644 --- a/sequencer.c +++ b/sequencer.c @@ -45,6 +45,35 @@ static GIT_PATH_FUNC(rebase_path_todo, "rebase-merge/git-rebase-todo") * actions. */ static GIT_PATH_FUNC(rebase_path_done, "rebase-merge/done") +/* + * The commit message that is planned to be used for any changes that + * need to be committed following a user interaction. + */ +static GIT_PATH_FUNC(rebase_path_message, "rebase-merge/message") +/* + * The file into which is accumulated the suggested commit message for + * squash/fixup commands. When the first of a series of squash/fixups + * is seen, the file is created and the commit message from the + * previous commit and from the first squash/fixup commit are written + * to it. The commit message for each subsequent squash/fixup commit + * is appended to the file as it is processed. + * + * The first line of the file is of the form + * # This is a combination of $count commits. + * where $count is the number of commits whose messages have been + * written to the file so far (including the initial "pick" commit). + * Each time that a commit message is processed, this line is read and + * updated. It is deleted just before the combined commit is made. + */ +static GIT_PATH_FUNC(rebase_path_squash_msg, "rebase-merge/message-squash") +/* + * If the current series of squash/fixups has not yet included a squash + * command, then this file exists and holds the commit message of the + * original "pick" commit. (If the series ends without a "squash" + * command, then this can be used as the commit message of the combined + * commit without opening the editor.) + */ +static GIT_PATH_FUNC(rebase_path_fixup_msg, "rebase-merge/message-fixup") /* * A script to set the GIT_AUTHOR_NAME, GIT_AUTHOR_EMAIL, and * GIT_AUTHOR_DATE that will be used for the commit that is currently @@ -688,6 +717,8 @@ enum todo_command { TODO_PICK = 0, TODO_REVERT, TODO_EDIT, + TODO_FIXUP, + TODO_SQUASH, /* commands that do something else than handling a single commit */ TODO_EXEC, /* commands that do nothing but are counted for reporting progress */ @@ -698,6 +729,8 @@ static const char *todo_command_strings[] = { "pick", "revert", "edit", + "fixup", + "squash", "exec", "noop" }; @@ -714,15 +747,114 @@ static int is_noop(const enum todo_command command) return TODO_NOOP <= (size_t)command; } -static int do_pick_commit(enum todo_command command, struct commit *commit, - struct replay_opts *opts) +static int is_fixup(enum todo_command command) { + return command == TODO_FIXUP || command == TODO_SQUASH; +} + +static int update_squash_messages(enum todo_command command, + struct commit *commit, struct replay_opts *opts) +{ + struct strbuf buf = STRBUF_INIT; + int count, res; + const char *message, *body; + + if (file_exists(rebase_path_squash_msg())) { + struct strbuf header = STRBUF_INIT; + char *eol, *p; + + if (strbuf_read_file(&buf, rebase_path_squash_msg(), 2048) <= 0) + return error(_("could not read '%s'"), + rebase_path_squash_msg()); + + p = buf.buf + 1; + eol = strchrnul(buf.buf, '\n'); + if (buf.buf[0] != comment_line_char || + (p += strcspn(p, "0123456789\n")) == eol) + return error(_("unexpected 1st line of squash message:" + "\n\n\t%.*s"), + (int)(eol - buf.buf), buf.buf); + count = strtol(p, NULL, 10); + + if (count < 1) + return error(_("invalid 1st line of squash message:\n" + "\n\t%.*s"), + (int)(eol - buf.buf), buf.buf); + + strbuf_addf(&header, "%c ", comment_line_char); + strbuf_addf(&header, + _("This is a combination of %d commits."), ++count); + strbuf_splice(&buf, 0, eol - buf.buf, header.buf, header.len); + strbuf_release(&header); + } else { + unsigned char head[20]; + struct commit *head_commit; + const char *head_message, *body; + + if (get_sha1("HEAD", head)) + return error(_("need a HEAD to fixup")); + if (!(head_commit = lookup_commit_reference(head))) + return error(_("could not read HEAD")); + if (!(head_message = get_commit_buffer(head_commit, NULL))) + return error(_("could not read HEAD's commit message")); + + find_commit_subject(head_message, &body); + if (write_message(body, strlen(body), + rebase_path_fixup_msg(), 0)) { + unuse_commit_buffer(head_commit, head_message); + return error(_("cannot write '%s'"), + rebase_path_fixup_msg()); + } + + count = 2; + strbuf_addf(&buf, "%c ", comment_line_char); + strbuf_addf(&buf, _("This is a combination of %d commits."), + count); + strbuf_addf(&buf, "\n%c ", comment_line_char); + strbuf_addstr(&buf, _("This is the 1st commit message:")); + strbuf_addstr(&buf, "\n\n"); + strbuf_addstr(&buf, body); + + unuse_commit_buffer(head_commit, head_message); + } + + if (!(message = get_commit_buffer(commit, NULL))) + return error(_("could not read commit message of %s"), + oid_to_hex(&commit->object.oid)); + find_commit_subject(message, &body); + + if (command == TODO_SQUASH) { + unlink(rebase_path_fixup_msg()); + strbuf_addf(&buf, "\n%c ", comment_line_char); + strbuf_addf(&buf, _("This is the commit message #%d:"), count); + strbuf_addstr(&buf, "\n\n"); + strbuf_addstr(&buf, body); + } else if (command == TODO_FIXUP) { + strbuf_addf(&buf, "\n%c ", comment_line_char); + strbuf_addf(&buf, _("The commit message #%d will be skipped:"), + count); + strbuf_addstr(&buf, "\n\n"); + strbuf_add_commented_lines(&buf, body, strlen(body)); + } else + return error(_("unknown command: %d"), command); + unuse_commit_buffer(commit, message); + + res = write_message(buf.buf, buf.len, rebase_path_squash_msg(), 0); + strbuf_release(&buf); + return res; +} + +static int do_pick_commit(enum todo_command command, struct commit *commit, + struct replay_opts *opts, int final_fixup) +{ + int edit = opts->edit, cleanup_commit_message = 0; + const char *msg_file = edit ? NULL : git_path_merge_msg(); unsigned char head[20]; struct commit *base, *next, *parent; const char *base_label, *next_label; struct commit_message msg = { NULL, NULL, NULL, NULL }; struct strbuf msgbuf = STRBUF_INIT; - int res, unborn = 0, allow; + int res, unborn = 0, amend = 0, allow; if (opts->no_commit) { /* @@ -767,7 +899,7 @@ static int do_pick_commit(enum todo_command command, struct commit *commit, else parent = commit->parents->item; - if (opts->allow_ff && + if (opts->allow_ff && !is_fixup(command) && ((parent && !hashcmp(parent->object.oid.hash, head)) || (!parent && unborn))) return fast_forward_to(commit->object.oid.hash, head, unborn, opts); @@ -826,6 +958,27 @@ static int do_pick_commit(enum todo_command command, struct commit *commit, } } + if (is_fixup(command)) { + if (update_squash_messages(command, commit, opts)) + return -1; + amend = 1; + if (!final_fixup) + msg_file = rebase_path_squash_msg(); + else if (file_exists(rebase_path_fixup_msg())) { + cleanup_commit_message = 1; + msg_file = rebase_path_fixup_msg(); + } else { + const char *dest = git_path("SQUASH_MSG"); + unlink(dest); + if (copy_file(dest, rebase_path_squash_msg(), 0666)) + return error(_("could not rename '%s' to '%s'"), + rebase_path_squash_msg(), dest); + unlink(git_path("MERGE_MSG")); + msg_file = dest; + edit = 1; + } + } + if (!opts->strategy || !strcmp(opts->strategy, "recursive") || command == TODO_REVERT) { res = do_recursive_merge(base, next, base_label, next_label, head, &msgbuf, opts); @@ -881,8 +1034,13 @@ static int do_pick_commit(enum todo_command command, struct commit *commit, goto leave; } if (!opts->no_commit) - res = run_git_commit(opts->edit ? NULL : git_path_merge_msg(), - opts, allow, opts->edit, 0, 0); + res = run_git_commit(msg_file, opts, allow, edit, amend, + cleanup_commit_message); + + if (!res && final_fixup) { + unlink(rebase_path_fixup_msg()); + unlink(rebase_path_squash_msg()); + } leave: free_message(commit, &msg); @@ -1023,7 +1181,7 @@ static int parse_insn_buffer(char *buf, struct todo_list *todo_list) { struct todo_item *item; char *p = buf, *next_p; - int i, res = 0; + int i, res = 0, fixup_okay = file_exists(rebase_path_done()); for (i = 1; *p; i++, p = next_p) { char *eol = strchrnul(p, '\n'); @@ -1038,8 +1196,16 @@ static int parse_insn_buffer(char *buf, struct todo_list *todo_list) if (parse_insn_line(item, p, eol)) { res = error(_("invalid line %d: %.*s"), i, (int)(eol - p), p); - item->command = -1; + item->command = TODO_NOOP; } + + if (fixup_okay) + ; /* do nothing */ + else if (is_fixup(item->command)) + return error(_("cannot '%s' without a previous commit"), + command_to_string(item->command)); + else if (!is_noop(item->command)) + fixup_okay = 1; } if (!todo_list->nr) return error(_("no commits parsed.")); @@ -1482,6 +1648,20 @@ static int error_with_patch(struct commit *commit, return exit_code; } +static int error_failed_squash(struct commit *commit, + struct replay_opts *opts, int subject_len, const char *subject) +{ + if (rename(rebase_path_squash_msg(), rebase_path_message())) + return error(_("could not rename '%s' to '%s'"), + rebase_path_squash_msg(), rebase_path_message()); + unlink(rebase_path_fixup_msg()); + unlink(git_path("MERGE_MSG")); + if (copy_file(git_path("MERGE_MSG"), rebase_path_message(), 0666)) + return error(_("could not copy '%s' to '%s'"), + rebase_path_message(), git_path("MERGE_MSG")); + return error_with_patch(commit, subject, subject_len, opts, 1, 0); +} + static int do_exec(const char *command_line) { const char *child_argv[] = { NULL, NULL }; @@ -1522,6 +1702,21 @@ static int do_exec(const char *command_line) return status; } +static int is_final_fixup(struct todo_list *todo_list) +{ + int i = todo_list->current; + + if (!is_fixup(todo_list->items[i].command)) + return 0; + + while (++i < todo_list->nr) + if (is_fixup(todo_list->items[i].command)) + return 0; + else if (!is_noop(todo_list->items[i].command)) + break; + return 1; +} + static int pick_commits(struct todo_list *todo_list, struct replay_opts *opts) { int res = 0; @@ -1537,9 +1732,15 @@ static int pick_commits(struct todo_list *todo_list, struct replay_opts *opts) struct todo_item *item = todo_list->items + todo_list->current; if (save_todo(todo_list, opts)) return -1; - if (item->command <= TODO_EDIT) { + if (is_rebase_i(opts)) { + unlink(rebase_path_message()); + unlink(rebase_path_author_script()); + unlink(rebase_path_stopped_sha()); + unlink(rebase_path_amend()); + } + if (item->command <= TODO_SQUASH) { res = do_pick_commit(item->command, item->commit, - opts); + opts, is_final_fixup(todo_list)); if (item->command == TODO_EDIT) { struct commit *commit = item->commit; if (!res) @@ -1550,6 +1751,12 @@ static int pick_commits(struct todo_list *todo_list, struct replay_opts *opts) item->arg, item->arg_len, opts, res, !res); } + if (res && is_fixup(item->command)) { + if (res == 1) + intend_to_amend(); + return error_failed_squash(item->commit, opts, + item->arg_len, item->arg); + } } else if (item->command == TODO_EXEC) { char *end_of_arg = (char *)(item->arg + item->arg_len); int saved = *end_of_arg; @@ -1648,7 +1855,7 @@ static int single_pick(struct commit *cmit, struct replay_opts *opts) { setenv(GIT_REFLOG_ACTION, action_name(opts), 0); return do_pick_commit(opts->action == REPLAY_PICK ? - TODO_PICK : TODO_REVERT, cmit, opts); + TODO_PICK : TODO_REVERT, cmit, opts, 0); } int sequencer_pick_revisions(struct replay_opts *opts) From c9eae7798f02e8312ce77847320022cb4b84adb7 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 13 Apr 2016 14:08:09 +0200 Subject: [PATCH 239/288] sequencer (rebase -i): implement the short commands For users' convenience, most rebase commands can be abbreviated, e.g. 'p' instead of 'pick' and 'x' instead of 'exec'. Let's teach the sequencer to handle those abbreviated commands just fine. Signed-off-by: Johannes Schindelin --- sequencer.c | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/sequencer.c b/sequencer.c index de4065b614..83a58da822 100644 --- a/sequencer.c +++ b/sequencer.c @@ -725,20 +725,23 @@ enum todo_command { TODO_NOOP }; -static const char *todo_command_strings[] = { - "pick", - "revert", - "edit", - "fixup", - "squash", - "exec", - "noop" +static struct { + char c; + const char *str; +} todo_command_info[] = { + { 'p', "pick" }, + { 0, "revert" }, + { 'e', "edit" }, + { 'f', "fixup" }, + { 's', "squash" }, + { 'x', "exec" }, + { 0, "noop" } }; static const char *command_to_string(const enum todo_command command) { - if ((size_t)command < ARRAY_SIZE(todo_command_strings)) - return todo_command_strings[command]; + if ((size_t)command < ARRAY_SIZE(todo_command_info)) + return todo_command_info[command].str; die("Unknown command: %d", command); } @@ -1134,12 +1137,16 @@ static int parse_insn_line(struct todo_item *item, const char *bol, char *eol) return 0; } - for (i = 0; i < ARRAY_SIZE(todo_command_strings); i++) - if (skip_prefix(bol, todo_command_strings[i], &bol)) { + for (i = 0; i < ARRAY_SIZE(todo_command_info); i++) + if (skip_prefix(bol, todo_command_info[i].str, &bol)) { + item->command = i; + break; + } else if (bol[1] == ' ' && *bol == todo_command_info[i].c) { + bol++; item->command = i; break; } - if (i >= ARRAY_SIZE(todo_command_strings)) + if (i >= ARRAY_SIZE(todo_command_info)) return -1; if (item->command == TODO_NOOP) { @@ -1334,7 +1341,7 @@ static int walk_revs_populate_todo(struct todo_list *todo_list, { enum todo_command command = opts->action == REPLAY_PICK ? TODO_PICK : TODO_REVERT; - const char *command_string = todo_command_strings[command]; + const char *command_string = todo_command_info[command].str; struct commit *commit; if (prepare_revs(opts)) From 30918cfc713fef3a6739c4e1a2619b4e8d032596 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 13 Apr 2016 14:11:12 +0200 Subject: [PATCH 240/288] sequencer (rebase -i): write an author-script file When the interactive rebase aborts, it writes out an author-script file to record the author information for the current commit. As we are about to teach the sequencer how to perform the actions behind an interactive rebase, it needs to write those author-script files, too. Signed-off-by: Johannes Schindelin --- sequencer.c | 50 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/sequencer.c b/sequencer.c index 83a58da822..1dda1b78cc 100644 --- a/sequencer.c +++ b/sequencer.c @@ -530,6 +530,52 @@ static int is_index_unchanged(void) return !hashcmp(active_cache_tree->sha1, head_commit->tree->object.oid.hash); } +static int write_author_script(const char *message) +{ + struct strbuf buf = STRBUF_INIT; + const char *eol; + int res; + + for (;;) + if (!*message || starts_with(message, "\n")) { +missing_author: + /* Missing 'author' line? */ + unlink(rebase_path_author_script()); + return 0; + } else if (skip_prefix(message, "author ", &message)) + break; + else if ((eol = strchr(message, '\n'))) + message = eol + 1; + else + goto missing_author; + + strbuf_addstr(&buf, "GIT_AUTHOR_NAME='"); + while (*message && *message != '\n' && *message != '\r') + if (skip_prefix(message, " <", &message)) + break; + else if (*message != '\'') + strbuf_addch(&buf, *(message++)); + else + strbuf_addf(&buf, "'\\\\%c'", *(message++)); + strbuf_addstr(&buf, "'\nGIT_AUTHOR_EMAIL='"); + while (*message && *message != '\n' && *message != '\r') + if (skip_prefix(message, "> ", &message)) + break; + else if (*message != '\'') + strbuf_addch(&buf, *(message++)); + else + strbuf_addf(&buf, "'\\\\%c'", *(message++)); + strbuf_addstr(&buf, "'\nGIT_AUTHOR_DATE='@"); + while (*message && *message != '\n' && *message != '\r') + if (*message != '\'') + strbuf_addch(&buf, *(message++)); + else + strbuf_addf(&buf, "'\\\\%c'", *(message++)); + res = write_message(buf.buf, buf.len, rebase_path_author_script(), 1); + strbuf_release(&buf); + return res; +} + /* * Read the author-script file into an environment block, ready for use in * run_command(), that can be free()d afterwards. @@ -982,7 +1028,9 @@ static int do_pick_commit(enum todo_command command, struct commit *commit, } } - if (!opts->strategy || !strcmp(opts->strategy, "recursive") || command == TODO_REVERT) { + if (is_rebase_i(opts) && write_author_script(msg.message) < 0) + res = -1; + else if (!opts->strategy || !strcmp(opts->strategy, "recursive") || command == TODO_REVERT) { res = do_recursive_merge(base, next, base_label, next_label, head, &msgbuf, opts); if (res < 0) From 8f33b4dfe28c5a39e8740cda93bf0f34c200af07 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 9 Mar 2016 17:12:20 +0100 Subject: [PATCH 241/288] sequencer (rebase -i): allow continuing with staged changes When an interactive rebase is interrupted, the user may stage changes before continuing, and we need to commit those changes in that case. Please note that the nested "if" added to the sequencer_continue() is not combined into a single "if" because it will be extended with an "else" clause in a later patch in this patch series. Signed-off-by: Johannes Schindelin --- sequencer.c | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/sequencer.c b/sequencer.c index 1dda1b78cc..7dd7ea53e0 100644 --- a/sequencer.c +++ b/sequencer.c @@ -1873,6 +1873,42 @@ static int continue_single_pick(void) return run_command_v_opt(argv, RUN_GIT_CMD); } +static int commit_staged_changes(struct replay_opts *opts) +{ + int amend = 0; + + if (has_unstaged_changes(1)) + return error(_("cannot rebase: You have unstaged changes.")); + if (!has_uncommitted_changes(0)) + return 0; + + if (file_exists(rebase_path_amend())) { + struct strbuf rev = STRBUF_INIT; + unsigned char head[20], to_amend[20]; + + if (get_sha1("HEAD", head)) + return error(_("cannot amend non-existing commit")); + if (!read_oneliner(&rev, rebase_path_amend(), 0)) + return error(_("invalid file: '%s'"), rebase_path_amend()); + if (get_sha1_hex(rev.buf, to_amend)) + return error(_("invalid contents: '%s'"), + rebase_path_amend()); + if (hashcmp(head, to_amend)) + return error(_("\nYou have uncommitted changes in your " + "working tree. Please, commit them\n" + "first and then run 'git rebase " + "--continue' again.")); + + strbuf_release(&rev); + amend = 1; + } + + if (run_git_commit(rebase_path_message(), opts, 1, 1, amend, 0)) + return error(_("could not commit staged changes.")); + unlink(rebase_path_amend()); + return 0; +} + int sequencer_continue(struct replay_opts *opts) { struct todo_list todo_list = TODO_LIST_INIT; @@ -1881,6 +1917,10 @@ int sequencer_continue(struct replay_opts *opts) if (read_and_refresh_cache(opts)) return -1; + if (is_rebase_i(opts)) { + if (commit_staged_changes(opts)) + return -1; + } if (!file_exists(get_todo_path(opts))) return continue_single_pick(); if (read_populate_opts(opts)) From 585d294b734ba6ce7c59318b023079ed54aa54a6 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sat, 9 Apr 2016 16:29:21 +0200 Subject: [PATCH 242/288] sequencer (rebase -i): remove CHERRY_PICK_HEAD when no longer needed The scripted version of the interactive rebase already does that. Signed-off-by: Johannes Schindelin --- sequencer.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/sequencer.c b/sequencer.c index 7dd7ea53e0..4ba086b210 100644 --- a/sequencer.c +++ b/sequencer.c @@ -1879,8 +1879,13 @@ static int commit_staged_changes(struct replay_opts *opts) if (has_unstaged_changes(1)) return error(_("cannot rebase: You have unstaged changes.")); - if (!has_uncommitted_changes(0)) + if (!has_uncommitted_changes(0)) { + const char *cherry_pick_head = git_path("CHERRY_PICK_HEAD"); + + if (file_exists(cherry_pick_head) && unlink(cherry_pick_head)) + return error(_("could not remove CHERRY_PICK_HEAD")); return 0; + } if (file_exists(rebase_path_amend())) { struct strbuf rev = STRBUF_INIT; From 11d3a93953974c1abc2b780c6cc3b1bcf487b762 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 31 Aug 2016 08:43:51 +0200 Subject: [PATCH 243/288] sequencer (rebase -i): skip some revert/cherry-pick specific code path When a cherry-pick continues without a "todo script", the intention is simply to pick a single commit. However, when an interactive rebase is continued without a "todo script", it means that the last command has been completed and that we now need to clean up. This commit guards the revert/cherry-pick specific steps so that they are not executed in rebase -i mode. Signed-off-by: Johannes Schindelin --- sequencer.c | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/sequencer.c b/sequencer.c index 4ba086b210..99051ae19b 100644 --- a/sequencer.c +++ b/sequencer.c @@ -1925,26 +1925,28 @@ int sequencer_continue(struct replay_opts *opts) if (is_rebase_i(opts)) { if (commit_staged_changes(opts)) return -1; - } - if (!file_exists(get_todo_path(opts))) + } else if (!file_exists(get_todo_path(opts))) return continue_single_pick(); if (read_populate_opts(opts)) return -1; if ((res = read_populate_todo(&todo_list, opts))) goto release_todo_list; - /* Verify that the conflict has been resolved */ - if (file_exists(git_path_cherry_pick_head()) || - file_exists(git_path_revert_head())) { - res = continue_single_pick(); - if (res) + if (!is_rebase_i(opts)) { + /* Verify that the conflict has been resolved */ + if (file_exists(git_path_cherry_pick_head()) || + file_exists(git_path_revert_head())) { + res = continue_single_pick(); + if (res) + goto release_todo_list; + } + if (index_differs_from("HEAD", 0, 0)) { + res = error_dirty_index(opts); goto release_todo_list; + } + todo_list.current++; } - if (index_differs_from("HEAD", 0, 0)) { - res = error_dirty_index(opts); - goto release_todo_list; - } - todo_list.current++; + res = pick_commits(&todo_list, opts); release_todo_list: todo_list_release(&todo_list); From 05b4e9aaa9d8a2ac83d883e2252a2fa3c63a8ade Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 13 Apr 2016 14:11:54 +0200 Subject: [PATCH 244/288] sequencer (rebase -i): the todo can be empty when continuing When the last command of an interactive rebase fails, the user needs to resolve the problem and then continue the interactive rebase. Naturally, the todo script is empty by then. So let's not complain about that! To that end, let's move that test out of the function that parses the todo script, and into the more high-level function read_populate_todo(). This is also necessary by now because the lower-level parse_insn_buffer() has no idea whether we are performing an interactive rebase or not. Signed-off-by: Johannes Schindelin --- sequencer.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/sequencer.c b/sequencer.c index 99051ae19b..5b03e17221 100644 --- a/sequencer.c +++ b/sequencer.c @@ -1262,8 +1262,7 @@ static int parse_insn_buffer(char *buf, struct todo_list *todo_list) else if (!is_noop(item->command)) fixup_okay = 1; } - if (!todo_list->nr) - return error(_("no commits parsed.")); + return res; } @@ -1287,6 +1286,10 @@ static int read_populate_todo(struct todo_list *todo_list, if (res) return error(_("unusable instruction sheet: '%s'"), todo_file); + if (!todo_list->nr && + (!is_rebase_i(opts) || !file_exists(rebase_path_done()))) + return error(_("no commits parsed.")); + if (!is_rebase_i(opts)) { enum todo_command valid = opts->action == REPLAY_PICK ? TODO_PICK : TODO_REVERT; From 3ebedc7776e9c8f9c6dc39a939a5b7c4fbf29d54 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 13 Apr 2016 14:12:54 +0200 Subject: [PATCH 245/288] sequencer (rebase -i): update refs after a successful rebase An interactive rebase operates on a detached HEAD (to keep the reflog of the original branch relatively clean), and updates the branch only at the end. Now that the sequencer learns to perform interactive rebases, it also needs to learn the trick to update the branch before removing the directory containing the state of the interactive rebase. We introduce a new head_ref variable in a wider scope than necessary at the moment, to allow for a later patch that prints out "Successfully rebased and updated ". Signed-off-by: Johannes Schindelin --- sequencer.c | 46 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/sequencer.c b/sequencer.c index 5b03e17221..6198d4b9e7 100644 --- a/sequencer.c +++ b/sequencer.c @@ -101,6 +101,8 @@ static GIT_PATH_FUNC(rebase_path_stopped_sha, "rebase-merge/stopped-sha") static GIT_PATH_FUNC(rebase_path_gpg_sign_opt, "rebase-merge/gpg_sign_opt") static GIT_PATH_FUNC(rebase_path_orig_head, "rebase-merge/orig-head") static GIT_PATH_FUNC(rebase_path_verbose, "rebase-merge/verbose") +static GIT_PATH_FUNC(rebase_path_head_name, "rebase-merge/head-name") +static GIT_PATH_FUNC(rebase_path_onto, "rebase-merge/onto") static inline int is_rebase_i(const struct replay_opts *opts) { @@ -1831,12 +1833,53 @@ static int pick_commits(struct todo_list *todo_list, struct replay_opts *opts) } if (is_rebase_i(opts)) { - struct strbuf buf = STRBUF_INIT; + struct strbuf head_ref = STRBUF_INIT, buf = STRBUF_INIT; /* Stopped in the middle, as planned? */ if (todo_list->current < todo_list->nr) return 0; + if (read_oneliner(&head_ref, rebase_path_head_name(), 0) && + starts_with(head_ref.buf, "refs/")) { + unsigned char head[20], orig[20]; + int res; + + if (get_sha1("HEAD", head)) { + res = error(_("cannot read HEAD")); +cleanup_head_ref: + strbuf_release(&head_ref); + strbuf_release(&buf); + return res; + } + if (!read_oneliner(&buf, rebase_path_orig_head(), 0) || + get_sha1_hex(buf.buf, orig)) { + res = error(_("could not read orig-head")); + goto cleanup_head_ref; + } + strbuf_addf(&buf, "rebase -i (finish): %s onto ", + head_ref.buf); + if (!read_oneliner(&buf, rebase_path_onto(), 0)) { + res = error(_("could not read 'onto'")); + goto cleanup_head_ref; + } + if (update_ref(buf.buf, head_ref.buf, head, orig, + REF_NODEREF, UPDATE_REFS_MSG_ON_ERR)) { + res = error(_("could not update %s"), + head_ref.buf); + goto cleanup_head_ref; + } + strbuf_reset(&buf); + strbuf_addf(&buf, + "rebase -i (finish): returning to %s", + head_ref.buf); + if (create_symref("HEAD", head_ref.buf, buf.buf)) { + res = error(_("could not update HEAD to %s"), + head_ref.buf); + goto cleanup_head_ref; + } + strbuf_reset(&buf); + } + if (opts->verbose) { struct rev_info log_tree_opt; struct object_id orig, head; @@ -1857,6 +1900,7 @@ static int pick_commits(struct todo_list *todo_list, struct replay_opts *opts) } } strbuf_release(&buf); + strbuf_release(&head_ref); } /* From fd8b463154d90e69ceb740b854abab6901655c77 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 10 Mar 2016 21:47:32 +0100 Subject: [PATCH 246/288] sequencer (rebase -i): leave a patch upon error When doing an interactive rebase, we want to leave a 'patch' file for further inspection by the user (even if we never tried to actually apply that patch, since we're cherry-picking instead). Signed-off-by: Johannes Schindelin --- sequencer.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sequencer.c b/sequencer.c index 6198d4b9e7..b6c288fa4d 100644 --- a/sequencer.c +++ b/sequencer.c @@ -1816,7 +1816,9 @@ static int pick_commits(struct todo_list *todo_list, struct replay_opts *opts) intend_to_amend(); return error_failed_squash(item->commit, opts, item->arg_len, item->arg); - } + } else if (res && is_rebase_i(opts)) + return res | error_with_patch(item->commit, + item->arg, item->arg_len, opts, res, 0); } else if (item->command == TODO_EXEC) { char *end_of_arg = (char *)(item->arg + item->arg_len); int saved = *end_of_arg; From 3ddf56be6697e5a4e6cc558ad9314dd58992332f Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 13 Apr 2016 14:13:31 +0200 Subject: [PATCH 247/288] sequencer (rebase -i): implement the 'reword' command This is now trivial, as all the building blocks are in place: all we need to do is to flip the "edit" switch when committing. Signed-off-by: Johannes Schindelin --- sequencer.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/sequencer.c b/sequencer.c index b6c288fa4d..20fe6e9816 100644 --- a/sequencer.c +++ b/sequencer.c @@ -765,6 +765,7 @@ enum todo_command { TODO_PICK = 0, TODO_REVERT, TODO_EDIT, + TODO_REWORD, TODO_FIXUP, TODO_SQUASH, /* commands that do something else than handling a single commit */ @@ -780,6 +781,7 @@ static struct { { 'p', "pick" }, { 0, "revert" }, { 'e', "edit" }, + { 'r', "reword" }, { 'f', "fixup" }, { 's', "squash" }, { 'x', "exec" }, @@ -1009,7 +1011,9 @@ static int do_pick_commit(enum todo_command command, struct commit *commit, } } - if (is_fixup(command)) { + if (command == TODO_REWORD) + edit = 1; + else if (is_fixup(command)) { if (update_squash_messages(command, commit, opts)) return -1; amend = 1; @@ -1818,7 +1822,8 @@ static int pick_commits(struct todo_list *todo_list, struct replay_opts *opts) item->arg_len, item->arg); } else if (res && is_rebase_i(opts)) return res | error_with_patch(item->commit, - item->arg, item->arg_len, opts, res, 0); + item->arg, item->arg_len, opts, res, + item->command == TODO_REWORD); } else if (item->command == TODO_EXEC) { char *end_of_arg = (char *)(item->arg + item->arg_len); int saved = *end_of_arg; From e477129e279e27a1038d9115781d82ca3130a983 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 23 May 2016 16:56:21 +0200 Subject: [PATCH 248/288] sequencer (rebase -i): allow fast-forwarding for edit/reword The sequencer already knew how to fast-forward instead of cherry-picking, if possible. We want to continue to do this, of course, but in case of the 'reword' command, we will need to call `git commit` after fast-forwarding. Signed-off-by: Johannes Schindelin --- sequencer.c | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/sequencer.c b/sequencer.c index 20fe6e9816..daa07fa1d9 100644 --- a/sequencer.c +++ b/sequencer.c @@ -907,7 +907,7 @@ static int do_pick_commit(enum todo_command command, struct commit *commit, const char *base_label, *next_label; struct commit_message msg = { NULL, NULL, NULL, NULL }; struct strbuf msgbuf = STRBUF_INIT; - int res, unborn = 0, amend = 0, allow; + int res, unborn = 0, amend = 0, allow = 0; if (opts->no_commit) { /* @@ -952,11 +952,23 @@ static int do_pick_commit(enum todo_command command, struct commit *commit, else parent = commit->parents->item; + if (get_message(commit, &msg) != 0) + return error(_("cannot get commit message for %s"), + oid_to_hex(&commit->object.oid)); + if (opts->allow_ff && !is_fixup(command) && ((parent && !hashcmp(parent->object.oid.hash, head)) || - (!parent && unborn))) - return fast_forward_to(commit->object.oid.hash, head, unborn, opts); - + (!parent && unborn))) { + if (is_rebase_i(opts)) + write_author_script(msg.message); + res = fast_forward_to(commit->object.oid.hash, head, unborn, + opts); + if (res || command != TODO_REWORD) + goto leave; + edit = amend = 1; + msg_file = NULL; + goto fast_forward_edit; + } if (parent && parse_commit(parent) < 0) /* TRANSLATORS: The first %s will be a "todo" command like "revert" or "pick", the second %s a SHA1. */ @@ -964,10 +976,6 @@ static int do_pick_commit(enum todo_command command, struct commit *commit, command_to_string(command), oid_to_hex(&parent->object.oid)); - if (get_message(commit, &msg) != 0) - return error(_("cannot get commit message for %s"), - oid_to_hex(&commit->object.oid)); - /* * "commit" is an existing commit. We would want to apply * the difference it introduces since its first parent "prev" @@ -1091,6 +1099,7 @@ static int do_pick_commit(enum todo_command command, struct commit *commit, goto leave; } if (!opts->no_commit) +fast_forward_edit: res = run_git_commit(msg_file, opts, allow, edit, amend, cleanup_commit_message); From 6e36735e666cba852b343ea11369197495de505d Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sat, 9 Apr 2016 16:27:51 +0200 Subject: [PATCH 249/288] sequencer (rebase -i): refactor setting the reflog message This makes the code DRYer, with the obvious benefit that we can enhance the code further in a single place. We can also reuse the functionality elsewhere by calling this new function. Signed-off-by: Johannes Schindelin --- sequencer.c | 33 ++++++++++++++++++++++++++------- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/sequencer.c b/sequencer.c index daa07fa1d9..6f33652302 100644 --- a/sequencer.c +++ b/sequencer.c @@ -1790,6 +1790,26 @@ static int is_final_fixup(struct todo_list *todo_list) return 1; } +static const char *reflog_message(struct replay_opts *opts, + const char *sub_action, const char *fmt, ...) +{ + va_list ap; + static struct strbuf buf = STRBUF_INIT; + + va_start(ap, fmt); + strbuf_reset(&buf); + strbuf_addstr(&buf, action_name(opts)); + if (sub_action) + strbuf_addf(&buf, " (%s)", sub_action); + if (fmt) { + strbuf_addstr(&buf, ": "); + strbuf_vaddf(&buf, fmt, ap); + } + va_end(ap); + + return buf.buf; +} + static int pick_commits(struct todo_list *todo_list, struct replay_opts *opts) { int res = 0; @@ -1857,6 +1877,7 @@ static int pick_commits(struct todo_list *todo_list, struct replay_opts *opts) if (read_oneliner(&head_ref, rebase_path_head_name(), 0) && starts_with(head_ref.buf, "refs/")) { + const char *msg; unsigned char head[20], orig[20]; int res; @@ -1872,23 +1893,21 @@ cleanup_head_ref: res = error(_("could not read orig-head")); goto cleanup_head_ref; } - strbuf_addf(&buf, "rebase -i (finish): %s onto ", - head_ref.buf); if (!read_oneliner(&buf, rebase_path_onto(), 0)) { res = error(_("could not read 'onto'")); goto cleanup_head_ref; } - if (update_ref(buf.buf, head_ref.buf, head, orig, + msg = reflog_message(opts, "finish", "%s onto %s", + head_ref.buf, buf.buf); + if (update_ref(msg, head_ref.buf, head, orig, REF_NODEREF, UPDATE_REFS_MSG_ON_ERR)) { res = error(_("could not update %s"), head_ref.buf); goto cleanup_head_ref; } - strbuf_reset(&buf); - strbuf_addf(&buf, - "rebase -i (finish): returning to %s", + msg = reflog_message(opts, "finish", "returning to %s", head_ref.buf); - if (create_symref("HEAD", head_ref.buf, buf.buf)) { + if (create_symref("HEAD", head_ref.buf, msg)) { res = error(_("could not update HEAD to %s"), head_ref.buf); goto cleanup_head_ref; From e70fdc47febe77b27adfdcbb5f3eb63f875d1d86 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sat, 9 Apr 2016 16:28:07 +0200 Subject: [PATCH 250/288] sequencer (rebase -i): set the reflog message consistently We already used the same reflog message as the scripted version of rebase -i when finishing. With this commit, we do that also for all the commands before that. Signed-off-by: Johannes Schindelin --- sequencer.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sequencer.c b/sequencer.c index 6f33652302..e6319bb9eb 100644 --- a/sequencer.c +++ b/sequencer.c @@ -1832,6 +1832,10 @@ static int pick_commits(struct todo_list *todo_list, struct replay_opts *opts) unlink(rebase_path_amend()); } if (item->command <= TODO_SQUASH) { + if (is_rebase_i(opts)) + setenv("GIT_REFLOG_ACTION", reflog_message(opts, + command_to_string(item->command), NULL), + 1); res = do_pick_commit(item->command, item->commit, opts, is_final_fixup(todo_list)); if (item->command == TODO_EDIT) { From 1e7657cb05d68622efc4faf10761df9acae887f1 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 12 Apr 2016 17:26:31 +0200 Subject: [PATCH 251/288] sequencer (rebase -i): copy commit notes at end When rebasing commits that have commit notes attached, the interactive rebase rewrites those notes faithfully at the end. The sequencer must do this, too, if it wishes to do interactive rebase's job. Signed-off-by: Johannes Schindelin --- sequencer.c | 76 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/sequencer.c b/sequencer.c index e6319bb9eb..a3c2a31195 100644 --- a/sequencer.c +++ b/sequencer.c @@ -94,6 +94,15 @@ static GIT_PATH_FUNC(rebase_path_amend, "rebase-merge/amend") * the abbreviated commit name of the corresponding patch. */ static GIT_PATH_FUNC(rebase_path_stopped_sha, "rebase-merge/stopped-sha") +/* + * For the post-rewrite hook, we make a list of rewritten commits and + * their new sha1s. The rewritten-pending list keeps the sha1s of + * commits that have been processed, but not committed yet, + * e.g. because they are waiting for a 'squash' command. + */ +static GIT_PATH_FUNC(rebase_path_rewritten_list, "rebase-merge/rewritten-list") +static GIT_PATH_FUNC(rebase_path_rewritten_pending, + "rebase-merge/rewritten-pending") /* * The following files are written by git-rebase just after parsing the * command-line (and are only consumed, not modified, by the sequencer). @@ -897,6 +906,44 @@ static int update_squash_messages(enum todo_command command, return res; } +static void flush_rewritten_pending(void) { + struct strbuf buf = STRBUF_INIT; + unsigned char newsha1[20]; + FILE *out; + + if (strbuf_read_file(&buf, rebase_path_rewritten_pending(), 82) > 0 && + !get_sha1("HEAD", newsha1) && + (out = fopen(rebase_path_rewritten_list(), "a"))) { + char *bol = buf.buf, *eol; + + while (*bol) { + eol = strchrnul(bol, '\n'); + fprintf(out, "%.*s %s\n", (int)(eol - bol), + bol, sha1_to_hex(newsha1)); + if (!*eol) + break; + bol = eol + 1; + } + fclose(out); + unlink(rebase_path_rewritten_pending()); + } + strbuf_release(&buf); +} + +static void record_in_rewritten(struct object_id *oid, + enum todo_command next_command) { + FILE *out = fopen(rebase_path_rewritten_pending(), "a"); + + if (!out) + return; + + fprintf(out, "%s\n", oid_to_hex(oid)); + fclose(out); + + if (!is_fixup(next_command)) + flush_rewritten_pending(); +} + static int do_pick_commit(enum todo_command command, struct commit *commit, struct replay_opts *opts, int final_fixup) { @@ -1790,6 +1837,17 @@ static int is_final_fixup(struct todo_list *todo_list) return 1; } +static enum todo_command peek_command(struct todo_list *todo_list, int offset) +{ + int i; + + for (i = todo_list->current + offset; i < todo_list->nr; i++) + if (!is_noop(todo_list->items[i].command)) + return todo_list->items[i].command; + + return -1; +} + static const char *reflog_message(struct replay_opts *opts, const char *sub_action, const char *fmt, ...) { @@ -1848,6 +1906,9 @@ static int pick_commits(struct todo_list *todo_list, struct replay_opts *opts) item->arg, item->arg_len, opts, res, !res); } + if (is_rebase_i(opts) && !res) + record_in_rewritten(&item->commit->object.oid, + peek_command(todo_list, 1)); if (res && is_fixup(item->command)) { if (res == 1) intend_to_amend(); @@ -1874,6 +1935,7 @@ static int pick_commits(struct todo_list *todo_list, struct replay_opts *opts) if (is_rebase_i(opts)) { struct strbuf head_ref = STRBUF_INIT, buf = STRBUF_INIT; + struct stat st; /* Stopped in the middle, as planned? */ if (todo_list->current < todo_list->nr) @@ -1938,6 +2000,20 @@ cleanup_head_ref: log_tree_diff_flush(&log_tree_opt); } } + flush_rewritten_pending(); + if (!stat(rebase_path_rewritten_list(), &st) && + st.st_size > 0) { + struct child_process child = CHILD_PROCESS_INIT; + + child.in = open(rebase_path_rewritten_list(), O_RDONLY); + child.git_cmd = 1; + argv_array_push(&child.args, "notes"); + argv_array_push(&child.args, "copy"); + argv_array_push(&child.args, "--for-rewrite=rebase"); + /* we don't care if this copying failed */ + run_command(&child); + } + strbuf_release(&buf); strbuf_release(&head_ref); } From 6a45fef61b0dcd78549cd08862fb11fc7ac6e921 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 8 Apr 2016 17:30:25 +0200 Subject: [PATCH 252/288] sequencer (rebase -i): record interrupted commits in rewritten, too When continuing after a `pick` command failed, we want that commit to show up in the rewritten-list (and its notes to be rewritten), too. Signed-off-by: Johannes Schindelin --- sequencer.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/sequencer.c b/sequencer.c index a3c2a31195..b6ce2a19fe 100644 --- a/sequencer.c +++ b/sequencer.c @@ -2107,6 +2107,14 @@ int sequencer_continue(struct replay_opts *opts) goto release_todo_list; } todo_list.current++; + } else if (file_exists(rebase_path_stopped_sha())) { + struct strbuf buf = STRBUF_INIT; + struct object_id oid; + + if (read_oneliner(&buf, rebase_path_stopped_sha(), 1) && + !get_sha1_committish(buf.buf, oid.hash)) + record_in_rewritten(&oid, peek_command(&todo_list, 0)); + strbuf_release(&buf); } res = pick_commits(&todo_list, opts); From aa669c0c5c9c3ffc175455d2cf433219c1a6541b Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 8 Apr 2016 17:27:03 +0200 Subject: [PATCH 253/288] sequencer (rebase -i): run the post-rewrite hook, if needed Signed-off-by: Johannes Schindelin --- sequencer.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/sequencer.c b/sequencer.c index b6ce2a19fe..5bc8a86adb 100644 --- a/sequencer.c +++ b/sequencer.c @@ -2004,6 +2004,8 @@ cleanup_head_ref: if (!stat(rebase_path_rewritten_list(), &st) && st.st_size > 0) { struct child_process child = CHILD_PROCESS_INIT; + const char *post_rewrite_hook = + find_hook("post-rewrite"); child.in = open(rebase_path_rewritten_list(), O_RDONLY); child.git_cmd = 1; @@ -2012,6 +2014,18 @@ cleanup_head_ref: argv_array_push(&child.args, "--for-rewrite=rebase"); /* we don't care if this copying failed */ run_command(&child); + + if (post_rewrite_hook) { + struct child_process hook = CHILD_PROCESS_INIT; + + hook.in = open(rebase_path_rewritten_list(), + O_RDONLY); + hook.stdout_to_stderr = 1; + argv_array_push(&hook.args, post_rewrite_hook); + argv_array_push(&hook.args, "rebase"); + /* we don't care if this hook failed */ + run_command(&hook); + } } strbuf_release(&buf); From ef40e393ca7019777166e1449ffd338db4116296 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 8 Apr 2016 16:05:06 +0200 Subject: [PATCH 254/288] sequencer (rebase -i): respect the rebase.autostash setting Git's `rebase` command inspects the `rebase.autostash` config setting to determine whether it should stash any uncommitted changes before rebasing and re-apply them afterwards. As we introduce more bits and pieces to let the sequencer act as interactive rebase's backend, here is the part that adds support for the autostash feature. Signed-off-by: Johannes Schindelin --- sequencer.c | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/sequencer.c b/sequencer.c index 5bc8a86adb..a454ed8960 100644 --- a/sequencer.c +++ b/sequencer.c @@ -112,6 +112,7 @@ static GIT_PATH_FUNC(rebase_path_orig_head, "rebase-merge/orig-head") static GIT_PATH_FUNC(rebase_path_verbose, "rebase-merge/verbose") static GIT_PATH_FUNC(rebase_path_head_name, "rebase-merge/head-name") static GIT_PATH_FUNC(rebase_path_onto, "rebase-merge/onto") +static GIT_PATH_FUNC(rebase_path_autostash, "rebase-merge/autostash") static inline int is_rebase_i(const struct replay_opts *opts) { @@ -1848,6 +1849,47 @@ static enum todo_command peek_command(struct todo_list *todo_list, int offset) return -1; } +static int apply_autostash(struct replay_opts *opts) +{ + struct strbuf stash_sha1 = STRBUF_INIT; + struct child_process child = CHILD_PROCESS_INIT; + int ret = 0; + + if (!read_oneliner(&stash_sha1, rebase_path_autostash(), 1)) { + strbuf_release(&stash_sha1); + return 0; + } + strbuf_trim(&stash_sha1); + + child.git_cmd = 1; + argv_array_push(&child.args, "stash"); + argv_array_push(&child.args, "apply"); + argv_array_push(&child.args, stash_sha1.buf); + if (!run_command(&child)) + printf(_("Applied autostash.")); + else { + struct child_process store = CHILD_PROCESS_INIT; + + store.git_cmd = 1; + argv_array_push(&store.args, "stash"); + argv_array_push(&store.args, "store"); + argv_array_push(&store.args, "-m"); + argv_array_push(&store.args, "autostash"); + argv_array_push(&store.args, "-q"); + argv_array_push(&store.args, stash_sha1.buf); + if (run_command(&store)) + ret = error(_("cannot store %s"), stash_sha1.buf); + else + printf(_("Applying autostash resulted in conflicts.\n" + "Your changes are safe in the stash.\n" + "You can run \"git stash pop\" or" + " \"git stash drop\" at any time.\n")); + } + + strbuf_release(&stash_sha1); + return ret; +} + static const char *reflog_message(struct replay_opts *opts, const char *sub_action, const char *fmt, ...) { @@ -2027,6 +2069,7 @@ cleanup_head_ref: run_command(&hook); } } + apply_autostash(opts); strbuf_release(&buf); strbuf_release(&head_ref); From 0071680d1ce5fc820c561248600376c6c5b038dd Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 30 Mar 2016 17:58:55 +0200 Subject: [PATCH 255/288] sequencer (rebase -i): respect strategy/strategy_opts settings The sequencer already has an idea about using different merge strategies. We just piggy-back on top of that, using rebase -i's own settings, when running the sequencer in interactive rebase mode. Signed-off-by: Johannes Schindelin --- sequencer.c | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/sequencer.c b/sequencer.c index a454ed8960..3d695ba6ba 100644 --- a/sequencer.c +++ b/sequencer.c @@ -113,6 +113,8 @@ static GIT_PATH_FUNC(rebase_path_verbose, "rebase-merge/verbose") static GIT_PATH_FUNC(rebase_path_head_name, "rebase-merge/head-name") static GIT_PATH_FUNC(rebase_path_onto, "rebase-merge/onto") static GIT_PATH_FUNC(rebase_path_autostash, "rebase-merge/autostash") +static GIT_PATH_FUNC(rebase_path_strategy, "rebase-merge/strategy") +static GIT_PATH_FUNC(rebase_path_strategy_opts, "rebase-merge/strategy_opts") static inline int is_rebase_i(const struct replay_opts *opts) { @@ -1415,6 +1417,26 @@ static int populate_opts_cb(const char *key, const char *value, void *data) return 0; } +static void read_strategy_opts(struct replay_opts *opts, struct strbuf *buf) +{ + int i; + + strbuf_reset(buf); + if (!read_oneliner(buf, rebase_path_strategy(), 0)) + return; + opts->strategy = strbuf_detach(buf, NULL); + if (!read_oneliner(buf, rebase_path_strategy_opts(), 0)) + return; + + opts->xopts_nr = split_cmdline(buf->buf, (const char ***)&opts->xopts); + for (i = 0; i < opts->xopts_nr; i++) { + const char *arg = opts->xopts[i]; + + skip_prefix(arg, "--", &arg); + opts->xopts[i] = xstrdup(arg); + } +} + static int read_populate_opts(struct replay_opts *opts) { if (is_rebase_i(opts)) { @@ -1428,11 +1450,13 @@ static int read_populate_opts(struct replay_opts *opts) opts->gpg_sign = xstrdup(buf.buf + 2); } } - strbuf_release(&buf); if (file_exists(rebase_path_verbose())) opts->verbose = 1; + read_strategy_opts(opts, &buf); + strbuf_release(&buf); + return 0; } From 30e02629ffcc3d567d8a2e13898896c23581f1ed Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 7 Apr 2016 16:31:59 +0200 Subject: [PATCH 256/288] sequencer (rebase -i): allow rescheduling commands The interactive rebase has the very special magic that a cherry-pick that exits with a status different from 0 and 1 signifies a failure to even record that a cherry-pick was started. This can happen e.g. when a fast-forward fails because it would overwrite untracked files. In that case, we must reschedule the command that we thought we already had at least started successfully. Signed-off-by: Johannes Schindelin --- sequencer.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sequencer.c b/sequencer.c index 3d695ba6ba..3ec7f9a3bc 100644 --- a/sequencer.c +++ b/sequencer.c @@ -1962,6 +1962,12 @@ static int pick_commits(struct todo_list *todo_list, struct replay_opts *opts) 1); res = do_pick_commit(item->command, item->commit, opts, is_final_fixup(todo_list)); + if (is_rebase_i(opts) && res < 0) { + /* Reschedule */ + todo_list->current--; + if (save_todo(todo_list, opts)) + return -1; + } if (item->command == TODO_EDIT) { struct commit *commit = item->commit; if (!res) From 839a3ebfa21f53fb73eeb0e671a853a88c3f648e Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 13 Apr 2016 15:18:29 +0200 Subject: [PATCH 257/288] sequencer (rebase -i): implement the 'drop' command The parsing part of a 'drop' command is almost identical to parsing a 'pick', while the operation is the same as that of a 'noop'. Signed-off-by: Johannes Schindelin --- sequencer.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/sequencer.c b/sequencer.c index 3ec7f9a3bc..0bad984dfc 100644 --- a/sequencer.c +++ b/sequencer.c @@ -783,7 +783,8 @@ enum todo_command { /* commands that do something else than handling a single commit */ TODO_EXEC, /* commands that do nothing but are counted for reporting progress */ - TODO_NOOP + TODO_NOOP, + TODO_DROP }; static struct { @@ -797,7 +798,8 @@ static struct { { 'f', "fixup" }, { 's', "squash" }, { 'x', "exec" }, - { 0, "noop" } + { 0, "noop" }, + { 'd', "drop" } }; static const char *command_to_string(const enum todo_command command) @@ -809,7 +811,7 @@ static const char *command_to_string(const enum todo_command command) static int is_noop(const enum todo_command command) { - return TODO_NOOP <= (size_t)command; + return TODO_NOOP <= command; } static int is_fixup(enum todo_command command) From 16a746222c24d8707c91db1c847995b6b4c35e49 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 8 Apr 2016 08:45:17 +0200 Subject: [PATCH 258/288] sequencer (rebase -i): differentiate between comments and 'noop' In the upcoming patch, we will support rebase -i's progress reporting. The progress skips comments but counts 'noop's. Signed-off-by: Johannes Schindelin --- sequencer.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/sequencer.c b/sequencer.c index 0bad984dfc..18d7e26d90 100644 --- a/sequencer.c +++ b/sequencer.c @@ -784,7 +784,9 @@ enum todo_command { TODO_EXEC, /* commands that do nothing but are counted for reporting progress */ TODO_NOOP, - TODO_DROP + TODO_DROP, + /* comments (not counted for reporting progress) */ + TODO_COMMENT }; static struct { @@ -799,12 +801,13 @@ static struct { { 's', "squash" }, { 'x', "exec" }, { 0, "noop" }, - { 'd', "drop" } + { 'd', "drop" }, + { 0, NULL } }; static const char *command_to_string(const enum todo_command command) { - if ((size_t)command < ARRAY_SIZE(todo_command_info)) + if (command < TODO_COMMENT) return todo_command_info[command].str; die("Unknown command: %d", command); } @@ -1245,14 +1248,14 @@ static int parse_insn_line(struct todo_item *item, const char *bol, char *eol) bol += strspn(bol, " \t"); if (bol == eol || *bol == '\r' || *bol == comment_line_char) { - item->command = TODO_NOOP; + item->command = TODO_COMMENT; item->commit = NULL; item->arg = bol; item->arg_len = eol - bol; return 0; } - for (i = 0; i < ARRAY_SIZE(todo_command_info); i++) + for (i = 0; i < TODO_COMMENT; i++) if (skip_prefix(bol, todo_command_info[i].str, &bol)) { item->command = i; break; @@ -1261,7 +1264,7 @@ static int parse_insn_line(struct todo_item *item, const char *bol, char *eol) item->command = i; break; } - if (i >= ARRAY_SIZE(todo_command_info)) + if (i >= TODO_COMMENT) return -1; if (item->command == TODO_NOOP) { From 8d772490a1736cd567186d283cd9cad5413152e8 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 2 Jan 2017 14:55:10 +0100 Subject: [PATCH 259/288] sequencer: make reading author-script more elegant Rather than abusing a strbuf to come up with an environment block, let's just use the argv_array structure which serves the same purpose much better. While at it, rename the function to reflect the fact that it does not really care exactly what environment variables are defined in said file. Suggested-by: Jeff King Signed-off-by: Johannes Schindelin --- sequencer.c | 34 ++++++++++++---------------------- t/t3404-rebase-interactive.sh | 16 ++++++++++++++++ 2 files changed, 28 insertions(+), 22 deletions(-) diff --git a/sequencer.c b/sequencer.c index 18d7e26d90..b8efd33f01 100644 --- a/sequencer.c +++ b/sequencer.c @@ -591,18 +591,17 @@ missing_author: } /* - * Read the author-script file into an environment block, ready for use in - * run_command(), that can be free()d afterwards. + * Read a list of environment variable assignments (such as the author-script + * file) into an environment block. Returns -1 on error, 0 otherwise. */ -static char **read_author_script(void) +static int read_env_script(struct argv_array *env) { struct strbuf script = STRBUF_INIT; int i, count = 0; - char *p, *p2, **env; - size_t env_size; + char *p, *p2; if (strbuf_read_file(&script, rebase_path_author_script(), 256) <= 0) - return NULL; + return -1; for (p = script.buf; *p; p++) if (skip_prefix(p, "'\\\\''", (const char **)&p2)) @@ -614,19 +613,12 @@ static char **read_author_script(void) count++; } - env_size = (count + 1) * sizeof(*env); - strbuf_grow(&script, env_size); - memmove(script.buf + env_size, script.buf, script.len); - p = script.buf + env_size; - env = (char **)strbuf_detach(&script, NULL); - - for (i = 0; i < count; i++) { - env[i] = p; + for (i = 0, p = script.buf; i < count; i++) { + argv_array_push(env, p); p += strlen(p) + 1; } - env[count] = NULL; - return env; + return 0; } static const char staged_changes_advice[] = @@ -659,14 +651,12 @@ static int run_git_commit(const char *defmsg, struct replay_opts *opts, int allow_empty, int edit, int amend, int cleanup_commit_message) { - char **env = NULL; - struct argv_array array; + struct argv_array env = ARGV_ARRAY_INIT, array; int rc; const char *value; if (is_rebase_i(opts)) { - env = read_author_script(); - if (!env) { + if (!read_env_script(&env)) { const char *gpg_opt = gpg_sign_opt_quoted(opts); return error(_(staged_changes_advice), @@ -702,9 +692,9 @@ static int run_git_commit(const char *defmsg, struct replay_opts *opts, argv_array_push(&array, "--allow-empty-message"); rc = run_command_v_opt_cd_env(array.argv, RUN_GIT_CMD, NULL, - (const char *const *)env); + (const char *const *)env.argv); argv_array_clear(&array); - free(env); + argv_array_clear(&env); return rc; } diff --git a/t/t3404-rebase-interactive.sh b/t/t3404-rebase-interactive.sh index c896a4c106..e2f18d11f6 100755 --- a/t/t3404-rebase-interactive.sh +++ b/t/t3404-rebase-interactive.sh @@ -237,6 +237,22 @@ test_expect_success 'retain authorship' ' git show HEAD | grep "^Author: Twerp Snog" ' +test_expect_success 'retain authorship w/ conflicts' ' + git reset --hard twerp && + test_commit a conflict a conflict-a && + git reset --hard twerp && + GIT_AUTHOR_NAME=AttributeMe \ + test_commit b conflict b conflict-b && + set_fake_editor && + test_must_fail git rebase -i conflict-a && + echo resolved >conflict && + git add conflict && + git rebase --continue && + test $(git rev-parse conflict-a^0) = $(git rev-parse HEAD^) && + git show >out && + grep AttributeMe out +' + test_expect_success 'squash' ' git reset --hard twerp && echo B > file7 && From ab3e98563267ddd01c94583b71ebfe0d2acfca8d Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 2 Jan 2017 15:23:42 +0100 Subject: [PATCH 260/288] sequencer: use run_command() directly Instead of using the convenience function run_command_v_opt_cd_env(), we now use the run_command() function. The former function is simply a wrapper of the latter, trying to make it more convenient to use. However, we already have to construct the argv and the env parameters, and we will need even finer control e.g. over the output of the command, so let's just stop using the convenience function. Based on patches and suggestions by Johannes Sixt and Jeff King. Signed-off-by: Johannes Schindelin --- sequencer.c | 36 ++++++++++++++++-------------------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/sequencer.c b/sequencer.c index b8efd33f01..4008134333 100644 --- a/sequencer.c +++ b/sequencer.c @@ -651,12 +651,13 @@ static int run_git_commit(const char *defmsg, struct replay_opts *opts, int allow_empty, int edit, int amend, int cleanup_commit_message) { - struct argv_array env = ARGV_ARRAY_INIT, array; - int rc; + struct child_process cmd = CHILD_PROCESS_INIT; const char *value; + cmd.git_cmd = 1; + if (is_rebase_i(opts)) { - if (!read_env_script(&env)) { + if (read_env_script(&cmd.env_array)) { const char *gpg_opt = gpg_sign_opt_quoted(opts); return error(_(staged_changes_advice), @@ -664,39 +665,34 @@ static int run_git_commit(const char *defmsg, struct replay_opts *opts, } } - argv_array_init(&array); - argv_array_push(&array, "commit"); - argv_array_push(&array, "-n"); + argv_array_push(&cmd.args, "commit"); + argv_array_push(&cmd.args, "-n"); if (amend) - argv_array_push(&array, "--amend"); + argv_array_push(&cmd.args, "--amend"); if (opts->gpg_sign) - argv_array_pushf(&array, "-S%s", opts->gpg_sign); + argv_array_pushf(&cmd.args, "-S%s", opts->gpg_sign); if (opts->signoff) - argv_array_push(&array, "-s"); + argv_array_push(&cmd.args, "-s"); if (defmsg) - argv_array_pushl(&array, "-F", defmsg, NULL); + argv_array_pushl(&cmd.args, "-F", defmsg, NULL); if (cleanup_commit_message) - argv_array_push(&array, "--cleanup=strip"); + argv_array_push(&cmd.args, "--cleanup=strip"); if (edit) - argv_array_push(&array, "-e"); + argv_array_push(&cmd.args, "-e"); else if (!cleanup_commit_message && !opts->signoff && !opts->record_origin && git_config_get_value("commit.cleanup", &value)) - argv_array_push(&array, "--cleanup=verbatim"); + argv_array_push(&cmd.args, "--cleanup=verbatim"); if (allow_empty) - argv_array_push(&array, "--allow-empty"); + argv_array_push(&cmd.args, "--allow-empty"); if (opts->allow_empty_message) - argv_array_push(&array, "--allow-empty-message"); + argv_array_push(&cmd.args, "--allow-empty-message"); - rc = run_command_v_opt_cd_env(array.argv, RUN_GIT_CMD, NULL, - (const char *const *)env.argv); - argv_array_clear(&array); - argv_array_clear(&env); - return rc; + return run_command(&cmd); } static int is_original_commit_empty(struct commit *commit) From 5ee5bc109bd48bf284a4dcadbb39a070f5542d39 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 12 Apr 2016 14:56:52 +0200 Subject: [PATCH 261/288] sequencer (rebase -i): show only failed `git commit`'s output This is the behavior of the shell script version of the interactive rebase, by using the `output` function defined in `git-rebase.sh`. Signed-off-by: Johannes Schindelin --- sequencer.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/sequencer.c b/sequencer.c index 4008134333..6f9b6cc398 100644 --- a/sequencer.c +++ b/sequencer.c @@ -657,6 +657,11 @@ static int run_git_commit(const char *defmsg, struct replay_opts *opts, cmd.git_cmd = 1; if (is_rebase_i(opts)) { + if (!edit) { + cmd.stdout_to_stderr = 1; + cmd.err = -1; + } + if (read_env_script(&cmd.env_array)) { const char *gpg_opt = gpg_sign_opt_quoted(opts); @@ -691,6 +696,19 @@ static int run_git_commit(const char *defmsg, struct replay_opts *opts, if (opts->allow_empty_message) argv_array_push(&cmd.args, "--allow-empty-message"); + if (cmd.err == -1) { + /* hide stderr on success */ + struct strbuf buf = STRBUF_INIT; + int rc = pipe_command(&cmd, + NULL, 0, + /* stdout is already redirected */ + NULL, 0, + &buf, 0); + if (rc) + fputs(buf.buf, stderr); + strbuf_release(&buf); + return rc; + } return run_command(&cmd); } From 52dc4a443c3ddb95cca328fc4dd8716b89714dff Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 12 Apr 2016 14:56:52 +0200 Subject: [PATCH 262/288] sequencer (rebase -i): show only failed cherry-picks' output This is the behavior of the shell script version of the interactive rebase, by using the `output` function defined in `git-rebase.sh`. Signed-off-by: Johannes Schindelin --- sequencer.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sequencer.c b/sequencer.c index 6f9b6cc398..364b5a28a9 100644 --- a/sequencer.c +++ b/sequencer.c @@ -480,6 +480,8 @@ static int do_recursive_merge(struct commit *base, struct commit *next, o.ancestor = base ? base_label : "(empty tree)"; o.branch1 = "HEAD"; o.branch2 = next ? next_label : "(empty tree)"; + if (is_rebase_i(opts)) + o.buffer_output = 2; head_tree = parse_tree_indirect(head); next_tree = next ? next->tree : empty_tree(); @@ -491,6 +493,8 @@ static int do_recursive_merge(struct commit *base, struct commit *next, clean = merge_trees(&o, head_tree, next_tree, base_tree, &result); + if (is_rebase_i(opts) && clean <= 0) + fputs(o.obuf.buf, stdout); strbuf_release(&o.obuf); if (clean < 0) return clean; From 786e0939238d92ded14a875faaba16b676cd51b2 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 21 Apr 2016 12:51:50 +0200 Subject: [PATCH 263/288] sequencer (rebase -i): suggest --edit-todo upon unknown command This is the same behavior as known from `git rebase -i`. Signed-off-by: Johannes Schindelin --- sequencer.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sequencer.c b/sequencer.c index 364b5a28a9..86282cee4b 100644 --- a/sequencer.c +++ b/sequencer.c @@ -1361,8 +1361,12 @@ static int read_populate_todo(struct todo_list *todo_list, close(fd); res = parse_insn_buffer(todo_list->buf.buf, todo_list); - if (res) + if (res) { + if (is_rebase_i(opts)) + return error(_("please fix this using " + "'git rebase --edit-todo'.")); return error(_("unusable instruction sheet: '%s'"), todo_file); + } if (!todo_list->nr && (!is_rebase_i(opts) || !file_exists(rebase_path_done()))) From 6705c9db256ac33b85e36b7378bee9d6e422a727 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 8 Apr 2016 15:51:19 +0200 Subject: [PATCH 264/288] sequencer (rebase -i): show the progress The interactive rebase keeps the user informed about its progress. If the sequencer wants to do the grunt work of the interactive rebase, it also needs to show that progress. Signed-off-by: Johannes Schindelin --- sequencer.c | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/sequencer.c b/sequencer.c index 86282cee4b..3b6ea5b2fd 100644 --- a/sequencer.c +++ b/sequencer.c @@ -1228,6 +1228,7 @@ struct todo_list { struct strbuf buf; struct todo_item *items; int nr, alloc, current; + int done_nr, total_nr; }; #define TODO_LIST_INIT { STRBUF_INIT } @@ -1344,6 +1345,17 @@ static int parse_insn_buffer(char *buf, struct todo_list *todo_list) return res; } +static int count_commands(struct todo_list *todo_list) +{ + int count = 0, i; + + for (i = 0; i < todo_list->nr; i++) + if (todo_list->items[i].command != TODO_COMMENT) + count++; + + return count; +} + static int read_populate_todo(struct todo_list *todo_list, struct replay_opts *opts) { @@ -1386,6 +1398,21 @@ static int read_populate_todo(struct todo_list *todo_list, return error(_("cannot revert during a cherry-pick.")); } + if (is_rebase_i(opts)) { + struct todo_list done = TODO_LIST_INIT; + + if (strbuf_read_file(&done.buf, rebase_path_done(), 0) > 0 && + !parse_insn_buffer(done.buf.buf, &done)) + todo_list->done_nr = count_commands(&done); + else + todo_list->done_nr = 0; + + todo_list->total_nr = todo_list->done_nr + + count_commands(todo_list); + + todo_list_release(&done); + } + return 0; } @@ -1967,6 +1994,11 @@ static int pick_commits(struct todo_list *todo_list, struct replay_opts *opts) if (save_todo(todo_list, opts)) return -1; if (is_rebase_i(opts)) { + if (item->command != TODO_COMMENT) + fprintf(stderr, "Rebasing (%d/%d)%s", + ++(todo_list->done_nr), + todo_list->total_nr, + opts->verbose ? "\n" : "\r"); unlink(rebase_path_message()); unlink(rebase_path_author_script()); unlink(rebase_path_stopped_sha()); From d2e2e1ecb2511c55f9e3ecef6cdb4dad11e71f90 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 8 Apr 2016 16:40:13 +0200 Subject: [PATCH 265/288] sequencer (rebase -i): write the progress into files For the benefit of e.g. the shell prompt, the interactive rebase not only displays the progress for the user to see, but also writes it into the msgnum/end files in the state directory. Teach the sequencer this new trick. Signed-off-by: Johannes Schindelin --- sequencer.c | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/sequencer.c b/sequencer.c index 3b6ea5b2fd..66b217a993 100644 --- a/sequencer.c +++ b/sequencer.c @@ -45,6 +45,16 @@ static GIT_PATH_FUNC(rebase_path_todo, "rebase-merge/git-rebase-todo") * actions. */ static GIT_PATH_FUNC(rebase_path_done, "rebase-merge/done") +/* + * The file to keep track of how many commands were already processed (e.g. + * for the prompt). + */ +static GIT_PATH_FUNC(rebase_path_msgnum, "rebase-merge/msgnum"); +/* + * The file to keep track of how many commands are to be processed in total + * (e.g. for the prompt). + */ +static GIT_PATH_FUNC(rebase_path_msgtotal, "rebase-merge/end"); /* * The commit message that is planned to be used for any changes that * need to be committed following a user interaction. @@ -1400,6 +1410,7 @@ static int read_populate_todo(struct todo_list *todo_list, if (is_rebase_i(opts)) { struct todo_list done = TODO_LIST_INIT; + FILE *f = fopen(rebase_path_msgtotal(), "w"); if (strbuf_read_file(&done.buf, rebase_path_done(), 0) > 0 && !parse_insn_buffer(done.buf.buf, &done)) @@ -1409,8 +1420,12 @@ static int read_populate_todo(struct todo_list *todo_list, todo_list->total_nr = todo_list->done_nr + count_commands(todo_list); - todo_list_release(&done); + + if (f) { + fprintf(f, "%d\n", todo_list->total_nr); + fclose(f); + } } return 0; @@ -1994,11 +2009,20 @@ static int pick_commits(struct todo_list *todo_list, struct replay_opts *opts) if (save_todo(todo_list, opts)) return -1; if (is_rebase_i(opts)) { - if (item->command != TODO_COMMENT) + if (item->command != TODO_COMMENT) { + FILE *f = fopen(rebase_path_msgnum(), "w"); + + todo_list->done_nr++; + + if (f) { + fprintf(f, "%d\n", todo_list->done_nr); + fclose(f); + } fprintf(stderr, "Rebasing (%d/%d)%s", - ++(todo_list->done_nr), + todo_list->done_nr, todo_list->total_nr, opts->verbose ? "\n" : "\r"); + } unlink(rebase_path_message()); unlink(rebase_path_author_script()); unlink(rebase_path_stopped_sha()); From 0bd5edc6de8c6132c63b5c50b7bfab2ef17c9803 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 8 Apr 2016 17:28:04 +0200 Subject: [PATCH 266/288] sequencer (rebase -i): write out the final message The shell script version of the interactive rebase has a very specific final message. Teach the sequencer to print the same. Signed-off-by: Johannes Schindelin --- sequencer.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sequencer.c b/sequencer.c index 66b217a993..7aad134b05 100644 --- a/sequencer.c +++ b/sequencer.c @@ -2174,6 +2174,9 @@ cleanup_head_ref: } apply_autostash(opts); + fprintf(stderr, "Successfully rebased and updated %s.\n", + head_ref.buf); + strbuf_release(&buf); strbuf_release(&head_ref); } From be15aaaa53433af54b3a6399b3e0cadfd3f32ac8 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 12 Apr 2016 16:02:27 +0200 Subject: [PATCH 267/288] Add a builtin helper for interactive rebases Git's interactive rebase is still implemented as a shell script, despite its complexity. This implies that it suffers from the portability point of view, from lack of expressibility, and of course also from performance. The latter issue is particularly serious on Windows, where we pay a hefty price for relying so much on POSIX. Unfortunately, being such a huge shell script also means that we missed the train when it would have been relatively easy to port it to C, and instead piled feature upon feature onto that poor script that originally never intended to be more than a slightly pimped cherry-pick in a loop. To open the road toward better performance (in addition to all the other benefits of C over shell scripts), let's just start *somewhere*. The approach taken here is to add a builtin helper that at first intends to take care of the parts of the interactive rebase that are most affected by the performance penalties mentioned above. In particular, after we spent all those efforts on preparing the sequencer to process rebase -i's git-rebase-todo scripts, we implement the `git rebase -i --continue` functionality as a new builtin, git-rebase--helper. Once that is in place, we can work gradually on tackling the rest of the technical debt. Note that the rebase--helper needs to learn about the transient --ff/--no-ff options of git-rebase, as the corresponding flag is not persisted to, and re-read from, the state directory. Signed-off-by: Johannes Schindelin --- .gitignore | 1 + Makefile | 1 + builtin.h | 1 + builtin/rebase--helper.c | 40 ++++++++++++++++++++++++++++++++++++++++ git.c | 1 + 5 files changed, 44 insertions(+) create mode 100644 builtin/rebase--helper.c diff --git a/.gitignore b/.gitignore index 6722f78f9a..2846ef91c1 100644 --- a/.gitignore +++ b/.gitignore @@ -114,6 +114,7 @@ /git-read-tree /git-rebase /git-rebase--am +/git-rebase--helper /git-rebase--interactive /git-rebase--merge /git-receive-pack diff --git a/Makefile b/Makefile index 76267262c1..3de0bcbb4f 100644 --- a/Makefile +++ b/Makefile @@ -929,6 +929,7 @@ BUILTIN_OBJS += builtin/prune.o BUILTIN_OBJS += builtin/pull.o BUILTIN_OBJS += builtin/push.o BUILTIN_OBJS += builtin/read-tree.o +BUILTIN_OBJS += builtin/rebase--helper.o BUILTIN_OBJS += builtin/receive-pack.o BUILTIN_OBJS += builtin/reflog.o BUILTIN_OBJS += builtin/remote.o diff --git a/builtin.h b/builtin.h index b9122bc5f4..6442c9bd6e 100644 --- a/builtin.h +++ b/builtin.h @@ -102,6 +102,7 @@ extern int cmd_prune_packed(int argc, const char **argv, const char *prefix); extern int cmd_pull(int argc, const char **argv, const char *prefix); extern int cmd_push(int argc, const char **argv, const char *prefix); extern int cmd_read_tree(int argc, const char **argv, const char *prefix); +extern int cmd_rebase__helper(int argc, const char **argv, const char *prefix); extern int cmd_receive_pack(int argc, const char **argv, const char *prefix); extern int cmd_reflog(int argc, const char **argv, const char *prefix); extern int cmd_remote(int argc, const char **argv, const char *prefix); diff --git a/builtin/rebase--helper.c b/builtin/rebase--helper.c new file mode 100644 index 0000000000..ca1ebb2fa1 --- /dev/null +++ b/builtin/rebase--helper.c @@ -0,0 +1,40 @@ +#include "builtin.h" +#include "cache.h" +#include "parse-options.h" +#include "sequencer.h" + +static const char * const builtin_rebase_helper_usage[] = { + N_("git rebase--helper []"), + NULL +}; + +int cmd_rebase__helper(int argc, const char **argv, const char *prefix) +{ + struct replay_opts opts = REPLAY_OPTS_INIT; + enum { + CONTINUE = 1, ABORT + } command = 0; + struct option options[] = { + OPT_BOOL(0, "ff", &opts.allow_ff, N_("allow fast-forward")), + OPT_CMDMODE(0, "continue", &command, N_("continue rebase"), + CONTINUE), + OPT_CMDMODE(0, "abort", &command, N_("abort rebase"), + ABORT), + OPT_END() + }; + + git_config(git_default_config, NULL); + + opts.action = REPLAY_INTERACTIVE_REBASE; + opts.allow_ff = 1; + opts.allow_empty = 1; + + argc = parse_options(argc, argv, NULL, options, + builtin_rebase_helper_usage, PARSE_OPT_KEEP_ARGV0); + + if (command == CONTINUE && argc == 1) + return !!sequencer_continue(&opts); + if (command == ABORT && argc == 1) + return !!sequencer_remove_state(&opts); + usage_with_options(builtin_rebase_helper_usage, options); +} diff --git a/git.c b/git.c index c8fe6637df..c4e6083284 100644 --- a/git.c +++ b/git.c @@ -472,6 +472,7 @@ static struct cmd_struct commands[] = { { "pull", cmd_pull, RUN_SETUP | NEED_WORK_TREE }, { "push", cmd_push, RUN_SETUP }, { "read-tree", cmd_read_tree, RUN_SETUP }, + { "rebase--helper", cmd_rebase__helper, RUN_SETUP | NEED_WORK_TREE }, { "receive-pack", cmd_receive_pack }, { "reflog", cmd_reflog, RUN_SETUP }, { "remote", cmd_remote, RUN_SETUP }, From a7167ae47d8a267ff13357400e0b8aaaef953d1d Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 28 Mar 2016 13:34:50 +0200 Subject: [PATCH 268/288] rebase -i: use the rebase--helper builtin Now that the sequencer learned to process a "normal" interactive rebase, we use it. The original shell script is still used for "non-normal" interactive rebases, i.e. when --root or --preserve-merges was passed. Please note that the --root option (via the $squash_onto variable) needs special handling only for the very first command, hence it is still okay to use the helper upon continue/skip. Also please note that the --no-ff setting is volatile, i.e. when the interactive rebase is interrupted at any stage, there is no record of it. Therefore, we have to pass it from the shell script to the rebase--helper. Note: the test t3404 had to be adjusted because the the error messages produced by the sequencer comply with our current convention to start with a lower-case letter. Signed-off-by: Johannes Schindelin --- git-rebase--interactive.sh | 13 +++++++++++++ t/t3404-rebase-interactive.sh | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/git-rebase--interactive.sh b/git-rebase--interactive.sh index f5f58c483a..c2ff97d865 100644 --- a/git-rebase--interactive.sh +++ b/git-rebase--interactive.sh @@ -1068,6 +1068,10 @@ git_rebase__interactive () { case "$action" in continue) + if test ! -d "$rewritten" + then + exec git rebase--helper ${force_rebase:+--no-ff} --continue + fi # do we have anything to commit? if git diff-index --cached --quiet HEAD -- then @@ -1127,6 +1131,10 @@ first and then run 'git rebase --continue' again.")" skip) git rerere clear + if test ! -d "$rewritten" + then + exec git rebase--helper ${force_rebase:+--no-ff} --continue + fi do_rest return 0 ;; @@ -1313,6 +1321,11 @@ expand_todo_ids test -d "$rewritten" || test -n "$force_rebase" || skip_unnecessary_picks checkout_onto +if test -z "$rebase_root" && test ! -d "$rewritten" +then + require_clean_work_tree "rebase" + exec git rebase--helper ${force_rebase:+--no-ff} --continue +fi do_rest } diff --git a/t/t3404-rebase-interactive.sh b/t/t3404-rebase-interactive.sh index e2f18d11f6..33d392ba11 100755 --- a/t/t3404-rebase-interactive.sh +++ b/t/t3404-rebase-interactive.sh @@ -556,7 +556,7 @@ test_expect_success 'clean error after failed "exec"' ' echo "edited again" > file7 && git add file7 && test_must_fail git rebase --continue 2>error && - test_i18ngrep "You have staged changes in your working tree." error + test_i18ngrep "you have staged changes in your working tree" error ' test_expect_success 'rebase a detached HEAD' ' From 984b68fddaede9fdfb1b887d2d5ca37609987e60 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 13 Apr 2016 17:43:05 +0200 Subject: [PATCH 269/288] rebase -i: generate the script via rebase--helper The first step of an interactive rebase is to generate the so-called "todo script", to be stored in the state directory as "git-rebase-todo" and to be edited by the user. Originally, we adjusted the output of `git log ` using a simple sed script. Over the course of the years, the code became more complicated. We now use shell scripting to edit the output of `git log` conditionally, depending whether to keep "empty" commits (i.e. commits that do not change any files). On platforms where shell scripting is not native, this can be a serious drag. And it opens the door for incompatibilities between platforms when it comes to shell scripting or to Unix-y commands. Let's just re-implement the todo script generation in plain C, using the revision machinery directly. This is substantially faster, improving the speed relative to the shell script version of the interactive rebase from 2x to 3x on Windows. Note that the rearrange_squash() function in git-rebase--interactive relied on the fact that we set the "format" variable to the config setting rebase.instructionFormat. Relying on a side effect like this is no good, hence we explicitly perform that assignment (possibly again) in rearrange_squash(). Signed-off-by: Johannes Schindelin --- builtin/rebase--helper.c | 8 ++++++- git-rebase--interactive.sh | 42 +++++++++++++++++++----------------- sequencer.c | 44 ++++++++++++++++++++++++++++++++++++++ sequencer.h | 3 +++ 4 files changed, 76 insertions(+), 21 deletions(-) diff --git a/builtin/rebase--helper.c b/builtin/rebase--helper.c index ca1ebb2fa1..821058d452 100644 --- a/builtin/rebase--helper.c +++ b/builtin/rebase--helper.c @@ -11,15 +11,19 @@ static const char * const builtin_rebase_helper_usage[] = { int cmd_rebase__helper(int argc, const char **argv, const char *prefix) { struct replay_opts opts = REPLAY_OPTS_INIT; + int keep_empty = 0; enum { - CONTINUE = 1, ABORT + CONTINUE = 1, ABORT, MAKE_SCRIPT } command = 0; struct option options[] = { OPT_BOOL(0, "ff", &opts.allow_ff, N_("allow fast-forward")), + OPT_BOOL(0, "keep-empty", &keep_empty, N_("keep empty commits")), OPT_CMDMODE(0, "continue", &command, N_("continue rebase"), CONTINUE), OPT_CMDMODE(0, "abort", &command, N_("abort rebase"), ABORT), + OPT_CMDMODE(0, "make-script", &command, + N_("make rebase script"), MAKE_SCRIPT), OPT_END() }; @@ -36,5 +40,7 @@ int cmd_rebase__helper(int argc, const char **argv, const char *prefix) return !!sequencer_continue(&opts); if (command == ABORT && argc == 1) return !!sequencer_remove_state(&opts); + if (command == MAKE_SCRIPT && argc > 1) + return !!sequencer_make_script(keep_empty, stdout, argc, argv); usage_with_options(builtin_rebase_helper_usage, options); } diff --git a/git-rebase--interactive.sh b/git-rebase--interactive.sh index c2ff97d865..ea406052df 100644 --- a/git-rebase--interactive.sh +++ b/git-rebase--interactive.sh @@ -784,6 +784,7 @@ collapse_todo_ids() { # each log message will be re-retrieved in order to normalize the # autosquash arrangement rearrange_squash () { + format=$(git config --get rebase.instructionFormat) # extract fixup!/squash! lines and resolve any referenced sha1's while read -r pick sha1 message do @@ -1209,26 +1210,27 @@ else revisions=$onto...$orig_head shortrevisions=$shorthead fi -format=$(git config --get rebase.instructionFormat) -# the 'rev-list .. | sed' requires %m to parse; the instruction requires %H to parse -git rev-list $merges_option --format="%m%H ${format:-%s}" \ - --reverse --left-right --topo-order \ - $revisions ${restrict_revision+^$restrict_revision} | \ - sed -n "s/^>//p" | -while read -r sha1 rest -do +if test t != "$preserve_merges" +then + git rebase--helper --make-script ${keep_empty:+--keep-empty} \ + $revisions ${restrict_revision+^$restrict_revision} >"$todo" +else + format=$(git config --get rebase.instructionFormat) + # the 'rev-list .. | sed' requires %m to parse; the instruction requires %H to parse + git rev-list $merges_option --format="%m%H ${format:-%s}" \ + --reverse --left-right --topo-order \ + $revisions ${restrict_revision+^$restrict_revision} | \ + sed -n "s/^>//p" | + while read -r sha1 rest + do - if test -z "$keep_empty" && is_empty_commit $sha1 && ! is_merge_commit $sha1 - then - comment_out="$comment_char " - else - comment_out= - fi + if test -z "$keep_empty" && is_empty_commit $sha1 && ! is_merge_commit $sha1 + then + comment_out="$comment_char " + else + comment_out= + fi - if test t != "$preserve_merges" - then - printf '%s\n' "${comment_out}pick $sha1 $rest" >>"$todo" - else if test -z "$rebase_root" then preserve=t @@ -1247,8 +1249,8 @@ do touch "$rewritten"/$sha1 printf '%s\n' "${comment_out}pick $sha1 $rest" >>"$todo" fi - fi -done + done +fi # Watch for commits that been dropped by --cherry-pick if test t = "$preserve_merges" diff --git a/sequencer.c b/sequencer.c index 7aad134b05..031a99697c 100644 --- a/sequencer.c +++ b/sequencer.c @@ -2426,3 +2426,47 @@ void append_signoff(struct strbuf *msgbuf, int ignore_footer, unsigned flag) strbuf_release(&sob); } + +int sequencer_make_script(int keep_empty, FILE *out, + int argc, const char **argv) +{ + char *format = "%s"; + struct pretty_print_context pp = {0}; + struct strbuf buf = STRBUF_INIT; + struct rev_info revs; + struct commit *commit; + + init_revisions(&revs, NULL); + revs.verbose_header = 1; + revs.max_parents = 1; + revs.cherry_pick = 1; + revs.limited = 1; + revs.reverse = 1; + revs.right_only = 1; + revs.sort_order = REV_SORT_IN_GRAPH_ORDER; + revs.topo_order = 1; + + revs.pretty_given = 1; + git_config_get_string("rebase.instructionFormat", &format); + get_commit_format(format, &revs); + pp.fmt = revs.commit_format; + pp.output_encoding = get_log_output_encoding(); + + if (setup_revisions(argc, argv, &revs, NULL) > 1) + return error(_("make_script: unhandled options")); + + if (prepare_revision_walk(&revs) < 0) + return error(_("make_script: error preparing revisions")); + + while ((commit = get_revision(&revs))) { + strbuf_reset(&buf); + if (!keep_empty && is_original_commit_empty(commit)) + strbuf_addf(&buf, "%c ", comment_line_char); + strbuf_addf(&buf, "pick %s ", oid_to_hex(&commit->object.oid)); + pretty_print_commit(&pp, commit, &buf); + strbuf_addch(&buf, '\n'); + fputs(buf.buf, out); + } + strbuf_release(&buf); + return 0; +} diff --git a/sequencer.h b/sequencer.h index f885b68395..83f2943b7a 100644 --- a/sequencer.h +++ b/sequencer.h @@ -45,6 +45,9 @@ int sequencer_continue(struct replay_opts *opts); int sequencer_rollback(struct replay_opts *opts); int sequencer_remove_state(struct replay_opts *opts); +int sequencer_make_script(int keep_empty, FILE *out, + int argc, const char **argv); + extern const char sign_off_header[]; void append_signoff(struct strbuf *msgbuf, int ignore_footer, unsigned flag); From bfe87426a831648cd7b6cb9b493eb3dce196fbf2 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 15 Apr 2016 10:00:39 +0200 Subject: [PATCH 270/288] rebase -i: remove useless indentation The commands used to be indented, and it is nice to look at, but when we transform the SHA-1s, the indentation is removed. So let's do away with it. For the moment, at least: when we will use the upcoming rebase--helper to transform the SHA-1s, we *will* keep the indentation and can reintroduce it. Yet, to be able to validate the rebase--helper against the output of the current shell script version, we need to remove the extra indentation. Signed-off-by: Johannes Schindelin --- git-rebase--interactive.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/git-rebase--interactive.sh b/git-rebase--interactive.sh index ea406052df..c96485e503 100644 --- a/git-rebase--interactive.sh +++ b/git-rebase--interactive.sh @@ -155,13 +155,13 @@ reschedule_last_action () { append_todo_help () { gettext " Commands: - p, pick = use commit - r, reword = use commit, but edit the commit message - e, edit = use commit, but stop for amending - s, squash = use commit, but meld into previous commit - f, fixup = like \"squash\", but discard this commit's log message - x, exec = run command (the rest of the line) using shell - d, drop = remove commit +p, pick = use commit +r, reword = use commit, but edit the commit message +e, edit = use commit, but stop for amending +s, squash = use commit, but meld into previous commit +f, fixup = like \"squash\", but discard this commit's log message +x, exec = run command (the rest of the line) using shell +d, drop = remove commit These lines can be re-ordered; they are executed from top to bottom. " | git stripspace --comment-lines >>"$todo" From b55a1562d9a0f49d155084521a540a8c59b801da Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sun, 17 Apr 2016 08:48:44 +0200 Subject: [PATCH 271/288] rebase -i: do not invent onelines when expanding/collapsing SHA-1s To avoid problems with short SHA-1s that become non-unique during the rebase, we rewrite the todo script with short/long SHA-1s before and after letting the user edit the script. Since SHA-1s are not intuitive for humans, rebase -i also provides the onelines (commit message subjects) in the script, purely for the user's convenience. It is very possible to generate a todo script via different means than rebase -i and then to let rebase -i run with it; In this case, these onelines are not required. And this is where the expand/collapse machinery has a bug: it *expects* that oneline, and failing to find one reuses the previous SHA-1 as "oneline". It was most likely an oversight, and made implementation in the (quite limiting) shell script language less convoluted. However, we are about to reimplement performance-critical parts in C (and due to spawning a git.exe process for every single line of the todo script, the expansion/collapsing of the SHA-1s *is* performance-hampering on Windows), therefore let's fix this bug to make cross-validation with the C version of that functionality possible. Signed-off-by: Johannes Schindelin --- git-rebase--interactive.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/git-rebase--interactive.sh b/git-rebase--interactive.sh index c96485e503..0cfde2189e 100644 --- a/git-rebase--interactive.sh +++ b/git-rebase--interactive.sh @@ -759,7 +759,12 @@ transform_todo_ids () { ;; *) sha1=$(git rev-parse --verify --quiet "$@" ${rest%%[ ]*}) && - rest="$sha1 ${rest#*[ ]}" + if test "a$rest" = "a${rest#*[ ]}" + then + rest=$sha1 + else + rest="$sha1 ${rest#*[ ]}" + fi ;; esac printf '%s\n' "$command${rest:+ }$rest" From 8a06c978fd7f1c4c181d6ca41abe87702dade948 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 13 Apr 2016 18:17:19 +0200 Subject: [PATCH 272/288] rebase -i: also expand/collapse the SHA-1s via the rebase--helper This is crucial to improve performance on Windows, as the speed is now mostly dominated by the SHA-1 transformation (because it spawns a new rev-parse process for *every* line, and spawning processes is pretty slow from Git for Windows' MSYS2 Bash). Signed-off-by: Johannes Schindelin --- builtin/rebase--helper.c | 10 ++++++- git-rebase--interactive.sh | 4 +-- sequencer.c | 59 ++++++++++++++++++++++++++++++++++++++ sequencer.h | 2 ++ 4 files changed, 72 insertions(+), 3 deletions(-) diff --git a/builtin/rebase--helper.c b/builtin/rebase--helper.c index 821058d452..9444c8d6c6 100644 --- a/builtin/rebase--helper.c +++ b/builtin/rebase--helper.c @@ -13,7 +13,7 @@ int cmd_rebase__helper(int argc, const char **argv, const char *prefix) struct replay_opts opts = REPLAY_OPTS_INIT; int keep_empty = 0; enum { - CONTINUE = 1, ABORT, MAKE_SCRIPT + CONTINUE = 1, ABORT, MAKE_SCRIPT, SHORTEN_SHA1S, EXPAND_SHA1S } command = 0; struct option options[] = { OPT_BOOL(0, "ff", &opts.allow_ff, N_("allow fast-forward")), @@ -24,6 +24,10 @@ int cmd_rebase__helper(int argc, const char **argv, const char *prefix) ABORT), OPT_CMDMODE(0, "make-script", &command, N_("make rebase script"), MAKE_SCRIPT), + OPT_CMDMODE(0, "shorten-sha1s", &command, + N_("shorten SHA-1s in the todo list"), SHORTEN_SHA1S), + OPT_CMDMODE(0, "expand-sha1s", &command, + N_("expand SHA-1s in the todo list"), EXPAND_SHA1S), OPT_END() }; @@ -42,5 +46,9 @@ int cmd_rebase__helper(int argc, const char **argv, const char *prefix) return !!sequencer_remove_state(&opts); if (command == MAKE_SCRIPT && argc > 1) return !!sequencer_make_script(keep_empty, stdout, argc, argv); + if (command == SHORTEN_SHA1S && argc == 1) + return !!transform_todo_ids(1); + if (command == EXPAND_SHA1S && argc == 1) + return !!transform_todo_ids(0); usage_with_options(builtin_rebase_helper_usage, options); } diff --git a/git-rebase--interactive.sh b/git-rebase--interactive.sh index 0cfde2189e..9c7f654ef2 100644 --- a/git-rebase--interactive.sh +++ b/git-rebase--interactive.sh @@ -773,11 +773,11 @@ transform_todo_ids () { } expand_todo_ids() { - transform_todo_ids + git rebase--helper --expand-sha1s } collapse_todo_ids() { - transform_todo_ids --short + git rebase--helper --shorten-sha1s } # Rearrange the todo list that has both "pick sha1 msg" and diff --git a/sequencer.c b/sequencer.c index 031a99697c..687b0549c2 100644 --- a/sequencer.c +++ b/sequencer.c @@ -2470,3 +2470,62 @@ int sequencer_make_script(int keep_empty, FILE *out, strbuf_release(&buf); return 0; } + + +int transform_todo_ids(int shorten_sha1s) +{ + const char *todo_file = rebase_path_todo(); + struct todo_list todo_list = TODO_LIST_INIT; + int fd, res, i; + FILE *out; + + strbuf_reset(&todo_list.buf); + fd = open(todo_file, O_RDONLY); + if (fd < 0) + return error_errno(_("could not open '%s'"), todo_file); + if (strbuf_read(&todo_list.buf, fd, 0) < 0) { + close(fd); + return error(_("could not read '%s'."), todo_file); + } + close(fd); + + res = parse_insn_buffer(todo_list.buf.buf, &todo_list); + if (res) { + todo_list_release(&todo_list); + return error(_("unusable instruction sheet: '%s'"), todo_file); + } + + out = fopen(todo_file, "w"); + if (!out) { + todo_list_release(&todo_list); + return error(_("unable to open '%s' for writing"), todo_file); + } + for (i = 0; i < todo_list.nr; i++) { + struct todo_item *item = todo_list.items + i; + int bol = item->offset_in_buf; + const char *p = todo_list.buf.buf + bol; + int eol = i + 1 < todo_list.nr ? + todo_list.items[i + 1].offset_in_buf : + todo_list.buf.len; + + if (item->command >= TODO_EXEC && item->command != TODO_DROP) + fwrite(p, eol - bol, 1, out); + else { + int eoc = strcspn(p, " \t"); + const char *sha1 = shorten_sha1s ? + short_commit_name(item->commit) : + oid_to_hex(&item->commit->object.oid); + + if (!eoc) { + p += strspn(p, " \t"); + eoc = strcspn(p, " \t"); + } + + fprintf(out, "%.*s %s %.*s\n", + eoc, p, sha1, item->arg_len, item->arg); + } + } + fclose(out); + todo_list_release(&todo_list); + return 0; +} diff --git a/sequencer.h b/sequencer.h index 83f2943b7a..47a81034e7 100644 --- a/sequencer.h +++ b/sequencer.h @@ -48,6 +48,8 @@ int sequencer_remove_state(struct replay_opts *opts); int sequencer_make_script(int keep_empty, FILE *out, int argc, const char **argv); +int transform_todo_ids(int shorten_sha1s); + extern const char sign_off_header[]; void append_signoff(struct strbuf *msgbuf, int ignore_footer, unsigned flag); From 6343e836fafda454ce8cda52ebe9e40417d8d1eb Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 14 Jun 2016 16:54:43 +0200 Subject: [PATCH 273/288] t3404: relax rebase.missingCommitsCheck tests These tests were a bit anal about the *exact* warning/error message printed by git rebase. But those messages are intended for the *end user*, therefore it does not make sense to test so rigidly for the *exact* wording. In the following, we will reimplement the missing commits check in the sequencer, with slightly different words. So let's just test for the parts in the warning/error message that we *really* care about, nothing more, nothing less. Signed-off-by: Johannes Schindelin --- t/t3404-rebase-interactive.sh | 22 ++++------------------ 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/t/t3404-rebase-interactive.sh b/t/t3404-rebase-interactive.sh index 33d392ba11..61113be08a 100755 --- a/t/t3404-rebase-interactive.sh +++ b/t/t3404-rebase-interactive.sh @@ -1242,20 +1242,13 @@ test_expect_success 'rebase -i respects rebase.missingCommitsCheck = error' ' test B = $(git cat-file commit HEAD^ | sed -ne \$p) ' -cat >expect <actual && - test_i18ncmp expect actual && + test_i18ngrep "badcmd $(git rev-list --oneline -1 master~1)" actual && + test_i18ngrep "You can fix this with .git rebase --edit-todo.." actual && FAKE_LINES="1 2 3 drop 4 5" git rebase --edit-todo && git rebase --continue && test E = $(git cat-file commit HEAD | sed -ne \$p) && @@ -1277,20 +1270,13 @@ test_expect_success 'tabs and spaces are accepted in the todolist' ' test E = $(git cat-file commit HEAD | sed -ne \$p) ' -cat >expect <actual && - test_i18ncmp expect actual && + test_i18ngrep "edit XXXXXXX False commit" actual && + test_i18ngrep "You can fix this with .git rebase --edit-todo.." actual && FAKE_LINES="1 2 4 5 6" git rebase --edit-todo && git rebase --continue && test E = $(git cat-file commit HEAD | sed -ne \$p) From 94242bb69113d0fc15c1c3c5fbfe0cfcf87127d0 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 14 Jun 2016 16:58:59 +0200 Subject: [PATCH 274/288] rebase -i: check for missing commits in the rebase--helper In particular on Windows, where shell scripts are even more expensive than on MacOSX or Linux, it makes sense to move a loop that forks Git at least once for every line in the todo list into a builtin. Signed-off-by: Johannes Schindelin --- builtin/rebase--helper.c | 7 +- git-rebase--interactive.sh | 164 ++----------------------------------- sequencer.c | 125 ++++++++++++++++++++++++++++ sequencer.h | 1 + 4 files changed, 137 insertions(+), 160 deletions(-) diff --git a/builtin/rebase--helper.c b/builtin/rebase--helper.c index 9444c8d6c6..e706eac710 100644 --- a/builtin/rebase--helper.c +++ b/builtin/rebase--helper.c @@ -13,7 +13,8 @@ int cmd_rebase__helper(int argc, const char **argv, const char *prefix) struct replay_opts opts = REPLAY_OPTS_INIT; int keep_empty = 0; enum { - CONTINUE = 1, ABORT, MAKE_SCRIPT, SHORTEN_SHA1S, EXPAND_SHA1S + CONTINUE = 1, ABORT, MAKE_SCRIPT, SHORTEN_SHA1S, EXPAND_SHA1S, + CHECK_TODO_LIST } command = 0; struct option options[] = { OPT_BOOL(0, "ff", &opts.allow_ff, N_("allow fast-forward")), @@ -28,6 +29,8 @@ int cmd_rebase__helper(int argc, const char **argv, const char *prefix) N_("shorten SHA-1s in the todo list"), SHORTEN_SHA1S), OPT_CMDMODE(0, "expand-sha1s", &command, N_("expand SHA-1s in the todo list"), EXPAND_SHA1S), + OPT_CMDMODE(0, "check-todo-list", &command, + N_("check the todo list"), CHECK_TODO_LIST), OPT_END() }; @@ -50,5 +53,7 @@ int cmd_rebase__helper(int argc, const char **argv, const char *prefix) return !!transform_todo_ids(1); if (command == EXPAND_SHA1S && argc == 1) return !!transform_todo_ids(0); + if (command == CHECK_TODO_LIST && argc == 1) + return !!check_todo_list(); usage_with_options(builtin_rebase_helper_usage, options); } diff --git a/git-rebase--interactive.sh b/git-rebase--interactive.sh index 9c7f654ef2..e3022690e6 100644 --- a/git-rebase--interactive.sh +++ b/git-rebase--interactive.sh @@ -889,96 +889,6 @@ add_exec_commands () { mv "$1.new" "$1" } -# Check if the SHA-1 passed as an argument is a -# correct one, if not then print $2 in "$todo".badsha -# $1: the SHA-1 to test -# $2: the line number of the input -# $3: the input filename -check_commit_sha () { - badsha=0 - if test -z "$1" - then - badsha=1 - else - sha1_verif="$(git rev-parse --verify --quiet $1^{commit})" - if test -z "$sha1_verif" - then - badsha=1 - fi - fi - - if test $badsha -ne 0 - then - line="$(sed -n -e "${2}p" "$3")" - warn "$(eval_gettext "\ -Warning: the SHA-1 is missing or isn't a commit in the following line: - - \$line")" - warn - fi - - return $badsha -} - -# prints the bad commits and bad commands -# from the todolist in stdin -check_bad_cmd_and_sha () { - retval=0 - lineno=0 - while read -r command rest - do - lineno=$(( $lineno + 1 )) - case $command in - "$comment_char"*|''|noop|x|exec) - # Doesn't expect a SHA-1 - ;; - "$cr") - # Work around CR left by "read" (e.g. with Git for - # Windows' Bash). - ;; - pick|p|drop|d|reword|r|edit|e|squash|s|fixup|f) - if ! check_commit_sha "${rest%%[ ]*}" "$lineno" "$1" - then - retval=1 - fi - ;; - *) - line="$(sed -n -e "${lineno}p" "$1")" - warn "$(eval_gettext "\ -Warning: the command isn't recognized in the following line: - - \$line")" - warn - retval=1 - ;; - esac - done <"$1" - return $retval -} - -# Print the list of the SHA-1 of the commits -# from stdin to stdout -todo_list_to_sha_list () { - git stripspace --strip-comments | - while read -r command sha1 rest - do - case $command in - "$comment_char"*|''|noop|x|"exec") - ;; - *) - long_sha=$(git rev-list --no-walk "$sha1" 2>/dev/null) - printf "%s\n" "$long_sha" - ;; - esac - done -} - -# Use warn for each line in stdin -warn_lines () { - while read -r line - do - warn " - $line" - done -} - # Switch to the branch in $into and notify it in the reflog checkout_onto () { GIT_REFLOG_ACTION="$GIT_REFLOG_ACTION: checkout $onto_name" @@ -993,74 +903,6 @@ get_missing_commit_check_level () { printf '%s' "$check_level" | tr 'A-Z' 'a-z' } -# Check if the user dropped some commits by mistake -# Behaviour determined by rebase.missingCommitsCheck. -# Check if there is an unrecognized command or a -# bad SHA-1 in a command. -check_todo_list () { - raise_error=f - - check_level=$(get_missing_commit_check_level) - - case "$check_level" in - warn|error) - # Get the SHA-1 of the commits - todo_list_to_sha_list <"$todo".backup >"$todo".oldsha1 - todo_list_to_sha_list <"$todo" >"$todo".newsha1 - - # Sort the SHA-1 and compare them - sort -u "$todo".oldsha1 >"$todo".oldsha1+ - mv "$todo".oldsha1+ "$todo".oldsha1 - sort -u "$todo".newsha1 >"$todo".newsha1+ - mv "$todo".newsha1+ "$todo".newsha1 - comm -2 -3 "$todo".oldsha1 "$todo".newsha1 >"$todo".miss - - # Warn about missing commits - if test -s "$todo".miss - then - test "$check_level" = error && raise_error=t - - warn "$(gettext "\ -Warning: some commits may have been dropped accidentally. -Dropped commits (newer to older):")" - - # Make the list user-friendly and display - opt="--no-walk=sorted --format=oneline --abbrev-commit --stdin" - git rev-list $opt <"$todo".miss | warn_lines - - warn "$(gettext "\ -To avoid this message, use \"drop\" to explicitly remove a commit. - -Use 'git config rebase.missingCommitsCheck' to change the level of warnings. -The possible behaviours are: ignore, warn, error.")" - warn - fi - ;; - ignore) - ;; - *) - warn "$(eval_gettext "Unrecognized setting \$check_level for option rebase.missingCommitsCheck. Ignoring.")" - ;; - esac - - if ! check_bad_cmd_and_sha "$todo" - then - raise_error=t - fi - - if test $raise_error = t - then - # Checkout before the first commit of the - # rebase: this way git rebase --continue - # will work correctly as it expects HEAD to be - # placed before the commit of the next action - checkout_onto - - warn "$(gettext "You can fix this with 'git rebase --edit-todo' and then run 'git rebase --continue'.")" - die "$(gettext "Or you can abort the rebase with 'git rebase --abort'.")" - fi -} - # The whole contents of this file is run by dot-sourcing it from # inside a shell function. It used to be that "return"s we see # below were not inside any function, and expected to return @@ -1321,7 +1163,11 @@ git_sequence_editor "$todo" || has_action "$todo" || return 2 -check_todo_list +git rebase--helper --check-todo-list || { + ret=$? + checkout_onto + exit $ret +} expand_todo_ids diff --git a/sequencer.c b/sequencer.c index 687b0549c2..7c8338133c 100644 --- a/sequencer.c +++ b/sequencer.c @@ -2529,3 +2529,128 @@ int transform_todo_ids(int shorten_sha1s) todo_list_release(&todo_list); return 0; } + +enum check_level { + CHECK_IGNORE = 0, CHECK_WARN, CHECK_ERROR +}; + +static enum check_level get_missing_commit_check_level(void) +{ + const char *value; + + if (git_config_get_value("rebase.missingcommitscheck", &value) || + !strcasecmp("ignore", value)) + return CHECK_IGNORE; + if (!strcasecmp("warn", value)) + return CHECK_WARN; + if (!strcasecmp("error", value)) + return CHECK_ERROR; + warning(_("unrecognized setting %s for option" + "rebase.missingCommitsCheck. Ignoring."), value); + return CHECK_IGNORE; +} + +/* + * Check if the user dropped some commits by mistake + * Behaviour determined by rebase.missingCommitsCheck. + * Check if there is an unrecognized command or a + * bad SHA-1 in a command. + */ +int check_todo_list(void) +{ + enum check_level check_level = get_missing_commit_check_level(); + struct strbuf todo_file = STRBUF_INIT; + struct todo_list todo_list = TODO_LIST_INIT; + struct commit_list *missing = NULL; + int raise_error = 0, res = 0, fd, i; + + strbuf_addstr(&todo_file, rebase_path_todo()); + fd = open(todo_file.buf, O_RDONLY); + if (fd < 0) { + res = error_errno(_("could not open '%s'"), todo_file.buf); + goto leave_check; + } + if (strbuf_read(&todo_list.buf, fd, 0) < 0) { + close(fd); + res = error(_("could not read '%s'."), todo_file.buf); + goto leave_check; + } + close(fd); + raise_error = res = + parse_insn_buffer(todo_list.buf.buf, &todo_list); + + if (check_level == CHECK_IGNORE) + goto leave_check; + + /* Get the SHA-1 of the commits */ + for (i = 0; i < todo_list.nr; i++) { + struct commit *commit = todo_list.items[i].commit; + if (commit) + commit->util = todo_list.items + i; + } + + todo_list_release(&todo_list); + strbuf_addstr(&todo_file, ".backup"); + fd = open(todo_file.buf, O_RDONLY); + if (fd < 0) { + res = error_errno(_("could not open '%s'"), todo_file.buf); + goto leave_check; + } + if (strbuf_read(&todo_list.buf, fd, 0) < 0) { + close(fd); + res = error(_("could not read '%s'."), todo_file.buf); + goto leave_check; + } + close(fd); + strbuf_release(&todo_file); + res = !!parse_insn_buffer(todo_list.buf.buf, &todo_list); + + /* Find commits that are missing after editing */ + for (i = 0; i < todo_list.nr; i++) { + struct commit *commit = todo_list.items[i].commit; + if (commit && !commit->util) { + commit_list_insert(commit, &missing); + commit->util = todo_list.items + i; + } + } + + /* Warn about missing commits */ + if (!missing) + goto leave_check; + + if (check_level == CHECK_ERROR) + raise_error = res = 1; + + fprintf(stderr, + _("Warning: some commits may have been dropped accidentally.\n" + "Dropped commits (newer to older):\n")); + + /* Make the list user-friendly and display */ + while (missing) { + struct commit *commit = pop_commit(&missing); + struct todo_item *item = commit->util; + + fprintf(stderr, " - %s %.*s\n", short_commit_name(commit), + item->arg_len, item->arg); + } + free_commit_list(missing); + + fprintf(stderr, _("To avoid this message, use \"drop\" to " + "explicitly remove a commit.\n\n" + "Use 'git config rebase.missingCommitsCheck' to change " + "the level of warnings.\n" + "The possible behaviours are: ignore, warn, error.\n\n")); + +leave_check: + strbuf_release(&todo_file); + todo_list_release(&todo_list); + + if (raise_error) + fprintf(stderr, + _("You can fix this with 'git rebase --edit-todo' " + "and then run 'git rebase --continue'.\n" + "Or you can abort the rebase with 'git rebase" + " --abort'.\n")); + + return res; +} diff --git a/sequencer.h b/sequencer.h index 47a81034e7..4978a61b83 100644 --- a/sequencer.h +++ b/sequencer.h @@ -49,6 +49,7 @@ int sequencer_make_script(int keep_empty, FILE *out, int argc, const char **argv); int transform_todo_ids(int shorten_sha1s); +int check_todo_list(void); extern const char sign_off_header[]; From 212ccece788905bd05d58c8ba0b9b3a46e5df9a4 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 15 Jun 2016 09:09:09 +0200 Subject: [PATCH 275/288] rebase -i: skip unnecessary picks using the rebase--helper In particular on Windows, where shell scripts are even more expensive than on MacOSX or Linux, it makes sense to move a loop that forks Git at least once for every line in the todo list into a builtin. Note: The original code did not try to skip unnecessary picks of root commits but punts instead (probably --root was not considered common enough of a use case to bother optimizing). We do the same, for now. Signed-off-by: Johannes Schindelin --- builtin/rebase--helper.c | 6 ++- git-rebase--interactive.sh | 41 ++--------------- sequencer.c | 90 ++++++++++++++++++++++++++++++++++++++ sequencer.h | 1 + 4 files changed, 99 insertions(+), 39 deletions(-) diff --git a/builtin/rebase--helper.c b/builtin/rebase--helper.c index e706eac710..de3ccd9bfb 100644 --- a/builtin/rebase--helper.c +++ b/builtin/rebase--helper.c @@ -14,7 +14,7 @@ int cmd_rebase__helper(int argc, const char **argv, const char *prefix) int keep_empty = 0; enum { CONTINUE = 1, ABORT, MAKE_SCRIPT, SHORTEN_SHA1S, EXPAND_SHA1S, - CHECK_TODO_LIST + CHECK_TODO_LIST, SKIP_UNNECESSARY_PICKS } command = 0; struct option options[] = { OPT_BOOL(0, "ff", &opts.allow_ff, N_("allow fast-forward")), @@ -31,6 +31,8 @@ int cmd_rebase__helper(int argc, const char **argv, const char *prefix) N_("expand SHA-1s in the todo list"), EXPAND_SHA1S), OPT_CMDMODE(0, "check-todo-list", &command, N_("check the todo list"), CHECK_TODO_LIST), + OPT_CMDMODE(0, "skip-unnecessary-picks", &command, + N_("skip unnecessary picks"), SKIP_UNNECESSARY_PICKS), OPT_END() }; @@ -55,5 +57,7 @@ int cmd_rebase__helper(int argc, const char **argv, const char *prefix) return !!transform_todo_ids(0); if (command == CHECK_TODO_LIST && argc == 1) return !!check_todo_list(); + if (command == SKIP_UNNECESSARY_PICKS && argc == 1) + return !!skip_unnecessary_picks(); usage_with_options(builtin_rebase_helper_usage, options); } diff --git a/git-rebase--interactive.sh b/git-rebase--interactive.sh index e3022690e6..168c041e54 100644 --- a/git-rebase--interactive.sh +++ b/git-rebase--interactive.sh @@ -712,43 +712,6 @@ do_rest () { done } -# skip picking commits whose parents are unchanged -skip_unnecessary_picks () { - fd=3 - while read -r command rest - do - # fd=3 means we skip the command - case "$fd,$command" in - 3,pick|3,p) - # pick a commit whose parent is current $onto -> skip - sha1=${rest%% *} - case "$(git rev-parse --verify --quiet "$sha1"^)" in - "$onto"*) - onto=$sha1 - ;; - *) - fd=1 - ;; - esac - ;; - 3,"$comment_char"*|3,) - # copy comments - ;; - *) - fd=1 - ;; - esac - printf '%s\n' "$command${rest:+ }$rest" >&$fd - done <"$todo" >"$todo.new" 3>>"$done" && - mv -f "$todo".new "$todo" && - case "$(peek_next_command)" in - squash|s|fixup|f) - record_in_rewritten "$onto" - ;; - esac || - die "$(gettext "Could not skip unnecessary pick commands")" -} - transform_todo_ids () { while read -r command rest do @@ -1171,7 +1134,9 @@ git rebase--helper --check-todo-list || { expand_todo_ids -test -d "$rewritten" || test -n "$force_rebase" || skip_unnecessary_picks +test -d "$rewritten" || test -n "$force_rebase" || +onto="$(git rebase--helper --skip-unnecessary-picks)" || +die "Could not skip unnecessary pick commands" checkout_onto if test -z "$rebase_root" && test ! -d "$rewritten" diff --git a/sequencer.c b/sequencer.c index 7c8338133c..b6e1d78015 100644 --- a/sequencer.c +++ b/sequencer.c @@ -2654,3 +2654,93 @@ leave_check: return res; } + +/* skip picking commits whose parents are unchanged */ +int skip_unnecessary_picks(void) +{ + const char *todo_file = rebase_path_todo(); + struct strbuf buf = STRBUF_INIT; + struct todo_list todo_list = TODO_LIST_INIT; + struct object_id onto_oid, *oid = &onto_oid, *parent_oid; + int fd, i; + + if (!read_oneliner(&buf, rebase_path_onto(), 0)) + return error(_("could not read 'onto'")); + if (get_sha1(buf.buf, onto_oid.hash)) { + strbuf_release(&buf); + return error(_("need a HEAD to fixup")); + } + strbuf_release(&buf); + + fd = open(todo_file, O_RDONLY); + if (fd < 0) { + return error_errno(_("could not open '%s'"), todo_file); + } + if (strbuf_read(&todo_list.buf, fd, 0) < 0) { + close(fd); + return error(_("could not read '%s'."), todo_file); + } + close(fd); + if (parse_insn_buffer(todo_list.buf.buf, &todo_list) < 0) { + todo_list_release(&todo_list); + return -1; + } + + for (i = 0; i < todo_list.nr; i++) { + struct todo_item *item = todo_list.items + i; + + if (item->command >= TODO_NOOP) + continue; + if (item->command != TODO_PICK) + break; + if (parse_commit(item->commit)) { + todo_list_release(&todo_list); + return error(_("could not parse commit '%s'"), + oid_to_hex(&item->commit->object.oid)); + } + if (!item->commit->parents) + break; /* root commit */ + if (item->commit->parents->next) + break; /* merge commit */ + parent_oid = &item->commit->parents->item->object.oid; + if (hashcmp(parent_oid->hash, oid->hash)) + break; + oid = &item->commit->object.oid; + } + if (i > 0) { + int offset = i < todo_list.nr ? + todo_list.items[i].offset_in_buf : todo_list.buf.len; + const char *done_path = rebase_path_done(); + + fd = open(done_path, O_CREAT | O_WRONLY | O_APPEND, 0666); + if (write_in_full(fd, todo_list.buf.buf, offset) < 0) { + todo_list_release(&todo_list); + return error_errno(_("could not write to '%s'"), + done_path); + } + close(fd); + + fd = open(rebase_path_todo(), O_WRONLY, 0666); + if (write_in_full(fd, todo_list.buf.buf + offset, + todo_list.buf.len - offset) < 0) { + todo_list_release(&todo_list); + return error_errno(_("could not write to '%s'"), + rebase_path_todo()); + } + if (ftruncate(fd, todo_list.buf.len - offset) < 0) { + todo_list_release(&todo_list); + return error_errno(_("could not truncate '%s'"), + rebase_path_todo()); + } + close(fd); + + todo_list.current = i; + if (is_fixup(peek_command(&todo_list, 0))) + record_in_rewritten(oid, peek_command(&todo_list, 0)); + } + + todo_list_release(&todo_list); + printf("%s\n", oid_to_hex(oid)); + + return 0; +} diff --git a/sequencer.h b/sequencer.h index 4978a61b83..28e1fc1e9b 100644 --- a/sequencer.h +++ b/sequencer.h @@ -50,6 +50,7 @@ int sequencer_make_script(int keep_empty, FILE *out, int transform_todo_ids(int shorten_sha1s); int check_todo_list(void); +int skip_unnecessary_picks(void); extern const char sign_off_header[]; From 440aa63ebf65c3e35d4514f35e04528e3f5f36b2 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sun, 24 Jul 2016 10:22:13 +0200 Subject: [PATCH 276/288] t3415: test fixup with wrapped oneline The `git commit --fixup` command unwraps wrapped onelines when constructing the commit message, without wrapping the result. We need to make sure that `git rebase --autosquash` keeps handling such cases correctly, in particular since we are about to move the autosquash handling into the rebase--helper. Signed-off-by: Johannes Schindelin --- t/t3415-rebase-autosquash.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/t/t3415-rebase-autosquash.sh b/t/t3415-rebase-autosquash.sh index 48346f1cc0..9fd629a6e2 100755 --- a/t/t3415-rebase-autosquash.sh +++ b/t/t3415-rebase-autosquash.sh @@ -304,4 +304,18 @@ test_expect_success 'extra spaces after fixup!' ' test $base = $parent ' +test_expect_success 'wrapped original subject' ' + if test -d .git/rebase-merge; then git rebase --abort; fi && + base=$(git rev-parse HEAD) && + echo "wrapped subject" >wrapped && + git add wrapped && + test_tick && + git commit --allow-empty -m "$(printf "To\nfixup")" && + test_tick && + git commit --allow-empty -m "fixup! To fixup" && + git rebase -i --autosquash --keep-empty HEAD~2 && + parent=$(git rev-parse HEAD^) && + test $base = $parent +' + test_done From 019fcb8d5ced90acb8c198a620c8d0fa15a1f5a7 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 16 Jun 2016 15:37:45 +0200 Subject: [PATCH 277/288] rebase -i: rearrange fixup/squash lines using the rebase--helper This operation has quadratic complexity, which is especially painful on Windows, where shell scripts are *already* slow (mainly due to the overhead of the POSIX emulation layer). Let's reimplement this with linear complexity (using a hash map to match the commits' subject lines) for the common case; Sadly, the fixup/squash feature's design neglected performance considerations, allowing arbitrary prefixes (read: `fixup! hell` will match the commit subject `hello world`), which means that we are stuck with quadratic performance in the worst case. The reimplemented logic also happens to fix a bug where commented-out lines (representing empty patches) were dropped by the previous code. Signed-off-by: Johannes Schindelin --- builtin/rebase--helper.c | 6 +- git-rebase--interactive.sh | 90 +--------------- sequencer.c | 197 +++++++++++++++++++++++++++++++++++ sequencer.h | 1 + t/t3415-rebase-autosquash.sh | 2 +- 5 files changed, 205 insertions(+), 91 deletions(-) diff --git a/builtin/rebase--helper.c b/builtin/rebase--helper.c index de3ccd9bfb..e6591f0111 100644 --- a/builtin/rebase--helper.c +++ b/builtin/rebase--helper.c @@ -14,7 +14,7 @@ int cmd_rebase__helper(int argc, const char **argv, const char *prefix) int keep_empty = 0; enum { CONTINUE = 1, ABORT, MAKE_SCRIPT, SHORTEN_SHA1S, EXPAND_SHA1S, - CHECK_TODO_LIST, SKIP_UNNECESSARY_PICKS + CHECK_TODO_LIST, SKIP_UNNECESSARY_PICKS, REARRANGE_SQUASH } command = 0; struct option options[] = { OPT_BOOL(0, "ff", &opts.allow_ff, N_("allow fast-forward")), @@ -33,6 +33,8 @@ int cmd_rebase__helper(int argc, const char **argv, const char *prefix) N_("check the todo list"), CHECK_TODO_LIST), OPT_CMDMODE(0, "skip-unnecessary-picks", &command, N_("skip unnecessary picks"), SKIP_UNNECESSARY_PICKS), + OPT_CMDMODE(0, "rearrange-squash", &command, + N_("rearrange fixup/squash lines"), REARRANGE_SQUASH), OPT_END() }; @@ -59,5 +61,7 @@ int cmd_rebase__helper(int argc, const char **argv, const char *prefix) return !!check_todo_list(); if (command == SKIP_UNNECESSARY_PICKS && argc == 1) return !!skip_unnecessary_picks(); + if (command == REARRANGE_SQUASH && argc == 1) + return !!rearrange_squash(); usage_with_options(builtin_rebase_helper_usage, options); } diff --git a/git-rebase--interactive.sh b/git-rebase--interactive.sh index 168c041e54..226b213e5c 100644 --- a/git-rebase--interactive.sh +++ b/git-rebase--interactive.sh @@ -743,94 +743,6 @@ collapse_todo_ids() { git rebase--helper --shorten-sha1s } -# Rearrange the todo list that has both "pick sha1 msg" and -# "pick sha1 fixup!/squash! msg" appears in it so that the latter -# comes immediately after the former, and change "pick" to -# "fixup"/"squash". -# -# Note that if the config has specified a custom instruction format -# each log message will be re-retrieved in order to normalize the -# autosquash arrangement -rearrange_squash () { - format=$(git config --get rebase.instructionFormat) - # extract fixup!/squash! lines and resolve any referenced sha1's - while read -r pick sha1 message - do - test -z "${format}" || message=$(git log -n 1 --format="%s" ${sha1}) - case "$message" in - "squash! "*|"fixup! "*) - action="${message%%!*}" - rest=$message - prefix= - # skip all squash! or fixup! (but save for later) - while : - do - case "$rest" in - "squash! "*|"fixup! "*) - prefix="$prefix${rest%%!*}," - rest="${rest#*! }" - ;; - *) - break - ;; - esac - done - printf '%s %s %s %s\n' "$sha1" "$action" "$prefix" "$rest" - # if it's a single word, try to resolve to a full sha1 and - # emit a second copy. This allows us to match on both message - # and on sha1 prefix - if test "${rest#* }" = "$rest"; then - fullsha="$(git rev-parse -q --verify "$rest" 2>/dev/null)" - if test -n "$fullsha"; then - # prefix the action to uniquely identify this line as - # intended for full sha1 match - echo "$sha1 +$action $prefix $fullsha" - fi - fi - esac - done >"$1.sq" <"$1" - test -s "$1.sq" || return - - used= - while read -r pick sha1 message - do - case " $used" in - *" $sha1 "*) continue ;; - esac - printf '%s\n' "$pick $sha1 $message" - test -z "${format}" || message=$(git log -n 1 --format="%s" ${sha1}) - used="$used$sha1 " - while read -r squash action msg_prefix msg_content - do - case " $used" in - *" $squash "*) continue ;; - esac - emit=0 - case "$action" in - +*) - action="${action#+}" - # full sha1 prefix test - case "$msg_content" in "$sha1"*) emit=1;; esac ;; - *) - # message prefix test - case "$message" in "$msg_content"*) emit=1;; esac ;; - esac - if test $emit = 1; then - if test -n "${format}" - then - msg_content=$(git log -n 1 --format="${format}" ${squash}) - else - msg_content="$(echo "$msg_prefix" | sed "s/,/! /g")$msg_content" - fi - printf '%s\n' "$action $squash $msg_content" - used="$used$squash " - fi - done <"$1.sq" - done >"$1.rearranged" <"$1" - cat "$1.rearranged" >"$1" - rm -f "$1.sq" "$1.rearranged" -} - # Add commands after a pick or after a squash/fixup serie # in the todo list. add_exec_commands () { @@ -1090,7 +1002,7 @@ then fi test -s "$todo" || echo noop >> "$todo" -test -n "$autosquash" && rearrange_squash "$todo" +test -z "$autosquash" || git rebase--helper --rearrange-squash || exit test -n "$cmd" && add_exec_commands "$todo" todocount=$(git stripspace --strip-comments <"$todo" | wc -l) diff --git a/sequencer.c b/sequencer.c index b6e1d78015..758e39b540 100644 --- a/sequencer.c +++ b/sequencer.c @@ -18,6 +18,7 @@ #include "quote.h" #include "log-tree.h" #include "wt-status.h" +#include "hashmap.h" #define GIT_REFLOG_ACTION "GIT_REFLOG_ACTION" @@ -2744,3 +2745,199 @@ int skip_unnecessary_picks(void) return 0; } + +struct subject2item_entry { + struct hashmap_entry entry; + int i; + char subject[FLEX_ARRAY]; +}; + +static int subject2item_cmp(const struct subject2item_entry *a, + const struct subject2item_entry *b, const void *key) +{ + return key ? strcmp(a->subject, key) : strcmp(a->subject, b->subject); +} + +/* + * Rearrange the todo list that has both "pick sha1 msg" and "pick sha1 + * fixup!/squash! msg" in it so that the latter is put immediately after the + * former, and change "pick" to "fixup"/"squash". + * + * Note that if the config has specified a custom instruction format, each log + * message will have to be retrieved from the commit (as the oneline in the + * script cannot be trusted) in order to normalize the autosquash arrangement. + */ +int rearrange_squash(void) +{ + const char *todo_file = rebase_path_todo(); + struct todo_list todo_list = TODO_LIST_INIT; + struct hashmap subject2item; + int res = 0, rearranged = 0, *next, *tail, fd, i; + char **subjects; + + fd = open(todo_file, O_RDONLY); + if (fd < 0) + return error_errno(_("could not open '%s'"), todo_file); + if (strbuf_read(&todo_list.buf, fd, 0) < 0) { + close(fd); + return error(_("could not read '%s'."), todo_file); + } + close(fd); + if (parse_insn_buffer(todo_list.buf.buf, &todo_list) < 0) { + todo_list_release(&todo_list); + return -1; + } + + /* + * The hashmap maps onelines to the respective todo list index. + * + * If any items need to be rearranged, the next[i] value will indicate + * which item was moved directly after the i'th. + * + * In that case, last[i] will indicate the index of the latest item to + * be moved to appear after the i'th. + */ + hashmap_init(&subject2item, (hashmap_cmp_fn) subject2item_cmp, + todo_list.nr); + ALLOC_ARRAY(next, todo_list.nr); + ALLOC_ARRAY(tail, todo_list.nr); + ALLOC_ARRAY(subjects, todo_list.nr); + for (i = 0; i < todo_list.nr; i++) { + struct strbuf buf = STRBUF_INIT; + struct todo_item *item = todo_list.items + i; + const char *commit_buffer, *subject, *p; + int i2 = -1; + struct subject2item_entry *entry; + + next[i] = tail[i] = -1; + if (item->command >= TODO_EXEC) { + subjects[i] = NULL; + continue; + } + + if (is_fixup(item->command)) { + todo_list_release(&todo_list); + return error(_("the script was already rearranged.")); + } + + item->commit->util = item; + + parse_commit(item->commit); + commit_buffer = get_commit_buffer(item->commit, NULL); + find_commit_subject(commit_buffer, &subject); + format_subject(&buf, subject, " "); + subject = subjects[i] = buf.buf; + unuse_commit_buffer(item->commit, commit_buffer); + if ((skip_prefix(subject, "fixup! ", &p) || + skip_prefix(subject, "squash! ", &p))) { + struct commit *commit2; + + for (;;) { + while (isspace(*p)) + p++; + if (!skip_prefix(p, "fixup! ", &p) && + !skip_prefix(p, "squash! ", &p)) + break; + } + + if ((entry = hashmap_get_from_hash(&subject2item, + strhash(p), p))) + /* found by title */ + i2 = entry->i; + else if (!strchr(p, ' ') && + (commit2 = + lookup_commit_reference_by_name(p)) && + commit2->util) + /* found by commit name */ + i2 = (struct todo_item *)commit2->util + - todo_list.items; + else { + /* copy can be a prefix of the commit subject */ + for (i2 = 0; i2 < i; i2++) + if (subjects[i2] && + starts_with(subjects[i2], p)) + break; + if (i2 == i) + i2 = -1; + } + } + if (i2 >= 0) { + rearranged = 1; + todo_list.items[i].command = + starts_with(subject, "fixup!") ? + TODO_FIXUP : TODO_SQUASH; + if (next[i2] < 0) + next[i2] = i; + else + next[tail[i2]] = i; + tail[i2] = i; + } else if (!hashmap_get_from_hash(&subject2item, + strhash(subject), subject)) { + FLEX_ALLOC_MEM(entry, subject, buf.buf, buf.len); + entry->i = i; + hashmap_entry_init(entry, strhash(entry->subject)); + hashmap_put(&subject2item, entry); + } + strbuf_detach(&buf, NULL); + } + + if (rearranged) { + struct strbuf buf = STRBUF_INIT; + char *format = NULL; + + git_config_get_string("rebase.instructionFormat", &format); + for (i = 0; i < todo_list.nr; i++) { + enum todo_command command = todo_list.items[i].command; + int cur = i; + + /* + * Initially, all commands are 'pick's. If it is a + * fixup or a squash now, we have rearranged it. + */ + if (is_fixup(command)) + continue; + + while (cur >= 0) { + int offset = todo_list.items[cur].offset_in_buf; + int end_offset = cur + 1 < todo_list.nr ? + todo_list.items[cur + 1].offset_in_buf : + todo_list.buf.len; + char *bol = todo_list.buf.buf + offset; + char *eol = todo_list.buf.buf + end_offset; + + /* replace 'pick', by 'fixup' or 'squash' */ + command = todo_list.items[cur].command; + if (is_fixup(command)) { + strbuf_addstr(&buf, + todo_command_info[command].str); + bol += strcspn(bol, " \t"); + } + + strbuf_add(&buf, bol, eol - bol); + + cur = next[cur]; + } + } + + fd = open(todo_file, O_WRONLY); + if (fd < 0) + res = error_errno(_("could not open '%s'"), todo_file); + else if (write(fd, buf.buf, buf.len) < 0) + res = error_errno(_("could not read '%s'."), todo_file); + else if (ftruncate(fd, buf.len) < 0) + res = error_errno(_("could not finish '%s'"), + todo_file); + close(fd); + strbuf_release(&buf); + } + + free(next); + free(tail); + for (i = 0; i < todo_list.nr; i++) + free(subjects[i]); + free(subjects); + hashmap_free(&subject2item, 1); + todo_list_release(&todo_list); + + return res; +} diff --git a/sequencer.h b/sequencer.h index 28e1fc1e9b..1c94bec762 100644 --- a/sequencer.h +++ b/sequencer.h @@ -51,6 +51,7 @@ int sequencer_make_script(int keep_empty, FILE *out, int transform_todo_ids(int shorten_sha1s); int check_todo_list(void); int skip_unnecessary_picks(void); +int rearrange_squash(void); extern const char sign_off_header[]; diff --git a/t/t3415-rebase-autosquash.sh b/t/t3415-rebase-autosquash.sh index 9fd629a6e2..b9e26008a7 100755 --- a/t/t3415-rebase-autosquash.sh +++ b/t/t3415-rebase-autosquash.sh @@ -278,7 +278,7 @@ set_backup_editor () { test_set_editor "$PWD/backup-editor.sh" } -test_expect_failure 'autosquash with multiple empty patches' ' +test_expect_success 'autosquash with multiple empty patches' ' test_tick && git commit --allow-empty -m "empty" && test_tick && From 3929a313f0097afbc99d87b0b4140b752c633d27 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 11 Jan 2017 21:08:15 +0100 Subject: [PATCH 278/288] t9116: work around hard-to-debug hangs As of a couple of weeks ago, t9116 hangs sometimes -- but not always! -- when being run in the Git for Windows SDK. The issue seems to be related to redirection via a pipe, but it is really hard to diagnose, what with git.exe (a non-MSYS2 program) calling a Perl script (which is executed by an MSYS2 Perl), piping into another MSYS2 program. As hunting time is scarce these days, simply work around this for now and leave the real diagnosis and resolution for later. Signed-off-by: Johannes Schindelin --- t/t9116-git-svn-log.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/t/t9116-git-svn-log.sh b/t/t9116-git-svn-log.sh index 45773ee560..0a9f1ef366 100755 --- a/t/t9116-git-svn-log.sh +++ b/t/t9116-git-svn-log.sh @@ -43,14 +43,18 @@ test_expect_success 'setup repository and import' ' test_expect_success 'run log' " git reset --hard origin/a && - git svn log -r2 origin/trunk | grep ^r2 && - git svn log -r4 origin/trunk | grep ^r4 && - git svn log -r3 | grep ^r3 + git svn log -r2 origin/trunk >out && + grep ^r2 out && + git svn log -r4 origin/trunk >out && + grep ^r4 out && + git svn log -r3 >out && + grep ^r3 out " test_expect_success 'run log against a from trunk' " git reset --hard origin/trunk && - git svn log -r3 origin/a | grep ^r3 + git svn log -r3 origin/a >out && + grep ^r3 out " printf 'r1 \nr2 \nr4 \n' > expected-range-r1-r2-r4 From c1a423a58902febaefe5d346e2171cdd5cf035db Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 11 Jan 2017 21:42:45 +0100 Subject: [PATCH 279/288] t5580: verify that alternates can be UNC paths On Windows, UNC paths are a very convenient way to share data, and alternates are all about sharing data. We fixed a bug where alternates specifying UNC paths were not handled properly, and it is high time that we add a regression test to ensure that this bug is not reintroduced. Signed-off-by: Johannes Schindelin --- t/t5580-clone-push-unc.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/t/t5580-clone-push-unc.sh b/t/t5580-clone-push-unc.sh index e06d230724..5da5d2927d 100755 --- a/t/t5580-clone-push-unc.sh +++ b/t/t5580-clone-push-unc.sh @@ -43,4 +43,16 @@ test_expect_success push ' ) ' +test_expect_success 'unc alternates' ' + tree="$(git rev-parse HEAD:)" && + mkdir test-unc-alternate && + ( + cd test-unc-alternate && + git init && + test_must_fail git show $tree && + echo "$UNCPATH/.git/objects" >.git/objects/info/alternates && + git show $tree + ) +' + test_done From 755b9533b6fe740563e20547335686de345d1564 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 11 Jan 2017 21:08:15 +0100 Subject: [PATCH 280/288] t9001: work around hard-to-debug hangs Just like the workaround we added for t9116, t9001.83 hangs sometimes -- but not always! -- when being run in the Git for Windows SDK. The issue seems to be related to redirection via a pipe, but it is really hard to diagnose, what with git.exe (a non-MSYS2 program) calling a Perl script (which is executed by an MSYS2 Perl), piping into another MSYS2 program. As hunting time is scarce these days, simply work around this for now and leave the real diagnosis and resolution for later. Signed-off-by: Johannes Schindelin --- t/t9001-send-email.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/t/t9001-send-email.sh b/t/t9001-send-email.sh index 3dc4a3454d..d9b1d725c9 100755 --- a/t/t9001-send-email.sh +++ b/t/t9001-send-email.sh @@ -1088,8 +1088,8 @@ test_expect_success $PREREQ 'in-reply-to but no threading' ' --to=nobody@example.com \ --in-reply-to="" \ --no-thread \ - $patches | - grep "In-Reply-To: " + $patches >out && + grep "In-Reply-To: " out ' test_expect_success $PREREQ 'no in-reply-to and no threading' ' From 2b3a55c4347d45090b8f58e3984742e3c2927da4 Mon Sep 17 00:00:00 2001 From: Max Kirillov Date: Wed, 18 Jan 2017 21:01:09 +0200 Subject: [PATCH 281/288] git-gui: correctly restore GIT_DIR after invoking gitk git-gui tries to temporary set GIT_DIR for starting gitk and restore it back after they are started. But in case of GIT_DIR which was not set prior to invocation it is not unset after it. This affects commands which can be later started from that git gui, for example "Git Bash". Fix it. Signed-off-by: Max Kirillov Signed-off-by: Johannes Schindelin --- git-gui/git-gui.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/git-gui/git-gui.sh b/git-gui/git-gui.sh index 0787c10fcd..ccf057966a 100755 --- a/git-gui/git-gui.sh +++ b/git-gui/git-gui.sh @@ -2209,6 +2209,8 @@ proc do_gitk {revs {is_submodule false}} { if {$old_GIT_DIR ne {}} { set env(GIT_DIR) $old_GIT_DIR + } else { + unset env(GIT_DIR) } cd $pwd From 7e36a998e218ea3b38115309d37813e1331c0c95 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Tue, 24 Jan 2017 15:12:13 -0500 Subject: [PATCH 282/288] fscache: add key for GIT_TRACE_FSCACHE Signed-off-by: Jeff Hostetler Signed-off-by: Johannes Schindelin --- compat/win32/fscache.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/compat/win32/fscache.c b/compat/win32/fscache.c index 63d522f49a..abefd3bb4f 100644 --- a/compat/win32/fscache.c +++ b/compat/win32/fscache.c @@ -7,6 +7,7 @@ static int initialized; static volatile long enabled; static struct hashmap map; static CRITICAL_SECTION mutex; +static struct trace_key trace_fscache = TRACE_KEY_INIT(FSCACHE); /* * An entry in the file system cache. Used for both entire directory listings @@ -190,6 +191,8 @@ static struct fsentry *fsentry_create_list(const struct fsentry *dir) if (h == INVALID_HANDLE_VALUE) { err = GetLastError(); errno = (err == ERROR_DIRECTORY) ? ENOTDIR : err_win_to_posix(err); + trace_printf_key(&trace_fscache, "fscache: error(%d) '%.*s'\n", + errno, dir->len, dir->name); return NULL; } @@ -391,6 +394,7 @@ int fscache_enable(int enable) fscache_clear(); LeaveCriticalSection(&mutex); } + trace_printf_key(&trace_fscache, "fscache: enable(%d)\n", enable); return result; } From 1bde9255b8cade12328a1be25a1c77ba4ffaca01 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Tue, 13 Dec 2016 14:05:32 -0500 Subject: [PATCH 283/288] fscache: remember not-found directories Teach FSCACHE to remember "not found" directories. This is a performance optimization. FSCACHE is a performance optimization available for Windows. It intercepts Posix-style lstat() calls into an in-memory directory using FindFirst/FindNext. It improves performance on Windows by catching the first lstat() call in a directory, using FindFirst/ FindNext to read the list of files (and attribute data) for the entire directory into the cache, and short-cut subsequent lstat() calls in the same directory. This gives a major performance boost on Windows. However, it does not remember "not found" directories. When STATUS runs and there are missing directories, the lstat() interception fails to find the parent directory and simply return ENOENT for the file -- it does not remember that the FindFirst on the directory failed. Thus subsequent lstat() calls in the same directory, each re-attempt the FindFirst. This completely defeats any performance gains. This can be seen by doing a sparse-checkout on a large repo and then doing a read-tree to reset the skip-worktree bits and then running status. This change reduced status times for my very large repo by 60%. Signed-off-by: Jeff Hostetler Signed-off-by: Johannes Schindelin --- compat/win32/fscache.c | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/compat/win32/fscache.c b/compat/win32/fscache.c index abefd3bb4f..e0f0681bb3 100644 --- a/compat/win32/fscache.c +++ b/compat/win32/fscache.c @@ -163,7 +163,8 @@ static struct fsentry *fseentry_create_entry(struct fsentry *list, * Dir should not contain trailing '/'. Use an empty string for the current * directory (not "."!). */ -static struct fsentry *fsentry_create_list(const struct fsentry *dir) +static struct fsentry *fsentry_create_list(const struct fsentry *dir, + int *dir_not_found) { wchar_t pattern[MAX_LONG_PATH + 2]; /* + 2 for "\*" */ WIN32_FIND_DATAW fdata; @@ -172,6 +173,8 @@ static struct fsentry *fsentry_create_list(const struct fsentry *dir) struct fsentry *list, **phead; DWORD err; + *dir_not_found = 0; + /* convert name to UTF-16 and check length */ if ((wlen = xutftowcs_path_ex(pattern, dir->name, MAX_LONG_PATH, dir->len, MAX_PATH - 2, core_long_paths)) < 0) @@ -190,6 +193,7 @@ static struct fsentry *fsentry_create_list(const struct fsentry *dir) h = FindFirstFileW(pattern, &fdata); if (h == INVALID_HANDLE_VALUE) { err = GetLastError(); + *dir_not_found = 1; /* or empty directory */ errno = (err == ERROR_DIRECTORY) ? ENOTDIR : err_win_to_posix(err); trace_printf_key(&trace_fscache, "fscache: error(%d) '%.*s'\n", errno, dir->len, dir->name); @@ -198,6 +202,7 @@ static struct fsentry *fsentry_create_list(const struct fsentry *dir) /* allocate object to hold directory listing */ list = fsentry_alloc(NULL, dir->name, dir->len); + list->st_mode = S_IFDIR; /* walk directory and build linked list of fsentry structures */ phead = &list->next; @@ -298,12 +303,16 @@ static struct fsentry *fscache_get_wait(struct fsentry *key) static struct fsentry *fscache_get(struct fsentry *key) { struct fsentry *fse, *future, *waiter; + int dir_not_found; EnterCriticalSection(&mutex); /* check if entry is in cache */ fse = fscache_get_wait(key); if (fse) { - fsentry_addref(fse); + if (fse->st_mode) + fsentry_addref(fse); + else + fse = NULL; /* non-existing directory */ LeaveCriticalSection(&mutex); return fse; } @@ -312,7 +321,10 @@ static struct fsentry *fscache_get(struct fsentry *key) fse = fscache_get_wait(key->list); if (fse) { LeaveCriticalSection(&mutex); - /* dir entry without file entry -> file doesn't exist */ + /* + * dir entry without file entry, or dir does not + * exist -> file doesn't exist + */ errno = ENOENT; return NULL; } @@ -326,7 +338,7 @@ static struct fsentry *fscache_get(struct fsentry *key) /* create the directory listing (outside mutex!) */ LeaveCriticalSection(&mutex); - fse = fsentry_create_list(future); + fse = fsentry_create_list(future, &dir_not_found); EnterCriticalSection(&mutex); /* remove future entry and signal waiting threads */ @@ -340,6 +352,17 @@ static struct fsentry *fscache_get(struct fsentry *key) /* leave on error (errno set by fsentry_create_list) */ if (!fse) { + if (dir_not_found && key->list) { + /* + * Record that the directory does not exist (or is + * empty, which for all practical matters is the same + * thing as far as fscache is concerned). + */ + fse = fsentry_alloc(key->list->list, + key->list->name, key->list->len); + fse->st_mode = 0; + hashmap_add(&map, fse); + } LeaveCriticalSection(&mutex); return NULL; } @@ -351,6 +374,9 @@ static struct fsentry *fscache_get(struct fsentry *key) if (key->list) fse = hashmap_get(&map, key, NULL); + if (fse && !fse->st_mode) + fse = NULL; /* non-existing directory */ + /* return entry or ENOENT */ if (fse) fsentry_addref(fse); From 0f3555ff0c0123e0c73010c1846dc821e9280e6b Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 25 Jan 2017 18:39:16 +0100 Subject: [PATCH 284/288] fscache: add a test for the dir-not-found optimization Signed-off-by: Johannes Schindelin --- t/t1090-sparse-checkout-scope.sh | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/t/t1090-sparse-checkout-scope.sh b/t/t1090-sparse-checkout-scope.sh index 1f61eb3e88..33a69a9c71 100755 --- a/t/t1090-sparse-checkout-scope.sh +++ b/t/t1090-sparse-checkout-scope.sh @@ -49,4 +49,24 @@ test_expect_success 'return to full checkout of master' ' test "$(cat b)" = "modified" ' +test_expect_success 'no unnecessary opendir() with fscache' ' + git clone . fscache-test && + ( + cd fscache-test && + git config core.fscache 1 && + echo "/excluded/*" >.git/info/sparse-checkout && + for f in $(test_seq 10) + do + sha1=$(echo $f | git hash-object -w --stdin) && + git update-index --add \ + --cacheinfo 100644,$sha1,excluded/$f || break + done && + test_tick && + git commit -m excluded && + GIT_TRACE_FSCACHE=1 git status >out 2>err && + grep excluded err >grep.out && + test_line_count = 1 grep.out + ) +' + test_done From 1c6c2420ff6683d93a61fc790842ab712f2a926b Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 13 Dec 2016 17:52:14 +0000 Subject: [PATCH 285/288] mingw: allow hooks to be .exe files This change is necessary to allow the files in .git/hooks/ to optionally have the file extension `.exe` on Windows, as the file names are hardcoded otherwise. Signed-off-by: Johannes Schindelin --- run-command.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/run-command.c b/run-command.c index 73bfba7ef9..5227f78aea 100644 --- a/run-command.c +++ b/run-command.c @@ -871,8 +871,14 @@ const char *find_hook(const char *name) strbuf_reset(&path); strbuf_git_path(&path, "hooks/%s", name); - if (access(path.buf, X_OK) < 0) + if (access(path.buf, X_OK) < 0) { +#ifdef STRIP_EXTENSION + strbuf_addstr(&path, STRIP_EXTENSION); + if (access(path.buf, X_OK) >= 0) + return path.buf; +#endif return NULL; + } return path.buf; } From df6944828780819f7003de767cf43cd29b62067e Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sat, 4 Feb 2017 09:36:45 +0100 Subject: [PATCH 286/288] fixup! fscache: add a test for the dir-not-found optimization The fscache feature only exists on Windows, therefore we can only successfully test it on that platform. Signed-off-by: Johannes Schindelin --- t/t1090-sparse-checkout-scope.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/t1090-sparse-checkout-scope.sh b/t/t1090-sparse-checkout-scope.sh index 33a69a9c71..bea7e18233 100755 --- a/t/t1090-sparse-checkout-scope.sh +++ b/t/t1090-sparse-checkout-scope.sh @@ -49,7 +49,7 @@ test_expect_success 'return to full checkout of master' ' test "$(cat b)" = "modified" ' -test_expect_success 'no unnecessary opendir() with fscache' ' +test_expect_success MINGW 'no unnecessary opendir() with fscache' ' git clone . fscache-test && ( cd fscache-test && From 87ad093f001e7146e5e521914255199e49c212a7 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 13 Feb 2017 23:23:20 +0100 Subject: [PATCH 287/288] mingw: make stderr unbuffered again When removing the hack for isatty(), we actually removed more than just an isatty() hack: we removed the hack where internal data structures of the MSVC runtime are modified in order to redirect stdout/stderr. Instead of using that hack (that does not work with newer versions of the runtime, anyway), we replaced it by reopening the respective file descriptors. What we forgot was to mark stderr as unbuffered again. Reported by Hannes Sixt. Fixed with Jeff Hostetler's assistance. Signed-off-by: Johannes Schindelin --- compat/winansi.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/compat/winansi.c b/compat/winansi.c index 664b672d40..986bfb30cf 100644 --- a/compat/winansi.c +++ b/compat/winansi.c @@ -505,6 +505,8 @@ static HANDLE swap_osfhnd(int fd, HANDLE new_handle) */ close(new_fd); + if (fd == 2) + setvbuf(stderr, NULL, _IONBF, BUFSIZ); fd_is_interactive[fd] |= FD_SWAPPED; return duplicate; @@ -555,6 +557,8 @@ static void detect_msys_tty(int fd) !wcsstr(name, L"-pty")) return; + if (fd == 2) + setvbuf(stderr, NULL, _IONBF, BUFSIZ); fd_is_interactive[fd] |= FD_MSYS; } From 116813629e688d12c36677180a90b7731cbe457f Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 21 Feb 2017 13:28:58 +0100 Subject: [PATCH 288/288] mingw: ensure valid CTYPE A change between versions 2.4.1 and 2.6.0 of the MSYS2 runtime modified how Cygwin's runtime (and hence Git for Windows' MSYS2 runtime derivative) handles locales: d16a56306d (Consolidate wctomb/mbtowc calls for POSIX-1.2008, 2016-07-20). An unintended side-effect is that "cold-calling" into the POSIX emulation will start with a locale based on the current code page, something that Git for Windows is very ill-prepared for, as it expects to be able to pass a command-line containing non-ASCII characters to the shell without having those characters munged. One symptom of this behavior: when `git clone` or `git fetch` shell out to call `git-upload-pack` with a path that contains non-ASCII characters, the shell tried to interpret the entire command-line (including command-line parameters) as executable path, which obviously must fail. This fixes https://github.com/git-for-windows/git/issues/1036 Signed-off-by: Johannes Schindelin --- compat/mingw.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/compat/mingw.c b/compat/mingw.c index 13fa2f14ae..40ac29eb8a 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2913,6 +2913,9 @@ static void setup_windows_environment(void) */ if (!(tmp = getenv("MSYS")) || !strstr(tmp, "winsymlinks:nativestrict")) has_symlinks = 0; + + if (!getenv("LC_ALL") && !getenv("LC_CTYPE") && !getenv("LANG")) + setenv("LC_CTYPE", "C", 1); } int handle_long_path(wchar_t *path, int len, int max_path, int expand)