From a2de79ddeb163f105ca28fe763da047847dff3ef Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Wed, 5 Apr 2017 10:58:09 -0600 Subject: [PATCH 001/118] reset --stdin: trim carriage return from the paths While using the reset --stdin feature on windows path added may have a \r at the end of the path that wasn't getting removed so didn't match the path in the index and wasn't reset. Signed-off-by: Kevin Willford --- builtin/reset.c | 2 +- t/t7108-reset-stdin.sh | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/builtin/reset.c b/builtin/reset.c index 023fe12be048b2..a27a90952ea549 100644 --- a/builtin/reset.c +++ b/builtin/reset.c @@ -327,7 +327,7 @@ int cmd_reset(int argc, const char **argv, const char *prefix) if (read_from_stdin) { strbuf_getline_fn getline_fn = nul_term_line ? - strbuf_getline_nul : strbuf_getline_lf; + strbuf_getline_nul : strbuf_getline; int flags = PATHSPEC_PREFER_FULL; struct strbuf buf = STRBUF_INIT; struct strbuf unquoted = STRBUF_INIT; diff --git a/t/t7108-reset-stdin.sh b/t/t7108-reset-stdin.sh index b7cbcbf869296c..db5483b8f10052 100755 --- a/t/t7108-reset-stdin.sh +++ b/t/t7108-reset-stdin.sh @@ -29,4 +29,13 @@ test_expect_success '--stdin requires --mixed' ' git reset --mixed --stdin list && + git reset --stdin Date: Tue, 24 Jan 2017 17:44:31 +0100 Subject: [PATCH 002/118] gvfs: start by adding the -gvfs suffix to the version Signed-off-by: Saeed Noursalehi --- GIT-VERSION-GEN | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GIT-VERSION-GEN b/GIT-VERSION-GEN index 13f835f9cde4fd..2f341df65fd807 100755 --- a/GIT-VERSION-GEN +++ b/GIT-VERSION-GEN @@ -1,7 +1,7 @@ #!/bin/sh GVF=GIT-VERSION-FILE -DEF_VER=v2.21.0 +DEF_VER=v2.21.0.vfs.1.1 LF=' ' From 9fc99b086a1c967db62a176ba78c0541fcca18bd Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 4 Apr 2017 12:04:11 +0200 Subject: [PATCH 003/118] gvfs: ensure that the version is based on a GVFS tag Signed-off-by: Johannes Schindelin --- GIT-VERSION-GEN | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GIT-VERSION-GEN b/GIT-VERSION-GEN index 2f341df65fd807..d8e71cb60c9c39 100755 --- a/GIT-VERSION-GEN +++ b/GIT-VERSION-GEN @@ -12,7 +12,7 @@ if test -f version then VN=$(cat version) || VN="$DEF_VER" elif test -d ${GIT_DIR:-.git} -o -f .git && - VN=$(git describe --match "v[0-9]*" HEAD 2>/dev/null) && + VN=$(git describe --match "v[0-9]*vfs*" HEAD 2>/dev/null) && case "$VN" in *$LF*) (exit 1) ;; v[0-9]*) From 7e3102069053da6c8f662618aebc40e407d8a55f Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Tue, 24 Jan 2017 17:30:59 +0100 Subject: [PATCH 004/118] gvfs: add a GVFS-specific header file This header file will accumulate GVFS-specific definitions. Signed-off-by: Kevin Willford --- gvfs.h | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 gvfs.h diff --git a/gvfs.h b/gvfs.h new file mode 100644 index 00000000000000..b6dbe85eae4071 --- /dev/null +++ b/gvfs.h @@ -0,0 +1,9 @@ +#ifndef GVFS_H +#define GVFS_H + +/* + * This file is for the specific settings and methods + * used for GVFS functionality + */ + +#endif /* GVFS_H */ From 7305e3856500f76d2d90543c6a28019f661eebc0 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Tue, 24 Jan 2017 17:34:12 +0100 Subject: [PATCH 005/118] gvfs: add the core.gvfs config setting This does not do anything yet. The next patches will add various values for that config setting that correspond to the various features offered/required by GVFS. Signed-off-by: Kevin Willford --- Documentation/config/core.txt | 3 +++ cache.h | 1 + config.c | 6 ++++++ environment.c | 1 + gvfs.h | 31 +++++++++++++++++++++++++++++++ 5 files changed, 42 insertions(+) diff --git a/Documentation/config/core.txt b/Documentation/config/core.txt index 71dce85e76926c..8cd133321bb80e 100644 --- a/Documentation/config/core.txt +++ b/Documentation/config/core.txt @@ -603,6 +603,9 @@ core.multiPackIndex:: single index. See link:technical/multi-pack-index.html[the multi-pack-index design document]. +core.gvfs:: + Enable the features needed for GVFS. + core.sparseCheckout:: Enable "sparse checkout" feature. See section "Sparse checkout" in linkgit:git-read-tree[1] for more information. diff --git a/cache.h b/cache.h index 877697571c8585..e976871a0bc85f 100644 --- a/cache.h +++ b/cache.h @@ -886,6 +886,7 @@ extern char *git_replace_ref_base; extern int fsync_object_files; extern int core_preload_index; extern int core_apply_sparse_checkout; +extern int core_gvfs; extern int precomposed_unicode; extern int protect_hfs; extern int protect_ntfs; diff --git a/config.c b/config.c index 279b9e8b187d9f..eea77e608e3e19 100644 --- a/config.c +++ b/config.c @@ -19,6 +19,7 @@ #include "utf8.h" #include "dir.h" #include "color.h" +#include "gvfs.h" struct config_source { struct config_source *prev; @@ -1324,6 +1325,11 @@ static int git_default_core_config(const char *var, const char *value, void *cb) return 0; } + if (!strcmp(var, "core.gvfs")) { + gvfs_load_config_value(value); + return 0; + } + if (!strcmp(var, "core.sparsecheckout")) { core_apply_sparse_checkout = git_config_bool(var, value); return 0; diff --git a/environment.c b/environment.c index 89af47cb850490..2bd89c627612f6 100644 --- a/environment.c +++ b/environment.c @@ -69,6 +69,7 @@ enum object_creation_mode object_creation_mode = OBJECT_CREATION_MODE; char *notes_ref_name; int grafts_replace_parents = 1; int core_apply_sparse_checkout; +int core_gvfs; int merge_log_config = -1; int precomposed_unicode = -1; /* see probe_utf8_pathname_composition() */ unsigned long pack_size_limit_cfg; diff --git a/gvfs.h b/gvfs.h index b6dbe85eae4071..2e9e10f4e49ef6 100644 --- a/gvfs.h +++ b/gvfs.h @@ -1,9 +1,40 @@ #ifndef GVFS_H #define GVFS_H +#include "cache.h" +#include "config.h" + /* * This file is for the specific settings and methods * used for GVFS functionality */ +static inline int gvfs_config_is_set(int mask) { + return (core_gvfs & mask) == mask; +} + +static inline int gvfs_config_is_set_any(void) { + return core_gvfs > 0; +} + +static inline void gvfs_load_config_value(const char *value) { + int is_bool = 0; + + if (value) + core_gvfs = git_config_bool_or_int("core.gvfs", value, &is_bool); + else + git_config_get_bool_or_int("core.gvfs", &is_bool, &core_gvfs); + + /* Turn on all bits if a bool was set in the settings */ + if (is_bool && core_gvfs) + core_gvfs = -1; +} + + +static inline int gvfs_config_load_and_is_set(int mask) { + gvfs_load_config_value(0); + return gvfs_config_is_set(mask); +} + + #endif /* GVFS_H */ From ee378e46705e93eeff25866a235b8e7c5e383f70 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Tue, 24 Jan 2017 17:38:59 +0100 Subject: [PATCH 006/118] gvfs: add the feature to skip writing the index' SHA-1 This takes a substantial amount of time, and if the user is reasonably sure that the files' integrity is not compromised, that time can be saved. Git no longer verifies the SHA-1 by default, anyway. Signed-off-by: Kevin Willford --- Documentation/config/core.txt | 10 +++++++++- gvfs.h | 6 ++++++ read-cache.c | 11 ++++++++--- t/t1016-read-tree-skip-sha-on-read.sh | 22 ++++++++++++++++++++++ 4 files changed, 45 insertions(+), 4 deletions(-) create mode 100755 t/t1016-read-tree-skip-sha-on-read.sh diff --git a/Documentation/config/core.txt b/Documentation/config/core.txt index 8cd133321bb80e..72cc6e949c4715 100644 --- a/Documentation/config/core.txt +++ b/Documentation/config/core.txt @@ -604,7 +604,15 @@ core.multiPackIndex:: multi-pack-index design document]. core.gvfs:: - Enable the features needed for GVFS. + Enable the features needed for GVFS. This value can be set to true + to indicate all features should be turned on or the bit values listed + below can be used to turn on specific features. ++ +-- + GVFS_SKIP_SHA_ON_INDEX:: + Bit value 1 + Disables the calculation of the sha when writing the index +-- core.sparseCheckout:: Enable "sparse checkout" feature. See section "Sparse checkout" in diff --git a/gvfs.h b/gvfs.h index 2e9e10f4e49ef6..690419127a72dd 100644 --- a/gvfs.h +++ b/gvfs.h @@ -9,6 +9,12 @@ * used for GVFS functionality */ + +/* + * The list of bits in the core_gvfs setting + */ +#define GVFS_SKIP_SHA_ON_INDEX (1 << 0) + static inline int gvfs_config_is_set(int mask) { return (core_gvfs & mask) == mask; } diff --git a/read-cache.c b/read-cache.c index bbcf488c20a483..a6c71ff0c8420e 100644 --- a/read-cache.c +++ b/read-cache.c @@ -24,6 +24,7 @@ #include "fsmonitor.h" #include "thread-utils.h" #include "progress.h" +#include "gvfs.h" /* Mask for the name length in ce_flags in the on-disk index */ @@ -2409,7 +2410,9 @@ static int ce_write_flush(git_hash_ctx *context, int fd) { unsigned int buffered = write_buffer_len; if (buffered) { - the_hash_algo->update_fn(context, write_buffer, buffered); + if (!gvfs_config_is_set(GVFS_SKIP_SHA_ON_INDEX)) + the_hash_algo->update_fn(context, write_buffer, + buffered); if (write_in_full(fd, write_buffer, buffered) < 0) return -1; write_buffer_len = 0; @@ -2458,7 +2461,8 @@ static int ce_flush(git_hash_ctx *context, int fd, unsigned char *hash) if (left) { write_buffer_len = 0; - the_hash_algo->update_fn(context, write_buffer, left); + if (!gvfs_config_is_set(GVFS_SKIP_SHA_ON_INDEX)) + the_hash_algo->update_fn(context, write_buffer, left); } /* Flush first if not enough space for hash signature */ @@ -2469,7 +2473,8 @@ static int ce_flush(git_hash_ctx *context, int fd, unsigned char *hash) } /* Append the hash signature at the end */ - the_hash_algo->final_fn(write_buffer + left, context); + if (!gvfs_config_is_set(GVFS_SKIP_SHA_ON_INDEX)) + the_hash_algo->final_fn(write_buffer + left, context); hashcpy(hash, write_buffer + left); left += the_hash_algo->rawsz; return (write_in_full(fd, write_buffer, left) < 0) ? -1 : 0; diff --git a/t/t1016-read-tree-skip-sha-on-read.sh b/t/t1016-read-tree-skip-sha-on-read.sh new file mode 100755 index 00000000000000..5b76a80a0020dc --- /dev/null +++ b/t/t1016-read-tree-skip-sha-on-read.sh @@ -0,0 +1,22 @@ +#!/bin/sh + +test_description='check that read-tree works with core.gvfs config value' + +. ./test-lib.sh +. "$TEST_DIRECTORY"/lib-read-tree.sh + +test_expect_success setup ' + echo one >a && + git add a && + git commit -m initial +' +test_expect_success 'read-tree without core.gvsf' ' + read_tree_u_must_succeed -m -u HEAD +' + +test_expect_success 'read-tree with core.gvfs set to 1' ' + git config core.gvfs 1 && + read_tree_u_must_succeed -m -u HEAD +' + +test_done From dea359752a5bfd6164806e03d1cc68370b3e9e0e Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Tue, 24 Jan 2017 17:54:55 +0100 Subject: [PATCH 007/118] gvfs: add the feature that blobs may be missing Signed-off-by: Kevin Willford --- Documentation/config/core.txt | 4 ++++ cache-tree.c | 4 +++- gvfs.h | 1 + t/t0000-basic.sh | 5 +++++ 4 files changed, 13 insertions(+), 1 deletion(-) diff --git a/Documentation/config/core.txt b/Documentation/config/core.txt index 72cc6e949c4715..f0ce1df43c47b3 100644 --- a/Documentation/config/core.txt +++ b/Documentation/config/core.txt @@ -612,6 +612,10 @@ core.gvfs:: GVFS_SKIP_SHA_ON_INDEX:: Bit value 1 Disables the calculation of the sha when writing the index + GVFS_MISSING_OK:: + Bit value 4 + Normally git write-tree ensures that the objects referenced by the + directory exist in the object database. This option disables this check. -- core.sparseCheckout:: diff --git a/cache-tree.c b/cache-tree.c index 706ffcf1883a01..e580bb649eb899 100644 --- a/cache-tree.c +++ b/cache-tree.c @@ -5,6 +5,7 @@ #include "cache-tree.h" #include "object-store.h" #include "replace-object.h" +#include "gvfs.h" #ifndef DEBUG_CACHE_TREE #define DEBUG_CACHE_TREE 0 @@ -243,7 +244,8 @@ static int update_one(struct cache_tree *it, int flags) { struct strbuf buffer; - int missing_ok = flags & WRITE_TREE_MISSING_OK; + int missing_ok = gvfs_config_is_set(GVFS_MISSING_OK) ? + WRITE_TREE_MISSING_OK : (flags & WRITE_TREE_MISSING_OK); int dryrun = flags & WRITE_TREE_DRY_RUN; int repair = flags & WRITE_TREE_REPAIR; int to_invalidate = 0; diff --git a/gvfs.h b/gvfs.h index 690419127a72dd..dabbf67f94c852 100644 --- a/gvfs.h +++ b/gvfs.h @@ -14,6 +14,7 @@ * The list of bits in the core_gvfs setting */ #define GVFS_SKIP_SHA_ON_INDEX (1 << 0) +#define GVFS_MISSING_OK (1 << 2) static inline int gvfs_config_is_set(int mask) { return (core_gvfs & mask) == mask; diff --git a/t/t0000-basic.sh b/t/t0000-basic.sh index b6566003dd8704..c5e6ec3f3c7e96 100755 --- a/t/t0000-basic.sh +++ b/t/t0000-basic.sh @@ -1079,6 +1079,11 @@ test_expect_success 'writing this tree with --missing-ok' ' git write-tree --missing-ok ' +test_expect_success 'writing this tree with missing ok config value' ' + git config core.gvfs 4 && + git write-tree +' + ################################################################ test_expect_success 'git read-tree followed by write-tree should be idempotent' ' From cd7014a3bb41c4474be9a1316386dc04ece0543f Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Wed, 18 May 2016 13:40:39 +0000 Subject: [PATCH 008/118] gvfs: prevent files to be deleted outside the sparse checkout Prevent the sparse checkout to delete files that were marked with skip-worktree bit and are not in the sparse-checkout file. This is because everything with the skip-worktree bit turned on is being virtualized and will be removed with the change of HEAD. There was only one failing test when running with these changes that was checking to make sure the worktree narrows on checkout which was expected since we would no longer be narrowing the worktree. Signed-off-by: Kevin Willford --- Documentation/config/core.txt | 9 +++++++++ gvfs.h | 1 + t/t1090-sparse-checkout-scope.sh | 17 +++++++++++++++++ unpack-trees.c | 22 ++++++++++++++++++++++ 4 files changed, 49 insertions(+) diff --git a/Documentation/config/core.txt b/Documentation/config/core.txt index f0ce1df43c47b3..77430dcbd66f62 100644 --- a/Documentation/config/core.txt +++ b/Documentation/config/core.txt @@ -616,6 +616,15 @@ core.gvfs:: Bit value 4 Normally git write-tree ensures that the objects referenced by the directory exist in the object database. This option disables this check. + GVFS_NO_DELETE_OUTSIDE_SPARSECHECKOUT:: + Bit value 8 + When marking entries to remove from the index and the working + directory this option will take into account what the + skip-worktree bit was set to so that if the entry has the + skip-worktree bit set it will not be removed from the working + directory. This will allow virtualized working directories to + detect the change to HEAD and use the new commit tree to show + the files that are in the working directory. -- core.sparseCheckout:: diff --git a/gvfs.h b/gvfs.h index dabbf67f94c852..f9144d2ad9ab92 100644 --- a/gvfs.h +++ b/gvfs.h @@ -15,6 +15,7 @@ */ #define GVFS_SKIP_SHA_ON_INDEX (1 << 0) #define GVFS_MISSING_OK (1 << 2) +#define GVFS_NO_DELETE_OUTSIDE_SPARSECHECKOUT (1 << 3) static inline int gvfs_config_is_set(int mask) { return (core_gvfs & mask) == mask; diff --git a/t/t1090-sparse-checkout-scope.sh b/t/t1090-sparse-checkout-scope.sh index 6e61b17c84d3e8..62735a6113c980 100755 --- a/t/t1090-sparse-checkout-scope.sh +++ b/t/t1090-sparse-checkout-scope.sh @@ -96,6 +96,23 @@ test_expect_success 'in partial clone, sparse checkout only fetches needed blobs test_cmp expect actual ' +test_expect_success 'checkout does not delete items outside the sparse checkout file' ' + git checkout master && + git config core.gvfs 8 && + git checkout -b outside && + echo "new file1" >d && + git add d && + git commit -m "branch initial" && + echo "new file1" >e && + git add e && + git commit -m "skipped worktree" && + git update-index --skip-worktree e && + echo "/d" >.git/info/sparse-checkout && + git checkout HEAD^ && + test_path_is_file d && + test_path_is_file e +' + test_expect_success MINGW 'no unnecessary opendir() with fscache' ' git clone . fscache-test && ( diff --git a/unpack-trees.c b/unpack-trees.c index 3563daae1aa6bc..7b410b0421e072 100644 --- a/unpack-trees.c +++ b/unpack-trees.c @@ -17,6 +17,7 @@ #include "fsmonitor.h" #include "object-store.h" #include "fetch-object.h" +#include "gvfs.h" /* * Error messages expected by scripts out of plumbing commands such as @@ -2088,6 +2089,27 @@ static int deleted_entry(const struct cache_entry *ce, } if (!(old->ce_flags & CE_CONFLICTED) && verify_uptodate(old, o)) return -1; + + /* + * When marking entries to remove from the index and the working + * directory this option will take into account what the + * skip-worktree bit was set to so that if the entry has the + * skip-worktree bit set it will not be removed from the working + * directory. This will allow virtualized working directories to + * detect the change to HEAD and use the new commit tree to show + * the files that are in the working directory. + * + * old is the cache_entry that will have the skip-worktree bit set + * which will need to be preserved when the CE_REMOVE entry is added + */ + if (gvfs_config_is_set(GVFS_NO_DELETE_OUTSIDE_SPARSECHECKOUT) && + old && + old->ce_flags & CE_SKIP_WORKTREE) { + add_entry(o, old, CE_REMOVE, 0); + invalidate_ce_path(old, o); + return 1; + } + add_entry(o, ce, CE_REMOVE, 0); invalidate_ce_path(ce, o); return 1; From e439e6a3a27be11527ab212dd03e8d2a92e039c8 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Wed, 22 Feb 2017 12:50:43 -0700 Subject: [PATCH 009/118] sparse-checkout: update files with a modify/delete conflict When using the sparse-checkout feature, the file might not be on disk because the skip-worktree bit is on. Signed-off-by: Kevin Willford --- merge-recursive.c | 2 +- t/t7615-merge-sparse-checkout.sh | 30 ++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) create mode 100755 t/t7615-merge-sparse-checkout.sh diff --git a/merge-recursive.c b/merge-recursive.c index 255347d23ed261..78183cbce827c2 100644 --- a/merge-recursive.c +++ b/merge-recursive.c @@ -1512,7 +1512,7 @@ static int handle_change_delete(struct merge_options *o, * path. We could call update_file_flags() with update_cache=0 * and update_wd=0, but that's a no-op. */ - if (change_branch != o->branch1 || alt_path) + if (change_branch != o->branch1 || alt_path || !file_exists(update_path)) ret = update_file(o, 0, changed_oid, changed_mode, update_path); } free(alt_path); diff --git a/t/t7615-merge-sparse-checkout.sh b/t/t7615-merge-sparse-checkout.sh new file mode 100755 index 00000000000000..8e8208a61d39eb --- /dev/null +++ b/t/t7615-merge-sparse-checkout.sh @@ -0,0 +1,30 @@ +#!/bin/sh + +test_description='merge can handle sparse-checkout' + +. ./test-lib.sh + +# merges with conflicts + +test_expect_success 'setup' ' + test_commit a && + test_commit file && + git checkout -b delete-file && + git rm file.t && + test_tick && + git commit -m "remove file" && + git checkout master && + test_commit modify file.t changed +' + +test_expect_success 'merge conflict deleted file and modified' ' + echo "/a.t" >.git/info/sparse-checkout && + test_config core.sparsecheckout true && + git checkout -f && + test_path_is_missing file.t && + test_must_fail git merge delete-file && + test_path_is_file file.t && + test "changed" = "$(cat file.t)" +' + +test_done From 9e94a98e82997bea4a978073f347cc72482eab41 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Wed, 1 Mar 2017 15:17:12 -0800 Subject: [PATCH 010/118] sparse-checkout: avoid writing entries with the skip-worktree bit When using the sparse-checkout feature git should not write to the working directory for files with the skip-worktree bit on. With the skip-worktree bit on the file may or may not be in the working directory and if it is not we don't want or need to create it by calling checkout_entry. There are two callers of checkout_target. Both of which check that the file does not exist before calling checkout_target. load_current which make a call to lstat right before calling checkout_target and check_preimage which will only run checkout_taret it stat_ret is less than zero. It sets stat_ret to zero and only if !stat->cached will it lstat the file and set stat_ret to something other than zero. This patch checks if skip-worktree bit is on in checkout_target and just returns so that the entry doesn't not end up in the working directory. This is so that apply will not create a file in the working directory, then update the index but not keep the working directory up to date with the changes that happened in the index. Signed-off-by: Kevin Willford --- apply.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/apply.c b/apply.c index 38283788b1772a..9add67802916cf 100644 --- a/apply.c +++ b/apply.c @@ -3349,6 +3349,24 @@ static int checkout_target(struct index_state *istate, { struct checkout costate = CHECKOUT_INIT; + /* + * Do not checkout the entry if the skipworktree bit is set + * + * Both callers of this method (check_preimage and load_current) + * check for the existance of the file before calling this + * method so we know that the file doesn't exist at this point + * and we don't need to perform that check again here. + * We just need to check the skip-worktree and return. + * + * This is to prevent git from creating a file in the + * working directory that has the skip-worktree bit on, + * then updating the index from the patch and not keeping + * the working directory version up to date with what it + * changed the index version to be. + */ + if (ce_skip_worktree(ce)) + return 0; + costate.refresh_cache = 1; costate.istate = istate; if (checkout_entry(ce, &costate, NULL, NULL) || From 5f8e1023e071dc981006d61499965a80d6d044e0 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Mon, 24 Sep 2018 19:07:28 +0000 Subject: [PATCH 011/118] fsck: use ERROR_MULTI_PACK_INDEX The multi-pack-index was added to the data verified by git-fsck in ea5ae6c3 "fsck: verify multi-pack-index". This implementation was based on the implementation for verifying the commit-graph, and a copy-paste error kept the ERROR_COMMIT_GRAPH flag as the bit set when an error appears in the multi-pack-index. Add a new flag, ERROR_MULTI_PACK_INDEX, and use that instead. Signed-off-by: Derrick Stolee --- builtin/fsck.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/builtin/fsck.c b/builtin/fsck.c index bb4227bebc606d..59a96e8c0dd1be 100644 --- a/builtin/fsck.c +++ b/builtin/fsck.c @@ -49,6 +49,7 @@ static int name_objects; #define ERROR_PACK 04 #define ERROR_REFS 010 #define ERROR_COMMIT_GRAPH 020 +#define ERROR_MULTI_PACK_INDEX 040 static const char *describe_object(struct object *obj) { @@ -906,7 +907,7 @@ int cmd_fsck(int argc, const char **argv, const char *prefix) midx_argv[2] = "--object-dir"; midx_argv[3] = odb->path; if (run_command(&midx_verify)) - errors_found |= ERROR_COMMIT_GRAPH; + errors_found |= ERROR_MULTI_PACK_INDEX; } } From e06eb8e8ec6f60ac5110856ebae4ed6192adb327 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Mon, 17 Dec 2018 16:42:53 -0500 Subject: [PATCH 012/118] repack: refactor pack deletion for future use The repack builtin deletes redundant pack-files and their associated .idx, .promisor, .bitmap, and .keep files. We will want to re-use this logic in the future for other types of repack, so pull the logic into 'unlink_pack_path()' in packfile.c. The 'ignore_keep' parameter is enabled for the use in repack, but will be important for a future caller. Signed-off-by: Derrick Stolee --- builtin/repack.c | 14 ++------------ packfile.c | 28 ++++++++++++++++++++++++++++ packfile.h | 7 +++++++ 3 files changed, 37 insertions(+), 12 deletions(-) diff --git a/builtin/repack.c b/builtin/repack.c index 67f8978043a439..1f9e6fad1bc035 100644 --- a/builtin/repack.c +++ b/builtin/repack.c @@ -129,19 +129,9 @@ static void get_non_kept_pack_filenames(struct string_list *fname_list, static void remove_redundant_pack(const char *dir_name, const char *base_name) { - const char *exts[] = {".pack", ".idx", ".keep", ".bitmap", ".promisor"}; - int i; struct strbuf buf = STRBUF_INIT; - size_t plen; - - strbuf_addf(&buf, "%s/%s", dir_name, base_name); - plen = buf.len; - - for (i = 0; i < ARRAY_SIZE(exts); i++) { - strbuf_setlen(&buf, plen); - strbuf_addstr(&buf, exts[i]); - unlink(buf.buf); - } + strbuf_addf(&buf, "%s/%s.pack", dir_name, base_name); + unlink_pack_path(buf.buf, 1); strbuf_release(&buf); } diff --git a/packfile.c b/packfile.c index 16bcb75262d918..713476b9af2fda 100644 --- a/packfile.c +++ b/packfile.c @@ -352,6 +352,34 @@ void close_all_packs(struct raw_object_store *o) } } +void unlink_pack_path(const char *pack_name, int force_delete) +{ + static const char *exts[] = {".pack", ".idx", ".keep", ".bitmap", ".promisor"}; + int i; + struct strbuf buf = STRBUF_INIT; + size_t plen; + + strbuf_addstr(&buf, pack_name); + strip_suffix_mem(buf.buf, &buf.len, ".pack"); + plen = buf.len; + + if (!force_delete) { + strbuf_addstr(&buf, ".keep"); + if (!access(buf.buf, F_OK)) { + strbuf_release(&buf); + return; + } + } + + for (i = 0; i < ARRAY_SIZE(exts); i++) { + strbuf_setlen(&buf, plen); + strbuf_addstr(&buf, exts[i]); + unlink(buf.buf); + } + + strbuf_release(&buf); +} + /* * The LRU pack is the one with the oldest MRU window, preferring packs * with no used windows, or the oldest mtime if it has no windows allocated. diff --git a/packfile.h b/packfile.h index d70c6d9afb94c7..cd757f6e69d0f5 100644 --- a/packfile.h +++ b/packfile.h @@ -86,6 +86,13 @@ extern void unuse_pack(struct pack_window **); extern void clear_delta_base_cache(void); extern struct packed_git *add_packed_git(const char *path, size_t path_len, int local); +/* + * Unlink the .pack and associated extension files. + * Does not unlink if 'force_delete' is false and the pack-file is + * marked as ".keep". + */ +extern void unlink_pack_path(const char *pack_name, int force_delete); + /* * Make sure that a pointer access into an mmap'd index file is within bounds, * and can provide at least 8 bytes of data. From 13e95e66f7557df680eb5d1677d6f5bb3b8c0d2b Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Fri, 21 Dec 2018 09:27:26 -0500 Subject: [PATCH 013/118] Docs: rearrange subcommands for multi-pack-index MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We will add new subcommands to the multi-pack-index, and that will make the documentation a bit messier. Clean up the 'verb' descriptions by renaming the concept to 'subcommand' and removing the reference to the object directory. Helped-by: Stefan Beller Helped-by: Szeder Gábor Signed-off-by: Derrick Stolee --- Documentation/git-multi-pack-index.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Documentation/git-multi-pack-index.txt b/Documentation/git-multi-pack-index.txt index f7778a2c85c1aa..1af406aca21922 100644 --- a/Documentation/git-multi-pack-index.txt +++ b/Documentation/git-multi-pack-index.txt @@ -9,7 +9,7 @@ git-multi-pack-index - Write and verify multi-pack-indexes SYNOPSIS -------- [verse] -'git multi-pack-index' [--object-dir=] +'git multi-pack-index' [--object-dir=] DESCRIPTION ----------- @@ -23,13 +23,13 @@ OPTIONS `/packs/multi-pack-index` for the current MIDX file, and `/packs` for the pack-files to index. +The following subcommands are available: + write:: - When given as the verb, write a new MIDX file to - `/packs/multi-pack-index`. + Write a new MIDX file. verify:: - When given as the verb, verify the contents of the MIDX file - at `/packs/multi-pack-index`. + Verify the contents of the MIDX file. EXAMPLES From 05443a544034ad0685db3eaabbc1843d53e4fbb7 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Wed, 5 Dec 2018 08:11:39 -0500 Subject: [PATCH 014/118] multi-pack-index: prepare for 'expire' subcommand The multi-pack-index tracks objects in a collection of pack-files. Only one copy of each object is indexed, using the modified time of the pack-files to determine tie-breakers. It is possible to have a pack-file with no referenced objects because all objects have a duplicate in a newer pack-file. Introduce a new 'expire' subcommand to the multi-pack-index builtin. This subcommand will delete these unused pack-files and rewrite the multi-pack-index to no longer refer to those files. More details about the specifics will follow as the method is implemented. Add a test that verifies the 'expire' subcommand is correctly wired, but will still be valid when the verb is implemented. Specifically, create a set of packs that should all have referenced objects and should not be removed during an 'expire' operation. Signed-off-by: Derrick Stolee --- Documentation/git-multi-pack-index.txt | 5 +++ builtin/multi-pack-index.c | 4 ++- midx.c | 5 +++ midx.h | 1 + t/t5319-multi-pack-index.sh | 47 ++++++++++++++++++++++++++ 5 files changed, 61 insertions(+), 1 deletion(-) diff --git a/Documentation/git-multi-pack-index.txt b/Documentation/git-multi-pack-index.txt index 1af406aca21922..6186c4c9369a9c 100644 --- a/Documentation/git-multi-pack-index.txt +++ b/Documentation/git-multi-pack-index.txt @@ -31,6 +31,11 @@ write:: verify:: Verify the contents of the MIDX file. +expire:: + Delete the pack-files that are tracked by the MIDX file, but + have no objects referenced by the MIDX. Rewrite the MIDX file + afterward to remove all references to these pack-files. + EXAMPLES -------- diff --git a/builtin/multi-pack-index.c b/builtin/multi-pack-index.c index fca70f8e4fcca8..145de3a46c2418 100644 --- a/builtin/multi-pack-index.c +++ b/builtin/multi-pack-index.c @@ -5,7 +5,7 @@ #include "midx.h" static char const * const builtin_multi_pack_index_usage[] = { - N_("git multi-pack-index [--object-dir=] (write|verify)"), + N_("git multi-pack-index [--object-dir=] (write|verify|expire)"), NULL }; @@ -44,6 +44,8 @@ int cmd_multi_pack_index(int argc, const char **argv, return write_midx_file(opts.object_dir); if (!strcmp(argv[0], "verify")) return verify_midx_file(opts.object_dir); + if (!strcmp(argv[0], "expire")) + return expire_midx_packs(opts.object_dir); die(_("unrecognized verb: %s"), argv[0]); } diff --git a/midx.c b/midx.c index 8a505fd423efcc..4c930876ef6cf5 100644 --- a/midx.c +++ b/midx.c @@ -1025,3 +1025,8 @@ int verify_midx_file(const char *object_dir) return verify_midx_error; } + +int expire_midx_packs(const char *object_dir) +{ + return 0; +} diff --git a/midx.h b/midx.h index 774f652530c429..e3a2b740b59057 100644 --- a/midx.h +++ b/midx.h @@ -49,6 +49,7 @@ int prepare_multi_pack_index_one(struct repository *r, const char *object_dir, i int write_midx_file(const char *object_dir); void clear_midx_file(struct repository *r); int verify_midx_file(const char *object_dir); +int expire_midx_packs(const char *object_dir); void close_midx(struct multi_pack_index *m); diff --git a/t/t5319-multi-pack-index.sh b/t/t5319-multi-pack-index.sh index 70926b5bc04643..948effc1eed544 100755 --- a/t/t5319-multi-pack-index.sh +++ b/t/t5319-multi-pack-index.sh @@ -348,4 +348,51 @@ test_expect_success 'verify incorrect 64-bit offset' ' "incorrect object offset" ' +test_expect_success 'setup expire tests' ' + mkdir dup && + ( + cd dup && + git init && + for i in $(test_seq 1 20) + do + test_commit $i + done && + git branch A HEAD && + git branch B HEAD~8 && + git branch C HEAD~13 && + git branch D HEAD~16 && + git branch E HEAD~18 && + git pack-objects --revs .git/objects/pack/pack-E <<-EOF && + refs/heads/E + EOF + git pack-objects --revs .git/objects/pack/pack-D <<-EOF && + refs/heads/D + ^refs/heads/E + EOF + git pack-objects --revs .git/objects/pack/pack-C <<-EOF && + refs/heads/C + ^refs/heads/D + EOF + git pack-objects --revs .git/objects/pack/pack-B <<-EOF && + refs/heads/B + ^refs/heads/C + EOF + git pack-objects --revs .git/objects/pack/pack-A <<-EOF && + refs/heads/A + ^refs/heads/B + EOF + git multi-pack-index write + ) +' + +test_expect_success 'expire does not remove any packs' ' + ( + cd dup && + ls .git/objects/pack >expect && + git multi-pack-index expire && + ls .git/objects/pack >actual && + test_cmp expect actual + ) +' + test_done From c9772d2430f020824c7b318db5c1116969a7d44f Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Wed, 5 Dec 2018 09:08:57 -0500 Subject: [PATCH 015/118] midx: refactor permutation logic When writing a multi-pack-index, we keep track of an integer permutation, tracking the list of pack-files that we know about (both from the existing multi-pack-index and the new pack-files being introduced) and converting them into a sorted order for the new multi-pack-index. In anticipation of dropping pack-files from the existing multi- pack-index, refactor the logic around how we track this permutation. First, insert the permutation into the pack_list structure. This allows us to grow the permutation dynamically as we add packs. Second, fill the permutation with values corresponding to their position in the list of pack-files, sorted as follows: 1. The pack-files in the existing multi-pack-index, sorted lexicographically. 2. The pack-files not in the existing multi-pack-index, sorted as discovered from the filesystem. There is a subtle thing in how we initialize this permutation, specifically how we use 'i' for the initial value. This will matter more when we implement the logic for dropping existing packs, as we will create holes in the ordering. Signed-off-by: Derrick Stolee --- midx.c | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/midx.c b/midx.c index 4c930876ef6cf5..6e0a5e392eaf1c 100644 --- a/midx.c +++ b/midx.c @@ -380,9 +380,11 @@ static size_t write_midx_header(struct hashfile *f, struct pack_list { struct packed_git **list; char **names; + uint32_t *perm; uint32_t nr; uint32_t alloc_list; uint32_t alloc_names; + uint32_t alloc_perm; size_t pack_name_concat_len; struct multi_pack_index *m; }; @@ -398,6 +400,7 @@ static void add_pack_to_midx(const char *full_path, size_t full_path_len, ALLOC_GROW(packs->list, packs->nr + 1, packs->alloc_list); ALLOC_GROW(packs->names, packs->nr + 1, packs->alloc_names); + ALLOC_GROW(packs->perm, packs->nr + 1, packs->alloc_perm); packs->list[packs->nr] = add_packed_git(full_path, full_path_len, @@ -417,6 +420,7 @@ static void add_pack_to_midx(const char *full_path, size_t full_path_len, return; } + packs->perm[packs->nr] = packs->nr; packs->names[packs->nr] = xstrdup(file_name); packs->pack_name_concat_len += strlen(file_name) + 1; packs->nr++; @@ -443,7 +447,7 @@ static void sort_packs_by_name(char **pack_names, uint32_t nr_packs, uint32_t *p ALLOC_ARRAY(pairs, nr_packs); for (i = 0; i < nr_packs; i++) { - pairs[i].pack_int_id = i; + pairs[i].pack_int_id = perm[i]; pairs[i].pack_name = pack_names[i]; } @@ -755,7 +759,6 @@ int write_midx_file(const char *object_dir) struct hashfile *f = NULL; struct lock_file lk; struct pack_list packs; - uint32_t *pack_perm = NULL; uint64_t written = 0; uint32_t chunk_ids[MIDX_MAX_CHUNKS + 1]; uint64_t chunk_offsets[MIDX_MAX_CHUNKS + 1]; @@ -774,18 +777,22 @@ int write_midx_file(const char *object_dir) packs.nr = 0; packs.alloc_list = packs.m ? packs.m->num_packs : 16; - packs.alloc_names = packs.alloc_list; + packs.alloc_perm = packs.alloc_names = packs.alloc_list; packs.list = NULL; packs.names = NULL; + packs.perm = NULL; packs.pack_name_concat_len = 0; ALLOC_ARRAY(packs.list, packs.alloc_list); ALLOC_ARRAY(packs.names, packs.alloc_names); + ALLOC_ARRAY(packs.perm, packs.alloc_perm); if (packs.m) { for (i = 0; i < packs.m->num_packs; i++) { ALLOC_GROW(packs.list, packs.nr + 1, packs.alloc_list); ALLOC_GROW(packs.names, packs.nr + 1, packs.alloc_names); + ALLOC_GROW(packs.perm, packs.nr + 1, packs.alloc_perm); + packs.perm[packs.nr] = i; packs.list[packs.nr] = NULL; packs.names[packs.nr] = xstrdup(packs.m->pack_names[i]); packs.pack_name_concat_len += strlen(packs.names[packs.nr]) + 1; @@ -802,10 +809,9 @@ int write_midx_file(const char *object_dir) packs.pack_name_concat_len += MIDX_CHUNK_ALIGNMENT - (packs.pack_name_concat_len % MIDX_CHUNK_ALIGNMENT); - ALLOC_ARRAY(pack_perm, packs.nr); - sort_packs_by_name(packs.names, packs.nr, pack_perm); + sort_packs_by_name(packs.names, packs.nr, packs.perm); - entries = get_sorted_entries(packs.m, packs.list, pack_perm, packs.nr, &nr_entries); + entries = get_sorted_entries(packs.m, packs.list, packs.perm, packs.nr, &nr_entries); for (i = 0; i < nr_entries; i++) { if (entries[i].offset > 0x7fffffff) @@ -923,8 +929,8 @@ int write_midx_file(const char *object_dir) free(packs.list); free(packs.names); + free(packs.perm); free(entries); - free(pack_perm); free(midx_name); return 0; } From 6e5954710283ccf7e3a3b4b8f0ad1940686868b5 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Wed, 5 Dec 2018 08:39:29 -0500 Subject: [PATCH 016/118] multi-pack-index: implement 'expire' verb The 'git multi-pack-index expire' command looks at the existing mult-pack-index, counts the number of objects referenced in each pack-file, deletes the pack-fils with no referenced objects, and rewrites the multi-pack-index to no longer reference those packs. Refactor the write_midx_file() method to call write_midx_internal() which now takes an existing 'struct multi_pack_index' and a list of pack-files to drop (as specified by the names of their pack- indexes). As we write the new multi-pack-index, we drop those file names from the list of known pack-files. The expire_midx_packs() method removes the unreferenced pack-files after carefully closing the packs to avoid open handles. Test that a new pack-file that covers the contents of two other pack-files leads to those pack-files being deleted during the expire command. Be sure to read the multi-pack-index to ensure it no longer references those packs. Signed-off-by: Derrick Stolee --- midx.c | 82 +++++++++++++++++++++++++++++++++++-- t/t5319-multi-pack-index.sh | 15 +++++++ 2 files changed, 93 insertions(+), 4 deletions(-) diff --git a/midx.c b/midx.c index 6e0a5e392eaf1c..b30f1e2665499f 100644 --- a/midx.c +++ b/midx.c @@ -751,7 +751,8 @@ static size_t write_midx_large_offsets(struct hashfile *f, uint32_t nr_large_off return written; } -int write_midx_file(const char *object_dir) +static int write_midx_internal(const char *object_dir, struct multi_pack_index *m, + struct string_list *packs_to_drop) { unsigned char cur_chunk, num_chunks = 0; char *midx_name; @@ -765,6 +766,7 @@ int write_midx_file(const char *object_dir) uint32_t nr_entries, num_large_offsets = 0; struct pack_midx_entry *entries = NULL; int large_offsets_needed = 0; + int result = 0; midx_name = get_midx_filename(object_dir); if (safe_create_leading_directories(midx_name)) { @@ -773,7 +775,10 @@ int write_midx_file(const char *object_dir) midx_name); } - packs.m = load_multi_pack_index(object_dir, 1); + if (m) + packs.m = m; + else + packs.m = load_multi_pack_index(object_dir, 1); packs.nr = 0; packs.alloc_list = packs.m ? packs.m->num_packs : 16; @@ -787,7 +792,25 @@ int write_midx_file(const char *object_dir) ALLOC_ARRAY(packs.perm, packs.alloc_perm); if (packs.m) { + int drop_index = 0, missing_drops = 0; for (i = 0; i < packs.m->num_packs; i++) { + if (packs_to_drop && drop_index < packs_to_drop->nr) { + int cmp = strcmp(packs.m->pack_names[i], + packs_to_drop->items[drop_index].string); + + if (!cmp) { + drop_index++; + continue; + } else if (cmp > 0) { + error(_("did not see pack-file %s to drop"), + packs_to_drop->items[drop_index].string); + drop_index++; + i--; + missing_drops++; + continue; + } + } + ALLOC_GROW(packs.list, packs.nr + 1, packs.alloc_list); ALLOC_GROW(packs.names, packs.nr + 1, packs.alloc_names); ALLOC_GROW(packs.perm, packs.nr + 1, packs.alloc_perm); @@ -798,6 +821,12 @@ int write_midx_file(const char *object_dir) packs.pack_name_concat_len += strlen(packs.names[packs.nr]) + 1; packs.nr++; } + + if (packs_to_drop && (drop_index < packs_to_drop->nr || missing_drops)) { + error(_("did not see all pack-files to drop")); + result = 1; + goto cleanup; + } } for_each_file_in_pack_dir(object_dir, add_pack_to_midx, &packs); @@ -932,7 +961,12 @@ int write_midx_file(const char *object_dir) free(packs.perm); free(entries); free(midx_name); - return 0; + return result; +} + +int write_midx_file(const char *object_dir) +{ + return write_midx_internal(object_dir, NULL, NULL); } void clear_midx_file(struct repository *r) @@ -1034,5 +1068,45 @@ int verify_midx_file(const char *object_dir) int expire_midx_packs(const char *object_dir) { - return 0; + uint32_t i, *count, result = 0; + struct string_list packs_to_drop = STRING_LIST_INIT_DUP; + struct multi_pack_index *m = load_multi_pack_index(object_dir, 1); + + if (!m) + return 0; + + count = xcalloc(m->num_packs, sizeof(uint32_t)); + for (i = 0; i < m->num_objects; i++) { + int pack_int_id = nth_midxed_pack_int_id(m, i); + count[pack_int_id]++; + } + + for (i = 0; i < m->num_packs; i++) { + char *pack_name; + + if (count[i]) + continue; + + if (prepare_midx_pack(m, i)) + continue; + + if (m->packs[i]->pack_keep) + continue; + + pack_name = xstrdup(m->packs[i]->pack_name); + close_pack(m->packs[i]); + FREE_AND_NULL(m->packs[i]); + + string_list_insert(&packs_to_drop, m->pack_names[i]); + unlink_pack_path(pack_name, 0); + free(pack_name); + } + + free(count); + + if (packs_to_drop.nr) + result = write_midx_internal(object_dir, m, &packs_to_drop); + + string_list_clear(&packs_to_drop, 0); + return result; } diff --git a/t/t5319-multi-pack-index.sh b/t/t5319-multi-pack-index.sh index 948effc1eed544..210279a3cf55e4 100755 --- a/t/t5319-multi-pack-index.sh +++ b/t/t5319-multi-pack-index.sh @@ -395,4 +395,19 @@ test_expect_success 'expire does not remove any packs' ' ) ' +test_expect_success 'expire removes unreferenced packs' ' + ( + cd dup && + git pack-objects --revs .git/objects/pack/pack-combined <<-EOF && + refs/heads/A + ^refs/heads/C + EOF + git multi-pack-index write && + ls .git/objects/pack | grep -v -e pack-[AB] >expect && + git multi-pack-index expire && + ls .git/objects/pack >actual && + test_cmp expect actual + ) +' + test_done From aeadfe89a37ad75fbcc6e63db7bab541bedbabb0 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Wed, 5 Dec 2018 09:37:13 -0500 Subject: [PATCH 017/118] multi-pack-index: prepare 'repack' subcommand In an environment where the multi-pack-index is useful, it is due to many pack-files and an inability to repack the object store into a single pack-file. However, it is likely that many of these pack-files are rather small, and could be repacked into a slightly larger pack-file without too much effort. It may also be important to ensure the object store is highly available and the repack operation does not interrupt concurrent git commands. Introduce a 'repack' subcommand to 'git multi-pack-index' that takes a '--batch-size' option. The verb will inspect the multi-pack-index for referenced pack-files whose size is smaller than the batch size, until collecting a list of pack-files whose sizes sum to larger than the batch size. Then, a new pack-file will be created containing the objects from those pack-files that are referenced by the multi-pack-index. The resulting pack is likely to actually be smaller than the batch size due to compression and the fact that there may be objects in the pack- files that have duplicate copies in other pack-files. The current change introduces the command-line arguments, and we add a test that ensures we parse these options properly. Since we specify a small batch size, we will guarantee that future implementations do not change the list of pack-files. Signed-off-by: Derrick Stolee --- Documentation/git-multi-pack-index.txt | 11 +++++++++++ builtin/multi-pack-index.c | 10 +++++++++- midx.c | 5 +++++ midx.h | 1 + t/t5319-multi-pack-index.sh | 11 +++++++++++ 5 files changed, 37 insertions(+), 1 deletion(-) diff --git a/Documentation/git-multi-pack-index.txt b/Documentation/git-multi-pack-index.txt index 6186c4c9369a9c..cc63531cc02b77 100644 --- a/Documentation/git-multi-pack-index.txt +++ b/Documentation/git-multi-pack-index.txt @@ -36,6 +36,17 @@ expire:: have no objects referenced by the MIDX. Rewrite the MIDX file afterward to remove all references to these pack-files. +repack:: + Collect a batch of pack-files whose size are all at most the + size given by --batch-size, but whose sizes sum to larger + than --batch-size. The batch is selected by greedily adding + small pack-files starting with the oldest pack-files that fit + the size. Create a new pack-file containing the objects the + multi-pack-index indexes into those pack-files, and rewrite + the multi-pack-index to contain that pack-file. A later run + of 'git multi-pack-index expire' will delete the pack-files + that were part of this batch. + EXAMPLES -------- diff --git a/builtin/multi-pack-index.c b/builtin/multi-pack-index.c index 145de3a46c2418..d87a2235e39715 100644 --- a/builtin/multi-pack-index.c +++ b/builtin/multi-pack-index.c @@ -5,12 +5,13 @@ #include "midx.h" static char const * const builtin_multi_pack_index_usage[] = { - N_("git multi-pack-index [--object-dir=] (write|verify|expire)"), + N_("git multi-pack-index [--object-dir=] (write|verify|expire|repack --batch-size=)"), NULL }; static struct opts_multi_pack_index { const char *object_dir; + unsigned long batch_size; } opts; int cmd_multi_pack_index(int argc, const char **argv, @@ -19,6 +20,8 @@ int cmd_multi_pack_index(int argc, const char **argv, static struct option builtin_multi_pack_index_options[] = { OPT_FILENAME(0, "object-dir", &opts.object_dir, N_("object directory containing set of packfile and pack-index pairs")), + OPT_MAGNITUDE(0, "batch-size", &opts.batch_size, + N_("during repack, collect pack-files of smaller size into a batch that is larger than this size")), OPT_END(), }; @@ -40,6 +43,11 @@ int cmd_multi_pack_index(int argc, const char **argv, return 1; } + if (!strcmp(argv[0], "repack")) + return midx_repack(opts.object_dir, (size_t)opts.batch_size); + if (opts.batch_size) + die(_("--batch-size option is only for 'repack' verb")); + if (!strcmp(argv[0], "write")) return write_midx_file(opts.object_dir); if (!strcmp(argv[0], "verify")) diff --git a/midx.c b/midx.c index b30f1e2665499f..596a8e974ad03b 100644 --- a/midx.c +++ b/midx.c @@ -1110,3 +1110,8 @@ int expire_midx_packs(const char *object_dir) string_list_clear(&packs_to_drop, 0); return result; } + +int midx_repack(const char *object_dir, size_t batch_size) +{ + return 0; +} diff --git a/midx.h b/midx.h index e3a2b740b59057..394a21ee96f0fd 100644 --- a/midx.h +++ b/midx.h @@ -50,6 +50,7 @@ int write_midx_file(const char *object_dir); void clear_midx_file(struct repository *r); int verify_midx_file(const char *object_dir); int expire_midx_packs(const char *object_dir); +int midx_repack(const char *object_dir, size_t batch_size); void close_midx(struct multi_pack_index *m); diff --git a/t/t5319-multi-pack-index.sh b/t/t5319-multi-pack-index.sh index 210279a3cf55e4..f6756210802524 100755 --- a/t/t5319-multi-pack-index.sh +++ b/t/t5319-multi-pack-index.sh @@ -410,4 +410,15 @@ test_expect_success 'expire removes unreferenced packs' ' ) ' +test_expect_success 'repack with minimum size does not alter existing packs' ' + ( + cd dup && + ls .git/objects/pack >expect && + MINSIZE=$(ls -l .git/objects/pack/*pack | awk "{print \$5;}" | sort -n | head -n 1) && + git multi-pack-index repack --batch-size=$MINSIZE && + ls .git/objects/pack >actual && + test_cmp expect actual + ) +' + test_done From f295fe686ced5f64533e4443ad584b51fd3a0bb5 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Wed, 5 Dec 2018 10:04:17 -0500 Subject: [PATCH 018/118] midx: implement midx_repack() To repack using a multi-pack-index, first sort all pack-files by their modified time. Second, walk those pack-files from oldest to newest, adding the packs to a list if they are smaller than the given pack-size. Finally, collect the objects from the multi-pack- index that are in those packs and send them to 'git pack-objects'. While first designing a 'git multi-pack-index repack' operation, I started by collecting the batches based on the size of the objects instead of the size of the pack-files. This allows repacking a large pack-file that has very few referencd objects. However, this came at a significant cost of parsing pack-files instead of simply reading the multi-pack-index and getting the file information for the pack-files. This object-size idea could be a direction for future expansion in this area. Signed-off-by: Derrick Stolee --- midx.c | 109 +++++++++++++++++++++++++++++++++++- t/t5319-multi-pack-index.sh | 25 +++++++++ 2 files changed, 133 insertions(+), 1 deletion(-) diff --git a/midx.c b/midx.c index 596a8e974ad03b..d672e3f729cc1f 100644 --- a/midx.c +++ b/midx.c @@ -8,6 +8,7 @@ #include "sha1-lookup.h" #include "midx.h" #include "progress.h" +#include "run-command.h" #define MIDX_SIGNATURE 0x4d494458 /* "MIDX" */ #define MIDX_VERSION 1 @@ -1111,7 +1112,113 @@ int expire_midx_packs(const char *object_dir) return result; } -int midx_repack(const char *object_dir, size_t batch_size) +struct time_and_id { + timestamp_t mtime; + uint32_t pack_int_id; +}; + +static int compare_by_mtime(const void *a_, const void *b_) { + const struct time_and_id *a, *b; + + a = (const struct time_and_id *)a_; + b = (const struct time_and_id *)b_; + + if (a->mtime < b->mtime) + return -1; + if (a->mtime > b->mtime) + return 1; return 0; } + +int midx_repack(const char *object_dir, size_t batch_size) +{ + int result = 0; + uint32_t i, packs_to_repack; + size_t total_size; + struct time_and_id *pack_ti; + unsigned char *include_pack; + struct child_process cmd = CHILD_PROCESS_INIT; + struct strbuf base_name = STRBUF_INIT; + struct multi_pack_index *m = load_multi_pack_index(object_dir, 1); + + if (!m) + return 0; + + include_pack = xcalloc(m->num_packs, sizeof(unsigned char)); + pack_ti = xcalloc(m->num_packs, sizeof(struct time_and_id)); + + for (i = 0; i < m->num_packs; i++) { + pack_ti[i].pack_int_id = i; + + if (prepare_midx_pack(m, i)) + continue; + + pack_ti[i].mtime = m->packs[i]->mtime; + } + QSORT(pack_ti, m->num_packs, compare_by_mtime); + + total_size = 0; + packs_to_repack = 0; + for (i = 0; total_size < batch_size && i < m->num_packs; i++) { + int pack_int_id = pack_ti[i].pack_int_id; + struct packed_git *p = m->packs[pack_int_id]; + + if (!p) + continue; + if (p->pack_size >= batch_size) + continue; + + packs_to_repack++; + total_size += p->pack_size; + include_pack[pack_int_id] = 1; + } + + if (total_size < batch_size || packs_to_repack < 2) + goto cleanup; + + argv_array_push(&cmd.args, "pack-objects"); + + strbuf_addstr(&base_name, object_dir); + strbuf_addstr(&base_name, "/pack/pack"); + argv_array_push(&cmd.args, base_name.buf); + strbuf_release(&base_name); + + cmd.git_cmd = 1; + cmd.in = cmd.out = -1; + + if (start_command(&cmd)) { + error(_("could not start pack-objects")); + result = 1; + goto cleanup; + } + + for (i = 0; i < m->num_objects; i++) { + struct object_id oid; + uint32_t pack_int_id = nth_midxed_pack_int_id(m, i); + + if (!include_pack[pack_int_id]) + continue; + + nth_midxed_object_oid(&oid, m, i); + xwrite(cmd.in, oid_to_hex(&oid), the_hash_algo->hexsz); + xwrite(cmd.in, "\n", 1); + } + close(cmd.in); + + if (finish_command(&cmd)) { + error(_("could not finish pack-objects")); + result = 1; + goto cleanup; + } + + result = write_midx_internal(object_dir, m, NULL); + m = NULL; + +cleanup: + if (m) + close_midx(m); + free(include_pack); + free(pack_ti); + return result; +} diff --git a/t/t5319-multi-pack-index.sh b/t/t5319-multi-pack-index.sh index f6756210802524..3f5e9ea6534206 100755 --- a/t/t5319-multi-pack-index.sh +++ b/t/t5319-multi-pack-index.sh @@ -421,4 +421,29 @@ test_expect_success 'repack with minimum size does not alter existing packs' ' ) ' +test_expect_success 'repack creates a new pack' ' + ( + cd dup && + SECOND_SMALLEST_SIZE=$(ls -l .git/objects/pack/*pack | awk "{print \$5;}" | sort -n | head -n 2 | tail -n 1) && + BATCH_SIZE=$(($SECOND_SMALLEST_SIZE + 1)) && + git multi-pack-index repack --batch-size=$BATCH_SIZE && + ls .git/objects/pack/*idx >idx-list && + test_line_count = 5 idx-list && + test-tool read-midx .git/objects | grep idx >midx-list && + test_line_count = 5 midx-list + ) +' + +test_expect_success 'expire removes repacked packs' ' + ( + cd dup && + ls -S .git/objects/pack/*pack | head -n 3 >expect && + git multi-pack-index expire && + ls -S .git/objects/pack/*pack >actual && + test_cmp expect actual && + test-tool read-midx .git/objects | grep idx >midx-list && + test_line_count = 3 midx-list + ) +' + test_done From 57a505d5e0db1ea415b2e21b73f798f689af907a Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Fri, 4 Jan 2019 09:37:43 -0500 Subject: [PATCH 019/118] midx: use more structured data for expire Signed-off-by: Derrick Stolee --- midx.c | 250 ++++++++++++++++++------------------ t/t5319-multi-pack-index.sh | 28 ++++ 2 files changed, 154 insertions(+), 124 deletions(-) diff --git a/midx.c b/midx.c index d672e3f729cc1f..0ed96391eccd8f 100644 --- a/midx.c +++ b/midx.c @@ -378,14 +378,26 @@ static size_t write_midx_header(struct hashfile *f, return MIDX_HEADER_SIZE; } +struct midx_info { + uint32_t orig_pack_int_id; + uint32_t new_pack_int_id; + char *pack_name; + struct packed_git *p; + unsigned expired : 1; +}; + +static int midx_info_compare(const void *_a, const void *_b) +{ + struct midx_info *a = (struct midx_info *)_a; + struct midx_info *b = (struct midx_info *)_b; + return strcmp(a->pack_name, b->pack_name); +} + struct pack_list { - struct packed_git **list; - char **names; + struct midx_info *info; uint32_t *perm; uint32_t nr; - uint32_t alloc_list; - uint32_t alloc_names; - uint32_t alloc_perm; + uint32_t alloc; size_t pack_name_concat_len; struct multi_pack_index *m; }; @@ -399,69 +411,33 @@ static void add_pack_to_midx(const char *full_path, size_t full_path_len, if (packs->m && midx_contains_pack(packs->m, file_name)) return; - ALLOC_GROW(packs->list, packs->nr + 1, packs->alloc_list); - ALLOC_GROW(packs->names, packs->nr + 1, packs->alloc_names); - ALLOC_GROW(packs->perm, packs->nr + 1, packs->alloc_perm); + ALLOC_GROW(packs->info, packs->nr + 1, packs->alloc); - packs->list[packs->nr] = add_packed_git(full_path, - full_path_len, - 0); + packs->info[packs->nr].p = add_packed_git(full_path, + full_path_len, + 0); + packs->info[packs->nr].expired = 0; - if (!packs->list[packs->nr]) { + if (!packs->info[packs->nr].p) { warning(_("failed to add packfile '%s'"), full_path); return; } - if (open_pack_index(packs->list[packs->nr])) { + if (open_pack_index(packs->info[packs->nr].p)) { warning(_("failed to open pack-index '%s'"), full_path); - close_pack(packs->list[packs->nr]); - FREE_AND_NULL(packs->list[packs->nr]); + close_pack(packs->info[packs->nr].p); + FREE_AND_NULL(packs->info[packs->nr].p); return; } - packs->perm[packs->nr] = packs->nr; - packs->names[packs->nr] = xstrdup(file_name); - packs->pack_name_concat_len += strlen(file_name) + 1; + packs->info[packs->nr].pack_name = xstrdup(file_name); + packs->info[packs->nr].orig_pack_int_id = packs->nr; packs->nr++; } } -struct pack_pair { - uint32_t pack_int_id; - char *pack_name; -}; - -static int pack_pair_compare(const void *_a, const void *_b) -{ - struct pack_pair *a = (struct pack_pair *)_a; - struct pack_pair *b = (struct pack_pair *)_b; - return strcmp(a->pack_name, b->pack_name); -} - -static void sort_packs_by_name(char **pack_names, uint32_t nr_packs, uint32_t *perm) -{ - uint32_t i; - struct pack_pair *pairs; - - ALLOC_ARRAY(pairs, nr_packs); - - for (i = 0; i < nr_packs; i++) { - pairs[i].pack_int_id = perm[i]; - pairs[i].pack_name = pack_names[i]; - } - - QSORT(pairs, nr_packs, pack_pair_compare); - - for (i = 0; i < nr_packs; i++) { - pack_names[i] = pairs[i].pack_name; - perm[pairs[i].pack_int_id] = i; - } - - free(pairs); -} - struct pack_midx_entry { struct object_id oid; uint32_t pack_int_id; @@ -487,7 +463,6 @@ static int midx_oid_compare(const void *_a, const void *_b) } static int nth_midxed_pack_midx_entry(struct multi_pack_index *m, - uint32_t *pack_perm, struct pack_midx_entry *e, uint32_t pos) { @@ -495,7 +470,7 @@ static int nth_midxed_pack_midx_entry(struct multi_pack_index *m, return 1; nth_midxed_object_oid(&e->oid, m, pos); - e->pack_int_id = pack_perm[nth_midxed_pack_int_id(m, pos)]; + e->pack_int_id = nth_midxed_pack_int_id(m, pos); e->offset = nth_midxed_offset(m, pos); /* consider objects in midx to be from "old" packs */ @@ -529,8 +504,7 @@ static void fill_pack_entry(uint32_t pack_int_id, * of a packfile containing the object). */ static struct pack_midx_entry *get_sorted_entries(struct multi_pack_index *m, - struct packed_git **p, - uint32_t *perm, + struct midx_info *info, uint32_t nr_packs, uint32_t *nr_objects) { @@ -541,7 +515,7 @@ static struct pack_midx_entry *get_sorted_entries(struct multi_pack_index *m, uint32_t start_pack = m ? m->num_packs : 0; for (cur_pack = start_pack; cur_pack < nr_packs; cur_pack++) - total_objects += p[cur_pack]->num_objects; + total_objects += info[cur_pack].p->num_objects; /* * As we de-duplicate by fanout value, we expect the fanout @@ -566,7 +540,7 @@ static struct pack_midx_entry *get_sorted_entries(struct multi_pack_index *m, for (cur_object = start; cur_object < end; cur_object++) { ALLOC_GROW(entries_by_fanout, nr_fanout + 1, alloc_fanout); - nth_midxed_pack_midx_entry(m, perm, + nth_midxed_pack_midx_entry(m, &entries_by_fanout[nr_fanout], cur_object); nr_fanout++; @@ -577,12 +551,12 @@ static struct pack_midx_entry *get_sorted_entries(struct multi_pack_index *m, uint32_t start = 0, end; if (cur_fanout) - start = get_pack_fanout(p[cur_pack], cur_fanout - 1); - end = get_pack_fanout(p[cur_pack], cur_fanout); + start = get_pack_fanout(info[cur_pack].p, cur_fanout - 1); + end = get_pack_fanout(info[cur_pack].p, cur_fanout); for (cur_object = start; cur_object < end; cur_object++) { ALLOC_GROW(entries_by_fanout, nr_fanout + 1, alloc_fanout); - fill_pack_entry(perm[cur_pack], p[cur_pack], cur_object, &entries_by_fanout[nr_fanout]); + fill_pack_entry(cur_pack, info[cur_pack].p, cur_object, &entries_by_fanout[nr_fanout]); nr_fanout++; } } @@ -611,7 +585,7 @@ static struct pack_midx_entry *get_sorted_entries(struct multi_pack_index *m, } static size_t write_midx_pack_names(struct hashfile *f, - char **pack_names, + struct midx_info *info, uint32_t num_packs) { uint32_t i; @@ -619,14 +593,19 @@ static size_t write_midx_pack_names(struct hashfile *f, size_t written = 0; for (i = 0; i < num_packs; i++) { - size_t writelen = strlen(pack_names[i]) + 1; + size_t writelen; + + if (info[i].expired) + continue; - if (i && strcmp(pack_names[i], pack_names[i - 1]) <= 0) + writelen = strlen(info[i].pack_name) + 1; + + if (i && strcmp(info[i].pack_name, info[i - 1].pack_name) <= 0) BUG("incorrect pack-file order: %s before %s", - pack_names[i - 1], - pack_names[i]); + info[i - 1].pack_name, + info[i].pack_name); - hashwrite(f, pack_names[i], writelen); + hashwrite(f, info[i].pack_name, writelen); written += writelen; } @@ -697,6 +676,7 @@ static size_t write_midx_oid_lookup(struct hashfile *f, unsigned char hash_len, } static size_t write_midx_object_offsets(struct hashfile *f, int large_offset_needed, + uint32_t *perm, struct pack_midx_entry *objects, uint32_t nr_objects) { struct pack_midx_entry *list = objects; @@ -705,8 +685,13 @@ static size_t write_midx_object_offsets(struct hashfile *f, int large_offset_nee for (i = 0; i < nr_objects; i++) { struct pack_midx_entry *obj = list++; + int pack_int_id = perm[obj->pack_int_id]; + + if (pack_int_id == UINT_MAX) + BUG("tried to write an object %s with expired pack-int-id", + oid_to_hex(&obj->oid)); - hashwrite_be32(f, obj->pack_int_id); + hashwrite_be32(f, pack_int_id); if (large_offset_needed && obj->offset >> 31) hashwrite_be32(f, MIDX_LARGE_OFFSET_NEEDED | nr_large_offset++); @@ -757,7 +742,7 @@ static int write_midx_internal(const char *object_dir, struct multi_pack_index * { unsigned char cur_chunk, num_chunks = 0; char *midx_name; - uint32_t i; + uint32_t i, drop_count; struct hashfile *f = NULL; struct lock_file lk; struct pack_list packs; @@ -782,66 +767,30 @@ static int write_midx_internal(const char *object_dir, struct multi_pack_index * packs.m = load_multi_pack_index(object_dir, 1); packs.nr = 0; - packs.alloc_list = packs.m ? packs.m->num_packs : 16; - packs.alloc_perm = packs.alloc_names = packs.alloc_list; - packs.list = NULL; - packs.names = NULL; + packs.alloc = packs.m ? packs.m->num_packs : 16; + packs.info = NULL; packs.perm = NULL; packs.pack_name_concat_len = 0; - ALLOC_ARRAY(packs.list, packs.alloc_list); - ALLOC_ARRAY(packs.names, packs.alloc_names); - ALLOC_ARRAY(packs.perm, packs.alloc_perm); + ALLOC_ARRAY(packs.info, packs.alloc); if (packs.m) { - int drop_index = 0, missing_drops = 0; for (i = 0; i < packs.m->num_packs; i++) { - if (packs_to_drop && drop_index < packs_to_drop->nr) { - int cmp = strcmp(packs.m->pack_names[i], - packs_to_drop->items[drop_index].string); - - if (!cmp) { - drop_index++; - continue; - } else if (cmp > 0) { - error(_("did not see pack-file %s to drop"), - packs_to_drop->items[drop_index].string); - drop_index++; - i--; - missing_drops++; - continue; - } - } + ALLOC_GROW(packs.info, packs.nr + 1, packs.alloc); - ALLOC_GROW(packs.list, packs.nr + 1, packs.alloc_list); - ALLOC_GROW(packs.names, packs.nr + 1, packs.alloc_names); - ALLOC_GROW(packs.perm, packs.nr + 1, packs.alloc_perm); - - packs.perm[packs.nr] = i; - packs.list[packs.nr] = NULL; - packs.names[packs.nr] = xstrdup(packs.m->pack_names[i]); - packs.pack_name_concat_len += strlen(packs.names[packs.nr]) + 1; + packs.info[packs.nr].pack_name = xstrdup(packs.m->pack_names[i]); + packs.info[packs.nr].orig_pack_int_id = i; + packs.info[packs.nr].p = NULL; + packs.info[packs.nr].expired = 0; packs.nr++; } - - if (packs_to_drop && (drop_index < packs_to_drop->nr || missing_drops)) { - error(_("did not see all pack-files to drop")); - result = 1; - goto cleanup; - } } for_each_file_in_pack_dir(object_dir, add_pack_to_midx, &packs); - if (packs.m && packs.nr == packs.m->num_packs) + if (packs.m && packs.nr == packs.m->num_packs && !packs_to_drop) goto cleanup; - if (packs.pack_name_concat_len % MIDX_CHUNK_ALIGNMENT) - packs.pack_name_concat_len += MIDX_CHUNK_ALIGNMENT - - (packs.pack_name_concat_len % MIDX_CHUNK_ALIGNMENT); - - sort_packs_by_name(packs.names, packs.nr, packs.perm); - - entries = get_sorted_entries(packs.m, packs.list, packs.perm, packs.nr, &nr_entries); + entries = get_sorted_entries(packs.m, packs.info, packs.nr, &nr_entries); for (i = 0; i < nr_entries; i++) { if (entries[i].offset > 0x7fffffff) @@ -850,6 +799,60 @@ static int write_midx_internal(const char *object_dir, struct multi_pack_index * large_offsets_needed = 1; } + QSORT(packs.info, packs.nr, midx_info_compare); + + if (packs_to_drop && packs_to_drop->nr) { + int drop_index = 0; + int missing_drops = 0; + + for (i = 0; i < packs.nr && drop_index < packs_to_drop->nr; i++) { + int cmp = strcmp(packs.info[i].pack_name, + packs_to_drop->items[drop_index].string); + + if (!cmp) { + drop_index++; + packs.info[i].expired = 1; + } else if (cmp > 0) { + error(_("did not see pack-file %s to drop"), + packs_to_drop->items[drop_index].string); + drop_index++; + missing_drops++; + i--; + } + } + + if (missing_drops) { + result = 1; + goto cleanup; + } + } + + drop_count = 0; + for (i = 0; i < packs.nr; i++) { + if (packs.info[i].expired) + drop_count++; + else + packs.info[i].new_pack_int_id = i - drop_count; + } + + packs.perm = xcalloc(packs.nr, sizeof(uint32_t)); + for (i = 0; i < packs.nr; i++) { + if (packs.info[i].expired) + packs.perm[packs.info[i].orig_pack_int_id] = UINT_MAX; + else + packs.perm[packs.info[i].orig_pack_int_id] = + packs.info[i].new_pack_int_id; + } + + for (i = 0; i < packs.nr; i++) { + if (!packs.info[i].expired) + packs.pack_name_concat_len += strlen(packs.info[i].pack_name) + 1; + } + + if (packs.pack_name_concat_len % MIDX_CHUNK_ALIGNMENT) + packs.pack_name_concat_len += MIDX_CHUNK_ALIGNMENT - + (packs.pack_name_concat_len % MIDX_CHUNK_ALIGNMENT); + hold_lock_file_for_update(&lk, midx_name, LOCK_DIE_ON_ERROR); f = hashfd(lk.tempfile->fd, lk.tempfile->filename.buf); FREE_AND_NULL(midx_name); @@ -860,7 +863,7 @@ static int write_midx_internal(const char *object_dir, struct multi_pack_index * cur_chunk = 0; num_chunks = large_offsets_needed ? 5 : 4; - written = write_midx_header(f, num_chunks, packs.nr); + written = write_midx_header(f, num_chunks, packs.nr - drop_count); chunk_ids[cur_chunk] = MIDX_CHUNKID_PACKNAMES; chunk_offsets[cur_chunk] = written + (num_chunks + 1) * MIDX_CHUNKLOOKUP_WIDTH; @@ -915,7 +918,7 @@ static int write_midx_internal(const char *object_dir, struct multi_pack_index * switch (chunk_ids[i]) { case MIDX_CHUNKID_PACKNAMES: - written += write_midx_pack_names(f, packs.names, packs.nr); + written += write_midx_pack_names(f, packs.info, packs.nr); break; case MIDX_CHUNKID_OIDFANOUT: @@ -927,7 +930,7 @@ static int write_midx_internal(const char *object_dir, struct multi_pack_index * break; case MIDX_CHUNKID_OBJECTOFFSETS: - written += write_midx_object_offsets(f, large_offsets_needed, entries, nr_entries); + written += write_midx_object_offsets(f, large_offsets_needed, packs.perm, entries, nr_entries); break; case MIDX_CHUNKID_LARGEOFFSETS: @@ -950,15 +953,14 @@ static int write_midx_internal(const char *object_dir, struct multi_pack_index * cleanup: for (i = 0; i < packs.nr; i++) { - if (packs.list[i]) { - close_pack(packs.list[i]); - free(packs.list[i]); + if (packs.info[i].p) { + close_pack(packs.info[i].p); + free(packs.info[i].p); } - free(packs.names[i]); + free(packs.info[i].pack_name); } - free(packs.list); - free(packs.names); + free(packs.info); free(packs.perm); free(entries); free(midx_name); diff --git a/t/t5319-multi-pack-index.sh b/t/t5319-multi-pack-index.sh index 3f5e9ea6534206..13af72e994df00 100755 --- a/t/t5319-multi-pack-index.sh +++ b/t/t5319-multi-pack-index.sh @@ -446,4 +446,32 @@ test_expect_success 'expire removes repacked packs' ' ) ' +test_expect_success 'expire works when adding new packs' ' + ( + cd dup && + git pack-objects --revs .git/objects/pack/pack-combined <<-EOF && + refs/heads/A + ^refs/heads/B + EOF + git pack-objects --revs .git/objects/pack/pack-combined <<-EOF && + refs/heads/B + ^refs/heads/C + EOF + git pack-objects --revs .git/objects/pack/pack-combined <<-EOF && + refs/heads/C + ^refs/heads/D + EOF + git multi-pack-index write && + git pack-objects --revs .git/objects/pack/pack-a <<-EOF && + refs/heads/D + ^refs/heads/E + EOF + git multi-pack-index write && + git pack-objects --revs .git/objects/pack/pack-z <<-EOF && + refs/heads/E + EOF + git multi-pack-index expire && + git multi-pack-index verify + ) +' test_done From 0253f80587019b248ead2a3f07fb6df132037fce Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Mon, 30 May 2016 10:55:53 -0400 Subject: [PATCH 020/118] gvfs: optionally skip reachability checks/upload pack during fetch While performing a fetch with a virtual file system we know that there will be missing objects and we don't want to download them just because of the reachability of the commits. We also don't want to download a pack file with commits, trees, and blobs since these will be downloaded on demand. This flag will skip the first connectivity check and by returning zero will skip the upload pack. It will also skip the second connectivity check but continue to update the branches to the latest commit ids. Signed-off-by: Kevin Willford --- Documentation/config/core.txt | 9 +++++++++ connected.c | 19 +++++++++++++++++++ gvfs.h | 1 + t/t5582-vfs.sh | 24 ++++++++++++++++++++++++ 4 files changed, 53 insertions(+) create mode 100755 t/t5582-vfs.sh diff --git a/Documentation/config/core.txt b/Documentation/config/core.txt index 77430dcbd66f62..4849855502867a 100644 --- a/Documentation/config/core.txt +++ b/Documentation/config/core.txt @@ -625,6 +625,15 @@ core.gvfs:: directory. This will allow virtualized working directories to detect the change to HEAD and use the new commit tree to show the files that are in the working directory. + GVFS_FETCH_SKIP_REACHABILITY_AND_UPLOADPACK:: + Bit value 16 + While performing a fetch with a virtual file system we know + that there will be missing objects and we don't want to download + them just because of the reachability of the commits. We also + don't want to download a pack file with commits, trees, and blobs + since these will be downloaded on demand. This flag will skip the + checks on the reachability of objects during a fetch as well as + the upload pack so that extraneous objects don't get downloaded. -- core.sparseCheckout:: diff --git a/connected.c b/connected.c index 1bba888eff90a2..e3b8573f193e2e 100644 --- a/connected.c +++ b/connected.c @@ -4,6 +4,7 @@ #include "connected.h" #include "transport.h" #include "packfile.h" +#include "gvfs.h" /* * If we feed all the commits we want to verify to this command @@ -28,6 +29,24 @@ int check_connected(oid_iterate_fn fn, void *cb_data, struct transport *transport; size_t base_len; + /* + * Running a virtual file system there will be objects that are + * missing locally and we don't want to download a bunch of + * commits, trees, and blobs just to make sure everything is + * reachable locally so this option will skip reachablility + * checks below that use rev-list. This will stop the check + * before uploadpack runs to determine if there is anything to + * fetch. Returning zero for the first check will also prevent the + * uploadpack from happening. It will also skip the check after + * the fetch is finished to make sure all the objects where + * downloaded in the pack file. This will allow the fetch to + * run and get all the latest tip commit ids for all the branches + * in the fetch but not pull down commits, trees, or blobs via + * upload pack. + */ + if (gvfs_config_is_set(GVFS_FETCH_SKIP_REACHABILITY_AND_UPLOADPACK)) + return 0; + if (!opt) opt = &defaults; transport = opt->transport; diff --git a/gvfs.h b/gvfs.h index f9144d2ad9ab92..8ee12df31c8a1d 100644 --- a/gvfs.h +++ b/gvfs.h @@ -16,6 +16,7 @@ #define GVFS_SKIP_SHA_ON_INDEX (1 << 0) #define GVFS_MISSING_OK (1 << 2) #define GVFS_NO_DELETE_OUTSIDE_SPARSECHECKOUT (1 << 3) +#define GVFS_FETCH_SKIP_REACHABILITY_AND_UPLOADPACK (1 << 4) static inline int gvfs_config_is_set(int mask) { return (core_gvfs & mask) == mask; diff --git a/t/t5582-vfs.sh b/t/t5582-vfs.sh new file mode 100755 index 00000000000000..8a703cbb640387 --- /dev/null +++ b/t/t5582-vfs.sh @@ -0,0 +1,24 @@ +#!/bin/sh + +test_description='fetch using the flag to skip reachability and upload pack' + +. ./test-lib.sh + + +test_expect_success setup ' + echo inital >a && + git add a && + git commit -m initial && + git clone . one +' + +test_expect_success "fetch test" ' + cd one && + git config core.gvfs 16 && + rm -rf .git/objects/* && + git -C .. cat-file commit HEAD | git hash-object -w --stdin -t commit && + git fetch && + test_must_fail git rev-parse --verify HEAD^{tree} +' + +test_done \ No newline at end of file From 94e5d2a7440579759f6c202af3d7175f81c7ce90 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Wed, 15 Mar 2017 16:36:53 -0600 Subject: [PATCH 021/118] Fix reset when using the sparse-checkout feature. When using the sparse checkout feature the git reset command will add entries to the index that will have the skip-worktree bit off but will leave the working directory empty. File data is lost because the index version of the files has been changed but there is nothing that is in the working directory. This will cause the next status call to show either deleted for files modified or deleting or nothing for files added. The added files should be shown as untracked and modified files should be shown as modified. To fix this when the reset is running if there is not a file in the working directory and if it will be missing with the new index entry or was not missing in the previous version, we create the previous index version of the file in the working directory so that status will report correctly and the files will be availble for the user to deal with. Signed-off-by: Kevin Willford --- builtin/reset.c | 34 +++++++++++++++++++ t/t7114-reset-sparse-checkout.sh | 58 ++++++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+) create mode 100755 t/t7114-reset-sparse-checkout.sh diff --git a/builtin/reset.c b/builtin/reset.c index 023fe12be048b2..91875a2657aab5 100644 --- a/builtin/reset.c +++ b/builtin/reset.c @@ -27,6 +27,7 @@ #include "submodule-config.h" #include "strbuf.h" #include "quote.h" +#include "dir.h" #define REFRESH_INDEX_DELAY_WARNING_IN_MS (2 * 1000) @@ -128,12 +129,45 @@ static void update_index_from_diff(struct diff_queue_struct *q, struct diff_options *opt, void *data) { int i; + int pos; int intent_to_add = *(int *)data; for (i = 0; i < q->nr; i++) { struct diff_filespec *one = q->queue[i]->one; + struct diff_filespec *two = q->queue[i]->two; int is_missing = !(one->mode && !is_null_oid(&one->oid)); + int was_missing = !two->mode && is_null_oid(&two->oid); struct cache_entry *ce; + struct cache_entry *ceBefore; + struct checkout state = CHECKOUT_INIT; + + /* + * When using the sparse-checkout feature the cache entries that are + * added here will not have the skip-worktree bit set. + * Without this code there is data that is lost because the files that + * would normally be in the working directory are not there and show as + * deleted for the next status or in the case of added files just disappear. + * We need to create the previous version of the files in the working + * directory so that they will have the right content and the next + * status call will show modified or untracked files correctly. + */ + if (core_apply_sparse_checkout && !file_exists(two->path)) + { + pos = cache_name_pos(two->path, strlen(two->path)); + if ((pos >= 0 && ce_skip_worktree(active_cache[pos])) && (is_missing || !was_missing)) + { + state.force = 1; + state.refresh_cache = 1; + state.istate = &the_index; + ceBefore = make_cache_entry(&the_index, two->mode, &two->oid, two->path, + 0, 0); + if (!ceBefore) + die(_("make_cache_entry failed for path '%s'"), + two->path); + + checkout_entry(ceBefore, &state, NULL, NULL); + } + } if (is_missing && !intent_to_add) { remove_file_from_cache(one->path); diff --git a/t/t7114-reset-sparse-checkout.sh b/t/t7114-reset-sparse-checkout.sh new file mode 100755 index 00000000000000..8dd88fd46d3938 --- /dev/null +++ b/t/t7114-reset-sparse-checkout.sh @@ -0,0 +1,58 @@ +#!/bin/sh + +test_description='reset when using a sparse-checkout' + +. ./test-lib.sh + +# reset using a sparse-checkout file + +test_expect_success 'setup' ' + test_tick && + echo "checkout file" >c && + echo "modify file" >m && + echo "delete file" >d && + git add . && + git commit -m "initial commit" && + echo "added file" >a && + echo "modification of a file" >m && + git rm d && + git add . && + git commit -m "second commit" && + git checkout -b endCommit +' + +test_expect_success 'reset when there is a sparse-checkout' ' + echo "/c" >.git/info/sparse-checkout && + test_config core.sparsecheckout true && + git checkout -b resetBranch && + test_path_is_missing m && + test_path_is_missing a && + test_path_is_missing d && + git reset HEAD~1 && + test "checkout file" = "$(cat c)" && + test "modification of a file" = "$(cat m)" && + test "added file" = "$(cat a)" && + test_path_is_missing d +' + +test_expect_success 'reset after deleting file without skip-worktree bit' ' + git checkout -f endCommit && + git clean -xdf && + echo "/c +/m" >.git/info/sparse-checkout && + test_config core.sparsecheckout true && + git checkout -b resetAfterDelete && + test_path_is_file m && + test_path_is_missing a && + test_path_is_missing d && + rm -f m && + git reset HEAD~1 && + test "checkout file" = "$(cat c)" && + test "added file" = "$(cat a)" && + test_path_is_missing m && + test_path_is_missing d +' + + + +test_done From 551acd0c86514a95ec1bddf81360bbd5a5bb5ae0 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Wed, 5 Apr 2017 10:55:32 -0600 Subject: [PATCH 022/118] Do not remove files outside the sparse-checkout Signed-off-by: Kevin Willford --- unpack-trees.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/unpack-trees.c b/unpack-trees.c index 3563daae1aa6bc..4fae9142aa0a1d 100644 --- a/unpack-trees.c +++ b/unpack-trees.c @@ -521,7 +521,9 @@ static int apply_sparse_checkout(struct index_state *istate, */ if (!(ce->ce_flags & CE_UPDATE) && verify_uptodate_sparse(ce, o)) return -1; - ce->ce_flags |= CE_WT_REMOVE; + if (!gvfs_config_is_set(GVFS_NO_DELETE_OUTSIDE_SPARSECHECKOUT)) + ce->ce_flags |= CE_WT_REMOVE; + ce->ce_flags &= ~CE_UPDATE; } if (was_skip_worktree && !ce_skip_worktree(ce)) { From ae101e7b91f357e4480ae64168162c38bb999274 Mon Sep 17 00:00:00 2001 From: Ben Peart Date: Wed, 15 Jun 2016 14:59:16 +0000 Subject: [PATCH 023/118] gvfs: ensure all filters and EOL conversions are blocked Ensure all filters and EOL conversions are blocked when running under GVFS so that our projected file sizes will match the actual file size when it is hydrated on the local machine. Signed-off-by: Ben Peart --- Documentation/config/core.txt | 9 +++++++++ convert.c | 22 +++++++++++++++++++++ gvfs.h | 1 + t/t0021-conversion.sh | 37 +++++++++++++++++++++++++++++++++++ t/t0027-auto-crlf.sh | 12 ++++++++++++ 5 files changed, 81 insertions(+) diff --git a/Documentation/config/core.txt b/Documentation/config/core.txt index 4849855502867a..cf0510cc2c9979 100644 --- a/Documentation/config/core.txt +++ b/Documentation/config/core.txt @@ -634,6 +634,15 @@ core.gvfs:: since these will be downloaded on demand. This flag will skip the checks on the reachability of objects during a fetch as well as the upload pack so that extraneous objects don't get downloaded. + GVFS_BLOCK_FILTERS_AND_EOL_CONVERSIONS:: + Bit value 64 + With a virtual file system we only know the file size before any + CRLF or smudge/clean filters processing is done on the client. + To prevent file corruption due to truncation or expansion with + garbage at the end, these filters must not run when the file + is first accessed and brought down to the client. Git.exe can't + currently tell the first access vs subsequent accesses so this + flag just blocks them from occurring at all. -- core.sparseCheckout:: diff --git a/convert.c b/convert.c index 5d0307fc1004f2..f799d8e167d52e 100644 --- a/convert.c +++ b/convert.c @@ -8,6 +8,7 @@ #include "pkt-line.h" #include "sub-process.h" #include "utf8.h" +#include "gvfs.h" /* * convert.c - convert a file when checking it out and checking it in. @@ -564,6 +565,9 @@ static int crlf_to_git(const struct index_state *istate, if (!buf) return 1; + if (gvfs_config_is_set(GVFS_BLOCK_FILTERS_AND_EOL_CONVERSIONS)) + die("CRLF conversions not supported when running under GVFS"); + /* only grow if not in place */ if (strbuf_avail(buf) + buf->len < len) strbuf_grow(buf, len - buf->len); @@ -603,6 +607,9 @@ static int crlf_to_worktree(const char *src, size_t len, if (!will_convert_lf_to_crlf(&stats, crlf_action)) return 0; + if (gvfs_config_is_set(GVFS_BLOCK_FILTERS_AND_EOL_CONVERSIONS)) + die("CRLF conversions not supported when running under GVFS"); + /* are we "faking" in place editing ? */ if (src == buf->buf) to_free = strbuf_detach(buf, NULL); @@ -717,6 +724,9 @@ static int apply_single_file_filter(const char *path, const char *src, size_t le struct async async; struct filter_params params; + if (gvfs_config_is_set(GVFS_BLOCK_FILTERS_AND_EOL_CONVERSIONS)) + die("Filter \"%s\" not supported when running under GVFS", cmd); + memset(&async, 0, sizeof(async)); async.proc = filter_buffer_or_fd; async.data = ¶ms; @@ -1101,6 +1111,9 @@ static int ident_to_git(const char *src, size_t len, if (!buf) return 1; + if (gvfs_config_is_set(GVFS_BLOCK_FILTERS_AND_EOL_CONVERSIONS)) + die("ident conversions not supported when running under GVFS"); + /* only grow if not in place */ if (strbuf_avail(buf) + buf->len < len) strbuf_grow(buf, len - buf->len); @@ -1148,6 +1161,9 @@ static int ident_to_worktree(const char *src, size_t len, if (!cnt) return 0; + if (gvfs_config_is_set(GVFS_BLOCK_FILTERS_AND_EOL_CONVERSIONS)) + die("ident conversions not supported when running under GVFS"); + /* are we "faking" in place editing ? */ if (src == buf->buf) to_free = strbuf_detach(buf, NULL); @@ -1580,6 +1596,9 @@ static int lf_to_crlf_filter_fn(struct stream_filter *filter, size_t count, o = 0; struct lf_to_crlf_filter *lf_to_crlf = (struct lf_to_crlf_filter *)filter; + if (gvfs_config_is_set(GVFS_BLOCK_FILTERS_AND_EOL_CONVERSIONS)) + die("CRLF conversions not supported when running under GVFS"); + /* * We may be holding onto the CR to see if it is followed by a * LF, in which case we would need to go to the main loop. @@ -1824,6 +1843,9 @@ static int ident_filter_fn(struct stream_filter *filter, struct ident_filter *ident = (struct ident_filter *)filter; static const char head[] = "$Id"; + if (gvfs_config_is_set(GVFS_BLOCK_FILTERS_AND_EOL_CONVERSIONS)) + die("ident conversions not supported when running under GVFS"); + if (!input) { /* drain upon eof */ switch (ident->state) { diff --git a/gvfs.h b/gvfs.h index 8ee12df31c8a1d..2d6de575bf4a65 100644 --- a/gvfs.h +++ b/gvfs.h @@ -17,6 +17,7 @@ #define GVFS_MISSING_OK (1 << 2) #define GVFS_NO_DELETE_OUTSIDE_SPARSECHECKOUT (1 << 3) #define GVFS_FETCH_SKIP_REACHABILITY_AND_UPLOADPACK (1 << 4) +#define GVFS_BLOCK_FILTERS_AND_EOL_CONVERSIONS (1 << 6) static inline int gvfs_config_is_set(int mask) { return (core_gvfs & mask) == mask; diff --git a/t/t0021-conversion.sh b/t/t0021-conversion.sh index 27c3d0afb29106..1a95d4a73dcf82 100755 --- a/t/t0021-conversion.sh +++ b/t/t0021-conversion.sh @@ -314,6 +314,43 @@ test_expect_success "filter: smudge empty file" ' test_cmp expected filtered-empty-in-repo ' +test_expect_success "filter: clean filters blocked when under GVFS" ' + test_config filter.empty-in-repo.clean "cat >/dev/null" && + test_config filter.empty-in-repo.smudge "echo smudged && cat" && + test_config core.gvfs 64 && + + echo dead data walking >empty-in-repo && + test_must_fail git add empty-in-repo +' + +test_expect_success "filter: smudge filters blocked when under GVFS" ' + test_config filter.empty-in-repo.clean "cat >/dev/null" && + test_config filter.empty-in-repo.smudge "echo smudged && cat" && + test_config core.gvfs 64 && + + test_must_fail git checkout +' + +test_expect_success "ident blocked on add when under GVFS" ' + test_config core.gvfs 64 && + test_config core.autocrlf false && + + echo "*.i ident" >.gitattributes && + echo "\$Id\$" > ident.i && + + test_must_fail git add ident.i +' + +test_expect_success "ident blocked when under GVFS" ' + git add ident.i && + + git commit -m "added ident.i" && + test_config core.gvfs 64 && + rm ident.i && + + test_must_fail git checkout -- ident.i +' + test_expect_success 'disable filter with empty override' ' test_config_global filter.disable.smudge false && test_config_global filter.disable.clean false && diff --git a/t/t0027-auto-crlf.sh b/t/t0027-auto-crlf.sh index 3587e454f19d9b..8abb4bdfbd7ead 100755 --- a/t/t0027-auto-crlf.sh +++ b/t/t0027-auto-crlf.sh @@ -331,6 +331,18 @@ checkout_files () { " } +test_expect_success 'crlf conversions blocked when under GVFS' ' + git checkout -b gvfs && + test_commit initial && + rm initial.t && + test_config core.gvfs 64 && + test_config core.autocrlf true && + test_must_fail git read-tree --reset -u HEAD && + + git config core.autocrlf false && + git read-tree --reset -u HEAD +' + # Test control characters # NUL SOH CR EOF==^Z test_expect_success 'ls-files --eol -o Text/Binary' ' From e85d1ba91f21d464f46abec4bd6159920071aac3 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 30 Nov 2016 23:11:36 +0100 Subject: [PATCH 024/118] Add a new run_hook_argv() function The two existing members of the run_hook*() family, run_hook_ve() and run_hook_le(), are good for callers that know the precise number of parameters already. Let's introduce a new sibling that takes an argv array for callers that want to pass a variable number of parameters. Signed-off-by: Johannes Schindelin --- run-command.c | 20 +++++++++++++++++--- run-command.h | 2 ++ 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/run-command.c b/run-command.c index 3db26b7b0e2ab9..1b8e5e2004c2e5 100644 --- a/run-command.c +++ b/run-command.c @@ -1304,7 +1304,8 @@ const char *find_hook(const char *name) return path.buf; } -int run_hook_ve(const char *const *env, const char *name, va_list args) +int run_hook_argv(const char *const *env, const char *name, + const char **argv) { struct child_process hook = CHILD_PROCESS_INIT; const char *p; @@ -1314,8 +1315,7 @@ int run_hook_ve(const char *const *env, const char *name, va_list args) return 0; argv_array_push(&hook.args, p); - while ((p = va_arg(args, const char *))) - argv_array_push(&hook.args, p); + argv_array_pushv(&hook.args, argv); hook.env = env; hook.no_stdin = 1; hook.stdout_to_stderr = 1; @@ -1323,6 +1323,20 @@ int run_hook_ve(const char *const *env, const char *name, va_list args) return run_command(&hook); } +int run_hook_ve(const char *const *env, const char *name, va_list args) +{ + struct argv_array argv = ARGV_ARRAY_INIT; + const char *p; + int ret; + + while ((p = va_arg(args, const char *))) + argv_array_push(&argv, p); + + ret = run_hook_argv(env, name, argv.argv); + argv_array_clear(&argv); + return ret; +} + int run_hook_le(const char *const *env, const char *name, ...) { va_list args; diff --git a/run-command.h b/run-command.h index 68f5369fc2d432..b17abb44a43722 100644 --- a/run-command.h +++ b/run-command.h @@ -65,6 +65,8 @@ extern const char *find_hook(const char *name); LAST_ARG_MUST_BE_NULL extern int run_hook_le(const char *const *env, const char *name, ...); extern int run_hook_ve(const char *const *env, const char *name, va_list args); +extern int run_hook_argv(const char *const *env, const char *name, + const char **argv); #define RUN_COMMAND_NO_STDIN 1 #define RUN_GIT_CMD 2 /*If this is to be git sub-command */ From 4445cbefae8f66322e7874e0ed035f452a605515 Mon Sep 17 00:00:00 2001 From: Ben Peart Date: Tue, 10 Jan 2017 18:47:14 +0000 Subject: [PATCH 025/118] gvfs: allow "virtualizing" objects The idea is to allow blob objects to be missing from the local repository, and to load them lazily on demand. After discussing this idea on the mailing list, we will rename the feature to "lazy clone" and work more on this. Signed-off-by: Ben Peart --- cache.h | 2 ++ config.c | 5 +++++ connected.c | 2 ++ environment.c | 1 + sha1-file.c | 22 ++++++++++++++++++++++ 5 files changed, 32 insertions(+) diff --git a/cache.h b/cache.h index e976871a0bc85f..ff1c630f984f16 100644 --- a/cache.h +++ b/cache.h @@ -912,6 +912,8 @@ int use_optional_locks(void); extern char comment_line_char; extern int auto_comment_line_char; +extern int core_virtualize_objects; + enum log_refs_config { LOG_REFS_UNSET = -1, LOG_REFS_NONE = 0, diff --git a/config.c b/config.c index eea77e608e3e19..0f236e53a84e9a 100644 --- a/config.c +++ b/config.c @@ -1360,6 +1360,11 @@ static int git_default_core_config(const char *var, const char *value, void *cb) return 0; } + if (!strcmp(var, "core.virtualizeobjects")) { + core_virtualize_objects = git_config_bool(var, value); + return 0; + } + /* Add other config variables here and to Documentation/config.txt. */ return platform_core_config(var, value, cb); } diff --git a/connected.c b/connected.c index e3b8573f193e2e..078ec309572b4e 100644 --- a/connected.c +++ b/connected.c @@ -46,6 +46,8 @@ int check_connected(oid_iterate_fn fn, void *cb_data, */ if (gvfs_config_is_set(GVFS_FETCH_SKIP_REACHABILITY_AND_UPLOADPACK)) return 0; + if (core_virtualize_objects) + return 0; if (!opt) opt = &defaults; diff --git a/environment.c b/environment.c index 2bd89c627612f6..d50080a4f8e74e 100644 --- a/environment.c +++ b/environment.c @@ -73,6 +73,7 @@ int core_gvfs; int merge_log_config = -1; int precomposed_unicode = -1; /* see probe_utf8_pathname_composition() */ unsigned long pack_size_limit_cfg; +int core_virtualize_objects; enum log_refs_config log_all_ref_updates = LOG_REFS_UNSET; #ifndef PROTECT_HFS_DEFAULT diff --git a/sha1-file.c b/sha1-file.c index 494606f7716923..914fe4749a604f 100644 --- a/sha1-file.c +++ b/sha1-file.c @@ -1310,6 +1310,21 @@ static int loose_object_info(struct repository *r, return (status < 0) ? status : 0; } +static int run_read_object_hook(const struct object_id *oid) +{ + struct argv_array args = ARGV_ARRAY_INIT; + int ret; + uint64_t start; + + start = getnanotime(); + argv_array_push(&args, oid_to_hex(oid)); + ret = run_hook_argv(NULL, "read-object", args.argv); + argv_array_clear(&args); + trace_performance_since(start, "run_read_object_hook"); + + return ret; +} + int fetch_if_missing = 1; int oid_object_info_extended(struct repository *r, const struct object_id *oid, @@ -1320,6 +1335,7 @@ int oid_object_info_extended(struct repository *r, const struct object_id *oid, int rtype; const struct object_id *real = oid; int already_retried = 0; + int tried_hook = 0; if (flags & OBJECT_INFO_LOOKUP_REPLACE) real = lookup_replace_object(r, oid); @@ -1330,6 +1346,7 @@ int oid_object_info_extended(struct repository *r, const struct object_id *oid, if (!oi) oi = &blank_oi; +retry: if (!(flags & OBJECT_INFO_SKIP_CACHED)) { struct cached_object *co = find_cached_object(real); if (co) { @@ -1366,6 +1383,11 @@ int oid_object_info_extended(struct repository *r, const struct object_id *oid, reprepare_packed_git(r); if (find_pack_entry(r, real, &e)) break; + if (core_virtualize_objects && !tried_hook) { + tried_hook = 1; + if (!run_read_object_hook(oid)) + goto retry; + } } /* Check if it is a missing object */ From fe1e47aa39aeae41ac244139cef4514dc933c1b9 Mon Sep 17 00:00:00 2001 From: Ben Peart Date: Tue, 24 May 2016 00:32:38 +0000 Subject: [PATCH 026/118] gvfs: add global command pre and post hook procs This adds hard-coded call to GVFS.hooks.exe before and after each Git command runs. To make sure that this is only called on repositories cloned with GVFS, we test for the tell-tale .gvfs. Signed-off-by: Ben Peart --- git.c | 147 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 144 insertions(+), 3 deletions(-) diff --git a/git.c b/git.c index c2330275b38cc4..35a744de28aff3 100644 --- a/git.c +++ b/git.c @@ -380,6 +380,132 @@ static int handle_alias(int *argcp, const char ***argv) return ret; } +/* + * Runs pre/post-command hook. + */ +struct argv_array sargv = ARGV_ARRAY_INIT; +int run_post_hook = 0; +int exit_code = -1; + +static int is_gvfs_repo(void) +{ + wchar_t pwd[MAX_PATH]; + DWORD dwRet; + WIN32_FIND_DATAW FindFileData; + HANDLE hFind; + wchar_t *lastslash; + + dwRet = GetCurrentDirectoryW(MAX_PATH-7, pwd); + if (dwRet == 0 || dwRet > MAX_PATH) + die("GetCurrentDirectory failed (%d)\n", (int)GetLastError()); + + if ('\\' != pwd[wcslen(pwd) - 1]) + wcscat(pwd, L"\\"); + lastslash = pwd + wcslen(pwd) - 1; + while (1) { + wcscat(lastslash, L".gvfs"); + + hFind = FindFirstFileW(pwd, &FindFileData); + if (hFind != INVALID_HANDLE_VALUE) { + FindClose(hFind); + if (FindFileData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) + return 1; + } + + lastslash--; + while ((pwd != lastslash) && (*lastslash != '\\')) + lastslash--; + if (pwd == lastslash) + return 0; + *(lastslash + 1) = 0; + }; + + return 0; +} + +static int run_GVFS_Hooks_argv(const char *const *env, const char *name, + const char *const *argv) +{ + struct child_process hook = CHILD_PROCESS_INIT; + + if (!is_gvfs_repo()) + return 0; + + argv_array_push(&hook.args, "GVFS.Hooks.exe"); + argv_array_push(&hook.args, name); + argv_array_pushv(&hook.args, (const char **)argv); + hook.env = env; + hook.no_stdin = 1; + + return run_command(&hook); +} + +static int run_pre_command_hook(const char **argv) +{ + char *lock; + int ret = 0; + + /* + * Ensure the global pre/post command hook is only called for + * the outer command and not when git is called recursively + * or spawns multiple commands (like with the alias command) + */ + lock = getenv("COMMAND_HOOK_LOCK"); + if (lock && !strcmp(lock, "true")) + return 0; + setenv("COMMAND_HOOK_LOCK", "true", 1); + + argv_array_pushv(&sargv, argv); + /* + * TODO: This is a temporary hack until we can get config settings + * before executing various git commands without messing up git's state. + * Once we can safely read settings, use the normal hook functions. + * + * ret = run_hook_argv(NULL, "pre-command", sargv.argv); + */ + ret = run_GVFS_Hooks_argv(NULL, "pre-command", sargv.argv); + + if (!ret) + run_post_hook = 1; + return ret; +} + +static int run_post_command_hook(void) +{ + char *lock; + int ret = 0; + + /* + * Only run post_command if pre_command succeeded in this process + */ + if (!run_post_hook) + return 0; + lock = getenv("COMMAND_HOOK_LOCK"); + if (!lock || strcmp(lock, "true")) + return 0; + + argv_array_pushf(&sargv, "--exit_code=%u", exit_code); + /* + * TODO: This is a temporary hack until we can get config settings + * before executing various git commands without messing up git's state. + * Once we can safely read settings, use the normal hook functions. + * + * ret = run_hook_argv(NULL, "post-command", sargv.argv); + */ + ret = run_GVFS_Hooks_argv(NULL, "post-command", sargv.argv); + + run_post_hook = 0; + argv_array_clear(&sargv); + setenv("COMMAND_HOOK_LOCK", "false", 1); + return ret; +} + +static void post_command_hook_atexit(void) +{ + fflush(NULL); + run_post_command_hook(); +} + static int run_builtin(struct cmd_struct *p, int argc, const char **argv) { int status, help; @@ -416,15 +542,20 @@ static int run_builtin(struct cmd_struct *p, int argc, const char **argv) if (!help && p->option & NEED_WORK_TREE) setup_work_tree(); + if (run_pre_command_hook(argv)) + die("pre-command hook aborted command"); + trace_argv_printf(argv, "trace: built-in: git"); validate_cache_entries(the_repository->index); - status = p->fn(argc, argv, prefix); + exit_code = status = p->fn(argc, argv, prefix); validate_cache_entries(the_repository->index); if (status) return status; + run_post_command_hook(); + /* Somebody closed stdout? */ if (fstat(fileno(stdout), &st)) return 0; @@ -673,6 +804,9 @@ static void execv_dashed_external(const char **argv) cmd.wait_after_clean = 1; cmd.silent_exec_failure = 1; + if (run_pre_command_hook(cmd.args.argv)) + die("pre-command hook aborted command"); + trace_argv_printf(cmd.args.argv, "trace: exec:"); /* @@ -681,11 +815,13 @@ static void execv_dashed_external(const char **argv) * or our usual generic code if we were not even able to exec * the program. */ - status = run_command(&cmd); + exit_code = status = run_command(&cmd); if (status >= 0) exit(status); else if (errno != ENOENT) exit(128); + + run_post_command_hook(); } static int run_argv(int *argcp, const char ***argv) @@ -784,6 +920,7 @@ int cmd_main(int argc, const char **argv) } trace_command_performance(argv); + atexit(post_command_hook_atexit); /* * "git-xxxx" is the same as "git xxxx", but we obviously: @@ -811,10 +948,14 @@ int cmd_main(int argc, const char **argv) } else { /* The user didn't specify a command; give them help */ commit_pager_choice(); + if (run_pre_command_hook(argv)) + die("pre-command hook aborted command"); printf(_("usage: %s\n\n"), git_usage_string); list_common_cmds_help(); printf("\n%s\n", _(git_more_info_string)); - exit(1); + exit_code = 1; + run_post_command_hook(); + exit(exit_code); } cmd = argv[0]; From df7ff8ed7e472ec6ac5abfca22157c1ea8bf386e Mon Sep 17 00:00:00 2001 From: Ben Peart Date: Wed, 15 Mar 2017 18:43:05 +0000 Subject: [PATCH 027/118] Hydrate missing loose objects in check_and_freshen() Hydrate missing loose objects in check_and_freshen() when running virtualized. Add test cases to verify read-object hook works when running virtualized. This hook is called in check_and_freshen() rather than check_and_freshen_local() to make the hook work also with alternates. Helped-by: Kevin Willford Signed-off-by: Ben Peart --- sha1-file.c | 46 +++++++++++++++++++++++++++--------------- t/t0411-read-object.sh | 27 +++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 16 deletions(-) create mode 100755 t/t0411-read-object.sh diff --git a/sha1-file.c b/sha1-file.c index 914fe4749a604f..5368dd352ef142 100644 --- a/sha1-file.c +++ b/sha1-file.c @@ -760,6 +760,24 @@ void prepare_alt_odb(struct repository *r) r->objects->loaded_alternates = 1; } +static int run_read_object_hook(const struct object_id *oid) +{ + struct child_process hook = CHILD_PROCESS_INIT; + const char *p; + + p = find_hook("read-object"); + if (!p) + return 1; + + argv_array_push(&hook.args, p); + argv_array_push(&hook.args, oid_to_hex(oid)); + hook.env = NULL; + hook.no_stdin = 1; + hook.stdout_to_stderr = 1; + + return run_command(&hook); +} + /* Returns 1 if we have successfully freshened the file, 0 otherwise. */ static int freshen_file(const char *fn) { @@ -812,8 +830,19 @@ static int check_and_freshen_nonlocal(const struct object_id *oid, int freshen) static int check_and_freshen(const struct object_id *oid, int freshen) { - return check_and_freshen_local(oid, freshen) || + int ret; + int tried_hook = 0; + +retry: + ret = check_and_freshen_local(oid, freshen) || check_and_freshen_nonlocal(oid, freshen); + if (!ret && core_virtualize_objects && !tried_hook) { + tried_hook = 1; + if (!run_read_object_hook(oid)) + goto retry; + } + + return ret; } int has_loose_object_nonlocal(const struct object_id *oid) @@ -1310,21 +1339,6 @@ static int loose_object_info(struct repository *r, return (status < 0) ? status : 0; } -static int run_read_object_hook(const struct object_id *oid) -{ - struct argv_array args = ARGV_ARRAY_INIT; - int ret; - uint64_t start; - - start = getnanotime(); - argv_array_push(&args, oid_to_hex(oid)); - ret = run_hook_argv(NULL, "read-object", args.argv); - argv_array_clear(&args); - trace_performance_since(start, "run_read_object_hook"); - - return ret; -} - int fetch_if_missing = 1; int oid_object_info_extended(struct repository *r, const struct object_id *oid, diff --git a/t/t0411-read-object.sh b/t/t0411-read-object.sh new file mode 100755 index 00000000000000..0c3bfea1886cf9 --- /dev/null +++ b/t/t0411-read-object.sh @@ -0,0 +1,27 @@ +#!/bin/sh + +test_description='tests for read-object hook' + +. ./test-lib.sh + +test_expect_success 'setup host and guest repos' ' + test_commit zero && + hash1=$(git ls-tree HEAD | grep zero.t | cut -f1 | cut -d\ -f3) && + git init guest-repo && + cd guest-repo && + git config core.virtualizeobjects true && + write_script .git/hooks/read-object <<-\EOF + # pass core.virtualizeobjects=false so we dont end up calling the hook proc recursively + git --git-dir=../.git/ cat-file blob "$1" | git -c core.virtualizeobjects=false hash-object -w --stdin >/dev/null 2>&1 + EOF +' + +test_expect_success 'blobs can be retrieved from the host repo' ' + git cat-file blob "$hash1" +' + +test_expect_success 'invalid blobs generate errors' ' + test_must_fail git cat-file blob "invalid" +' + +test_done From 1d229a90573940341f6a00ea80d96dcbaedff28b Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 6 Mar 2017 12:38:08 +0100 Subject: [PATCH 028/118] Allow hooks to be run before setup_git_directory() The use case here is to allow usage statistics to be gathered by running hooks before and after every hook, and to make that configurable via hooks. Signed-off-by: Johannes Schindelin --- run-command.c | 51 ++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/run-command.c b/run-command.c index 1b8e5e2004c2e5..94254e9d8d29af 100644 --- a/run-command.c +++ b/run-command.c @@ -7,6 +7,7 @@ #include "strbuf.h" #include "string-list.h" #include "quote.h" +#include "config.h" void child_process_init(struct child_process *child) { @@ -1270,12 +1271,60 @@ int async_with_fork(void) #endif } +static int early_hooks_path_config(const char *var, const char *value, void *data) +{ + if (!strcmp(var, "core.hookspath")) + return git_config_pathname((const char **)data, var, value); + + return 0; +} + +/* Discover the hook before setup_git_directory() was called */ +static const char *hook_path_early(const char *name, struct strbuf *result) +{ + static struct strbuf hooks_dir = STRBUF_INIT; + static int initialized; + + if (initialized < 0) + return NULL; + + if (!initialized) { + struct strbuf gitdir = STRBUF_INIT, commondir = STRBUF_INIT; + const char *early_hooks_dir = NULL; + + if (discover_git_directory(&commondir, &gitdir) < 0) { + initialized = -1; + return NULL; + } + + read_early_config(early_hooks_path_config, &early_hooks_dir); + if (!early_hooks_dir) + strbuf_addf(&hooks_dir, "%s/hooks/", commondir.buf); + else { + strbuf_add_absolute_path(&hooks_dir, early_hooks_dir); + strbuf_addch(&hooks_dir, '/'); + } + + strbuf_release(&gitdir); + strbuf_release(&commondir); + + initialized = 1; + } + + strbuf_addf(result, "%s%s", hooks_dir.buf, name); + return result->buf; +} + const char *find_hook(const char *name) { static struct strbuf path = STRBUF_INIT; strbuf_reset(&path); - strbuf_git_path(&path, "hooks/%s", name); + if (have_git_dir()) + strbuf_git_path(&path, "hooks/%s", name); + else if (!hook_path_early(name, &path)) + return NULL; + if (access(path.buf, X_OK) < 0) { int err = errno; From 502045b65edabb76d540060b9ad646c12b36acd0 Mon Sep 17 00:00:00 2001 From: Ben Peart Date: Tue, 18 Jul 2017 12:04:44 +0200 Subject: [PATCH 029/118] Add support for read-object as a background process to retrieve missing objects This commit converts the existing read_object hook proc model for downloading missing blobs to use a background process that is started the first time git encounters a missing blob and stays running until git exits. Git and the read-object process communicate via stdin/stdout and a versioned, capability negotiated interface as documented in Documentation/technical/read-object-protocol.txt. The advantage of this over the previous hook proc is that it saves the overhead of spawning a new hook process for every missing blob. The model for the background process was refactored from the recent git LFS work. I refactored that code into a shared module (sub-process.c/h) and then updated convert.c to consume the new library. I then used the same sub-process module when implementing the read-object background process. Signed-off-by: Ben Peart --- .../technical/read-object-protocol.txt | 102 +++++++++++++++ contrib/long-running-read-object/example.pl | 114 ++++++++++++++++ sha1-file.c | 122 ++++++++++++++++-- t/t0410/read-object | 114 ++++++++++++++++ t/t0411-read-object.sh | 24 ++-- 5 files changed, 450 insertions(+), 26 deletions(-) create mode 100644 Documentation/technical/read-object-protocol.txt create mode 100644 contrib/long-running-read-object/example.pl create mode 100755 t/t0410/read-object diff --git a/Documentation/technical/read-object-protocol.txt b/Documentation/technical/read-object-protocol.txt new file mode 100644 index 00000000000000..a893b46e7c28a9 --- /dev/null +++ b/Documentation/technical/read-object-protocol.txt @@ -0,0 +1,102 @@ +Read Object Process +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The read-object process enables Git to read all missing blobs with a +single process invocation for the entire life of a single Git command. +This is achieved by using a packet format (pkt-line, see technical/ +protocol-common.txt) based protocol over standard input and standard +output as follows. All packets, except for the "*CONTENT" packets and +the "0000" flush packet, are considered text and therefore are +terminated by a LF. + +Git starts the process when it encounters the first missing object that +needs to be retrieved. After the process is started, Git sends a welcome +message ("git-read-object-client"), a list of supported protocol version +numbers, and a flush packet. Git expects to read a welcome response +message ("git-read-object-server"), exactly one protocol version number +from the previously sent list, and a flush packet. All further +communication will be based on the selected version. + +The remaining protocol description below documents "version=1". Please +note that "version=42" in the example below does not exist and is only +there to illustrate how the protocol would look with more than one +version. + +After the version negotiation Git sends a list of all capabilities that +it supports and a flush packet. Git expects to read a list of desired +capabilities, which must be a subset of the supported capabilities list, +and a flush packet as response: +------------------------ +packet: git> git-read-object-client +packet: git> version=1 +packet: git> version=42 +packet: git> 0000 +packet: git< git-read-object-server +packet: git< version=1 +packet: git< 0000 +packet: git> capability=get +packet: git> capability=have +packet: git> capability=put +packet: git> capability=not-yet-invented +packet: git> 0000 +packet: git< capability=get +packet: git< 0000 +------------------------ +The only supported capability in version 1 is "get". + +Afterwards Git sends a list of "key=value" pairs terminated with a flush +packet. The list will contain at least the command (based on the +supported capabilities) and the sha1 of the object to retrieve. Please +note, that the process must not send any response before it received the +final flush packet. + +When the process receives the "get" command, it should make the requested +object available in the git object store and then return success. Git will +then check the object store again and this time find it and proceed. +------------------------ +packet: git> command=get +packet: git> sha1=0a214a649e1b3d5011e14a3dc227753f2bd2be05 +packet: git> 0000 +------------------------ + +The process is expected to respond with a list of "key=value" pairs +terminated with a flush packet. If the process does not experience +problems then the list must contain a "success" status. +------------------------ +packet: git< status=success +packet: git< 0000 +------------------------ + +In case the process cannot or does not want to process the content, it +is expected to respond with an "error" status. +------------------------ +packet: git< status=error +packet: git< 0000 +------------------------ + +In case the process cannot or does not want to process the content as +well as any future content for the lifetime of the Git process, then it +is expected to respond with an "abort" status at any point in the +protocol. +------------------------ +packet: git< status=abort +packet: git< 0000 +------------------------ + +Git neither stops nor restarts the process in case the "error"/"abort" +status is set. + +If the process dies during the communication or does not adhere to the +protocol then Git will stop the process and restart it with the next +object that needs to be processed. + +After the read-object process has processed an object it is expected to +wait for the next "key=value" list containing a command. Git will close +the command pipe on exit. The process is expected to detect EOF and exit +gracefully on its own. Git will wait until the process has stopped. + +A long running read-object process demo implementation can be found in +`contrib/long-running-read-object/example.pl` located in the Git core +repository. If you develop your own long running process then the +`GIT_TRACE_PACKET` environment variables can be very helpful for +debugging (see linkgit:git[1]). diff --git a/contrib/long-running-read-object/example.pl b/contrib/long-running-read-object/example.pl new file mode 100644 index 00000000000000..b8f37f836a813c --- /dev/null +++ b/contrib/long-running-read-object/example.pl @@ -0,0 +1,114 @@ +#!/usr/bin/perl +# +# Example implementation for the Git read-object protocol version 1 +# See Documentation/technical/read-object-protocol.txt +# +# Allows you to test the ability for blobs to be pulled from a host git repo +# "on demand." Called when git needs a blob it couldn't find locally due to +# a lazy clone that only cloned the commits and trees. +# +# A lazy clone can be simulated via the following commands from the host repo +# you wish to create a lazy clone of: +# +# cd /host_repo +# git rev-parse HEAD +# git init /guest_repo +# git cat-file --batch-check --batch-all-objects | grep -v 'blob' | +# cut -d' ' -f1 | git pack-objects /guest_repo/.git/objects/pack/noblobs +# cd /guest_repo +# git config core.virtualizeobjects true +# git reset --hard +# +# Please note, this sample is a minimal skeleton. No proper error handling +# was implemented. +# + +use strict; +use warnings; + +# +# Point $DIR to the folder where your host git repo is located so we can pull +# missing objects from it +# +my $DIR = "/host_repo/.git/"; + +sub packet_bin_read { + my $buffer; + my $bytes_read = read STDIN, $buffer, 4; + if ( $bytes_read == 0 ) { + + # EOF - Git stopped talking to us! + exit(); + } + elsif ( $bytes_read != 4 ) { + die "invalid packet: '$buffer'"; + } + my $pkt_size = hex($buffer); + if ( $pkt_size == 0 ) { + return ( 1, "" ); + } + elsif ( $pkt_size > 4 ) { + my $content_size = $pkt_size - 4; + $bytes_read = read STDIN, $buffer, $content_size; + if ( $bytes_read != $content_size ) { + die "invalid packet ($content_size bytes expected; $bytes_read bytes read)"; + } + return ( 0, $buffer ); + } + else { + die "invalid packet size: $pkt_size"; + } +} + +sub packet_txt_read { + my ( $res, $buf ) = packet_bin_read(); + unless ( $buf =~ s/\n$// ) { + die "A non-binary line MUST be terminated by an LF."; + } + return ( $res, $buf ); +} + +sub packet_bin_write { + my $buf = shift; + print STDOUT sprintf( "%04x", length($buf) + 4 ); + print STDOUT $buf; + STDOUT->flush(); +} + +sub packet_txt_write { + packet_bin_write( $_[0] . "\n" ); +} + +sub packet_flush { + print STDOUT sprintf( "%04x", 0 ); + STDOUT->flush(); +} + +( packet_txt_read() eq ( 0, "git-read-object-client" ) ) || die "bad initialize"; +( packet_txt_read() eq ( 0, "version=1" ) ) || die "bad version"; +( packet_bin_read() eq ( 1, "" ) ) || die "bad version end"; + +packet_txt_write("git-read-object-server"); +packet_txt_write("version=1"); +packet_flush(); + +( packet_txt_read() eq ( 0, "capability=get" ) ) || die "bad capability"; +( packet_bin_read() eq ( 1, "" ) ) || die "bad capability end"; + +packet_txt_write("capability=get"); +packet_flush(); + +while (1) { + my ($command) = packet_txt_read() =~ /^command=([^=]+)$/; + + if ( $command eq "get" ) { + my ($sha1) = packet_txt_read() =~ /^sha1=([0-9a-f]{40})$/; + packet_bin_read(); + + system ('git --git-dir="' . $DIR . '" cat-file blob ' . $sha1 . ' | git -c core.virtualizeobjects=false hash-object -w --stdin >/dev/null 2>&1'); + packet_txt_write(($?) ? "status=error" : "status=success"); + packet_flush(); + } else { + die "bad command '$command'"; + } +} diff --git a/sha1-file.c b/sha1-file.c index 5368dd352ef142..65828b82a1f5f3 100644 --- a/sha1-file.c +++ b/sha1-file.c @@ -32,6 +32,9 @@ #include "packfile.h" #include "fetch-object.h" #include "object-store.h" +#include "sigchain.h" +#include "sub-process.h" +#include "pkt-line.h" /* The maximum size for an object header. */ #define MAX_HEADER_LEN 32 @@ -760,22 +763,113 @@ void prepare_alt_odb(struct repository *r) r->objects->loaded_alternates = 1; } -static int run_read_object_hook(const struct object_id *oid) +#define CAP_GET (1u<<0) + +static int subprocess_map_initialized; +static struct hashmap subprocess_map; + +struct read_object_process { + struct subprocess_entry subprocess; + unsigned int supported_capabilities; +}; + +static int start_read_object_fn(struct subprocess_entry *subprocess) { - struct child_process hook = CHILD_PROCESS_INIT; - const char *p; + struct read_object_process *entry = (struct read_object_process *)subprocess; + static int versions[] = {1, 0}; + static struct subprocess_capability capabilities[] = { + { "get", CAP_GET }, + { NULL, 0 } + }; - p = find_hook("read-object"); - if (!p) - return 1; + return subprocess_handshake(subprocess, "git-read-object", versions, + NULL, capabilities, + &entry->supported_capabilities); +} + +static int read_object_process(const struct object_id *oid) +{ + int err; + struct read_object_process *entry; + struct child_process *process; + struct strbuf status = STRBUF_INIT; + const char *cmd = find_hook("read-object"); + uint64_t start; + + start = getnanotime(); + + if (!subprocess_map_initialized) { + subprocess_map_initialized = 1; + hashmap_init(&subprocess_map, (hashmap_cmp_fn)cmd2process_cmp, + NULL, 0); + entry = NULL; + } else { + entry = (struct read_object_process *) subprocess_find_entry(&subprocess_map, cmd); + } + + if (!entry) { + entry = xmalloc(sizeof(*entry)); + entry->supported_capabilities = 0; + + if (subprocess_start(&subprocess_map, &entry->subprocess, cmd, + start_read_object_fn)) { + free(entry); + return -1; + } + } + process = &entry->subprocess.process; + + if (!(CAP_GET & entry->supported_capabilities)) + return -1; + + sigchain_push(SIGPIPE, SIG_IGN); + + err = packet_write_fmt_gently(process->in, "command=get\n"); + if (err) + goto done; + + err = packet_write_fmt_gently(process->in, "sha1=%s\n", oid_to_hex(oid)); + if (err) + goto done; + + err = packet_flush_gently(process->in); + if (err) + goto done; + + err = subprocess_read_status(process->out, &status); + err = err ? err : strcmp(status.buf, "success"); + +done: + sigchain_pop(SIGPIPE); + + if (err || errno == EPIPE) { + err = err ? err : errno; + if (!strcmp(status.buf, "error")) { + /* The process signaled a problem with the file. */ + } + else if (!strcmp(status.buf, "abort")) { + /* + * The process signaled a permanent problem. Don't try to read + * objects with the same command for the lifetime of the current + * Git process. + */ + entry->supported_capabilities &= ~CAP_GET; + } + else { + /* + * Something went wrong with the read-object process. + * Force shutdown and restart if needed. + */ + error("external process '%s' failed", cmd); + subprocess_stop(&subprocess_map, + (struct subprocess_entry *)entry); + free(entry); + } + } - argv_array_push(&hook.args, p); - argv_array_push(&hook.args, oid_to_hex(oid)); - hook.env = NULL; - hook.no_stdin = 1; - hook.stdout_to_stderr = 1; + trace_performance_since(start, "read_object_process"); - return run_command(&hook); + return err; } /* Returns 1 if we have successfully freshened the file, 0 otherwise. */ @@ -838,7 +932,7 @@ static int check_and_freshen(const struct object_id *oid, int freshen) check_and_freshen_nonlocal(oid, freshen); if (!ret && core_virtualize_objects && !tried_hook) { tried_hook = 1; - if (!run_read_object_hook(oid)) + if (!read_object_process(oid)) goto retry; } @@ -1399,7 +1493,7 @@ int oid_object_info_extended(struct repository *r, const struct object_id *oid, break; if (core_virtualize_objects && !tried_hook) { tried_hook = 1; - if (!run_read_object_hook(oid)) + if (!read_object_process(oid)) goto retry; } } diff --git a/t/t0410/read-object b/t/t0410/read-object new file mode 100755 index 00000000000000..85e997c930581c --- /dev/null +++ b/t/t0410/read-object @@ -0,0 +1,114 @@ +#!/usr/bin/perl +# +# Example implementation for the Git read-object protocol version 1 +# See Documentation/technical/read-object-protocol.txt +# +# Allows you to test the ability for blobs to be pulled from a host git repo +# "on demand." Called when git needs a blob it couldn't find locally due to +# a lazy clone that only cloned the commits and trees. +# +# A lazy clone can be simulated via the following commands from the host repo +# you wish to create a lazy clone of: +# +# cd /host_repo +# git rev-parse HEAD +# git init /guest_repo +# git cat-file --batch-check --batch-all-objects | grep -v 'blob' | +# cut -d' ' -f1 | git pack-objects /guest_repo/.git/objects/pack/noblobs +# cd /guest_repo +# git config core.virtualizeobjects true +# git reset --hard +# +# Please note, this sample is a minimal skeleton. No proper error handling +# was implemented. +# + +use strict; +use warnings; + +# +# Point $DIR to the folder where your host git repo is located so we can pull +# missing objects from it +# +my $DIR = "../.git/"; + +sub packet_bin_read { + my $buffer; + my $bytes_read = read STDIN, $buffer, 4; + if ( $bytes_read == 0 ) { + + # EOF - Git stopped talking to us! + exit(); + } + elsif ( $bytes_read != 4 ) { + die "invalid packet: '$buffer'"; + } + my $pkt_size = hex($buffer); + if ( $pkt_size == 0 ) { + return ( 1, "" ); + } + elsif ( $pkt_size > 4 ) { + my $content_size = $pkt_size - 4; + $bytes_read = read STDIN, $buffer, $content_size; + if ( $bytes_read != $content_size ) { + die "invalid packet ($content_size bytes expected; $bytes_read bytes read)"; + } + return ( 0, $buffer ); + } + else { + die "invalid packet size: $pkt_size"; + } +} + +sub packet_txt_read { + my ( $res, $buf ) = packet_bin_read(); + unless ( $buf =~ s/\n$// ) { + die "A non-binary line MUST be terminated by an LF."; + } + return ( $res, $buf ); +} + +sub packet_bin_write { + my $buf = shift; + print STDOUT sprintf( "%04x", length($buf) + 4 ); + print STDOUT $buf; + STDOUT->flush(); +} + +sub packet_txt_write { + packet_bin_write( $_[0] . "\n" ); +} + +sub packet_flush { + print STDOUT sprintf( "%04x", 0 ); + STDOUT->flush(); +} + +( packet_txt_read() eq ( 0, "git-read-object-client" ) ) || die "bad initialize"; +( packet_txt_read() eq ( 0, "version=1" ) ) || die "bad version"; +( packet_bin_read() eq ( 1, "" ) ) || die "bad version end"; + +packet_txt_write("git-read-object-server"); +packet_txt_write("version=1"); +packet_flush(); + +( packet_txt_read() eq ( 0, "capability=get" ) ) || die "bad capability"; +( packet_bin_read() eq ( 1, "" ) ) || die "bad capability end"; + +packet_txt_write("capability=get"); +packet_flush(); + +while (1) { + my ($command) = packet_txt_read() =~ /^command=([^=]+)$/; + + if ( $command eq "get" ) { + my ($sha1) = packet_txt_read() =~ /^sha1=([0-9a-f]{40})$/; + packet_bin_read(); + + system ('git --git-dir="' . $DIR . '" cat-file blob ' . $sha1 . ' | git -c core.virtualizeobjects=false hash-object -w --stdin >/dev/null 2>&1'); + packet_txt_write(($?) ? "status=error" : "status=success"); + packet_flush(); + } else { + die "bad command '$command'"; + } +} diff --git a/t/t0411-read-object.sh b/t/t0411-read-object.sh index 0c3bfea1886cf9..b8d7521c2c9106 100755 --- a/t/t0411-read-object.sh +++ b/t/t0411-read-object.sh @@ -1,27 +1,27 @@ #!/bin/sh -test_description='tests for read-object hook' +test_description='tests for long running read-object process' . ./test-lib.sh -test_expect_success 'setup host and guest repos' ' +test_expect_success 'setup host repo with a root commit' ' test_commit zero && - hash1=$(git ls-tree HEAD | grep zero.t | cut -f1 | cut -d\ -f3) && - git init guest-repo && - cd guest-repo && - git config core.virtualizeobjects true && - write_script .git/hooks/read-object <<-\EOF - # pass core.virtualizeobjects=false so we dont end up calling the hook proc recursively - git --git-dir=../.git/ cat-file blob "$1" | git -c core.virtualizeobjects=false hash-object -w --stdin >/dev/null 2>&1 - EOF + hash1=$(git ls-tree HEAD | grep zero.t | cut -f1 | cut -d\ -f3) ' test_expect_success 'blobs can be retrieved from the host repo' ' - git cat-file blob "$hash1" + git init guest-repo && + (cd guest-repo && + mkdir -p .git/hooks && + cp $TEST_DIRECTORY/t0410/read-object .git/hooks/ && + git config core.virtualizeobjects true && + git cat-file blob "$hash1") ' test_expect_success 'invalid blobs generate errors' ' - test_must_fail git cat-file blob "invalid" + (cd guest-repo && + test_must_fail git cat-file blob "invalid") ' + test_done From a160dbdbc0845dc00ef7536e63e9ec4034f110d2 Mon Sep 17 00:00:00 2001 From: Ben Peart Date: Tue, 24 Jan 2017 20:48:27 +0100 Subject: [PATCH 030/118] gvfs: introduce pre/post command hooks GVFS Git introduced pre-command and post-command hooks, to gather usage statistics and to be able to adjust the worktree if necessary. As run_hooks() implicitly calls setup_git_directory(), and that function does surprising things to the global state (sometimes even changing the current working directory), it cannot be used here. This commit introduces the pre-command/post-command hooks, based on the previous patches that culminate in support for running hooks early, i.e. before setup_git_directory() was called. Signed-off-by: Ben Peart --- git.c | 84 ++++-------------------------------- t/t0400-pre-command-hook.sh | 34 +++++++++++++++ t/t0401-post-command-hook.sh | 32 ++++++++++++++ 3 files changed, 74 insertions(+), 76 deletions(-) create mode 100755 t/t0400-pre-command-hook.sh create mode 100755 t/t0401-post-command-hook.sh diff --git a/git.c b/git.c index 35a744de28aff3..be09df70adaab2 100644 --- a/git.c +++ b/git.c @@ -4,6 +4,7 @@ #include "help.h" #include "run-command.h" #include "alias.h" +#include "dir.h" #define RUN_SETUP (1<<0) #define RUN_SETUP_GENTLY (1<<1) @@ -380,65 +381,10 @@ static int handle_alias(int *argcp, const char ***argv) return ret; } -/* - * Runs pre/post-command hook. - */ -struct argv_array sargv = ARGV_ARRAY_INIT; -int run_post_hook = 0; -int exit_code = -1; - -static int is_gvfs_repo(void) -{ - wchar_t pwd[MAX_PATH]; - DWORD dwRet; - WIN32_FIND_DATAW FindFileData; - HANDLE hFind; - wchar_t *lastslash; - - dwRet = GetCurrentDirectoryW(MAX_PATH-7, pwd); - if (dwRet == 0 || dwRet > MAX_PATH) - die("GetCurrentDirectory failed (%d)\n", (int)GetLastError()); - - if ('\\' != pwd[wcslen(pwd) - 1]) - wcscat(pwd, L"\\"); - lastslash = pwd + wcslen(pwd) - 1; - while (1) { - wcscat(lastslash, L".gvfs"); - - hFind = FindFirstFileW(pwd, &FindFileData); - if (hFind != INVALID_HANDLE_VALUE) { - FindClose(hFind); - if (FindFileData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) - return 1; - } - - lastslash--; - while ((pwd != lastslash) && (*lastslash != '\\')) - lastslash--; - if (pwd == lastslash) - return 0; - *(lastslash + 1) = 0; - }; - - return 0; -} - -static int run_GVFS_Hooks_argv(const char *const *env, const char *name, - const char *const *argv) -{ - struct child_process hook = CHILD_PROCESS_INIT; - - if (!is_gvfs_repo()) - return 0; - - argv_array_push(&hook.args, "GVFS.Hooks.exe"); - argv_array_push(&hook.args, name); - argv_array_pushv(&hook.args, (const char **)argv); - hook.env = env; - hook.no_stdin = 1; - - return run_command(&hook); -} +/* Runs pre/post-command hook */ +static struct argv_array sargv = ARGV_ARRAY_INIT; +static int run_post_hook = 0; +static int exit_code = -1; static int run_pre_command_hook(const char **argv) { @@ -455,15 +401,9 @@ static int run_pre_command_hook(const char **argv) return 0; setenv("COMMAND_HOOK_LOCK", "true", 1); + /* call the hook proc */ argv_array_pushv(&sargv, argv); - /* - * TODO: This is a temporary hack until we can get config settings - * before executing various git commands without messing up git's state. - * Once we can safely read settings, use the normal hook functions. - * - * ret = run_hook_argv(NULL, "pre-command", sargv.argv); - */ - ret = run_GVFS_Hooks_argv(NULL, "pre-command", sargv.argv); + ret = run_hook_argv(NULL, "pre-command", sargv.argv); if (!ret) run_post_hook = 1; @@ -485,14 +425,7 @@ static int run_post_command_hook(void) return 0; argv_array_pushf(&sargv, "--exit_code=%u", exit_code); - /* - * TODO: This is a temporary hack until we can get config settings - * before executing various git commands without messing up git's state. - * Once we can safely read settings, use the normal hook functions. - * - * ret = run_hook_argv(NULL, "post-command", sargv.argv); - */ - ret = run_GVFS_Hooks_argv(NULL, "post-command", sargv.argv); + ret = run_hook_argv(NULL, "post-command", sargv.argv); run_post_hook = 0; argv_array_clear(&sargv); @@ -502,7 +435,6 @@ static int run_post_command_hook(void) static void post_command_hook_atexit(void) { - fflush(NULL); run_post_command_hook(); } diff --git a/t/t0400-pre-command-hook.sh b/t/t0400-pre-command-hook.sh new file mode 100755 index 00000000000000..4f4f610b52b0a0 --- /dev/null +++ b/t/t0400-pre-command-hook.sh @@ -0,0 +1,34 @@ +#!/bin/sh + +test_description='pre-command hook' + +. ./test-lib.sh + +test_expect_success 'with no hook' ' + echo "first" > file && + git add file && + git commit -m "first" +' + +test_expect_success 'with succeeding hook' ' + mkdir -p .git/hooks && + write_script .git/hooks/pre-command <<-EOF && + echo "\$*" >\$(git rev-parse --git-dir)/pre-command.out + EOF + echo "second" >> file && + git add file && + test "add file" = "$(cat .git/pre-command.out)" && + echo Hello | git hash-object --stdin && + test "hash-object --stdin" = "$(cat .git/pre-command.out)" +' + +test_expect_success 'with failing hook' ' + write_script .git/hooks/pre-command <<-EOF && + exit 1 + EOF + echo "third" >> file && + test_must_fail git add file && + test_path_is_missing "$(cat .git/pre-command.out)" +' + +test_done diff --git a/t/t0401-post-command-hook.sh b/t/t0401-post-command-hook.sh new file mode 100755 index 00000000000000..64646f7ad03b57 --- /dev/null +++ b/t/t0401-post-command-hook.sh @@ -0,0 +1,32 @@ +#!/bin/sh + +test_description='post-command hook' + +. ./test-lib.sh + +test_expect_success 'with no hook' ' + echo "first" > file && + git add file && + git commit -m "first" +' + +test_expect_success 'with succeeding hook' ' + mkdir -p .git/hooks && + write_script .git/hooks/post-command <<-EOF && + echo "\$*" >\$(git rev-parse --git-dir)/post-command.out + EOF + echo "second" >> file && + git add file && + test "add file --exit_code=0" = "$(cat .git/post-command.out)" +' + +test_expect_success 'with failing pre-command hook' ' + write_script .git/hooks/pre-command <<-EOF && + exit 1 + EOF + echo "third" >> file && + test_must_fail git add file && + test_path_is_missing "$(cat .git/post-command.out)" +' + +test_done From b9f19b3fe436eb44d073bf2a6680f3a414049616 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 8 Sep 2017 11:32:43 +0200 Subject: [PATCH 031/118] sha1_file: when writing objects, skip the read_object_hook If we are going to write an object there is no use in calling the read object hook to get an object from a potentially remote source. We would rather just write out the object and avoid the potential round trip for an object that doesn't exist. This change adds a flag to the check_and_freshen() and freshen_loose_object() functions' signatures so that the hook is bypassed when the functions are called before writing loose objects. The check for a local object is still performed so we don't overwrite something that has already been written to one of the objects directories. Based on a patch by Kevin Willford. Signed-off-by: Johannes Schindelin --- sha1-file.c | 17 ++++++++++------- t/t0410/read-object | 4 ++++ t/t0411-read-object.sh | 7 +++++++ 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/sha1-file.c b/sha1-file.c index 65828b82a1f5f3..3bd01b93b19cba 100644 --- a/sha1-file.c +++ b/sha1-file.c @@ -922,7 +922,8 @@ static int check_and_freshen_nonlocal(const struct object_id *oid, int freshen) return 0; } -static int check_and_freshen(const struct object_id *oid, int freshen) +static int check_and_freshen(const struct object_id *oid, int freshen, + int skip_virtualized_objects) { int ret; int tried_hook = 0; @@ -930,7 +931,8 @@ static int check_and_freshen(const struct object_id *oid, int freshen) retry: ret = check_and_freshen_local(oid, freshen) || check_and_freshen_nonlocal(oid, freshen); - if (!ret && core_virtualize_objects && !tried_hook) { + if (!ret && core_virtualize_objects && !skip_virtualized_objects && + !tried_hook) { tried_hook = 1; if (!read_object_process(oid)) goto retry; @@ -946,7 +948,7 @@ int has_loose_object_nonlocal(const struct object_id *oid) static int has_loose_object(const struct object_id *oid) { - return check_and_freshen(oid, 0); + return check_and_freshen(oid, 0, 0); } static void mmap_limit_check(size_t length) @@ -1876,9 +1878,10 @@ static int write_loose_object(const struct object_id *oid, char *hdr, return finalize_object_file(tmp_file.buf, filename.buf); } -static int freshen_loose_object(const struct object_id *oid) +static int freshen_loose_object(const struct object_id *oid, + int skip_virtualized_objects) { - return check_and_freshen(oid, 1); + return check_and_freshen(oid, 1, skip_virtualized_objects); } static int freshen_packed_object(const struct object_id *oid) @@ -1904,7 +1907,7 @@ int write_object_file(const void *buf, unsigned long len, const char *type, * it out into .git/objects/??/?{38} file. */ write_object_file_prepare(buf, len, type, oid, hdr, &hdrlen); - if (freshen_packed_object(oid) || freshen_loose_object(oid)) + if (freshen_packed_object(oid) || freshen_loose_object(oid, 1)) return 0; return write_loose_object(oid, hdr, hdrlen, buf, len, 0); } @@ -1923,7 +1926,7 @@ int hash_object_file_literally(const void *buf, unsigned long len, if (!(flags & HASH_WRITE_OBJECT)) goto cleanup; - if (freshen_packed_object(oid) || freshen_loose_object(oid)) + if (freshen_packed_object(oid) || freshen_loose_object(oid, 1)) goto cleanup; status = write_loose_object(oid, header, hdrlen, buf, len, 0); diff --git a/t/t0410/read-object b/t/t0410/read-object index 85e997c930581c..2360d4916ee81c 100755 --- a/t/t0410/read-object +++ b/t/t0410/read-object @@ -108,6 +108,10 @@ while (1) { system ('git --git-dir="' . $DIR . '" cat-file blob ' . $sha1 . ' | git -c core.virtualizeobjects=false hash-object -w --stdin >/dev/null 2>&1'); packet_txt_write(($?) ? "status=error" : "status=success"); packet_flush(); + + open my $log, '>>.git/read-object-hook.log'; + print $log "Read object $sha1, exit code $?\n"; + close $log; } else { die "bad command '$command'"; } diff --git a/t/t0411-read-object.sh b/t/t0411-read-object.sh index b8d7521c2c9106..af97ec5b50f356 100755 --- a/t/t0411-read-object.sh +++ b/t/t0411-read-object.sh @@ -23,5 +23,12 @@ test_expect_success 'invalid blobs generate errors' ' test_must_fail git cat-file blob "invalid") ' +test_expect_success 'read-object-hook is bypassed when writing objects' ' + (cd guest-repo && + echo hello >hello.txt && + git add hello.txt && + hash="$(git rev-parse --verify :hello.txt)" && + ! grep "$hash" .git/read-object-hook.log) +' test_done From 5e5e4968e637e266e17ced3a605cf00a6f3f4242 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 16 Mar 2017 21:07:54 +0100 Subject: [PATCH 032/118] t0400: verify that the hook is called correctly from a subdirectory Suggested by Ben Peart. Signed-off-by: Johannes Schindelin --- t/t0400-pre-command-hook.sh | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/t/t0400-pre-command-hook.sh b/t/t0400-pre-command-hook.sh index 4f4f610b52b0a0..8e0e30cd845769 100755 --- a/t/t0400-pre-command-hook.sh +++ b/t/t0400-pre-command-hook.sh @@ -31,4 +31,27 @@ test_expect_success 'with failing hook' ' test_path_is_missing "$(cat .git/pre-command.out)" ' +test_expect_success 'in a subdirectory' ' + echo touch i-was-here | write_script .git/hooks/pre-command && + mkdir sub && + ( + cd sub && + git version + ) && + test_path_is_file sub/i-was-here +' + +test_expect_success 'in a subdirectory, using an alias' ' + git reset --hard && + echo "echo \"\$@; \$(pwd)\" >>log" | + write_script .git/hooks/pre-command && + mkdir -p sub && + ( + cd sub && + git -c alias.r="rebase HEAD" r + ) && + test_path_is_missing log && + test_line_count = 2 sub/log +' + test_done From 05e24c6f2ec8b3c3a99ee65c0994ebe2c41f2a12 Mon Sep 17 00:00:00 2001 From: Alejandro Pauly Date: Mon, 10 Apr 2017 13:26:14 -0400 Subject: [PATCH 033/118] Pass PID of git process to hooks. Signed-off-by: Alejandro Pauly --- git.c | 1 + t/t0400-pre-command-hook.sh | 3 ++- t/t0401-post-command-hook.sh | 3 ++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/git.c b/git.c index be09df70adaab2..99b547cc3065ea 100644 --- a/git.c +++ b/git.c @@ -403,6 +403,7 @@ static int run_pre_command_hook(const char **argv) /* call the hook proc */ argv_array_pushv(&sargv, argv); + argv_array_pushf(&sargv, "--git-pid=%"PRIuMAX, (uintmax_t)getpid()); ret = run_hook_argv(NULL, "pre-command", sargv.argv); if (!ret) diff --git a/t/t0400-pre-command-hook.sh b/t/t0400-pre-command-hook.sh index 8e0e30cd845769..492bf31c799492 100755 --- a/t/t0400-pre-command-hook.sh +++ b/t/t0400-pre-command-hook.sh @@ -13,7 +13,8 @@ test_expect_success 'with no hook' ' test_expect_success 'with succeeding hook' ' mkdir -p .git/hooks && write_script .git/hooks/pre-command <<-EOF && - echo "\$*" >\$(git rev-parse --git-dir)/pre-command.out + echo "\$*" | sed "s/ --git-pid=[0-9]*//" \ + >\$(git rev-parse --git-dir)/pre-command.out EOF echo "second" >> file && git add file && diff --git a/t/t0401-post-command-hook.sh b/t/t0401-post-command-hook.sh index 64646f7ad03b57..fcbfc4a0c79c1e 100755 --- a/t/t0401-post-command-hook.sh +++ b/t/t0401-post-command-hook.sh @@ -13,7 +13,8 @@ test_expect_success 'with no hook' ' test_expect_success 'with succeeding hook' ' mkdir -p .git/hooks && write_script .git/hooks/post-command <<-EOF && - echo "\$*" >\$(git rev-parse --git-dir)/post-command.out + echo "\$*" | sed "s/ --git-pid=[0-9]*//" \ + >\$(git rev-parse --git-dir)/post-command.out EOF echo "second" >> file && git add file && From 43eaae491e9d797ddeefa03a1083393689bf9040 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 8 Aug 2017 00:27:50 +0200 Subject: [PATCH 034/118] pre-command: always respect core.hooksPath We need to respect that config setting even if we already know that we have a repository, but have not yet read the config. The regression test was written by Alejandro Pauly. Signed-off-by: Johannes Schindelin --- run-command.c | 13 +++++++++++-- t/t0400-pre-command-hook.sh | 11 +++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/run-command.c b/run-command.c index 94254e9d8d29af..f8a7f733718378 100644 --- a/run-command.c +++ b/run-command.c @@ -1320,9 +1320,18 @@ const char *find_hook(const char *name) static struct strbuf path = STRBUF_INIT; strbuf_reset(&path); - if (have_git_dir()) + if (have_git_dir()) { + static int forced_config; + + if (!forced_config) { + if (!git_hooks_path) + git_config_get_pathname("core.hookspath", + &git_hooks_path); + forced_config = 1; + } + strbuf_git_path(&path, "hooks/%s", name); - else if (!hook_path_early(name, &path)) + } else if (!hook_path_early(name, &path)) return NULL; if (access(path.buf, X_OK) < 0) { diff --git a/t/t0400-pre-command-hook.sh b/t/t0400-pre-command-hook.sh index 492bf31c799492..2ab9833af5cce1 100755 --- a/t/t0400-pre-command-hook.sh +++ b/t/t0400-pre-command-hook.sh @@ -55,4 +55,15 @@ test_expect_success 'in a subdirectory, using an alias' ' test_line_count = 2 sub/log ' +test_expect_success 'with core.hooksPath' ' + mkdir -p .git/alternateHooks && + write_script .git/alternateHooks/pre-command <<-EOF && + echo "alternate" >\$(git rev-parse --git-dir)/pre-command.out + EOF + write_script .git/hooks/pre-command <<-EOF && + echo "original" >\$(git rev-parse --git-dir)/pre-command.out + EOF + git -c core.hooksPath=.git/alternateHooks status && + test "alternate" = "$(cat .git/pre-command.out)" +' test_done From 66bcffe8b341c997e3554eb75332933d435d52c9 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Tue, 9 Jan 2018 14:16:07 -0500 Subject: [PATCH 035/118] status: add status.aheadbehind setting Add "status.aheadbehind" config setting to change the default behavior of ALL git status formats. Signed-off-by: Jeff Hostetler --- Documentation/config/status.txt | 6 ++++++ builtin/commit.c | 17 ++++++++++++++++- t/t6040-tracking-info.sh | 31 +++++++++++++++++++++++++++++++ t/t7064-wtstatus-pv2.sh | 4 ++++ 4 files changed, 57 insertions(+), 1 deletion(-) diff --git a/Documentation/config/status.txt b/Documentation/config/status.txt index a2d09a8846af08..764de31fc46e0a 100644 --- a/Documentation/config/status.txt +++ b/Documentation/config/status.txt @@ -12,6 +12,12 @@ status.branch:: Set to true to enable --branch by default in linkgit:git-status[1]. The option --no-branch takes precedence over this variable. +status.aheadBehind:: + EXPERIMENTAL, Set to true to enable --ahead-behind and false to + enable --no-ahead-behind by default in linkgit:git-status[1] for + all status formats. Defaults to true. Changing this may + break backwards compatibility for existing scripts. + status.displayCommentPrefix:: If set to true, linkgit:git-status[1] will insert a comment prefix before each output line (starting with diff --git a/builtin/commit.c b/builtin/commit.c index daf48f696e62c0..3d866338129014 100644 --- a/builtin/commit.c +++ b/builtin/commit.c @@ -1127,9 +1127,11 @@ static const char *read_commit_message(const char *name) static struct status_deferred_config { enum wt_status_format status_format; int show_branch; + enum ahead_behind_flags ahead_behind; } status_deferred_config = { STATUS_FORMAT_UNSPECIFIED, - -1 /* unspecified */ + -1, /* unspecified */ + AHEAD_BEHIND_UNSPECIFIED, }; static void finalize_deferred_config(struct wt_status *s) @@ -1156,6 +1158,15 @@ static void finalize_deferred_config(struct wt_status *s) if (s->show_branch < 0) s->show_branch = 0; + /* + * If the user did not give a "--[no]-ahead-behind" command + * line argument, then we inherit the a/b config setting. + * If is not set, then we inherit _FULL for backwards + * compatibility. + */ + if (s->ahead_behind_flags == AHEAD_BEHIND_UNSPECIFIED) + s->ahead_behind_flags = status_deferred_config.ahead_behind; + if (s->ahead_behind_flags == AHEAD_BEHIND_UNSPECIFIED) s->ahead_behind_flags = AHEAD_BEHIND_FULL; } @@ -1307,6 +1318,10 @@ static int git_status_config(const char *k, const char *v, void *cb) status_deferred_config.show_branch = git_config_bool(k, v); return 0; } + if (!strcmp(k, "status.aheadbehind")) { + status_deferred_config.ahead_behind = git_config_bool(k, v); + return 0; + } if (!strcmp(k, "status.showstash")) { s->show_stash = git_config_bool(k, v); return 0; diff --git a/t/t6040-tracking-info.sh b/t/t6040-tracking-info.sh index 716283b274677d..febf63f28a54cd 100755 --- a/t/t6040-tracking-info.sh +++ b/t/t6040-tracking-info.sh @@ -159,6 +159,19 @@ test_expect_success 'status -s -b --no-ahead-behind (diverged from upstream)' ' test_i18ncmp expect actual ' +cat >expect <<\EOF +## b1...origin/master [different] +EOF + +test_expect_success 'status.aheadbehind=false status -s -b (diverged from upstream)' ' + ( + cd test && + git checkout b1 >/dev/null && + git -c status.aheadbehind=false status -s -b | head -1 + ) >actual && + test_i18ncmp expect actual +' + cat >expect <<\EOF On branch b1 Your branch and 'origin/master' have diverged, @@ -174,6 +187,15 @@ test_expect_success 'status --long --branch' ' test_i18ncmp expect actual ' +test_expect_success 'status --long --branch' ' + ( + cd test && + git checkout b1 >/dev/null && + git -c status.aheadbehind=true status --long -b | head -3 + ) >actual && + test_i18ncmp expect actual +' + cat >expect <<\EOF On branch b1 Your branch and 'origin/master' refer to different commits. @@ -188,6 +210,15 @@ test_expect_success 'status --long --branch --no-ahead-behind' ' test_i18ncmp expect actual ' +test_expect_success 'status.aheadbehind=false status --long --branch' ' + ( + cd test && + git checkout b1 >/dev/null && + git -c status.aheadbehind=false status --long -b | head -2 + ) >actual && + test_i18ncmp expect actual +' + cat >expect <<\EOF ## b5...brokenbase [gone] EOF diff --git a/t/t7064-wtstatus-pv2.sh b/t/t7064-wtstatus-pv2.sh index 11eccc231a792b..a0baf6e8b09f95 100755 --- a/t/t7064-wtstatus-pv2.sh +++ b/t/t7064-wtstatus-pv2.sh @@ -436,6 +436,10 @@ test_expect_success 'verify --[no-]ahead-behind with V2 format' ' git status --no-ahead-behind --porcelain=v2 --branch --untracked-files=all >actual && test_cmp expect actual && + # Confirmat that "status.aheadbehind" works on V2 format. + git -c status.aheadbehind=false status --porcelain=v2 --branch --untracked-files=all >actual && + test_cmp expect actual && + # Confirm --ahead-behind reports traditional branch.ab with 1/0. cat >expect <<-EOF && # branch.oid $HUF From 4e6e83e264ffa71343999e826394153be9164aa5 Mon Sep 17 00:00:00 2001 From: Jameson Miller Date: Wed, 10 Jan 2018 11:56:26 -0500 Subject: [PATCH 036/118] Teach ahead-behind and serialized status to play nicely together --- t/t7524-serialized-status.sh | 26 ++++++++++++++++++++++++++ wt-status-deserialize.c | 2 +- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/t/t7524-serialized-status.sh b/t/t7524-serialized-status.sh index a0fc1bc2daf6ec..6bcd301dbb7913 100755 --- a/t/t7524-serialized-status.sh +++ b/t/t7524-serialized-status.sh @@ -137,4 +137,30 @@ test_expect_success 'verify serialized status handles path scopes' ' test_i18ncmp expect output ' +test_expect_success 'verify no-ahead-behind and serialized status integration' ' + test_when_finished "rm serialized_status.dat new_change.txt output" && + cat >expect <<-\EOF && + # branch.oid 68d4a437ea4c2de65800f48c053d4d543b55c410 + # branch.head alt_branch + # branch.upstream master + # branch.ab +1 -0 + ? expect + ? serialized_status.dat + ? untracked/ + ? untracked_1.txt + EOF + + git checkout -b alt_branch master --track >/dev/null && + touch alt_branch_changes.txt && + git add alt_branch_changes.txt && + test_tick && + git commit -m"New commit on alt branch" && + + git status --untracked-files=complete --ignored=matching --serialize >serialized_status.dat && + touch new_change.txt && + + git -c status.aheadBehind=false status --porcelain=v2 --branch --ahead-behind --deserialize=serialized_status.dat >output && + test_i18ncmp expect output +' + test_done diff --git a/wt-status-deserialize.c b/wt-status-deserialize.c index 82eb8d8fca591b..f8232bb7524060 100644 --- a/wt-status-deserialize.c +++ b/wt-status-deserialize.c @@ -548,6 +548,7 @@ static int wt_deserialize_fd(const struct wt_status *cmd_s, struct wt_status *de des_s->show_branch = cmd_s->show_branch; des_s->show_stash = cmd_s->show_stash; /* hints */ + des_s->ahead_behind_flags = cmd_s->ahead_behind_flags; des_s->status_format = cmd_s->status_format; des_s->fp = cmd_s->fp; if (cmd_s->prefix && *cmd_s->prefix) @@ -556,7 +557,6 @@ static int wt_deserialize_fd(const struct wt_status *cmd_s, struct wt_status *de return DESERIALIZE_OK; } - /* * Read raw serialized status data from the given file * From 0d9c5fd481d50317f395cd35145fd8e442c92162 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Fri, 14 Apr 2017 10:59:20 -0600 Subject: [PATCH 037/118] gvfs: refactor loading the core.gvfs config value This code change makes sure that the config value for core_gvfs is always loaded before checking it. Signed-off-by: Kevin Willford --- Makefile | 1 + gvfs.c | 38 ++++++++++++++++++++++++++++++++++++++ gvfs.h | 31 ++----------------------------- 3 files changed, 41 insertions(+), 29 deletions(-) create mode 100644 gvfs.c diff --git a/Makefile b/Makefile index 7b9668c1a747a8..ab2fe57df4daa2 100644 --- a/Makefile +++ b/Makefile @@ -917,6 +917,7 @@ LIB_OBJS += gettext.o LIB_OBJS += gpg-interface.o LIB_OBJS += graph.o LIB_OBJS += grep.o +LIB_OBJS += gvfs.o LIB_OBJS += hashmap.o LIB_OBJS += linear-assignment.o LIB_OBJS += help.o diff --git a/gvfs.c b/gvfs.c new file mode 100644 index 00000000000000..7235199c04ac39 --- /dev/null +++ b/gvfs.c @@ -0,0 +1,38 @@ +#include "cache.h" +#include "gvfs.h" +#include "config.h" + +static int gvfs_config_loaded; +static int core_gvfs_is_bool; + +static int early_core_gvfs_config(const char *var, const char *value, void *data) +{ + if (!strcmp(var, "core.gvfs")) + core_gvfs = git_config_bool_or_int("core.gvfs", value, &core_gvfs_is_bool); + return 0; +} + +void gvfs_load_config_value(const char *value) +{ + if (gvfs_config_loaded) + return; + + if (value) + core_gvfs = git_config_bool_or_int("core.gvfs", value, &core_gvfs_is_bool); + else if (startup_info->have_repository == 0) + read_early_config(early_core_gvfs_config, NULL); + else + git_config_get_bool_or_int("core.gvfs", &core_gvfs_is_bool, &core_gvfs); + + /* Turn on all bits if a bool was set in the settings */ + if (core_gvfs_is_bool && core_gvfs) + core_gvfs = -1; + + gvfs_config_loaded = 1; +} + +int gvfs_config_is_set(int mask) +{ + gvfs_load_config_value(0); + return (core_gvfs & mask) == mask; +} diff --git a/gvfs.h b/gvfs.h index 2d6de575bf4a65..7c9367866f502a 100644 --- a/gvfs.h +++ b/gvfs.h @@ -1,8 +1,6 @@ #ifndef GVFS_H #define GVFS_H -#include "cache.h" -#include "config.h" /* * This file is for the specific settings and methods @@ -19,32 +17,7 @@ #define GVFS_FETCH_SKIP_REACHABILITY_AND_UPLOADPACK (1 << 4) #define GVFS_BLOCK_FILTERS_AND_EOL_CONVERSIONS (1 << 6) -static inline int gvfs_config_is_set(int mask) { - return (core_gvfs & mask) == mask; -} - -static inline int gvfs_config_is_set_any(void) { - return core_gvfs > 0; -} - -static inline void gvfs_load_config_value(const char *value) { - int is_bool = 0; - - if (value) - core_gvfs = git_config_bool_or_int("core.gvfs", value, &is_bool); - else - git_config_get_bool_or_int("core.gvfs", &is_bool, &core_gvfs); - - /* Turn on all bits if a bool was set in the settings */ - if (is_bool && core_gvfs) - core_gvfs = -1; -} - - -static inline int gvfs_config_load_and_is_set(int mask) { - gvfs_load_config_value(0); - return gvfs_config_is_set(mask); -} - +void gvfs_load_config_value(const char *value); +int gvfs_config_is_set(int mask); #endif /* GVFS_H */ From 8a3d1e9498e4983af76167ee39d0f0a916d48b24 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 10 Jan 2018 13:50:24 -0500 Subject: [PATCH 038/118] status: add warning when a/b calculation takes too long for long/normal format Signed-off-by: Jeff Hostetler --- advice.c | 2 ++ advice.h | 1 + wt-status.c | 17 +++++++++++++++++ 3 files changed, 20 insertions(+) diff --git a/advice.c b/advice.c index 567209aa79afee..b6f09947d2b7ac 100644 --- a/advice.c +++ b/advice.c @@ -12,6 +12,7 @@ int advice_push_needs_force = 1; int advice_push_unqualified_ref_name = 1; int advice_status_hints = 1; int advice_status_u_option = 1; +int advice_status_ahead_behind_warning = 1; int advice_commit_before_merge = 1; int advice_reset_quiet_warning = 1; int advice_resolve_conflict = 1; @@ -67,6 +68,7 @@ static struct { { "pushUnqualifiedRefName", &advice_push_unqualified_ref_name }, { "statusHints", &advice_status_hints }, { "statusUoption", &advice_status_u_option }, + { "statusAheadBehindWarning", &advice_status_ahead_behind_warning }, { "commitBeforeMerge", &advice_commit_before_merge }, { "resetQuiet", &advice_reset_quiet_warning }, { "resolveConflict", &advice_resolve_conflict }, diff --git a/advice.h b/advice.h index f875f8cd8da5fd..9dc9e491094de8 100644 --- a/advice.h +++ b/advice.h @@ -12,6 +12,7 @@ extern int advice_push_needs_force; extern int advice_push_unqualified_ref_name; extern int advice_status_hints; extern int advice_status_u_option; +extern int advice_status_ahead_behind_warning; extern int advice_commit_before_merge; extern int advice_reset_quiet_warning; extern int advice_resolve_conflict; diff --git a/wt-status.c b/wt-status.c index 639509bb6fdc43..2e8683ee084579 100644 --- a/wt-status.c +++ b/wt-status.c @@ -18,6 +18,8 @@ #include "worktree.h" #include "lockfile.h" +#define AB_DELAY_WARNING_IN_MS (2 * 1000) + static const char cut_line[] = "------------------------ >8 ------------------------\n"; @@ -1070,14 +1072,29 @@ static void wt_longstatus_print_tracking(struct wt_status *s) struct branch *branch; char comment_line_string[3]; int i; + uint64_t t_begin = 0; assert(s->branch && !s->is_initial); if (!skip_prefix(s->branch, "refs/heads/", &branch_name)) return; branch = branch_get(branch_name); + + t_begin = getnanotime(); + if (!format_tracking_info(branch, &sb, s->ahead_behind_flags)) return; + if (advice_status_ahead_behind_warning && + s->ahead_behind_flags == AHEAD_BEHIND_FULL) { + uint64_t t_delta_in_ms = (getnanotime() - t_begin) / 1000000; + if (t_delta_in_ms > AB_DELAY_WARNING_IN_MS) { + strbuf_addf(&sb, _("\n" + "It took %.2f seconds to compute the branch ahead/behind values.\n" + "You can use '--no-ahead-behind' to avoid this.\n"), + t_delta_in_ms / 1000.0); + } + } + i = 0; if (s->display_comment_prefix) { comment_line_string[i++] = comment_line_char; From 1cce3bef75a54bb2fae8d1f678d1e88b10682950 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Tue, 22 Aug 2017 11:54:23 -0400 Subject: [PATCH 039/118] status: add status serialization mechanism Teach STATUS to optionally serialize the results of a status computation to a file. Teach STATUS to optionally read an existing serialization file and simply print the results, rather than actually scanning. This is intended for immediate status results on extremely large repos and assumes the use of a service/daemon to maintain a fresh current status snapshot. Signed-off-by: Jeff Hostetler --- Documentation/config/status.txt | 6 + Documentation/git-status.txt | 33 + .../technical/status-serialization-format.txt | 107 ++++ Makefile | 2 + builtin/commit.c | 123 +++- pkt-line.c | 2 +- pkt-line.h | 1 + t/t7523-status-complete-untracked.sh | 39 ++ t/t7524-serialized-status.sh | 140 ++++ wt-status-deserialize.c | 597 ++++++++++++++++++ wt-status-serialize.c | 213 +++++++ wt-status.c | 6 + wt-status.h | 52 +- 13 files changed, 1318 insertions(+), 3 deletions(-) create mode 100644 Documentation/technical/status-serialization-format.txt create mode 100755 t/t7523-status-complete-untracked.sh create mode 100755 t/t7524-serialized-status.sh create mode 100644 wt-status-deserialize.c create mode 100644 wt-status-serialize.c diff --git a/Documentation/config/status.txt b/Documentation/config/status.txt index ed72fa7daece09..a2d09a8846af08 100644 --- a/Documentation/config/status.txt +++ b/Documentation/config/status.txt @@ -70,3 +70,9 @@ status.submoduleSummary:: the --ignore-submodules=dirty command-line option or the 'git submodule summary' command, which shows a similar output but does not honor these settings. + +status.deserializePath:: + EXPERIMENTAL, Pathname to a file containing cached status results + generated by `--serialize`. This will be overridden by + `--deserialize=` on the command line. If the cache file is + invalid or stale, git will fall-back and compute status normally. diff --git a/Documentation/git-status.txt b/Documentation/git-status.txt index e5756a6b668373..6bf716bf43c6a4 100644 --- a/Documentation/git-status.txt +++ b/Documentation/git-status.txt @@ -152,6 +152,19 @@ ignored, then the directory is not shown, but all contents are shown. update it afterwards if any changes were detected. Defaults to `--lock-index`. +--serialize[=]:: + (EXPERIMENTAL) Serialize raw status results to stdout in a + format suitable for use by `--deserialize`. Valid values for + `` are "1" and "v1". + +--deserialize[=]:: + (EXPERIMENTAL) Deserialize raw status results from a file or + stdin rather than scanning the worktree. If `` is omitted + and `status.deserializePath` is unset, input is read from stdin. +--no-deserialize:: + (EXPERIMENTAL) Disable implicit deserialization of status results + from the value of `status.deserializePath`. + ...:: See the 'pathspec' entry in linkgit:gitglossary[7]. @@ -397,6 +410,26 @@ quoted as explained for the configuration variable `core.quotePath` (see linkgit:git-config[1]). +SERIALIZATION and DESERIALIZATION (EXPERIMENTAL) +------------------------------------------------ + +The `--serialize` option allows git to cache the result of a +possibly time-consuming status scan to a binary file. A local +service/daemon watching file system events could use this to +periodically pre-compute a fresh status result. + +Interactive users could then use `--deserialize` to simply +(and immediately) print the last-known-good result without +waiting for the status scan. + +The binary serialization file format includes some worktree state +information allowing `--deserialize` to reject the cached data +and force a normal status scan if, for example, the commit, branch, +or status modes/options change. The format cannot, however, indicate +when the cached data is otherwise stale -- that coordination belongs +to the task driving the serializations. + + CONFIGURATION ------------- diff --git a/Documentation/technical/status-serialization-format.txt b/Documentation/technical/status-serialization-format.txt new file mode 100644 index 00000000000000..475ae814495581 --- /dev/null +++ b/Documentation/technical/status-serialization-format.txt @@ -0,0 +1,107 @@ +Git status serialization format +=============================== + +Git status serialization enables git to dump the results of a status scan +to a binary file. This file can then be loaded by later status invocations +to print the cached status results. + +The file contains the essential fields from: +() the index +() the "struct wt_status" for the overall results +() the contents of "struct wt_status_change_data" for tracked changed files +() the list of untracked and ignored files + +Version 1 Format: +================= + +The V1 file begins with a required header section followed by optional +sections for each type of item (changed, untracked, ignored). Individual +item sections are only present if necessary. Each item section begins +with an item-type header with the number of items in the section. + +Each "line" in the format is encoded using pkt-line with a final LF. +Flush packets are used to terminate sections. + +----------------- +PKT-LINE("version" SP "1") + +[] +[] +[] +----------------- + + +V1 Header +--------- + +The v1-header-section fields are taken directly from "struct wt_status". +Each field is printed on a separate pkt-line. Lines for NULL string +values are omitted. All integers are printed with "%d". OIDs are +printed in hex. + +v1-header-section = + + PKT-LINE() + +v1-index-headers = PKT-LINE("index_mtime" SP SP LF) + +v1-wt-status-headers = PKT-LINE("is_initial" SP LF) + [ PKT-LINE("branch" SP LF) ] + [ PKT-LINE("reference" SP LF) ] + PKT-LINE("show_ignored_files" SP LF) + PKT-LINE("show_untracked_files" SP LF) + PKT-LINE("show_ignored_directory" SP LF) + [ PKT-LINE("ignore_submodule_arg" SP LF) ] + PKT-LINE("detect_rename" SP LF) + PKT-LINE("rename_score" SP LF) + PKT-LINE("rename_limit" SP LF) + PKT-LINE("detect_break" SP LF) + PKT-LINE("sha1_commit" SP LF) + PKT-LINE("committable" SP LF) + PKT-LINE("workdir_dirty" SP LF) + + +V1 Changed Items +---------------- + +The v1-changed-item-section lists all of the changed items with one +item per pkt-line. Each pkt-line contains: a binary block of data +from "struct wt_status_serialize_data_fixed" in a fixed header where +integers are in network byte order and OIDs are in raw (non-hex) form. +This is followed by one or two raw pathnames (not c-quoted) with NUL +terminators (both NULs are always present even if there is no rename). + +v1-changed-item-section = PKT-LINE("changed" SP LF) + [ PKT-LINE( LF) ]+ + PKT-LINE() + +changed_item = + + + + + + + + + + + + NUL + [ ] + NUL + + +V1 Untracked and Ignored Items +------------------------------ + +These sections are simple lists of pathnames. They ARE NOT +c-quoted. + +v1-untracked-item-section = PKT-LINE("untracked" SP LF) + [ PKT-LINE( LF) ]+ + PKT-LINE() + +v1-ignored-item-section = PKT-LINE("ignored" SP LF) + [ PKT-LINE( LF) ]+ + PKT-LINE() diff --git a/Makefile b/Makefile index ab2fe57df4daa2..66dbf6cc71211b 100644 --- a/Makefile +++ b/Makefile @@ -1044,6 +1044,8 @@ LIB_OBJS += wrapper.o LIB_OBJS += write-or-die.o LIB_OBJS += ws.o LIB_OBJS += wt-status.o +LIB_OBJS += wt-status-deserialize.o +LIB_OBJS += wt-status-serialize.o LIB_OBJS += xdiff-interface.o LIB_OBJS += zlib.o diff --git a/builtin/commit.c b/builtin/commit.c index 8348694141cdd0..daf48f696e62c0 100644 --- a/builtin/commit.c +++ b/builtin/commit.c @@ -143,6 +143,70 @@ static int opt_parse_porcelain(const struct option *opt, const char *arg, int un return 0; } +static int do_serialize = 0; +static int do_implicit_deserialize = 0; +static int do_explicit_deserialize = 0; +static char *deserialize_path = NULL; + +/* + * --serialize | --serialize=1 | --serialize=v1 + * + * Request that we serialize our output rather than printing in + * any of the established formats. Optionally specify serialization + * version. + */ +static int opt_parse_serialize(const struct option *opt, const char *arg, int unset) +{ + enum wt_status_format *value = (enum wt_status_format *)opt->value; + if (unset || !arg) + *value = STATUS_FORMAT_SERIALIZE_V1; + else if (!strcmp(arg, "v1") || !strcmp(arg, "1")) + *value = STATUS_FORMAT_SERIALIZE_V1; + else + die("unsupported serialize version '%s'", arg); + + if (do_explicit_deserialize) + die("cannot mix --serialize and --deserialize"); + do_implicit_deserialize = 0; + + do_serialize = 1; + return 0; +} + +/* + * --deserialize | --deserialize= | + * --no-deserialize + * + * Request that we deserialize status data from some existing resource + * rather than performing a status scan. + * + * The input source can come from stdin or a path given here -- or be + * inherited from the config settings. + */ +static int opt_parse_deserialize(const struct option *opt, const char *arg, int unset) +{ + if (unset) { + do_implicit_deserialize = 0; + do_explicit_deserialize = 0; + } else { + if (do_serialize) + die("cannot mix --serialize and --deserialize"); + if (arg) { + /* override config or stdin */ + free(deserialize_path); + deserialize_path = xstrdup(arg); + } + if (deserialize_path && *deserialize_path + && (access(deserialize_path, R_OK) != 0)) + die("cannot find serialization file '%s'", + deserialize_path); + + do_explicit_deserialize = 1; + } + + return 0; +} + static int opt_parse_m(const struct option *opt, const char *arg, int unset) { struct strbuf *buf = opt->value; @@ -1038,6 +1102,8 @@ static void handle_untracked_files_arg(struct wt_status *s) s->show_untracked_files = SHOW_NORMAL_UNTRACKED_FILES; else if (!strcmp(untracked_files_arg, "all")) s->show_untracked_files = SHOW_ALL_UNTRACKED_FILES; + else if (!strcmp(untracked_files_arg,"complete")) + s->show_untracked_files = SHOW_COMPLETE_UNTRACKED_FILES; else die(_("Invalid untracked files mode '%s'"), untracked_files_arg); } @@ -1266,6 +1332,19 @@ static int git_status_config(const char *k, const char *v, void *cb) s->relative_paths = git_config_bool(k, v); return 0; } + if (!strcmp(k, "status.deserializepath")) { + /* + * Automatically assume deserialization if this is + * set in the config and the file exists. Do not + * complain if the file does not exist, because we + * silently fall back to normal mode. + */ + if (v && *v && access(v, R_OK) == 0) { + do_implicit_deserialize = 1; + deserialize_path = xstrdup(v); + } + return 0; + } if (!strcmp(k, "status.showuntrackedfiles")) { if (!v) return config_error_nonbool(k); @@ -1308,7 +1387,8 @@ int cmd_status(int argc, const char **argv, const char *prefix) static int show_ignored_directory = 0; static struct wt_status s; unsigned int progress_flag = 0; - int fd; + int try_deserialize; + int fd = -1; struct object_id oid; static struct option builtin_status_options[] = { OPT__VERBOSE(&verbose, N_("be verbose")), @@ -1323,6 +1403,12 @@ int cmd_status(int argc, const char **argv, const char *prefix) { OPTION_CALLBACK, 0, "porcelain", &status_format, N_("version"), N_("machine-readable output"), PARSE_OPT_OPTARG, opt_parse_porcelain }, + { OPTION_CALLBACK, 0, "serialize", &status_format, + N_("version"), N_("serialize raw status data to stdout"), + PARSE_OPT_OPTARG | PARSE_OPT_NONEG, opt_parse_serialize }, + { OPTION_CALLBACK, 0, "deserialize", NULL, + N_("path"), N_("deserialize raw status data from file"), + PARSE_OPT_OPTARG, opt_parse_deserialize }, OPT_SET_INT(0, "long", &status_format, N_("show status in long format (default)"), STATUS_FORMAT_LONG), @@ -1383,10 +1469,26 @@ int cmd_status(int argc, const char **argv, const char *prefix) s.show_untracked_files == SHOW_NO_UNTRACKED_FILES) die(_("Unsupported combination of ignored and untracked-files arguments")); + if (s.show_untracked_files == SHOW_COMPLETE_UNTRACKED_FILES && + s.show_ignored_mode == SHOW_NO_IGNORED) + die(_("Complete Untracked only supported with ignored files")); + parse_pathspec(&s.pathspec, 0, PATHSPEC_PREFER_FULL, prefix, argv); + /* + * If we want to try to deserialize status data from a cache file, + * we need to re-order the initialization code. The problem is that + * this makes for a very nasty diff and causes merge conflicts as we + * carry it forward. And it easy to mess up the merge, so we + * duplicate some code here to hopefully reduce conflicts. + */ + try_deserialize = (!do_serialize && + (do_implicit_deserialize || do_explicit_deserialize)); + if (try_deserialize) + goto skip_init; + enable_fscache(0); if (status_format != STATUS_FORMAT_PORCELAIN && status_format != STATUS_FORMAT_PORCELAIN_V2) @@ -1401,6 +1503,7 @@ int cmd_status(int argc, const char **argv, const char *prefix) else fd = -1; +skip_init: s.is_initial = get_oid(s.reference, &oid) ? 1 : 0; if (!s.is_initial) hashcpy(s.sha1_commit, oid.hash); @@ -1417,6 +1520,24 @@ int cmd_status(int argc, const char **argv, const char *prefix) s.rename_score = parse_rename_score(&rename_score_arg); } + if (try_deserialize) { + if (s.relative_paths) + s.prefix = prefix; + + if (wt_status_deserialize(&s, deserialize_path) == DESERIALIZE_OK) + return 0; + + /* deserialize failed, so force the initialization we skipped above. */ + enable_fscache(1); + read_cache_preload(&s.pathspec); + refresh_index(&the_index, REFRESH_QUIET|REFRESH_UNMERGED, &s.pathspec, NULL, NULL); + + if (use_optional_locks()) + fd = hold_locked_index(&index_lock, 0); + else + fd = -1; + } + wt_status_collect(&s); if (0 <= fd) diff --git a/pkt-line.c b/pkt-line.c index d4b71d3e82b0e5..e977aec307c9e8 100644 --- a/pkt-line.c +++ b/pkt-line.c @@ -185,7 +185,7 @@ int packet_write_fmt_gently(int fd, const char *fmt, ...) return status; } -static int packet_write_gently(const int fd_out, const char *buf, size_t size) +int packet_write_gently(const int fd_out, const char *buf, size_t size) { static char packet_write_buffer[LARGE_PACKET_MAX]; size_t packet_size; diff --git a/pkt-line.h b/pkt-line.h index ad9a4a2cd7c332..aaf6442a01ddb7 100644 --- a/pkt-line.h +++ b/pkt-line.h @@ -30,6 +30,7 @@ void packet_buf_write(struct strbuf *buf, const char *fmt, ...) __attribute__((f void packet_buf_write_len(struct strbuf *buf, const char *data, size_t len); int packet_flush_gently(int fd); int packet_write_fmt_gently(int fd, const char *fmt, ...) __attribute__((format (printf, 2, 3))); +int packet_write_gently(const int fd_out, const char *buf, size_t size); int write_packetized_from_fd(int fd_in, int fd_out); int write_packetized_from_buf(const char *src_in, size_t len, int fd_out); diff --git a/t/t7523-status-complete-untracked.sh b/t/t7523-status-complete-untracked.sh new file mode 100755 index 00000000000000..2f4476654a5570 --- /dev/null +++ b/t/t7523-status-complete-untracked.sh @@ -0,0 +1,39 @@ +#!/bin/sh + +test_description='git status untracked complete tests' + +. ./test-lib.sh + +test_expect_success 'setup' ' + cat >.gitignore <<-\EOF && + *.ign + ignored_dir/ + EOF + + mkdir tracked ignored_dir && + touch tracked_1.txt tracked/tracked_1.txt && + git add . && + test_tick && + git commit -m"Adding original file." && + mkdir untracked && + touch ignored.ign ignored_dir/ignored_2.txt \ + untracked_1.txt untracked/untracked_2.txt untracked/untracked_3.txt +' + +test_expect_success 'verify untracked-files=complete' ' + cat >expect <<-\EOF && + ? expect + ? output + ? untracked/ + ? untracked/untracked_2.txt + ? untracked/untracked_3.txt + ? untracked_1.txt + ! ignored.ign + ! ignored_dir/ + EOF + + git status --porcelain=v2 --untracked-files=complete --ignored >output && + test_i18ncmp expect output +' + +test_done diff --git a/t/t7524-serialized-status.sh b/t/t7524-serialized-status.sh new file mode 100755 index 00000000000000..a0fc1bc2daf6ec --- /dev/null +++ b/t/t7524-serialized-status.sh @@ -0,0 +1,140 @@ +#!/bin/sh + +test_description='git serialized status tests' + +. ./test-lib.sh + +# This file includes tests for serializing / deserializing +# status data. These tests cover two basic features: +# +# [1] Because users can request different types of untracked-file +# and ignored file reporting, the cache data generated by +# serialize must use either the same untracked and ignored +# parameters as the later deserialize invocation; otherwise, +# the deserialize invocation must disregard the cached data +# and run a full scan itself. +# +# To increase the number of cases where the cached status can +# be used, we have added a "--untracked-file=complete" option +# that reports a superset or union of the results from the +# "-u normal" and "-u all". We combine this with a filter in +# deserialize to filter the results. +# +# Ignored file reporting is simpler in that is an all or +# nothing; there are no subsets. +# +# The tests here (in addition to confirming that a cache +# file can be generated and used by a subsequent status +# command) need to test this untracked-file filtering. +# +# [2] ensuring the status calls are using data from the status +# cache as expected. This includes verifying cached data +# is used when appropriate as well as falling back to +# performing a new status scan when the data in the cache +# is insufficient/known stale. + +test_expect_success 'setup' ' + cat >.gitignore <<-\EOF && + *.ign + ignored_dir/ + EOF + + mkdir tracked ignored_dir && + touch tracked_1.txt tracked/tracked_1.txt && + git add . && + test_tick && + git commit -m"Adding original file." && + mkdir untracked && + touch ignored.ign ignored_dir/ignored_2.txt \ + untracked_1.txt untracked/untracked_2.txt untracked/untracked_3.txt +' + +test_expect_success 'verify untracked-files=complete with no conversion' ' + test_when_finished "rm serialized_status.dat new_change.txt output" && + cat >expect <<-\EOF && + ? expect + ? serialized_status.dat + ? untracked/ + ? untracked/untracked_2.txt + ? untracked/untracked_3.txt + ? untracked_1.txt + ! ignored.ign + ! ignored_dir/ + EOF + + git status --untracked-files=complete --ignored=matching --serialize >serialized_status.dat && + touch new_change.txt && + + git status --porcelain=v2 --untracked-files=complete --ignored=matching --deserialize=serialized_status.dat >output && + test_i18ncmp expect output +' + +test_expect_success 'verify untracked-files=complete to untracked-files=normal conversion' ' + test_when_finished "rm serialized_status.dat new_change.txt output" && + cat >expect <<-\EOF && + ? expect + ? serialized_status.dat + ? untracked/ + ? untracked_1.txt + EOF + + git status --untracked-files=complete --ignored=matching --serialize >serialized_status.dat && + touch new_change.txt && + + git status --porcelain=v2 --deserialize=serialized_status.dat >output && + test_i18ncmp expect output +' + +test_expect_success 'verify untracked-files=complete to untracked-files=all conversion' ' + test_when_finished "rm serialized_status.dat new_change.txt output" && + cat >expect <<-\EOF && + ? expect + ? serialized_status.dat + ? untracked/untracked_2.txt + ? untracked/untracked_3.txt + ? untracked_1.txt + ! ignored.ign + ! ignored_dir/ + EOF + + git status --untracked-files=complete --ignored=matching --serialize >serialized_status.dat && + touch new_change.txt && + + git status --porcelain=v2 --untracked-files=all --ignored=matching --deserialize=serialized_status.dat >output && + test_i18ncmp expect output +' + +test_expect_success 'verify serialized status with non-convertible ignore mode does new scan' ' + test_when_finished "rm serialized_status.dat new_change.txt output" && + cat >expect <<-\EOF && + ? expect + ? new_change.txt + ? output + ? serialized_status.dat + ? untracked/ + ? untracked_1.txt + ! ignored.ign + ! ignored_dir/ + EOF + + git status --untracked-files=complete --ignored=matching --serialize >serialized_status.dat && + touch new_change.txt && + + git status --porcelain=v2 --ignored --deserialize=serialized_status.dat >output && + test_i18ncmp expect output +' + +test_expect_success 'verify serialized status handles path scopes' ' + test_when_finished "rm serialized_status.dat new_change.txt output" && + cat >expect <<-\EOF && + ? untracked/ + EOF + + git status --untracked-files=complete --ignored=matching --serialize >serialized_status.dat && + touch new_change.txt && + + git status --porcelain=v2 --deserialize=serialized_status.dat untracked >output && + test_i18ncmp expect output +' + +test_done diff --git a/wt-status-deserialize.c b/wt-status-deserialize.c new file mode 100644 index 00000000000000..82eb8d8fca591b --- /dev/null +++ b/wt-status-deserialize.c @@ -0,0 +1,597 @@ +#include "cache.h" +#include "wt-status.h" +#include "pkt-line.h" +#include "trace.h" + +static struct trace_key trace_deserialize = TRACE_KEY_INIT(DESERIALIZE); + +enum deserialize_parse_strategy { + DESERIALIZE_STRATEGY_AS_IS, + DESERIALIZE_STRATEGY_SKIP, + DESERIALIZE_STRATEGY_NORMAL, + DESERIALIZE_STRATEGY_ALL +}; + +static int check_path_contains(const char *out, int out_len, const char *in, int in_len) +{ + return (out_len > 0 && + out_len < in_len && + (out[out_len - 1] == '/') && + !memcmp(out, in, out_len)); +} + +static const char *my_packet_read_line(int fd, int *line_len) +{ + static char buf[LARGE_PACKET_MAX]; + + *line_len = packet_read(fd, NULL, NULL, buf, sizeof(buf), + PACKET_READ_CHOMP_NEWLINE | + PACKET_READ_GENTLE_ON_EOF); + return (*line_len > 0) ? buf : NULL; +} + +/* + * mtime_reported contains the mtime of the index when the + * serialization snapshot was computed. + * + * mtime_observed_on_disk contains the mtime of the index now. + * + * If these 2 times are different, then the .git/index has + * changed since the serialization cache was created and we + * must reject the cache because anything could have changed. + * + * If they are the same, we continue trying to use the cache. + */ +static int my_validate_index(const struct cache_time *mtime_reported) +{ + const char *path = get_index_file(); + struct stat st; + struct cache_time mtime_observed_on_disk; + + if (lstat(path, &st)) { + trace_printf_key(&trace_deserialize, "could not stat index"); + return DESERIALIZE_ERR; + } + mtime_observed_on_disk.sec = st.st_mtime; + mtime_observed_on_disk.nsec = ST_MTIME_NSEC(st); + if ((mtime_observed_on_disk.sec != mtime_reported->sec) || + (mtime_observed_on_disk.nsec != mtime_reported->nsec)) { + trace_printf_key(&trace_deserialize, "index mtime changed [des %d.%d][obs %d.%d]", + mtime_reported->sec, mtime_reported->nsec, + mtime_observed_on_disk.sec, mtime_observed_on_disk.nsec); + return DESERIALIZE_ERR; + } + + return DESERIALIZE_OK; +} + +static int wt_deserialize_v1_header(struct wt_status *s, int fd) +{ + struct cache_time index_mtime; + int line_len, nr_fields; + const char *line; + const char *arg; + + /* + * parse header lines up to the first flush packet. + */ + while ((line = my_packet_read_line(fd, &line_len))) { + + if (skip_prefix(line, "index_mtime ", &arg)) { + nr_fields = sscanf(arg, "%d %d", + &index_mtime.sec, + &index_mtime.nsec); + if (nr_fields != 2) { + trace_printf_key(&trace_deserialize, "invalid index_mtime (%d) '%s'", + nr_fields, line); + return DESERIALIZE_ERR; + } + continue; + } + + if (skip_prefix(line, "is_initial ", &arg)) { + s->is_initial = (int)strtol(arg, NULL, 10); + continue; + } + if (skip_prefix(line, "branch ", &arg)) { + s->branch = xstrdup(arg); + continue; + } + if (skip_prefix(line, "reference ", &arg)) { + s->reference = xstrdup(arg); + continue; + } + /* pathspec */ + /* verbose */ + /* amend */ + if (skip_prefix(line, "whence ", &arg)) { + s->whence = (int)strtol(arg, NULL, 10); + continue; + } + /* nowarn */ + /* use_color */ + /* no_gettext */ + /* display_comment_prefix */ + /* relative_paths */ + /* submodule_summary */ + if (skip_prefix(line, "show_ignored_mode ", &arg)) { + s->show_ignored_mode = (int)strtol(arg, NULL, 10); + continue; + } + if (skip_prefix(line, "show_untracked_files ", &arg)) { + s->show_untracked_files = (int)strtol(arg, NULL, 10); + continue; + } + if (skip_prefix(line, "ignore_submodule_arg ", &arg)) { + s->ignore_submodule_arg = xstrdup(arg); + continue; + } + /* color_palette */ + /* colopts */ + /* null_termination */ + /* commit_template */ + /* show_branch */ + /* show_stash */ + if (skip_prefix(line, "hints ", &arg)) { + s->hints = (int)strtol(arg, NULL, 10); + continue; + } + if (skip_prefix(line, "detect_rename ", &arg)) { + s->detect_rename = (int)strtol(arg, NULL, 10); + continue; + } + if (skip_prefix(line, "rename_score ", &arg)) { + s->rename_score = (int)strtol(arg, NULL, 10); + continue; + } + if (skip_prefix(line, "rename_limit ", &arg)) { + s->rename_limit = (int)strtol(arg, NULL, 10); + continue; + } + /* status_format */ + if (skip_prefix(line, "sha1_commit ", &arg)) { + if (get_sha1_hex(arg, s->sha1_commit)) { + trace_printf_key(&trace_deserialize, "invalid sha1_commit"); + return DESERIALIZE_ERR; + } + continue; + } + if (skip_prefix(line, "committable ", &arg)) { + s->committable = (int)strtol(arg, NULL, 10); + continue; + } + if (skip_prefix(line, "workdir_dirty ", &arg)) { + s->workdir_dirty = (int)strtol(arg, NULL, 10); + continue; + } + /* prefix */ + + trace_printf_key(&trace_deserialize, "unexpected line '%s'", line); + return DESERIALIZE_ERR; + } + + return my_validate_index(&index_mtime); +} + +/* + * Build a string-list of (count) lines from the input. + */ +static int wt_deserialize_v1_changed_items(struct wt_status *s, int fd, int count) +{ + struct wt_status_serialize_data *sd; + char *p; + int line_len; + const char *line; + struct string_list_item *item; + + string_list_init(&s->change, 1); + + /* + * + + * + * + * NUL [] NUL + */ + while ((line = my_packet_read_line(fd, &line_len))) { + struct wt_status_change_data *d = xcalloc(1, sizeof(*d)); + sd = (struct wt_status_serialize_data *)line; + + d->worktree_status = ntohl(sd->fixed.worktree_status); + d->index_status = ntohl(sd->fixed.index_status); + d->stagemask = ntohl(sd->fixed.stagemask); + d->rename_score = ntohl(sd->fixed.rename_score); + d->mode_head = ntohl(sd->fixed.mode_head); + d->mode_index = ntohl(sd->fixed.mode_index); + d->mode_worktree = ntohl(sd->fixed.mode_worktree); + d->dirty_submodule = ntohl(sd->fixed.dirty_submodule); + d->new_submodule_commits = ntohl(sd->fixed.new_submodule_commits); + oidcpy(&d->oid_head, &sd->fixed.oid_head); + oidcpy(&d->oid_index, &sd->fixed.oid_index); + + p = sd->variant; + item = string_list_append(&s->change, p); + p += strlen(p) + 1; + if (*p) + d->rename_source = xstrdup(p); + item->util = d; + + trace_printf_key( + &trace_deserialize, + "change: %d %d %d %d %o %o %o %d %d %s %s '%s' '%s'", + d->worktree_status, + d->index_status, + d->stagemask, + d->rename_score, + d->mode_head, + d->mode_index, + d->mode_worktree, + d->dirty_submodule, + d->new_submodule_commits, + oid_to_hex(&d->oid_head), + oid_to_hex(&d->oid_index), + item->string, + (d->rename_source ? d->rename_source : "")); + } + + return DESERIALIZE_OK; +} + +static int wt_deserialize_v1_untracked_items(struct wt_status *s, + int fd, + int count, + enum deserialize_parse_strategy strategy) +{ + int line_len; + const char *line; + char *out = NULL; + int out_len = 0; + + string_list_init(&s->untracked, 1); + + /* + * + + * + */ + while ((line = my_packet_read_line(fd, &line_len))) { + if (strategy == DESERIALIZE_STRATEGY_AS_IS) + string_list_append(&s->untracked, line); + if (strategy == DESERIALIZE_STRATEGY_SKIP) + continue; + if (strategy == DESERIALIZE_STRATEGY_NORMAL) { + + /* Only add "normal" entries to list */ + if (out && + check_path_contains(out, out_len, line, line_len)) { + continue; + } + else { + out = string_list_append(&s->untracked, line)->string; + out_len = line_len; + } + } + if (strategy == DESERIALIZE_STRATEGY_ALL) { + /* Only add "all" entries to list */ + if (line[line_len - 1] != '/') + string_list_append(&s->untracked, line); + } + } + + return DESERIALIZE_OK; +} + +static int wt_deserialize_v1_ignored_items(struct wt_status *s, + int fd, + int count, + enum deserialize_parse_strategy strategy) +{ + int line_len; + const char *line; + + string_list_init(&s->ignored, 1); + + /* + * + + * + */ + while ((line = my_packet_read_line(fd, &line_len))) { + if (strategy == DESERIALIZE_STRATEGY_AS_IS) + string_list_append(&s->ignored, line); + else + continue; + } + + return DESERIALIZE_OK; +} + +static int validate_untracked_files_arg(enum untracked_status_type cmd, + enum untracked_status_type des, + enum deserialize_parse_strategy *strategy) +{ + *strategy = DESERIALIZE_STRATEGY_AS_IS; + + if (cmd == des) { + *strategy = DESERIALIZE_STRATEGY_AS_IS; + } else if (cmd == SHOW_NO_UNTRACKED_FILES) { + *strategy = DESERIALIZE_STRATEGY_SKIP; + } else if (des == SHOW_COMPLETE_UNTRACKED_FILES) { + if (cmd == SHOW_ALL_UNTRACKED_FILES) + *strategy = DESERIALIZE_STRATEGY_ALL; + else if (cmd == SHOW_NORMAL_UNTRACKED_FILES) + *strategy = DESERIALIZE_STRATEGY_NORMAL; + } else { + return DESERIALIZE_ERR; + } + + return DESERIALIZE_OK; +} + +static int validate_ignored_files_arg(enum show_ignored_type cmd, + enum show_ignored_type des, + enum deserialize_parse_strategy *strategy) +{ + *strategy = DESERIALIZE_STRATEGY_AS_IS; + + if (cmd == SHOW_NO_IGNORED) { + *strategy = DESERIALIZE_STRATEGY_SKIP; + } + else if (cmd != des) { + return DESERIALIZE_ERR; + } + + return DESERIALIZE_OK; +} + +static int wt_deserialize_v1(const struct wt_status *cmd_s, struct wt_status *s, int fd) +{ + int line_len; + const char *line; + const char *arg; + int nr_changed = 0; + int nr_untracked = 0; + int nr_ignored = 0; + + enum deserialize_parse_strategy ignored_strategy = DESERIALIZE_STRATEGY_AS_IS, untracked_strategy = DESERIALIZE_STRATEGY_AS_IS; + + if (wt_deserialize_v1_header(s, fd) == DESERIALIZE_ERR) + return DESERIALIZE_ERR; + + /* + * We now have the header parsed. Look at the command args (as passed in), and see how to parse + * the serialized data + */ + if (validate_untracked_files_arg(cmd_s->show_untracked_files, s->show_untracked_files, &untracked_strategy)) { + trace_printf_key(&trace_deserialize, "reject: show_untracked_file: command: %d, serialized : %d", + cmd_s->show_untracked_files, + s->show_untracked_files); + return DESERIALIZE_ERR; + } + + if (validate_ignored_files_arg(cmd_s->show_ignored_mode, s->show_ignored_mode, &ignored_strategy)) { + trace_printf_key(&trace_deserialize, "reject: show_ignored_mode: command: %d, serialized: %d", + cmd_s->show_ignored_mode, + s->show_ignored_mode); + return DESERIALIZE_ERR; + } + + /* + * [ [+] ] + * [ [+] ] + * [ [+] ] + */ + while ((line = my_packet_read_line(fd, &line_len))) { + if (skip_prefix(line, "changed ", &arg)) { + nr_changed = (int)strtol(arg, NULL, 10); + if (wt_deserialize_v1_changed_items(s, fd, nr_changed) + == DESERIALIZE_ERR) + return DESERIALIZE_ERR; + continue; + } + if (skip_prefix(line, "untracked ", &arg)) { + nr_untracked = (int)strtol(arg, NULL, 10); + if (wt_deserialize_v1_untracked_items(s, fd, nr_untracked, untracked_strategy) + == DESERIALIZE_ERR) + return DESERIALIZE_ERR; + continue; + } + if (skip_prefix(line, "ignored ", &arg)) { + nr_ignored = (int)strtol(arg, NULL, 10); + if (wt_deserialize_v1_ignored_items(s, fd, nr_ignored, ignored_strategy) + == DESERIALIZE_ERR) + return DESERIALIZE_ERR; + continue; + } + trace_printf_key(&trace_deserialize, "unexpected line '%s'", line); + return DESERIALIZE_ERR; + } + + return DESERIALIZE_OK; +} + +static int wt_deserialize_parse(const struct wt_status *cmd_s, struct wt_status *s, int fd) +{ + int line_len; + const char *line; + const char *arg; + + memset(s, 0, sizeof(*s)); + + if ((line = my_packet_read_line(fd, &line_len)) && + (skip_prefix(line, "version ", &arg))) { + int version = (int)strtol(arg, NULL, 10); + if (version == 1) + return wt_deserialize_v1(cmd_s, s, fd); + } + trace_printf_key(&trace_deserialize, "missing/unsupported version"); + return DESERIALIZE_ERR; +} + +static inline int my_strcmp_null(const char *a, const char *b) +{ + const char *alt_a = (a) ? a : ""; + const char *alt_b = (b) ? b : ""; + + return strcmp(alt_a, alt_b); +} + +static int wt_deserialize_fd(const struct wt_status *cmd_s, struct wt_status *des_s, int fd) +{ + /* + * Check the path spec on the current command + */ + if (cmd_s->pathspec.nr > 1) { + trace_printf_key(&trace_deserialize, "reject: multiple pathspecs"); + return DESERIALIZE_ERR; + } + + /* + * If we have a pathspec, but it maches the root (e.g. no filtering) + * then this is OK. + */ + if (cmd_s->pathspec.nr == 1 && + my_strcmp_null(cmd_s->pathspec.items[0].match, "")) { + trace_printf_key(&trace_deserialize, "reject: pathspec"); + return DESERIALIZE_ERR; + } + + /* + * Deserialize cached status + */ + if (wt_deserialize_parse(cmd_s, des_s, fd) == DESERIALIZE_ERR) + return DESERIALIZE_ERR; + + /* + * Compare fields in cmd_s with those observed in des_s and + * complain if they are incompatible (such as different "-u" + * or "--ignored" settings). + */ + if (cmd_s->is_initial != des_s->is_initial) { + trace_printf_key(&trace_deserialize, "reject: is_initial"); + return DESERIALIZE_ERR; + } + if (my_strcmp_null(cmd_s->branch, des_s->branch)) { + trace_printf_key(&trace_deserialize, "reject: branch"); + return DESERIALIZE_ERR; + } + if (my_strcmp_null(cmd_s->reference, des_s->reference)) { + trace_printf_key(&trace_deserialize, "reject: reference"); + return DESERIALIZE_ERR; + } + /* verbose */ + /* amend */ + if (cmd_s->whence != des_s->whence) { + trace_printf_key(&trace_deserialize, "reject: whence"); + return DESERIALIZE_ERR; + } + /* nowarn */ + /* use_color */ + /* no_gettext */ + /* display_comment_prefix */ + /* relative_paths */ + /* submodule_summary */ + + /* show_ignored_files - already validated */ + /* show_untrackes_files - already validated */ + + /* + * Submodules are not supported by status serialization. + * The status will not be serialized if it contains submodules, + * and so this check is not needed. + * + * if (my_strcmp_null(cmd_s->ignore_submodule_arg, des_s->ignore_submodule_arg)) { + * trace_printf_key(&trace_deserialize, "reject: ignore_submodule_arg"); + * return DESERIALIZE_ERR; + * } + */ + + /* color_palette */ + /* colopts */ + /* null_termination */ + /* commit_template */ + /* show_branch */ + /* show_stash */ + /* hints */ + if (cmd_s->detect_rename != des_s->detect_rename) { + trace_printf_key(&trace_deserialize, "reject: detect_rename"); + return DESERIALIZE_ERR; + } + if (cmd_s->rename_score != des_s->rename_score) { + trace_printf_key(&trace_deserialize, "reject: rename_score"); + return DESERIALIZE_ERR; + } + if (cmd_s->rename_limit != des_s->rename_limit) { + trace_printf_key(&trace_deserialize, "reject: rename_limit"); + return DESERIALIZE_ERR; + } + /* status_format */ + if (!hasheq(cmd_s->sha1_commit, des_s->sha1_commit)) { + trace_printf_key(&trace_deserialize, "reject: sha1_commit"); + return DESERIALIZE_ERR; + } + + /* + * Copy over display-related fields from the current command. + */ + des_s->verbose = cmd_s->verbose; + /* amend */ + /* whence */ + des_s->nowarn = cmd_s->nowarn; + des_s->use_color = cmd_s->use_color; + des_s->no_gettext = cmd_s->no_gettext; + des_s->display_comment_prefix = cmd_s->display_comment_prefix; + des_s->relative_paths = cmd_s->relative_paths; + des_s->submodule_summary = cmd_s->submodule_summary; + memcpy(des_s->color_palette, cmd_s->color_palette, + sizeof(char)*WT_STATUS_MAXSLOT*COLOR_MAXLEN); + des_s->colopts = cmd_s->colopts; + des_s->null_termination = cmd_s->null_termination; + /* commit_template */ + des_s->show_branch = cmd_s->show_branch; + des_s->show_stash = cmd_s->show_stash; + /* hints */ + des_s->status_format = cmd_s->status_format; + des_s->fp = cmd_s->fp; + if (cmd_s->prefix && *cmd_s->prefix) + des_s->prefix = xstrdup(cmd_s->prefix); + + return DESERIALIZE_OK; +} + + +/* + * Read raw serialized status data from the given file + * + * Verify that the args specified in the current command + * are compatible with the deserialized data (such as "-uno"). + * + * Copy display-related fields from the current command + * into the deserialized data (so that the user can request + * long or short as they please). + */ +int wt_status_deserialize(const struct wt_status *cmd_s, + const char *path) +{ + struct wt_status des_s; + int result; + + if (path && *path && strcmp(path, "0")) { + int fd = xopen(path, O_RDONLY); + if (fd == -1) { + trace_printf_key(&trace_deserialize, "could not read '%s'", path); + return DESERIALIZE_ERR; + } + trace_printf_key(&trace_deserialize, "reading serialization file '%s'", path); + result = wt_deserialize_fd(cmd_s, &des_s, fd); + close(fd); + } else { + trace_printf_key(&trace_deserialize, "reading stdin"); + result = wt_deserialize_fd(cmd_s, &des_s, 0); + } + + if (result == DESERIALIZE_OK) { + wt_status_get_state(cmd_s->repo, &des_s.state, des_s.branch && + !strcmp(des_s.branch, "HEAD")); + wt_status_print(&des_s); + } + + return result; +} diff --git a/wt-status-serialize.c b/wt-status-serialize.c new file mode 100644 index 00000000000000..c393c7292aea7a --- /dev/null +++ b/wt-status-serialize.c @@ -0,0 +1,213 @@ +#include "cache.h" +#include "wt-status.h" +#include "pkt-line.h" + +static struct trace_key trace_serialize = TRACE_KEY_INIT(SERIALIZE); + +/* + * Write V1 header fields. + */ +static void wt_serialize_v1_header(struct wt_status *s, int fd) +{ + /* + * Write select fields from the current index to help + * the deserializer recognize a stale data set. + */ + packet_write_fmt(fd, "index_mtime %d %d\n", + s->repo->index->timestamp.sec, + s->repo->index->timestamp.nsec); + + /* + * Write data from wt_status to qualify this status report. + * That is, if this run specified "-uno", the consumer of + * our serialization should know that. + */ + packet_write_fmt(fd, "is_initial %d\n", s->is_initial); + if (s->branch) + packet_write_fmt(fd, "branch %s\n", s->branch); + if (s->reference) + packet_write_fmt(fd, "reference %s\n", s->reference); + /* pathspec */ + /* verbose */ + /* amend */ + packet_write_fmt(fd, "whence %d\n", s->whence); + /* nowarn */ + /* use_color */ + /* no_gettext */ + /* display_comment_prefix */ + /* relative_paths */ + /* submodule_summary */ + packet_write_fmt(fd, "show_ignored_mode %d\n", s->show_ignored_mode); + packet_write_fmt(fd, "show_untracked_files %d\n", s->show_untracked_files); + if (s->ignore_submodule_arg) + packet_write_fmt(fd, "ignore_submodule_arg %s\n", s->ignore_submodule_arg); + /* color_palette */ + /* colopts */ + /* null_termination */ + /* commit_template */ + /* show_branch */ + /* show_stash */ + packet_write_fmt(fd, "hints %d\n", s->hints); + packet_write_fmt(fd, "detect_rename %d\n", s->detect_rename); + packet_write_fmt(fd, "rename_score %d\n", s->rename_score); + packet_write_fmt(fd, "rename_limit %d\n", s->rename_limit); + /* status_format */ + packet_write_fmt(fd, "sha1_commit %s\n", sha1_to_hex(s->sha1_commit)); + packet_write_fmt(fd, "committable %d\n", s->committable); + packet_write_fmt(fd, "workdir_dirty %d\n", s->workdir_dirty); + /* prefix */ + packet_flush(fd); +} + +/* + * Print changed/unmerged items. + * We write raw (not c-quoted) pathname(s). The rename_source is only + * set when status computed a rename/copy. + * + * We ALWAYS write a final LF to the packet-line (for debugging) + * even though Linux pathnames allow LFs. + */ +static inline void wt_serialize_v1_changed(struct wt_status *s, int fd, + struct string_list_item *item) +{ + struct wt_status_change_data *d = item->util; + struct wt_status_serialize_data sd; + char *begin; + char *end; + char *p; + int len_path, len_rename_source; + + trace_printf_key(&trace_serialize, + "change: %d %d %d %d %o %o %o %d %d %s %s '%s' '%s'", + d->worktree_status, + d->index_status, + d->stagemask, + d->rename_score, + d->mode_head, + d->mode_index, + d->mode_worktree, + d->dirty_submodule, + d->new_submodule_commits, + oid_to_hex(&d->oid_head), + oid_to_hex(&d->oid_index), + item->string, + (d->rename_source ? d->rename_source : "")); + + sd.fixed.worktree_status = htonl(d->worktree_status); + sd.fixed.index_status = htonl(d->index_status); + sd.fixed.stagemask = htonl(d->stagemask); + sd.fixed.rename_score = htonl(d->rename_score); + sd.fixed.mode_head = htonl(d->mode_head); + sd.fixed.mode_index = htonl(d->mode_index); + sd.fixed.mode_worktree = htonl(d->mode_worktree); + sd.fixed.dirty_submodule = htonl(d->dirty_submodule); + sd.fixed.new_submodule_commits = htonl(d->new_submodule_commits); + oidcpy(&sd.fixed.oid_head, &d->oid_head); + oidcpy(&sd.fixed.oid_index, &d->oid_index); + + begin = (char *)&sd; + end = begin + sizeof(sd); + + p = sd.variant; + + /* + * Write NUL [] NUL LF at the end of the buffer. + */ + len_path = strlen(item->string); + len_rename_source = d->rename_source ? strlen(d->rename_source) : 0; + + /* + * This is a bit of a hack, but I don't want to split the + * status detail record across multiple pkt-lines. + */ + if (p + len_path + 1 + len_rename_source + 1 + 1 >= end) + BUG("path to long to serialize '%s'", item->string); + + memcpy(p, item->string, len_path); + p += len_path; + *p++ = '\0'; + + if (len_rename_source) { + memcpy(p, d->rename_source, len_rename_source); + p += len_rename_source; + } + *p++ = '\0'; + *p++ = '\n'; + + if (packet_write_gently(fd, begin, (p - begin))) + BUG("cannot serialize '%s'", item->string); +} + +/* + * Write raw (not c-quoted) pathname for an untracked item. + * We ALWAYS write a final LF to the packet-line (for debugging) + * even though Linux pathnames allows LFs. That is, deserialization + * should use the packet-line length and omit the final LF. + */ +static inline void wt_serialize_v1_untracked(struct wt_status *s, int fd, + struct string_list_item *item) +{ + packet_write_fmt(fd, "%s\n", item->string); +} + +/* + * Write raw (not c-quoted) pathname for an ignored item. + * We ALWAYS write a final LF to the packet-line (for debugging) + * even though Linux pathnames allows LFs. + */ +static inline void wt_serialize_v1_ignored(struct wt_status *s, int fd, + struct string_list_item *item) +{ + packet_write_fmt(fd, "%s\n", item->string); +} + +/* + * Serialize the list of changes to stdout. The goal of this + * is to just serialize the key fields in wt_status so that a + * later command can rebuilt it and do the printing. + * + * We DO NOT include the contents of wt_status_state NOR + * current branch info. This info easily gets stale and + * is relatively quick for the status consumer to compute + * as necessary. + */ +void wt_status_serialize_v1(struct wt_status *s) +{ + int fd = 1; /* we always write to stdout */ + struct string_list_item *iter; + int k; + + /* + * version header must be first line. + */ + packet_write_fmt(fd, "version 1\n"); + + wt_serialize_v1_header(s, fd); + + if (s->change.nr > 0) { + packet_write_fmt(fd, "changed %d\n", s->change.nr); + for (k = 0; k < s->change.nr; k++) { + iter = &(s->change.items[k]); + wt_serialize_v1_changed(s, fd, iter); + } + packet_flush(fd); + } + + if (s->untracked.nr > 0) { + packet_write_fmt(fd, "untracked %d\n", s->untracked.nr); + for (k = 0; k < s->untracked.nr; k++) { + iter = &(s->untracked.items[k]); + wt_serialize_v1_untracked(s, fd, iter); + } + packet_flush(fd); + } + + if (s->ignored.nr > 0) { + packet_write_fmt(fd, "ignored %d\n", s->ignored.nr); + for (k = 0; k < s->ignored.nr; k++) { + iter = &(s->ignored.items[k]); + wt_serialize_v1_ignored(s, fd, iter); + } + packet_flush(fd); + } +} diff --git a/wt-status.c b/wt-status.c index 1f564b12d25995..639509bb6fdc43 100644 --- a/wt-status.c +++ b/wt-status.c @@ -696,6 +696,9 @@ static void wt_status_collect_untracked(struct wt_status *s) if (s->show_untracked_files != SHOW_ALL_UNTRACKED_FILES) dir.flags |= DIR_SHOW_OTHER_DIRECTORIES | DIR_HIDE_EMPTY_DIRECTORIES; + if (s->show_untracked_files == SHOW_COMPLETE_UNTRACKED_FILES) + dir.flags |= DIR_KEEP_UNTRACKED_CONTENTS; + if (s->show_ignored_mode) { dir.flags |= DIR_SHOW_IGNORED_TOO; @@ -2308,6 +2311,9 @@ void wt_status_print(struct wt_status *s) case STATUS_FORMAT_LONG: wt_longstatus_print(s); break; + case STATUS_FORMAT_SERIALIZE_V1: + wt_status_serialize_v1(s); + break; } } diff --git a/wt-status.h b/wt-status.h index 3a95975032bb1e..c473a6785e4bf4 100644 --- a/wt-status.h +++ b/wt-status.h @@ -5,6 +5,7 @@ #include "string-list.h" #include "color.h" #include "pathspec.h" +#include "pkt-line.h" #include "remote.h" struct repository; @@ -26,7 +27,8 @@ enum color_wt_status { enum untracked_status_type { SHOW_NO_UNTRACKED_FILES, SHOW_NORMAL_UNTRACKED_FILES, - SHOW_ALL_UNTRACKED_FILES + SHOW_ALL_UNTRACKED_FILES, + SHOW_COMPLETE_UNTRACKED_FILES, }; enum show_ignored_type { @@ -61,6 +63,7 @@ enum wt_status_format { STATUS_FORMAT_SHORT, STATUS_FORMAT_PORCELAIN, STATUS_FORMAT_PORCELAIN_V2, + STATUS_FORMAT_SERIALIZE_V1, STATUS_FORMAT_UNSPECIFIED }; @@ -158,4 +161,51 @@ int require_clean_work_tree(struct repository *repo, int ignore_submodules, int gently); +#define DESERIALIZE_OK 0 +#define DESERIALIZE_ERR 1 + +struct wt_status_serialize_data_fixed +{ + uint32_t worktree_status; + uint32_t index_status; + uint32_t stagemask; + uint32_t rename_score; + uint32_t mode_head; + uint32_t mode_index; + uint32_t mode_worktree; + uint32_t dirty_submodule; + uint32_t new_submodule_commits; + struct object_id oid_head; + struct object_id oid_index; +}; + +/* + * Consume the maximum amount of data possible in a + * packet-line record. This is overkill because we + * have at most 2 relative pathnames, but means we + * don't need to allocate a variable length structure. + */ +struct wt_status_serialize_data +{ + struct wt_status_serialize_data_fixed fixed; + char variant[LARGE_PACKET_DATA_MAX + - sizeof(struct wt_status_serialize_data_fixed)]; +}; + +/* + * Serialize computed status scan results using "version 1" format + * to the given file. + */ +void wt_status_serialize_v1(struct wt_status *s); + +/* + * Deserialize existing status results from the given file and + * populate a (new) "struct wt_status". Use the contents of "cmd_s" + * (computed from the command line arguments) to verify that the + * cached data is compatible and overlay various display-related + * fields. + */ +int wt_status_deserialize(const struct wt_status *cmd_s, + const char *path); + #endif /* STATUS_H */ From 1ccd8b277dac388ebd0e0af61cb608c05fe0a3d1 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Mon, 3 Jul 2017 13:39:45 -0600 Subject: [PATCH 040/118] cache-tree: remove use of strbuf_addf in update_one String formatting can be a performance issue when there are hundreds of thousands of trees. Change to stop using the strbuf_addf and just add the strings or characters individually. There are a limited number of modes so added a switch for the known ones and a default case if something comes through that are not a known one for git. In one scenario regarding a huge worktree, this reduces the time required for a `git checkout ` from 44 seconds to 38 seconds, i.e. it is a non-negligible performance improvement. Signed-off-by: Kevin Willford --- cache-tree.c | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/cache-tree.c b/cache-tree.c index e580bb649eb899..67c0a1c7f8f519 100644 --- a/cache-tree.c +++ b/cache-tree.c @@ -397,7 +397,29 @@ static int update_one(struct cache_tree *it, continue; strbuf_grow(&buffer, entlen + 100); - strbuf_addf(&buffer, "%o %.*s%c", mode, entlen, path + baselen, '\0'); + + switch (mode) { + case 0100644: + strbuf_add(&buffer, "100644 ", 7); + break; + case 0100664: + strbuf_add(&buffer, "100664 ", 7); + break; + case 0100755: + strbuf_add(&buffer, "100755 ", 7); + break; + case 0120000: + strbuf_add(&buffer, "120000 ", 7); + break; + case 0160000: + strbuf_add(&buffer, "160000 ", 7); + break; + default: + strbuf_addf(&buffer, "%o ", mode); + break; + } + strbuf_add(&buffer, path + baselen, entlen); + strbuf_addch(&buffer, '\0'); strbuf_add(&buffer, oid->hash, the_hash_algo->rawsz); #if DEBUG_CACHE_TREE From cbc122057c3407e9bdd3e40a8f54a81ed58e4a19 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Mon, 5 Feb 2018 09:44:04 -0500 Subject: [PATCH 041/118] status: ignore status.aheadbehind in porcelain formats Teach porcelain V[12] formats to ignore the status.aheadbehind config setting. They only respect the --[no-]ahead-behind command line argument. This is for backwards compatibility with existing scripts. Signed-off-by: Jeff Hostetler --- Documentation/config/status.txt | 3 +-- builtin/commit.c | 10 ++++++---- t/t7064-wtstatus-pv2.sh | 12 ++++++++---- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/Documentation/config/status.txt b/Documentation/config/status.txt index 764de31fc46e0a..d845d2dd4d36a4 100644 --- a/Documentation/config/status.txt +++ b/Documentation/config/status.txt @@ -15,8 +15,7 @@ status.branch:: status.aheadBehind:: EXPERIMENTAL, Set to true to enable --ahead-behind and false to enable --no-ahead-behind by default in linkgit:git-status[1] for - all status formats. Defaults to true. Changing this may - break backwards compatibility for existing scripts. + non-porcelain status formats. Defaults to true. status.displayCommentPrefix:: If set to true, linkgit:git-status[1] will insert a comment diff --git a/builtin/commit.c b/builtin/commit.c index 3d866338129014..5eba4acb66b01b 100644 --- a/builtin/commit.c +++ b/builtin/commit.c @@ -1160,11 +1160,13 @@ static void finalize_deferred_config(struct wt_status *s) /* * If the user did not give a "--[no]-ahead-behind" command - * line argument, then we inherit the a/b config setting. - * If is not set, then we inherit _FULL for backwards - * compatibility. + * line argument *AND* we will print in a human-readable format + * (short, long etc.) then we inherit from the status.aheadbehind + * config setting. In all other cases (and porcelain V[12] formats + * in particular), we inherit _FULL for backwards compatibility. */ - if (s->ahead_behind_flags == AHEAD_BEHIND_UNSPECIFIED) + if (use_deferred_config && + s->ahead_behind_flags == AHEAD_BEHIND_UNSPECIFIED) s->ahead_behind_flags = status_deferred_config.ahead_behind; if (s->ahead_behind_flags == AHEAD_BEHIND_UNSPECIFIED) diff --git a/t/t7064-wtstatus-pv2.sh b/t/t7064-wtstatus-pv2.sh index a0baf6e8b09f95..537787e598b414 100755 --- a/t/t7064-wtstatus-pv2.sh +++ b/t/t7064-wtstatus-pv2.sh @@ -436,10 +436,6 @@ test_expect_success 'verify --[no-]ahead-behind with V2 format' ' git status --no-ahead-behind --porcelain=v2 --branch --untracked-files=all >actual && test_cmp expect actual && - # Confirmat that "status.aheadbehind" works on V2 format. - git -c status.aheadbehind=false status --porcelain=v2 --branch --untracked-files=all >actual && - test_cmp expect actual && - # Confirm --ahead-behind reports traditional branch.ab with 1/0. cat >expect <<-EOF && # branch.oid $HUF @@ -449,6 +445,14 @@ test_expect_success 'verify --[no-]ahead-behind with V2 format' ' EOF git status --ahead-behind --porcelain=v2 --branch --untracked-files=all >actual && + test_cmp expect actual && + + # Confirm that "status.aheadbehind" DOES NOT work on V2 format. + git -c status.aheadbehind=false status --porcelain=v2 --branch --untracked-files=all >actual && + test_cmp expect actual && + + # Confirm that "status.aheadbehind" DOES NOT work on V2 format. + git -c status.aheadbehind=true status --porcelain=v2 --branch --untracked-files=all >actual && test_cmp expect actual ) ' From 6ebc4eed25da73e14051dd959ec50e5a2b34dec3 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 2 Feb 2018 14:17:05 -0500 Subject: [PATCH 042/118] status: serialize to path Teach status serialization to take an optional pathname on the command line to direct that cache data be written there rather than to stdout. When used this way, normal status results will still be written to stdout. When no path is given, only binary serialization data is written to stdout. Usage: git status --serialize[=] Signed-off-by: Jeff Hostetler --- Documentation/git-status.txt | 10 ++++++---- builtin/commit.c | 36 +++++++++++++++++++++++++++--------- t/t7524-serialized-status.sh | 23 +++++++++++++++++++++++ wt-status-serialize.c | 5 ++--- wt-status.c | 2 +- wt-status.h | 2 +- 6 files changed, 60 insertions(+), 18 deletions(-) diff --git a/Documentation/git-status.txt b/Documentation/git-status.txt index 6bf716bf43c6a4..b0ffe5a9e8ecda 100644 --- a/Documentation/git-status.txt +++ b/Documentation/git-status.txt @@ -152,10 +152,12 @@ ignored, then the directory is not shown, but all contents are shown. update it afterwards if any changes were detected. Defaults to `--lock-index`. ---serialize[=]:: - (EXPERIMENTAL) Serialize raw status results to stdout in a - format suitable for use by `--deserialize`. Valid values for - `` are "1" and "v1". +--serialize[=]:: + (EXPERIMENTAL) Serialize raw status results to a file or stdout + in a format suitable for use by `--deserialize`. If a path is + given, serialize data will be written to that path *and* normal + status output will be written to stdout. If path is omitted, + only binary serialization data will be written to stdout. --deserialize[=]:: (EXPERIMENTAL) Deserialize raw status results from a file or diff --git a/builtin/commit.c b/builtin/commit.c index 5eba4acb66b01b..510501235bb257 100644 --- a/builtin/commit.c +++ b/builtin/commit.c @@ -144,26 +144,34 @@ static int opt_parse_porcelain(const struct option *opt, const char *arg, int un } static int do_serialize = 0; +static char *serialize_path = NULL; + static int do_implicit_deserialize = 0; static int do_explicit_deserialize = 0; static char *deserialize_path = NULL; /* - * --serialize | --serialize=1 | --serialize=v1 + * --serialize | --serialize= + * + * Request that we serialize status output rather than or in addition to + * printing in any of the established formats. + * + * Without a path, we write binary serialization data to stdout (and omit + * the normal status output). * - * Request that we serialize our output rather than printing in - * any of the established formats. Optionally specify serialization - * version. + * With a path, we write binary serialization data to the and then + * write normal status output. */ static int opt_parse_serialize(const struct option *opt, const char *arg, int unset) { enum wt_status_format *value = (enum wt_status_format *)opt->value; if (unset || !arg) *value = STATUS_FORMAT_SERIALIZE_V1; - else if (!strcmp(arg, "v1") || !strcmp(arg, "1")) - *value = STATUS_FORMAT_SERIALIZE_V1; - else - die("unsupported serialize version '%s'", arg); + + if (arg) { + free(serialize_path); + serialize_path = xstrdup(arg); + } if (do_explicit_deserialize) die("cannot mix --serialize and --deserialize"); @@ -1421,7 +1429,7 @@ int cmd_status(int argc, const char **argv, const char *prefix) N_("version"), N_("machine-readable output"), PARSE_OPT_OPTARG, opt_parse_porcelain }, { OPTION_CALLBACK, 0, "serialize", &status_format, - N_("version"), N_("serialize raw status data to stdout"), + N_("path"), N_("serialize raw status data to path or stdout"), PARSE_OPT_OPTARG | PARSE_OPT_NONEG, opt_parse_serialize }, { OPTION_CALLBACK, 0, "deserialize", NULL, N_("path"), N_("deserialize raw status data from file"), @@ -1563,6 +1571,16 @@ int cmd_status(int argc, const char **argv, const char *prefix) if (s.relative_paths) s.prefix = prefix; + if (serialize_path) { + int fd_serialize = xopen(serialize_path, + O_WRONLY | O_CREAT | O_TRUNC, 0666); + if (fd_serialize < 0) + die_errno(_("could not serialize to '%s'"), + serialize_path); + wt_status_serialize_v1(fd_serialize, &s); + close(fd_serialize); + } + wt_status_print(&s); wt_status_collect_free_buffers(&s); diff --git a/t/t7524-serialized-status.sh b/t/t7524-serialized-status.sh index 6bcd301dbb7913..d1ca3af3f96546 100755 --- a/t/t7524-serialized-status.sh +++ b/t/t7524-serialized-status.sh @@ -163,4 +163,27 @@ test_expect_success 'verify no-ahead-behind and serialized status integration' ' test_i18ncmp expect output ' +test_expect_success 'verify new --serialize=path mode' ' + #test_when_finished "rm serialized_status.dat expect new_change.txt output.1 output.2" && + cat >expect <<-\EOF && + ? expect + ? output.1 + ? untracked/ + ? untracked_1.txt + EOF + + git checkout -b serialize_path_branch master --track >/dev/null && + touch alt_branch_changes.txt && + git add alt_branch_changes.txt && + test_tick && + git commit -m"New commit on serialize_path_branch" && + + git status --porcelain=v2 --serialize=serialized_status.dat >output.1 && + touch new_change.txt && + + git status --porcelain=v2 --deserialize=serialized_status.dat >output.2 && + test_i18ncmp expect output.1 && + test_i18ncmp expect output.2 +' + test_done diff --git a/wt-status-serialize.c b/wt-status-serialize.c index c393c7292aea7a..43b9c25daadc0b 100644 --- a/wt-status-serialize.c +++ b/wt-status-serialize.c @@ -162,7 +162,7 @@ static inline void wt_serialize_v1_ignored(struct wt_status *s, int fd, } /* - * Serialize the list of changes to stdout. The goal of this + * Serialize the list of changes to the given file. The goal of this * is to just serialize the key fields in wt_status so that a * later command can rebuilt it and do the printing. * @@ -171,9 +171,8 @@ static inline void wt_serialize_v1_ignored(struct wt_status *s, int fd, * is relatively quick for the status consumer to compute * as necessary. */ -void wt_status_serialize_v1(struct wt_status *s) +void wt_status_serialize_v1(int fd, struct wt_status *s) { - int fd = 1; /* we always write to stdout */ struct string_list_item *iter; int k; diff --git a/wt-status.c b/wt-status.c index 2e8683ee084579..48789cf26c5303 100644 --- a/wt-status.c +++ b/wt-status.c @@ -2329,7 +2329,7 @@ void wt_status_print(struct wt_status *s) wt_longstatus_print(s); break; case STATUS_FORMAT_SERIALIZE_V1: - wt_status_serialize_v1(s); + wt_status_serialize_v1(1, s); break; } } diff --git a/wt-status.h b/wt-status.h index c473a6785e4bf4..f639ee1ed17f92 100644 --- a/wt-status.h +++ b/wt-status.h @@ -196,7 +196,7 @@ struct wt_status_serialize_data * Serialize computed status scan results using "version 1" format * to the given file. */ -void wt_status_serialize_v1(struct wt_status *s); +void wt_status_serialize_v1(int fd, struct wt_status *s); /* * Deserialize existing status results from the given file and From a49236d6c459757dda4becbac7503b5c316f884e Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Thu, 15 Feb 2018 14:44:48 -0500 Subject: [PATCH 043/118] fetch: Warn about forced updates after branch list Signed-off-by: Derrick Stolee --- builtin/fetch.c | 34 +++++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/builtin/fetch.c b/builtin/fetch.c index a6f3654608f1e8..5f66b8301d903a 100644 --- a/builtin/fetch.c +++ b/builtin/fetch.c @@ -24,6 +24,8 @@ #include "list-objects-filter-options.h" #include "commit-reach.h" +#define FORCED_UPDATES_DELAY_WARNING_IN_MS (10 * 1000) + static const char * const builtin_fetch_usage[] = { N_("git fetch [] [ [...]]"), N_("git fetch [] "), @@ -41,6 +43,7 @@ enum { static int fetch_prune_config = -1; /* unspecified */ static int fetch_show_forced_updates = 1; static int fetch_show_forced_updates_warning = 0; +static uint64_t forced_updates_ms = 0; static int prune = -1; /* unspecified */ #define PRUNE_BY_DEFAULT 0 /* do we prune by default? */ @@ -708,6 +711,7 @@ static int update_local_ref(struct ref *ref, enum object_type type; struct branch *current_branch = branch_get(NULL); const char *pretty_ref = prettify_refname(ref->name); + int fast_forward = 0; type = oid_object_info(the_repository, &ref->new_oid, NULL); if (type < 0) @@ -782,18 +786,19 @@ static int update_local_ref(struct ref *ref, return r; } - if (!fetch_show_forced_updates || in_merge_bases(current, updated)) { + if (fetch_show_forced_updates) { + uint64_t t_before = getnanotime(); + fast_forward = in_merge_bases(current, updated); + forced_updates_ms += (getnanotime() - t_before) / 1000000; + } else { + fetch_show_forced_updates_warning = 1; + fast_forward = 1; + } + + if (fast_forward) { struct strbuf quickref = STRBUF_INIT; int r; - if (!fetch_show_forced_updates && !fetch_show_forced_updates_warning) { - fetch_show_forced_updates_warning = 1; - warning(_("Fetch normally indicates which branches had a forced update, but that check has been disabled.")); - warning(_("To re-enable, use '--show-forced-updates' flag or run 'git config fetch.showForcedUpdates true'.")); - warning(_("OR if you just want to check if a branch with name was force-updated after fetching, run:")); - warning(_(" git rev-list --left-right --count ...@{1}")); - } - strbuf_add_unique_abbrev(&quickref, ¤t->object.oid, DEFAULT_ABBREV); strbuf_addstr(&quickref, ".."); strbuf_add_unique_abbrev(&quickref, &ref->new_oid, DEFAULT_ABBREV); @@ -989,6 +994,17 @@ static int store_updated_refs(const char *raw_url, const char *remote_name, " 'git remote prune %s' to remove any old, conflicting " "branches"), remote_name); + if (!fetch_show_forced_updates && fetch_show_forced_updates_warning) { + warning(_("Fetch normally indicates which branches had a forced update, but that check has been disabled.")); + warning(_("To re-enable, use '--show-forced-updates' flag or run 'git config fetch.showForcedUpdates true'.")); + } + if (fetch_show_forced_updates && + forced_updates_ms > FORCED_UPDATES_DELAY_WARNING_IN_MS) { + warning(_("It took %.2f seconds to check forced updates. You can use '--no-show-forced-updates'\n"), + forced_updates_ms / 1000.0); + warning(_("or run 'git config fetch.showForcedUpdates false' to avoid this check.\n")); + } + abort: strbuf_release(¬e); free(url); From bbdfe60687c4ebe99c3689fb7f607bde3d9ae109 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 7 Feb 2018 10:59:03 -0500 Subject: [PATCH 044/118] status: reject deserialize in V2 and conflicts Teach status deserialize code to reject status cache when printing in porcelain V2 and there are unresolved conflicts in the cache file. A follow-on task might extend the cache format to include this additiona data. See code for longer explanation. Signed-off-by: Jeff Hostetler --- t/t7524-serialized-status.sh | 85 +++++++++++++++++++++++++++++++++++- wt-status-deserialize.c | 28 +++++++++++- 2 files changed, 110 insertions(+), 3 deletions(-) diff --git a/t/t7524-serialized-status.sh b/t/t7524-serialized-status.sh index d1ca3af3f96546..04c45ef71221f5 100755 --- a/t/t7524-serialized-status.sh +++ b/t/t7524-serialized-status.sh @@ -164,7 +164,7 @@ test_expect_success 'verify no-ahead-behind and serialized status integration' ' ' test_expect_success 'verify new --serialize=path mode' ' - #test_when_finished "rm serialized_status.dat expect new_change.txt output.1 output.2" && + test_when_finished "rm serialized_status.dat expect new_change.txt output.1 output.2" && cat >expect <<-\EOF && ? expect ? output.1 @@ -186,4 +186,87 @@ test_expect_success 'verify new --serialize=path mode' ' test_i18ncmp expect output.2 ' +test_expect_success 'merge conflicts' ' + + # create a merge conflict. + + git init conflicts && + echo x >conflicts/x.txt && + git -C conflicts add x.txt && + git -C conflicts commit -m x && + git -C conflicts branch a && + git -C conflicts branch b && + git -C conflicts checkout a && + echo y >conflicts/x.txt && + git -C conflicts add x.txt && + git -C conflicts commit -m a && + git -C conflicts checkout b && + echo z >conflicts/x.txt && + git -C conflicts add x.txt && + git -C conflicts commit -m b && + test_must_fail git -C conflicts merge --no-commit a && + + # verify that regular status correctly identifies it + # in each format. + + cat >expect.v2 <observed.v2 && + test_cmp expect.v2 observed.v2 && + + cat >expect.long <..." to mark resolution) + + both modified: x.txt + +no changes added to commit (use "git add" and/or "git commit -a") +EOF + git -C conflicts status --long >observed.long && + test_cmp expect.long observed.long && + + cat >expect.short <observed.short && + test_cmp expect.short observed.short && + + # save status data in serialized cache. + + git -C conflicts status --serialize >serialized && + + # make some dirt in the worktree so we can tell whether subsequent + # status commands used the cached data or did a fresh status. + + echo dirt >conflicts/dirt.txt && + + # run status using the cached data. + + git -C conflicts status --long --deserialize=../serialized >observed.long && + test_cmp expect.long observed.long && + + git -C conflicts status --short --deserialize=../serialized >observed.short && + test_cmp expect.short observed.short && + + # currently, the cached data does not have enough information about + # merge conflicts for porcelain V2 format. (And V2 format looks at + # the index to get that data, but the whole point of the serialization + # is to avoid reading the index unnecessarily.) So V2 always rejects + # the cached data when there is an unresolved conflict. + + cat >expect.v2.dirty <observed.v2 && + test_cmp expect.v2.dirty observed.v2 + +' + test_done diff --git a/wt-status-deserialize.c b/wt-status-deserialize.c index f8232bb7524060..a2eb107fe3b7c6 100644 --- a/wt-status-deserialize.c +++ b/wt-status-deserialize.c @@ -176,7 +176,8 @@ static int wt_deserialize_v1_header(struct wt_status *s, int fd) /* * Build a string-list of (count) lines from the input. */ -static int wt_deserialize_v1_changed_items(struct wt_status *s, int fd, int count) +static int wt_deserialize_v1_changed_items(const struct wt_status *cmd_s, + struct wt_status *s, int fd, int count) { struct wt_status_serialize_data *sd; char *p; @@ -231,6 +232,29 @@ static int wt_deserialize_v1_changed_items(struct wt_status *s, int fd, int coun oid_to_hex(&d->oid_index), item->string, (d->rename_source ? d->rename_source : "")); + + if (d->stagemask && + cmd_s->status_format == STATUS_FORMAT_PORCELAIN_V2) { + /* + * We have an unresolved conflict and the user wants + * to see porcelain V2 output. The cached status data + * does not contain enough information for V2 (because + * the main status computation does not capture it). + * We only get a single change record for the file with + * a single SHA -- we don't get the stage [123] mode + * and SHA data. The V2 detail-line print code looks + * up this information directly from the index. The + * whole point of this serialization cache is to avoid + * reading the index, so the V2 print code gets zeros. + * So we reject the status cache and let the fallback + * code run. + */ + trace_printf_key( + &trace_deserialize, + "reject: V2 format and umerged file: %s", + item->string); + return DESERIALIZE_ERR; + } } return DESERIALIZE_OK; @@ -381,7 +405,7 @@ static int wt_deserialize_v1(const struct wt_status *cmd_s, struct wt_status *s, while ((line = my_packet_read_line(fd, &line_len))) { if (skip_prefix(line, "changed ", &arg)) { nr_changed = (int)strtol(arg, NULL, 10); - if (wt_deserialize_v1_changed_items(s, fd, nr_changed) + if (wt_deserialize_v1_changed_items(cmd_s, s, fd, nr_changed) == DESERIALIZE_ERR) return DESERIALIZE_ERR; continue; From 029499b21ca527fbfdd50c6011d42c7059555282 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Fri, 16 Feb 2018 09:28:07 -0500 Subject: [PATCH 045/118] push: add --[no-]show-forced-updates passthrough to fetch --- builtin/pull.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/builtin/pull.c b/builtin/pull.c index 701d1473dc59e0..f096d428b94c06 100644 --- a/builtin/pull.c +++ b/builtin/pull.c @@ -122,6 +122,7 @@ static char *opt_update_shallow; static char *opt_refmap; static char *opt_ipv4; static char *opt_ipv6; +static int opt_show_forced_updates = -1; static struct option pull_options[] = { /* Shared options */ @@ -233,6 +234,8 @@ static struct option pull_options[] = { OPT_PASSTHRU('6', "ipv6", &opt_ipv6, NULL, N_("use IPv6 addresses only"), PARSE_OPT_NOARG), + OPT_BOOL(0, "show-forced-updates", &opt_show_forced_updates, + N_("check for forced-updates on all updated branches")), OPT_END() }; @@ -541,6 +544,10 @@ static int run_fetch(const char *repo, const char **refspecs) argv_array_push(&args, opt_ipv4); if (opt_ipv6) argv_array_push(&args, opt_ipv6); + if (opt_show_forced_updates > 0) + argv_array_push(&args, "--show-forced-updates"); + else if (opt_show_forced_updates == 0) + argv_array_push(&args, "--no-show-forced-updates"); if (repo) { argv_array_push(&args, repo); From ec4b1844cd4353194920d99351080fe80fb1545d Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Thu, 1 Feb 2018 09:47:12 -0500 Subject: [PATCH 046/118] fetch: Add --[no-]show-forced-updates argument Also include fetch.showForcedUpdates config setting that is overridden by the command-line arguments. Shows a warning when skipping the work. Signed-off-by: Derrick Stolee --- builtin/fetch.c | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/builtin/fetch.c b/builtin/fetch.c index b620fd54b4e95c..a6f3654608f1e8 100644 --- a/builtin/fetch.c +++ b/builtin/fetch.c @@ -39,6 +39,8 @@ enum { }; static int fetch_prune_config = -1; /* unspecified */ +static int fetch_show_forced_updates = 1; +static int fetch_show_forced_updates_warning = 0; static int prune = -1; /* unspecified */ #define PRUNE_BY_DEFAULT 0 /* do we prune by default? */ @@ -79,6 +81,11 @@ static int git_fetch_config(const char *k, const char *v, void *cb) return 0; } + if (!strcmp(k, "fetch.showforcedupdates")) { + fetch_show_forced_updates = git_config_bool(k, v); + return 0; + } + if (!strcmp(k, "submodule.recurse")) { int r = git_config_bool(k, v) ? RECURSE_SUBMODULES_ON : RECURSE_SUBMODULES_OFF; @@ -169,6 +176,8 @@ static struct option builtin_fetch_options[] = { OPT_STRING_LIST(0, "negotiation-tip", &negotiation_tip, N_("revision"), N_("report that we have only objects reachable from this object")), OPT_PARSE_LIST_OBJECTS_FILTER(&filter_options), + OPT_BOOL(0, "show-forced-updates", &fetch_show_forced_updates, + N_("check for forced-updates on all updated branches")), OPT_END() }; @@ -773,9 +782,18 @@ static int update_local_ref(struct ref *ref, return r; } - if (in_merge_bases(current, updated)) { + if (!fetch_show_forced_updates || in_merge_bases(current, updated)) { struct strbuf quickref = STRBUF_INIT; int r; + + if (!fetch_show_forced_updates && !fetch_show_forced_updates_warning) { + fetch_show_forced_updates_warning = 1; + warning(_("Fetch normally indicates which branches had a forced update, but that check has been disabled.")); + warning(_("To re-enable, use '--show-forced-updates' flag or run 'git config fetch.showForcedUpdates true'.")); + warning(_("OR if you just want to check if a branch with name was force-updated after fetching, run:")); + warning(_(" git rev-list --left-right --count ...@{1}")); + } + strbuf_add_unique_abbrev(&quickref, ¤t->object.oid, DEFAULT_ABBREV); strbuf_addstr(&quickref, ".."); strbuf_add_unique_abbrev(&quickref, &ref->new_oid, DEFAULT_ABBREV); From ee7dcf5d9449cd7ee6dac760c4a6e4ceabccfa1d Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Wed, 21 Feb 2018 08:48:05 -0500 Subject: [PATCH 047/118] fetch: add documentation for --[no-]show-forced-updates Signed-off-by: Derrick Stolee --- Documentation/fetch-options.txt | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/Documentation/fetch-options.txt b/Documentation/fetch-options.txt index fa0a3151b3f7e9..3680082ae7ebc6 100644 --- a/Documentation/fetch-options.txt +++ b/Documentation/fetch-options.txt @@ -220,6 +220,19 @@ endif::git-pull[] When multiple `--server-option=