Skip to content

Commit

Permalink
Merge pull request #212 fsmonitor updates for improved performance
Browse files Browse the repository at this point in the history
fsmonitor updates for improved performance
  • Loading branch information
kewillford authored and dscho committed May 20, 2020
2 parents 92aa019 + 40de6f1 commit d4ad347
Show file tree
Hide file tree
Showing 4 changed files with 137 additions and 26 deletions.
2 changes: 1 addition & 1 deletion cache.h
Original file line number Diff line number Diff line change
Expand Up @@ -239,7 +239,7 @@ static inline unsigned create_ce_flags(unsigned stage)
#define ce_namelen(ce) ((ce)->ce_namelen)
#define ce_size(ce) cache_entry_size(ce_namelen(ce))
#define ce_stage(ce) ((CE_STAGEMASK & (ce)->ce_flags) >> CE_STAGESHIFT)
#define ce_uptodate(ce) ((ce)->ce_flags & CE_UPTODATE)
#define ce_uptodate(ce) (((ce)->ce_flags & CE_UPTODATE) || ((ce)->ce_flags & CE_FSMONITOR_VALID))
#define ce_skip_worktree(ce) ((ce)->ce_flags & CE_SKIP_WORKTREE)
#define ce_mark_uptodate(ce) ((ce)->ce_flags |= CE_UPTODATE)
#define ce_intent_to_add(ce) ((ce)->ce_flags & CE_INTENT_TO_ADD)
Expand Down
11 changes: 7 additions & 4 deletions fsmonitor.c
Original file line number Diff line number Diff line change
Expand Up @@ -261,10 +261,13 @@ void refresh_fsmonitor(struct index_state *istate)
if (istate->untracked)
istate->untracked->use_fsmonitor = 1;
} else {

/* We only want to run the post index changed hook if we've actually changed entries, so keep track
* if we actually changed entries or not */
/*
* We only want to run the post index changed hook if we've
* actually changed entries, so keep track if we actually
* changed entries or not
*/
int is_cache_changed = 0;

/* Mark all entries invalid */
for (i = 0; i < istate->cache_nr; i++) {
if (istate->cache[i]->ce_flags & CE_FSMONITOR_VALID) {
Expand All @@ -273,7 +276,7 @@ void refresh_fsmonitor(struct index_state *istate)
}
}

/* If we're going to check every file, ensure we save the results */
/* If we're going to check every file, ensure we save results */
if (is_cache_changed)
istate->cache_changed |= FSMONITOR_CHANGED;

Expand Down
22 changes: 1 addition & 21 deletions t/t7519/fsmonitor-watchman
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ use IPC::Open2;
# 'git config core.fsmonitor .git/hooks/query-watchman'
#
my ($version, $time) = @ARGV;
#print STDERR "$0 $version $time\n";

# Check the hook interface version

Expand All @@ -44,7 +43,7 @@ launch_watchman();

sub launch_watchman {

my $pid = open2(\*CHLD_OUT, \*CHLD_IN, 'watchman -j')
my $pid = open2(\*CHLD_OUT, \*CHLD_IN, 'watchman -j --no-pretty')
or die "open2() failed: $!\n" .
"Falling back to scanning...\n";

Expand All @@ -62,19 +61,11 @@ sub launch_watchman {
"fields": ["name"]
}]
END

open (my $fh, ">", ".git/watchman-query.json");
print $fh $query;
close $fh;

print CHLD_IN $query;
close CHLD_IN;
my $response = do {local $/; <CHLD_OUT>};

open ($fh, ">", ".git/watchman-response.json");
print $fh $response;
close $fh;

die "Watchman: command returned no output.\n" .
"Falling back to scanning...\n" if $response eq "";
die "Watchman: command returned invalid output: $response\n" .
Expand All @@ -93,7 +84,6 @@ sub launch_watchman {
my $o = $json_pkg->new->utf8->decode($response);

if ($retry > 0 and $o->{error} and $o->{error} =~ m/unable to resolve root .* directory (.*) is not watched/) {
print STDERR "Adding '$git_work_tree' to watchman's watch list.\n";
$retry--;
qx/watchman watch "$git_work_tree"/;
die "Failed to make watchman watch '$git_work_tree'.\n" .
Expand All @@ -103,11 +93,6 @@ sub launch_watchman {
# return the fast "everything is dirty" flag to git and do the
# Watchman query just to get it over with now so we won't pay
# the cost in git to look up each individual file.

open ($fh, ">", ".git/watchman-output.out");
print "/\0";
close $fh;

print "/\0";
eval { launch_watchman() };
exit 0;
Expand All @@ -116,11 +101,6 @@ sub launch_watchman {
die "Watchman: $o->{error}.\n" .
"Falling back to scanning...\n" if $o->{error};

open ($fh, ">", ".git/watchman-output.out");
binmode $fh, ":utf8";
print $fh @{$o->{files}};
close $fh;

binmode STDOUT, ":utf8";
local $, = "\0";
print @{$o->{files}};
Expand Down
128 changes: 128 additions & 0 deletions t/t7519/fsmonitor-watchman-debug
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
#!/usr/bin/perl

use strict;
use warnings;
use IPC::Open2;

# An example hook script to integrate Watchman
# (https://facebook.github.io/watchman/) with git to speed up detecting
# new and modified files.
#
# The hook is passed a version (currently 1) and a time in nanoseconds
# formatted as a string and outputs to stdout all files that have been
# modified since the given time. Paths must be relative to the root of
# the working tree and separated by a single NUL.
#
# To enable this hook, rename this file to "query-watchman" and set
# 'git config core.fsmonitor .git/hooks/query-watchman'
#
my ($version, $time) = @ARGV;
#print STDERR "$0 $version $time\n";

# Check the hook interface version

if ($version == 1) {
# convert nanoseconds to seconds
# subtract one second to make sure watchman will return all changes
$time = int ($time / 1000000000) - 1;
} else {
die "Unsupported query-fsmonitor hook version '$version'.\n" .
"Falling back to scanning...\n";
}

my $git_work_tree;
if ($^O =~ 'msys' || $^O =~ 'cygwin') {
$git_work_tree = Win32::GetCwd();
$git_work_tree =~ tr/\\/\//;
} else {
require Cwd;
$git_work_tree = Cwd::cwd();
}

my $retry = 1;

launch_watchman();

sub launch_watchman {

my $pid = open2(\*CHLD_OUT, \*CHLD_IN, 'watchman -j')
or die "open2() failed: $!\n" .
"Falling back to scanning...\n";

# In the query expression below we're asking for names of files that
# changed since $time but were not transient (ie created after
# $time but no longer exist).
#
# To accomplish this, we're using the "since" generator to use the
# recency index to select candidate nodes and "fields" to limit the
# output to file names only.

my $query = <<" END";
["query", "$git_work_tree", {
"since": $time,
"fields": ["name"]
}]
END

open (my $fh, ">", ".git/watchman-query.json");
print $fh $query;
close $fh;

print CHLD_IN $query;
close CHLD_IN;
my $response = do {local $/; <CHLD_OUT>};

open ($fh, ">", ".git/watchman-response.json");
print $fh $response;
close $fh;

die "Watchman: command returned no output.\n" .
"Falling back to scanning...\n" if $response eq "";
die "Watchman: command returned invalid output: $response\n" .
"Falling back to scanning...\n" unless $response =~ /^\{/;

my $json_pkg;
eval {
require JSON::XS;
$json_pkg = "JSON::XS";
1;
} or do {
require JSON::PP;
$json_pkg = "JSON::PP";
};

my $o = $json_pkg->new->utf8->decode($response);

if ($retry > 0 and $o->{error} and $o->{error} =~ m/unable to resolve root .* directory (.*) is not watched/) {
print STDERR "Adding '$git_work_tree' to watchman's watch list.\n";
$retry--;
qx/watchman watch "$git_work_tree"/;
die "Failed to make watchman watch '$git_work_tree'.\n" .
"Falling back to scanning...\n" if $? != 0;

# Watchman will always return all files on the first query so
# return the fast "everything is dirty" flag to git and do the
# Watchman query just to get it over with now so we won't pay
# the cost in git to look up each individual file.

open ($fh, ">", ".git/watchman-output.out");
print "/\0";
close $fh;

print "/\0";
eval { launch_watchman() };
exit 0;
}

die "Watchman: $o->{error}.\n" .
"Falling back to scanning...\n" if $o->{error};

open ($fh, ">", ".git/watchman-output.out");
binmode $fh, ":utf8";
print $fh @{$o->{files}};
close $fh;

binmode STDOUT, ":utf8";
local $, = "\0";
print @{$o->{files}};
}

0 comments on commit d4ad347

Please sign in to comment.