From 65cf85103f1939da84038b3e13e49adf2e9db0ca Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Fri, 13 Apr 2012 10:21:18 -0400 Subject: [PATCH 1/4] build-remote: CopyClosure::copyTo takes a new argument --- scripts/build-remote.pl.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build-remote.pl.in b/scripts/build-remote.pl.in index fca00241bf1..da5561cd219 100755 --- a/scripts/build-remote.pl.in +++ b/scripts/build-remote.pl.in @@ -248,7 +248,7 @@ if ($@) { print STDERR "somebody is hogging $uploadLock, continuing...\n"; unlink $uploadLock; } -Nix::CopyClosure::copyTo($hostName, [ @sshOpts ], [ $drvPath, @inputs ], "", "", 0, 0, $maybeSign ne ""); +Nix::CopyClosure::copyTo($hostName, [ @sshOpts ], [ $drvPath, @inputs ], "", "", 0, 0, $maybeSign ne "", ""); close UPLOADLOCK; From 6af2be4a120f32e521da693caa561cea0ee16ce9 Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Fri, 13 Apr 2012 10:23:59 -0400 Subject: [PATCH 2/4] nix-copy-closure: Have pv be size-aware when used --- perl/lib/Nix/CopyClosure.pm | 5 +++-- scripts/nix-copy-closure.in | 7 ++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/perl/lib/Nix/CopyClosure.pm b/perl/lib/Nix/CopyClosure.pm index 79c6dfcccbf..3a064ed51aa 100644 --- a/perl/lib/Nix/CopyClosure.pm +++ b/perl/lib/Nix/CopyClosure.pm @@ -10,7 +10,6 @@ sub copyTo { $compressor = "$compressor |" if $compressor ne ""; $decompressor = "$decompressor |" if $decompressor ne ""; - $progressViewer = "$progressViewer |" if $progressViewer ne ""; # Get the closure of this path. my @closure = reverse(topoSortPaths(computeFSClosure(0, $includeOutputs, @@ -35,7 +34,9 @@ sub copyTo { if (scalar @missing > 0) { print STDERR "copying ", scalar @missing, " missing paths to ‘$sshHost’...\n"; unless ($dryRun) { - open SSH, "| $compressor $progressViewer ssh $sshHost @{$sshOpts} '$decompressor nix-store --import' > /dev/null" or die; + my $size = `nix-store -q --size @missing | awk '{n+=\$1} END {printf n}'`; + $progressViewer = "$progressViewer -s $size |" if $progressViewer ne ""; + open SSH, "| $progressViewer $compressor ssh $sshHost @{$sshOpts} '$decompressor nix-store --import' > /dev/null" or die; exportPaths(fileno(SSH), $sign, @missing); close SSH or die "copying store paths to remote machine `$sshHost' failed: $?"; } diff --git a/scripts/nix-copy-closure.in b/scripts/nix-copy-closure.in index 12a83cff979..4a446a23e5b 100755 --- a/scripts/nix-copy-closure.in +++ b/scripts/nix-copy-closure.in @@ -63,7 +63,7 @@ while (@ARGV) { $includeOutputs = 1; } elsif ($arg eq "--show-progress") { - $progressViewer = "@pv@"; + $progressViewer = "@pv@ -apebt"; } elsif ($arg eq "--dry-run") { $dryRun = 1; @@ -106,10 +106,11 @@ else { # Copy FROM the remote machine. print STDERR "copying ", scalar @missing, " missing paths from ‘$sshHost’...\n"; $compressor = "| $compressor" if $compressor ne ""; $decompressor = "$decompressor |" if $decompressor ne ""; - $progressViewer = "$progressViewer |" if $progressViewer ne ""; + my $size = `ssh $sshHost @sshOpts "nix-store -q --size $extraOpts @missing | awk '{n+=\\\$1} END {printf n}'"`; + $progressViewer = "$progressViewer -s $size |" if $progressViewer ne ""; unless ($dryRun) { my $extraOpts = $sign ? "--sign" : ""; - system("set -f; ssh $sshHost @sshOpts 'nix-store --export $extraOpts @missing $compressor' | $progressViewer $decompressor $Nix::Config::binDir/nix-store --import > /dev/null") == 0 + system("set -f; ssh $sshHost @sshOpts 'nix-store --export $extraOpts @missing $compressor' | $decompressor $progressViewer $Nix::Config::binDir/nix-store --import > /dev/null") == 0 or die "copying store paths from remote machine `$sshHost' failed: $?"; } } From aba00bb86c0899da7cab8651d8059533a3d0176f Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Fri, 13 Apr 2012 11:08:11 -0400 Subject: [PATCH 3/4] nix-copy-closure: Put the progressViewer after decompression always to avoid a delay between 'completion' according to pv and actual completion --- perl/lib/Nix/CopyClosure.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perl/lib/Nix/CopyClosure.pm b/perl/lib/Nix/CopyClosure.pm index 3a064ed51aa..69e5940289a 100644 --- a/perl/lib/Nix/CopyClosure.pm +++ b/perl/lib/Nix/CopyClosure.pm @@ -36,7 +36,7 @@ sub copyTo { unless ($dryRun) { my $size = `nix-store -q --size @missing | awk '{n+=\$1} END {printf n}'`; $progressViewer = "$progressViewer -s $size |" if $progressViewer ne ""; - open SSH, "| $progressViewer $compressor ssh $sshHost @{$sshOpts} '$decompressor nix-store --import' > /dev/null" or die; + open SSH, "| $compressor ssh $sshHost @{$sshOpts} '$decompressor $progressViewer nix-store --import' > /dev/null" or die; exportPaths(fileno(SSH), $sign, @missing); close SSH or die "copying store paths to remote machine `$sshHost' failed: $?"; } From a1425014a456ee5583bd89530052d98233cde364 Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Tue, 17 Apr 2012 17:05:03 -0400 Subject: [PATCH 4/4] nix-copy-closure: Don't hard-code the pv path when invoking remotely --- scripts/nix-copy-closure.in | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/nix-copy-closure.in b/scripts/nix-copy-closure.in index 4a446a23e5b..e0ff23ab728 100755 --- a/scripts/nix-copy-closure.in +++ b/scripts/nix-copy-closure.in @@ -63,7 +63,7 @@ while (@ARGV) { $includeOutputs = 1; } elsif ($arg eq "--show-progress") { - $progressViewer = "@pv@ -apebt"; + $progressViewer = "-apebt"; } elsif ($arg eq "--dry-run") { $dryRun = 1; @@ -81,10 +81,12 @@ openSSHConnection $sshHost or die "$0: unable to start SSH\n"; if ($toMode) { # Copy TO the remote machine. + $progressViewer = "pv $progressViewer" if $progressViewer ne ""; Nix::CopyClosure::copyTo($sshHost, [ @sshOpts ], [ @storePaths ], $compressor, $decompressor, $includeOutputs, $dryRun, $sign, $progressViewer); } else { # Copy FROM the remote machine. + $progressViewer = "@pv@ $progressViewer" if $progressViewer ne ""; # Query the closure of the given store paths on the remote # machine. Paths are assumed to be store paths; there is no