Dataset created from zone is downloaded to local file system

- Add WIP documentation to README
- Add Rsync util for downloading file from GZ in a fraction of
  the time that net/scp takes
- [refs #26]

README.md

@@ -114,6 +114,14 @@ vagrant zones start [name]
 vagrant zones stop [name]
 ```
 
+Export a running zone as a dataset (image) in the local file system:
+
+```bash
+vagrant dataset create [zone] [identifier]
+vagrant dataset delete [identifier]
+vagrant dataset list
+```
+
 ## Single zone usage
 
 When a single zone is configured (currently the only configuration
@@ -232,19 +240,6 @@ group with which to connect. This user should have `Primary
 Administrator` privileges. When creating a local zone, a `vagrant`
 user and group are also created in the zone.
 
-## Tests
-
-There is a basic test suite that uses `test-kitchen` to converge
-different brands of zones. Although it might not be comprehensive, it
-should be ran after new features or after any significant refactoring to
-ensure that nothing breaks the ability to stand up a zone.
-
-```bash
-bundle exec kitchen test
-```
-
-This may take a while...
-
 ## Plugin configuration
 
 The plugin allows for local configuration through the `vagrant zones
@@ -277,6 +272,32 @@ create zones. It should be pretty obvious when this is the case.
 
 * Does your network overlap with VirtualBox's local network space?
 
+### Using local zone images
+
+When running vagrant from a location far away from where SmartOS images
+are hosted (anywhere other than the east coast of the USA, basically),
+setting up a zone can be very slow.
+
+Zones can be modified in Vagrant, then exported locally to save time on
+repeated tasks. After modifying a zone, for instance by installing build
+tools, you can export the zone as follows (warning: this will take a
+while):
+
+```bash
+# vagrant dataset create [zone] [identifier]
+vagrant dataset create base64 base64-15.1.1-build-essential
+```
+
+NOTE: config overrides are currently WIP. Do not rely on them working
+until this officially released.
+
+The vagrant-smartos-zones plugin can be configured to use a local zone
+image in place of a remote one as follows:
+
+```bash
+vagrant zones config dataset.0edf00aa-0562-11e5-b92f-879647d45790 base64-15.1.1-build-essential
+```
+
 ### Pkgsrc mirror
 
 ```bash
@@ -286,6 +307,19 @@ vagrant zones config local.pkgsrc http://mirror.domain.com
 This will replace the protocol and domain of the pkgsrc mirror used by
 pkgin in a SmartOS zone.
 
+## Tests
+
+There is a basic test suite that uses `test-kitchen` to converge
+different brands of zones. Although it might not be comprehensive, it
+should be ran after new features or after any significant refactoring to
+ensure that nothing breaks the ability to stand up a zone.
+
+```bash
+bundle exec kitchen test
+```
+
+This may take a while...
+
 ## References / Alternatives
 
 Any success of this project depends heavily on the work of others,

lib/vagrant/smartos/locales/en.yml

@@ -3,6 +3,17 @@ en:
     smartos:
       zones:
         commands:
+          dataset:
+            create: |-
+              Creating a new dataset %{dataset} from zone: %{uuid}
+            prepare_image: |-
+              Preparing zone for dataset
+            save: |-
+              Saving zone file system
+            start_zone: |-
+              Restarting zone
+            stop_zone: |-
+              Stopping zone
           zones:
             list: |-
               name                      state      uuid 

lib/vagrant/smartos/zones/commands/dataset.rb

@@ -39,7 +39,9 @@ def host
           end
 
           def create(zonename, dataset)
+            setup_local_directory
             with_zone(zonename) do |zone|
+              ui.info(I18n.t('vagrant.smartos.zones.commands.dataset.create', uuid: zone.uuid, dataset: dataset))
               Models::Dataset.create(dataset, zone)
             end
           end
@@ -80,6 +82,12 @@ def with_zone(name)
             ui.warn(I18n.t('vagrant.smartos.zones.warning.zone_not_found',
                            name: name), prefix: false)
           end
+
+          def setup_local_directory
+            home_path = @env.respond_to?(:home_path) ? @env.home_path : @env[:home_path]
+            image_path = home_path.join('smartos', 'datasets')
+            FileUtils.mkdir_p(image_path)
+          end
         end
       end
     end

lib/vagrant/smartos/zones/communicator/smartos.rb

@@ -1,4 +1,5 @@
 require 'vagrant/smartos/zones/util/global_zone/connection'
+require 'vagrant/smartos/zones/util/rsync'
 
 module Vagrant
   module Smartos
@@ -10,6 +11,23 @@ def initialize(machine)
             super
           end
 
+          def gz_download(from, to = nil)
+            @logger.debug("Downloading from global zone: #{from} to #{to}")
+
+            if gz_rsync_connector.available?
+              gz_rsync_connector.download(from, to)
+            else
+              gz_scp_connect do |scp|
+                binding.pry
+                scp.download!(from, to) do |ch, name, sent, total|
+                  percent = (sent.to_f / total) * 100
+                  print "#{name}: #{sent}/#{total} : #{percent.to_i}%\r"
+                  $stdout.flush
+                end
+              end
+            end
+          end
+
           # rubocop:disable Metrics/MethodLength
           def gz_execute(command, opts = {}, &block)
             opts = {
@@ -70,6 +88,15 @@ def gz_execute(command, opts = {}, &block)
             exit_status
           end
 
+          def gz_scp_connect
+            global_zone_connector.with_connection do |connection|
+              yield connection.scp
+            end
+          rescue Net::SCP::Error => e
+            raise Vagrant::Errors::SCPUnavailable if e.message =~ /\(127\)/
+            raise
+          end
+
           def gz_test(command, opts = nil)
             opts = { error_check: false }.merge(opts || {})
             gz_execute(command, opts) == 0
@@ -80,6 +107,10 @@ def gz_test(command, opts = nil)
           def global_zone_connector
             @global_zone_connector ||= Vagrant::Smartos::Zones::Util::GlobalZone::Connection.new(@machine, @logger)
           end
+
+          def gz_rsync_connector
+            @gz_rsync_connector ||= Vagrant::Smartos::Zones::Util::Rsync.new(global_zone_connector.connect)
+          end
         end
       end
     end

lib/vagrant/smartos/zones/models/dataset.rb

@@ -8,18 +8,67 @@ module Models
         class Dataset
           include Util::GlobalZone::Helper
 
+          attr_reader :name, :zone, :machine
+
+          def initialize(name, zone)
+            @name = name
+            @zone = zone
+            @machine = zone.machine
+          end
+
           def self.create(name, zone)
+            new(name, zone).create
+          end
+
+          def create
+            create_dataset
+            download
+          end
+
+          def exists?
+            cmd = 'ls %s 2>/dev/null' % remote_filename
+            with_gz(cmd) do |output|
+              return true if output.strip == remote_filename
+            end
+            false
+          end
+
+          private
+
+          def create_dataset
+            return machine.ui.info('Dataset already exists in global zone') if exists?
             Models::Snapshot.around(zone) do
+              machine.ui.info(I18n.t('vagrant.smartos.zones.commands.dataset.prepare_image'))
               zone.zlogin('\'find /var/log -type f -exec truncate --size 0 {} \;\'')
               zone.zlogin('sm-prepare-image -y')
+              machine.ui.info(I18n.t('vagrant.smartos.zones.commands.dataset.stop_zone'))
               zone.stop
               Models::Snapshot.around(zone) do |snapshot|
-                cmd = 'pfexec /usr/bin/bash -l -c "/usr/sbin/zfs send %s | /usr/bin/bzip2 > /zones/%s.zfs.bz2"'
-                with_gz(zone.machine, cmd % [snapshot.path, name])
+                machine.ui.info(I18n.t('vagrant.smartos.zones.commands.dataset.save'))
+                cmd = 'pfexec /usr/bin/bash -l -c "/usr/sbin/zfs send %s | /usr/bin/bzip2 > %s"'
+                with_gz(cmd % [snapshot.path, remote_filename])
               end
             end
+            machine.ui.info(I18n.t('vagrant.smartos.zones.commands.dataset.start_zone'))
             zone.start
           end
+
+          def download
+            machine.ui.info 'Downloading %s' % filename
+            machine.communicate.gz_download(remote_filename, local_filename)
+          end
+
+          def filename
+            '%s.zfs.bz2' % name
+          end
+
+          def local_filename
+            machine.env.home_path.join('smartos', 'datasets', filename).to_s
+          end
+
+          def remote_filename
+            '/zones/%s' % filename
+          end
         end
       end
     end

lib/vagrant/smartos/zones/util/rsync.rb

@@ -0,0 +1,48 @@
+module Vagrant
+  module Smartos
+    module Zones
+      module Util
+        class Rsync
+          attr_reader :connection
+
+          def initialize(connection)
+            @connection = connection
+          end
+
+          def available?
+            Vagrant::Util::Subprocess.execute(*%w'which rsync').exit_code == 0
+          end
+
+          def download(from, to)
+            command = %W'
+              rsync
+              -avz
+              --progress
+            '
+            Vagrant::Util::Subprocess.execute(*command, '-e', ssh_command, remote(from), to, notify: [:stdout])
+          end
+
+          def ssh_command
+            "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i #{identity_file} -p #{remote_port}"
+          end
+
+          def identity_file
+            connection.options[:keys].first
+          end
+
+          def remote(file)
+            "#{remote_user}@#{connection.host}:#{file}"
+          end
+
+          def remote_port
+            connection.options[:port]
+          end
+
+          def remote_user
+            connection.options[:user]
+          end
+        end
+      end
+    end
+  end
+end