Add FTP support to file_splitter.rb

Allows for sending both anonymous and authorized FTP uploads directly
from the script.  Splits are done in memory, so the file never is
created on the host machine, which is ideal for large backups and such
that are being split up.

Spec for testing this functionality are done via a live FTP (vsftpd)
server setup with the following config:

    listen=NO
    listen_ipv6=YES

    local_enable=YES
    local_umask=022
    write_enable=YES
    connect_from_port_20=YES

    anonymous_enable=YES
    anon_root=/var/ftp/pub
    anon_umask=022
    anon_upload_enable=YES
    anon_mkdir_write_enable=YES
    anon_other_write_enable=YES

    pam_service_name=vsftpd
    userlist_enable=YES
    userlist_deny=NO
    tcp_wrappers=YES

Running on the 192.168.50.3 IP address.  Full vagrant setup for the
above can be found here:

    https://gist.github.com/NickLaMuro/c883a997c7ae943dd684bccd469cea43

Put the `Vagrantfile` into a directory and run:

    $ vagrant up

(with `vagrant` installed on that machine, of course)

The specs will only run if the user specifically targets the tests using
a tag flag:

    $ bundle exec rspec --tag with_real_ftp

lib/manageiq/util/file_splitter.rb

@@ -13,11 +13,14 @@
 # platform, without having to be concerned with differences in `split`
 # functionality.
 
+require_relative 'ftp_lib'
 require 'optparse'
 
 module ManageIQ
   module Util
     class FileSplitter
+      include ManageIQ::Util::FtpLib
+
       KILOBYTE = 1024
       MEGABYTE = KILOBYTE * 1024
       GIGABYTE = MEGABYTE * 1024
@@ -30,6 +33,15 @@ class FileSplitter
 
       attr_accessor :input_file, :byte_count
 
+      class << self
+        attr_writer :instance_logger
+
+        # Don't log by default, but allow this to work with FtpLib logging.
+        def instance_logger
+          @instance_logger ||= Logger.new(File::NULL)
+        end
+      end
+
       def self.run(options = nil)
         options ||= parse_argv
         new(options).split
@@ -41,6 +53,15 @@ def self.parse_argv
           opt.on("-b", "--byte-count=BYTES", "Number of bytes for each split") do |bytes|
             options[:byte_count] = parse_byte_value(bytes)
           end
+          opt.on("--ftp-host=HOST", "Host of the FTP server") do |host|
+            options[:ftp_host] = host
+          end
+          opt.on("--ftp-dir=DIR", "Dir on the FTP server to save files") do |dir|
+            options[:ftp_dir] = dir
+          end
+          opt.on("-v", "--verbose", "Turn on logging") do
+            options[:verbose] = logging
+          end
         end.parse!
 
         input_file, file_pattern = determine_input_file_and_file_pattern
@@ -56,21 +77,86 @@ def initialize(options = {})
         @input_filename = options[:input_filename]
         @byte_count     = options[:byte_count] || (10 * MEGABYTE)
         @position       = 0
+
+        setup_logging(options)
+        setup_ftp(options)
       end
 
       def split
         until input_file.eof?
-          File.open(next_split_filename, "w") do |split_file|
-            split_file << input_file.read(byte_count)
-            @position += byte_count
+          if ftp
+            split_ftp
+          else
+            split_local
           end
+          @position += byte_count
         end
       ensure
         input_file.close
+        ftp.close if ftp
       end
 
       private
 
+      def setup_logging(options)
+        self.class.instance_logger = Logger.new(STDOUT) if options[:verbose]
+      end
+
+      def setup_ftp(options)
+        if options[:ftp_host]
+          @uri      = options[:ftp_host]
+          @ftp_user = options[:ftp_user] || ENV["FTP_USERNAME"] || "anonymous"
+          @ftp_pass = options[:ftp_pass] || ENV["FTP_PASSWORD"]
+          @ftp      = connect
+
+          @input_filename = File.join(options[:ftp_dir] || "", File.basename(input_filename))
+        end
+      end
+
+      def login_credentials
+        [@ftp_user, @ftp_pass]
+      end
+
+      def split_local
+        File.open(next_split_filename, "w") do |split_file|
+          split_file << input_file.read(byte_count)
+        end
+      end
+
+      # Specific version of Net::FTP#storbinary that doesn't use an existing local
+      # file, and only uploads a specific size from the input_file
+      FTP_CHUNKSIZE = ::Net::FTP::DEFAULT_BLOCKSIZE
+      def split_ftp
+        ftp_mkdir_p
+        ftp.synchronize do
+          ftp.send(:with_binary, true) do
+            conn         = ftp.send(:transfercmd, "STOR #{next_split_filename}")
+            buf_left     = byte_count
+            while buf_left.positive?
+              cur_readsize = buf_left - FTP_CHUNKSIZE >= 0 ? FTP_CHUNKSIZE : buf_left
+              buf = input_file.read(cur_readsize)
+              break if buf == nil # rubocop:disable Style/NilComparison (from original)
+              conn.write(buf)
+              buf_left -= FTP_CHUNKSIZE
+            end
+            conn.close
+            ftp.send(:voidresp)
+          end
+        end
+      rescue Errno::EPIPE
+        # EPIPE, in this case, means that the data connection was unexpectedly
+        # terminated.  Rather than just raising EPIPE to the caller, check the
+        # response on the control connection.  If getresp doesn't raise a more
+        # appropriate exception, re-raise the original exception.
+        getresp
+        raise
+      end
+
+      def ftp_mkdir_p
+        dir_path = File.dirname(input_filename)[1..-1].split('/') - ftp.pwd[1..-1].split("/")
+        create_directory_structure(dir_path.join('/'))
+      end
+
       def input_filename
         @input_filename ||= File.expand_path(input_file.path)
       end

spec/lib/manageiq/util/file_splitter_spec.rb

@@ -2,6 +2,18 @@
 require 'pathname'
 require 'manageiq/util/file_splitter'
 
+# Putting this here since this config option can die with this script, and
+# doesn't need to live in the global config.
+RSpec.configure do |config|
+  # These are tests that shouldn't run on CI, but should be semi-automated to
+  # be triggered manaually to test in an automated fasion.  There will be setup
+  # steps with a vagrant file to spinup a endpoint to use for this.
+  #
+  # TODO:  Maybe we should just use VCR for this?  Still would required the
+  # vagrant VM I guess to record the tests from... so for now, skipping.
+  config.filter_run_excluding :with_real_ftp => true
+end
+
 module ManageIQ::Util
   describe FileSplitter do
     shared_context "generated tmp files" do
@@ -22,6 +34,25 @@ module ManageIQ::Util
       end
     end
 
+    shared_context "ftp context" do
+      let(:ftp) do
+        Net::FTP.new(URI(ftp_host).hostname).tap do |ftp|
+          ftp.login(ftp_creds[:ftp_user], ftp_creds[:ftp_pass])
+        end
+      end
+
+      let(:ftp_dir)    { File.join("", "uploads") }
+      let(:ftp_host)   { ENV["FTP_HOST_FOR_SPECS"] || "ftp://192.168.50.3" }
+      let(:ftp_creds)  { { :ftp_user => "anonymous", :ftp_pass => nil } }
+      let(:ftp_config) { { :ftp_host => ftp_host, :ftp_dir => ftp_dir } }
+
+      let(:ftp_user_1) { ENV["FTP_USER_1_FOR_SPECS"] || "vagrant" }
+      let(:ftp_pass_1) { ENV["FTP_USER_1_FOR_SPECS"] || "vagrant" }
+
+      let(:ftp_user_2) { ENV["FTP_USER_2_FOR_SPECS"] || "foo" }
+      let(:ftp_pass_2) { ENV["FTP_USER_2_FOR_SPECS"] || "bar" }
+    end
+
     describe ".run" do
       include_context "generated tmp files"
 
@@ -59,6 +90,112 @@ module ManageIQ::Util
           expect(Pathname.new(filename).lstat.size).to eq(1.megabyte)
         end
       end
+
+      context "to an ftp target", :with_real_ftp => true do
+        include_context "ftp context"
+
+        let(:base_config) { { :byte_count => 1.megabyte, :input_file => File.open(source_path) } }
+        let(:run_config)  { ftp_config.merge(base_config) }
+
+        let(:expected_splitfiles) do
+          (1..10).map do |suffix|
+            File.join(ftp_dir, "#{source_path.basename}.000#{'%02d' % suffix}")
+          end
+        end
+
+        it "uploads the split files as an annoymous user by default" do
+          FileSplitter.run(run_config)
+
+          expected_splitfiles.each do |filename|
+            expect(ftp.nlst(filename)).to eq([filename])
+            expect(ftp.size(filename)).to eq(1.megabyte)
+            ftp.delete(filename)
+          end
+        end
+
+        context "with slightly a slightly smaller input file than 10MB" do
+          let(:tmpfile_size) { 10.megabytes - 1.kilobyte }
+
+          it "properly chunks the file" do
+            FileSplitter.run(run_config)
+
+            expected_splitfiles[0, 9].each do |filename|
+              expect(ftp.nlst(filename)).to eq([filename])
+              expect(ftp.size(filename)).to eq(1.megabyte)
+              ftp.delete(filename)
+            end
+
+            expect(ftp.nlst(expected_splitfiles.last)).to eq([expected_splitfiles.last])
+            expect(ftp.size(expected_splitfiles.last)).to eq(1.megabyte - 1.kilobyte)
+            ftp.delete(expected_splitfiles.last)
+          end
+        end
+
+        context "with slightly a slightly larger input file than 10MB" do
+          let(:tmpfile_size) { 10.megabytes + 1.kilobyte }
+
+          it "properly chunks the file" do
+            FileSplitter.run(run_config)
+
+            expected_splitfiles.each do |filename|
+              expect(ftp.nlst(filename)).to eq([filename])
+              expect(ftp.size(filename)).to eq(1.megabyte)
+              ftp.delete(filename)
+            end
+
+            filename = File.join(ftp_dir, "#{source_path.basename}.00011")
+            expect(ftp.nlst(filename)).to eq([filename])
+            expect(ftp.size(filename)).to eq(1.kilobyte)
+            ftp.delete(filename)
+          end
+        end
+
+        context "with a dir that doesn't exist" do
+          let(:ftp_dir) { File.join("", "uploads", "backups", "current") }
+
+          it "uploads the split files" do
+            FileSplitter.run(run_config)
+
+            expected_splitfiles.each do |filename|
+              expect(ftp.nlst(filename)).to eq([filename])
+              expect(ftp.size(filename)).to eq(1.megabyte)
+              ftp.delete(filename)
+            end
+            ftp.rmdir(ftp_dir)
+          end
+        end
+
+        context "with a specified user" do
+          let(:ftp_dir)    { File.join("", "home", ftp_user_1) }
+          let(:ftp_creds)  { { :ftp_user => ftp_user_1, :ftp_pass => ftp_pass_1 } }
+          let(:run_config) { ftp_config.merge(ftp_creds).merge(base_config) }
+
+          it "uploads the split files" do
+            FileSplitter.run(run_config)
+
+            expected_splitfiles.each do |filename|
+              expect(ftp.nlst(filename)).to eq([filename])
+              expect(ftp.size(filename)).to eq(1.megabyte)
+              ftp.delete(filename)
+            end
+          end
+
+          context "with a dir that doesn't exist" do
+            let(:ftp_dir) { File.join("", "home", ftp_user_1, "backups") }
+
+            it "uploads the split files" do
+              FileSplitter.run(run_config)
+
+              expected_splitfiles.each do |filename|
+                expect(ftp.nlst(filename)).to eq([filename])
+                expect(ftp.size(filename)).to eq(1.megabyte)
+                ftp.delete(filename)
+              end
+              ftp.rmdir(ftp_dir)
+            end
+          end
+        end
+      end
     end
 
     describe ".parse_argv" do
@@ -175,6 +312,49 @@ module ManageIQ::Util
           expect(Pathname.new(filename).lstat.size).to eq(2.megabyte)
         end
       end
+
+      context "to a ftp target", :with_real_ftp => true do
+        include_context "ftp context"
+
+        let(:expected_splitfiles) do
+          (1..5).map do |suffix|
+            File.join(ftp_dir, "#{source_path.basename}.000#{'%02d' % suffix}")
+          end
+        end
+
+        it "it uploads with an anonymous user by default" do
+          cmd_opts = "-b 2M --ftp-host=#{ftp_host} --ftp-dir #{ftp_dir}"
+          `cat #{source_path.expand_path} | #{script_file} #{cmd_opts} - #{source_path.basename}`
+
+          expected_splitfiles.each do |filename|
+            expect(ftp.nlst(filename)).to eq([filename])
+            expect(ftp.size(filename)).to eq(2.megabyte)
+            ftp.delete(filename)
+          end
+        end
+
+        context "with a specified user via ENV vars" do
+          let(:ftp_dir)    { File.join("", "home", ftp_user_1, "backups") }
+          let(:ftp_creds)  { { :ftp_user => ftp_user_1, :ftp_pass => ftp_pass_1 } }
+          let(:run_config) { ftp_config.merge(ftp_creds).merge(base_config) }
+
+          it "uploads the split files and creates necessary dirs" do
+            env      = { 'FTP_USERNAME' => ftp_user_1, 'FTP_PASSWORD' => ftp_pass_1 }
+            cmd_opts = "-b 2M --ftp-host=#{ftp_host} --ftp-dir #{ftp_dir}"
+            cmd      = "cat #{source_path.expand_path} | #{script_file} #{cmd_opts} - #{source_path.basename}"
+            pid      = Kernel.spawn(env, cmd)
+            Process.wait(pid)
+
+            expect($CHILD_STATUS).to eq(0)
+            expected_splitfiles.each do |filename|
+              expect(ftp.nlst(filename)).to eq([filename])
+              expect(ftp.size(filename)).to eq(2.megabyte)
+              ftp.delete(filename)
+            end
+            ftp.rmdir(ftp_dir)
+          end
+        end
+      end
     end
   end
 end