diff --git a/lib/manageiq/util/file_splitter.rb b/lib/manageiq/util/file_splitter.rb index ecdcdf13fce0..2f6f9ebd02d9 100755 --- a/lib/manageiq/util/file_splitter.rb +++ b/lib/manageiq/util/file_splitter.rb @@ -13,11 +13,14 @@ # platform, without having to be concerned with differences in `split` # functionality. +require_relative 'ftp_lib' require 'optparse' module ManageIQ module Util class FileSplitter + include ManageIQ::Util::FtpLib + KILOBYTE = 1024 MEGABYTE = KILOBYTE * 1024 GIGABYTE = MEGABYTE * 1024 @@ -30,6 +33,15 @@ class FileSplitter attr_accessor :input_file, :byte_count + class << self + attr_writer :instance_logger + + # Don't log by default, but allow this to work with FtpLib logging. + def instance_logger + @instance_logger ||= Logger.new(File::NULL) + end + end + def self.run(options = nil) options ||= parse_argv new(options).split @@ -41,6 +53,15 @@ def self.parse_argv opt.on("-b", "--byte-count=BYTES", "Number of bytes for each split") do |bytes| options[:byte_count] = parse_byte_value(bytes) end + opt.on("--ftp-host=HOST", "Host of the FTP server") do |host| + options[:ftp_host] = host + end + opt.on("--ftp-dir=DIR", "Dir on the FTP server to save files") do |dir| + options[:ftp_dir] = dir + end + opt.on("-v", "--verbose", "Turn on logging") do + options[:verbose] = logging + end end.parse! input_file, file_pattern = determine_input_file_and_file_pattern @@ -56,21 +77,86 @@ def initialize(options = {}) @input_filename = options[:input_filename] @byte_count = options[:byte_count] || (10 * MEGABYTE) @position = 0 + + setup_logging(options) + setup_ftp(options) end def split until input_file.eof? - File.open(next_split_filename, "w") do |split_file| - split_file << input_file.read(byte_count) - @position += byte_count + if ftp + split_ftp + else + split_local end + @position += byte_count end ensure input_file.close + ftp.close if ftp end private + def setup_logging(options) + self.class.instance_logger = Logger.new(STDOUT) if options[:verbose] + end + + def setup_ftp(options) + if options[:ftp_host] + @uri = options[:ftp_host] + @ftp_user = options[:ftp_user] || ENV["FTP_USERNAME"] || "anonymous" + @ftp_pass = options[:ftp_pass] || ENV["FTP_PASSWORD"] + @ftp = connect + + @input_filename = File.join(options[:ftp_dir] || "", File.basename(input_filename)) + end + end + + def login_credentials + [@ftp_user, @ftp_pass] + end + + def split_local + File.open(next_split_filename, "w") do |split_file| + split_file << input_file.read(byte_count) + end + end + + # Specific version of Net::FTP#storbinary that doesn't use an existing local + # file, and only uploads a specific size from the input_file + FTP_CHUNKSIZE = ::Net::FTP::DEFAULT_BLOCKSIZE + def split_ftp + ftp_mkdir_p + ftp.synchronize do + ftp.send(:with_binary, true) do + conn = ftp.send(:transfercmd, "STOR #{next_split_filename}") + buf_left = byte_count + while buf_left.positive? + cur_readsize = buf_left - FTP_CHUNKSIZE >= 0 ? FTP_CHUNKSIZE : buf_left + buf = input_file.read(cur_readsize) + break if buf == nil # rubocop:disable Style/NilComparison (from original) + conn.write(buf) + buf_left -= FTP_CHUNKSIZE + end + conn.close + ftp.send(:voidresp) + end + end + rescue Errno::EPIPE + # EPIPE, in this case, means that the data connection was unexpectedly + # terminated. Rather than just raising EPIPE to the caller, check the + # response on the control connection. If getresp doesn't raise a more + # appropriate exception, re-raise the original exception. + getresp + raise + end + + def ftp_mkdir_p + dir_path = File.dirname(input_filename)[1..-1].split('/') - ftp.pwd[1..-1].split("/") + create_directory_structure(dir_path.join('/')) + end + def input_filename @input_filename ||= File.expand_path(input_file.path) end diff --git a/spec/lib/manageiq/util/file_splitter_spec.rb b/spec/lib/manageiq/util/file_splitter_spec.rb index f3e45fb6c3e5..b5443e4f98cd 100644 --- a/spec/lib/manageiq/util/file_splitter_spec.rb +++ b/spec/lib/manageiq/util/file_splitter_spec.rb @@ -2,6 +2,18 @@ require 'pathname' require 'manageiq/util/file_splitter' +# Putting this here since this config option can die with this script, and +# doesn't need to live in the global config. +RSpec.configure do |config| + # These are tests that shouldn't run on CI, but should be semi-automated to + # be triggered manaually to test in an automated fasion. There will be setup + # steps with a vagrant file to spinup a endpoint to use for this. + # + # TODO: Maybe we should just use VCR for this? Still would required the + # vagrant VM I guess to record the tests from... so for now, skipping. + config.filter_run_excluding :with_real_ftp => true +end + module ManageIQ::Util describe FileSplitter do shared_context "generated tmp files" do @@ -22,6 +34,25 @@ module ManageIQ::Util end end + shared_context "ftp context" do + let(:ftp) do + Net::FTP.new(URI(ftp_host).hostname).tap do |ftp| + ftp.login(ftp_creds[:ftp_user], ftp_creds[:ftp_pass]) + end + end + + let(:ftp_dir) { File.join("", "uploads") } + let(:ftp_host) { ENV["FTP_HOST_FOR_SPECS"] || "ftp://192.168.50.3" } + let(:ftp_creds) { { :ftp_user => "anonymous", :ftp_pass => nil } } + let(:ftp_config) { { :ftp_host => ftp_host, :ftp_dir => ftp_dir } } + + let(:ftp_user_1) { ENV["FTP_USER_1_FOR_SPECS"] || "vagrant" } + let(:ftp_pass_1) { ENV["FTP_USER_1_FOR_SPECS"] || "vagrant" } + + let(:ftp_user_2) { ENV["FTP_USER_2_FOR_SPECS"] || "foo" } + let(:ftp_pass_2) { ENV["FTP_USER_2_FOR_SPECS"] || "bar" } + end + describe ".run" do include_context "generated tmp files" @@ -59,6 +90,112 @@ module ManageIQ::Util expect(Pathname.new(filename).lstat.size).to eq(1.megabyte) end end + + context "to an ftp target", :with_real_ftp => true do + include_context "ftp context" + + let(:base_config) { { :byte_count => 1.megabyte, :input_file => File.open(source_path) } } + let(:run_config) { ftp_config.merge(base_config) } + + let(:expected_splitfiles) do + (1..10).map do |suffix| + File.join(ftp_dir, "#{source_path.basename}.000#{'%02d' % suffix}") + end + end + + it "uploads the split files as an annoymous user by default" do + FileSplitter.run(run_config) + + expected_splitfiles.each do |filename| + expect(ftp.nlst(filename)).to eq([filename]) + expect(ftp.size(filename)).to eq(1.megabyte) + ftp.delete(filename) + end + end + + context "with slightly a slightly smaller input file than 10MB" do + let(:tmpfile_size) { 10.megabytes - 1.kilobyte } + + it "properly chunks the file" do + FileSplitter.run(run_config) + + expected_splitfiles[0, 9].each do |filename| + expect(ftp.nlst(filename)).to eq([filename]) + expect(ftp.size(filename)).to eq(1.megabyte) + ftp.delete(filename) + end + + expect(ftp.nlst(expected_splitfiles.last)).to eq([expected_splitfiles.last]) + expect(ftp.size(expected_splitfiles.last)).to eq(1.megabyte - 1.kilobyte) + ftp.delete(expected_splitfiles.last) + end + end + + context "with slightly a slightly larger input file than 10MB" do + let(:tmpfile_size) { 10.megabytes + 1.kilobyte } + + it "properly chunks the file" do + FileSplitter.run(run_config) + + expected_splitfiles.each do |filename| + expect(ftp.nlst(filename)).to eq([filename]) + expect(ftp.size(filename)).to eq(1.megabyte) + ftp.delete(filename) + end + + filename = File.join(ftp_dir, "#{source_path.basename}.00011") + expect(ftp.nlst(filename)).to eq([filename]) + expect(ftp.size(filename)).to eq(1.kilobyte) + ftp.delete(filename) + end + end + + context "with a dir that doesn't exist" do + let(:ftp_dir) { File.join("", "uploads", "backups", "current") } + + it "uploads the split files" do + FileSplitter.run(run_config) + + expected_splitfiles.each do |filename| + expect(ftp.nlst(filename)).to eq([filename]) + expect(ftp.size(filename)).to eq(1.megabyte) + ftp.delete(filename) + end + ftp.rmdir(ftp_dir) + end + end + + context "with a specified user" do + let(:ftp_dir) { File.join("", "home", ftp_user_1) } + let(:ftp_creds) { { :ftp_user => ftp_user_1, :ftp_pass => ftp_pass_1 } } + let(:run_config) { ftp_config.merge(ftp_creds).merge(base_config) } + + it "uploads the split files" do + FileSplitter.run(run_config) + + expected_splitfiles.each do |filename| + expect(ftp.nlst(filename)).to eq([filename]) + expect(ftp.size(filename)).to eq(1.megabyte) + ftp.delete(filename) + end + end + + context "with a dir that doesn't exist" do + let(:ftp_dir) { File.join("", "home", ftp_user_1, "backups") } + + it "uploads the split files" do + FileSplitter.run(run_config) + + expected_splitfiles.each do |filename| + expect(ftp.nlst(filename)).to eq([filename]) + expect(ftp.size(filename)).to eq(1.megabyte) + ftp.delete(filename) + end + ftp.rmdir(ftp_dir) + end + end + end + end end describe ".parse_argv" do @@ -175,6 +312,49 @@ module ManageIQ::Util expect(Pathname.new(filename).lstat.size).to eq(2.megabyte) end end + + context "to a ftp target", :with_real_ftp => true do + include_context "ftp context" + + let(:expected_splitfiles) do + (1..5).map do |suffix| + File.join(ftp_dir, "#{source_path.basename}.000#{'%02d' % suffix}") + end + end + + it "it uploads with an anonymous user by default" do + cmd_opts = "-b 2M --ftp-host=#{ftp_host} --ftp-dir #{ftp_dir}" + `cat #{source_path.expand_path} | #{script_file} #{cmd_opts} - #{source_path.basename}` + + expected_splitfiles.each do |filename| + expect(ftp.nlst(filename)).to eq([filename]) + expect(ftp.size(filename)).to eq(2.megabyte) + ftp.delete(filename) + end + end + + context "with a specified user via ENV vars" do + let(:ftp_dir) { File.join("", "home", ftp_user_1, "backups") } + let(:ftp_creds) { { :ftp_user => ftp_user_1, :ftp_pass => ftp_pass_1 } } + let(:run_config) { ftp_config.merge(ftp_creds).merge(base_config) } + + it "uploads the split files and creates necessary dirs" do + env = { 'FTP_USERNAME' => ftp_user_1, 'FTP_PASSWORD' => ftp_pass_1 } + cmd_opts = "-b 2M --ftp-host=#{ftp_host} --ftp-dir #{ftp_dir}" + cmd = "cat #{source_path.expand_path} | #{script_file} #{cmd_opts} - #{source_path.basename}" + pid = Kernel.spawn(env, cmd) + Process.wait(pid) + + expect($CHILD_STATUS).to eq(0) + expected_splitfiles.each do |filename| + expect(ftp.nlst(filename)).to eq([filename]) + expect(ftp.size(filename)).to eq(2.megabyte) + ftp.delete(filename) + end + ftp.rmdir(ftp_dir) + end + end + end end end end