From b0b08ff5ad18d97f07aacc1265200caac9c7891b Mon Sep 17 00:00:00 2001 From: Nick LaMuro Date: Sun, 15 Jul 2018 15:09:04 -0500 Subject: [PATCH] Add FTP support to file_splitter.rb Allows for sending both anonymous and authorized FTP uploads directly from the script. Splits are done in memory, so the file never is created on the host machine, which is ideal for large backups and such that are being split up. Spec for testing this functionality are done via a live FTP (vsftpd) server setup with the following config: listen=NO listen_ipv6=YES local_enable=YES local_umask=022 write_enable=YES connect_from_port_20=YES anonymous_enable=YES anon_root=/var/ftp/pub anon_umask=022 anon_upload_enable=YES anon_mkdir_write_enable=YES anon_other_write_enable=YES pam_service_name=vsftpd userlist_enable=YES userlist_deny=NO tcp_wrappers=YES Running on the 192.168.50.3 IP address. Full vagrant setup for the above can be found here: https://gist.github.com/NickLaMuro/c883a997c7ae943dd684bccd469cea43 Put the `Vagrantfile` into a directory and run: $ vagrant up (with `vagrant` installed on that machine, of course) The specs will only run if the user specifically targets the tests using a tag flag: $ bundle exec rspec --tag with_real_ftp --- lib/manageiq/util/file_splitter.rb | 92 +++++++++- spec/lib/manageiq/util/file_splitter_spec.rb | 180 +++++++++++++++++++ 2 files changed, 269 insertions(+), 3 deletions(-) diff --git a/lib/manageiq/util/file_splitter.rb b/lib/manageiq/util/file_splitter.rb index ecdcdf13fce0..2f6f9ebd02d9 100755 --- a/lib/manageiq/util/file_splitter.rb +++ b/lib/manageiq/util/file_splitter.rb @@ -13,11 +13,14 @@ # platform, without having to be concerned with differences in `split` # functionality. +require_relative 'ftp_lib' require 'optparse' module ManageIQ module Util class FileSplitter + include ManageIQ::Util::FtpLib + KILOBYTE = 1024 MEGABYTE = KILOBYTE * 1024 GIGABYTE = MEGABYTE * 1024 @@ -30,6 +33,15 @@ class FileSplitter attr_accessor :input_file, :byte_count + class << self + attr_writer :instance_logger + + # Don't log by default, but allow this to work with FtpLib logging. + def instance_logger + @instance_logger ||= Logger.new(File::NULL) + end + end + def self.run(options = nil) options ||= parse_argv new(options).split @@ -41,6 +53,15 @@ def self.parse_argv opt.on("-b", "--byte-count=BYTES", "Number of bytes for each split") do |bytes| options[:byte_count] = parse_byte_value(bytes) end + opt.on("--ftp-host=HOST", "Host of the FTP server") do |host| + options[:ftp_host] = host + end + opt.on("--ftp-dir=DIR", "Dir on the FTP server to save files") do |dir| + options[:ftp_dir] = dir + end + opt.on("-v", "--verbose", "Turn on logging") do + options[:verbose] = logging + end end.parse! input_file, file_pattern = determine_input_file_and_file_pattern @@ -56,21 +77,86 @@ def initialize(options = {}) @input_filename = options[:input_filename] @byte_count = options[:byte_count] || (10 * MEGABYTE) @position = 0 + + setup_logging(options) + setup_ftp(options) end def split until input_file.eof? - File.open(next_split_filename, "w") do |split_file| - split_file << input_file.read(byte_count) - @position += byte_count + if ftp + split_ftp + else + split_local end + @position += byte_count end ensure input_file.close + ftp.close if ftp end private + def setup_logging(options) + self.class.instance_logger = Logger.new(STDOUT) if options[:verbose] + end + + def setup_ftp(options) + if options[:ftp_host] + @uri = options[:ftp_host] + @ftp_user = options[:ftp_user] || ENV["FTP_USERNAME"] || "anonymous" + @ftp_pass = options[:ftp_pass] || ENV["FTP_PASSWORD"] + @ftp = connect + + @input_filename = File.join(options[:ftp_dir] || "", File.basename(input_filename)) + end + end + + def login_credentials + [@ftp_user, @ftp_pass] + end + + def split_local + File.open(next_split_filename, "w") do |split_file| + split_file << input_file.read(byte_count) + end + end + + # Specific version of Net::FTP#storbinary that doesn't use an existing local + # file, and only uploads a specific size from the input_file + FTP_CHUNKSIZE = ::Net::FTP::DEFAULT_BLOCKSIZE + def split_ftp + ftp_mkdir_p + ftp.synchronize do + ftp.send(:with_binary, true) do + conn = ftp.send(:transfercmd, "STOR #{next_split_filename}") + buf_left = byte_count + while buf_left.positive? + cur_readsize = buf_left - FTP_CHUNKSIZE >= 0 ? FTP_CHUNKSIZE : buf_left + buf = input_file.read(cur_readsize) + break if buf == nil # rubocop:disable Style/NilComparison (from original) + conn.write(buf) + buf_left -= FTP_CHUNKSIZE + end + conn.close + ftp.send(:voidresp) + end + end + rescue Errno::EPIPE + # EPIPE, in this case, means that the data connection was unexpectedly + # terminated. Rather than just raising EPIPE to the caller, check the + # response on the control connection. If getresp doesn't raise a more + # appropriate exception, re-raise the original exception. + getresp + raise + end + + def ftp_mkdir_p + dir_path = File.dirname(input_filename)[1..-1].split('/') - ftp.pwd[1..-1].split("/") + create_directory_structure(dir_path.join('/')) + end + def input_filename @input_filename ||= File.expand_path(input_file.path) end diff --git a/spec/lib/manageiq/util/file_splitter_spec.rb b/spec/lib/manageiq/util/file_splitter_spec.rb index f3e45fb6c3e5..b5443e4f98cd 100644 --- a/spec/lib/manageiq/util/file_splitter_spec.rb +++ b/spec/lib/manageiq/util/file_splitter_spec.rb @@ -2,6 +2,18 @@ require 'pathname' require 'manageiq/util/file_splitter' +# Putting this here since this config option can die with this script, and +# doesn't need to live in the global config. +RSpec.configure do |config| + # These are tests that shouldn't run on CI, but should be semi-automated to + # be triggered manaually to test in an automated fasion. There will be setup + # steps with a vagrant file to spinup a endpoint to use for this. + # + # TODO: Maybe we should just use VCR for this? Still would required the + # vagrant VM I guess to record the tests from... so for now, skipping. + config.filter_run_excluding :with_real_ftp => true +end + module ManageIQ::Util describe FileSplitter do shared_context "generated tmp files" do @@ -22,6 +34,25 @@ module ManageIQ::Util end end + shared_context "ftp context" do + let(:ftp) do + Net::FTP.new(URI(ftp_host).hostname).tap do |ftp| + ftp.login(ftp_creds[:ftp_user], ftp_creds[:ftp_pass]) + end + end + + let(:ftp_dir) { File.join("", "uploads") } + let(:ftp_host) { ENV["FTP_HOST_FOR_SPECS"] || "ftp://192.168.50.3" } + let(:ftp_creds) { { :ftp_user => "anonymous", :ftp_pass => nil } } + let(:ftp_config) { { :ftp_host => ftp_host, :ftp_dir => ftp_dir } } + + let(:ftp_user_1) { ENV["FTP_USER_1_FOR_SPECS"] || "vagrant" } + let(:ftp_pass_1) { ENV["FTP_USER_1_FOR_SPECS"] || "vagrant" } + + let(:ftp_user_2) { ENV["FTP_USER_2_FOR_SPECS"] || "foo" } + let(:ftp_pass_2) { ENV["FTP_USER_2_FOR_SPECS"] || "bar" } + end + describe ".run" do include_context "generated tmp files" @@ -59,6 +90,112 @@ module ManageIQ::Util expect(Pathname.new(filename).lstat.size).to eq(1.megabyte) end end + + context "to an ftp target", :with_real_ftp => true do + include_context "ftp context" + + let(:base_config) { { :byte_count => 1.megabyte, :input_file => File.open(source_path) } } + let(:run_config) { ftp_config.merge(base_config) } + + let(:expected_splitfiles) do + (1..10).map do |suffix| + File.join(ftp_dir, "#{source_path.basename}.000#{'%02d' % suffix}") + end + end + + it "uploads the split files as an annoymous user by default" do + FileSplitter.run(run_config) + + expected_splitfiles.each do |filename| + expect(ftp.nlst(filename)).to eq([filename]) + expect(ftp.size(filename)).to eq(1.megabyte) + ftp.delete(filename) + end + end + + context "with slightly a slightly smaller input file than 10MB" do + let(:tmpfile_size) { 10.megabytes - 1.kilobyte } + + it "properly chunks the file" do + FileSplitter.run(run_config) + + expected_splitfiles[0, 9].each do |filename| + expect(ftp.nlst(filename)).to eq([filename]) + expect(ftp.size(filename)).to eq(1.megabyte) + ftp.delete(filename) + end + + expect(ftp.nlst(expected_splitfiles.last)).to eq([expected_splitfiles.last]) + expect(ftp.size(expected_splitfiles.last)).to eq(1.megabyte - 1.kilobyte) + ftp.delete(expected_splitfiles.last) + end + end + + context "with slightly a slightly larger input file than 10MB" do + let(:tmpfile_size) { 10.megabytes + 1.kilobyte } + + it "properly chunks the file" do + FileSplitter.run(run_config) + + expected_splitfiles.each do |filename| + expect(ftp.nlst(filename)).to eq([filename]) + expect(ftp.size(filename)).to eq(1.megabyte) + ftp.delete(filename) + end + + filename = File.join(ftp_dir, "#{source_path.basename}.00011") + expect(ftp.nlst(filename)).to eq([filename]) + expect(ftp.size(filename)).to eq(1.kilobyte) + ftp.delete(filename) + end + end + + context "with a dir that doesn't exist" do + let(:ftp_dir) { File.join("", "uploads", "backups", "current") } + + it "uploads the split files" do + FileSplitter.run(run_config) + + expected_splitfiles.each do |filename| + expect(ftp.nlst(filename)).to eq([filename]) + expect(ftp.size(filename)).to eq(1.megabyte) + ftp.delete(filename) + end + ftp.rmdir(ftp_dir) + end + end + + context "with a specified user" do + let(:ftp_dir) { File.join("", "home", ftp_user_1) } + let(:ftp_creds) { { :ftp_user => ftp_user_1, :ftp_pass => ftp_pass_1 } } + let(:run_config) { ftp_config.merge(ftp_creds).merge(base_config) } + + it "uploads the split files" do + FileSplitter.run(run_config) + + expected_splitfiles.each do |filename| + expect(ftp.nlst(filename)).to eq([filename]) + expect(ftp.size(filename)).to eq(1.megabyte) + ftp.delete(filename) + end + end + + context "with a dir that doesn't exist" do + let(:ftp_dir) { File.join("", "home", ftp_user_1, "backups") } + + it "uploads the split files" do + FileSplitter.run(run_config) + + expected_splitfiles.each do |filename| + expect(ftp.nlst(filename)).to eq([filename]) + expect(ftp.size(filename)).to eq(1.megabyte) + ftp.delete(filename) + end + ftp.rmdir(ftp_dir) + end + end + end + end end describe ".parse_argv" do @@ -175,6 +312,49 @@ module ManageIQ::Util expect(Pathname.new(filename).lstat.size).to eq(2.megabyte) end end + + context "to a ftp target", :with_real_ftp => true do + include_context "ftp context" + + let(:expected_splitfiles) do + (1..5).map do |suffix| + File.join(ftp_dir, "#{source_path.basename}.000#{'%02d' % suffix}") + end + end + + it "it uploads with an anonymous user by default" do + cmd_opts = "-b 2M --ftp-host=#{ftp_host} --ftp-dir #{ftp_dir}" + `cat #{source_path.expand_path} | #{script_file} #{cmd_opts} - #{source_path.basename}` + + expected_splitfiles.each do |filename| + expect(ftp.nlst(filename)).to eq([filename]) + expect(ftp.size(filename)).to eq(2.megabyte) + ftp.delete(filename) + end + end + + context "with a specified user via ENV vars" do + let(:ftp_dir) { File.join("", "home", ftp_user_1, "backups") } + let(:ftp_creds) { { :ftp_user => ftp_user_1, :ftp_pass => ftp_pass_1 } } + let(:run_config) { ftp_config.merge(ftp_creds).merge(base_config) } + + it "uploads the split files and creates necessary dirs" do + env = { 'FTP_USERNAME' => ftp_user_1, 'FTP_PASSWORD' => ftp_pass_1 } + cmd_opts = "-b 2M --ftp-host=#{ftp_host} --ftp-dir #{ftp_dir}" + cmd = "cat #{source_path.expand_path} | #{script_file} #{cmd_opts} - #{source_path.basename}" + pid = Kernel.spawn(env, cmd) + Process.wait(pid) + + expect($CHILD_STATUS).to eq(0) + expected_splitfiles.each do |filename| + expect(ftp.nlst(filename)).to eq([filename]) + expect(ftp.size(filename)).to eq(2.megabyte) + ftp.delete(filename) + end + ftp.rmdir(ftp_dir) + end + end + end end end end