Merge pull request #386 from aws/1.7.x

1.7.x

bin/install

@@ -71,12 +71,24 @@ class IMDS
   BASE_PATH = '/latest/meta-data'
   IDENTITY_DOCUMENT_PATH = '/latest/dynamic/instance-identity/document'
   DOMAIN_PATH = '/latest/meta-data/services/domain'
+  HTTP_MAX_RETRY_COUNT = 2
 
-  def self.imds_supported?
+  attr :disable_imds_v1
+  attr :log
+  def initialize(disable_imds_v1, log)
+    @disable_imds_v1 = disable_imds_v1
+    @log = log
+  end
+
+  def imds_supported?
     imds_v2? || imds_v1?
   end
 
-  def self.imds_v1?
+  def disable_imds_v1?
+    !!@disable_imds_v1
+  end
+
+  def imds_v1?
     begin
       get_request(BASE_PATH) { |response|
         return response.kind_of? Net::HTTPSuccess
@@ -86,72 +98,90 @@ class IMDS
     end
   end
 
-  def self.imds_v2?
+  def imds_v2?
     begin
-      put_request(TOKEN_PATH) { |token_response|
-        (token_response.kind_of? Net::HTTPSuccess) && get_request(BASE_PATH, token_response.body) { |response|
-          return response.kind_of? Net::HTTPSuccess
-        }
+      token = get_imds_v2_token(TOKEN_PATH)
+      get_request(BASE_PATH, token) { |response|
+        return response.kind_of? Net::HTTPSuccess
       }
     rescue
       false
     end
   end
 
-  def self.region
+  def region
     begin
       identity_document()['region'].strip
     rescue
       nil
     end
   end
 
-  def self.domain
+  def domain
     begin
       get_instance_metadata(DOMAIN_PATH).strip
     rescue
       nil
     end
   end
 
-  def self.identity_document
+  def identity_document
     # JSON is lazy loaded to ensure we dont break older ruby runtimes
     require 'json'
     JSON.parse(get_instance_metadata(IDENTITY_DOCUMENT_PATH).strip)
   end
 
   private
-  def self.get_instance_metadata(path)
+  def get_instance_metadata(path)
     begin
-      token = put_request(TOKEN_PATH)
-      get_request(path, token)
+      token = get_imds_v2_token(TOKEN_PATH)
     rescue
-      get_request(path)
+      if disable_imds_v1?
+        @log.error('HTTP error from metadata service to get imdsv2 token.')
+        raise 'HTTP error from metadata service to get imdsv2 token.'
+      end
+      @log.warn('IMDSv2 http request failed, falling back to IMDSv1.')
+      return get_request(path)
     end
+    get_request(path, token)
   end
 
+  private
+  def get_imds_v2_token(path)
+    @current_imds_v2_token ||= put_request(path)
+  end
 
   private
-  def self.http_request(request)
-    Net::HTTP.start(IP_ADDRESS, 80, :read_timeout => 10, :open_timeout => 10) do |http|
-      response = http.request(request)
-      if block_given?
-        yield(response)
-      elsif response.kind_of? Net::HTTPSuccess
-        response.body
+  def http_request(request)
+    retry_interval_in_sec = [1, 2]
+    begin
+      Net::HTTP.start(IP_ADDRESS, 80, :read_timeout => 10, :open_timeout => 10, :max_retries => HTTP_MAX_RETRY_COUNT) do |http|
+        response = http.request(request)
+        if block_given?
+          yield(response)
+        elsif response.kind_of? Net::HTTPSuccess
+          response.body
+        else
+          raise "HTTP error from metadata service: #{response.message}, code #{response.code}"
+        end
+      end
+    rescue
+      if delay = retry_interval_in_sec.shift # will be nil if the list is empty
+        sleep delay
+        retry # backs up to just after the "begin"
       else
-        raise "HTTP error from metadata service: #{response.message}, code #{response.code}"
+        raise # with no args re-raises original error
       end
     end
   end
 
-  def self.put_request(path, &block)
+  def put_request(path, &block)
     request = Net::HTTP::Put.new(path)
     request['X-aws-ec2-metadata-token-ttl-seconds'] = '21600'
     http_request(request, &block)
   end
 
-  def self.get_request(path, token = nil, &block)
+  def get_request(path, token = nil, &block)
     request = Net::HTTP::Get.new(path)
     unless token.nil?
       request['X-aws-ec2-metadata-token'] = token
@@ -187,8 +217,9 @@ begin
   def usage
     print <<EOF
 
-install [--sanity-check] [--proxy http://hostname:port] <package-type>
+install [--sanity-check] [--disable-imds-v1] [--proxy http://hostname:port] <package-type>
    --sanity-check [optional]
+   --disable-imds-v1 [optional]
    --proxy [optional]
    package-type: 'rpm', 'deb', or 'auto'
 
@@ -208,6 +239,9 @@ the gdebi will be installed using apt-get first.
 If --sanity-check is specified, the install script will wait for 3 minutes post installation
 to check for a running agent.
 
+If --disable-imds-v1 is specified, the install script will not fallback to IMDS v1 call when
+IMDS v2 call is failed
+
 To use a HTTP proxy, specify --proxy followed by the proxy server
 defined by http://hostname:port
 
@@ -267,13 +301,15 @@ EOF
     end
 
     @sanity_check = false
+    @disable_imds_v1 = false
     @reexeced = false
     @http_proxy = nil
     @target_version_arg = nil
 
     @args = Array.new(ARGV)
     opts = GetoptLong.new(
       ['--sanity-check', GetoptLong::NO_ARGUMENT],
+      ['--disable-imds-v1', GetoptLong::NO_ARGUMENT],
       ['--help', GetoptLong::NO_ARGUMENT],
       ['--re-execed', GetoptLong::NO_ARGUMENT],
       ['--proxy', GetoptLong::OPTIONAL_ARGUMENT],
@@ -283,6 +319,8 @@ EOF
       case opt
       when '--sanity-check'
         @sanity_check = true
+      when '--disable-imds-v1'
+        @disable_imds_v1 = true
       when '--help'
         usage
         exit(0)
@@ -347,9 +385,10 @@ EOF
   end
 
   def get_ec2_metadata_property(property)
-    if IMDS.imds_supported?
+    imds = IMDS.new(@disable_imds_v1, @log)
+    if imds.imds_supported?
       begin
-        return IMDS.send(property)
+        return imds.send(property)
       rescue => error
         @log.warn("Could not get #{property} from EC2 metadata service at '#{error.message}'")
       end
@@ -559,7 +598,7 @@ EOF
     else
       #use -n for non-interactive mode
       #use -o to not overwrite config files unless they have not been changed
-      install_cmd = ['/usr/bin/gdebi', '-n', '-o', 'Dpkg::Options::=--force-confdef', '-o', 'Dkpg::Options::=--force-confold']
+      install_cmd = ['/usr/bin/gdebi', '-n', '-o', 'Dpkg::Options::=--force-confdef', '-o', 'Dpkg::Options::=--force-confold']
       install_from_s3(s3_bucket, target_version, install_cmd)
       do_sanity_check('/usr/sbin/service')
     end

codedeploy_agent.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = 'aws_codedeploy_agent'
-  spec.version       = '1.6.0'
+  spec.version       = '1.7.0'
   spec.summary       = 'Packages AWS CodeDeploy agent libraries'
   spec.description   = 'AWS CodeDeploy agent is responsible for doing the actual work of deploying software on an individual EC2 instance'
   spec.author        = 'Amazon Web Services'
@@ -11,7 +11,7 @@ Gem::Specification.new do |spec|
   spec.license        = 'Apache-2.0'
   spec.required_ruby_version = '>= 2.7.0'
 
-  spec.add_dependency('gli', '~> 2.5')
+  spec.add_dependency('gli', '~> 2.21')
   spec.add_dependency('json_pure', '~> 1.6')
   spec.add_dependency('archive-tar-minitar', '~> 0.5.2')
   spec.add_dependency('rubyzip', '~> 1.3.0')

conf/codedeployagent.yml

@@ -7,4 +7,5 @@
 :verbose: false
 :wait_between_runs: 1
 :proxy_uri:
-:max_revisions: 5
+:max_revisions: 5
+:disable_imds_v1: false

lib/instance_agent/config.rb

@@ -62,6 +62,18 @@ def validate_use_fips_mode errors
     def region 
       ENV['AWS_REGION'] || InstanceMetadata.region
     end
-
+
+    # @param opts [Hash]
+    # @return [Hash]
+    def self.common_client_config(opts = {})
+      # See: https://github.com/aws/aws-sdk-ruby/blob/3136ed9f74da28e9d742b1f5b3a2d5abda28ecba/gems/aws-sdk-core/lib/aws-sdk-core/credential_provider_chain.rb#L36-L38
+      #
+      # Latest ruby sdk may have better default behavior for imds retry, but we are pinned to an old version to support ruby 2.0. So, we need
+      # to configure for more imds retries manually.
+      {
+        :instance_profile_credentials_retries => 3,
+        :instance_profile_credentials_timeout => 1,
+      }.merge(opts)
+    end
   end
 end

lib/instance_agent/file_credentials.rb

@@ -17,6 +17,8 @@ def refresh
       @credentials = Aws::SharedCredentials.new(path: @path).credentials
       raise "Failed to load credentials from path #{@path}" if @credentials.nil?
       @expiration = Time.new + 1800
+    rescue Aws::Errors::NoSuchProfileError
+      raise "Failed to load credentials from path #{@path}"
     end
   end
 end

lib/instance_agent/platform/linux_util.rb

@@ -43,9 +43,9 @@ def self.extract_tar(bundle_file, dst)
       FileUtils.mkdir_p(dst)
       working_dir = FileUtils.pwd()
       absolute_bundle_path = File.expand_path(bundle_file)
-      FileUtils.cd(dst)
-      execute_tar_command("/bin/tar -xpsf #{absolute_bundle_path}")
-      FileUtils.cd(working_dir)
+      FileUtils.cd(dst) do
+        execute_tar_command("/bin/tar -xpsf #{absolute_bundle_path}")
+      end
     end
 
     def self.extract_zip(bundle_file, dst)
@@ -60,9 +60,9 @@ def self.extract_tgz(bundle_file, dst)
       FileUtils.mkdir_p(dst)
       working_dir = FileUtils.pwd()
       absolute_bundle_path = File.expand_path(bundle_file)
-      FileUtils.cd(dst)
-      execute_tar_command("/bin/tar -zxpsf #{absolute_bundle_path}")
-      FileUtils.cd(working_dir)
+      FileUtils.cd(dst) do 
+        execute_tar_command("/bin/tar -zxpsf #{absolute_bundle_path}")
+      end
     end
 
     def self.supports_process_groups?()

lib/instance_agent/plugins/codedeploy/codedeploy_control.rb

@@ -12,6 +12,8 @@ def initialize(options = {})
             :http_read_timeout => InstanceAgent::Config.config[:http_read_timeout]
           })
 
+          @options = options.update(InstanceAgent::Config.common_client_config)
+
           if InstanceAgent::Config.config[:log_aws_wire]
             @options = options.update({
               # wire logs might be huge; customers should be careful about turning them on

lib/instance_agent/plugins/codedeploy/command_executor.rb

@@ -325,6 +325,7 @@ def most_recent_install_file_path(deployment_group)
         def download_from_s3(deployment_spec, bucket, key, version, etag)
           log(:info, "Downloading artifact bundle from bucket '#{bucket}' and key '#{key}', version '#{version}', etag '#{etag}'")
           options = s3_options()
+          options = InstanceAgent::Config.common_client_config(options)
           s3 = Aws::S3::Client.new(options)
 
           File.open(artifact_bundle(deployment_spec), 'wb') do |file|

lib/instance_agent/plugins/codedeploy/deployment_command_tracker.rb

@@ -14,8 +14,23 @@ class DeploymentCommandTracker
             DEPLOYMENT_EVENT_FILE_STALE_TIMELIMIT_SECONDS = 86400 # 24 hour limit in secounds
 
             def self.create_ongoing_deployment_tracking_file(deployment_id, host_command_identifier)
-              FileUtils.mkdir_p(deployment_dir_path())
-              File.write(deployment_event_tracking_file_path(deployment_id), host_command_identifier)
+              retry_interval_in_sec = [1, 2, 5]
+
+              # deployment tracking file creations intermittently fails on recent windows versions
+              begin
+                FileUtils.mkdir_p(deployment_dir_path())
+                File.write(deployment_event_tracking_file_path(deployment_id), host_command_identifier)
+              rescue Errno::EACCES => error
+                InstanceAgent::Log.warn("Received Errno::EACCESS when creating deployment tracking file, retrying creation")
+                InstanceAgent::Log.warn(error.message)
+                if delay = retry_interval_in_sec.shift
+                  sleep delay
+                  retry
+                else
+                  InstanceAgent::Log.error("Exhausted retries on creating tracking file, rethrowing exception")
+                  raise
+                end
+              end
             end
 
             def self.delete_deployment_tracking_file_if_stale?(deployment_id, timeout)

lib/instance_agent/plugins/codedeploy/hook_executor.rb

@@ -159,6 +159,7 @@ def execute_script(script, script_log_file)
           script_command = InstanceAgent::Platform.util.prepare_script_command(script, script_absolute_path(script))
           log_script("Script - " + script.location + "\n", script_log_file)
           exit_status = 1
+          diagnostics = nil
           signal = nil
 
           if !InstanceAgent::Platform.util.supports_process_groups?
@@ -176,6 +177,7 @@ def execute_script(script, script_log_file)
             stderr_thread = Thread.new{stderr.each_line { |line| log_script("[stderr]" + line.to_s, script_log_file)}}
             thread_joiner = InstanceAgent::ThreadJoiner.new(script.timeout)
             thread_joiner.joinOrFail(wait_thr) do
+              log :info, "Script at specified location: #{script.location} timed out"
               Process.kill(signal, wait_thr.pid)
               raise Timeout::Error
             end
@@ -189,9 +191,13 @@ def execute_script(script, script_log_file)
               log :error, script_error
               raise ScriptError.new(ScriptError::OUTPUTS_LEFT_OPEN_CODE, script.location, @script_log), script_error
             end
+
             exit_status = wait_thr.value.exitstatus
+            diagnostics = process_status_diagnostics(wait_thr.value)
           end
-          if(exit_status != 0)
+
+          if (exit_status != 0)
+            log :debug, "Script failed. Diagnostics=#{diagnostics.to_s}"
             script_error = "#{script_error_prefix(script.location, script.runas)} failed with exit code #{exit_status.to_s}"
             raise ScriptError.new(ScriptError::SCRIPT_FAILED_CODE, script.location, @script_log), script_error
           end
@@ -301,6 +307,26 @@ def log_script(message, script_log_file)
             script_log_file.flush
           end
         end
+
+        private
+        # @param status [Process::Status]
+        #
+        # @return [Hash]
+        def process_status_diagnostics(status)
+          {
+            coredump?: status.coredump?,
+            exited: status.exited?,
+            exitstatus: status.exitstatus,
+            inspect: status.inspect,
+            pid: status.pid,
+            signaled?: status.signaled?,
+            stopped?: status.stopped?,
+            stopsig: status.stopsig,
+            success?: status.success?,
+            termsig: status.termsig,
+            to_i: status.to_i,
+          }
+        end
       end
     end
   end