Added skip error logging

CHANGELOG.md

@@ -1,3 +1,7 @@
+## 4.1.2
+
+- Added a configuration called failure_type_logging_whitelist which takes a list of strings, that are error types from elasticsearch, so we prevent logging WARN if elasticsearch fails with that action. See https://github.com/logstash-plugins/logstash-output-elasticsearch/issues/423
+
 ## 4.1.1
 - Fix bug where setting credentials would cause fatal errors. See https://github.com/logstash-plugins/logstash-output-elasticsearch/issues/441
 

CONTRIBUTORS

@@ -25,6 +25,7 @@ Contributors:
 * Tal Levy (talevy)
 * Tom Hodder (tolland)
 * jimmyjones2
+* Gabriel Moskovicz (gmoskovicz)
 
 Note: If you've sent us patches, bug reports, or otherwise contributed to
 Logstash, and you aren't on the list above and want to be, please let us know

lib/logstash/outputs/elasticsearch.rb

@@ -133,6 +133,11 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   # a timeout occurs, the request will be retried.
   config :timeout, :validate => :number
 
+  # Set the Elasticsearch errors in the whitelist that you don't want to log. 
+  # A useful example is when you want to skip all 409 errors
+  # which are `document_already_exists_exception`.
+  config :failure_type_logging_whitelist, :validate => :array, :default => []
+
   # While the output tries to reuse connections efficiently we have a maximum.
   # This sets the maximum number of open connections the output will create.
   # Setting this too low may mean frequently closing / opening connections

lib/logstash/outputs/elasticsearch/common.rb

@@ -112,17 +112,18 @@ def submit(actions)
       actions_to_retry = []
       bulk_response["items"].each_with_index do |response,idx|
         action_type, action_props = response.first
+
         status = action_props["status"]
-        error  = action_props["error"]
+        failure  = action_props["error"]
         action = actions[idx]
 
         if SUCCESS_CODES.include?(status)
           next
         elsif RETRYABLE_CODES.include?(status)
-          @logger.info "retrying failed action with response code: #{status} (#{error})"
+          @logger.info "retrying failed action with response code: #{status} (#{failure})"
           actions_to_retry << action
-        else
-          @logger.warn "Failed action. ", status: status, action: action, response: response
+        elsif !failure_type_logging_whitelist.include?(failure["type"])
+          @logger.warn "Failed action.", status: status, action: action, response: response
         end
       end
 

logstash-output-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '4.1.1'
+  s.version         = '4.1.2'
   s.licenses        = ['apache-2.0']
   s.summary         = "Logstash Output to Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

spec/unit/outputs/error_whitelist_spec.rb

@@ -0,0 +1,52 @@
+require "logstash/outputs/elasticsearch"
+require_relative "../../../spec/es_spec_helper"
+
+describe "whitelisting error types in expected behavior" do
+  let(:template) { '{"template" : "not important, will be updated by :index"}' }
+  let(:event1) { LogStash::Event.new("somevalue" => 100, "@timestamp" => "2014-11-17T20:37:17.223Z") }
+  let(:action1) { ["index", {:_id=>1, :_routing=>nil, :_index=>"logstash-2014.11.17", :_type=>"logs"}, event1] }
+  let(:settings) { {"manage_template" => true, "index" => "logstash-2014.11.17", "template_overwrite" => true, "hosts" => get_host_port() } }
+
+  subject { LogStash::Outputs::ElasticSearch.new(settings) }
+
+  before :each do
+    allow(subject.logger).to receive(:warn)
+
+    subject.register
+
+    allow(subject.client).to receive(:bulk).and_return(
+      {
+        "errors" => true,
+        "items" => [{
+          "create" => {
+            "status" => 409, 
+            "error" => {
+              "type" => "document_already_exists_exception",
+              "reason" => "[shard] document already exists"
+            }
+          }
+        }]
+      })
+
+    subject.multi_receive([event1])
+  end
+
+  after :each do
+    subject.close
+  end
+
+  describe "when failure logging is enabled for everything" do
+    it "should log a failure on the action" do
+      expect(subject.logger).to have_received(:warn).with("Failed action.", anything)
+    end
+  end
+
+  describe "when failure logging is disabled for docuemnt exists error" do
+    let(:settings) { super.merge("failure_type_logging_whitelist" => ["document_already_exists_exception"]) }
+
+    it "should log a failure on the action" do
+      expect(subject.logger).not_to have_received(:warn).with("Failed action.", anything)
+    end
+  end
+
+end