Update template to use ES 5.x mapping

Fixes logstash-plugins#386
suyograo · Aug 4, 2016 · fc43eb2 · fc43eb2
1 parent 2e944ef
commit fc43eb2
Show file tree

Hide file tree

Showing 5 changed files with 19 additions and 20 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -3,9 +3,9 @@ sudo: false
 jdk: oraclejdk8
 env:
 - INTEGRATION=false
-- INTEGRATION=true ES_VERSION=2.2.0
 - INTEGRATION=true ES_VERSION=1.7.5
-- INTEGRATION=true ES_VERSION=5.0.0-alpha1
+- INTEGRATION=true ES_VERSION=2.3.4
+- INTEGRATION=true ES_VERSION=5.0.0-alpha4
 language: ruby
 cache: bundler
 rvm:

diff --git a/lib/logstash/outputs/elasticsearch/elasticsearch-template-es5x.json b/lib/logstash/outputs/elasticsearch/elasticsearch-template-es5x.json
@@ -5,13 +5,13 @@
   },
   "mappings" : {
     "_default_" : {
-      "_all" : {"enabled" : true, "omit_norms" : true},
+      "_all" : {"enabled" : true, "norms" : false},
       "dynamic_templates" : [ {
         "message_field" : {
           "match" : "message",
           "match_mapping_type" : "string",
           "mapping" : {
-            "type" : "string", "index" : "analyzed", "omit_norms" : true,
+            "type" : "string", "index" : "analyzed", "norms" : false,
             "fielddata" : { "format" : "disabled" }
           }
         }
@@ -20,24 +20,23 @@
           "match" : "*",
           "match_mapping_type" : "string",
           "mapping" : {
-            "type" : "string", "index" : "analyzed", "omit_norms" : true,
-            "fielddata" : { "format" : "disabled" },
+            "type" : "text", "norms" : false,
             "fields" : {
-              "raw" : {"type": "string", "index" : "not_analyzed", "ignore_above" : 256}
+              "keyword" : { "type": "keyword" }
             }
           }
         }
       } ],
       "properties" : {
-        "@timestamp": { "type": "date" },
-        "@version": { "type": "string", "index": "not_analyzed" },
+        "@timestamp": { "type": "date", "include_in_all": false },
+        "@version": { "type": "keyword", "include_in_all": false },
         "geoip"  : {
           "dynamic": true,
           "properties" : {
             "ip": { "type": "ip" },
             "location" : { "type" : "geo_point" },
-            "latitude" : { "type" : "float" },
-            "longitude" : { "type" : "float" }
+            "latitude" : { "type" : "half_float" },
+            "longitude" : { "type" : "half_float" }
           }
         }
       }

diff --git a/spec/integration/outputs/pipeline_spec.rb b/spec/integration/outputs/pipeline_spec.rb
@@ -19,7 +19,7 @@
         {
           "grok": {
             "field": "message",
-            "pattern": "%{COMBINEDAPACHELOG}"
+            "patterns": ["%{COMBINEDAPACHELOG}"]
           }
         }
       ]

diff --git a/spec/integration/outputs/templates_spec.rb b/spec/integration/outputs/templates_spec.rb
@@ -59,27 +59,27 @@
     reject { values }.include?(1)
   end
 
-  it "does not create .raw field for the message field" do
-    results = @es.search(:q => "message.raw:\"sample message here\"")
+  it "does not create .keyword field for the message field" do
+    results = @es.search(:q => "message.keyword:\"sample message here\"")
     insist { results["hits"]["total"] } == 0
   end
 
-  it "creates .raw field from any string field which is not_analyzed" do
-    results = @es.search(:q => "country.raw:\"us\"")
+  it "creates .keyword field from any string field which is not_analyzed" do
+    results = @es.search(:q => "country.keyword:\"us\"")
     insist { results["hits"]["total"] } == 1
     insist { results["hits"]["hits"][0]["_source"]["country"] } == "us"
 
     # partial or terms should not work.
-    results = @es.search(:q => "country.raw:\"u\"")
+    results = @es.search(:q => "country.keyword:\"u\"")
     insist { results["hits"]["total"] } == 0
   end
 
   it "make [geoip][location] a geo_point" do
     expect(@es.indices.get_template(name: "logstash")["logstash"]["mappings"]["_default_"]["properties"]["geoip"]["properties"]["location"]["type"]).to eq("geo_point")
   end
 
-  it "aggregate .raw results correctly " do
-    results = @es.search(:body => { "aggregations" => { "my_agg" => { "terms" => { "field" => "country.raw" } } } })["aggregations"]["my_agg"]
+  it "aggregate .keyword results correctly " do
+    results = @es.search(:body => { "aggregations" => { "my_agg" => { "terms" => { "field" => "country.keyword" } } } })["aggregations"]["my_agg"]
     terms = results["buckets"].collect { |b| b["key"] }
 
     insist { terms }.include?("us")

diff --git a/travis-run.sh b/travis-run.sh
@@ -35,7 +35,7 @@ if [[ "$INTEGRATION" != "true" ]]; then
 else
   if [[ "$ES_VERSION" == 5.* ]]; then
     setup_es https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/$ES_VERSION/elasticsearch-$ES_VERSION.tar.gz
-    start_es -Ees.script.inline=true -Ees.script.indexed=true -Ees.script.file=true
+    start_es -Escript.inline=true -Escript.stored=true -Escript.file=true
     bundle exec rspec -fd spec --tag integration --tag version_5x --tag integration_2x_plus
   elif [[ "$ES_VERSION" == 2.* ]]; then
     setup_es https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch-$ES_VERSION.tar.gz