-
Notifications
You must be signed in to change notification settings - Fork 4.9k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add elasticsearch module with stats metricset
This adds the stats metricset for the elasticsearch module. The stats metricset gives and overview over the current state of the cluster.
Showing
27 changed files
with
2,570 additions
and
16 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,37 @@ | ||
//// | ||
This file is generated! See scripts/docs_collector.py | ||
//// | ||
|
||
[[metricbeat-module-elasticsearch]] | ||
== elasticsearch Module | ||
|
||
This is the elasticsearch Module. The elasticsearch module contains a minimal set of metrics to enable monitoring of elasticsearch across multiple versions. | ||
|
||
|
||
|
||
[float] | ||
=== Example Configuration | ||
|
||
The elasticsearch module supports the standard configuration options that are described | ||
in <<configuration-metricbeat>>. Here is an example configuration: | ||
|
||
[source,yaml] | ||
---- | ||
metricbeat.modules: | ||
#- module: elasticsearch | ||
# metricsets: ["stats"] | ||
# enabled: true | ||
# period: 10s | ||
# hosts: ["localhost:9200"] | ||
---- | ||
|
||
[float] | ||
=== Metricsets | ||
|
||
The following metricsets are available: | ||
|
||
* <<metricbeat-metricset-elasticsearch-stats,stats>> | ||
|
||
include::elasticsearch/stats.asciidoc[] | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,19 @@ | ||
//// | ||
This file is generated! See scripts/docs_collector.py | ||
//// | ||
|
||
[[metricbeat-metricset-elasticsearch-stats]] | ||
include::../../../module/elasticsearch/stats/_meta/docs.asciidoc[] | ||
|
||
|
||
==== Fields | ||
|
||
For a description of each field in the metricset, see the | ||
<<exported-fields-elasticsearch,exported fields>> section. | ||
|
||
Here is an example document generated by this metricset: | ||
|
||
[source,json] | ||
---- | ||
include::../../../module/elasticsearch/stats/_meta/data.json[] | ||
---- |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,6 @@ | ||
#- module: elasticsearch | ||
# metricsets: ["stats"] | ||
# enabled: true | ||
# period: 10s | ||
# hosts: ["localhost:9200"] | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,4 @@ | ||
== elasticsearch Module | ||
|
||
This is the elasticsearch Module. The elasticsearch module contains a minimal set of metrics to enable monitoring of elasticsearch across multiple versions. | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,2 @@ | ||
ELASTICSEARCH_HOST=elasticsearch | ||
ELASTICSEARCH_PORT=9200 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,12 @@ | ||
- key: elasticsearch | ||
title: "elasticsearch" | ||
description: > | ||
[]experimental | ||
elasticsearch Module | ||
short_config: false | ||
fields: | ||
- name: elasticsearch | ||
type: group | ||
description: > | ||
fields: |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,4 @@ | ||
/* | ||
Package elasticsearch is a Metricbeat module that contains MetricSets. | ||
*/ | ||
package elasticsearch |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,38 @@ | ||
{ | ||
"@timestamp": "2016-05-23T08:05:34.853Z", | ||
"beat": { | ||
"hostname": "host.example.com", | ||
"name": "host.example.com" | ||
}, | ||
"elasticsearch": { | ||
"stats": { | ||
"docs": { | ||
"count": 4, | ||
"deleted": 0 | ||
}, | ||
"segments": { | ||
"count": 4, | ||
"memory": { | ||
"bytes": 6647 | ||
} | ||
}, | ||
"shards": { | ||
"failed": 0, | ||
"successful": 11, | ||
"total": 22 | ||
}, | ||
"store": { | ||
"size": { | ||
"bytes": 16148 | ||
} | ||
} | ||
} | ||
}, | ||
"metricset": { | ||
"host": "127.0.0.1:9200", | ||
"module": "elasticsearch", | ||
"name": "stats", | ||
"rtt": 115 | ||
}, | ||
"type": "metricsets" | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,3 @@ | ||
=== elasticsearch stats MetricSet | ||
|
||
This is the stats metricset of the module elasticsearch. This provides statistics across all indices. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,38 @@ | ||
- name: stats | ||
type: group | ||
description: > | ||
stats | ||
fields: | ||
- name: docs.count | ||
type: long | ||
description: > | ||
Total number of existing documents. | ||
- name: docs.deleted | ||
type: long | ||
description: > | ||
Total number of deleted documents. | ||
- name: segments.count | ||
type: long | ||
description: > | ||
Total number of segments. | ||
- name: segments.memory.bytes | ||
type: long | ||
format: bytes | ||
description: > | ||
Total size of segments in bytes. | ||
- name: shards.failed | ||
type: long | ||
description: > | ||
Number of failed shards. | ||
- name: shards.successful | ||
type: long | ||
description: > | ||
Number of successful shards. | ||
- name: shards.total | ||
type: long | ||
description: > | ||
Total number of shards. | ||
- name: store.size.bytes | ||
type: long | ||
description: > | ||
Total size of the store in bytes. |
30 changes: 30 additions & 0 deletions
30
metricbeat/module/elasticsearch/stats/_meta/test/generate.sh
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,30 @@ | ||
#!/usr/bin/env bash | ||
|
||
# This script is to generate test input files for different elasticsearch versions | ||
# | ||
# The script creates an index, adds a document and writes the output from _stats | ||
# to a document. The document name is based on the first param passed to the script. | ||
# For es 5.1.2 pass 512 | ||
# | ||
# Note: Small corrections were made to the output documents as size of the index | ||
# is not the same across all versions | ||
|
||
# Create index | ||
curl -XPUT 'http://localhost:9200/testindex' | ||
|
||
# Add document | ||
curl -XPUT 'http://localhost:9200/testindex/test/1?pretty' -H 'Content-Type: application/json' -d' | ||
{ | ||
"user" : "kimchy", | ||
"message" : "trying out Elasticsearch" | ||
} | ||
' | ||
|
||
# Make sure index is created | ||
# For 1x / 2x releases | ||
curl -XPOST 'http://localhost:9200/_optimize' | ||
# For 5x releases | ||
curl -XPOST 'http://localhost:9200/_forcemerge' | ||
|
||
# Read stats output | ||
curl -XGET 'http://localhost:9200/_stats?pretty' > stats.${1}.json |
457 changes: 457 additions & 0 deletions
457
metricbeat/module/elasticsearch/stats/_meta/test/stats.175.json
Large diffs are not rendered by default.
Oops, something went wrong.
509 changes: 509 additions & 0 deletions
509
metricbeat/module/elasticsearch/stats/_meta/test/stats.201.json
Large diffs are not rendered by default.
Oops, something went wrong.
513 changes: 513 additions & 0 deletions
513
metricbeat/module/elasticsearch/stats/_meta/test/stats.240.json
Large diffs are not rendered by default.
Oops, something went wrong.
12 changes: 12 additions & 0 deletions
12
metricbeat/module/elasticsearch/stats/_meta/test/stats.512.empty.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,12 @@ | ||
{ | ||
"_shards" : { | ||
"total" : 0, | ||
"successful" : 0, | ||
"failed" : 0 | ||
}, | ||
"_all" : { | ||
"primaries" : { }, | ||
"total" : { } | ||
}, | ||
"indices" : { } | ||
} |
481 changes: 481 additions & 0 deletions
481
metricbeat/module/elasticsearch/stats/_meta/test/stats.512.json
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,64 @@ | ||
package stats | ||
|
||
import ( | ||
"encoding/json" | ||
|
||
"github.com/elastic/beats/libbeat/common" | ||
s "github.com/elastic/beats/metricbeat/schema" | ||
c "github.com/elastic/beats/metricbeat/schema/mapstriface" | ||
) | ||
|
||
var ( | ||
shards = s.Schema{ | ||
"shards": c.Dict("_shards", s.Schema{ | ||
"total": c.Int("total"), | ||
"successful": c.Int("successful"), | ||
"failed": c.Int("failed"), | ||
}), | ||
} | ||
|
||
total = s.Schema{ | ||
"docs": c.Dict("docs", s.Schema{ | ||
"count": c.Int("count"), | ||
"deleted": c.Int("deleted"), | ||
}), | ||
"store": c.Dict("store", s.Schema{ | ||
"size": s.Object{ | ||
"bytes": c.Int("size_in_bytes"), | ||
}, | ||
}), | ||
"segments": c.Dict("segments", s.Schema{ | ||
"count": c.Int("count"), | ||
"memory": s.Object{ | ||
"bytes": c.Int("memory_in_bytes"), | ||
}, | ||
}), | ||
} | ||
) | ||
|
||
func eventMapping(content []byte) (common.MapStr, error) { | ||
|
||
// Empty struct needed every time | ||
var allStruct struct { | ||
All struct { | ||
Total map[string]interface{} `json:"total"` | ||
} `json:"_all"` | ||
} | ||
var shardsStruct map[string]interface{} | ||
|
||
json.Unmarshal(content, &allStruct) | ||
|
||
// This happens before elasticsearch has any shards. Return empty document. | ||
if len(allStruct.All.Total) == 0 { | ||
return common.MapStr{}, nil | ||
} | ||
|
||
json.Unmarshal(content, &shardsStruct) | ||
|
||
allData, errs1 := total.Apply(allStruct.All.Total) | ||
shards, errs2 := shards.Apply(shardsStruct) | ||
errs1.AddErrors(errs2) | ||
|
||
allData.Update(shards) | ||
return allData, errs1 | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,47 @@ | ||
// +build !integration | ||
|
||
package stats | ||
|
||
import ( | ||
"io/ioutil" | ||
"path/filepath" | ||
"testing" | ||
|
||
"github.com/elastic/beats/libbeat/common" | ||
"github.com/elastic/beats/metricbeat/schema" | ||
"github.com/stretchr/testify/assert" | ||
) | ||
|
||
func TestStats(t *testing.T) { | ||
|
||
files, err := filepath.Glob("./_meta/test/stats.*.json") | ||
assert.NoError(t, err) | ||
|
||
for _, f := range files { | ||
content, err := ioutil.ReadFile(f) | ||
assert.NoError(t, err) | ||
|
||
_, errs := eventMapping(content) | ||
if errs == nil { | ||
continue | ||
} | ||
errors, ok := errs.(*schema.Errors) | ||
if ok { | ||
assert.False(t, errors.HasRequiredErrors(), "mapping error: %s", errors) | ||
} else { | ||
t.Error(err) | ||
} | ||
} | ||
} | ||
|
||
func TestEmptyStats(t *testing.T) { | ||
|
||
file := "./_meta/test/stats.512.empty.json" | ||
|
||
content, err := ioutil.ReadFile(file) | ||
assert.NoError(t, err) | ||
|
||
event, errs := eventMapping(content) | ||
assert.Equal(t, event, common.MapStr{}) | ||
assert.Nil(t, errs) | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,53 @@ | ||
package stats | ||
|
||
import ( | ||
"github.com/elastic/beats/libbeat/common" | ||
"github.com/elastic/beats/libbeat/logp" | ||
"github.com/elastic/beats/metricbeat/helper" | ||
"github.com/elastic/beats/metricbeat/mb" | ||
"github.com/elastic/beats/metricbeat/mb/parse" | ||
) | ||
|
||
// init registers the MetricSet with the central registry. | ||
// The New method will be called after the setup of the module and before starting to fetch data | ||
func init() { | ||
if err := mb.Registry.AddMetricSet("elasticsearch", "stats", New, hostParser); err != nil { | ||
panic(err) | ||
} | ||
} | ||
|
||
var ( | ||
hostParser = parse.URLHostParserBuilder{ | ||
DefaultScheme: "http", | ||
PathConfigKey: "path", | ||
DefaultPath: "_stats", | ||
}.Build() | ||
) | ||
|
||
// MetricSet type defines all fields of the MetricSet | ||
type MetricSet struct { | ||
mb.BaseMetricSet | ||
http *helper.HTTP | ||
} | ||
|
||
// New create a new instance of the MetricSet | ||
func New(base mb.BaseMetricSet) (mb.MetricSet, error) { | ||
logp.Experimental("Elasticsearch stats metricset is enabled.") | ||
return &MetricSet{ | ||
base, | ||
helper.NewHTTP(base), | ||
}, nil | ||
} | ||
|
||
// Fetch gathers stats for each overall _stats API (not specific to an index) | ||
// More details can be found here: https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-stats.html | ||
func (m *MetricSet) Fetch() (common.MapStr, error) { | ||
|
||
content, err := m.http.FetchContent() | ||
if err != nil { | ||
return nil, err | ||
} | ||
|
||
event, _ := eventMapping(content) | ||
return event, nil | ||
} |
57 changes: 57 additions & 0 deletions
57
metricbeat/module/elasticsearch/stats/stats_integration_test.go
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,57 @@ | ||
// +build integration | ||
|
||
package stats | ||
|
||
import ( | ||
"fmt" | ||
"net/http" | ||
"strings" | ||
"testing" | ||
|
||
mbtest "github.com/elastic/beats/metricbeat/mb/testing" | ||
"github.com/elastic/beats/metricbeat/module/elasticsearch" | ||
"github.com/stretchr/testify/assert" | ||
) | ||
|
||
func TestFetch(t *testing.T) { | ||
err := loadData() | ||
if err != nil { | ||
t.Fatal("write", err) | ||
} | ||
|
||
f := mbtest.NewEventFetcher(t, elasticsearch.GetConfig("stats")) | ||
event, err := f.Fetch() | ||
if !assert.NoError(t, err) { | ||
t.FailNow() | ||
} | ||
|
||
assert.NotNil(t, event) | ||
t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event) | ||
} | ||
|
||
func TestData(t *testing.T) { | ||
err := loadData() | ||
if err != nil { | ||
t.Fatal("write", err) | ||
} | ||
|
||
f := mbtest.NewEventFetcher(t, elasticsearch.GetConfig("stats")) | ||
err = mbtest.WriteEvent(f, t) | ||
if err != nil { | ||
t.Fatal("write", err) | ||
} | ||
} | ||
|
||
func loadData() error { | ||
client := &http.Client{} | ||
url := fmt.Sprintf("http://%s:%s/tests/stats", elasticsearch.GetEnvHost(), elasticsearch.GetEnvPort()) | ||
|
||
request, err := http.NewRequest("POST", url, strings.NewReader(`{"hello":"world"}`)) | ||
request.Header.Set("Content-Type", "application/json") | ||
if err != nil { | ||
return err | ||
} | ||
res, err := client.Do(request) | ||
defer res.Body.Close() | ||
return err | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,29 @@ | ||
package elasticsearch | ||
|
||
import "os" | ||
|
||
func GetEnvHost() string { | ||
host := os.Getenv("ELASTICSEARCH_HOST") | ||
|
||
if len(host) == 0 { | ||
host = "127.0.0.1" | ||
} | ||
return host | ||
} | ||
|
||
func GetEnvPort() string { | ||
port := os.Getenv("ELASTICSEARCH_PORT") | ||
|
||
if len(port) == 0 { | ||
port = "9200" | ||
} | ||
return port | ||
} | ||
|
||
func GetConfig(metricset string) map[string]interface{} { | ||
return map[string]interface{}{ | ||
"module": "elasticsearch", | ||
"metricsets": []string{metricset}, | ||
"hosts": []string{GetEnvHost() + ":" + GetEnvPort()}, | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,32 @@ | ||
import os | ||
import metricbeat | ||
import unittest | ||
|
||
|
||
class Test(metricbeat.BaseTest): | ||
|
||
@unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") | ||
def test_stats(self): | ||
""" | ||
elasticsearch stats metricset test | ||
""" | ||
self.render_config_template(modules=[{ | ||
"name": "elasticsearch", | ||
"metricsets": ["stats"], | ||
"hosts": self.get_hosts(), | ||
"period": "1s", | ||
}]) | ||
proc = self.start_beat() | ||
self.wait_until(lambda: self.output_lines() > 0, max_timeout=20) | ||
proc.check_kill_and_wait() | ||
|
||
output = self.read_output_json() | ||
self.assertTrue(len(output) >= 1) | ||
evt = output[0] | ||
print(evt) | ||
|
||
self.assert_fields_are_documented(evt) | ||
|
||
def get_hosts(self): | ||
return [os.getenv('ELASTICSEARCH_HOST', 'localhost') + ':' + | ||
os.getenv('ELASTICSEARCH_PORT', '9200')] |