Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add elasticsearch module with stats metricset
Browse files Browse the repository at this point in the history
This adds the stats metricset for the elasticsearch module. The stats metricset gives and overview over the current state of the cluster.
ruflin committed Apr 10, 2017

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
1 parent ca6caf1 commit a6cc12f
Showing 27 changed files with 2,570 additions and 16 deletions.
39 changes: 23 additions & 16 deletions metricbeat/docker-compose.yml
Original file line number Diff line number Diff line change
@@ -19,6 +19,7 @@ services:
- ${PWD}/module/apache/_meta/env
- ${PWD}/module/ceph/_meta/env
- ${PWD}/module/couchbase/_meta/env
- ${PWD}/module/elasticsearch/_meta/env
- ${PWD}/module/haproxy/_meta/env
- ${PWD}/module/jolokia/_meta/env
- ${PWD}/module/kafka/_meta/env
@@ -38,22 +39,23 @@ services:
proxy_dep:
image: busybox
depends_on:
apache: { condition: service_healthy }
ceph: { condition: service_healthy }
couchbase: { condition: service_healthy }
haproxy: { condition: service_healthy }
jolokia: { condition: service_healthy }
kafka: { condition: service_healthy }
kibana: { condition: service_healthy }
memcached: { condition: service_healthy }
mongodb: { condition: service_healthy }
mysql: { condition: service_healthy }
nginx: { condition: service_healthy }
phpfpm: { condition: service_healthy }
postgresql: { condition: service_healthy }
prometheus: { condition: service_healthy }
redis: { condition: service_healthy }
zookeeper: { condition: service_healthy }
apache: { condition: service_healthy }
ceph: { condition: service_healthy }
couchbase: { condition: service_healthy }
elasticsearch: { condition: service_healthy }
haproxy: { condition: service_healthy }
jolokia: { condition: service_healthy }
kafka: { condition: service_healthy }
kibana: { condition: service_healthy }
memcached: { condition: service_healthy }
mongodb: { condition: service_healthy }
mysql: { condition: service_healthy }
nginx: { condition: service_healthy }
phpfpm: { condition: service_healthy }
postgresql: { condition: service_healthy }
prometheus: { condition: service_healthy }
redis: { condition: service_healthy }
zookeeper: { condition: service_healthy }

# Modules
apache:
@@ -65,6 +67,11 @@ services:
couchbase:
build: ${PWD}/module/couchbase/_meta

elasticsearch:
extends:
file: ${ES_BEATS}/testing/environments/${TESTING_ENVIRONMENT}.yml
service: elasticsearch

haproxy:
build: ${PWD}/module/haproxy/_meta

88 changes: 88 additions & 0 deletions metricbeat/docs/fields.asciidoc
Original file line number Diff line number Diff line change
@@ -19,6 +19,7 @@ grouped in the following categories:
* <<exported-fields-common>>
* <<exported-fields-couchbase>>
* <<exported-fields-docker>>
* <<exported-fields-elasticsearch>>
* <<exported-fields-golang>>
* <<exported-fields-haproxy>>
* <<exported-fields-jolokia>>
@@ -1843,6 +1844,93 @@ type: long
Total number of outgoing packets.
[[exported-fields-elasticsearch]]
== elasticsearch Fields
[]experimental
elasticsearch Module
[float]
== elasticsearch Fields
[float]
== stats Fields
stats
[float]
=== elasticsearch.stats.docs.count
type: long
Total number of existing documents.
[float]
=== elasticsearch.stats.docs.deleted
type: long
Total number of deleted documents.
[float]
=== elasticsearch.stats.segments.count
type: long
Total number of segments.
[float]
=== elasticsearch.stats.segments.memory.bytes
type: long
format: bytes
Total size of segments in bytes.
[float]
=== elasticsearch.stats.shards.failed
type: long
Number of failed shards.
[float]
=== elasticsearch.stats.shards.successful
type: long
Number of successful shards.
[float]
=== elasticsearch.stats.shards.total
type: long
Total number of shards.
[float]
=== elasticsearch.stats.store.size.bytes
type: long
Total size of the store in bytes.
[[exported-fields-golang]]
== golang Fields
37 changes: 37 additions & 0 deletions metricbeat/docs/modules/elasticsearch.asciidoc
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
////
This file is generated! See scripts/docs_collector.py
////

[[metricbeat-module-elasticsearch]]
== elasticsearch Module

This is the elasticsearch Module. The elasticsearch module contains a minimal set of metrics to enable monitoring of elasticsearch across multiple versions.



[float]
=== Example Configuration

The elasticsearch module supports the standard configuration options that are described
in <<configuration-metricbeat>>. Here is an example configuration:

[source,yaml]
----
metricbeat.modules:
#- module: elasticsearch
# metricsets: ["stats"]
# enabled: true
# period: 10s
# hosts: ["localhost:9200"]
----

[float]
=== Metricsets

The following metricsets are available:

* <<metricbeat-metricset-elasticsearch-stats,stats>>

include::elasticsearch/stats.asciidoc[]

19 changes: 19 additions & 0 deletions metricbeat/docs/modules/elasticsearch/stats.asciidoc
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
////
This file is generated! See scripts/docs_collector.py
////

[[metricbeat-metricset-elasticsearch-stats]]
include::../../../module/elasticsearch/stats/_meta/docs.asciidoc[]


==== Fields

For a description of each field in the metricset, see the
<<exported-fields-elasticsearch,exported fields>> section.

Here is an example document generated by this metricset:

[source,json]
----
include::../../../module/elasticsearch/stats/_meta/data.json[]
----
2 changes: 2 additions & 0 deletions metricbeat/docs/modules_list.asciidoc
Original file line number Diff line number Diff line change
@@ -6,6 +6,7 @@ This file is generated! See scripts/docs_collector.py
* <<metricbeat-module-ceph,ceph>>
* <<metricbeat-module-couchbase,Couchbase>>
* <<metricbeat-module-docker,Docker>>
* <<metricbeat-module-elasticsearch,elasticsearch>>
* <<metricbeat-module-golang,golang>>
* <<metricbeat-module-haproxy,HAProxy>>
* <<metricbeat-module-jolokia,Jolokia>>
@@ -31,6 +32,7 @@ include::modules/apache.asciidoc[]
include::modules/ceph.asciidoc[]
include::modules/couchbase.asciidoc[]
include::modules/docker.asciidoc[]
include::modules/elasticsearch.asciidoc[]
include::modules/golang.asciidoc[]
include::modules/haproxy.asciidoc[]
include::modules/jolokia.asciidoc[]
2 changes: 2 additions & 0 deletions metricbeat/include/list.go
Original file line number Diff line number Diff line change
@@ -28,6 +28,8 @@ import (
_ "github.com/elastic/beats/metricbeat/module/docker/info"
_ "github.com/elastic/beats/metricbeat/module/docker/memory"
_ "github.com/elastic/beats/metricbeat/module/docker/network"
_ "github.com/elastic/beats/metricbeat/module/elasticsearch"
_ "github.com/elastic/beats/metricbeat/module/elasticsearch/stats"
_ "github.com/elastic/beats/metricbeat/module/golang"
_ "github.com/elastic/beats/metricbeat/module/golang/expvar"
_ "github.com/elastic/beats/metricbeat/module/golang/heap"
8 changes: 8 additions & 0 deletions metricbeat/metricbeat.full.yml
Original file line number Diff line number Diff line change
@@ -124,6 +124,14 @@ metricbeat.modules:
#certificate: "/etc/pki/client/cert.pem"
#key: "/etc/pki/client/cert.key"

#---------------------------- elasticsearch Module ---------------------------
#- module: elasticsearch
# metricsets: ["stats"]
# enabled: true
# period: 10s
# hosts: ["localhost:9200"]


#------------------------------- golang Module -------------------------------
#- module: golang
# metricsets: ["expvar","heap"]
6 changes: 6 additions & 0 deletions metricbeat/module/elasticsearch/_meta/config.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
#- module: elasticsearch
# metricsets: ["stats"]
# enabled: true
# period: 10s
# hosts: ["localhost:9200"]

4 changes: 4 additions & 0 deletions metricbeat/module/elasticsearch/_meta/docs.asciidoc
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
== elasticsearch Module

This is the elasticsearch Module. The elasticsearch module contains a minimal set of metrics to enable monitoring of elasticsearch across multiple versions.

2 changes: 2 additions & 0 deletions metricbeat/module/elasticsearch/_meta/env
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
ELASTICSEARCH_HOST=elasticsearch
ELASTICSEARCH_PORT=9200
12 changes: 12 additions & 0 deletions metricbeat/module/elasticsearch/_meta/fields.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
- key: elasticsearch
title: "elasticsearch"
description: >
[]experimental
elasticsearch Module
short_config: false
fields:
- name: elasticsearch
type: group
description: >
fields:
4 changes: 4 additions & 0 deletions metricbeat/module/elasticsearch/doc.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
/*
Package elasticsearch is a Metricbeat module that contains MetricSets.
*/
package elasticsearch
38 changes: 38 additions & 0 deletions metricbeat/module/elasticsearch/stats/_meta/data.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
{
"@timestamp": "2016-05-23T08:05:34.853Z",
"beat": {
"hostname": "host.example.com",
"name": "host.example.com"
},
"elasticsearch": {
"stats": {
"docs": {
"count": 4,
"deleted": 0
},
"segments": {
"count": 4,
"memory": {
"bytes": 6647
}
},
"shards": {
"failed": 0,
"successful": 11,
"total": 22
},
"store": {
"size": {
"bytes": 16148
}
}
}
},
"metricset": {
"host": "127.0.0.1:9200",
"module": "elasticsearch",
"name": "stats",
"rtt": 115
},
"type": "metricsets"
}
3 changes: 3 additions & 0 deletions metricbeat/module/elasticsearch/stats/_meta/docs.asciidoc
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
=== elasticsearch stats MetricSet

This is the stats metricset of the module elasticsearch. This provides statistics across all indices.
38 changes: 38 additions & 0 deletions metricbeat/module/elasticsearch/stats/_meta/fields.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
- name: stats
type: group
description: >
stats
fields:
- name: docs.count
type: long
description: >
Total number of existing documents.
- name: docs.deleted
type: long
description: >
Total number of deleted documents.
- name: segments.count
type: long
description: >
Total number of segments.
- name: segments.memory.bytes
type: long
format: bytes
description: >
Total size of segments in bytes.
- name: shards.failed
type: long
description: >
Number of failed shards.
- name: shards.successful
type: long
description: >
Number of successful shards.
- name: shards.total
type: long
description: >
Total number of shards.
- name: store.size.bytes
type: long
description: >
Total size of the store in bytes.
30 changes: 30 additions & 0 deletions metricbeat/module/elasticsearch/stats/_meta/test/generate.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
#!/usr/bin/env bash

# This script is to generate test input files for different elasticsearch versions
#
# The script creates an index, adds a document and writes the output from _stats
# to a document. The document name is based on the first param passed to the script.
# For es 5.1.2 pass 512
#
# Note: Small corrections were made to the output documents as size of the index
# is not the same across all versions

# Create index
curl -XPUT 'http://localhost:9200/testindex'

# Add document
curl -XPUT 'http://localhost:9200/testindex/test/1?pretty' -H 'Content-Type: application/json' -d'
{
"user" : "kimchy",
"message" : "trying out Elasticsearch"
}
'

# Make sure index is created
# For 1x / 2x releases
curl -XPOST 'http://localhost:9200/_optimize'
# For 5x releases
curl -XPOST 'http://localhost:9200/_forcemerge'

# Read stats output
curl -XGET 'http://localhost:9200/_stats?pretty' > stats.${1}.json
457 changes: 457 additions & 0 deletions metricbeat/module/elasticsearch/stats/_meta/test/stats.175.json

Large diffs are not rendered by default.

509 changes: 509 additions & 0 deletions metricbeat/module/elasticsearch/stats/_meta/test/stats.201.json

Large diffs are not rendered by default.

513 changes: 513 additions & 0 deletions metricbeat/module/elasticsearch/stats/_meta/test/stats.240.json

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
{
"_shards" : {
"total" : 0,
"successful" : 0,
"failed" : 0
},
"_all" : {
"primaries" : { },
"total" : { }
},
"indices" : { }
}
481 changes: 481 additions & 0 deletions metricbeat/module/elasticsearch/stats/_meta/test/stats.512.json

Large diffs are not rendered by default.

64 changes: 64 additions & 0 deletions metricbeat/module/elasticsearch/stats/data.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
package stats

import (
"encoding/json"

"github.com/elastic/beats/libbeat/common"
s "github.com/elastic/beats/metricbeat/schema"
c "github.com/elastic/beats/metricbeat/schema/mapstriface"
)

var (
shards = s.Schema{
"shards": c.Dict("_shards", s.Schema{
"total": c.Int("total"),
"successful": c.Int("successful"),
"failed": c.Int("failed"),
}),
}

total = s.Schema{
"docs": c.Dict("docs", s.Schema{
"count": c.Int("count"),
"deleted": c.Int("deleted"),
}),
"store": c.Dict("store", s.Schema{
"size": s.Object{
"bytes": c.Int("size_in_bytes"),
},
}),
"segments": c.Dict("segments", s.Schema{
"count": c.Int("count"),
"memory": s.Object{
"bytes": c.Int("memory_in_bytes"),
},
}),
}
)

func eventMapping(content []byte) (common.MapStr, error) {

// Empty struct needed every time
var allStruct struct {
All struct {
Total map[string]interface{} `json:"total"`
} `json:"_all"`
}
var shardsStruct map[string]interface{}

json.Unmarshal(content, &allStruct)

// This happens before elasticsearch has any shards. Return empty document.
if len(allStruct.All.Total) == 0 {
return common.MapStr{}, nil
}

json.Unmarshal(content, &shardsStruct)

allData, errs1 := total.Apply(allStruct.All.Total)
shards, errs2 := shards.Apply(shardsStruct)
errs1.AddErrors(errs2)

allData.Update(shards)
return allData, errs1
}
47 changes: 47 additions & 0 deletions metricbeat/module/elasticsearch/stats/data_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
// +build !integration

package stats

import (
"io/ioutil"
"path/filepath"
"testing"

"github.com/elastic/beats/libbeat/common"
"github.com/elastic/beats/metricbeat/schema"
"github.com/stretchr/testify/assert"
)

func TestStats(t *testing.T) {

files, err := filepath.Glob("./_meta/test/stats.*.json")
assert.NoError(t, err)

for _, f := range files {
content, err := ioutil.ReadFile(f)
assert.NoError(t, err)

_, errs := eventMapping(content)
if errs == nil {
continue
}
errors, ok := errs.(*schema.Errors)
if ok {
assert.False(t, errors.HasRequiredErrors(), "mapping error: %s", errors)
} else {
t.Error(err)
}
}
}

func TestEmptyStats(t *testing.T) {

file := "./_meta/test/stats.512.empty.json"

content, err := ioutil.ReadFile(file)
assert.NoError(t, err)

event, errs := eventMapping(content)
assert.Equal(t, event, common.MapStr{})
assert.Nil(t, errs)
}
53 changes: 53 additions & 0 deletions metricbeat/module/elasticsearch/stats/stats.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
package stats

import (
"github.com/elastic/beats/libbeat/common"
"github.com/elastic/beats/libbeat/logp"
"github.com/elastic/beats/metricbeat/helper"
"github.com/elastic/beats/metricbeat/mb"
"github.com/elastic/beats/metricbeat/mb/parse"
)

// init registers the MetricSet with the central registry.
// The New method will be called after the setup of the module and before starting to fetch data
func init() {
if err := mb.Registry.AddMetricSet("elasticsearch", "stats", New, hostParser); err != nil {
panic(err)
}
}

var (
hostParser = parse.URLHostParserBuilder{
DefaultScheme: "http",
PathConfigKey: "path",
DefaultPath: "_stats",
}.Build()
)

// MetricSet type defines all fields of the MetricSet
type MetricSet struct {
mb.BaseMetricSet
http *helper.HTTP
}

// New create a new instance of the MetricSet
func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
logp.Experimental("Elasticsearch stats metricset is enabled.")
return &MetricSet{
base,
helper.NewHTTP(base),
}, nil
}

// Fetch gathers stats for each overall _stats API (not specific to an index)
// More details can be found here: https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-stats.html
func (m *MetricSet) Fetch() (common.MapStr, error) {

content, err := m.http.FetchContent()
if err != nil {
return nil, err
}

event, _ := eventMapping(content)
return event, nil
}
57 changes: 57 additions & 0 deletions metricbeat/module/elasticsearch/stats/stats_integration_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
// +build integration

package stats

import (
"fmt"
"net/http"
"strings"
"testing"

mbtest "github.com/elastic/beats/metricbeat/mb/testing"
"github.com/elastic/beats/metricbeat/module/elasticsearch"
"github.com/stretchr/testify/assert"
)

func TestFetch(t *testing.T) {
err := loadData()
if err != nil {
t.Fatal("write", err)
}

f := mbtest.NewEventFetcher(t, elasticsearch.GetConfig("stats"))
event, err := f.Fetch()
if !assert.NoError(t, err) {
t.FailNow()
}

assert.NotNil(t, event)
t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
}

func TestData(t *testing.T) {
err := loadData()
if err != nil {
t.Fatal("write", err)
}

f := mbtest.NewEventFetcher(t, elasticsearch.GetConfig("stats"))
err = mbtest.WriteEvent(f, t)
if err != nil {
t.Fatal("write", err)
}
}

func loadData() error {
client := &http.Client{}
url := fmt.Sprintf("http://%s:%s/tests/stats", elasticsearch.GetEnvHost(), elasticsearch.GetEnvPort())

request, err := http.NewRequest("POST", url, strings.NewReader(`{"hello":"world"}`))
request.Header.Set("Content-Type", "application/json")
if err != nil {
return err
}
res, err := client.Do(request)
defer res.Body.Close()
return err
}
29 changes: 29 additions & 0 deletions metricbeat/module/elasticsearch/testing.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
package elasticsearch

import "os"

func GetEnvHost() string {
host := os.Getenv("ELASTICSEARCH_HOST")

if len(host) == 0 {
host = "127.0.0.1"
}
return host
}

func GetEnvPort() string {
port := os.Getenv("ELASTICSEARCH_PORT")

if len(port) == 0 {
port = "9200"
}
return port
}

func GetConfig(metricset string) map[string]interface{} {
return map[string]interface{}{
"module": "elasticsearch",
"metricsets": []string{metricset},
"hosts": []string{GetEnvHost() + ":" + GetEnvPort()},
}
}
32 changes: 32 additions & 0 deletions metricbeat/tests/system/test_elasticsearch.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
import os
import metricbeat
import unittest


class Test(metricbeat.BaseTest):

@unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test")
def test_stats(self):
"""
elasticsearch stats metricset test
"""
self.render_config_template(modules=[{
"name": "elasticsearch",
"metricsets": ["stats"],
"hosts": self.get_hosts(),
"period": "1s",
}])
proc = self.start_beat()
self.wait_until(lambda: self.output_lines() > 0, max_timeout=20)
proc.check_kill_and_wait()

output = self.read_output_json()
self.assertTrue(len(output) >= 1)
evt = output[0]
print(evt)

self.assert_fields_are_documented(evt)

def get_hosts(self):
return [os.getenv('ELASTICSEARCH_HOST', 'localhost') + ':' +
os.getenv('ELASTICSEARCH_PORT', '9200')]

0 comments on commit a6cc12f

Please sign in to comment.