From 386d48a41e3f59c8a7611688bad190b09591f8b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9F=A6=E9=92=B0=E8=BF=AA?= Date: Thu, 2 Nov 2017 09:57:58 +0800 Subject: [PATCH 1/8] Add osd_tree code --- metricbeat/include/list.go | 1 + .../module/ceph/osd_tree/_meta/data.json | 35 +++++++ .../module/ceph/osd_tree/_meta/docs.asciidoc | 3 + .../module/ceph/osd_tree/_meta/fields.yml | 57 +++++++++++ metricbeat/module/ceph/osd_tree/data.go | 95 +++++++++++++++++++ metricbeat/module/ceph/osd_tree/osd_tree.go | 58 +++++++++++ .../osd_tree/osd_tree_integration_test.go | 48 ++++++++++ .../module/ceph/osd_tree/osd_tree_test.go | 88 +++++++++++++++++ 8 files changed, 385 insertions(+) create mode 100644 metricbeat/module/ceph/osd_tree/_meta/data.json create mode 100644 metricbeat/module/ceph/osd_tree/_meta/docs.asciidoc create mode 100644 metricbeat/module/ceph/osd_tree/_meta/fields.yml create mode 100644 metricbeat/module/ceph/osd_tree/data.go create mode 100644 metricbeat/module/ceph/osd_tree/osd_tree.go create mode 100644 metricbeat/module/ceph/osd_tree/osd_tree_integration_test.go create mode 100644 metricbeat/module/ceph/osd_tree/osd_tree_test.go diff --git a/metricbeat/include/list.go b/metricbeat/include/list.go index f4dfb760b13e..5b9b308d03f2 100644 --- a/metricbeat/include/list.go +++ b/metricbeat/include/list.go @@ -17,6 +17,7 @@ import ( _ "github.com/elastic/beats/metricbeat/module/ceph/cluster_health" _ "github.com/elastic/beats/metricbeat/module/ceph/cluster_status" _ "github.com/elastic/beats/metricbeat/module/ceph/monitor_health" + _ "github.com/elastic/beats/metricbeat/module/ceph/osd_tree" _ "github.com/elastic/beats/metricbeat/module/ceph/pool_disk" _ "github.com/elastic/beats/metricbeat/module/couchbase" _ "github.com/elastic/beats/metricbeat/module/couchbase/bucket" diff --git a/metricbeat/module/ceph/osd_tree/_meta/data.json b/metricbeat/module/ceph/osd_tree/_meta/data.json new file mode 100644 index 000000000000..277cdd677e83 --- /dev/null +++ b/metricbeat/module/ceph/osd_tree/_meta/data.json @@ -0,0 +1,35 @@ +{ + "@timestamp": "2017-11-01T07:26:34.876Z", + "@metadata": { + "beat": "metricbeat", + "type": "doc", + "version": "7.0.0-alpha1" + }, + "ceph": { + "osd_tree": { + "status": "up", + "name": "osd.0", + "type": "osd", + "primary_affinity": 1, + "exists": 1, + "id": 0, + "type_id": 0, + "crush_weight": 0.048691, + "device_class": "hdd", + "reweight": 1, + "father": "ceph-mon1", + "depth": 2 + } + }, + "metricset": { + "rtt": 1331122, + "module": "ceph", + "name": "osd_tree", + "host": "192.168.56.131:5000" + }, + "beat": { + "hostname": "centos7gui", + "version": "7.0.0-alpha1", + "name": "centos7gui" + } +} \ No newline at end of file diff --git a/metricbeat/module/ceph/osd_tree/_meta/docs.asciidoc b/metricbeat/module/ceph/osd_tree/_meta/docs.asciidoc new file mode 100644 index 000000000000..4a14002a18be --- /dev/null +++ b/metricbeat/module/ceph/osd_tree/_meta/docs.asciidoc @@ -0,0 +1,3 @@ +=== Ceph cluster_health metricset + +This is the `osd_tree` metricset of the Ceph module. \ No newline at end of file diff --git a/metricbeat/module/ceph/osd_tree/_meta/fields.yml b/metricbeat/module/ceph/osd_tree/_meta/fields.yml new file mode 100644 index 000000000000..c63100c6a328 --- /dev/null +++ b/metricbeat/module/ceph/osd_tree/_meta/fields.yml @@ -0,0 +1,57 @@ +- name: osd_tree + type: group + description: > + ceph osd tree info + fields: + - name: id + type: long + description: > + osd or bucket node id + - name: name + type: string + description: > + osd or bucket node name + - name: type + type: string + description: > + osd or bucket node type, illegal type include osd, host, root etc. + - name: type_id + type: long + description: > + osd or bucket node typeID + - name: children + type: string + description: > + bucket children list, seperate by comma. + - name: crush_weight + type: float + description: > + osd node crush weight + - name: depth + type: long + description: > + node depth + - name: exists + type: long + description: > + is node still exist or not(1-yes, 0-no) + - name: primary_affinity + type: float + description: > + the weight of reading data from primary osd + - name: reweight + type: long + description: > + the reweight of osd + - name: status + type: string + description: > + status of osd, it should be up or down + - name: device_class + type: string + description: > + the device class of osd, like hdd, ssd etc. + - name: father + type: string + description: > + the parent node of this osd or bucket node \ No newline at end of file diff --git a/metricbeat/module/ceph/osd_tree/data.go b/metricbeat/module/ceph/osd_tree/data.go new file mode 100644 index 000000000000..a7d54e79991c --- /dev/null +++ b/metricbeat/module/ceph/osd_tree/data.go @@ -0,0 +1,95 @@ +package osd_tree + +import ( + "encoding/json" + "strconv" + "strings" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/logp" +) + +type Node struct { + Id int64 `json:"id"` + Name string `json:"name"` + Type string `json:"type"` + TypeId int64 `json:"type_id"` + Children []int64 `json:"children"` + + CrushWeight float64 `json:"crush_weight"` + Depth int64 `json:"depth"` + Exist int64 `json:"exists"` + PrimaryAffinity float64 `json:"primary_affinity"` + Reweight float64 `json:"reweight"` + Status string `json:"status"` + DeviceClass string `json:"device_class"` +} + +type Output struct { + Nodes []Node `json:"nodes"` +} + +type OsdTreeRequest struct { + Status string `json:"status"` + Output Output `json:"output"` +} + +func eventsMapping(content []byte) ([]common.MapStr, error) { + var d OsdTreeRequest + err := json.Unmarshal(content, &d) + if err != nil { + logp.Err("Error: ", err) + return nil, err + } + + nodeList := d.Output.Nodes + + //generate fatherNode and children map + fatherMap := make(map[string]string) + childrenMap := make(map[string]string) + + for _, node := range nodeList { + if node.Id >= 0 { + //it's osd node + continue + } + childrenList := []string{} + for _, child := range node.Children { + childIdStr := strconv.FormatInt(child, 10) + childrenList = append(childrenList, childIdStr) + fatherMap[childIdStr] = node.Name + } + //generate bucket node's children list + childrenMap[node.Name] = strings.Join(childrenList, ",") + } + + //osd node list + events := []common.MapStr{} + for _, node := range nodeList { + nodeInfo := common.MapStr{} + if node.Id < 0 { + //bucket node + nodeInfo["children"] = childrenMap[node.Name] + } else { + //osd node + nodeInfo["crush_weight"] = node.CrushWeight + nodeInfo["depth"] = node.Depth + nodeInfo["exists"] = node.Exist + nodeInfo["primary_affinity"] = node.PrimaryAffinity + nodeInfo["reweight"] = node.Reweight + nodeInfo["status"] = node.Status + nodeInfo["device_class"] = node.DeviceClass + } + nodeInfo["id"] = node.Id + nodeInfo["name"] = node.Name + nodeInfo["type"] = node.Type + nodeInfo["type_id"] = node.TypeId + + idStr := strconv.FormatInt(node.Id, 10) + nodeInfo["father"] = fatherMap[idStr] + + events = append(events, nodeInfo) + } + + return events, nil +} diff --git a/metricbeat/module/ceph/osd_tree/osd_tree.go b/metricbeat/module/ceph/osd_tree/osd_tree.go new file mode 100644 index 000000000000..c54cf5f9a63d --- /dev/null +++ b/metricbeat/module/ceph/osd_tree/osd_tree.go @@ -0,0 +1,58 @@ +package osd_tree + +import ( + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/common/cfgwarn" + "github.com/elastic/beats/metricbeat/helper" + "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/metricbeat/mb/parse" +) + +const ( + defaultScheme = "http" + defaultPath = "/api/v0.1/osd/tree" +) + +var ( + hostParser = parse.URLHostParserBuilder{ + DefaultScheme: defaultScheme, + DefaultPath: defaultPath, + }.Build() +) + +func init() { + if err := mb.Registry.AddMetricSet("ceph", "osd_tree", New, hostParser); err != nil { + panic(err) + } +} + +type MetricSet struct { + mb.BaseMetricSet + *helper.HTTP +} + +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + cfgwarn.Beta("The ceph osd_tree metricset is beta") + + http := helper.NewHTTP(base) + http.SetHeader("Accept", "application/json") + + return &MetricSet{ + base, + http, + }, nil +} + +func (m *MetricSet) Fetch() ([]common.MapStr, error) { + content, err := m.HTTP.FetchContent() + if err != nil { + return nil, err + } + + events, err := eventsMapping(content) + if err != nil { + return nil, err + } + + return events, nil +} diff --git a/metricbeat/module/ceph/osd_tree/osd_tree_integration_test.go b/metricbeat/module/ceph/osd_tree/osd_tree_integration_test.go new file mode 100644 index 000000000000..12464c6b131c --- /dev/null +++ b/metricbeat/module/ceph/osd_tree/osd_tree_integration_test.go @@ -0,0 +1,48 @@ +package osd_tree + +import ( + "fmt" + "os" + "testing" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" +) + +func TestData(t *testing.T) { + f := mbtest.NewEventsFetcher(t, getConfig()) + err := mbtest.WriteEvents(f, t) + if err != nil { + t.Fatal("write", err) + } +} + +func getConfig() map[string]interface{} { + return map[string]interface{}{ + "module": "ceph", + "metricsets": []string{"osd_tree"}, + "hosts": getTestCephHost(), + } +} + +const ( + cephDefaultHost = "127.0.0.1" + cephDefaultPort = "5000" +) + +func getTestCephHost() string { + return fmt.Sprintf("%v:%v", + getenv("CEPH_HOST", cephDefaultHost), + getenv("CEPH_PORT", cephDefaultPort), + ) +} + +func getenv(name, defaultValue string) string { + return strDefault(os.Getenv(name), defaultValue) +} + +func strDefault(a, defaults string) string { + if len(a) == 0 { + return defaults + } + return a +} diff --git a/metricbeat/module/ceph/osd_tree/osd_tree_test.go b/metricbeat/module/ceph/osd_tree/osd_tree_test.go new file mode 100644 index 000000000000..2d877045181d --- /dev/null +++ b/metricbeat/module/ceph/osd_tree/osd_tree_test.go @@ -0,0 +1,88 @@ +package osd_tree + +import ( + "io/ioutil" + "net/http" + "net/http/httptest" + "path/filepath" + "testing" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + + "github.com/stretchr/testify/assert" +) + +func TestFetchEventContents(t *testing.T) { + absPath, err := filepath.Abs("../_meta/testdata/") + assert.NoError(t, err) + + response, err := ioutil.ReadFile(absPath + "/osd_tree_sample_response.json") + assert.NoError(t, err) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + w.Header().Set("Content-Type", "application/json;") + w.Write([]byte(response)) + })) + defer server.Close() + + config := map[string]interface{}{ + "module": "ceph", + "metricsets": []string{"osd_tree"}, + "hosts": []string{server.URL}, + } + + f := mbtest.NewEventsFetcher(t, config) + events, err := f.Fetch() + event := events[0] + + t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event.StringToPrint()) + + //check root bucket info + nodeInfo := events[0] + assert.EqualValues(t, "default", nodeInfo["name"]) + assert.EqualValues(t, "root", nodeInfo["type"]) + assert.EqualValues(t, "-3", nodeInfo["children"]) + assert.EqualValues(t, -1, nodeInfo["id"]) + assert.EqualValues(t, 10, nodeInfo["type_id"]) + assert.EqualValues(t, "", nodeInfo["father"]) + + //check host bucket info + nodeInfo = events[1] + assert.EqualValues(t, "ceph-mon1", nodeInfo["name"]) + assert.EqualValues(t, "host", nodeInfo["type"]) + assert.EqualValues(t, "1,0", nodeInfo["children"]) + assert.EqualValues(t, -3, nodeInfo["id"]) + assert.EqualValues(t, 1, nodeInfo["type_id"]) + assert.EqualValues(t, "default", nodeInfo["father"]) + + //check osd bucket info + nodeInfo = events[2] + assert.EqualValues(t, "up", nodeInfo["status"]) + assert.EqualValues(t, "osd.0", nodeInfo["name"]) + assert.EqualValues(t, "osd", nodeInfo["type"]) + assert.EqualValues(t, 1, nodeInfo["primary_affinity"]) + assert.EqualValues(t, 1, nodeInfo["exists"]) + assert.EqualValues(t, 0, nodeInfo["id"]) + assert.EqualValues(t, 0, nodeInfo["type_id"]) + assert.EqualValues(t, 0.048691, nodeInfo["crush_weight"]) + assert.EqualValues(t, "hdd", nodeInfo["device_class"]) + assert.EqualValues(t, 1, nodeInfo["reweight"]) + assert.EqualValues(t, "ceph-mon1", nodeInfo["father"]) + assert.EqualValues(t, 2, nodeInfo["depth"]) + + nodeInfo = events[3] + assert.EqualValues(t, "up", nodeInfo["status"]) + assert.EqualValues(t, "osd.1", nodeInfo["name"]) + assert.EqualValues(t, "osd", nodeInfo["type"]) + assert.EqualValues(t, 1, nodeInfo["primary_affinity"]) + assert.EqualValues(t, 1, nodeInfo["exists"]) + assert.EqualValues(t, 1, nodeInfo["id"]) + assert.EqualValues(t, 0, nodeInfo["type_id"]) + assert.EqualValues(t, 0.048691, nodeInfo["crush_weight"]) + assert.EqualValues(t, "hdd", nodeInfo["device_class"]) + assert.EqualValues(t, 1, nodeInfo["reweight"]) + assert.EqualValues(t, "ceph-mon1", nodeInfo["father"]) + assert.EqualValues(t, 2, nodeInfo["depth"]) + +} From 8582680198ae137b2e30717ac4b348f1a432aa81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9F=A6=E9=92=B0=E8=BF=AA?= Date: Thu, 2 Nov 2017 10:04:03 +0800 Subject: [PATCH 2/8] add osd_tree sample response file --- .../testdata/osd_tree_sample_response.json | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 metricbeat/module/ceph/_meta/testdata/osd_tree_sample_response.json diff --git a/metricbeat/module/ceph/_meta/testdata/osd_tree_sample_response.json b/metricbeat/module/ceph/_meta/testdata/osd_tree_sample_response.json new file mode 100644 index 000000000000..1f43d7c68f15 --- /dev/null +++ b/metricbeat/module/ceph/_meta/testdata/osd_tree_sample_response.json @@ -0,0 +1,56 @@ +{ + "output": { + "nodes": [ + { + "children": [ + -3 + ], + "id": -1, + "name": "default", + "type": "root", + "type_id": 10 + }, + { + "children": [ + 1, + 0 + ], + "id": -3, + "name": "ceph-mon1", + "pool_weights": {}, + "type": "host", + "type_id": 1 + }, + { + "crush_weight": 0.048691, + "depth": 2, + "device_class": "hdd", + "exists": 1, + "id": 0, + "name": "osd.0", + "pool_weights": {}, + "primary_affinity": 1.0, + "reweight": 1.0, + "status": "up", + "type": "osd", + "type_id": 0 + }, + { + "crush_weight": 0.048691, + "depth": 2, + "device_class": "hdd", + "exists": 1, + "id": 1, + "name": "osd.1", + "pool_weights": {}, + "primary_affinity": 1.0, + "reweight": 1.0, + "status": "up", + "type": "osd", + "type_id": 0 + } + ], + "stray": [] + }, + "status": "OK" +} \ No newline at end of file From 9d121700ff92b8b0c74631f187473e6d72ece108 Mon Sep 17 00:00:00 2001 From: elaron Date: Wed, 1 Nov 2017 21:07:44 +0800 Subject: [PATCH 3/8] update asciidoc files --- metricbeat/docs/fields.asciidoc | 111 ++++++++++++++++++ metricbeat/docs/modules/ceph.asciidoc | 4 + .../docs/modules/ceph/osd_tree.asciidoc | 19 +++ 3 files changed, 134 insertions(+) create mode 100644 metricbeat/docs/modules/ceph/osd_tree.asciidoc diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index a4e3eab94c9b..5d559506f42b 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -1192,6 +1192,117 @@ type: long Last updated +[float] +== osd_tree fields + +ceph osd tree info + + + +[float] +=== `ceph.osd_tree.id` + +type: long + +osd or bucket node id + + +[float] +=== `ceph.osd_tree.name` + +type: string + +osd or bucket node name + + +[float] +=== `ceph.osd_tree.type` + +type: string + +osd or bucket node type, illegal type include osd, host, root etc. + + +[float] +=== `ceph.osd_tree.type_id` + +type: long + +osd or bucket node typeID + + +[float] +=== `ceph.osd_tree.children` + +type: string + +bucket children list, seperate by comma. + + +[float] +=== `ceph.osd_tree.crush_weight` + +type: float + +osd node crush weight + + +[float] +=== `ceph.osd_tree.depth` + +type: long + +node depth + + +[float] +=== `ceph.osd_tree.exists` + +type: long + +is node still exist or not(1-yes, 0-no) + + +[float] +=== `ceph.osd_tree.primary_affinity` + +type: float + +the weight of reading data from primary osd + + +[float] +=== `ceph.osd_tree.reweight` + +type: long + +the reweight of osd + + +[float] +=== `ceph.osd_tree.status` + +type: string + +status of osd, it should be up or down + + +[float] +=== `ceph.osd_tree.device_class` + +type: string + +the device class of osd, like hdd, ssd etc. + + +[float] +=== `ceph.osd_tree.father` + +type: string + +the parent node of this osd or bucket node + + [float] == pool_disk fields diff --git a/metricbeat/docs/modules/ceph.asciidoc b/metricbeat/docs/modules/ceph.asciidoc index 5b2843395c1e..6bdab755554a 100644 --- a/metricbeat/docs/modules/ceph.asciidoc +++ b/metricbeat/docs/modules/ceph.asciidoc @@ -39,6 +39,8 @@ The following metricsets are available: * <> +* <> + * <> include::ceph/cluster_disk.asciidoc[] @@ -49,5 +51,7 @@ include::ceph/cluster_status.asciidoc[] include::ceph/monitor_health.asciidoc[] +include::ceph/osd_tree.asciidoc[] + include::ceph/pool_disk.asciidoc[] diff --git a/metricbeat/docs/modules/ceph/osd_tree.asciidoc b/metricbeat/docs/modules/ceph/osd_tree.asciidoc new file mode 100644 index 000000000000..a6b08c502b4d --- /dev/null +++ b/metricbeat/docs/modules/ceph/osd_tree.asciidoc @@ -0,0 +1,19 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[metricbeat-metricset-ceph-osd_tree]] +include::../../../module/ceph/osd_tree/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/ceph/osd_tree/_meta/data.json[] +---- From 1f1aa1f177e7d64b919957a1ca864ce116c1c90a Mon Sep 17 00:00:00 2001 From: elaron Date: Thu, 2 Nov 2017 04:42:53 +0800 Subject: [PATCH 4/8] update fields.yml and CHANGLOG --- CHANGELOG.asciidoc | 1 + metricbeat/docs/fields.asciidoc | 8 ++++---- metricbeat/module/ceph/osd_tree/_meta/data.json | 2 +- metricbeat/module/ceph/osd_tree/_meta/fields.yml | 8 ++++---- metricbeat/module/ceph/osd_tree/data.go | 6 +++++- metricbeat/module/ceph/osd_tree/osd_tree_test.go | 4 ++-- 6 files changed, 17 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 429e4b0e5498..b371a3d10b76 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -108,6 +108,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di - Add Kubernetes manifests to deploy Metricbeat. {pull}5349[5349] - Add etcd module. {issue}4970[4970] - Add ip address of docker containers to event. {pull}5379[5379] +- Add ceph osd tree infomation to metricbeat {pull}5498[5498] *Packetbeat* diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 5d559506f42b..2f25ff6de917 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -1210,7 +1210,7 @@ osd or bucket node id [float] === `ceph.osd_tree.name` -type: string +type: text osd or bucket node name @@ -1218,7 +1218,7 @@ osd or bucket node name [float] === `ceph.osd_tree.type` -type: string +type: keyword osd or bucket node type, illegal type include osd, host, root etc. @@ -1258,7 +1258,7 @@ node depth [float] === `ceph.osd_tree.exists` -type: long +type: boolean is node still exist or not(1-yes, 0-no) @@ -1290,7 +1290,7 @@ status of osd, it should be up or down [float] === `ceph.osd_tree.device_class` -type: string +type: keyword the device class of osd, like hdd, ssd etc. diff --git a/metricbeat/module/ceph/osd_tree/_meta/data.json b/metricbeat/module/ceph/osd_tree/_meta/data.json index 277cdd677e83..b6def740376e 100644 --- a/metricbeat/module/ceph/osd_tree/_meta/data.json +++ b/metricbeat/module/ceph/osd_tree/_meta/data.json @@ -11,7 +11,7 @@ "name": "osd.0", "type": "osd", "primary_affinity": 1, - "exists": 1, + "exists": true, "id": 0, "type_id": 0, "crush_weight": 0.048691, diff --git a/metricbeat/module/ceph/osd_tree/_meta/fields.yml b/metricbeat/module/ceph/osd_tree/_meta/fields.yml index c63100c6a328..78d54731b8b6 100644 --- a/metricbeat/module/ceph/osd_tree/_meta/fields.yml +++ b/metricbeat/module/ceph/osd_tree/_meta/fields.yml @@ -8,11 +8,11 @@ description: > osd or bucket node id - name: name - type: string + type: text description: > osd or bucket node name - name: type - type: string + type: keyword description: > osd or bucket node type, illegal type include osd, host, root etc. - name: type_id @@ -32,7 +32,7 @@ description: > node depth - name: exists - type: long + type: boolean description: > is node still exist or not(1-yes, 0-no) - name: primary_affinity @@ -48,7 +48,7 @@ description: > status of osd, it should be up or down - name: device_class - type: string + type: keyword description: > the device class of osd, like hdd, ssd etc. - name: father diff --git a/metricbeat/module/ceph/osd_tree/data.go b/metricbeat/module/ceph/osd_tree/data.go index a7d54e79991c..fd460ce022ff 100644 --- a/metricbeat/module/ceph/osd_tree/data.go +++ b/metricbeat/module/ceph/osd_tree/data.go @@ -74,11 +74,15 @@ func eventsMapping(content []byte) ([]common.MapStr, error) { //osd node nodeInfo["crush_weight"] = node.CrushWeight nodeInfo["depth"] = node.Depth - nodeInfo["exists"] = node.Exist nodeInfo["primary_affinity"] = node.PrimaryAffinity nodeInfo["reweight"] = node.Reweight nodeInfo["status"] = node.Status nodeInfo["device_class"] = node.DeviceClass + if node.Exist > 0 { + nodeInfo["exists"] = true + } else { + nodeInfo["exists"] = false + } } nodeInfo["id"] = node.Id nodeInfo["name"] = node.Name diff --git a/metricbeat/module/ceph/osd_tree/osd_tree_test.go b/metricbeat/module/ceph/osd_tree/osd_tree_test.go index 2d877045181d..4a4763239520 100644 --- a/metricbeat/module/ceph/osd_tree/osd_tree_test.go +++ b/metricbeat/module/ceph/osd_tree/osd_tree_test.go @@ -62,7 +62,7 @@ func TestFetchEventContents(t *testing.T) { assert.EqualValues(t, "osd.0", nodeInfo["name"]) assert.EqualValues(t, "osd", nodeInfo["type"]) assert.EqualValues(t, 1, nodeInfo["primary_affinity"]) - assert.EqualValues(t, 1, nodeInfo["exists"]) + assert.EqualValues(t, true, nodeInfo["exists"]) assert.EqualValues(t, 0, nodeInfo["id"]) assert.EqualValues(t, 0, nodeInfo["type_id"]) assert.EqualValues(t, 0.048691, nodeInfo["crush_weight"]) @@ -76,7 +76,7 @@ func TestFetchEventContents(t *testing.T) { assert.EqualValues(t, "osd.1", nodeInfo["name"]) assert.EqualValues(t, "osd", nodeInfo["type"]) assert.EqualValues(t, 1, nodeInfo["primary_affinity"]) - assert.EqualValues(t, 1, nodeInfo["exists"]) + assert.EqualValues(t, true, nodeInfo["exists"]) assert.EqualValues(t, 1, nodeInfo["id"]) assert.EqualValues(t, 0, nodeInfo["type_id"]) assert.EqualValues(t, 0.048691, nodeInfo["crush_weight"]) From b6d4acd06432006ed6f4971ca6e7da762740b535 Mon Sep 17 00:00:00 2001 From: elaron Date: Thu, 2 Nov 2017 04:57:09 +0800 Subject: [PATCH 5/8] update ceph config.yml --- metricbeat/docs/modules/ceph.asciidoc | 2 +- metricbeat/metricbeat.reference.yml | 2 +- metricbeat/module/ceph/_meta/config.yml | 2 +- metricbeat/modules.d/ceph.yml.disabled | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/metricbeat/docs/modules/ceph.asciidoc b/metricbeat/docs/modules/ceph.asciidoc index 6bdab755554a..273c346e50d3 100644 --- a/metricbeat/docs/modules/ceph.asciidoc +++ b/metricbeat/docs/modules/ceph.asciidoc @@ -21,7 +21,7 @@ in <>. Here is an example configuration: ---- metricbeat.modules: - module: ceph - metricsets: ["cluster_disk", "cluster_health", "monitor_health", "pool_disk"] + metricsets: ["cluster_disk", "cluster_health", "monitor_health", "pool_disk", "osd_tree"] period: 10s hosts: ["localhost:5000"] ---- diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 53d585a83a00..d2943591bc8a 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -119,7 +119,7 @@ metricbeat.modules: #-------------------------------- Ceph Module -------------------------------- - module: ceph - metricsets: ["cluster_disk", "cluster_health", "monitor_health", "pool_disk"] + metricsets: ["cluster_disk", "cluster_health", "monitor_health", "pool_disk", "osd_tree"] period: 10s hosts: ["localhost:5000"] diff --git a/metricbeat/module/ceph/_meta/config.yml b/metricbeat/module/ceph/_meta/config.yml index 21ee68217332..a069c8e69b4b 100644 --- a/metricbeat/module/ceph/_meta/config.yml +++ b/metricbeat/module/ceph/_meta/config.yml @@ -1,4 +1,4 @@ - module: ceph - metricsets: ["cluster_disk", "cluster_health", "monitor_health", "pool_disk"] + metricsets: ["cluster_disk", "cluster_health", "monitor_health", "pool_disk", "osd_tree"] period: 10s hosts: ["localhost:5000"] diff --git a/metricbeat/modules.d/ceph.yml.disabled b/metricbeat/modules.d/ceph.yml.disabled index 21ee68217332..a069c8e69b4b 100644 --- a/metricbeat/modules.d/ceph.yml.disabled +++ b/metricbeat/modules.d/ceph.yml.disabled @@ -1,4 +1,4 @@ - module: ceph - metricsets: ["cluster_disk", "cluster_health", "monitor_health", "pool_disk"] + metricsets: ["cluster_disk", "cluster_health", "monitor_health", "pool_disk", "osd_tree"] period: 10s hosts: ["localhost:5000"] From e19e598a5396baed03f2199962a3909dacd8e262 Mon Sep 17 00:00:00 2001 From: elaron Date: Sat, 4 Nov 2017 02:10:34 +0800 Subject: [PATCH 6/8] update father and status field's type from string to keyword --- metricbeat/docs/fields.asciidoc | 4 ++-- metricbeat/module/ceph/osd_tree/_meta/fields.yml | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 2f25ff6de917..7152d3c0f7d3 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -1282,7 +1282,7 @@ the reweight of osd [float] === `ceph.osd_tree.status` -type: string +type: keyword status of osd, it should be up or down @@ -1298,7 +1298,7 @@ the device class of osd, like hdd, ssd etc. [float] === `ceph.osd_tree.father` -type: string +type: keyword the parent node of this osd or bucket node diff --git a/metricbeat/module/ceph/osd_tree/_meta/fields.yml b/metricbeat/module/ceph/osd_tree/_meta/fields.yml index 78d54731b8b6..a7c79bdc1762 100644 --- a/metricbeat/module/ceph/osd_tree/_meta/fields.yml +++ b/metricbeat/module/ceph/osd_tree/_meta/fields.yml @@ -44,7 +44,7 @@ description: > the reweight of osd - name: status - type: string + type: keyword description: > status of osd, it should be up or down - name: device_class @@ -52,6 +52,6 @@ description: > the device class of osd, like hdd, ssd etc. - name: father - type: string + type: keyword description: > - the parent node of this osd or bucket node \ No newline at end of file + the parent node of this osd or bucket node From c1b10d0e4ac8d575b9d37d6df69e44c19d0d73b8 Mon Sep 17 00:00:00 2001 From: elaron Date: Sat, 4 Nov 2017 02:39:28 +0800 Subject: [PATCH 7/8] update variable name from Id to ID --- metricbeat/module/ceph/osd_tree/data.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/metricbeat/module/ceph/osd_tree/data.go b/metricbeat/module/ceph/osd_tree/data.go index fd460ce022ff..abec86d61699 100644 --- a/metricbeat/module/ceph/osd_tree/data.go +++ b/metricbeat/module/ceph/osd_tree/data.go @@ -10,10 +10,10 @@ import ( ) type Node struct { - Id int64 `json:"id"` + ID int64 `json:"id"` Name string `json:"name"` Type string `json:"type"` - TypeId int64 `json:"type_id"` + TypeID int64 `json:"type_id"` Children []int64 `json:"children"` CrushWeight float64 `json:"crush_weight"` @@ -49,15 +49,15 @@ func eventsMapping(content []byte) ([]common.MapStr, error) { childrenMap := make(map[string]string) for _, node := range nodeList { - if node.Id >= 0 { + if node.ID >= 0 { //it's osd node continue } childrenList := []string{} for _, child := range node.Children { - childIdStr := strconv.FormatInt(child, 10) - childrenList = append(childrenList, childIdStr) - fatherMap[childIdStr] = node.Name + childIDStr := strconv.FormatInt(child, 10) + childrenList = append(childrenList, childIDStr) + fatherMap[childIDStr] = node.Name } //generate bucket node's children list childrenMap[node.Name] = strings.Join(childrenList, ",") @@ -67,7 +67,7 @@ func eventsMapping(content []byte) ([]common.MapStr, error) { events := []common.MapStr{} for _, node := range nodeList { nodeInfo := common.MapStr{} - if node.Id < 0 { + if node.ID < 0 { //bucket node nodeInfo["children"] = childrenMap[node.Name] } else { @@ -84,12 +84,12 @@ func eventsMapping(content []byte) ([]common.MapStr, error) { nodeInfo["exists"] = false } } - nodeInfo["id"] = node.Id + nodeInfo["id"] = node.ID nodeInfo["name"] = node.Name nodeInfo["type"] = node.Type - nodeInfo["type_id"] = node.TypeId + nodeInfo["type_id"] = node.TypeID - idStr := strconv.FormatInt(node.Id, 10) + idStr := strconv.FormatInt(node.ID, 10) nodeInfo["father"] = fatherMap[idStr] events = append(events, nodeInfo) From 292f10643153fc5cc646dd32610e14b9e0865463 Mon Sep 17 00:00:00 2001 From: elaron Date: Sat, 4 Nov 2017 14:35:57 +0800 Subject: [PATCH 8/8] update children field type from string to text --- metricbeat/docs/fields.asciidoc | 2 +- metricbeat/module/ceph/osd_tree/_meta/fields.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 7152d3c0f7d3..4d10b72c7710 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -1234,7 +1234,7 @@ osd or bucket node typeID [float] === `ceph.osd_tree.children` -type: string +type: text bucket children list, seperate by comma. diff --git a/metricbeat/module/ceph/osd_tree/_meta/fields.yml b/metricbeat/module/ceph/osd_tree/_meta/fields.yml index a7c79bdc1762..5a19be7203e2 100644 --- a/metricbeat/module/ceph/osd_tree/_meta/fields.yml +++ b/metricbeat/module/ceph/osd_tree/_meta/fields.yml @@ -20,7 +20,7 @@ description: > osd or bucket node typeID - name: children - type: string + type: text description: > bucket children list, seperate by comma. - name: crush_weight