From bb81640c4844c1244284ffdbda8ec9ce5d4d7fe3 Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Wed, 5 Feb 2020 16:25:53 +0100 Subject: [PATCH 01/41] Ceph: docker image for nautilus --- .../module/ceph/_meta/Dockerfile.nautilus | 21 +++++++++++++++++++ metricbeat/module/ceph/docker-compose.yml | 13 +++++++++++- 2 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 metricbeat/module/ceph/_meta/Dockerfile.nautilus diff --git a/metricbeat/module/ceph/_meta/Dockerfile.nautilus b/metricbeat/module/ceph/_meta/Dockerfile.nautilus new file mode 100644 index 000000000000..97912c581e14 --- /dev/null +++ b/metricbeat/module/ceph/_meta/Dockerfile.nautilus @@ -0,0 +1,21 @@ +ARG CEPH_VERSION +FROM ceph/daemon:${CEPH_VERSION} + +RUN yum -q install -y jq && yum clean all && rm -fr /var/cache/yum + +# Wait for the health endpoint to have monitors information +HEALTHCHECK --interval=1s --retries=300 \ + CMD ceph health | grep HEALTH_OK && \ + ceph restful list-keys > /opt/ceph-container/sree/static/restful-list-keys.json && \ + curl -s localhost:5000 >/dev/null + +EXPOSE 5000 8003 + +ENV NETWORK_AUTO_DETECT 4 +ENV CEPH_DAEMON demo +ENV CEPH_DEMO_UID beats +ENV CEPH_DEMO_BUCKET beats +ENV CEPH_DEMO_ACCESS_KEY demo +ENV CEPH_DEMO_SECRET_KEY demo + +CMD ["demo"] diff --git a/metricbeat/module/ceph/docker-compose.yml b/metricbeat/module/ceph/docker-compose.yml index e3da5ece1cc9..b951966699cc 100644 --- a/metricbeat/module/ceph/docker-compose.yml +++ b/metricbeat/module/ceph/docker-compose.yml @@ -1,11 +1,22 @@ -version: '2.3' +version: '3.7' services: ceph: image: docker.elastic.co/integrations-ci/beats-ceph:${CEPH_VERSION:-master-6373c6a-jewel-centos-7-x86_64}-1 build: context: ./_meta + dockerfile: Dockerfile args: CEPH_VERSION: ${CEPH_VERSION:-master-6373c6a-jewel-centos-7-x86_64} ports: - 5000 + ceph-nautilus: + image: docker.elastic.co/integrations-ci/beats-ceph:${CEPH_VERSION:-master-97985eb-nautilus-centos-7-x86_64}-1 + build: + context: ./_meta + dockerfile: Dockerfile.nautilus + args: + CEPH_VERSION: ${CEPH_VERSION:-master-97985eb-nautilus-centos-7-x86_64} + ports: + - 5000:5000 + - 8003:8003 From 2ad3479a3c1e2c8adfb8bc83d14df95b44ec8a9d Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Thu, 6 Feb 2020 09:21:53 +0100 Subject: [PATCH 02/41] Expose Prometheus port --- metricbeat/module/ceph/_meta/Dockerfile.nautilus | 2 +- metricbeat/module/ceph/docker-compose.yml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/metricbeat/module/ceph/_meta/Dockerfile.nautilus b/metricbeat/module/ceph/_meta/Dockerfile.nautilus index 97912c581e14..241356412a61 100644 --- a/metricbeat/module/ceph/_meta/Dockerfile.nautilus +++ b/metricbeat/module/ceph/_meta/Dockerfile.nautilus @@ -9,7 +9,7 @@ HEALTHCHECK --interval=1s --retries=300 \ ceph restful list-keys > /opt/ceph-container/sree/static/restful-list-keys.json && \ curl -s localhost:5000 >/dev/null -EXPOSE 5000 8003 +EXPOSE 5000 8003 9283 ENV NETWORK_AUTO_DETECT 4 ENV CEPH_DAEMON demo diff --git a/metricbeat/module/ceph/docker-compose.yml b/metricbeat/module/ceph/docker-compose.yml index b951966699cc..43fe4fdca43f 100644 --- a/metricbeat/module/ceph/docker-compose.yml +++ b/metricbeat/module/ceph/docker-compose.yml @@ -20,3 +20,4 @@ services: ports: - 5000:5000 - 8003:8003 + - 9283:9283 From 4ca3deabd2f7849bdc750b381bebda6405b95152 Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Thu, 6 Feb 2020 13:00:57 +0100 Subject: [PATCH 03/41] Fix: expose missing port --- metricbeat/module/ceph/_meta/Dockerfile | 1 - metricbeat/module/ceph/docker-compose.yml | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/module/ceph/_meta/Dockerfile b/metricbeat/module/ceph/_meta/Dockerfile index 5919c2da6251..19d23bd275c3 100644 --- a/metricbeat/module/ceph/_meta/Dockerfile +++ b/metricbeat/module/ceph/_meta/Dockerfile @@ -8,7 +8,6 @@ HEALTHCHECK --interval=1s --retries=300 \ CMD curl -s -H "Accept: application/json" localhost:5000/api/v0.1/health \ | jq .output.health.health_services[0].mons[0] \ | grep health -EXPOSE 5000 ENV NETWORK_AUTO_DETECT 4 ENV DEMO_DAEMONS osd,rest_api diff --git a/metricbeat/module/ceph/docker-compose.yml b/metricbeat/module/ceph/docker-compose.yml index 43fe4fdca43f..268b214855ae 100644 --- a/metricbeat/module/ceph/docker-compose.yml +++ b/metricbeat/module/ceph/docker-compose.yml @@ -20,4 +20,5 @@ services: ports: - 5000:5000 - 8003:8003 + - 8080:8080 - 9283:9283 From d6338b05c8a0b6a7bd399b49661146bbdb20dba2 Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Fri, 7 Feb 2020 10:39:18 +0100 Subject: [PATCH 04/41] Rename docker service --- metricbeat/module/ceph/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/module/ceph/docker-compose.yml b/metricbeat/module/ceph/docker-compose.yml index 268b214855ae..6a077f485418 100644 --- a/metricbeat/module/ceph/docker-compose.yml +++ b/metricbeat/module/ceph/docker-compose.yml @@ -10,7 +10,7 @@ services: CEPH_VERSION: ${CEPH_VERSION:-master-6373c6a-jewel-centos-7-x86_64} ports: - 5000 - ceph-nautilus: + ceph-mgr: image: docker.elastic.co/integrations-ci/beats-ceph:${CEPH_VERSION:-master-97985eb-nautilus-centos-7-x86_64}-1 build: context: ./_meta From fd38f16ed592013e6c4d6f7b66c23aa1410263f9 Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Sat, 8 Feb 2020 13:43:14 +0100 Subject: [PATCH 05/41] Boilerplate code for mgr_cluster_health --- .../ceph/mgr_cluster_health/_meta/data.json | 19 +++++++ .../mgr_cluster_health/_meta/docs.asciidoc | 1 + .../ceph/mgr_cluster_health/_meta/fields.yml | 10 ++++ .../mgr_cluster_health/mgr_cluster_health.go | 54 +++++++++++++++++++ 4 files changed, 84 insertions(+) create mode 100644 metricbeat/module/ceph/mgr_cluster_health/_meta/data.json create mode 100644 metricbeat/module/ceph/mgr_cluster_health/_meta/docs.asciidoc create mode 100644 metricbeat/module/ceph/mgr_cluster_health/_meta/fields.yml create mode 100644 metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health.go diff --git a/metricbeat/module/ceph/mgr_cluster_health/_meta/data.json b/metricbeat/module/ceph/mgr_cluster_health/_meta/data.json new file mode 100644 index 000000000000..0c492b55550f --- /dev/null +++ b/metricbeat/module/ceph/mgr_cluster_health/_meta/data.json @@ -0,0 +1,19 @@ +{ + "@timestamp":"2016-05-23T08:05:34.853Z", + "beat":{ + "hostname":"beathost", + "name":"beathost" + }, + "metricset":{ + "host":"localhost", + "module":"ceph", + "name":"mgr_cluster_health", + "rtt":44269 + }, + "ceph":{ + "mgr_cluster_health":{ + "example": "mgr_cluster_health" + } + }, + "type":"metricsets" +} diff --git a/metricbeat/module/ceph/mgr_cluster_health/_meta/docs.asciidoc b/metricbeat/module/ceph/mgr_cluster_health/_meta/docs.asciidoc new file mode 100644 index 000000000000..7224d6180951 --- /dev/null +++ b/metricbeat/module/ceph/mgr_cluster_health/_meta/docs.asciidoc @@ -0,0 +1 @@ +This is the mgr_cluster_health metricset of the module ceph. diff --git a/metricbeat/module/ceph/mgr_cluster_health/_meta/fields.yml b/metricbeat/module/ceph/mgr_cluster_health/_meta/fields.yml new file mode 100644 index 000000000000..8c1ba7cd5ee5 --- /dev/null +++ b/metricbeat/module/ceph/mgr_cluster_health/_meta/fields.yml @@ -0,0 +1,10 @@ +- name: mgr_cluster_health + type: group + release: beta + description: > + mgr_cluster_health + fields: + - name: example + type: keyword + description: > + Example field diff --git a/metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health.go b/metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health.go new file mode 100644 index 000000000000..fa7150fadd51 --- /dev/null +++ b/metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health.go @@ -0,0 +1,54 @@ +package mgr_cluster_health + +import ( + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/common/cfgwarn" + "github.com/elastic/beats/metricbeat/mb" +) + +// init registers the MetricSet with the central registry as soon as the program +// starts. The New function will be called later to instantiate an instance of +// the MetricSet for each host defined in the module's configuration. After the +// MetricSet has been created then Fetch will begin to be called periodically. +func init() { + mb.Registry.MustAddMetricSet("ceph", "mgr_cluster_health", New) +} + +// MetricSet holds any configuration or state information. It must implement +// the mb.MetricSet interface. And this is best achieved by embedding +// mb.BaseMetricSet because it implements all of the required mb.MetricSet +// interface methods except for Fetch. +type MetricSet struct { + mb.BaseMetricSet + counter int +} + +// New creates a new instance of the MetricSet. New is responsible for unpacking +// any MetricSet specific configuration options if there are any. +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + cfgwarn.Beta("The ceph mgr_cluster_health metricset is beta.") + + config := struct{}{} + if err := base.Module().UnpackConfig(&config); err != nil { + return nil, err + } + + return &MetricSet{ + BaseMetricSet: base, + counter: 1, + }, nil +} + +// Fetch methods implements the data gathering and data conversion to the right +// format. It publishes the event which is then forwarded to the output. In case +// of an error set the Error field of mb.Event or simply call report.Error(). +func (m *MetricSet) Fetch(report mb.ReporterV2) error { + report.Event(mb.Event{ + MetricSetFields: common.MapStr{ + "counter": m.counter, + }, + }) + m.counter++ + + return nil +} From 8652b6f7ae6305b93f6aff0f9676e64fab547dbb Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Sat, 8 Feb 2020 22:28:04 +0100 Subject: [PATCH 06/41] Use light module --- metricbeat/docs/fields.asciidoc | 5 ++ metricbeat/include/fields/fields.go | 2 +- metricbeat/module/ceph/fields.go | 4 +- .../ceph/mgr_cluster_health/_meta/data.json | 19 ------- .../mgr_cluster_health/_meta/docs.asciidoc | 1 - .../ceph/mgr_cluster_health/_meta/fields.yml | 9 +--- .../ceph/mgr_cluster_health/manifest.yml | 25 +++++++++ .../mgr_cluster_health/mgr_cluster_health.go | 54 ------------------- metricbeat/module/ceph/module.yml | 3 ++ 9 files changed, 38 insertions(+), 84 deletions(-) delete mode 100644 metricbeat/module/ceph/mgr_cluster_health/_meta/data.json delete mode 100644 metricbeat/module/ceph/mgr_cluster_health/_meta/docs.asciidoc create mode 100644 metricbeat/module/ceph/mgr_cluster_health/manifest.yml delete mode 100644 metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health.go create mode 100644 metricbeat/module/ceph/module.yml diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 754041ae8ba0..6c7704a0aea6 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -3932,6 +3932,11 @@ type: long -- +[float] +=== ceph + +Status metrics of Ceph cluster + [float] === monitor_health diff --git a/metricbeat/include/fields/fields.go b/metricbeat/include/fields/fields.go index 75e43d7e3cfd..c1a18bbe8893 100644 --- a/metricbeat/include/fields/fields.go +++ b/metricbeat/include/fields/fields.go @@ -24,7 +24,7 @@ import ( ) func init() { - if err := asset.SetFields("metricbeat", "../libbeat/fields.yml", asset.LibbeatFieldsPri, AssetLibbeatFieldsYml); err != nil { + if err := asset.SetFields("metricbeat", "/Users/marcin.tojek/go/src/github.com/elastic/beats/libbeat/fields.yml", asset.LibbeatFieldsPri, AssetUsersMarcinTojekGoSrcGithubComElasticBeatsLibbeatFieldsYml); err != nil { panic(err) } } diff --git a/metricbeat/module/ceph/fields.go b/metricbeat/module/ceph/fields.go index 8f8636152e87..ff6d517af979 100644 --- a/metricbeat/module/ceph/fields.go +++ b/metricbeat/module/ceph/fields.go @@ -30,7 +30,7 @@ func init() { } // AssetCeph returns asset data. -// This is the base64 encoded gzipped contents of ../metricbeat/module/ceph. +// This is the base64 encoded gzipped contents of /Users/marcin.tojek/go/src/github.com/elastic/beats/metricbeat/module/ceph. func AssetCeph() string { - return "eJzEms9v27gSx+/5KwY5vQckxntXHx7Q1y3QYDdtsGlPi4VKkyOJa0okOFS8/u8XpGTZ1k/boV2fWtv5zsczo+HMSI+wxu0SOJr8DsBJp3AJ9x/R5Pd3AAKJW2mc1OUS/ncHAOA/gkKLSuEdAOXauoTrMpXZElKmyL9rUSEjXELG/HfQOVlmtIQ/7onU/QPc586Z+z/vAFKJStAyKD9CyQpsWfzLbY1XsboyzTsDRP71w//RD+C6dEyWBC5HKNBZyf2/mYMNWgTilhkUkFpdwMdPL58XjcAhxhGKqsihTYSkdfvhENYEmn+N6Bz7affqwhwCsTcmFVspXKy2DunoOzsupcus88EEmn992KlCUAWdBgc21J2vp9oWzC2hD7CDdNoxFRXwm1eMA1cRiqhs3wnF5WjdTMuRKZfHyLWe0vnZpt/QMqUScsxVw/5a43ajrTjPZV9rXah1p9zWppQskOfI17RAo3keKXbPzMAbWpK6nDNrdVWKxRtTFUYy3opD0D4NIHYoXo9CcAzSzc2e7eHcHM6zEzK2pz+Vm0NhuzwW4VRrfDGZEJalqeQLi0wkZ1WR8co0D1f7B7xRcLnVVZabyoFBC4Rcj+VOw7qx0uHNYYPVC2iDZ7VJjE8I5LHie+hCqQ2d57yr8dReOgWokGQU47gIp2tkjJ04mAzKqliNVOGWQa/+Qu5inaA9ikb+JBTLnNSDIMSZQpGkSjM3kt8GLcey++m5uH2EHaXAzDKB4ipB24nPBK1luE7QWooTgtai/LygtbjjQTPZQjDHbl4xTQbe7kTvarJFGABujsaOB4QxuJDlN4dz++FgDMw3/TfnqtrBYAQrNDwYWjpM/JuRLs2XLHTWePi9aQSuq94Fdan111xvCHK9gYKVWzAZAbMIsmygdDr703t8kZu9JkA03+9pEou0UsOle6W1QtalmjH+RF4UeqKHFktk9ipWvfCM6apINIlYp0QnGbyyz4S5cW9HUplrw/jU1DVQm57fXyahZPkzoJ6+TEJZLJgxKBKT3Zrs90/PH15ePv0yyhdzYg9a3R6jbQp1KZ0+eYly9qB6rB88QOHMPnFm3W/vDI9Vbve7u6Yv2s3yz1+HM6bnHHjfBuFz7YwZq/ufvl5F/+W//n/OfL2SjGa63kfOmw3LxmhWw6Zx3qhi5JLKCOawG8rasv/kzN8rC4RNjiVsGMGQ9s74aBtzUW59YQXO/Vxy2mLoE2ihdBZ1t/ubzvar3S7AKdvmQ7hCEo9K9yyJx8MjclHpXl+/xYO7+i2F9wLOXnQXJB8j17vWDs7VRKTxjziOJg+topC0hopY5lv32hfHTfLUQSdj+SA0rVpgV/E65aa1NjqRCHyTHBOuGEVcwrd2vcYDSKUwYyr8D2TJVSUQciEegEgAOr6YOON8ut5gyG1TpJ6+37SqRlzW3nC7JVWduBNUF95IjcG2X6ZM8JksKasi0lVE3ZEclOa+rNS9vAzD4XjwxprVa+3qOlFsNFiGg1XQWcTZOnhKxfNCodbN1s+blT5tYVXxNbobF8GO3dFy6M1c1fJEQdQkHiDX5B7Aau0m6uLWYHLNsHipp+Hxl+dSCYvDm6uLnNQY3gmDkt4DhIbZcEWvtsB1UbBhX3BbUZ5sUGb58DU9dDGfenoFcRgQ35+eZmTyPD8OwWJfcGcK/5Y0cq/jogWapNoiOalUre7ToNTuX/993CI9wH8eS/3v4VpuZcHsNmFpKkvptrEc76ei2tu+g7XIhCybewjhCafG7mhxtziRCBc8z5BjK+mBxszGfoRh/xRJKAnS+fOuUgJWCJXxURJ6M7zevU475x1RK0NQbtGUXJ/SyKXM5b3Hh97JY5jFsqlXYaCuD/1OKesdsEZr9e6n34ZEfuLB+iR2KwUPdoMT9XCLMWqyHiNv+nxfj+W0oddzxr2X/CWsc8Nl0txEPs1bN3iW7x0+usLyb7064vonAAD//8IHLlo=" + return "eJzEmk2P2zYTx+/7KQZ7eh5g12ivPhRI0wBZtJssusmpKBSaHEmsKZEgKbv+9gWpF9t6t5fW+pTY3v/8PDMazoz0CFs8rIGiSu8ALLcC13D/EVV6fwfA0FDNleUyX8MvdwAA7iPIJCsE3gGYVGobUZnHPFlDTIRx72oUSAyuISHuO2gtzxOzhr/ujRH3D3CfWqvu/74DiDkKZtZe+RFykmHD4l72oJyKloWq3ukhcq8f7o9+AJW5JTw3YFOEDK3m1P2bWNijRjBUE4UMYi0z+Pjp5fOqEjjFOEMRhbGoI8bNtvmwD2sEzb0GdM79VL/aMKdAZEe4IBuBq83Bojn7Ts0lZJ60PhhBc68PtSp4VZCxd2BF3fp6LHVG7Bq6ADWklZaIoIDfnGIYuMIgC8r23SC7Hq2daSkSYdMQudZRujzb5A41ESIyltii319bPOylZpe57GupC6XumNualOIZ0hTp1qxQSZoGit0zUbBDbbjMp8xqWeRstSOiwEDGG3Hw2vMAQofi9SwE5yDt3OzY7s/N/jybkbEd/bHc7Avb9bHwp1rli9GE0CSOOV1pJCy6qIoMV6ZpuNI/4IyCTbUsklQVFhRqMEjlUO5UrHvNLS4O661eQes9K1WkXEIgDRXfUxdyqcxlzrsZT+mlOUAZN0oQiit/ugbGqMVBJZAX2WagCjcMcvMPUhvqBO1QVPKzUDSxXPaCGEoEsigWktiB/FaoKebtTy/F7SLUlAwTTRiymwStFp8IWsNwm6A1FDOC1qC8X9Aa3OGgqWTFiCWLV0yVgLM70ruqZOUHgMXRyPmAMATns3xxOHscDobAXNO/OFfRDAYDWL7hQd/SYeTeDHRpviS+s8bT740jUFl0Lqhrrb+mcm8glXvISH4AlRggGoHnFZSMJ396hy9ws1cFyEz3e9KwVVyI/tK9kVIgaVNNGH8yThQ6oqcWcyT6Jlad8ITpIoukYaFOiVYyOGWXCVPjXk1SqFvDuNSUJVCTnt9fRqF4/h5QT19GoTRmRClkkUqWJvvz0/OHl5dPvw3yhZzYvVa7x+jZHMK81Uk18tWrQhmXk2A3N5uhdoN2bH3S9Kcy51bO3udcPDOf6/tgGN8+zByfj4tERUNV/uMasWrR6rXC89f+5O04B962zPhcOmPC6vGnbzfBf/nvv06ZL7ejwUyXq9Fps37vGcyqX3pOGxXE2KhQjFhsh7K07D658PfyDGGfYg57YqBPuzY+2FFdlVtfSIZTP9dYqdG3LGYlZBJ0zfyHTI5b5jbAnMX3KVzGDQ1K98wNDYdnjA1K9/r6LRzcze9uvBVw8qK7IvmIsZ1r7eSIj1gc/ohzp7nvWhk3WygMSdwUUfrivF8fO+h4KB/4/lkybCveptw01gaHI4Y7TjGigpiA9wMau07jAbgQmBDh/wc8p6JgCCljD2AMA7R0NXLGuXRdYN5uUqRcBOykKAZc1tz7W5KqTNwRqivv6YZgO+51RvhUEuVFFugqMu3tAAhJXVkpxwru59Th4A01q7daG7aiWGmQBHuroNWIl80fAxXPCflaN1k/Fyt9UsOmoFu0CxfBlt3BcujM3NTySEGUhj1AKo19AC2lHamLB4XRLcPipJ76J3GacsE09i/RrnJSZbgWBsGdBwwqov0VvTkAlVlG+n1BdWHSaI88Sfuv6b6Lee7p5cWhR/x4eqqByfPyOHiLXcHaFP7LzcBtl6t2edyUFo3lQpTqLg1yaf/38+MBzQP89JjL//fXcs0zog8RiWOec3sI5Xg3FZXedh2sRsJ4Xt3O8A9bVXYHi7vGkUS44tGKFBtJBzRkNvTTFMcHWnxJ4Nadd4VgsEEolIsSk/v+TfNt2jnniFIZvHKDJvh2TiMXE5t2nmR6I48iGvOqXvmBujz0W6Wsc8AqKcWbH8TrE3nHg/WJ1SsFB7bAiXq6xRg0WY6Riz5q2GGZN/Q6zrC3tb/4zbK/TKr72fO8tcBjhW/w0Q2Wf9vNGdd/AQAA//91H1SV" } diff --git a/metricbeat/module/ceph/mgr_cluster_health/_meta/data.json b/metricbeat/module/ceph/mgr_cluster_health/_meta/data.json deleted file mode 100644 index 0c492b55550f..000000000000 --- a/metricbeat/module/ceph/mgr_cluster_health/_meta/data.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "@timestamp":"2016-05-23T08:05:34.853Z", - "beat":{ - "hostname":"beathost", - "name":"beathost" - }, - "metricset":{ - "host":"localhost", - "module":"ceph", - "name":"mgr_cluster_health", - "rtt":44269 - }, - "ceph":{ - "mgr_cluster_health":{ - "example": "mgr_cluster_health" - } - }, - "type":"metricsets" -} diff --git a/metricbeat/module/ceph/mgr_cluster_health/_meta/docs.asciidoc b/metricbeat/module/ceph/mgr_cluster_health/_meta/docs.asciidoc deleted file mode 100644 index 7224d6180951..000000000000 --- a/metricbeat/module/ceph/mgr_cluster_health/_meta/docs.asciidoc +++ /dev/null @@ -1 +0,0 @@ -This is the mgr_cluster_health metricset of the module ceph. diff --git a/metricbeat/module/ceph/mgr_cluster_health/_meta/fields.yml b/metricbeat/module/ceph/mgr_cluster_health/_meta/fields.yml index 8c1ba7cd5ee5..a65d2efbbe78 100644 --- a/metricbeat/module/ceph/mgr_cluster_health/_meta/fields.yml +++ b/metricbeat/module/ceph/mgr_cluster_health/_meta/fields.yml @@ -1,10 +1,5 @@ -- name: mgr_cluster_health +- name: ceph type: group + description: Status metrics of Ceph cluster release: beta - description: > - mgr_cluster_health fields: - - name: example - type: keyword - description: > - Example field diff --git a/metricbeat/module/ceph/mgr_cluster_health/manifest.yml b/metricbeat/module/ceph/mgr_cluster_health/manifest.yml new file mode 100644 index 000000000000..f1818ee0242e --- /dev/null +++ b/metricbeat/module/ceph/mgr_cluster_health/manifest.yml @@ -0,0 +1,25 @@ +default: true +input: + module: http + metricset: json + defaults: + method: POST + namespace: "json_namespace" + basepath: "/request" + headers: + Content-Type: application/json + query: + wait: "1" + body: '{"prefix": "status", "format": "json"}' + +processors: + - extract_array: + field: http.json_namespace.finished + mappings: + ceph.request: 0 + - decode_json_fields: + fields: ["ceph.request.outb"] + process_array: true + target: "ceph.mgr_cluster_status.output" + - drop_fields: + fields: ["http", "ceph.request"] diff --git a/metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health.go b/metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health.go deleted file mode 100644 index fa7150fadd51..000000000000 --- a/metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health.go +++ /dev/null @@ -1,54 +0,0 @@ -package mgr_cluster_health - -import ( - "github.com/elastic/beats/libbeat/common" - "github.com/elastic/beats/libbeat/common/cfgwarn" - "github.com/elastic/beats/metricbeat/mb" -) - -// init registers the MetricSet with the central registry as soon as the program -// starts. The New function will be called later to instantiate an instance of -// the MetricSet for each host defined in the module's configuration. After the -// MetricSet has been created then Fetch will begin to be called periodically. -func init() { - mb.Registry.MustAddMetricSet("ceph", "mgr_cluster_health", New) -} - -// MetricSet holds any configuration or state information. It must implement -// the mb.MetricSet interface. And this is best achieved by embedding -// mb.BaseMetricSet because it implements all of the required mb.MetricSet -// interface methods except for Fetch. -type MetricSet struct { - mb.BaseMetricSet - counter int -} - -// New creates a new instance of the MetricSet. New is responsible for unpacking -// any MetricSet specific configuration options if there are any. -func New(base mb.BaseMetricSet) (mb.MetricSet, error) { - cfgwarn.Beta("The ceph mgr_cluster_health metricset is beta.") - - config := struct{}{} - if err := base.Module().UnpackConfig(&config); err != nil { - return nil, err - } - - return &MetricSet{ - BaseMetricSet: base, - counter: 1, - }, nil -} - -// Fetch methods implements the data gathering and data conversion to the right -// format. It publishes the event which is then forwarded to the output. In case -// of an error set the Error field of mb.Event or simply call report.Error(). -func (m *MetricSet) Fetch(report mb.ReporterV2) error { - report.Event(mb.Event{ - MetricSetFields: common.MapStr{ - "counter": m.counter, - }, - }) - m.counter++ - - return nil -} diff --git a/metricbeat/module/ceph/module.yml b/metricbeat/module/ceph/module.yml new file mode 100644 index 000000000000..0172e994d1ff --- /dev/null +++ b/metricbeat/module/ceph/module.yml @@ -0,0 +1,3 @@ +name: ceph +metricsets: + - mgr_cluster_health From 02bda673da3fafbc2eb2255d517905d91ebb3028 Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Sun, 9 Feb 2020 10:16:38 +0100 Subject: [PATCH 07/41] Strip fields --- metricbeat/module/ceph/mgr_cluster_health/manifest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/module/ceph/mgr_cluster_health/manifest.yml b/metricbeat/module/ceph/mgr_cluster_health/manifest.yml index f1818ee0242e..9f17946fbdca 100644 --- a/metricbeat/module/ceph/mgr_cluster_health/manifest.yml +++ b/metricbeat/module/ceph/mgr_cluster_health/manifest.yml @@ -22,4 +22,4 @@ processors: process_array: true target: "ceph.mgr_cluster_status.output" - drop_fields: - fields: ["http", "ceph.request"] + fields: ["http", "ceph.request", "ceph.mgr_cluster_status.output.mgrmap"] From 965e4811795c567fb2003ee877b1493f0f67a3a9 Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Mon, 10 Feb 2020 11:32:30 +0100 Subject: [PATCH 08/41] Ceph: add mgr_cluster_disk metricset --- metricbeat/module/ceph/fields.go | 2 +- .../ceph/mgr_cluster_disk/_meta/fields.yml | 5 ++++ .../module/ceph/mgr_cluster_disk/manifest.yml | 25 +++++++++++++++++++ .../ceph/mgr_cluster_health/manifest.yml | 4 +-- metricbeat/module/ceph/module.yml | 1 + 5 files changed, 34 insertions(+), 3 deletions(-) create mode 100644 metricbeat/module/ceph/mgr_cluster_disk/_meta/fields.yml create mode 100644 metricbeat/module/ceph/mgr_cluster_disk/manifest.yml diff --git a/metricbeat/module/ceph/fields.go b/metricbeat/module/ceph/fields.go index ff6d517af979..0f2965533a27 100644 --- a/metricbeat/module/ceph/fields.go +++ b/metricbeat/module/ceph/fields.go @@ -32,5 +32,5 @@ func init() { // AssetCeph returns asset data. // This is the base64 encoded gzipped contents of /Users/marcin.tojek/go/src/github.com/elastic/beats/metricbeat/module/ceph. func AssetCeph() string { - return "eJzEmk2P2zYTx+/7KQZ7eh5g12ivPhRI0wBZtJssusmpKBSaHEmsKZEgKbv+9gWpF9t6t5fW+pTY3v/8PDMazoz0CFs8rIGiSu8ALLcC13D/EVV6fwfA0FDNleUyX8MvdwAA7iPIJCsE3gGYVGobUZnHPFlDTIRx72oUSAyuISHuO2gtzxOzhr/ujRH3D3CfWqvu/74DiDkKZtZe+RFykmHD4l72oJyKloWq3ukhcq8f7o9+AJW5JTw3YFOEDK3m1P2bWNijRjBUE4UMYi0z+Pjp5fOqEjjFOEMRhbGoI8bNtvmwD2sEzb0GdM79VL/aMKdAZEe4IBuBq83Bojn7Ts0lZJ60PhhBc68PtSp4VZCxd2BF3fp6LHVG7Bq6ADWklZaIoIDfnGIYuMIgC8r23SC7Hq2daSkSYdMQudZRujzb5A41ESIyltii319bPOylZpe57GupC6XumNualOIZ0hTp1qxQSZoGit0zUbBDbbjMp8xqWeRstSOiwEDGG3Hw2vMAQofi9SwE5yDt3OzY7s/N/jybkbEd/bHc7Avb9bHwp1rli9GE0CSOOV1pJCy6qIoMV6ZpuNI/4IyCTbUsklQVFhRqMEjlUO5UrHvNLS4O661eQes9K1WkXEIgDRXfUxdyqcxlzrsZT+mlOUAZN0oQiit/ugbGqMVBJZAX2WagCjcMcvMPUhvqBO1QVPKzUDSxXPaCGEoEsigWktiB/FaoKebtTy/F7SLUlAwTTRiymwStFp8IWsNwm6A1FDOC1qC8X9Aa3OGgqWTFiCWLV0yVgLM70ruqZOUHgMXRyPmAMATns3xxOHscDobAXNO/OFfRDAYDWL7hQd/SYeTeDHRpviS+s8bT740jUFl0Lqhrrb+mcm8glXvISH4AlRggGoHnFZSMJ396hy9ws1cFyEz3e9KwVVyI/tK9kVIgaVNNGH8yThQ6oqcWcyT6Jlad8ITpIoukYaFOiVYyOGWXCVPjXk1SqFvDuNSUJVCTnt9fRqF4/h5QT19GoTRmRClkkUqWJvvz0/OHl5dPvw3yhZzYvVa7x+jZHMK81Uk18tWrQhmXk2A3N5uhdoN2bH3S9Kcy51bO3udcPDOf6/tgGN8+zByfj4tERUNV/uMasWrR6rXC89f+5O04B962zPhcOmPC6vGnbzfBf/nvv06ZL7ejwUyXq9Fps37vGcyqX3pOGxXE2KhQjFhsh7K07D658PfyDGGfYg57YqBPuzY+2FFdlVtfSIZTP9dYqdG3LGYlZBJ0zfyHTI5b5jbAnMX3KVzGDQ1K98wNDYdnjA1K9/r6LRzcze9uvBVw8qK7IvmIsZ1r7eSIj1gc/ohzp7nvWhk3WygMSdwUUfrivF8fO+h4KB/4/lkybCveptw01gaHI4Y7TjGigpiA9wMau07jAbgQmBDh/wc8p6JgCCljD2AMA7R0NXLGuXRdYN5uUqRcBOykKAZc1tz7W5KqTNwRqivv6YZgO+51RvhUEuVFFugqMu3tAAhJXVkpxwru59Th4A01q7daG7aiWGmQBHuroNWIl80fAxXPCflaN1k/Fyt9UsOmoFu0CxfBlt3BcujM3NTySEGUhj1AKo19AC2lHamLB4XRLcPipJ76J3GacsE09i/RrnJSZbgWBsGdBwwqov0VvTkAlVlG+n1BdWHSaI88Sfuv6b6Lee7p5cWhR/x4eqqByfPyOHiLXcHaFP7LzcBtl6t2edyUFo3lQpTqLg1yaf/38+MBzQP89JjL//fXcs0zog8RiWOec3sI5Xg3FZXedh2sRsJ4Xt3O8A9bVXYHi7vGkUS44tGKFBtJBzRkNvTTFMcHWnxJ4Nadd4VgsEEolIsSk/v+TfNt2jnniFIZvHKDJvh2TiMXE5t2nmR6I48iGvOqXvmBujz0W6Wsc8AqKcWbH8TrE3nHg/WJ1SsFB7bAiXq6xRg0WY6Riz5q2GGZN/Q6zrC3tb/4zbK/TKr72fO8tcBjhW/w0Q2Wf9vNGdd/AQAA//91H1SV" + return "eJzMms2O2zYQx+/7FIM9tcCu0V59KJCmAbJoN1l0k1NRKDQ5klhTIkFS6/rtC1IftvVtm1bqU2J5//PTcDicGekRtrhfA0WV3gFYbgWu4f49qvT+DoChoZory2W+hl/uAADcJcgkKwTeAZhUahtRmcc8WUNMhHHfahRIDK4hIe43aC3PE7OGv+6NEfcPcJ9aq+7/vgOIOQpm1l75EXKSYcPiPnavnIqWhaq+6SFyn2/uj74BlbklPDdgU4QMrebU/ZtY2KFGMFQThQxiLTN4/+Hl46oSOMY4QRGFsagjxs22udiHNYLmPgM6p36qP22YYyDyRrggG4Grzd6iOflNzSVknrQujKC5z7taFbwqyNg7sKJu/TyWOiN2DV2AGtJKS0RQwC9OMQxcYZAFZftqkF2O1o60FImwaYhY6yidH23yDTURIjKW2KLfX1vc76Rm57nsc6kLpe6Y25qQ4hnSFOnWrFBJmgZau2ei4A214TKfMqtlkbPVGxEFBjLeiIPXngcQeileT5bgFKQdmx3b/bHZH2czIrajPxabfct2+Vr4U63yxWhAaBLHnK40EhadlUWGM9M0XOkfcEbBploWSaoKCwo1GKRyKHYq1p3mFheH9VYvoPWelSpSLiCQhlrfYxdyqcx5zrsZT+mlOUAZN0oQiit/ugbGqMVBJZAX2WYgCzcMcvMPUhvqBO1QVPKzUDSxXPaCGEoEsigWktiB+FaoKebtq+fidhFqSoaJJgzZTRatFp9YtIbhNovWUMxYtAbl+y1agzu8aCpZMWLJ4hlTJeDsjtSuKln5BmBxNHLaIAzB+ShfHM4emoMhMFf0L85VNI3BAJYveNCXdBi5LwNtzZfEV9Z4/LtxBCqLzoa61PprKncGUrmDjOR7UIkBohF4XkHJePLWO3yBi71qgcx0vScNW8WF6E/dGykFkjbVhPEn40ShI3psMUeib2LVCU+YLrJIGhbqlGgFg1N2kTDV7tUkhbo1jAtNWQI14fn1ZRSK598D6unTKJTGjCiFLFLJ0mR/fnh+9/Ly4bdBvpAdu9dq1xg9k0OYNzqpWr56VCjjshPsxmbT1G7Qjo1P/ocomcy5lbNHS2e376f6Pi6Mr2RmdvKHmaaioQ6hw0SzqhbrCcfz5/591HEOXDdX+Vg6Y8Lq4da3m+B3/vuvU+bLQW0w0+WUdtqsH8EGs+rnr9NGBTE2KhQjFttLWVp2V868X54h7FLMYUcM9GnXxgeLu4ti6xPJcOp2jZUaffVkVkImQSfef8jkMPBuA8yZwR/DZdzQoHTP3NBweMbYoHSvr1/Cwd38Qcu1gJOb7oLgI8Z29tpRtRGxOPwR505zX0AzbrZQGJK4hqb0xWnrMHbQ8VA+8KW8ZNhWvE26aawN9mkM3zjFiApiAj6aaOw6jQfgQmBChP8f8JyKgiGkjD2AMQzQ0tXIGefCda7zr2j9mxApZxJvUhQDLmseQy5JVQbuCNWFj5dDsB1GTCN8KonyIgu0i0x7UAFCUpdWyg6H+5Z5ePGGitVbTTBbq1hpkAR7s6DViOf1HwMZzwn5XDeZPxdLfVLDpqBbtAsnwZbdwXTozNzU8khClIY9QCqNfQAtpR3Ji3uF0S2XxUk99Q8FaMoF09g/z7vISZXhWhgEdx4wqIj2O3qzByqzjPT7gurCpNEOeZL27+m+zTz39PLi0CN+OD3VQOd5/jp4i13B2hT+y83AE6CLxorclBaN5UKU6i4Mcml/+Plxj+YBfnrM5Y/9uVzzjOh9ROKY59zuQznedUWlt10Fq5EwnldPVvx7X5XdweSucSQQLnjLI8VG0gENmQ39Ysfh3RqfErh1510hGGwQCuVWicld/9D7NuWcc0SpDF65QRN8O6eQi4lNOy9VXcmjiMa8yle+oS4P/VYq6xywSkpx9TuBfSLf8WB9YvVIwYEtcKIeTzEGTZZt5KJvPXZY5jW9jjPsE/ZPfsjtt0n1aH2etxZ4w/EKH91g+LfdnHD9FwAA///rbHrQ" } diff --git a/metricbeat/module/ceph/mgr_cluster_disk/_meta/fields.yml b/metricbeat/module/ceph/mgr_cluster_disk/_meta/fields.yml new file mode 100644 index 000000000000..a65d2efbbe78 --- /dev/null +++ b/metricbeat/module/ceph/mgr_cluster_disk/_meta/fields.yml @@ -0,0 +1,5 @@ +- name: ceph + type: group + description: Status metrics of Ceph cluster + release: beta + fields: diff --git a/metricbeat/module/ceph/mgr_cluster_disk/manifest.yml b/metricbeat/module/ceph/mgr_cluster_disk/manifest.yml new file mode 100644 index 000000000000..abc6f2a4c963 --- /dev/null +++ b/metricbeat/module/ceph/mgr_cluster_disk/manifest.yml @@ -0,0 +1,25 @@ +default: true +input: + module: http + metricset: json + defaults: + method: POST + namespace: "json_namespace" + basepath: "/request" + headers: + Content-Type: application/json + query: + wait: "1" + body: '{"prefix": "df", "format": "json"}' + +processors: + - extract_array: + field: http.json_namespace.finished + mappings: + ceph.request: 0 + - decode_json_fields: + fields: ["ceph.request.outb"] + process_array: true + target: "ceph.mgr_cluster_disk" + - drop_fields: + fields: ["http", "ceph.request"] diff --git a/metricbeat/module/ceph/mgr_cluster_health/manifest.yml b/metricbeat/module/ceph/mgr_cluster_health/manifest.yml index 9f17946fbdca..0434082f88a2 100644 --- a/metricbeat/module/ceph/mgr_cluster_health/manifest.yml +++ b/metricbeat/module/ceph/mgr_cluster_health/manifest.yml @@ -20,6 +20,6 @@ processors: - decode_json_fields: fields: ["ceph.request.outb"] process_array: true - target: "ceph.mgr_cluster_status.output" + target: "ceph.mgr_cluster_status" - drop_fields: - fields: ["http", "ceph.request", "ceph.mgr_cluster_status.output.mgrmap"] + fields: ["http", "ceph.request", "ceph.mgr_cluster_status.mgrmap"] diff --git a/metricbeat/module/ceph/module.yml b/metricbeat/module/ceph/module.yml index 0172e994d1ff..67121f26181a 100644 --- a/metricbeat/module/ceph/module.yml +++ b/metricbeat/module/ceph/module.yml @@ -1,3 +1,4 @@ name: ceph metricsets: - mgr_cluster_health + - mgr_cluster_disk From 9c42f10d235d85c09e47fbc7105dde486bffb1d6 Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Mon, 10 Feb 2020 12:01:29 +0100 Subject: [PATCH 09/41] Ceph: add mgr_osd_disk metricset --- metricbeat/docs/fields.asciidoc | 10 ++++++++ metricbeat/module/ceph/fields.go | 2 +- .../ceph/mgr_cluster_disk/_meta/fields.yml | 2 +- .../module/ceph/mgr_osd_disk/_meta/fields.yml | 5 ++++ .../module/ceph/mgr_osd_disk/manifest.yml | 25 +++++++++++++++++++ metricbeat/module/ceph/module.yml | 1 + 6 files changed, 43 insertions(+), 2 deletions(-) create mode 100644 metricbeat/module/ceph/mgr_osd_disk/_meta/fields.yml create mode 100644 metricbeat/module/ceph/mgr_osd_disk/manifest.yml diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 6c7704a0aea6..3a808f54f83d 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -3935,8 +3935,18 @@ type: long [float] === ceph +Disk metrics of Ceph cluster + +[float] +=== ceph + Status metrics of Ceph cluster +[float] +=== ceph + +OSD disk metrics of Ceph cluster + [float] === monitor_health diff --git a/metricbeat/module/ceph/fields.go b/metricbeat/module/ceph/fields.go index 0f2965533a27..3e8e6028358e 100644 --- a/metricbeat/module/ceph/fields.go +++ b/metricbeat/module/ceph/fields.go @@ -32,5 +32,5 @@ func init() { // AssetCeph returns asset data. // This is the base64 encoded gzipped contents of /Users/marcin.tojek/go/src/github.com/elastic/beats/metricbeat/module/ceph. func AssetCeph() string { - return "eJzMms2O2zYQx+/7FIM9tcCu0V59KJCmAbJoN1l0k1NRKDQ5klhTIkFS6/rtC1IftvVtm1bqU2J5//PTcDicGekRtrhfA0WV3gFYbgWu4f49qvT+DoChoZory2W+hl/uAADcJcgkKwTeAZhUahtRmcc8WUNMhHHfahRIDK4hIe43aC3PE7OGv+6NEfcPcJ9aq+7/vgOIOQpm1l75EXKSYcPiPnavnIqWhaq+6SFyn2/uj74BlbklPDdgU4QMrebU/ZtY2KFGMFQThQxiLTN4/+Hl46oSOMY4QRGFsagjxs22udiHNYLmPgM6p36qP22YYyDyRrggG4Grzd6iOflNzSVknrQujKC5z7taFbwqyNg7sKJu/TyWOiN2DV2AGtJKS0RQwC9OMQxcYZAFZftqkF2O1o60FImwaYhY6yidH23yDTURIjKW2KLfX1vc76Rm57nsc6kLpe6Y25qQ4hnSFOnWrFBJmgZau2ei4A214TKfMqtlkbPVGxEFBjLeiIPXngcQeileT5bgFKQdmx3b/bHZH2czIrajPxabfct2+Vr4U63yxWhAaBLHnK40EhadlUWGM9M0XOkfcEbBploWSaoKCwo1GKRyKHYq1p3mFheH9VYvoPWelSpSLiCQhlrfYxdyqcx5zrsZT+mlOUAZN0oQiit/ugbGqMVBJZAX2WYgCzcMcvMPUhvqBO1QVPKzUDSxXPaCGEoEsigWktiB+FaoKebtq+fidhFqSoaJJgzZTRatFp9YtIbhNovWUMxYtAbl+y1agzu8aCpZMWLJ4hlTJeDsjtSuKln5BmBxNHLaIAzB+ShfHM4emoMhMFf0L85VNI3BAJYveNCXdBi5LwNtzZfEV9Z4/LtxBCqLzoa61PprKncGUrmDjOR7UIkBohF4XkHJePLWO3yBi71qgcx0vScNW8WF6E/dGykFkjbVhPEn40ShI3psMUeib2LVCU+YLrJIGhbqlGgFg1N2kTDV7tUkhbo1jAtNWQI14fn1ZRSK598D6unTKJTGjCiFLFLJ0mR/fnh+9/Ly4bdBvpAdu9dq1xg9k0OYNzqpWr56VCjjshPsxmbT1G7Qjo1P/ocomcy5lbNHS2e376f6Pi6Mr2RmdvKHmaaioQ6hw0SzqhbrCcfz5/591HEOXDdX+Vg6Y8Lq4da3m+B3/vuvU+bLQW0w0+WUdtqsH8EGs+rnr9NGBTE2KhQjFttLWVp2V868X54h7FLMYUcM9GnXxgeLu4ti6xPJcOp2jZUaffVkVkImQSfef8jkMPBuA8yZwR/DZdzQoHTP3NBweMbYoHSvr1/Cwd38Qcu1gJOb7oLgI8Z29tpRtRGxOPwR505zX0AzbrZQGJK4hqb0xWnrMHbQ8VA+8KW8ZNhWvE26aawN9mkM3zjFiApiAj6aaOw6jQfgQmBChP8f8JyKgiGkjD2AMQzQ0tXIGefCda7zr2j9mxApZxJvUhQDLmseQy5JVQbuCNWFj5dDsB1GTCN8KonyIgu0i0x7UAFCUpdWyg6H+5Z5ePGGitVbTTBbq1hpkAR7s6DViOf1HwMZzwn5XDeZPxdLfVLDpqBbtAsnwZbdwXTozNzU8khClIY9QCqNfQAtpR3Ji3uF0S2XxUk99Q8FaMoF09g/z7vISZXhWhgEdx4wqIj2O3qzByqzjPT7gurCpNEOeZL27+m+zTz39PLi0CN+OD3VQOd5/jp4i13B2hT+y83AE6CLxorclBaN5UKU6i4Mcml/+Plxj+YBfnrM5Y/9uVzzjOh9ROKY59zuQznedUWlt10Fq5EwnldPVvx7X5XdweSucSQQLnjLI8VG0gENmQ39Ysfh3RqfErh1510hGGwQCuVWicld/9D7NuWcc0SpDF65QRN8O6eQi4lNOy9VXcmjiMa8yle+oS4P/VYq6xywSkpx9TuBfSLf8WB9YvVIwYEtcKIeTzEGTZZt5KJvPXZY5jW9jjPsE/ZPfsjtt0n1aH2etxZ4w/EKH91g+LfdnHD9FwAA///rbHrQ" + return "eJzMmk2P2zYTx+/7KQZ7eh5g12ivPhRIkwBZtJssusmpKBSaHEmsKZHgUOv62xekZK2tV9tLO/Upsbz/+WlmOOSMdA9r3C6Bo8lvAJx0Cpdw+x5NfnsDIJC4lcZJXS7hlxsAAH8JCi0qhTcAlGvrEq7LVGZLSJki/61FhYxwCRnzv0HnZJnREv68JVK3d3CbO2du/7oBSCUqQcugfA8lK7Bl8R+3NV7F6so03wwQ+c93/0ffgevSMVkSuByhQGcl9/9mDjZoEYhbZlBAanUB7z8+fVo0AvsYByiqIoc2EZLW7cUhrAk0/xnROfTT7tOF2QdiL0wqtlK4WG0d0sFvdlxKl1nnwgSa/7zbqUJQBZ0GBzbUnZ+n2hbMLaEPsIN02jEVFfCrV4wDVxGKqGzfCMX5aN1My5Epl8fItZ7S6dmmX9AypRJyzFXD/lrjdqOtOM1lX2pdqHWn3NamlCyQ58jXtECjeR4pdo/MwAtakrqcM2t1VYrFC1MVRjLeikPQPg4gdiieD0JwCNLNzZ7t4dwczrMjMranP5WbQ2E7PxZhV2t8MZkQlqWp5AuLTCQnVZHxyjQPV/sHvFFwudVVlpvKgUELhFyP5U7DurHS4dVhg9UzaINntUmMTwjkseK770KpDZ3mvIvx1F46BqiQZBTjuAi7a2SMnTiYDMqqWI1U4ZZBr/5G7mLtoD2KRv4oFMuc1IMgxJlCkaRKMzeS3wYtx7J79VTcPsKOUmBmmUBxkaDtxGeC1jJcJmgtxRFBa1F+XNBa3PGgmWwhmGNXr5gmA2934uxqskVoAK6Oxg4bhDG4kOVXh3OvzcEYmD/0X52rahuDEaxw4MFwpMPEfxlpaT5l4WSN+7+bRuC66i2oc60/53pDkOsNFKzcgskImEWQZQOl09lb7/FFPuw1AaL5854msUgrNVy6V1orZF2qGeMP5EWhJ7pvsURmL2LVC8+YropEk4i1S3SSwSv7TJhr93Yklbk0jE9NXQO16fntaRJKlj8C6uHzJJTFghmDIjHZtcn++Pj47unp44dRvpgde9DqnjEGJodw3Ojkg6R1OyjUad0H9jOzbWlX6KaGJ+eDNL3nfwHly/MHEJfxS6FL6fTRU66TJwmH+iFFKRyqJthgcLxqeKz98HW42hxcd8OWxy/DS7rnHHjbiOdT7YwZq6+3vl5Fv/Pffp0zX8+Mo5muB8bzZsM0OJrVMAqeN6oYuaQygjnshrK27K+ceL+yQNjkWMKGEQxp74yPnjPPyq3PrMC52yWnLYaDHC2UzqIO33/X2evsvQtwzOOAfbhCEo9K9yiJx8MjclHpnp+/xoO7+DOftwLOLrozko+R6621vYNPItL4W5zfz8NZPmzRFbHM91a1Lw67mKmNTsbyQegqtMCu4mXKTWtttGUU+CI5JlwxiviUpLXrNe5AKoUZU+F/IEuuKoGQC3EHRALQ8cXEHufT9Vjnv2EK0aZIPR550aoacVn7RPSaVHXiTlCd+aQ7BtvrtGuCz2RJWRWRVhF1ZyagNPdlpW62ZOjex4M3dli91DC1E8VGg2U4WAWdRTytAxmpeF4o1LrZ+nm10qctrCq+RnflItixO1oOvZmLWp4oiJrEHeSa3B1Yrd1EXdwaTC4ZFi/1MDyf4LlUwuLwaPEsJzWGd8KgpPcAoWE2rOjVFrguCjbsC24rypMNyiwfXtNDi/nY3SuIw4D46+5pRjrP0+MQLPYFd6bwH0kjD6POmnBKqi2Sk0rV6j4NSu3+9/P9FukOfrov9f+Ha7mVBbPbhKWpLKXbxnK874pqb/sTrEUmZNk85AmvoDV2R4u7xYlEOOOFkxxbSQ80Zjb2Oyavr/mEkiCd3+8qJWCFUBkfJaE3w/P3yxznvCNqZQjKLZqS62MOcilzee/9rjfyGGaxbOpVaKjrTb9TynobrNFavfn1xCGRH7ixPojdSMGDXWFH3Z9ijJqs28irvoDZYzmu6fWccR/2fw7z9rBMmqf8x3nrCi9bvsFHFxj+rVcHXP8GAAD//yDmoD8=" } diff --git a/metricbeat/module/ceph/mgr_cluster_disk/_meta/fields.yml b/metricbeat/module/ceph/mgr_cluster_disk/_meta/fields.yml index a65d2efbbe78..a846844d10a9 100644 --- a/metricbeat/module/ceph/mgr_cluster_disk/_meta/fields.yml +++ b/metricbeat/module/ceph/mgr_cluster_disk/_meta/fields.yml @@ -1,5 +1,5 @@ - name: ceph type: group - description: Status metrics of Ceph cluster + description: Disk metrics of Ceph cluster release: beta fields: diff --git a/metricbeat/module/ceph/mgr_osd_disk/_meta/fields.yml b/metricbeat/module/ceph/mgr_osd_disk/_meta/fields.yml new file mode 100644 index 000000000000..67051db010a5 --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_disk/_meta/fields.yml @@ -0,0 +1,5 @@ +- name: ceph + type: group + description: OSD disk metrics of Ceph cluster + release: beta + fields: diff --git a/metricbeat/module/ceph/mgr_osd_disk/manifest.yml b/metricbeat/module/ceph/mgr_osd_disk/manifest.yml new file mode 100644 index 000000000000..4bf2661c4ef2 --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_disk/manifest.yml @@ -0,0 +1,25 @@ +default: true +input: + module: http + metricset: json + defaults: + method: POST + namespace: "json_namespace" + basepath: "/request" + headers: + Content-Type: application/json + query: + wait: "1" + body: '{"prefix": "osd df", "format": "json"}' + +processors: + - extract_array: + field: http.json_namespace.finished + mappings: + ceph.request: 0 + - decode_json_fields: + fields: ["ceph.request.outb"] + process_array: true + target: "ceph.mgr_osd_disk" + - drop_fields: + fields: ["http", "ceph.request"] diff --git a/metricbeat/module/ceph/module.yml b/metricbeat/module/ceph/module.yml index 67121f26181a..cb7f7154091f 100644 --- a/metricbeat/module/ceph/module.yml +++ b/metricbeat/module/ceph/module.yml @@ -2,3 +2,4 @@ name: ceph metricsets: - mgr_cluster_health - mgr_cluster_disk + - mgr_osd_disk From 61dee28f6469a8cba9fb9632765de12331dc0526 Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Mon, 10 Feb 2020 13:36:10 +0100 Subject: [PATCH 10/41] Ceph: add mgr_osd_* metricsets --- metricbeat/module/ceph/fields.go | 2 +- .../module/ceph/mgr_osd_perf/_meta/fields.yml | 5 ++++ .../module/ceph/mgr_osd_perf/manifest.yml | 25 +++++++++++++++++++ .../ceph/mgr_osd_pool_stats/_meta/fields.yml | 5 ++++ .../ceph/mgr_osd_pool_stats/manifest.yml | 25 +++++++++++++++++++ .../module/ceph/mgr_osd_tree/_meta/fields.yml | 5 ++++ .../module/ceph/mgr_osd_tree/manifest.yml | 25 +++++++++++++++++++ metricbeat/module/ceph/module.yml | 3 +++ 8 files changed, 94 insertions(+), 1 deletion(-) create mode 100644 metricbeat/module/ceph/mgr_osd_perf/_meta/fields.yml create mode 100644 metricbeat/module/ceph/mgr_osd_perf/manifest.yml create mode 100644 metricbeat/module/ceph/mgr_osd_pool_stats/_meta/fields.yml create mode 100644 metricbeat/module/ceph/mgr_osd_pool_stats/manifest.yml create mode 100644 metricbeat/module/ceph/mgr_osd_tree/_meta/fields.yml create mode 100644 metricbeat/module/ceph/mgr_osd_tree/manifest.yml diff --git a/metricbeat/module/ceph/fields.go b/metricbeat/module/ceph/fields.go index 3e8e6028358e..8b37641a4dea 100644 --- a/metricbeat/module/ceph/fields.go +++ b/metricbeat/module/ceph/fields.go @@ -32,5 +32,5 @@ func init() { // AssetCeph returns asset data. // This is the base64 encoded gzipped contents of /Users/marcin.tojek/go/src/github.com/elastic/beats/metricbeat/module/ceph. func AssetCeph() string { - return "eJzMmk2P2zYTx+/7KQZ7eh5g12ivPhRIkwBZtJssusmpKBSaHEmsKZHgUOv62xekZK2tV9tLO/Upsbz/+WlmOOSMdA9r3C6Bo8lvAJx0Cpdw+x5NfnsDIJC4lcZJXS7hlxsAAH8JCi0qhTcAlGvrEq7LVGZLSJki/61FhYxwCRnzv0HnZJnREv68JVK3d3CbO2du/7oBSCUqQcugfA8lK7Bl8R+3NV7F6so03wwQ+c93/0ffgevSMVkSuByhQGcl9/9mDjZoEYhbZlBAanUB7z8+fVo0AvsYByiqIoc2EZLW7cUhrAk0/xnROfTT7tOF2QdiL0wqtlK4WG0d0sFvdlxKl1nnwgSa/7zbqUJQBZ0GBzbUnZ+n2hbMLaEPsIN02jEVFfCrV4wDVxGKqGzfCMX5aN1My5Epl8fItZ7S6dmmX9AypRJyzFXD/lrjdqOtOM1lX2pdqHWn3NamlCyQ58jXtECjeR4pdo/MwAtakrqcM2t1VYrFC1MVRjLeikPQPg4gdiieD0JwCNLNzZ7t4dwczrMjMranP5WbQ2E7PxZhV2t8MZkQlqWp5AuLTCQnVZHxyjQPV/sHvFFwudVVlpvKgUELhFyP5U7DurHS4dVhg9UzaINntUmMTwjkseK770KpDZ3mvIvx1F46BqiQZBTjuAi7a2SMnTiYDMqqWI1U4ZZBr/5G7mLtoD2KRv4oFMuc1IMgxJlCkaRKMzeS3wYtx7J79VTcPsKOUmBmmUBxkaDtxGeC1jJcJmgtxRFBa1F+XNBa3PGgmWwhmGNXr5gmA2934uxqskVoAK6Oxg4bhDG4kOVXh3OvzcEYmD/0X52rahuDEaxw4MFwpMPEfxlpaT5l4WSN+7+bRuC66i2oc60/53pDkOsNFKzcgskImEWQZQOl09lb7/FFPuw1AaL5854msUgrNVy6V1orZF2qGeMP5EWhJ7pvsURmL2LVC8+YropEk4i1S3SSwSv7TJhr93Yklbk0jE9NXQO16fntaRJKlj8C6uHzJJTFghmDIjHZtcn++Pj47unp44dRvpgde9DqnjEGJodw3Ojkg6R1OyjUad0H9jOzbWlX6KaGJ+eDNL3nfwHly/MHEJfxS6FL6fTRU66TJwmH+iFFKRyqJthgcLxqeKz98HW42hxcd8OWxy/DS7rnHHjbiOdT7YwZq6+3vl5Fv/Pffp0zX8+Mo5muB8bzZsM0OJrVMAqeN6oYuaQygjnshrK27K+ceL+yQNjkWMKGEQxp74yPnjPPyq3PrMC52yWnLYaDHC2UzqIO33/X2evsvQtwzOOAfbhCEo9K9yiJx8MjclHpnp+/xoO7+DOftwLOLrozko+R6621vYNPItL4W5zfz8NZPmzRFbHM91a1Lw67mKmNTsbyQegqtMCu4mXKTWtttGUU+CI5JlwxiviUpLXrNe5AKoUZU+F/IEuuKoGQC3EHRALQ8cXEHufT9Vjnv2EK0aZIPR550aoacVn7RPSaVHXiTlCd+aQ7BtvrtGuCz2RJWRWRVhF1ZyagNPdlpW62ZOjex4M3dli91DC1E8VGg2U4WAWdRTytAxmpeF4o1LrZ+nm10qctrCq+RnflItixO1oOvZmLWp4oiJrEHeSa3B1Yrd1EXdwaTC4ZFi/1MDyf4LlUwuLwaPEsJzWGd8KgpPcAoWE2rOjVFrguCjbsC24rypMNyiwfXtNDi/nY3SuIw4D46+5pRjrP0+MQLPYFd6bwH0kjD6POmnBKqi2Sk0rV6j4NSu3+9/P9FukOfrov9f+Ha7mVBbPbhKWpLKXbxnK874pqb/sTrEUmZNk85AmvoDV2R4u7xYlEOOOFkxxbSQ80Zjb2Oyavr/mEkiCd3+8qJWCFUBkfJaE3w/P3yxznvCNqZQjKLZqS62MOcilzee/9rjfyGGaxbOpVaKjrTb9TynobrNFavfn1xCGRH7ixPojdSMGDXWFH3Z9ijJqs28irvoDZYzmu6fWccR/2fw7z9rBMmqf8x3nrCi9bvsFHFxj+rVcHXP8GAAD//yDmoD8=" + return "eJzsmk1vGzcTx+/+FAOfngewhfaqQ4E0CRCjdWLUyakoNhQ5u8uKuyRIrlV9+4LcF0n7Lpla51CdEkn+z08zs0POkPewxf0aKKr0BsByK3ANt+9Rpbc3AAwN1VxZLvM1/HIDAOA+gkyyQuANgEmlthGVecyTNcREGPeuRoHE4BoS4r6D1vI8MWv489YYcXsHt6m16vavG4CYo2Bm7ZXvIScZNizuZffKqWhZqOqdHiL3+u7+6DtQmVvCcwM2RcjQak7dv4mFHWoEQzVRyCDWMoP3H58+rSqBY4wTFFEYizpi3GybD/uwRtDca0Dn1E/1qw1zDEReCBdkI3C12Vs0J9+puYTMk9YHI2ju9a5WBa8KMvYOrKhbX4+lzohdQxeghrTSEhEU8KtTDANXGGRB2b4ZZJejtTMtRSJsGiLXOkrnZ5t8QU2EiIwltuj31xb3O6nZeS77UupCqTvmtialeIY0Rbo1K1SSpoFi90gUvKA2XOZTZrUscrZ6IaLAQMYbcfDa8wBCh+L5JASnIO3c7Njuz83+PJuRsR39sdzsC9vlsfCrWuWL0YTQJI45XWkkLDqrigxXpmm40j/gjIJNtSySVBUWFGowSOVQ7lSsO80tLg7rrV5A6z0rVaRcQiANFd9jF3KpzHnOuxpP6aU5QBk3ShCKK7+6BsaoxUElkBfZZqAKNwxy8zdSG2oF7VBU8rNQNLFc9oIYSgSyKBaS2IH8Vqgp5u1Pz8XtItSUDBNNGLKrBK0Wnwhaw3CdoDUUM4LWoLxd0Brc4aCpZMWIJYtXTJWAszuyd1XJyjcAi6OR0wZhCM5n+eJw9tAcDIG5Tf/iXEXTGAxg+Q0P+i0dRu7NQI/mU+J31nj8vXEEKovOA3Wp9edU7gykcgcZyfegEgNEI/C8gpLx5E/v8AXe7FUBMtP7PWnYKi5Ef+neSCmQtKkmjD8YJwod0WOLORJ9FatOeMJ0kUXSsFCrRCsZnLLLhKl2ryYp1LVhXGrKEqhJz29Po1A8fwuoh8+jUBozohSySCVLk/3x8fHd09PHD4N8ITt2r9XeY/RMDmHe6OQDN9tmUCjjsg/sZmbT0m7Qjg1PLgepes8fAeXL8wdgP4pf/oN5A5hM5tzK2cPIswc+p/q+khi/9x1hg94puKKhti2HGXjVX9Qzsccv/ZW34xx43STuU+mMCauHn77dBP/lv/06Zb4c7QczXc71p836oX0wq35iP21UEGOjQjFisR3K0rL75MzfyzOEXYo57IiBPu3a+GA7cFFufSYZTv1cY6VGv982KyGToGckv8vkcETSBphzanMMl3FDg9I9ckPD4Rljg9I9P38NB3f1o7nXAk4+dBckHzG286wd7U8jFodf4tx67lsuv0QXhiSuBS59cdpsji10PJQPfPMnGbYVr1NuGmuDnT3DF04xooKYgIdZjV2ncQdcCEyI8P8DnlNRMISUsTswhgFauhpZ41y6znX+K4ZFTYqUU6wXKYoBlzUH10tSlYk7QnXhhYQQbIeh5AifSqK8yAI9RaY92gIhqSsrZU/M/ZBlOHhDm9VrzbxbUaw0SIK9VdBqxPM6kIGK54R8rZusn4uVPqlhU9At2oWLYMvuYDl0Zq5qeaQgSsPuIJXG3oGW0o7Uxb3C6JphcVIP/WMkmnLBNPZPgC9yUmW4FgbBnQcMKqL9E73ZA5VZRvp9QXVh0miHPEn7n+m+h3nu6uXFoUf8sHqqgc7z/Dh4i13B2hT+w83AmeFFg2huSovGciFKdZcGubT/+/l+j+YOfrrP5f/7a7nmGdH7iMQxz7ndh3K864pKb7sdrEbCeF6dxfmbgpXdweKucSQRLrgXlGIj6YCGzIa+CnS4jeVLArduvSsEgw1CoVyUmNz1H5NcZzvnHFEqg1du0ATfztnIxcSmnWt4r+RRRGNe1SvfUJeLfquUdRZYJaV49S3SPpE3XFgfWD1ScGALrKjHU4xBk2Ubueg92Q7LvKbXcYa9k/HZH4v4x6S6jDHPWwvciX2Fj64w/NtuTrj+DQAA//8X5BOG" } diff --git a/metricbeat/module/ceph/mgr_osd_perf/_meta/fields.yml b/metricbeat/module/ceph/mgr_osd_perf/_meta/fields.yml new file mode 100644 index 000000000000..67051db010a5 --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_perf/_meta/fields.yml @@ -0,0 +1,5 @@ +- name: ceph + type: group + description: OSD disk metrics of Ceph cluster + release: beta + fields: diff --git a/metricbeat/module/ceph/mgr_osd_perf/manifest.yml b/metricbeat/module/ceph/mgr_osd_perf/manifest.yml new file mode 100644 index 000000000000..e7b4d2454424 --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_perf/manifest.yml @@ -0,0 +1,25 @@ +default: true +input: + module: http + metricset: json + defaults: + method: POST + namespace: "json_namespace" + basepath: "/request" + headers: + Content-Type: application/json + query: + wait: "1" + body: '{"prefix": "osd perf", "format": "json"}' + +processors: + - extract_array: + field: http.json_namespace.finished + mappings: + ceph.request: 0 + - decode_json_fields: + fields: ["ceph.request.outb"] + process_array: true + target: "ceph.mgr_osd_perf" + - drop_fields: + fields: ["http", "ceph.request"] diff --git a/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/fields.yml b/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/fields.yml new file mode 100644 index 000000000000..67051db010a5 --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/fields.yml @@ -0,0 +1,5 @@ +- name: ceph + type: group + description: OSD disk metrics of Ceph cluster + release: beta + fields: diff --git a/metricbeat/module/ceph/mgr_osd_pool_stats/manifest.yml b/metricbeat/module/ceph/mgr_osd_pool_stats/manifest.yml new file mode 100644 index 000000000000..68cf73553243 --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_pool_stats/manifest.yml @@ -0,0 +1,25 @@ +default: true +input: + module: http + metricset: json + defaults: + method: POST + namespace: "json_namespace" + basepath: "/request" + headers: + Content-Type: application/json + query: + wait: "1" + body: '{"prefix": "osd pool stats", "format": "json"}' + +processors: + - extract_array: + field: http.json_namespace.finished + mappings: + ceph.request: 0 + - decode_json_fields: + fields: ["ceph.request.outb"] + process_array: true + target: "ceph.mgr_osd_pool_stats" + - drop_fields: + fields: ["http", "ceph.request"] diff --git a/metricbeat/module/ceph/mgr_osd_tree/_meta/fields.yml b/metricbeat/module/ceph/mgr_osd_tree/_meta/fields.yml new file mode 100644 index 000000000000..67051db010a5 --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_tree/_meta/fields.yml @@ -0,0 +1,5 @@ +- name: ceph + type: group + description: OSD disk metrics of Ceph cluster + release: beta + fields: diff --git a/metricbeat/module/ceph/mgr_osd_tree/manifest.yml b/metricbeat/module/ceph/mgr_osd_tree/manifest.yml new file mode 100644 index 000000000000..91702555f48e --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_tree/manifest.yml @@ -0,0 +1,25 @@ +default: true +input: + module: http + metricset: json + defaults: + method: POST + namespace: "json_namespace" + basepath: "/request" + headers: + Content-Type: application/json + query: + wait: "1" + body: '{"prefix": "osd tree", "format": "json"}' + +processors: + - extract_array: + field: http.json_namespace.finished + mappings: + ceph.request: 0 + - decode_json_fields: + fields: ["ceph.request.outb"] + process_array: true + target: "ceph.mgr_osd_tree" + - drop_fields: + fields: ["http", "ceph.request"] diff --git a/metricbeat/module/ceph/module.yml b/metricbeat/module/ceph/module.yml index cb7f7154091f..57b6c837edb5 100644 --- a/metricbeat/module/ceph/module.yml +++ b/metricbeat/module/ceph/module.yml @@ -3,3 +3,6 @@ metricsets: - mgr_cluster_health - mgr_cluster_disk - mgr_osd_disk + - mgr_osd_pool_stats + - mgr_osd_perf + - mgr_osd_tree From 18dc228724c6f0e58aff3495bb223278759b837a Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Mon, 10 Feb 2020 13:40:10 +0100 Subject: [PATCH 11/41] Update doc --- metricbeat/docs/fields.asciidoc | 27 ++++++++++++++++--- metricbeat/module/ceph/fields.go | 2 +- .../ceph/mgr_cluster_disk/_meta/fields.yml | 5 ++-- .../ceph/mgr_cluster_health/_meta/fields.yml | 5 ++-- .../module/ceph/mgr_osd_disk/_meta/fields.yml | 5 ++-- .../module/ceph/mgr_osd_perf/_meta/fields.yml | 5 ++-- .../ceph/mgr_osd_pool_stats/_meta/fields.yml | 5 ++-- .../module/ceph/mgr_osd_tree/_meta/fields.yml | 5 ++-- 8 files changed, 43 insertions(+), 16 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 3a808f54f83d..e3418ff1f2f6 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -3933,20 +3933,41 @@ type: long -- [float] -=== ceph +=== mgr_cluster_disk Disk metrics of Ceph cluster + [float] -=== ceph +=== mgr_cluster_health Status metrics of Ceph cluster + [float] -=== ceph +=== mgr_osd_disk OSD disk metrics of Ceph cluster + +[float] +=== mgr_osd_perf + +OSD performance metrics of Ceph cluster + + +[float] +=== mgr_osd_pool_stats + +OSD pool stats of Ceph cluster + + +[float] +=== mgr_osd_tree + +OSD tree stats of Ceph cluster + + [float] === monitor_health diff --git a/metricbeat/module/ceph/fields.go b/metricbeat/module/ceph/fields.go index 8b37641a4dea..3f337ac9539f 100644 --- a/metricbeat/module/ceph/fields.go +++ b/metricbeat/module/ceph/fields.go @@ -32,5 +32,5 @@ func init() { // AssetCeph returns asset data. // This is the base64 encoded gzipped contents of /Users/marcin.tojek/go/src/github.com/elastic/beats/metricbeat/module/ceph. func AssetCeph() string { - return "eJzsmk1vGzcTx+/+FAOfngewhfaqQ4E0CRCjdWLUyakoNhQ5u8uKuyRIrlV9+4LcF0n7Lpla51CdEkn+z08zs0POkPewxf0aKKr0BsByK3ANt+9Rpbc3AAwN1VxZLvM1/HIDAOA+gkyyQuANgEmlthGVecyTNcREGPeuRoHE4BoS4r6D1vI8MWv489YYcXsHt6m16vavG4CYo2Bm7ZXvIScZNizuZffKqWhZqOqdHiL3+u7+6DtQmVvCcwM2RcjQak7dv4mFHWoEQzVRyCDWMoP3H58+rSqBY4wTFFEYizpi3GybD/uwRtDca0Dn1E/1qw1zDEReCBdkI3C12Vs0J9+puYTMk9YHI2ju9a5WBa8KMvYOrKhbX4+lzohdQxeghrTSEhEU8KtTDANXGGRB2b4ZZJejtTMtRSJsGiLXOkrnZ5t8QU2EiIwltuj31xb3O6nZeS77UupCqTvmtialeIY0Rbo1K1SSpoFi90gUvKA2XOZTZrUscrZ6IaLAQMYbcfDa8wBCh+L5JASnIO3c7Njuz83+PJuRsR39sdzsC9vlsfCrWuWL0YTQJI45XWkkLDqrigxXpmm40j/gjIJNtSySVBUWFGowSOVQ7lSsO80tLg7rrV5A6z0rVaRcQiANFd9jF3KpzHnOuxpP6aU5QBk3ShCKK7+6BsaoxUElkBfZZqAKNwxy8zdSG2oF7VBU8rNQNLFc9oIYSgSyKBaS2IH8Vqgp5u1Pz8XtItSUDBNNGLKrBK0Wnwhaw3CdoDUUM4LWoLxd0Brc4aCpZMWIJYtXTJWAszuyd1XJyjcAi6OR0wZhCM5n+eJw9tAcDIG5Tf/iXEXTGAxg+Q0P+i0dRu7NQI/mU+J31nj8vXEEKovOA3Wp9edU7gykcgcZyfegEgNEI/C8gpLx5E/v8AXe7FUBMtP7PWnYKi5Ef+neSCmQtKkmjD8YJwod0WOLORJ9FatOeMJ0kUXSsFCrRCsZnLLLhKl2ryYp1LVhXGrKEqhJz29Po1A8fwuoh8+jUBozohSySCVLk/3x8fHd09PHD4N8ITt2r9XeY/RMDmHe6OQDN9tmUCjjsg/sZmbT0m7Qjg1PLgepes8fAeXL8wdgP4pf/oN5A5hM5tzK2cPIswc+p/q+khi/9x1hg94puKKhti2HGXjVX9Qzsccv/ZW34xx43STuU+mMCauHn77dBP/lv/06Zb4c7QczXc71p836oX0wq35iP21UEGOjQjFisR3K0rL75MzfyzOEXYo57IiBPu3a+GA7cFFufSYZTv1cY6VGv982KyGToGckv8vkcETSBphzanMMl3FDg9I9ckPD4Rljg9I9P38NB3f1o7nXAk4+dBckHzG286wd7U8jFodf4tx67lsuv0QXhiSuBS59cdpsji10PJQPfPMnGbYVr1NuGmuDnT3DF04xooKYgIdZjV2ncQdcCEyI8P8DnlNRMISUsTswhgFauhpZ41y6znX+K4ZFTYqUU6wXKYoBlzUH10tSlYk7QnXhhYQQbIeh5AifSqK8yAI9RaY92gIhqSsrZU/M/ZBlOHhDm9VrzbxbUaw0SIK9VdBqxPM6kIGK54R8rZusn4uVPqlhU9At2oWLYMvuYDl0Zq5qeaQgSsPuIJXG3oGW0o7Uxb3C6JphcVIP/WMkmnLBNPZPgC9yUmW4FgbBnQcMKqL9E73ZA5VZRvp9QXVh0miHPEn7n+m+h3nu6uXFoUf8sHqqgc7z/Dh4i13B2hT+w83AmeFFg2huSovGciFKdZcGubT/+/l+j+YOfrrP5f/7a7nmGdH7iMQxz7ndh3K864pKb7sdrEbCeF6dxfmbgpXdweKucSQRLrgXlGIj6YCGzIa+CnS4jeVLArduvSsEgw1CoVyUmNz1H5NcZzvnHFEqg1du0ATfztnIxcSmnWt4r+RRRGNe1SvfUJeLfquUdRZYJaV49S3SPpE3XFgfWD1ScGALrKjHU4xBk2Ubueg92Q7LvKbXcYa9k/HZH4v4x6S6jDHPWwvciX2Fj64w/NtuTrj+DQAA//8X5BOG" + return "eJzEms2O2zYQx+/7FIM9tcCu0V59KJAmAbJoN1l0k1NRKLQ4klhTGoKk1vXbF6Rk2danPyjHp8Ty/uen4XDIGfIR1rhdQowquwOwwkpcwv17VNn9HQBHE2uhrKBiCb/dAQC4R5ATLyXeAZiMtI1iKhKRLiFh0rhvNUpkBpeQMvcbtFYUqVnC3/fGyPsHuM+sVff/3AEkAiU3S6/8CAXLsWFxH7tVTkVTqepveojc57v7o+8QU2GZKAzYDCFHq0Xs/s0sbFAjmFgzhRwSTTm8//jyaVELHGIcocjSWNQRF2bdPOzDGkFznwGdYz/tPm2YQyD2xoRkK4mL1daiOfrNjktSkbYejKC5z7udKnhVoMQ7sKZu/TwhnTO7hC7ADtKSZTIo4FenGAauNMiDsn0zyC9Ha0dahkzaLESsdZTOjzZ6Q82kjIxltuz31xq3G9L8PJd9qXSh0h1zWxNSIsc4w3htFqgozgKN3TNT8IbaCCqmzGoqC754Y7LEQMYbcfDapwGEHorXoyE4BmnHZsd2f2z2x9kJEdvRH4vNvmG7fCz8qlb7YjQgNEsSES80Mh6dlUWGM9M0XOUfcEbBZprKNFOlBYUaDMY0FDs160YLizeH9VYvoPWeJRUpFxAYhxrfQxcKUuY8583GU3npFKBcGCVZjAu/ugbG2ImDSqEo89VAFm4YaPUvxjbUCtqhqOVPQtHMCuoFMTGTyKNEErMD8a1Qx1i0n56L20XYUXJMNePIZxm0nfjEoDUM8wxaQ3HCoDUoP27QGtzhQVPpgjPLbp4xVQrO7sjeVaULXwDcHI0dFwhDcD7Kbw5n98XBEJjb9N+cq2wKgwEsv+FBv6XDyH0ZaGq+pH5njYe/G0eIqexMqEutv2a0MZDRBnJWbEGlBphGEEUNRcnkq3f4Am/26gEy0/s9MnyRlLI/da+IJLI21YTxJ+NEoSN6aLFApmex6oQnTJd5RIaHWiVaweCUXSRMlXs7klLNDeNCkyqgJjy/vYxCieJHQD19HoXSmDOlkEcqvTXZXx+f3728fPwwyBeyYvda7T1GsylMdRSqZfdBmHXTQqSkqhC7MdsUuyu0Y22VPsTrez11xToPJhl+tRe/vH4APp8nHaJCnVyL6DTcgl/EOCcpUdVLm+6hTPESVd2zOTCtRrwW0GmEB6RCWDp53pzdhzrWr/HdlnyEDXqb8yoOtZvat+brsmfXqnv+0r8gdJwD1zUIP1XOmLC6f/X1Kvib//H7lPnqxCGY6eq4YdqsP0sIZtUfJEwblczYqFScWWwPZWXZPTnzfUWOsMmwgA0z0Ke9Mz5YpVwUW59ZjlOvayxprLLmQlIa9OjmT0r3JzdtgFMOkw7hcmHioHTPwsTh8IyxQeleX7+Gg5v9xPBawMlJd0HwMWM7c+1g2xzx6W3N+UctbiV2laDflJWGpa4yr3xxXAOPLXQilA98TUoc24rzpJvG2mDDgeObiDGKJTMBz9gau07jAYSUmDLp/weiiGXJETLOH8AYDmjjxcga58L1VOdf0cNqQqRqrr2RLAdc1pyn35KqCtwRqgvvSYRg2/dKR/hUGhVlHmgWmXbHDSTFLq1UpbrwvZ/hwRvarM7Vim+NYq3BUuzNgtdWI03G8yWJy3WT+fNmqY80rMp4jfbGSbBldzAdOjOzWh5JiGT4A2Rk7ANoIjuSF7cKozmHxUk99Xe34kxIrrG/MX2Rk2rDO2GQwnnAoGLaz+jVFmLKc9bvi1iXJos2KNKsf073TeZTVy8vDj3i+9VTDVSe54+Dt9gV3JnC/4QZOMq8qD8uTGXRWCFlpe7CoCD706+PWzQP8MtjQT/353Itcqa3EUsSUQi7DeV4VxVV3nY7WI2Mi6I+IvQXGGu7g8ld40ggXHBdKcNG0gENmQ19Q2l/ScynBGHdeldKDiuEUrlR4rTpP72ZZzvnHFEpg1du0KRYn7KRS5jNOrcDr+RRTGNR5ytfUFeLfiuVdRZY35G8tsfbJ/IDF9YnvmspOLAbrKiHXYxBk1UZedPrux2W04pexxn2qshnf1rjp0l9R+Q0b93gqu4VPpqh+bdeHXH9HwAA//8tPTqG" } diff --git a/metricbeat/module/ceph/mgr_cluster_disk/_meta/fields.yml b/metricbeat/module/ceph/mgr_cluster_disk/_meta/fields.yml index a846844d10a9..64eb95ee4f69 100644 --- a/metricbeat/module/ceph/mgr_cluster_disk/_meta/fields.yml +++ b/metricbeat/module/ceph/mgr_cluster_disk/_meta/fields.yml @@ -1,5 +1,6 @@ -- name: ceph +- name: mgr_cluster_disk type: group - description: Disk metrics of Ceph cluster + description: > + Disk metrics of Ceph cluster release: beta fields: diff --git a/metricbeat/module/ceph/mgr_cluster_health/_meta/fields.yml b/metricbeat/module/ceph/mgr_cluster_health/_meta/fields.yml index a65d2efbbe78..45072b45c0e4 100644 --- a/metricbeat/module/ceph/mgr_cluster_health/_meta/fields.yml +++ b/metricbeat/module/ceph/mgr_cluster_health/_meta/fields.yml @@ -1,5 +1,6 @@ -- name: ceph +- name: mgr_cluster_health type: group - description: Status metrics of Ceph cluster + description: > + Status metrics of Ceph cluster release: beta fields: diff --git a/metricbeat/module/ceph/mgr_osd_disk/_meta/fields.yml b/metricbeat/module/ceph/mgr_osd_disk/_meta/fields.yml index 67051db010a5..52c0ee98519b 100644 --- a/metricbeat/module/ceph/mgr_osd_disk/_meta/fields.yml +++ b/metricbeat/module/ceph/mgr_osd_disk/_meta/fields.yml @@ -1,5 +1,6 @@ -- name: ceph +- name: mgr_osd_disk type: group - description: OSD disk metrics of Ceph cluster + description: > + OSD disk metrics of Ceph cluster release: beta fields: diff --git a/metricbeat/module/ceph/mgr_osd_perf/_meta/fields.yml b/metricbeat/module/ceph/mgr_osd_perf/_meta/fields.yml index 67051db010a5..c6ceca85837d 100644 --- a/metricbeat/module/ceph/mgr_osd_perf/_meta/fields.yml +++ b/metricbeat/module/ceph/mgr_osd_perf/_meta/fields.yml @@ -1,5 +1,6 @@ -- name: ceph +- name: mgr_osd_perf type: group - description: OSD disk metrics of Ceph cluster + description: > + OSD performance metrics of Ceph cluster release: beta fields: diff --git a/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/fields.yml b/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/fields.yml index 67051db010a5..5d0ea5bf1503 100644 --- a/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/fields.yml +++ b/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/fields.yml @@ -1,5 +1,6 @@ -- name: ceph +- name: mgr_osd_pool_stats type: group - description: OSD disk metrics of Ceph cluster + description: > + OSD pool stats of Ceph cluster release: beta fields: diff --git a/metricbeat/module/ceph/mgr_osd_tree/_meta/fields.yml b/metricbeat/module/ceph/mgr_osd_tree/_meta/fields.yml index 67051db010a5..76a073160a1e 100644 --- a/metricbeat/module/ceph/mgr_osd_tree/_meta/fields.yml +++ b/metricbeat/module/ceph/mgr_osd_tree/_meta/fields.yml @@ -1,5 +1,6 @@ -- name: ceph +- name: mgr_osd_tree type: group - description: OSD disk metrics of Ceph cluster + description: > + OSD tree stats of Ceph cluster release: beta fields: From 6d8f9ad1de8d5b0445018d86a7afd17728f49eb5 Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Mon, 10 Feb 2020 15:09:47 +0100 Subject: [PATCH 12/41] Update metricset docs --- metricbeat/docs/modules/ceph.asciidoc | 28 ++++++++++++++++++- .../modules/ceph/mgr_cluster_disk.asciidoc | 18 ++++++++++++ .../modules/ceph/mgr_cluster_health.asciidoc | 18 ++++++++++++ .../docs/modules/ceph/mgr_osd_disk.asciidoc | 18 ++++++++++++ .../docs/modules/ceph/mgr_osd_perf.asciidoc | 18 ++++++++++++ .../modules/ceph/mgr_osd_pool_stats.asciidoc | 18 ++++++++++++ .../docs/modules/ceph/mgr_osd_tree.asciidoc | 18 ++++++++++++ metricbeat/docs/modules_list.asciidoc | 8 +++++- metricbeat/module/ceph/_meta/docs.asciidoc | 4 ++- .../ceph/mgr_cluster_disk/_meta/docs.asciidoc | 1 + .../mgr_cluster_health/_meta/docs.asciidoc | 1 + .../ceph/mgr_osd_disk/_meta/docs.asciidoc | 1 + .../ceph/mgr_osd_perf/_meta/docs.asciidoc | 1 + .../mgr_osd_pool_stats/_meta/docs.asciidoc | 1 + .../ceph/mgr_osd_tree/_meta/docs.asciidoc | 1 + 15 files changed, 151 insertions(+), 3 deletions(-) create mode 100644 metricbeat/docs/modules/ceph/mgr_cluster_disk.asciidoc create mode 100644 metricbeat/docs/modules/ceph/mgr_cluster_health.asciidoc create mode 100644 metricbeat/docs/modules/ceph/mgr_osd_disk.asciidoc create mode 100644 metricbeat/docs/modules/ceph/mgr_osd_perf.asciidoc create mode 100644 metricbeat/docs/modules/ceph/mgr_osd_pool_stats.asciidoc create mode 100644 metricbeat/docs/modules/ceph/mgr_osd_tree.asciidoc create mode 100644 metricbeat/module/ceph/mgr_cluster_disk/_meta/docs.asciidoc create mode 100644 metricbeat/module/ceph/mgr_cluster_health/_meta/docs.asciidoc create mode 100644 metricbeat/module/ceph/mgr_osd_disk/_meta/docs.asciidoc create mode 100644 metricbeat/module/ceph/mgr_osd_perf/_meta/docs.asciidoc create mode 100644 metricbeat/module/ceph/mgr_osd_pool_stats/_meta/docs.asciidoc create mode 100644 metricbeat/module/ceph/mgr_osd_tree/_meta/docs.asciidoc diff --git a/metricbeat/docs/modules/ceph.asciidoc b/metricbeat/docs/modules/ceph.asciidoc index 1fb296f1784c..558b0753b82c 100644 --- a/metricbeat/docs/modules/ceph.asciidoc +++ b/metricbeat/docs/modules/ceph.asciidoc @@ -11,7 +11,9 @@ the http://docs.ceph.com/docs/master/man/8/ceph-rest-api/[ceph-rest-api]. The de [float] === Compatibility -The Ceph module is tested with Ceph Jewel (10.2.10). +The Ceph module is tested with Ceph Jewel (10.2.10) and Ceph Nautilus (14.2.7). + +Metricsets with the `mgr_` prefix are compatible with Ceph releases using the Ceph Manager Daemon. [float] === Dashboard @@ -52,6 +54,18 @@ The following metricsets are available: * <> +* <> + +* <> + +* <> + +* <> + +* <> + +* <> + * <> * <> @@ -66,6 +80,18 @@ include::ceph/cluster_health.asciidoc[] include::ceph/cluster_status.asciidoc[] +include::ceph/mgr_cluster_disk.asciidoc[] + +include::ceph/mgr_cluster_health.asciidoc[] + +include::ceph/mgr_osd_disk.asciidoc[] + +include::ceph/mgr_osd_perf.asciidoc[] + +include::ceph/mgr_osd_pool_stats.asciidoc[] + +include::ceph/mgr_osd_tree.asciidoc[] + include::ceph/monitor_health.asciidoc[] include::ceph/osd_df.asciidoc[] diff --git a/metricbeat/docs/modules/ceph/mgr_cluster_disk.asciidoc b/metricbeat/docs/modules/ceph/mgr_cluster_disk.asciidoc new file mode 100644 index 000000000000..28718c360eb4 --- /dev/null +++ b/metricbeat/docs/modules/ceph/mgr_cluster_disk.asciidoc @@ -0,0 +1,18 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-ceph-mgr_cluster_disk]] +=== Ceph mgr_cluster_disk metricset + +beta[] + +include::../../../module/ceph/mgr_cluster_disk/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + diff --git a/metricbeat/docs/modules/ceph/mgr_cluster_health.asciidoc b/metricbeat/docs/modules/ceph/mgr_cluster_health.asciidoc new file mode 100644 index 000000000000..ec7ea91b8884 --- /dev/null +++ b/metricbeat/docs/modules/ceph/mgr_cluster_health.asciidoc @@ -0,0 +1,18 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-ceph-mgr_cluster_health]] +=== Ceph mgr_cluster_health metricset + +beta[] + +include::../../../module/ceph/mgr_cluster_health/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + diff --git a/metricbeat/docs/modules/ceph/mgr_osd_disk.asciidoc b/metricbeat/docs/modules/ceph/mgr_osd_disk.asciidoc new file mode 100644 index 000000000000..0493d91d127f --- /dev/null +++ b/metricbeat/docs/modules/ceph/mgr_osd_disk.asciidoc @@ -0,0 +1,18 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-ceph-mgr_osd_disk]] +=== Ceph mgr_osd_disk metricset + +beta[] + +include::../../../module/ceph/mgr_osd_disk/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + diff --git a/metricbeat/docs/modules/ceph/mgr_osd_perf.asciidoc b/metricbeat/docs/modules/ceph/mgr_osd_perf.asciidoc new file mode 100644 index 000000000000..f84269015613 --- /dev/null +++ b/metricbeat/docs/modules/ceph/mgr_osd_perf.asciidoc @@ -0,0 +1,18 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-ceph-mgr_osd_perf]] +=== Ceph mgr_osd_perf metricset + +beta[] + +include::../../../module/ceph/mgr_osd_perf/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + diff --git a/metricbeat/docs/modules/ceph/mgr_osd_pool_stats.asciidoc b/metricbeat/docs/modules/ceph/mgr_osd_pool_stats.asciidoc new file mode 100644 index 000000000000..052385e336ec --- /dev/null +++ b/metricbeat/docs/modules/ceph/mgr_osd_pool_stats.asciidoc @@ -0,0 +1,18 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-ceph-mgr_osd_pool_stats]] +=== Ceph mgr_osd_pool_stats metricset + +beta[] + +include::../../../module/ceph/mgr_osd_pool_stats/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + diff --git a/metricbeat/docs/modules/ceph/mgr_osd_tree.asciidoc b/metricbeat/docs/modules/ceph/mgr_osd_tree.asciidoc new file mode 100644 index 000000000000..007235ed1886 --- /dev/null +++ b/metricbeat/docs/modules/ceph/mgr_osd_tree.asciidoc @@ -0,0 +1,18 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-ceph-mgr_osd_tree]] +=== Ceph mgr_osd_tree metricset + +beta[] + +include::../../../module/ceph/mgr_osd_tree/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index e35f54b73510..4f3e8210926d 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -38,9 +38,15 @@ This file is generated! See scripts/mage/docs_collector.go .2+| .2+| |<> |<> |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | -.7+| .7+| |<> +.13+| .13+| |<> |<> |<> +|<> beta[] +|<> beta[] +|<> beta[] +|<> beta[] +|<> beta[] +|<> beta[] |<> |<> |<> diff --git a/metricbeat/module/ceph/_meta/docs.asciidoc b/metricbeat/module/ceph/_meta/docs.asciidoc index d055dc46dcc4..0831de00302e 100644 --- a/metricbeat/module/ceph/_meta/docs.asciidoc +++ b/metricbeat/module/ceph/_meta/docs.asciidoc @@ -4,7 +4,9 @@ the http://docs.ceph.com/docs/master/man/8/ceph-rest-api/[ceph-rest-api]. The de [float] === Compatibility -The Ceph module is tested with Ceph Jewel (10.2.10). +The Ceph module is tested with Ceph Jewel (10.2.10) and Ceph Nautilus (14.2.7). + +Metricsets with the `mgr_` prefix are compatible with Ceph releases using the Ceph Manager Daemon. [float] === Dashboard diff --git a/metricbeat/module/ceph/mgr_cluster_disk/_meta/docs.asciidoc b/metricbeat/module/ceph/mgr_cluster_disk/_meta/docs.asciidoc new file mode 100644 index 000000000000..00e5d01fd51e --- /dev/null +++ b/metricbeat/module/ceph/mgr_cluster_disk/_meta/docs.asciidoc @@ -0,0 +1 @@ +This is the `mgr_cluster_disk` metricset of the Ceph module. diff --git a/metricbeat/module/ceph/mgr_cluster_health/_meta/docs.asciidoc b/metricbeat/module/ceph/mgr_cluster_health/_meta/docs.asciidoc new file mode 100644 index 000000000000..73d74cd225cf --- /dev/null +++ b/metricbeat/module/ceph/mgr_cluster_health/_meta/docs.asciidoc @@ -0,0 +1 @@ +This is the `mgr_cluster_health` metricset of the Ceph module. diff --git a/metricbeat/module/ceph/mgr_osd_disk/_meta/docs.asciidoc b/metricbeat/module/ceph/mgr_osd_disk/_meta/docs.asciidoc new file mode 100644 index 000000000000..983b39f26a97 --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_disk/_meta/docs.asciidoc @@ -0,0 +1 @@ +This is the `mgr_osd_disk` metricset of the Ceph module. diff --git a/metricbeat/module/ceph/mgr_osd_perf/_meta/docs.asciidoc b/metricbeat/module/ceph/mgr_osd_perf/_meta/docs.asciidoc new file mode 100644 index 000000000000..2af82348aa65 --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_perf/_meta/docs.asciidoc @@ -0,0 +1 @@ +This is the `mgr_osd_perf` metricset of the Ceph module. diff --git a/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/docs.asciidoc b/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/docs.asciidoc new file mode 100644 index 000000000000..618cf1bdaabd --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/docs.asciidoc @@ -0,0 +1 @@ +This is the `mgr_osd_pool_stats` metricset of the Ceph module. diff --git a/metricbeat/module/ceph/mgr_osd_tree/_meta/docs.asciidoc b/metricbeat/module/ceph/mgr_osd_tree/_meta/docs.asciidoc new file mode 100644 index 000000000000..440f11414f99 --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_tree/_meta/docs.asciidoc @@ -0,0 +1 @@ +This is the `mgr_osd_tree` metricset of the Ceph module. From 1fcef899e5a6f88193dbf8a361105d5fc51b1cf4 Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Mon, 10 Feb 2020 15:25:23 +0100 Subject: [PATCH 13/41] Update module config --- metricbeat/module/ceph/_meta/config.yml | 20 ++++++++++++++++++++ metricbeat/modules.d/ceph.yml.disabled | 20 ++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/metricbeat/module/ceph/_meta/config.yml b/metricbeat/module/ceph/_meta/config.yml index 6df14df1d80d..9364f60bfac2 100644 --- a/metricbeat/module/ceph/_meta/config.yml +++ b/metricbeat/module/ceph/_meta/config.yml @@ -16,3 +16,23 @@ # - pool_disk period: 1m hosts: ["localhost:5000"] + +- module: ceph + #metricsets: + # - mgr_cluster_health + period: 10s + hosts: [ "https://localhost:8003" ] + #username: "user" + #password: "secret" + +- module: ceph + #metricsets: + # - mgr_cluster_disk + # - mgr_osd_disk + # - mgr_osd_perf + # - mgr_osd_pool_stats + # - mgr_osd_tree + period: 1m + hosts: [ "https://localhost:8003" ] + #username: "user" + #password: "secret" diff --git a/metricbeat/modules.d/ceph.yml.disabled b/metricbeat/modules.d/ceph.yml.disabled index 7e875b274bd9..6ee3e7867ef1 100644 --- a/metricbeat/modules.d/ceph.yml.disabled +++ b/metricbeat/modules.d/ceph.yml.disabled @@ -19,3 +19,23 @@ # - pool_disk period: 1m hosts: ["localhost:5000"] + +- module: ceph + #metricsets: + # - mgr_cluster_health + period: 10s + hosts: [ "https://localhost:8003" ] + #username: "user" + #password: "secret" + +- module: ceph + #metricsets: + # - mgr_cluster_disk + # - mgr_osd_disk + # - mgr_osd_perf + # - mgr_osd_pool_stats + # - mgr_osd_tree + period: 1m + hosts: [ "https://localhost:8003" ] + #username: "user" + #password: "secret" From 1a3ee038f6c12d75a3a85cfd700d8c58eb330bc0 Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Mon, 10 Feb 2020 15:30:31 +0100 Subject: [PATCH 14/41] Fix: comment --- metricbeat/module/ceph/fields.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/module/ceph/fields.go b/metricbeat/module/ceph/fields.go index 3f337ac9539f..85d979fff728 100644 --- a/metricbeat/module/ceph/fields.go +++ b/metricbeat/module/ceph/fields.go @@ -30,7 +30,7 @@ func init() { } // AssetCeph returns asset data. -// This is the base64 encoded gzipped contents of /Users/marcin.tojek/go/src/github.com/elastic/beats/metricbeat/module/ceph. +// This is the base64 encoded gzipped contents of ../metricbeat/module/ceph. func AssetCeph() string { return "eJzEms2O2zYQx+/7FIM9tcCu0V59KJAmAbJoN1l0k1NRKLQ4klhTGoKk1vXbF6Rk2danPyjHp8Ty/uen4XDIGfIR1rhdQowquwOwwkpcwv17VNn9HQBHE2uhrKBiCb/dAQC4R5ATLyXeAZiMtI1iKhKRLiFh0rhvNUpkBpeQMvcbtFYUqVnC3/fGyPsHuM+sVff/3AEkAiU3S6/8CAXLsWFxH7tVTkVTqepveojc57v7o+8QU2GZKAzYDCFHq0Xs/s0sbFAjmFgzhRwSTTm8//jyaVELHGIcocjSWNQRF2bdPOzDGkFznwGdYz/tPm2YQyD2xoRkK4mL1daiOfrNjktSkbYejKC5z7udKnhVoMQ7sKZu/TwhnTO7hC7ADtKSZTIo4FenGAauNMiDsn0zyC9Ha0dahkzaLESsdZTOjzZ6Q82kjIxltuz31xq3G9L8PJd9qXSh0h1zWxNSIsc4w3htFqgozgKN3TNT8IbaCCqmzGoqC754Y7LEQMYbcfDapwGEHorXoyE4BmnHZsd2f2z2x9kJEdvRH4vNvmG7fCz8qlb7YjQgNEsSES80Mh6dlUWGM9M0XOUfcEbBZprKNFOlBYUaDMY0FDs160YLizeH9VYvoPWeJRUpFxAYhxrfQxcKUuY8583GU3npFKBcGCVZjAu/ugbG2ImDSqEo89VAFm4YaPUvxjbUCtqhqOVPQtHMCuoFMTGTyKNEErMD8a1Qx1i0n56L20XYUXJMNePIZxm0nfjEoDUM8wxaQ3HCoDUoP27QGtzhQVPpgjPLbp4xVQrO7sjeVaULXwDcHI0dFwhDcD7Kbw5n98XBEJjb9N+cq2wKgwEsv+FBv6XDyH0ZaGq+pH5njYe/G0eIqexMqEutv2a0MZDRBnJWbEGlBphGEEUNRcnkq3f4Am/26gEy0/s9MnyRlLI/da+IJLI21YTxJ+NEoSN6aLFApmex6oQnTJd5RIaHWiVaweCUXSRMlXs7klLNDeNCkyqgJjy/vYxCieJHQD19HoXSmDOlkEcqvTXZXx+f3728fPwwyBeyYvda7T1GsylMdRSqZfdBmHXTQqSkqhC7MdsUuyu0Y22VPsTrez11xToPJhl+tRe/vH4APp8nHaJCnVyL6DTcgl/EOCcpUdVLm+6hTPESVd2zOTCtRrwW0GmEB6RCWDp53pzdhzrWr/HdlnyEDXqb8yoOtZvat+brsmfXqnv+0r8gdJwD1zUIP1XOmLC6f/X1Kvib//H7lPnqxCGY6eq4YdqsP0sIZtUfJEwblczYqFScWWwPZWXZPTnzfUWOsMmwgA0z0Ke9Mz5YpVwUW59ZjlOvayxprLLmQlIa9OjmT0r3JzdtgFMOkw7hcmHioHTPwsTh8IyxQeleX7+Gg5v9xPBawMlJd0HwMWM7c+1g2xzx6W3N+UctbiV2laDflJWGpa4yr3xxXAOPLXQilA98TUoc24rzpJvG2mDDgeObiDGKJTMBz9gau07jAYSUmDLp/weiiGXJETLOH8AYDmjjxcga58L1VOdf0cNqQqRqrr2RLAdc1pyn35KqCtwRqgvvSYRg2/dKR/hUGhVlHmgWmXbHDSTFLq1UpbrwvZ/hwRvarM7Vim+NYq3BUuzNgtdWI03G8yWJy3WT+fNmqY80rMp4jfbGSbBldzAdOjOzWh5JiGT4A2Rk7ANoIjuSF7cKozmHxUk99Xe34kxIrrG/MX2Rk2rDO2GQwnnAoGLaz+jVFmLKc9bvi1iXJos2KNKsf073TeZTVy8vDj3i+9VTDVSe54+Dt9gV3JnC/4QZOMq8qD8uTGXRWCFlpe7CoCD706+PWzQP8MtjQT/353Itcqa3EUsSUQi7DeV4VxVV3nY7WI2Mi6I+IvQXGGu7g8ld40ggXHBdKcNG0gENmQ19Q2l/ScynBGHdeldKDiuEUrlR4rTpP72ZZzvnHFEpg1du0KRYn7KRS5jNOrcDr+RRTGNR5ytfUFeLfiuVdRZY35G8tsfbJ/IDF9YnvmspOLAbrKiHXYxBk1UZedPrux2W04pexxn2qshnf1rjp0l9R+Q0b93gqu4VPpqh+bdeHXH9HwAA//8tPTqG" } From 5444bff64e5f67753e4de9c388018d372d23ea25 Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Mon, 10 Feb 2020 15:33:28 +0100 Subject: [PATCH 15/41] Fix: fields.go --- metricbeat/include/fields/fields.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/include/fields/fields.go b/metricbeat/include/fields/fields.go index c1a18bbe8893..75e43d7e3cfd 100644 --- a/metricbeat/include/fields/fields.go +++ b/metricbeat/include/fields/fields.go @@ -24,7 +24,7 @@ import ( ) func init() { - if err := asset.SetFields("metricbeat", "/Users/marcin.tojek/go/src/github.com/elastic/beats/libbeat/fields.yml", asset.LibbeatFieldsPri, AssetUsersMarcinTojekGoSrcGithubComElasticBeatsLibbeatFieldsYml); err != nil { + if err := asset.SetFields("metricbeat", "../libbeat/fields.yml", asset.LibbeatFieldsPri, AssetLibbeatFieldsYml); err != nil { panic(err) } } From e011b62ed65bfe207d5d0c51650ca538dc5fdba3 Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Tue, 11 Feb 2020 13:07:43 +0100 Subject: [PATCH 16/41] Implement basic Python system test --- metricbeat/module/ceph/test_ceph_mgr.py | 52 +++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 metricbeat/module/ceph/test_ceph_mgr.py diff --git a/metricbeat/module/ceph/test_ceph_mgr.py b/metricbeat/module/ceph/test_ceph_mgr.py new file mode 100644 index 000000000000..1b8281075563 --- /dev/null +++ b/metricbeat/module/ceph/test_ceph_mgr.py @@ -0,0 +1,52 @@ +import os +import sys +import time +import unittest +from parameterized import parameterized + +sys.path.append(os.path.join(os.path.dirname(__file__), '../../tests/system')) +import metricbeat + + +class Test(metricbeat.BaseTest): + COMPOSE_SERVICES = ['ceph-mgr'] + FIELDS = ["ceph"] + + def get_ceph_module_config(self, metricset, password): + return { + 'name': 'ceph', + 'metricsets': [metricset], + 'period': '1h', + 'hosts': self.get_hosts(), + 'username': 'demo', + 'password': password, + 'extras': { + 'ssl.verification_mode': 'none' + } + } + + def get_hosts(self): + return ['https://' + self.compose_host(port='8003/tcp')] + + @parameterized.expand([ + "mgr_cluster_disk", + "mgr_cluster_health", + "mgr_osd_disk", + "mgr_osd_perf", + "mgr_osd_pool_stats", + "mgr_osd_tree", + ]) + @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") + def test_ceph_mgr(self, metricset): + """ + ceph-mgr metricsets tests + """ + + self.render_config_template(modules=[self.get_ceph_module_config(metricset, 'password')]) + proc = self.start_beat(home=self.beat_path) + + self.wait_until(lambda: self.output_lines() > 0) + proc.check_kill_and_wait() + self.assert_no_logged_warnings(replace=['SSL/TLS verifications disabled.']) + + #output = self.read_output_json() From 539bb3e5e8a9ee59bf6996d3f1ddb5d6b3eddce8 Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Tue, 11 Feb 2020 19:07:42 +0100 Subject: [PATCH 17/41] Add system test --- .../module/ceph/_meta/Dockerfile.nautilus | 6 ++---- metricbeat/module/ceph/_meta/healthcheck.sh | 14 ++++++++++++++ metricbeat/module/ceph/docker-compose.yml | 7 +++---- metricbeat/module/ceph/test_ceph_mgr.py | 17 ++++++++++++----- 4 files changed, 31 insertions(+), 13 deletions(-) create mode 100755 metricbeat/module/ceph/_meta/healthcheck.sh diff --git a/metricbeat/module/ceph/_meta/Dockerfile.nautilus b/metricbeat/module/ceph/_meta/Dockerfile.nautilus index 241356412a61..130d771635af 100644 --- a/metricbeat/module/ceph/_meta/Dockerfile.nautilus +++ b/metricbeat/module/ceph/_meta/Dockerfile.nautilus @@ -4,10 +4,8 @@ FROM ceph/daemon:${CEPH_VERSION} RUN yum -q install -y jq && yum clean all && rm -fr /var/cache/yum # Wait for the health endpoint to have monitors information -HEALTHCHECK --interval=1s --retries=300 \ - CMD ceph health | grep HEALTH_OK && \ - ceph restful list-keys > /opt/ceph-container/sree/static/restful-list-keys.json && \ - curl -s localhost:5000 >/dev/null +ADD healthcheck.sh / +HEALTHCHECK --interval=1s --retries=300 CMD /healthcheck.sh EXPOSE 5000 8003 9283 diff --git a/metricbeat/module/ceph/_meta/healthcheck.sh b/metricbeat/module/ceph/_meta/healthcheck.sh new file mode 100755 index 000000000000..09f34f38ea73 --- /dev/null +++ b/metricbeat/module/ceph/_meta/healthcheck.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +KEYS_JSON=/opt/ceph-container/sree/static/restful-list-keys.json + +if [[ ! -f "${KEYS_JSON}" ]]; then + ceph restful list-keys | grep demo >/dev/null + if [[ $? -eq 0 ]]; then + ceph restful list-keys > ${KEYS_JSON} + else + exit 1 + fi +fi + +ceph health | grep HEALTH_OK && curl -s localhost:5000 >/dev/null diff --git a/metricbeat/module/ceph/docker-compose.yml b/metricbeat/module/ceph/docker-compose.yml index 6a077f485418..9afdcee7fc36 100644 --- a/metricbeat/module/ceph/docker-compose.yml +++ b/metricbeat/module/ceph/docker-compose.yml @@ -18,7 +18,6 @@ services: args: CEPH_VERSION: ${CEPH_VERSION:-master-97985eb-nautilus-centos-7-x86_64} ports: - - 5000:5000 - - 8003:8003 - - 8080:8080 - - 9283:9283 + - 5000 + - 8003 + - 8080 diff --git a/metricbeat/module/ceph/test_ceph_mgr.py b/metricbeat/module/ceph/test_ceph_mgr.py index 1b8281075563..5732473e55e8 100644 --- a/metricbeat/module/ceph/test_ceph_mgr.py +++ b/metricbeat/module/ceph/test_ceph_mgr.py @@ -1,4 +1,5 @@ import os +import requests import sys import time import unittest @@ -12,14 +13,14 @@ class Test(metricbeat.BaseTest): COMPOSE_SERVICES = ['ceph-mgr'] FIELDS = ["ceph"] - def get_ceph_module_config(self, metricset, password): + def get_ceph_module_config(self, metricset): return { 'name': 'ceph', 'metricsets': [metricset], 'period': '1h', 'hosts': self.get_hosts(), 'username': 'demo', - 'password': password, + 'password': self.get_password(), 'extras': { 'ssl.verification_mode': 'none' } @@ -28,6 +29,11 @@ def get_ceph_module_config(self, metricset, password): def get_hosts(self): return ['https://' + self.compose_host(port='8003/tcp')] + def get_password(self): + r = requests.get('http://' + self.compose_host(port='5000/tcp') + '/restful-list-keys.json') + keys = r.json() + return keys['demo'] + @parameterized.expand([ "mgr_cluster_disk", "mgr_cluster_health", @@ -42,11 +48,12 @@ def test_ceph_mgr(self, metricset): ceph-mgr metricsets tests """ - self.render_config_template(modules=[self.get_ceph_module_config(metricset, 'password')]) + self.render_config_template(modules=[self.get_ceph_module_config(metricset)]) proc = self.start_beat(home=self.beat_path) - self.wait_until(lambda: self.output_lines() > 0) proc.check_kill_and_wait() self.assert_no_logged_warnings(replace=['SSL/TLS verifications disabled.']) - #output = self.read_output_json() + output = self.read_output_json() + for evt in output: + assert 'ceph' in evt From 6a5ce3013800d84ea837c81ed47956d483112860 Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Wed, 12 Feb 2020 11:41:11 +0100 Subject: [PATCH 18/41] Update data.json --- .../modules/ceph/mgr_cluster_disk.asciidoc | 6 + .../modules/ceph/mgr_cluster_health.asciidoc | 6 + .../docs/modules/ceph/mgr_osd_disk.asciidoc | 6 + .../docs/modules/ceph/mgr_osd_perf.asciidoc | 6 + .../modules/ceph/mgr_osd_pool_stats.asciidoc | 6 + .../docs/modules/ceph/mgr_osd_tree.asciidoc | 6 + metricbeat/module/ceph/_meta/config.yml | 1 + .../ceph/mgr_cluster_disk/_meta/data.json | 152 ++++++++++++ .../ceph/mgr_cluster_health/_meta/data.json | 234 ++++++++++++++++++ .../module/ceph/mgr_osd_disk/_meta/data.json | 67 +++++ .../module/ceph/mgr_osd_perf/_meta/data.json | 46 ++++ .../ceph/mgr_osd_pool_stats/_meta/data.json | 100 ++++++++ .../module/ceph/mgr_osd_tree/_meta/data.json | 67 +++++ metricbeat/modules.d/ceph.yml.disabled | 1 + 14 files changed, 704 insertions(+) create mode 100644 metricbeat/module/ceph/mgr_cluster_disk/_meta/data.json create mode 100644 metricbeat/module/ceph/mgr_cluster_health/_meta/data.json create mode 100644 metricbeat/module/ceph/mgr_osd_disk/_meta/data.json create mode 100644 metricbeat/module/ceph/mgr_osd_perf/_meta/data.json create mode 100644 metricbeat/module/ceph/mgr_osd_pool_stats/_meta/data.json create mode 100644 metricbeat/module/ceph/mgr_osd_tree/_meta/data.json diff --git a/metricbeat/docs/modules/ceph/mgr_cluster_disk.asciidoc b/metricbeat/docs/modules/ceph/mgr_cluster_disk.asciidoc index 28718c360eb4..22f79ae4b3b3 100644 --- a/metricbeat/docs/modules/ceph/mgr_cluster_disk.asciidoc +++ b/metricbeat/docs/modules/ceph/mgr_cluster_disk.asciidoc @@ -16,3 +16,9 @@ This is a default metricset. If the host module is unconfigured, this metricset For a description of each field in the metricset, see the <> section. +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/ceph/mgr_cluster_disk/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/ceph/mgr_cluster_health.asciidoc b/metricbeat/docs/modules/ceph/mgr_cluster_health.asciidoc index ec7ea91b8884..80580cb8a420 100644 --- a/metricbeat/docs/modules/ceph/mgr_cluster_health.asciidoc +++ b/metricbeat/docs/modules/ceph/mgr_cluster_health.asciidoc @@ -16,3 +16,9 @@ This is a default metricset. If the host module is unconfigured, this metricset For a description of each field in the metricset, see the <> section. +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/ceph/mgr_cluster_health/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/ceph/mgr_osd_disk.asciidoc b/metricbeat/docs/modules/ceph/mgr_osd_disk.asciidoc index 0493d91d127f..5a5ccee54912 100644 --- a/metricbeat/docs/modules/ceph/mgr_osd_disk.asciidoc +++ b/metricbeat/docs/modules/ceph/mgr_osd_disk.asciidoc @@ -16,3 +16,9 @@ This is a default metricset. If the host module is unconfigured, this metricset For a description of each field in the metricset, see the <> section. +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/ceph/mgr_osd_disk/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/ceph/mgr_osd_perf.asciidoc b/metricbeat/docs/modules/ceph/mgr_osd_perf.asciidoc index f84269015613..e16a0db71167 100644 --- a/metricbeat/docs/modules/ceph/mgr_osd_perf.asciidoc +++ b/metricbeat/docs/modules/ceph/mgr_osd_perf.asciidoc @@ -16,3 +16,9 @@ This is a default metricset. If the host module is unconfigured, this metricset For a description of each field in the metricset, see the <> section. +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/ceph/mgr_osd_perf/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/ceph/mgr_osd_pool_stats.asciidoc b/metricbeat/docs/modules/ceph/mgr_osd_pool_stats.asciidoc index 052385e336ec..2710299b2abc 100644 --- a/metricbeat/docs/modules/ceph/mgr_osd_pool_stats.asciidoc +++ b/metricbeat/docs/modules/ceph/mgr_osd_pool_stats.asciidoc @@ -16,3 +16,9 @@ This is a default metricset. If the host module is unconfigured, this metricset For a description of each field in the metricset, see the <> section. +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/ceph/mgr_osd_pool_stats/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/ceph/mgr_osd_tree.asciidoc b/metricbeat/docs/modules/ceph/mgr_osd_tree.asciidoc index 007235ed1886..9c53e3284c5f 100644 --- a/metricbeat/docs/modules/ceph/mgr_osd_tree.asciidoc +++ b/metricbeat/docs/modules/ceph/mgr_osd_tree.asciidoc @@ -16,3 +16,9 @@ This is a default metricset. If the host module is unconfigured, this metricset For a description of each field in the metricset, see the <> section. +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/ceph/mgr_osd_tree/_meta/data.json[] +---- diff --git a/metricbeat/module/ceph/_meta/config.yml b/metricbeat/module/ceph/_meta/config.yml index 9364f60bfac2..105b67feaf04 100644 --- a/metricbeat/module/ceph/_meta/config.yml +++ b/metricbeat/module/ceph/_meta/config.yml @@ -28,6 +28,7 @@ - module: ceph #metricsets: # - mgr_cluster_disk + # - mgr_cluster_health # - mgr_osd_disk # - mgr_osd_perf # - mgr_osd_pool_stats diff --git a/metricbeat/module/ceph/mgr_cluster_disk/_meta/data.json b/metricbeat/module/ceph/mgr_cluster_disk/_meta/data.json new file mode 100644 index 000000000000..cb7522b0c999 --- /dev/null +++ b/metricbeat/module/ceph/mgr_cluster_disk/_meta/data.json @@ -0,0 +1,152 @@ +{ + "@timestamp": "2020-02-12T10:00:49.733Z", + "@metadata": { + "beat": "metricbeat", + "type": "_doc", + "version": "8.0.0" + }, + "event": { + "dataset": "http.json_namespace", + "module": "ceph", + "duration": 9219357 + }, + "metricset": { + "period": 60000, + "name": "mgr_cluster_disk" + }, + "service": { + "type": "ceph", + "address": "localhost:8003" + }, + "agent": { + "hostname": "host.example.com", + "name": "host.example.com" + }, + "ceph": { + "mgr_cluster_disk": { + "pools": [ + { + "stats": { + "max_avail": 4307550208, + "stored": 0, + "objects": 0, + "kb_used": 0, + "bytes_used": 0, + "percent_used": 0 + }, + "name": "rbd", + "id": 1 + }, + { + "name": "cephfs_data", + "id": 2, + "stats": { + "stored": 0, + "objects": 0, + "kb_used": 0, + "bytes_used": 0, + "percent_used": 0, + "max_avail": 4307550208 + } + }, + { + "name": "cephfs_metadata", + "id": 3, + "stats": { + "max_avail": 4307550208, + "stored": 2286, + "objects": 22, + "kb_used": 512, + "bytes_used": 524288, + "percent_used": 0.00012169891851954162 + } + }, + { + "name": ".rgw.root", + "id": 4, + "stats": { + "stored": 2398, + "objects": 6, + "kb_used": 384, + "bytes_used": 393216, + "percent_used": 9.127696830546483e-05, + "max_avail": 4307550208 + } + }, + { + "name": "default.rgw.control", + "id": 5, + "stats": { + "stored": 0, + "objects": 8, + "kb_used": 0, + "bytes_used": 0, + "percent_used": 0, + "max_avail": 4307550208 + } + }, + { + "stats": { + "stored": 736, + "objects": 5, + "kb_used": 256, + "bytes_used": 262144, + "percent_used": 6.085316272219643e-05, + "max_avail": 4307550208 + }, + "name": "default.rgw.meta", + "id": 6 + }, + { + "stats": { + "stored": 0, + "objects": 176, + "kb_used": 0, + "bytes_used": 0, + "percent_used": 0, + "max_avail": 4307550208 + }, + "name": "default.rgw.log", + "id": 7 + }, + { + "name": "default.rgw.buckets.index", + "id": 8, + "stats": { + "objects": 1, + "kb_used": 0, + "bytes_used": 0, + "percent_used": 0, + "max_avail": 4307550208, + "stored": 0 + } + }, + { + "name": "scbench", + "id": 9, + "stats": { + "stored": 5020581888, + "objects": 1188, + "kb_used": 4902912, + "bytes_used": 5020581888, + "percent_used": 0.5382194519042969, + "max_avail": 4307550208 + } + } + ], + "stats": { + "total_bytes": 10737418240, + "total_avail_bytes": 4844421120, + "total_used_bytes": 4819255296, + "total_used_raw_bytes": 5892997120, + "total_used_raw_ratio": 0.548828125, + "num_osds": 1, + "num_per_pool_osds": 1 + }, + "stats_by_class": {} + } + }, + "ecs": { + "version": "1.4.0" + } +} diff --git a/metricbeat/module/ceph/mgr_cluster_health/_meta/data.json b/metricbeat/module/ceph/mgr_cluster_health/_meta/data.json new file mode 100644 index 000000000000..26c1e1939d9e --- /dev/null +++ b/metricbeat/module/ceph/mgr_cluster_health/_meta/data.json @@ -0,0 +1,234 @@ +{ + "@timestamp": "2020-02-12T10:05:51.130Z", + "@metadata": { + "beat": "metricbeat", + "type": "_doc", + "version": "8.0.0" + }, + "service": { + "address": "localhost:8003", + "type": "ceph" + }, + "event": { + "dataset": "http.json_namespace", + "module": "ceph", + "duration": 12981066 + }, + "ceph": { + "mgr_cluster_status": { + "progress_events": {}, + "health": { + "checks": { + "POOL_APP_NOT_ENABLED": { + "severity": "HEALTH_WARN", + "summary": { + "message": "application not enabled on 1 pool(s)" + } + } + }, + "status": "HEALTH_WARN" + }, + "election_epoch": 3, + "quorum": [ + 0 + ], + "quorum_age": 669, + "monmap": { + "mons": [ + { + "rank": 0, + "name": "aa72f9faae85", + "public_addrs": { + "addrvec": [ + { + "type": "v2", + "addr": "172.30.0.2:3300", + "nonce": 0 + } + ] + }, + "addr": "172.30.0.2:3300/0", + "public_addr": "172.30.0.2:3300/0" + } + ], + "epoch": 1, + "fsid": "17af221e-4129-4886-950b-adfb9c2c851f", + "modified": "2020-02-12 09:54:42.262674", + "created": "2020-02-12 09:54:42.262674", + "min_mon_release": 14, + "min_mon_release_name": "nautilus", + "features": { + "persistent": [ + "kraken", + "luminous", + "mimic", + "osdmap-prune", + "nautilus" + ], + "optional": [] + } + }, + "pgmap": { + "pgs_by_state": [ + { + "state_name": "active+clean", + "count": 164 + } + ], + "num_pgs": 164, + "num_pools": 9, + "num_objects": 1964, + "data_bytes": 7319065924, + "bytes_used": 8397062144, + "bytes_avail": 2340356096, + "bytes_total": 10737418240 + }, + "fsid": "17af221e-4129-4886-950b-adfb9c2c851f", + "quorum_names": [ + "aa72f9faae85" + ], + "osdmap": { + "osdmap": { + "epoch": 21, + "num_osds": 1, + "num_up_osds": 1, + "num_in_osds": 1, + "num_remapped_pgs": 0 + } + }, + "fsmap": { + "epoch": 4, + "id": 1, + "up": 1, + "in": 1, + "max": 1, + "by_rank": [ + { + "name": "demo", + "status": "up:active", + "gid": 4129, + "filesystem_id": 1, + "rank": 0 + } + ], + "up:standby": 0 + }, + "servicemap": { + "epoch": 3, + "modified": "2020-02-12 09:55:02.463222", + "services": { + "rbd-mirror": { + "daemons": { + "4164": { + "start_epoch": 3, + "start_stamp": "2020-02-12 09:55:01.596669", + "gid": 4164, + "addr": "172.30.0.2:0/1853666642", + "metadata": { + "ceph_release": "nautilus", + "ceph_version": "ceph version 14.2.7 (3d58626ebeec02d8385a4cefb92c6cbc3a45bfe8) nautilus (stable)", + "mem_swap_kb": "2097148", + "os": "Linux", + "arch": "x86_64", + "instance_id": "4164", + "cpu": "Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz", + "hostname": "aa72f9faae85", + "id": "admin", + "kernel_description": "#1 SMP Tue Jul 2 22:58:16 UTC 2019", + "distro_version": "7", + "distro": "centos", + "distro_description": "CentOS Linux 7 (Core)", + "kernel_version": "4.9.184-linuxkit", + "mem_total_kb": "8164244", + "ceph_version_short": "14.2.7" + } + }, + "summary": "" + } + }, + "rgw": { + "daemons": { + "summary": "", + "aa72f9faae85": { + "start_epoch": 2, + "start_stamp": "2020-02-12 09:54:59.401638", + "gid": 4144, + "addr": "172.30.0.2:0/681525861", + "metadata": { + "distro_version": "7", + "mem_swap_kb": "2097148", + "zone_id": "1fb88fdf-f239-4fc8-8a84-932626130a03", + "zonegroup_id": "b5fba8f9-ec19-4925-a56d-e0440fdf2137", + "ceph_version_short": "14.2.7", + "frontend_config#0": "beast endpoint=0.0.0.0:8080", + "frontend_type#0": "beast", + "kernel_version": "4.9.184-linuxkit", + "arch": "x86_64", + "mem_total_kb": "8164244", + "pid": "966", + "zone_name": "default", + "distro_description": "CentOS Linux 7 (Core)", + "ceph_version": "ceph version 14.2.7 (3d58626ebeec02d8385a4cefb92c6cbc3a45bfe8) nautilus (stable)", + "cpu": "Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz", + "distro": "centos", + "hostname": "aa72f9faae85", + "kernel_description": "#1 SMP Tue Jul 2 22:58:16 UTC 2019", + "num_handles": "1", + "os": "Linux", + "ceph_release": "nautilus", + "zonegroup_name": "default" + } + } + } + }, + "rgw-nfs": { + "daemons": { + "summary": "", + "aa72f9faae85": { + "start_stamp": "2020-02-12 09:55:01.635512", + "gid": 4166, + "addr": "172.30.0.2:0/108157315", + "metadata": { + "ceph_version_short": "14.2.7", + "kernel_version": "4.9.184-linuxkit", + "mem_total_kb": "8164244", + "zonegroup_name": "default", + "hostname": "aa72f9faae85", + "num_handles": "1", + "os": "Linux", + "pid": "1997", + "ceph_release": "nautilus", + "ceph_version": "ceph version 14.2.7 (3d58626ebeec02d8385a4cefb92c6cbc3a45bfe8) nautilus (stable)", + "cpu": "Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz", + "frontend_type#0": "rgw-nfs", + "zone_name": "default", + "zonegroup_id": "b5fba8f9-ec19-4925-a56d-e0440fdf2137", + "frontend_config#0": "rgwlib", + "kernel_description": "#1 SMP Tue Jul 2 22:58:16 UTC 2019", + "mem_swap_kb": "2097148", + "zone_id": "1fb88fdf-f239-4fc8-8a84-932626130a03", + "arch": "x86_64", + "distro": "centos", + "distro_description": "CentOS Linux 7 (Core)", + "distro_version": "7" + }, + "start_epoch": 3 + } + } + } + } + } + } + }, + "agent": { + "hostname": "host.example.com", + "name": "host.example.com" + }, + "ecs": { + "version": "1.4.0" + }, + "metricset": { + "name": "mgr_cluster_health", + "period": 60000 + } +} diff --git a/metricbeat/module/ceph/mgr_osd_disk/_meta/data.json b/metricbeat/module/ceph/mgr_osd_disk/_meta/data.json new file mode 100644 index 000000000000..dcf16544b36e --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_disk/_meta/data.json @@ -0,0 +1,67 @@ +{ + "@timestamp": "2020-02-12T10:07:05.597Z", + "@metadata": { + "beat": "metricbeat", + "type": "_doc", + "version": "8.0.0" + }, + "service": { + "address": "localhost:8003", + "type": "ceph" + }, + "ceph": { + "mgr_osd_disk": { + "summary": { + "total_kb_used": 8200256, + "total_kb_used_data": 7151552, + "total_kb_used_meta": 1048576, + "total_kb_avail": 2285504, + "average_utilization": 78.2037353515625, + "max_var": 1, + "dev": 0, + "total_kb": 10485760, + "min_var": 1, + "total_kb_used_omap": 0 + }, + "nodes": [ + { + "pool_weights": {}, + "kb_avail": 2285504, + "status": "up", + "type": "osd", + "kb": 10485760, + "kb_used": 8200256, + "kb_used_data": 7151552, + "kb_used_meta": 1048576, + "id": 0, + "reweight": 1, + "kb_used_omap": 0, + "var": 1, + "pgs": 164, + "name": "osd.0", + "type_id": 0, + "crush_weight": 0.009796142578125, + "depth": 2, + "utilization": 78.2037353515625 + } + ], + "stray": [] + } + }, + "agent": { + "hostname": "host.example.com", + "name": "host.example.com" + }, + "ecs": { + "version": "1.4.0" + }, + "event": { + "dataset": "http.json_namespace", + "module": "ceph", + "duration": 10084520 + }, + "metricset": { + "name": "mgr_osd_disk", + "period": 60000 + } +} diff --git a/metricbeat/module/ceph/mgr_osd_perf/_meta/data.json b/metricbeat/module/ceph/mgr_osd_perf/_meta/data.json new file mode 100644 index 000000000000..5c5e20ba06fd --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_perf/_meta/data.json @@ -0,0 +1,46 @@ +{ + "@timestamp": "2020-02-12T10:19:09.058Z", + "@metadata": { + "beat": "metricbeat", + "type": "_doc", + "version": "8.0.0" + }, + "agent": { + "hostname": "host.example.com", + "name": "host.example.com" + }, + "service": { + "address": "localhost:8003", + "type": "ceph" + }, + "ecs": { + "version": "1.4.0" + }, + "event": { + "duration": 13115609, + "dataset": "http.json_namespace", + "module": "ceph" + }, + "metricset": { + "name": "mgr_osd_perf", + "period": 60000 + }, + "ceph": { + "mgr_osd_perf": { + "pg_ready": true, + "osdstats": { + "osd_perf_infos": [ + { + "id": 0, + "perf_stats": { + "commit_latency_ms": 0, + "apply_latency_ms": 0, + "commit_latency_ns": 3, + "apply_latency_ns": 0 + } + } + ] + } + } + } +} diff --git a/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/data.json b/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/data.json new file mode 100644 index 000000000000..6c8d2c2ae491 --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/data.json @@ -0,0 +1,100 @@ +{ + "@timestamp": "2020-02-12T10:21:15.766Z", + "@metadata": { + "beat": "metricbeat", + "type": "_doc", + "version": "8.0.0" + }, + "agent": { + "hostname": "host.example.com", + "name": "host.example.com" + }, + "event": { + "dataset": "http.json_namespace", + "module": "ceph", + "duration": 15642520 + }, + "metricset": { + "period": 60000, + "name": "mgr_osd_pool_stats" + }, + "service": { + "address": "localhost:8003", + "type": "ceph" + }, + "ecs": { + "version": "1.4.0" + }, + "ceph": { + "mgr_osd_pool_stats": [ + { + "pool_name": "rbd", + "pool_id": "1", + "recovery": {}, + "recovery_rate": {}, + "client_io_rate": {} + }, + { + "pool_name": "cephfs_data", + "pool_id": "2", + "recovery": {}, + "recovery_rate": {}, + "client_io_rate": {} + }, + { + "pool_name": "cephfs_metadata", + "pool_id": "3", + "recovery": {}, + "recovery_rate": {}, + "client_io_rate": {} + }, + { + "recovery_rate": {}, + "client_io_rate": {}, + "pool_name": ".rgw.root", + "pool_id": "4", + "recovery": {} + }, + { + "recovery_rate": {}, + "client_io_rate": {}, + "pool_name": "default.rgw.control", + "pool_id": "5", + "recovery": {} + }, + { + "client_io_rate": {}, + "pool_name": "default.rgw.meta", + "pool_id": "6", + "recovery": {}, + "recovery_rate": {} + }, + { + "client_io_rate": {}, + "pool_name": "default.rgw.log", + "pool_id": "7", + "recovery": {}, + "recovery_rate": {} + }, + { + "client_io_rate": {}, + "pool_name": "default.rgw.buckets.index", + "pool_id": "8", + "recovery": {}, + "recovery_rate": {} + }, + { + "pool_name": "scbench", + "pool_id": "9", + "recovery": {}, + "recovery_rate": {}, + "client_io_rate": { + "read_op_per_sec": "156", + "write_op_per_sec": "0", + "read_bytes_sec": "654312204", + "write_bytes_sec": "85", + } + } + ] + } +} diff --git a/metricbeat/module/ceph/mgr_osd_tree/_meta/data.json b/metricbeat/module/ceph/mgr_osd_tree/_meta/data.json new file mode 100644 index 000000000000..b86883f58134 --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_tree/_meta/data.json @@ -0,0 +1,67 @@ +{ + "@timestamp": "2020-02-12T10:25:58.283Z", + "@metadata": { + "beat": "metricbeat", + "type": "_doc", + "version": "8.0.0" + }, + "agent": { + "hostname": "host.example.com", + "name": "host.example.com" + }, + "service": { + "type": "ceph", + "address": "localhost:8003" + }, + "event": { + "dataset": "http.json_namespace", + "module": "ceph", + "duration": 21934403 + }, + "metricset": { + "period": 60000, + "name": "mgr_osd_tree" + }, + "ecs": { + "version": "1.4.0" + }, + "ceph": { + "mgr_osd_tree": { + "stray": [], + "nodes": [ + { + "id": -1, + "name": "default", + "type": "root", + "type_id": 11, + "children": [ + -2 + ] + }, + { + "children": [ + 0 + ], + "id": -2, + "name": "aa72f9faae85", + "type": "host", + "type_id": 1, + "pool_weights": {} + }, + { + "name": "osd.0", + "type": "osd", + "exists": 1, + "reweight": 1, + "id": 0, + "crush_weight": 0.009796142578125, + "depth": 2, + "pool_weights": {}, + "status": "up", + "primary_affinity": 1, + "type_id": 0 + } + ] + } + } +} diff --git a/metricbeat/modules.d/ceph.yml.disabled b/metricbeat/modules.d/ceph.yml.disabled index 6ee3e7867ef1..79d90f7186e7 100644 --- a/metricbeat/modules.d/ceph.yml.disabled +++ b/metricbeat/modules.d/ceph.yml.disabled @@ -31,6 +31,7 @@ - module: ceph #metricsets: # - mgr_cluster_disk + # - mgr_cluster_health # - mgr_osd_disk # - mgr_osd_perf # - mgr_osd_pool_stats From 15a34b98747bd662b0dfcbb01930201235ee8aa1 Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Wed, 12 Feb 2020 12:17:14 +0100 Subject: [PATCH 19/41] Update data.json --- .../ceph/mgr_cluster_health/_meta/data.json | 105 ------------------ .../ceph/mgr_cluster_health/manifest.yml | 2 +- 2 files changed, 1 insertion(+), 106 deletions(-) diff --git a/metricbeat/module/ceph/mgr_cluster_health/_meta/data.json b/metricbeat/module/ceph/mgr_cluster_health/_meta/data.json index 26c1e1939d9e..515728e3cfa4 100644 --- a/metricbeat/module/ceph/mgr_cluster_health/_meta/data.json +++ b/metricbeat/module/ceph/mgr_cluster_health/_meta/data.json @@ -112,111 +112,6 @@ } ], "up:standby": 0 - }, - "servicemap": { - "epoch": 3, - "modified": "2020-02-12 09:55:02.463222", - "services": { - "rbd-mirror": { - "daemons": { - "4164": { - "start_epoch": 3, - "start_stamp": "2020-02-12 09:55:01.596669", - "gid": 4164, - "addr": "172.30.0.2:0/1853666642", - "metadata": { - "ceph_release": "nautilus", - "ceph_version": "ceph version 14.2.7 (3d58626ebeec02d8385a4cefb92c6cbc3a45bfe8) nautilus (stable)", - "mem_swap_kb": "2097148", - "os": "Linux", - "arch": "x86_64", - "instance_id": "4164", - "cpu": "Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz", - "hostname": "aa72f9faae85", - "id": "admin", - "kernel_description": "#1 SMP Tue Jul 2 22:58:16 UTC 2019", - "distro_version": "7", - "distro": "centos", - "distro_description": "CentOS Linux 7 (Core)", - "kernel_version": "4.9.184-linuxkit", - "mem_total_kb": "8164244", - "ceph_version_short": "14.2.7" - } - }, - "summary": "" - } - }, - "rgw": { - "daemons": { - "summary": "", - "aa72f9faae85": { - "start_epoch": 2, - "start_stamp": "2020-02-12 09:54:59.401638", - "gid": 4144, - "addr": "172.30.0.2:0/681525861", - "metadata": { - "distro_version": "7", - "mem_swap_kb": "2097148", - "zone_id": "1fb88fdf-f239-4fc8-8a84-932626130a03", - "zonegroup_id": "b5fba8f9-ec19-4925-a56d-e0440fdf2137", - "ceph_version_short": "14.2.7", - "frontend_config#0": "beast endpoint=0.0.0.0:8080", - "frontend_type#0": "beast", - "kernel_version": "4.9.184-linuxkit", - "arch": "x86_64", - "mem_total_kb": "8164244", - "pid": "966", - "zone_name": "default", - "distro_description": "CentOS Linux 7 (Core)", - "ceph_version": "ceph version 14.2.7 (3d58626ebeec02d8385a4cefb92c6cbc3a45bfe8) nautilus (stable)", - "cpu": "Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz", - "distro": "centos", - "hostname": "aa72f9faae85", - "kernel_description": "#1 SMP Tue Jul 2 22:58:16 UTC 2019", - "num_handles": "1", - "os": "Linux", - "ceph_release": "nautilus", - "zonegroup_name": "default" - } - } - } - }, - "rgw-nfs": { - "daemons": { - "summary": "", - "aa72f9faae85": { - "start_stamp": "2020-02-12 09:55:01.635512", - "gid": 4166, - "addr": "172.30.0.2:0/108157315", - "metadata": { - "ceph_version_short": "14.2.7", - "kernel_version": "4.9.184-linuxkit", - "mem_total_kb": "8164244", - "zonegroup_name": "default", - "hostname": "aa72f9faae85", - "num_handles": "1", - "os": "Linux", - "pid": "1997", - "ceph_release": "nautilus", - "ceph_version": "ceph version 14.2.7 (3d58626ebeec02d8385a4cefb92c6cbc3a45bfe8) nautilus (stable)", - "cpu": "Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz", - "frontend_type#0": "rgw-nfs", - "zone_name": "default", - "zonegroup_id": "b5fba8f9-ec19-4925-a56d-e0440fdf2137", - "frontend_config#0": "rgwlib", - "kernel_description": "#1 SMP Tue Jul 2 22:58:16 UTC 2019", - "mem_swap_kb": "2097148", - "zone_id": "1fb88fdf-f239-4fc8-8a84-932626130a03", - "arch": "x86_64", - "distro": "centos", - "distro_description": "CentOS Linux 7 (Core)", - "distro_version": "7" - }, - "start_epoch": 3 - } - } - } - } } } }, diff --git a/metricbeat/module/ceph/mgr_cluster_health/manifest.yml b/metricbeat/module/ceph/mgr_cluster_health/manifest.yml index 0434082f88a2..968ddc652405 100644 --- a/metricbeat/module/ceph/mgr_cluster_health/manifest.yml +++ b/metricbeat/module/ceph/mgr_cluster_health/manifest.yml @@ -22,4 +22,4 @@ processors: process_array: true target: "ceph.mgr_cluster_status" - drop_fields: - fields: ["http", "ceph.request", "ceph.mgr_cluster_status.mgrmap"] + fields: ["http", "ceph.request", "ceph.mgr_cluster_status.mgrmap", "ceph.mgr_cluster_status.servicemap"] From ed6f58767e7a0776f2f650c8eeec6769060f302e Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Wed, 12 Feb 2020 12:32:33 +0100 Subject: [PATCH 20/41] Remove field with custom structs --- metricbeat/module/ceph/mgr_cluster_health/_meta/data.json | 8 -------- metricbeat/module/ceph/mgr_cluster_health/manifest.yml | 2 +- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/metricbeat/module/ceph/mgr_cluster_health/_meta/data.json b/metricbeat/module/ceph/mgr_cluster_health/_meta/data.json index 515728e3cfa4..f1a2c66662c3 100644 --- a/metricbeat/module/ceph/mgr_cluster_health/_meta/data.json +++ b/metricbeat/module/ceph/mgr_cluster_health/_meta/data.json @@ -18,14 +18,6 @@ "mgr_cluster_status": { "progress_events": {}, "health": { - "checks": { - "POOL_APP_NOT_ENABLED": { - "severity": "HEALTH_WARN", - "summary": { - "message": "application not enabled on 1 pool(s)" - } - } - }, "status": "HEALTH_WARN" }, "election_epoch": 3, diff --git a/metricbeat/module/ceph/mgr_cluster_health/manifest.yml b/metricbeat/module/ceph/mgr_cluster_health/manifest.yml index 968ddc652405..6de8b944615f 100644 --- a/metricbeat/module/ceph/mgr_cluster_health/manifest.yml +++ b/metricbeat/module/ceph/mgr_cluster_health/manifest.yml @@ -22,4 +22,4 @@ processors: process_array: true target: "ceph.mgr_cluster_status" - drop_fields: - fields: ["http", "ceph.request", "ceph.mgr_cluster_status.mgrmap", "ceph.mgr_cluster_status.servicemap"] + fields: ["http", "ceph.request", "ceph.mgr_cluster_status.mgrmap", "ceph.mgr_cluster_status.servicemap", "ceph.mgr_cluster_status.health.checks"] From 46dc360f65ff0ab83e22ae6a5e19654525c8a41e Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Wed, 12 Feb 2020 12:57:09 +0100 Subject: [PATCH 21/41] Update data.json --- .../ceph/mgr_cluster_health/_meta/data.json | 116 +++++++++--------- .../ceph/mgr_cluster_health/manifest.yml | 7 +- .../module/ceph/mgr_osd_disk/_meta/data.json | 23 +++- .../module/ceph/mgr_osd_tree/_meta/data.json | 13 +- 4 files changed, 98 insertions(+), 61 deletions(-) diff --git a/metricbeat/module/ceph/mgr_cluster_health/_meta/data.json b/metricbeat/module/ceph/mgr_cluster_health/_meta/data.json index f1a2c66662c3..bb023db43b00 100644 --- a/metricbeat/module/ceph/mgr_cluster_health/_meta/data.json +++ b/metricbeat/module/ceph/mgr_cluster_health/_meta/data.json @@ -1,35 +1,33 @@ { - "@timestamp": "2020-02-12T10:05:51.130Z", + "@timestamp": "2020-02-12T11:53:41.177Z", "@metadata": { "beat": "metricbeat", "type": "_doc", "version": "8.0.0" }, - "service": { - "address": "localhost:8003", - "type": "ceph" - }, - "event": { - "dataset": "http.json_namespace", - "module": "ceph", - "duration": 12981066 + "ecs": { + "version": "1.4.0" }, "ceph": { "mgr_cluster_status": { - "progress_events": {}, - "health": { - "status": "HEALTH_WARN" - }, - "election_epoch": 3, - "quorum": [ - 0 - ], - "quorum_age": 669, "monmap": { + "created": "2020-02-12 11:52:57.443798", + "min_mon_release": 14, + "min_mon_release_name": "nautilus", + "features": { + "persistent": [ + "kraken", + "luminous", + "mimic", + "osdmap-prune", + "nautilus" + ], + "optional": [] + }, "mons": [ { "rank": 0, - "name": "aa72f9faae85", + "name": "a7b86c4500b7", "public_addrs": { "addrvec": [ { @@ -44,55 +42,47 @@ } ], "epoch": 1, - "fsid": "17af221e-4129-4886-950b-adfb9c2c851f", - "modified": "2020-02-12 09:54:42.262674", - "created": "2020-02-12 09:54:42.262674", - "min_mon_release": 14, - "min_mon_release_name": "nautilus", - "features": { - "persistent": [ - "kraken", - "luminous", - "mimic", - "osdmap-prune", - "nautilus" - ], - "optional": [] - } + "fsid": "6032ef6c-b305-44c7-96fc-9a10e6e36783", + "modified": "2020-02-12 11:52:57.443798" }, - "pgmap": { - "pgs_by_state": [ - { - "state_name": "active+clean", - "count": 164 - } - ], - "num_pgs": 164, - "num_pools": 9, - "num_objects": 1964, - "data_bytes": 7319065924, - "bytes_used": 8397062144, - "bytes_avail": 2340356096, - "bytes_total": 10737418240 + "progress_events": {}, + "fsid": "6032ef6c-b305-44c7-96fc-9a10e6e36783", + "health": { + "status": "HEALTH_OK" }, - "fsid": "17af221e-4129-4886-950b-adfb9c2c851f", + "election_epoch": 3, + "quorum": [ + 0 + ], "quorum_names": [ - "aa72f9faae85" + "a7b86c4500b7" ], + "quorum_age": 43, "osdmap": { "osdmap": { - "epoch": 21, + "epoch": 18, "num_osds": 1, "num_up_osds": 1, "num_in_osds": 1, "num_remapped_pgs": 0 } }, + "pgmap": { + "pgs_by_state": [ + { + "state_name": "active+clean", + "count": 64 + } + ], + "num_pgs": 64, + "num_pools": 8, + "num_objects": 218, + "data_bytes": 5420, + "bytes_used": 1077542912, + "bytes_avail": 9659875328, + "bytes_total": 10737418240 + }, "fsmap": { - "epoch": 4, - "id": 1, - "up": 1, - "in": 1, "max": 1, "by_rank": [ { @@ -103,7 +93,11 @@ "rank": 0 } ], - "up:standby": 0 + "up:standby": 0, + "epoch": 4, + "id": 1, + "up": 1, + "in": 1 } } }, @@ -111,11 +105,17 @@ "hostname": "host.example.com", "name": "host.example.com" }, - "ecs": { - "version": "1.4.0" + "event": { + "module": "ceph", + "duration": 19257753, + "dataset": "http.json_namespace" }, "metricset": { "name": "mgr_cluster_health", "period": 60000 + }, + "service": { + "address": "localhost:8003", + "type": "ceph" } } diff --git a/metricbeat/module/ceph/mgr_cluster_health/manifest.yml b/metricbeat/module/ceph/mgr_cluster_health/manifest.yml index 6de8b944615f..07933b33af28 100644 --- a/metricbeat/module/ceph/mgr_cluster_health/manifest.yml +++ b/metricbeat/module/ceph/mgr_cluster_health/manifest.yml @@ -22,4 +22,9 @@ processors: process_array: true target: "ceph.mgr_cluster_status" - drop_fields: - fields: ["http", "ceph.request", "ceph.mgr_cluster_status.mgrmap", "ceph.mgr_cluster_status.servicemap", "ceph.mgr_cluster_status.health.checks"] + fields: + - "http" + - "ceph.request" + - "ceph.mgr_cluster_status.mgrmap" + - "ceph.mgr_cluster_status.servicemap" + - "ceph.mgr_cluster_status.health.checks" diff --git a/metricbeat/module/ceph/mgr_osd_disk/_meta/data.json b/metricbeat/module/ceph/mgr_osd_disk/_meta/data.json index dcf16544b36e..28275ee395dc 100644 --- a/metricbeat/module/ceph/mgr_osd_disk/_meta/data.json +++ b/metricbeat/module/ceph/mgr_osd_disk/_meta/data.json @@ -45,7 +45,28 @@ "utilization": 78.2037353515625 } ], - "stray": [] + "stray": [ + { + "pool_weights": {}, + "kb_avail": 2285504, + "status": "up", + "type": "osd", + "kb": 10485760, + "kb_used": 1256, + "kb_used_data": 864, + "kb_used_meta": 164, + "id": 0, + "reweight": 1, + "kb_used_omap": 0, + "var": 1, + "pgs": 164, + "name": "osd.0", + "type_id": 0, + "crush_weight": 0.005991142478123, + "depth": 2, + "utilization": 0.00001573562622 + } + ] } }, "agent": { diff --git a/metricbeat/module/ceph/mgr_osd_tree/_meta/data.json b/metricbeat/module/ceph/mgr_osd_tree/_meta/data.json index b86883f58134..97d7a59d018a 100644 --- a/metricbeat/module/ceph/mgr_osd_tree/_meta/data.json +++ b/metricbeat/module/ceph/mgr_osd_tree/_meta/data.json @@ -27,7 +27,18 @@ }, "ceph": { "mgr_osd_tree": { - "stray": [], + "stray": [ + { + "children": [ + 0 + ], + "id": -3, + "name": "bc30f9fcab61", + "type": "host", + "type_id": 1, + "pool_weights": {} + } + ], "nodes": [ { "id": -1, From 26f1c47add521b2dff6e49218be06af1159483d1 Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Wed, 12 Feb 2020 14:12:03 +0100 Subject: [PATCH 22/41] Use event processors --- .../ceph/mgr_cluster_disk/_meta/data.json | 128 ++++++++--------- .../module/ceph/mgr_cluster_disk/manifest.yml | 3 +- .../ceph/mgr_cluster_health/_meta/data.json | 108 +++++++------- .../ceph/mgr_cluster_health/manifest.yml | 2 + .../module/ceph/mgr_osd_disk/_meta/data.json | 133 ++++++++++-------- .../module/ceph/mgr_osd_disk/manifest.yml | 31 +++- .../module/ceph/mgr_osd_perf/_meta/data.json | 35 ++--- .../module/ceph/mgr_osd_perf/manifest.yml | 1 + .../ceph/mgr_osd_pool_stats/_meta/data.json | 79 +++++------ .../ceph/mgr_osd_pool_stats/manifest.yml | 19 +++ .../module/ceph/mgr_osd_tree/_meta/data.json | 75 +++++----- .../module/ceph/mgr_osd_tree/manifest.yml | 29 ++++ 12 files changed, 369 insertions(+), 274 deletions(-) diff --git a/metricbeat/module/ceph/mgr_cluster_disk/_meta/data.json b/metricbeat/module/ceph/mgr_cluster_disk/_meta/data.json index cb7522b0c999..44e14bb0a8d9 100644 --- a/metricbeat/module/ceph/mgr_cluster_disk/_meta/data.json +++ b/metricbeat/module/ceph/mgr_cluster_disk/_meta/data.json @@ -1,110 +1,105 @@ { - "@timestamp": "2020-02-12T10:00:49.733Z", + "@timestamp": "2020-02-12T12:21:29.890Z", "@metadata": { "beat": "metricbeat", "type": "_doc", "version": "8.0.0" }, - "event": { - "dataset": "http.json_namespace", - "module": "ceph", - "duration": 9219357 - }, - "metricset": { - "period": 60000, - "name": "mgr_cluster_disk" - }, - "service": { - "type": "ceph", - "address": "localhost:8003" - }, - "agent": { - "hostname": "host.example.com", - "name": "host.example.com" + "ecs": { + "version": "1.4.0" }, "ceph": { "mgr_cluster_disk": { + "stats": { + "total_bytes": 10737418240, + "total_avail_bytes": 8342470656, + "total_used_bytes": 1321205760, + "total_used_raw_bytes": 2394947584, + "total_used_raw_ratio": 0.22304686903953552, + "num_osds": 1, + "num_per_pool_osds": 1 + }, "pools": [ { + "name": "rbd", + "id": 1, "stats": { - "max_avail": 4307550208, + "bytes_used": 0, + "percent_used": 0, + "max_avail": 7805599744, "stored": 0, "objects": 0, - "kb_used": 0, - "bytes_used": 0, - "percent_used": 0 - }, - "name": "rbd", - "id": 1 + "kb_used": 0 + } }, { "name": "cephfs_data", "id": 2, "stats": { - "stored": 0, - "objects": 0, "kb_used": 0, "bytes_used": 0, "percent_used": 0, - "max_avail": 4307550208 + "max_avail": 7805599744, + "stored": 0, + "objects": 0 } }, { "name": "cephfs_metadata", "id": 3, "stats": { - "max_avail": 4307550208, + "max_avail": 7805599744, "stored": 2286, "objects": 22, "kb_used": 512, "bytes_used": 524288, - "percent_used": 0.00012169891851954162 + "percent_used": 6.716368079651147e-05 } }, { "name": ".rgw.root", "id": 4, "stats": { + "max_avail": 7805599744, "stored": 2398, "objects": 6, "kb_used": 384, "bytes_used": 393216, - "percent_used": 9.127696830546483e-05, - "max_avail": 4307550208 + "percent_used": 5.037360460846685e-05 } }, { "name": "default.rgw.control", "id": 5, "stats": { - "stored": 0, - "objects": 8, "kb_used": 0, "bytes_used": 0, "percent_used": 0, - "max_avail": 4307550208 + "max_avail": 7805599744, + "stored": 0, + "objects": 8 } }, { + "name": "default.rgw.meta", + "id": 6, "stats": { "stored": 736, "objects": 5, "kb_used": 256, "bytes_used": 262144, - "percent_used": 6.085316272219643e-05, - "max_avail": 4307550208 - }, - "name": "default.rgw.meta", - "id": 6 + "percent_used": 3.3582968171685934e-05, + "max_avail": 7805599744 + } }, { "stats": { + "max_avail": 7805599744, "stored": 0, "objects": 176, "kb_used": 0, "bytes_used": 0, - "percent_used": 0, - "max_avail": 4307550208 + "percent_used": 0 }, "name": "default.rgw.log", "id": 7 @@ -113,40 +108,47 @@ "name": "default.rgw.buckets.index", "id": 8, "stats": { - "objects": 1, "kb_used": 0, "bytes_used": 0, "percent_used": 0, - "max_avail": 4307550208, - "stored": 0 + "max_avail": 7805599744, + "stored": 0, + "objects": 1 } }, { "name": "scbench", "id": 9, "stats": { - "stored": 5020581888, - "objects": 1188, - "kb_used": 4902912, - "bytes_used": 5020581888, - "percent_used": 0.5382194519042969, - "max_avail": 4307550208 + "kb_used": 1425408, + "bytes_used": 1459617792, + "percent_used": 0.1575373411178589, + "max_avail": 7805599744, + "stored": 1459617792, + "objects": 339 } } - ], - "stats": { - "total_bytes": 10737418240, - "total_avail_bytes": 4844421120, - "total_used_bytes": 4819255296, - "total_used_raw_bytes": 5892997120, - "total_used_raw_ratio": 0.548828125, - "num_osds": 1, - "num_per_pool_osds": 1 - }, - "stats_by_class": {} + ] } }, - "ecs": { - "version": "1.4.0" + "agent": { + "hostname": "host", + "id": "993db589-f9d8-40f3-94cb-43b2b69fa9f4", + "version": "8.0.0", + "type": "metricbeat", + "ephemeral_id": "089f698c-74e3-44e1-ba4d-21010fc38433" + }, + "event": { + "duration": 49779315, + "dataset": "http.json_namespace", + "module": "ceph" + }, + "metricset": { + "name": "mgr_cluster_disk", + "period": 60000 + }, + "service": { + "type": "ceph", + "address": "localhost:8003" } } diff --git a/metricbeat/module/ceph/mgr_cluster_disk/manifest.yml b/metricbeat/module/ceph/mgr_cluster_disk/manifest.yml index abc6f2a4c963..051eee3a2e1f 100644 --- a/metricbeat/module/ceph/mgr_cluster_disk/manifest.yml +++ b/metricbeat/module/ceph/mgr_cluster_disk/manifest.yml @@ -22,4 +22,5 @@ processors: process_array: true target: "ceph.mgr_cluster_disk" - drop_fields: - fields: ["http", "ceph.request"] + ignore_missing: true + fields: ["http", "ceph.request", "ceph.mgr_cluster_disk.stats_by_class" ] diff --git a/metricbeat/module/ceph/mgr_cluster_health/_meta/data.json b/metricbeat/module/ceph/mgr_cluster_health/_meta/data.json index bb023db43b00..5ea3a9099be6 100644 --- a/metricbeat/module/ceph/mgr_cluster_health/_meta/data.json +++ b/metricbeat/module/ceph/mgr_cluster_health/_meta/data.json @@ -1,17 +1,38 @@ { - "@timestamp": "2020-02-12T11:53:41.177Z", + "@timestamp": "2020-02-12T12:22:57.142Z", "@metadata": { "beat": "metricbeat", "type": "_doc", "version": "8.0.0" }, + "agent": { + "id": "host", + "version": "8.0.0", + "type": "metricbeat", + "ephemeral_id": "59b80233-4d5c-48c9-a3db-a40d6e172aae", + "hostname": "MacBook-Elastic.local" + }, + "event": { + "duration": 29973749, + "dataset": "http.json_namespace", + "module": "ceph" + }, + "metricset": { + "period": 60000, + "name": "mgr_cluster_health" + }, + "service": { + "address": "localhost:8003", + "type": "ceph" + }, "ecs": { "version": "1.4.0" }, "ceph": { "mgr_cluster_status": { + "fsid": "ac9834a6-7dc8-434f-b9f8-86fca6849c5b", + "quorum_age": 226, "monmap": { - "created": "2020-02-12 11:52:57.443798", "min_mon_release": 14, "min_mon_release_name": "nautilus", "features": { @@ -26,8 +47,9 @@ }, "mons": [ { + "public_addr": "172.30.0.2:3300/0", "rank": 0, - "name": "a7b86c4500b7", + "name": "c213fde94857", "public_addrs": { "addrvec": [ { @@ -37,85 +59,69 @@ } ] }, - "addr": "172.30.0.2:3300/0", - "public_addr": "172.30.0.2:3300/0" + "addr": "172.30.0.2:3300/0" } ], "epoch": 1, - "fsid": "6032ef6c-b305-44c7-96fc-9a10e6e36783", - "modified": "2020-02-12 11:52:57.443798" + "fsid": "ac9834a6-7dc8-434f-b9f8-86fca6849c5b", + "modified": "2020-02-12 12:19:11.154113", + "created": "2020-02-12 12:19:11.154113" + }, + "pgmap": { + "num_pgs": 164, + "bytes_avail": 6950027264, + "read_bytes_sec": 85, + "write_bytes_sec": 746908239, + "bytes_used": 3787390976, + "bytes_total": 10737418240, + "read_op_per_sec": 0, + "write_op_per_sec": 318, + "pgs_by_state": [ + { + "state_name": "active+clean", + "count": 164 + } + ], + "num_pools": 9, + "num_objects": 950, + "data_bytes": 3070235948 }, - "progress_events": {}, - "fsid": "6032ef6c-b305-44c7-96fc-9a10e6e36783", "health": { - "status": "HEALTH_OK" + "status": "HEALTH_WARN" }, "election_epoch": 3, "quorum": [ 0 ], "quorum_names": [ - "a7b86c4500b7" + "c213fde94857" ], - "quorum_age": 43, "osdmap": { "osdmap": { - "epoch": 18, + "epoch": 21, "num_osds": 1, "num_up_osds": 1, "num_in_osds": 1, "num_remapped_pgs": 0 } }, - "pgmap": { - "pgs_by_state": [ - { - "state_name": "active+clean", - "count": 64 - } - ], - "num_pgs": 64, - "num_pools": 8, - "num_objects": 218, - "data_bytes": 5420, - "bytes_used": 1077542912, - "bytes_avail": 9659875328, - "bytes_total": 10737418240 - }, "fsmap": { + "in": 1, "max": 1, "by_rank": [ { + "filesystem_id": 1, + "rank": 0, "name": "demo", "status": "up:active", - "gid": 4129, - "filesystem_id": 1, - "rank": 0 + "gid": 4129 } ], "up:standby": 0, - "epoch": 4, + "epoch": 5, "id": 1, - "up": 1, - "in": 1 + "up": 1 } } - }, - "agent": { - "hostname": "host.example.com", - "name": "host.example.com" - }, - "event": { - "module": "ceph", - "duration": 19257753, - "dataset": "http.json_namespace" - }, - "metricset": { - "name": "mgr_cluster_health", - "period": 60000 - }, - "service": { - "address": "localhost:8003", - "type": "ceph" } } diff --git a/metricbeat/module/ceph/mgr_cluster_health/manifest.yml b/metricbeat/module/ceph/mgr_cluster_health/manifest.yml index 07933b33af28..0462312f4756 100644 --- a/metricbeat/module/ceph/mgr_cluster_health/manifest.yml +++ b/metricbeat/module/ceph/mgr_cluster_health/manifest.yml @@ -22,9 +22,11 @@ processors: process_array: true target: "ceph.mgr_cluster_status" - drop_fields: + ignore_missing: true fields: - "http" - "ceph.request" - "ceph.mgr_cluster_status.mgrmap" - "ceph.mgr_cluster_status.servicemap" - "ceph.mgr_cluster_status.health.checks" + - "ceph.mgr_cluster_status.progress_events" diff --git a/metricbeat/module/ceph/mgr_osd_disk/_meta/data.json b/metricbeat/module/ceph/mgr_osd_disk/_meta/data.json index 28275ee395dc..375d5b0a3242 100644 --- a/metricbeat/module/ceph/mgr_osd_disk/_meta/data.json +++ b/metricbeat/module/ceph/mgr_osd_disk/_meta/data.json @@ -1,88 +1,103 @@ { - "@timestamp": "2020-02-12T10:07:05.597Z", + "@timestamp": "2020-02-12T13:08:24.727Z", "@metadata": { "beat": "metricbeat", "type": "_doc", "version": "8.0.0" }, + "host": { + "name": "MacBook-Elastic.local", + "os": { + "build": "18G95", + "platform": "darwin", + "version": "10.14.6", + "family": "darwin", + "name": "Mac OS X", + "kernel": "18.7.0" + }, + "id": "24F065F8-4274-521D-8DD5-5D27557E15B4", + "hostname": "MacBook-Elastic.local", + "architecture": "x86_64" + }, + "agent": { + "type": "metricbeat", + "ephemeral_id": "ee373afc-c2ca-4308-91a2-651e4081d823", + "hostname": "MacBook-Elastic.local", + "id": "993db589-f9d8-40f3-94cb-43b2b69fa9f4", + "version": "8.0.0" + }, + "event": { + "dataset": "http.json_namespace", + "module": "ceph", + "duration": 43294329 + }, + "metricset": { + "name": "mgr_osd_disk", + "period": 60000 + }, "service": { "address": "localhost:8003", "type": "ceph" }, + "ecs": { + "version": "1.4.0" + }, "ceph": { "mgr_osd_disk": { - "summary": { - "total_kb_used": 8200256, - "total_kb_used_data": 7151552, - "total_kb_used_meta": 1048576, - "total_kb_avail": 2285504, - "average_utilization": 78.2037353515625, - "max_var": 1, - "dev": 0, - "total_kb": 10485760, - "min_var": 1, - "total_kb_used_omap": 0 - }, "nodes": [ { - "pool_weights": {}, - "kb_avail": 2285504, - "status": "up", - "type": "osd", - "kb": 10485760, - "kb_used": 8200256, - "kb_used_data": 7151552, - "kb_used_meta": 1048576, - "id": 0, + "depth": 2, "reweight": 1, - "kb_used_omap": 0, - "var": 1, "pgs": 164, + "crush_weight": 0.009796142578125, + "status": "up", + "id": 0, "name": "osd.0", "type_id": 0, - "crush_weight": 0.009796142578125, - "depth": 2, - "utilization": 78.2037353515625 + "kb_used_data": 6950784, + "kb_used_omap": 7, + "kb_used_meta": 1048568, + "kb_avail": 2486336, + "var": 1, + "type": "osd", + "kb": 10485760, + "kb_used": 7999424, + "utilization": 76.2884521484375 } ], "stray": [ { - "pool_weights": {}, - "kb_avail": 2285504, - "status": "up", - "type": "osd", - "kb": 10485760, - "kb_used": 1256, - "kb_used_data": 864, - "kb_used_meta": 164, - "id": 0, + "depth": 2, "reweight": 1, - "kb_used_omap": 0, - "var": 1, "pgs": 164, - "name": "osd.0", + "crush_weight": 0.009796142578125, + "status": "down", + "id": 1, + "name": "osd.1", "type_id": 0, - "crush_weight": 0.005991142478123, - "depth": 2, - "utilization": 0.00001573562622 + "kb_used_data": 6950784, + "kb_used_omap": 7, + "kb_used_meta": 1048568, + "kb_avail": 2486336, + "var": 1, + "type": "osd", + "kb": 10485760, + "kb_used": 7999424, + "utilization": 76.2884521484375 } - ] + ], + "summary": { + "total_kb_used_omap": 7, + "average_utilization": 76.2884521484375, + "dev": 0, + "total_kb_avail": 2486336, + "min_var": 1, + "max_var": 1, + "total_kb": 10485760, + "total_kb_used": 7999424, + "total_kb_used_data": 6950784, + "total_kb_used_meta": 1048568 + } } - }, - "agent": { - "hostname": "host.example.com", - "name": "host.example.com" - }, - "ecs": { - "version": "1.4.0" - }, - "event": { - "dataset": "http.json_namespace", - "module": "ceph", - "duration": 10084520 - }, - "metricset": { - "name": "mgr_osd_disk", - "period": 60000 } } diff --git a/metricbeat/module/ceph/mgr_osd_disk/manifest.yml b/metricbeat/module/ceph/mgr_osd_disk/manifest.yml index 4bf2661c4ef2..a58b2ba3455d 100644 --- a/metricbeat/module/ceph/mgr_osd_disk/manifest.yml +++ b/metricbeat/module/ceph/mgr_osd_disk/manifest.yml @@ -22,4 +22,33 @@ processors: process_array: true target: "ceph.mgr_osd_disk" - drop_fields: - fields: ["http", "ceph.request"] + ignore_missing: true + fields: ["http", "ceph.request", "ceph.mgr_osd_disk.nodes.pool_weight"] + - script: + lang: javascript + source: > + function process(event) { + var nodes = event.Get("ceph.mgr_osd_disk.nodes") + if (nodes != null) { + var filtered = []; + nodes.forEach(function(node, index) { + if (node["pool_weights"] !== undefined) { + delete node["pool_weights"]; + } + filtered.push(node); + }); + event.Put("ceph.mgr_osd_disk.nodes", filtered); + } + + var stray = event.Get("ceph.mgr_osd_disk.stray") + if (stray != null) { + var filtered = []; + stray.forEach(function(node, index) { + if (node["pool_weights"] !== undefined) { + delete node["pool_weights"]; + } + filtered.push(node); + }); + event.Put("ceph.mgr_osd_disk.stray", filtered); + } + } diff --git a/metricbeat/module/ceph/mgr_osd_perf/_meta/data.json b/metricbeat/module/ceph/mgr_osd_perf/_meta/data.json index 5c5e20ba06fd..5f4b958dcd06 100644 --- a/metricbeat/module/ceph/mgr_osd_perf/_meta/data.json +++ b/metricbeat/module/ceph/mgr_osd_perf/_meta/data.json @@ -1,30 +1,33 @@ { - "@timestamp": "2020-02-12T10:19:09.058Z", + "@timestamp": "2020-02-12T12:25:29.572Z", "@metadata": { "beat": "metricbeat", "type": "_doc", "version": "8.0.0" }, "agent": { - "hostname": "host.example.com", - "name": "host.example.com" - }, - "service": { - "address": "localhost:8003", - "type": "ceph" - }, - "ecs": { - "version": "1.4.0" + "type": "metricbeat", + "ephemeral_id": "62fb1ff9-4f2a-4347-bbd1-cee5df88c9da", + "hostname": "host", + "id": "993db589-f9d8-40f3-94cb-43b2b69fa9f4", + "version": "8.0.0" }, "event": { - "duration": 13115609, "dataset": "http.json_namespace", - "module": "ceph" + "module": "ceph", + "duration": 27722184 }, "metricset": { "name": "mgr_osd_perf", "period": 60000 }, + "service": { + "type": "ceph", + "address": "localhost:8003" + }, + "ecs": { + "version": "1.4.0" + }, "ceph": { "mgr_osd_perf": { "pg_ready": true, @@ -33,10 +36,10 @@ { "id": 0, "perf_stats": { - "commit_latency_ms": 0, - "apply_latency_ms": 0, - "commit_latency_ns": 3, - "apply_latency_ns": 0 + "commit_latency_ms": 23, + "apply_latency_ms": 23, + "commit_latency_ns": 23000000, + "apply_latency_ns": 23000000 } } ] diff --git a/metricbeat/module/ceph/mgr_osd_perf/manifest.yml b/metricbeat/module/ceph/mgr_osd_perf/manifest.yml index e7b4d2454424..4886520d7531 100644 --- a/metricbeat/module/ceph/mgr_osd_perf/manifest.yml +++ b/metricbeat/module/ceph/mgr_osd_perf/manifest.yml @@ -22,4 +22,5 @@ processors: process_array: true target: "ceph.mgr_osd_perf" - drop_fields: + ignore_missing: true fields: ["http", "ceph.request"] diff --git a/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/data.json b/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/data.json index 6c8d2c2ae491..1bb99f68b28f 100644 --- a/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/data.json +++ b/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/data.json @@ -1,100 +1,85 @@ { - "@timestamp": "2020-02-12T10:21:15.766Z", + "@timestamp": "2020-02-12T12:59:59.964Z", "@metadata": { "beat": "metricbeat", "type": "_doc", "version": "8.0.0" }, - "agent": { - "hostname": "host.example.com", - "name": "host.example.com" - }, - "event": { - "dataset": "http.json_namespace", - "module": "ceph", - "duration": 15642520 - }, - "metricset": { - "period": 60000, - "name": "mgr_osd_pool_stats" - }, "service": { "address": "localhost:8003", "type": "ceph" }, - "ecs": { - "version": "1.4.0" + "agent": { + "type": "metricbeat", + "ephemeral_id": "457300cc-fa5e-4818-b95e-54ca42b978cb", + "hostname": "host", + "id": "993db589-f9d8-40f3-94cb-43b2b69fa9f4", + "version": "8.0.0" }, "ceph": { "mgr_osd_pool_stats": [ { "pool_name": "rbd", "pool_id": "1", - "recovery": {}, - "recovery_rate": {}, "client_io_rate": {} }, { - "pool_name": "cephfs_data", "pool_id": "2", - "recovery": {}, - "recovery_rate": {}, - "client_io_rate": {} + "client_io_rate": {}, + "pool_name": "cephfs_data" }, { + "client_io_rate": {}, "pool_name": "cephfs_metadata", - "pool_id": "3", - "recovery": {}, - "recovery_rate": {}, - "client_io_rate": {} + "pool_id": "3" }, { - "recovery_rate": {}, "client_io_rate": {}, "pool_name": ".rgw.root", - "pool_id": "4", - "recovery": {} + "pool_id": "4" }, { - "recovery_rate": {}, - "client_io_rate": {}, "pool_name": "default.rgw.control", "pool_id": "5", - "recovery": {} + "client_io_rate": {} }, { - "client_io_rate": {}, "pool_name": "default.rgw.meta", "pool_id": "6", - "recovery": {}, - "recovery_rate": {} + "client_io_rate": {} }, { "client_io_rate": {}, "pool_name": "default.rgw.log", - "pool_id": "7", - "recovery": {}, - "recovery_rate": {} + "pool_id": "7" }, { - "client_io_rate": {}, "pool_name": "default.rgw.buckets.index", "pool_id": "8", - "recovery": {}, - "recovery_rate": {} + "client_io_rate": {} }, { "pool_name": "scbench", "pool_id": "9", - "recovery": {}, - "recovery_rate": {}, "client_io_rate": { - "read_op_per_sec": "156", - "write_op_per_sec": "0", - "read_bytes_sec": "654312204", - "write_bytes_sec": "85", + "read_bytes_sec": "85", + "write_bytes_sec": "802631707", + "read_op_per_sec": "0", + "write_op_per_sec": "336" } } ] + }, + "ecs": { + "version": "1.4.0" + }, + "event": { + "duration": 9206093, + "dataset": "http.json_namespace", + "module": "ceph" + }, + "metricset": { + "name": "mgr_osd_pool_stats", + "period": 60000 } } diff --git a/metricbeat/module/ceph/mgr_osd_pool_stats/manifest.yml b/metricbeat/module/ceph/mgr_osd_pool_stats/manifest.yml index 68cf73553243..0cbc0ace5ee6 100644 --- a/metricbeat/module/ceph/mgr_osd_pool_stats/manifest.yml +++ b/metricbeat/module/ceph/mgr_osd_pool_stats/manifest.yml @@ -23,3 +23,22 @@ processors: target: "ceph.mgr_osd_pool_stats" - drop_fields: fields: ["http", "ceph.request"] + - script: + lang: javascript + source: > + function process(event) { + var pools = event.Get("ceph.mgr_osd_pool_stats") + if (pools != null) { + var filtered = []; + pools.forEach(function(pool, index) { + if (pool["recovery"] !== undefined) { + delete pool["recovery"]; + } + if (pool["recovery_rate"] !== undefined) { + delete pool["recovery_rate"]; + } + filtered.push(pool); + }); + event.Put("ceph.mgr_osd_pool_stats", filtered); + } + } diff --git a/metricbeat/module/ceph/mgr_osd_tree/_meta/data.json b/metricbeat/module/ceph/mgr_osd_tree/_meta/data.json index 97d7a59d018a..e5b3cf637c85 100644 --- a/metricbeat/module/ceph/mgr_osd_tree/_meta/data.json +++ b/metricbeat/module/ceph/mgr_osd_tree/_meta/data.json @@ -1,78 +1,81 @@ { - "@timestamp": "2020-02-12T10:25:58.283Z", + "@timestamp": "2020-02-12T13:05:44.386Z", "@metadata": { "beat": "metricbeat", "type": "_doc", "version": "8.0.0" }, - "agent": { - "hostname": "host.example.com", - "name": "host.example.com" - }, "service": { - "type": "ceph", - "address": "localhost:8003" + "address": "localhost:8003", + "type": "ceph" }, "event": { "dataset": "http.json_namespace", "module": "ceph", - "duration": 21934403 - }, - "metricset": { - "period": 60000, - "name": "mgr_osd_tree" + "duration": 30426747 }, "ecs": { "version": "1.4.0" }, "ceph": { "mgr_osd_tree": { - "stray": [ - { - "children": [ - 0 - ], - "id": -3, - "name": "bc30f9fcab61", - "type": "host", - "type_id": 1, - "pool_weights": {} - } - ], "nodes": [ { - "id": -1, "name": "default", "type": "root", "type_id": 11, "children": [ -2 - ] + ], + "id": -1 }, { + "type_id": 1, "children": [ 0 ], "id": -2, - "name": "aa72f9faae85", - "type": "host", - "type_id": 1, - "pool_weights": {} + "name": "c213fde94857", + "type": "host" }, { - "name": "osd.0", - "type": "osd", - "exists": 1, "reweight": 1, "id": 0, - "crush_weight": 0.009796142578125, "depth": 2, - "pool_weights": {}, + "exists": 1, "status": "up", "primary_affinity": 1, - "type_id": 0 + "name": "osd.0", + "type": "osd", + "type_id": 0, + "crush_weight": 0.009796142578125 + } + ], + "stray": [ + { + "reweight": 1, + "id": 2, + "depth": 2, + "exists": 1, + "status": "down", + "primary_affinity": 1, + "name": "osd.1", + "type": "osd", + "type_id": 0, + "crush_weight": 0.004796132578111 } ] } + }, + "agent": { + "ephemeral_id": "fcabd35f-6792-47d9-9cb1-611030bff6e8", + "hostname": "host", + "id": "993db589-f9d8-40f3-94cb-43b2b69fa9f4", + "version": "8.0.0", + "type": "metricbeat" + }, + "metricset": { + "period": 60000, + "name": "mgr_osd_tree" } } diff --git a/metricbeat/module/ceph/mgr_osd_tree/manifest.yml b/metricbeat/module/ceph/mgr_osd_tree/manifest.yml index 91702555f48e..5b8230cb9a10 100644 --- a/metricbeat/module/ceph/mgr_osd_tree/manifest.yml +++ b/metricbeat/module/ceph/mgr_osd_tree/manifest.yml @@ -22,4 +22,33 @@ processors: process_array: true target: "ceph.mgr_osd_tree" - drop_fields: + ignore_missing: true fields: ["http", "ceph.request"] + - script: + lang: javascript + source: > + function process(event) { + var nodes = event.Get("ceph.mgr_osd_tree.nodes") + if (nodes != null) { + var filtered = []; + nodes.forEach(function(node, index) { + if (node["pool_weights"] !== undefined) { + delete node["pool_weights"]; + } + filtered.push(node); + }); + event.Put("ceph.mgr_osd_tree.nodes", filtered); + } + + var stray = event.Get("ceph.mgr_osd_tree.stray") + if (stray != null) { + var filtered = []; + stray.forEach(function(node, index) { + if (node["pool_weights"] !== undefined) { + delete node["pool_weights"]; + } + filtered.push(node); + }); + event.Put("ceph.mgr_osd_tree.stray", filtered); + } + } From 2cd10560836a67b4026dbf49233ed56fbf6378fc Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Wed, 12 Feb 2020 14:17:49 +0100 Subject: [PATCH 23/41] Update data.json --- .../ceph/mgr_cluster_health/_meta/data.json | 4 ++-- .../module/ceph/mgr_osd_disk/_meta/data.json | 16 +--------------- 2 files changed, 3 insertions(+), 17 deletions(-) diff --git a/metricbeat/module/ceph/mgr_cluster_health/_meta/data.json b/metricbeat/module/ceph/mgr_cluster_health/_meta/data.json index 5ea3a9099be6..f01eb0ea305f 100644 --- a/metricbeat/module/ceph/mgr_cluster_health/_meta/data.json +++ b/metricbeat/module/ceph/mgr_cluster_health/_meta/data.json @@ -6,11 +6,11 @@ "version": "8.0.0" }, "agent": { - "id": "host", + "id": "993db589-f9d8-40f3-94cb-43b2b69fa9f4", "version": "8.0.0", "type": "metricbeat", "ephemeral_id": "59b80233-4d5c-48c9-a3db-a40d6e172aae", - "hostname": "MacBook-Elastic.local" + "hostname": "host" }, "event": { "duration": 29973749, diff --git a/metricbeat/module/ceph/mgr_osd_disk/_meta/data.json b/metricbeat/module/ceph/mgr_osd_disk/_meta/data.json index 375d5b0a3242..f3a9f80d0b8f 100644 --- a/metricbeat/module/ceph/mgr_osd_disk/_meta/data.json +++ b/metricbeat/module/ceph/mgr_osd_disk/_meta/data.json @@ -5,24 +5,10 @@ "type": "_doc", "version": "8.0.0" }, - "host": { - "name": "MacBook-Elastic.local", - "os": { - "build": "18G95", - "platform": "darwin", - "version": "10.14.6", - "family": "darwin", - "name": "Mac OS X", - "kernel": "18.7.0" - }, - "id": "24F065F8-4274-521D-8DD5-5D27557E15B4", - "hostname": "MacBook-Elastic.local", - "architecture": "x86_64" - }, "agent": { "type": "metricbeat", "ephemeral_id": "ee373afc-c2ca-4308-91a2-651e4081d823", - "hostname": "MacBook-Elastic.local", + "hostname": "host", "id": "993db589-f9d8-40f3-94cb-43b2b69fa9f4", "version": "8.0.0" }, From 809277c5ab764741ba24e0c5250989aa45bfe2b0 Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Wed, 12 Feb 2020 15:37:35 +0100 Subject: [PATCH 24/41] Fields.yml: mgr_cluster_disk --- metricbeat/docs/fields.asciidoc | 166 ++++++++++++++++++ metricbeat/module/ceph/fields.go | 2 +- .../ceph/mgr_cluster_disk/_meta/fields.yml | 63 ++++++- .../ceph/mgr_cluster_health/_meta/fields.yml | 6 +- 4 files changed, 232 insertions(+), 5 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index e3418ff1f2f6..df297ec4bc75 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -3938,12 +3938,178 @@ type: long Disk metrics of Ceph cluster +[float] +=== stats + +`stats` contains disk availability statistics. + + + +*`ceph.mgr_cluster_disk.stats.total_bytes`*:: ++ +-- +Total bytes of the cluster + +type: long + +-- + +*`ceph.mgr_cluster_disk.stats.total_avail_bytes`*:: ++ +-- +Available bytes of the cluster + +type: long + +-- + +*`ceph.mgr_cluster_disk.stats.total_used_bytes`*:: ++ +-- +Used bytes of the cluster + +type: long + +-- + +*`ceph.mgr_cluster_disk.stats.total_used_raw_bytes`*:: ++ +-- +Used raw bytes of the cluster + +type: long + +-- + +*`ceph.mgr_cluster_disk.stats.total_used_raw_ratio`*:: ++ +-- +Used raw bytes ratio + +type: scaled_float + +-- + +*`ceph.mgr_cluster_disk.stats.num_osds`*:: ++ +-- +Number of OSDs in the cluster + +type: long + +-- + +*`ceph.mgr_cluster_disk.stats.num_per_pool_osds`*:: ++ +-- +Number of OSDs per pool in the cluster + +type: long + +-- + +[float] +=== pools + +`pools` contains disk pool statistics. + + + +*`ceph.mgr_cluster_disk.pools.name`*:: ++ +-- +Pool name + +type: keyword + +-- + +*`ceph.mgr_cluster_disk.pools.id`*:: ++ +-- +Pool ID + +type: long + +-- + +[float] +=== stats + +Pool statistics + + +*`ceph.mgr_cluster_disk.pools.stats.kb_used`*:: ++ +-- +Used space in kilobytes + +type: long + +-- + +*`ceph.mgr_cluster_disk.pools.stats.bytes_used`*:: ++ +-- +Used space in bytes + +type: long + +-- + +*`ceph.mgr_cluster_disk.pools.stats.percent_used`*:: ++ +-- +Used space in pct + +type: scaled_float + +format: percent + +-- + +*`ceph.mgr_cluster_disk.pools.stats.max_avail`*:: ++ +-- +Maximum available space in bytes + +type: long + +-- + +*`ceph.mgr_cluster_disk.pools.stats.stored`*:: ++ +-- +Stored bytes + +type: long + +-- + +*`ceph.mgr_cluster_disk.pools.stats.objects`*:: ++ +-- +Number of stored objects + +type: long + +-- + [float] === mgr_cluster_health Status metrics of Ceph cluster +*`ceph.mgr_cluster_health.fsid`*:: ++ +-- +File System ID, a unique identifier for the cluster + +type: keyword + +-- + [float] === mgr_osd_disk diff --git a/metricbeat/module/ceph/fields.go b/metricbeat/module/ceph/fields.go index 85d979fff728..34d63d0e7908 100644 --- a/metricbeat/module/ceph/fields.go +++ b/metricbeat/module/ceph/fields.go @@ -32,5 +32,5 @@ func init() { // AssetCeph returns asset data. // This is the base64 encoded gzipped contents of ../metricbeat/module/ceph. func AssetCeph() string { - return "eJzEms2O2zYQx+/7FIM9tcCu0V59KJAmAbJoN1l0k1NRKLQ4klhTGoKk1vXbF6Rk2danPyjHp8Ty/uen4XDIGfIR1rhdQowquwOwwkpcwv17VNn9HQBHE2uhrKBiCb/dAQC4R5ATLyXeAZiMtI1iKhKRLiFh0rhvNUpkBpeQMvcbtFYUqVnC3/fGyPsHuM+sVff/3AEkAiU3S6/8CAXLsWFxH7tVTkVTqepveojc57v7o+8QU2GZKAzYDCFHq0Xs/s0sbFAjmFgzhRwSTTm8//jyaVELHGIcocjSWNQRF2bdPOzDGkFznwGdYz/tPm2YQyD2xoRkK4mL1daiOfrNjktSkbYejKC5z7udKnhVoMQ7sKZu/TwhnTO7hC7ADtKSZTIo4FenGAauNMiDsn0zyC9Ha0dahkzaLESsdZTOjzZ6Q82kjIxltuz31xq3G9L8PJd9qXSh0h1zWxNSIsc4w3htFqgozgKN3TNT8IbaCCqmzGoqC754Y7LEQMYbcfDapwGEHorXoyE4BmnHZsd2f2z2x9kJEdvRH4vNvmG7fCz8qlb7YjQgNEsSES80Mh6dlUWGM9M0XOUfcEbBZprKNFOlBYUaDMY0FDs160YLizeH9VYvoPWeJRUpFxAYhxrfQxcKUuY8583GU3npFKBcGCVZjAu/ugbG2ImDSqEo89VAFm4YaPUvxjbUCtqhqOVPQtHMCuoFMTGTyKNEErMD8a1Qx1i0n56L20XYUXJMNePIZxm0nfjEoDUM8wxaQ3HCoDUoP27QGtzhQVPpgjPLbp4xVQrO7sjeVaULXwDcHI0dFwhDcD7Kbw5n98XBEJjb9N+cq2wKgwEsv+FBv6XDyH0ZaGq+pH5njYe/G0eIqexMqEutv2a0MZDRBnJWbEGlBphGEEUNRcnkq3f4Am/26gEy0/s9MnyRlLI/da+IJLI21YTxJ+NEoSN6aLFApmex6oQnTJd5RIaHWiVaweCUXSRMlXs7klLNDeNCkyqgJjy/vYxCieJHQD19HoXSmDOlkEcqvTXZXx+f3728fPwwyBeyYvda7T1GsylMdRSqZfdBmHXTQqSkqhC7MdsUuyu0Y22VPsTrez11xToPJhl+tRe/vH4APp8nHaJCnVyL6DTcgl/EOCcpUdVLm+6hTPESVd2zOTCtRrwW0GmEB6RCWDp53pzdhzrWr/HdlnyEDXqb8yoOtZvat+brsmfXqnv+0r8gdJwD1zUIP1XOmLC6f/X1Kvib//H7lPnqxCGY6eq4YdqsP0sIZtUfJEwblczYqFScWWwPZWXZPTnzfUWOsMmwgA0z0Ke9Mz5YpVwUW59ZjlOvayxprLLmQlIa9OjmT0r3JzdtgFMOkw7hcmHioHTPwsTh8IyxQeleX7+Gg5v9xPBawMlJd0HwMWM7c+1g2xzx6W3N+UctbiV2laDflJWGpa4yr3xxXAOPLXQilA98TUoc24rzpJvG2mDDgeObiDGKJTMBz9gau07jAYSUmDLp/weiiGXJETLOH8AYDmjjxcga58L1VOdf0cNqQqRqrr2RLAdc1pyn35KqCtwRqgvvSYRg2/dKR/hUGhVlHmgWmXbHDSTFLq1UpbrwvZ/hwRvarM7Vim+NYq3BUuzNgtdWI03G8yWJy3WT+fNmqY80rMp4jfbGSbBldzAdOjOzWh5JiGT4A2Rk7ANoIjuSF7cKozmHxUk99Xe34kxIrrG/MX2Rk2rDO2GQwnnAoGLaz+jVFmLKc9bvi1iXJos2KNKsf073TeZTVy8vDj3i+9VTDVSe54+Dt9gV3JnC/4QZOMq8qD8uTGXRWCFlpe7CoCD706+PWzQP8MtjQT/353Itcqa3EUsSUQi7DeV4VxVV3nY7WI2Mi6I+IvQXGGu7g8ld40ggXHBdKcNG0gENmQ19Q2l/ScynBGHdeldKDiuEUrlR4rTpP72ZZzvnHFEpg1du0KRYn7KRS5jNOrcDr+RRTGNR5ytfUFeLfiuVdRZY35G8tsfbJ/IDF9YnvmspOLAbrKiHXYxBk1UZedPrux2W04pexxn2qshnf1rjp0l9R+Q0b93gqu4VPpqh+bdeHXH9HwAA//8tPTqG" + return "eJzEm1tv47oRx9/zKQZ5aoGs0b7mocDpyRYnaLMb1GefikJLkyOJNSWyJBWvv/0BqYttibrYpr1+2o2V//w0HM4ML/kEW9w/A0WVPwBYbgU+w+OvqPLHBwCGhmquLJflM/ztAQDAfQWFZJXABwCTS20TKsuUZ8+QEmHcTzUKJAafISPuGbSWl5l5hv88GiMen+Axt1Y9/vcBIOUomHn2yp+gJAV2LO5j98qpaFmp5icBIvf57n7pO1BZWsJLAzZHKNBqTt2/iYUdagRDNVHIINWygF8/v/+2agSOMU5QRGUs6oRxs+2+DGFNoLnPiM6pn9pPH+YYiHwQLshG4Gqzt2hOnmm5hCyz3hcTaO7zS6sKXhVk6h3YUPceT6UuiH2GIUALaaUlIirg704xDlxlkEVl+2aQXY7Wj7QcibB5jFgbKJ0fbfIDNREiMZbYKuyvLe53UrPzXPa11oVad8ptXUjxAmmOdGtWqCTNI43dG1HwgdpwWc6Z1bIq2eqDiAojGe/EwWsvA4g9FOuTITgF6cfmwHY4NsNxtiBiB/pTsRkatsvHwle1xheTAaFJmnK60khYclYWGc9M83C1f8AZBZtrWWW5qiwo1GCQyrHYaVh3mlu8O6y3egGt96xUiXIBgTTW+B67kEtlznPezXhqLy0BKrhRglBc+eoaGaMVB5VBWRWbkSzcMcjN/5DaWBV0QNHIL0LRxHIZBDGUCGRJKiSxI/GtUFMs+9+eiztEaCkZZpowZDcZtFZ8ZtA6htsMWkexYNA6lJ83aB3u+KCpbMWIJXfPmCoDZ3eid1XZyi8A7o5GThcIY3A+yu8OZw+LgzEw1/TfnavqFgYjWL7hQd/SYeJ+GGlqvme+s8bj56YRqKwGE+pS6+tc7gzkcgcFKfegMgNEI/CygZLp7KsP+CI3e80Amfl+Txq2SisRTt0bKQWSPtWM8VfjRGEgemyxRKJvYtUJz5iuikQaFqtK9ILBKbtImFvutSSVujWMC01ZA3Xh+e19EoqXPwPq9csklMaCKIUsUdm9yf79+e2X9/fPL6N8MVfsXqvfY3RNYaaTy7fsXrjZdpuGMq3XhMMo7Za3G7RLN1Kcr8LD0ida4ILvXu1os9O9aluiueB27+1xYzk1q35dC1BCf+8uWCphatQG2It37Pqmxxuc8wDO2tPsQ4z2C+cxLN0aDJrXZBcLQZPdVRihdh2WtOxzOGHhlmKkFp3ngC8+VbgX/7p+mSw+fdNuxa+kFNEZ3ErfCS+phO65eJnDq/Uzh0e5PGMEGleY2ZMcoL47hKBQa4WHNJaPgDfw+jIqH0rQMOnqsImDGwOPj3nyGGS78dMu+MzMCw+I/GQzilDffW+5kGP55GDfP3ErhHnzzdp+HmAm6cyAKDr2S9ObDKewBflRl6sornojP3hRFUdr7TP8ZqzUkYZs7aUW2AzvJ11m9JAh6zcZiIdavEvOypo9/tu0eakZpKnFRzP/4AJhvTcWC3h9eQICVcn/XyFwhqXlKUftgjNYMI6dIw27+rj66/qlLg3x3NRHVKjTaxGdhpuuJcVbkroWoF8fLuNtK8QtMK1GvBbQacQHlCW3cvFsPfv08FS/wWdk8aQ9XKkYVoXrL1Q0daRtvN++hpfxA+fAdce6v9XOmLF6ePXtJvqb//Pvc+breyLRTNdLznmz/gZINKu+sZg3KoixSaUYsYMyXVt235z5vrxA2OVYwo4YCGlPtOhXxNYXUuDc6/oqXmfNlZBZ1As3/5LZYTXbB1hyBegYruCGRqV744bGwzPGRqVbr3+PB3fze17XAs5OuguCjxg7mGtHm50Jm29rzr8g4yqxNKxuyipDMrcsqH1xenIxVehGetPzfeBPEiTDvuJt0k1nbfSYiOEHp5hQQUzEm1GdXafxBFwIzIjw/wNeUlExhJyxJzCGAVq6mqhxLlyXOv+Kk8cuROoj0Q8pqhGXdbcg70lVB+4E1YW3W2OwHVbdE3wqS8qqiDSLTP+cFISkLq3UByzcn9iND95Ys3qrCxS9UWw0SIbBLHjtaqTLeH5J4nLdbP68W+qTGjYV3aK9cxLs2R1Nh87MTS1PJERp2BPk0tgn0FLaiby4V5jcclic1Gv4TJLmXDCN4esEFzmpMdwKg+DOAwYV0X5Gb/ZAZVGQsC+orkye7JBneXhOhybz0urlxSEgfqieamTlef44eItDwdYU/uBm5Bz0olsN3NQWjeVC1OouDEpp//TXT3s0T/CXT6X8cziXa14QvU9ImvKS230sx7tVUe1t18FqJIyXzcUu/2cnjd3R5K5xIhAuuGSeYyfpgMbMxr5Xfrja71MCt67eVYLBBqFSbpSY3IXv3NymnXOOqJXBK3dogm+XNHIpsfngwPBKHkU0lk2+8gvquuj3UtmgwPodyWv3eEMiP7GwvrJ2S8GB3aGiHu9ijJqsl5F3/aOrAcuyRa/jjHvB93Ak097sXeatO/yB1RU+usHm33ZzwvVHAAAA//9glLWa" } diff --git a/metricbeat/module/ceph/mgr_cluster_disk/_meta/fields.yml b/metricbeat/module/ceph/mgr_cluster_disk/_meta/fields.yml index 64eb95ee4f69..de265b599687 100644 --- a/metricbeat/module/ceph/mgr_cluster_disk/_meta/fields.yml +++ b/metricbeat/module/ceph/mgr_cluster_disk/_meta/fields.yml @@ -1,6 +1,65 @@ - name: mgr_cluster_disk type: group - description: > - Disk metrics of Ceph cluster + description: Disk metrics of Ceph cluster release: beta fields: + - name: stats + type: group + description: > + `stats` contains disk availability statistics. + fields: + - name: total_bytes + type: long + description: Total bytes of the cluster + - name: total_avail_bytes + type: long + description: Available bytes of the cluster + - name: total_used_bytes + type: long + description: Used bytes of the cluster + - name: total_used_raw_bytes + type: long + description: Used raw bytes of the cluster + - name: total_used_raw_ratio + type: scaled_float + description: Used raw bytes ratio + - name: num_osds + type: long + description: Number of OSDs in the cluster + - name: num_per_pool_osds + type: long + description: Number of OSDs per pool in the cluster + - name: pools + type: group + description: > + `pools` contains disk pool statistics. + fields: + - name: name + type: keyword + description: Pool name + - name: id + type: long + description: Pool ID + - name: stats + type: group + description: Pool statistics + fields: + - name: kb_used + type: long + description: Used space in kilobytes + - name: bytes_used + type: long + description: Used space in bytes + - name: percent_used + type: scaled_float + description: Used space in pct + format: percent + - name: max_avail + type: long + description: Maximum available space in bytes + - name: stored + type: long + description: Stored bytes + - name: objects + type: long + description: Number of stored objects diff --git a/metricbeat/module/ceph/mgr_cluster_health/_meta/fields.yml b/metricbeat/module/ceph/mgr_cluster_health/_meta/fields.yml index 45072b45c0e4..22753608aa0f 100644 --- a/metricbeat/module/ceph/mgr_cluster_health/_meta/fields.yml +++ b/metricbeat/module/ceph/mgr_cluster_health/_meta/fields.yml @@ -1,6 +1,8 @@ - name: mgr_cluster_health type: group - description: > - Status metrics of Ceph cluster + description: Status metrics of Ceph cluster release: beta fields: + - name: fsid + type: keyword + description: File System ID, a unique identifier for the cluster From c63294b6f1e48dd6ee69c1357b5a29eb8dd9647c Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Wed, 12 Feb 2020 16:06:12 +0100 Subject: [PATCH 25/41] Fields: mgr_cluster_health in progress --- metricbeat/docs/fields.asciidoc | 178 ++++++++++++++++++ metricbeat/module/ceph/fields.go | 2 +- .../ceph/mgr_cluster_health/_meta/data.json | 12 +- .../ceph/mgr_cluster_health/_meta/fields.yml | 72 +++++++ .../ceph/mgr_cluster_health/manifest.yml | 11 ++ 5 files changed, 267 insertions(+), 8 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index df297ec4bc75..86cf35ca5279 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -4110,6 +4110,184 @@ type: keyword -- +*`ceph.mgr_cluster_health.quorum_age`*:: ++ +-- +Cluster quarum age + +type: long + +-- + +[float] +=== monmap + +`monmap` contains information about cluster monitors. + + + +*`ceph.mgr_cluster_health.monmap.min_mon_release`*:: ++ +-- +Minimal monitor release + +type: long + +-- + +*`ceph.mgr_cluster_health.monmap.min_mon_release_name`*:: ++ +-- +Minimal monitor release name + +type: keyword + +-- + +[float] +=== features + +`features` contains information about available cluster features. + + + +*`ceph.mgr_cluster_health.monmap.features.persistent`*:: ++ +-- +Persistent feature + +type: keyword + +-- + +*`ceph.mgr_cluster_health.monmap.features.optional`*:: ++ +-- +Optional feature + +type: keyword + +-- + +[float] +=== mons + +`mons` contains information about monitor configurations. + + + +*`ceph.mgr_cluster_health.monmap.mons.public_addr`*:: ++ +-- +Monitor public address + +type: keyword + +-- + +*`ceph.mgr_cluster_health.monmap.mons.rank`*:: ++ +-- +Monitor rank + +type: long + +-- + +*`ceph.mgr_cluster_health.monmap.mons.name`*:: ++ +-- +Monitor name + +type: keyword + +-- + +[float] +=== public_addrs + +Monitor public addresses + + +[float] +=== addrvec + +`addrvec` contains details of all public addresses. + + + +*`ceph.mgr_cluster_health.monmap.mons.public_addrs.addrvec.type`*:: ++ +-- +Address type + +type: keyword + +-- + +*`ceph.mgr_cluster_health.monmap.mons.public_addrs.addrvec.addr`*:: ++ +-- +Host and port of monitor + +type: keyword + +-- + +*`ceph.mgr_cluster_health.monmap.mons.public_addrs.addrvec.nonce`*:: ++ +-- +Nonce value + +type: long + +-- + +*`ceph.mgr_cluster_health.monmap.mons.addr`*:: ++ +-- +Monitor address + +type: keyword + +-- + +*`ceph.mgr_cluster_health.monmap.epoch`*:: ++ +-- +Epoch version + +type: long + +-- + +*`ceph.mgr_cluster_health.monmap.fsid`*:: ++ +-- +File System ID, a unique identifier for the cluster + +type: keyword + +-- + +*`ceph.mgr_cluster_health.monmap.modified`*:: ++ +-- +Last modification time and date + +type: date + +-- + +*`ceph.mgr_cluster_health.monmap.created`*:: ++ +-- +Creation time and date + +type: date + +-- + [float] === mgr_osd_disk diff --git a/metricbeat/module/ceph/fields.go b/metricbeat/module/ceph/fields.go index 34d63d0e7908..35bd3a66cf62 100644 --- a/metricbeat/module/ceph/fields.go +++ b/metricbeat/module/ceph/fields.go @@ -32,5 +32,5 @@ func init() { // AssetCeph returns asset data. // This is the base64 encoded gzipped contents of ../metricbeat/module/ceph. func AssetCeph() string { - return "eJzEm1tv47oRx9/zKQZ5aoGs0b7mocDpyRYnaLMb1GefikJLkyOJNSWyJBWvv/0BqYttibrYpr1+2o2V//w0HM4ML/kEW9w/A0WVPwBYbgU+w+OvqPLHBwCGhmquLJflM/ztAQDAfQWFZJXABwCTS20TKsuUZ8+QEmHcTzUKJAafISPuGbSWl5l5hv88GiMen+Axt1Y9/vcBIOUomHn2yp+gJAV2LO5j98qpaFmp5icBIvf57n7pO1BZWsJLAzZHKNBqTt2/iYUdagRDNVHIINWygF8/v/+2agSOMU5QRGUs6oRxs+2+DGFNoLnPiM6pn9pPH+YYiHwQLshG4Gqzt2hOnmm5hCyz3hcTaO7zS6sKXhVk6h3YUPceT6UuiH2GIUALaaUlIirg704xDlxlkEVl+2aQXY7Wj7QcibB5jFgbKJ0fbfIDNREiMZbYKuyvLe53UrPzXPa11oVad8ptXUjxAmmOdGtWqCTNI43dG1HwgdpwWc6Z1bIq2eqDiAojGe/EwWsvA4g9FOuTITgF6cfmwHY4NsNxtiBiB/pTsRkatsvHwle1xheTAaFJmnK60khYclYWGc9M83C1f8AZBZtrWWW5qiwo1GCQyrHYaVh3mlu8O6y3egGt96xUiXIBgTTW+B67kEtlznPezXhqLy0BKrhRglBc+eoaGaMVB5VBWRWbkSzcMcjN/5DaWBV0QNHIL0LRxHIZBDGUCGRJKiSxI/GtUFMs+9+eiztEaCkZZpowZDcZtFZ8ZtA6htsMWkexYNA6lJ83aB3u+KCpbMWIJXfPmCoDZ3eid1XZyi8A7o5GThcIY3A+yu8OZw+LgzEw1/TfnavqFgYjWL7hQd/SYeJ+GGlqvme+s8bj56YRqKwGE+pS6+tc7gzkcgcFKfegMgNEI/CygZLp7KsP+CI3e80Amfl+Txq2SisRTt0bKQWSPtWM8VfjRGEgemyxRKJvYtUJz5iuikQaFqtK9ILBKbtImFvutSSVujWMC01ZA3Xh+e19EoqXPwPq9csklMaCKIUsUdm9yf79+e2X9/fPL6N8MVfsXqvfY3RNYaaTy7fsXrjZdpuGMq3XhMMo7Za3G7RLN1Kcr8LD0ida4ILvXu1os9O9aluiueB27+1xYzk1q35dC1BCf+8uWCphatQG2It37Pqmxxuc8wDO2tPsQ4z2C+cxLN0aDJrXZBcLQZPdVRihdh2WtOxzOGHhlmKkFp3ngC8+VbgX/7p+mSw+fdNuxa+kFNEZ3ErfCS+phO65eJnDq/Uzh0e5PGMEGleY2ZMcoL47hKBQa4WHNJaPgDfw+jIqH0rQMOnqsImDGwOPj3nyGGS78dMu+MzMCw+I/GQzilDffW+5kGP55GDfP3ErhHnzzdp+HmAm6cyAKDr2S9ObDKewBflRl6sornojP3hRFUdr7TP8ZqzUkYZs7aUW2AzvJ11m9JAh6zcZiIdavEvOypo9/tu0eakZpKnFRzP/4AJhvTcWC3h9eQICVcn/XyFwhqXlKUftgjNYMI6dIw27+rj66/qlLg3x3NRHVKjTaxGdhpuuJcVbkroWoF8fLuNtK8QtMK1GvBbQacQHlCW3cvFsPfv08FS/wWdk8aQ9XKkYVoXrL1Q0daRtvN++hpfxA+fAdce6v9XOmLF6ePXtJvqb//Pvc+breyLRTNdLznmz/gZINKu+sZg3KoixSaUYsYMyXVt235z5vrxA2OVYwo4YCGlPtOhXxNYXUuDc6/oqXmfNlZBZ1As3/5LZYTXbB1hyBegYruCGRqV744bGwzPGRqVbr3+PB3fze17XAs5OuguCjxg7mGtHm50Jm29rzr8g4yqxNKxuyipDMrcsqH1xenIxVehGetPzfeBPEiTDvuJt0k1nbfSYiOEHp5hQQUzEm1GdXafxBFwIzIjw/wNeUlExhJyxJzCGAVq6mqhxLlyXOv+Kk8cuROoj0Q8pqhGXdbcg70lVB+4E1YW3W2OwHVbdE3wqS8qqiDSLTP+cFISkLq3UByzcn9iND95Ys3qrCxS9UWw0SIbBLHjtaqTLeH5J4nLdbP68W+qTGjYV3aK9cxLs2R1Nh87MTS1PJERp2BPk0tgn0FLaiby4V5jcclic1Gv4TJLmXDCN4esEFzmpMdwKg+DOAwYV0X5Gb/ZAZVGQsC+orkye7JBneXhOhybz0urlxSEgfqieamTlef44eItDwdYU/uBm5Bz0olsN3NQWjeVC1OouDEpp//TXT3s0T/CXT6X8cziXa14QvU9ImvKS230sx7tVUe1t18FqJIyXzcUu/2cnjd3R5K5xIhAuuGSeYyfpgMbMxr5Xfrja71MCt67eVYLBBqFSbpSY3IXv3NymnXOOqJXBK3dogm+XNHIpsfngwPBKHkU0lk2+8gvquuj3UtmgwPodyWv3eEMiP7GwvrJ2S8GB3aGiHu9ijJqsl5F3/aOrAcuyRa/jjHvB93Ak097sXeatO/yB1RU+usHm33ZzwvVHAAAA//9glLWa" + return "eJzEXE+P27oRv+dTDPbUAhujve6hQJpNkUXfJovue6ei0NLiWGJNiQxJreNv/0Dqj2WJlGSbdnx6by3/5seZ0fzTKB9hi/sHSFHmHwAMMxwf4O4zyvzuAwBFnSomDRPlA/zjAwCA/QoKQSuOHwB0LpRJUlFuWPYAG8K1/atCjkTjA2TEXoPGsDLTD/DfO6353T3c5cbIu/99ANgw5FQ/OOSPUJICOy72Y/bSoihRyeYvHkb282Z/9AapKA1hpQaTIxRoFEvtfxMDO1QIOlVEIoWNEgV8/vLyddUA9GkcUeGVNqgSyvS2+9JHa4Ka/QRwjvXUfoZk+oTIO2GcrDmu1nuD+uialhcXZTb4YoKa/XxqUcGhgtg4BTasB5dvhCqIeYAxgZakEYbwqAR/t4hxyFUaaVRuf2ik51MbelqOhJs8hq+NkE73NvGOinCeaENM5dfXFvc7oehpKvte40KNO6W2zqVYgWmO6VavUIo0j2S7ZyLhHZVmopwTq0RV0tU74RVGEt6Bg8NeRiC2KV6PTHBMZOibI9l+3/T72QKPHeFP+abPbOfbwmW1RheTDqHIZsPSlUJCk5OiSDgyzZOr9QNWKJhciSrLZWVAogKNqQj5TsN1p5jBm5N1Us9g6zQrZCKtQ2Aay759FTIh9WnKuxqfWktLCBVMS05SXLnsGplGCw4yg7Iq1oEo3HEQ6/9jamJl0BGLBn4RFUUME14iOiUcabLhgpiAf0tUKZbDb0+lO6bQsqSYKUKRXsVoLfiM0ToO1zFax2KB0Toqv85oHd2w0WS2osSQm0dMmYGVO1G7ymzlGoCbUyPHDUKInPPym5Mzh+YgRMwW/TfnVXWNQYCWK3jQlXSY2D9GujVfMldZY/+6aQqpqEY31LnSX3Ox05CLHRSk3IPMNBCFwMqGlNjMHn3EL3Kx1xhIz9d7QtPVpuL+0L0WgiMZspoR/qQtKIxA+xJLJOoqUi3wjOiqSISmsbLEwBkssvWEuXavZVLJa5OxrilqQp17/vEySYqVv4LU07dJUgoLIiXSRGa3ZvafL8+fXl6+PAb5xezYHdawxuiKwkwl54/sHpnedkNDsal7wrGXdu3tGs3SQYrVld8sQ0YLVPDm0HrDTnvUNkUzzszeyWPasFSvhnnNwxKGsztvqoQpq41oL57YDUWHC5zTCJw00xySCNYLp3FYOhr0ildkF4uCIruLaPjKdVhSss/R8QO3LAK56DQFfHOhwh78++vjZPIZirYdvxSCR+dgO30LvCQT2uviRQ6HNowcjsr5EcNTuMLMTHJE9cVS8AK1UpgPY7kFnICnxyC8L0DDpKr9Ig5q9Fwe0mSfyHbtbjvvNTMHHjFyN5uWJHXV95ZxEYonB/nuimtRmBff9PbzBGaCzgwRmYZ+ND1kOCZbkJ91uoqiqmfykxVV0eu1T9CbNkJFMtmrg1og0z9POk/oIULWJxmB+0q8c56VNTP+65R5Gz0KU4sfzfyLcYTXvTZYwNPjPRCoSvajQmAUS8M2DJV1ztmE8aMSqioSkp03TWh74x8VUdYZM39nXoiyIMOQeH5equF6iYmV9Y3IRAlkLSrTntlKZkaok9NUwcqkEGXS2PWibPLMSlYQ3nKBEGZAtm/WA6emzACH6Sy6QWIqNVFSLsp0YwPaz1sLPmnGQ3RrDdr+bGhQWJgxJSrNtAkF63m1js730iG25KbjoPvZaLx/tvjvDV5QeO8mvI4lLfCkFVunqxd/KlfGl+dbsFpzliaE0nFB3j/SCTp8bgjW0GChUU+nM0XKbZxE3t6RIcCZyvmiAwcxPcqeTt0hBwqKPtZ1oHiY9og+UYvzPnrgehrPEVe/w7eft0Zmv0NCQxh3hQLhfHRGn8svPWj/sPYokxcucwnvmT/VZOdl9DV/LTJfhTZASgpSKGO12oSSRcRKUabL1DRxr3ppfbPI4NvuOWZwjRgVCk6tTN/4dPakR5K+uKGp/yHKZPW65FxRq1g4Sm/U/iDMiRLjs9YRod+INg1SWmcwwwp0Huj9ebfzpJCYC2V/thhhkf2WRmh68ZLp99fHeqATr7kZUpSoNpdStBi2nLA33BWZClHvLc7vq83xbec616BpFOKlBC1GfIJ1cFraY5+883eM39CnZHGrfViEHs9yLl+DbqY/7bj8+bv/4dtIOXDZMubXWhkzUg9H366jn/zf/5wTX293RxNdPyiaF+v2tqNJdePAeaGcaJNUknpSQTANzJ3X5oJdjiXsiAYf9kR7cIFvfSMFzh3Xzd7qqLniIou6Jv+byA7PoIYElizu98kVTKdR2T0zncajp7WJyu719fd45K7+dsalBGdvujOcz1aBQ8jeikJC58ua09fabSYWmtZFWaVJhv1RysJEF5gon64Dt/8jKA4RrxNuOmnB5S6K7yzFJOVk1PvEkGsx7oFxjhnh7v+AlSmvKEJO6T1oTQFNuprIcdZdlyr/gn3BzkXqRcZ3wauAyrp3l27JqnbcCVZnvpMWg9thmjzBT2ZJWRWR7iI93G4ELlIbVuq1KOb27MLGCxWr11p7Hlixweg/1elHwUu7kS7iuZbExrrZ+Hmz0CcUrKt0i+bGQXAgNxgOPdO5uJInAqLQ9B5yoc09KCHMRFzcS0yuaRYL9eTfJExzxqlC/xLwWUpqBLfAwJnVgEZJlLuj13tIRVEQvy5SVek82SHLcv897buZl2YvBw4e8EP2lIHO83Q7OIljwG7++JPpwPbiWbvITNcStWGc1+jWDUph/vL3j3vU9/C3j6X4qz+WK1YQtU/IZsNKZvaxFG+7olrbtoJVSCgrm9cx3MvijdxgcFc44QhnvBqaYwdpCYXExn4b9PBCrgsJzNh8V3EKa4RKWitRsfNvyl+nnLOKqJHBIXfUONsuKeQ2xOSjEfOFfCRRWDbxyjXUddIfhLJRgnUTyUtnvD6QX5hYn2g7UrDEbpBR+1OMoMi6jbzpP5Uw4rKs6bU8476Wd1ikat/HW6atG/yzCBfo6ArDv+36iNefAQAA///q5pTm" } diff --git a/metricbeat/module/ceph/mgr_cluster_health/_meta/data.json b/metricbeat/module/ceph/mgr_cluster_health/_meta/data.json index f01eb0ea305f..dbef986ed31e 100644 --- a/metricbeat/module/ceph/mgr_cluster_health/_meta/data.json +++ b/metricbeat/module/ceph/mgr_cluster_health/_meta/data.json @@ -97,13 +97,11 @@ "c213fde94857" ], "osdmap": { - "osdmap": { - "epoch": 21, - "num_osds": 1, - "num_up_osds": 1, - "num_in_osds": 1, - "num_remapped_pgs": 0 - } + "epoch": 21, + "num_osds": 1, + "num_up_osds": 1, + "num_in_osds": 1, + "num_remapped_pgs": 0 }, "fsmap": { "in": 1, diff --git a/metricbeat/module/ceph/mgr_cluster_health/_meta/fields.yml b/metricbeat/module/ceph/mgr_cluster_health/_meta/fields.yml index 22753608aa0f..b084b1780e30 100644 --- a/metricbeat/module/ceph/mgr_cluster_health/_meta/fields.yml +++ b/metricbeat/module/ceph/mgr_cluster_health/_meta/fields.yml @@ -6,3 +6,75 @@ - name: fsid type: keyword description: File System ID, a unique identifier for the cluster + - name: quorum_age + type: long + description: Cluster quarum age + - name: monmap + type: group + description: > + `monmap` contains information about cluster monitors. + fields: + - name: min_mon_release + type: long + description: Minimal monitor release + - name: min_mon_release_name + type: keyword + description: Minimal monitor release name + - name: features + type: group + description: > + `features` contains information about available cluster features. + fields: + - name: persistent + type: keyword + description: Persistent feature + - name: optional + type: keyword + description: Optional feature + - name: mons + type: group + description: > + `mons` contains information about monitor configurations. + fields: + - name: public_addr + type: keyword + description: Monitor public address + - name: rank + type: long + description: Monitor rank + - name: name + type: keyword + description: Monitor name + - name: public_addrs + type: group + description: Monitor public addresses + fields: + - name: addrvec + type: group + description: > + `addrvec` contains details of all public addresses. + fields: + - name: type + type: keyword + description: Address type + - name: addr + type: keyword + description: Host and port of monitor + - name: nonce + type: long + description: Nonce value + - name: addr + type: keyword + description: Monitor address + - name: epoch + type: long + description: Epoch version + - name: fsid + type: keyword + description: File System ID, a unique identifier for the cluster + - name: modified + type: date + description: Last modification time and date + - name: created + type: date + description: Creation time and date diff --git a/metricbeat/module/ceph/mgr_cluster_health/manifest.yml b/metricbeat/module/ceph/mgr_cluster_health/manifest.yml index 0462312f4756..57807639ba95 100644 --- a/metricbeat/module/ceph/mgr_cluster_health/manifest.yml +++ b/metricbeat/module/ceph/mgr_cluster_health/manifest.yml @@ -21,6 +21,11 @@ processors: fields: ["ceph.request.outb"] process_array: true target: "ceph.mgr_cluster_status" + - rename: + ignore_missing: true + fields: + - from: "ceph.mgr_cluster_status.osdmap.osdmap" + - to: "ceph.mgr_cluster_status._tmp_osdmap" - drop_fields: ignore_missing: true fields: @@ -30,3 +35,9 @@ processors: - "ceph.mgr_cluster_status.servicemap" - "ceph.mgr_cluster_status.health.checks" - "ceph.mgr_cluster_status.progress_events" + - "ceph.mgr_cluster_status.osdmap" + - rename: + ignore_missing: true + fields: + - from: "ceph.mgr_cluster_status._tmp_osdmap" + - to: "ceph.mgr_cluster_status.osdmap" From 516b4d5ae0021008fd0ddb0c1a14e9c8d41bef97 Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Wed, 12 Feb 2020 17:25:32 +0100 Subject: [PATCH 26/41] Fields: mgr_cluster_health --- metricbeat/docs/fields.asciidoc | 330 ++++++++++++++++++ metricbeat/module/ceph/fields.go | 2 +- .../ceph/mgr_cluster_health/_meta/fields.yml | 126 +++++++ 3 files changed, 457 insertions(+), 1 deletion(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 86cf35ca5279..8c71a29bd144 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -4288,6 +4288,336 @@ type: date -- +[float] +=== pgmap + +`pgmap` contains information about placement groups. + + + +*`ceph.mgr_cluster_health.pgmap.num_pgs`*:: ++ +-- +Number of PGs + +type: long + +-- + +*`ceph.mgr_cluster_health.pgmap.bytes_avail`*:: ++ +-- +Available bytes + +type: long + +-- + +*`ceph.mgr_cluster_health.pgmap.read_bytes_sec`*:: ++ +-- +Average number of bytes read per second + +type: long + +-- + +*`ceph.mgr_cluster_health.pgmap.write_bytes_sec`*:: ++ +-- +Average number of bytes written per second + +type: long + +-- + +*`ceph.mgr_cluster_health.pgmap.bytes_used`*:: ++ +-- +Used bytes + +type: long + +-- + +*`ceph.mgr_cluster_health.pgmap.bytes_total`*:: ++ +-- +Total bytes + +type: long + +-- + +*`ceph.mgr_cluster_health.pgmap.read_op_per_sec`*:: ++ +-- +Read operations per second + +type: long + +-- + +*`ceph.mgr_cluster_health.pgmap.write_op_per_sec`*:: ++ +-- +Write operations per second + +type: long + +-- + +[float] +=== pgs_by_state + +`pgs_by_state` contains information about PGs' states. + + + +*`ceph.mgr_cluster_health.pgmap.pgs_by_state.state_name`*:: ++ +-- +State name + +type: keyword + +-- + +*`ceph.mgr_cluster_health.pgmap.pgs_by_state.count`*:: ++ +-- +Count + +type: long + +-- + +*`ceph.mgr_cluster_health.pgmap.num_pools`*:: ++ +-- +Number of pools + +type: long + +-- + +*`ceph.mgr_cluster_health.pgmap.num_objects`*:: ++ +-- +Number of objects + +type: long + +-- + +*`ceph.mgr_cluster_health.pgmap.data_bytes`*:: ++ +-- +Data bytes + +type: long + +-- + +[float] +=== health + +`health` contains general information about cluster status. + + + +*`ceph.mgr_cluster_health.health.status`*:: ++ +-- +Health status + +type: keyword + +-- + +*`ceph.mgr_cluster_health.election_epoch`*:: ++ +-- +Election epoch + +type: long + +-- + +*`ceph.mgr_cluster_health.quorum`*:: ++ +-- +Quorum ID + +type: long + +-- + +*`ceph.mgr_cluster_health.quorum_names`*:: ++ +-- +Quorum names + +type: keyword + +-- + +[float] +=== osdmap + +`osdmap` contains information about OSDs. + + + +*`ceph.mgr_cluster_health.osdmap.epoch`*:: ++ +-- +Epoch version + +type: long + +-- + +*`ceph.mgr_cluster_health.osdmap.num_osds`*:: ++ +-- +Number of OSDs + +type: long + +-- + +*`ceph.mgr_cluster_health.osdmap.num_up_osds`*:: ++ +-- +Number of up OSDs + +type: long + +-- + +*`ceph.mgr_cluster_health.osdmap.num_in_osds`*:: ++ +-- +Number of in OSDs + +type: long + +-- + +*`ceph.mgr_cluster_health.osdmap.num_remapped_pgs`*:: ++ +-- +Number of remapped OSDs + +type: long + +-- + +[float] +=== fsmap + +`fsmap` contains information about file systems. + + + +*`ceph.mgr_cluster_health.fsmap.in`*:: ++ +-- +File system in + +type: long + +-- + +*`ceph.mgr_cluster_health.fsmap.max`*:: ++ +-- +File system max + +type: long + +-- + +*`ceph.mgr_cluster_health.fsmap.up:standby`*:: ++ +-- +File system up:standby + +type: long + +-- + +*`ceph.mgr_cluster_health.fsmap.epoch`*:: ++ +-- +Epoch version + +type: long + +-- + +*`ceph.mgr_cluster_health.fsmap.up`*:: ++ +-- +File system up + +type: long + +-- + +[float] +=== by_rank + +`by_rank` contains information about file systems sorted by rank. + + + +*`ceph.mgr_cluster_health.fsmap.by_rank.filesystem_id`*:: ++ +-- +File system ID + +type: long + +-- + +*`ceph.mgr_cluster_health.fsmap.by_rank.rank`*:: ++ +-- +File system rank + +type: long + +-- + +*`ceph.mgr_cluster_health.fsmap.by_rank.name`*:: ++ +-- +File system name + +type: keyword + +-- + +*`ceph.mgr_cluster_health.fsmap.by_rank.status`*:: ++ +-- +File system status + +type: keyword + +-- + +*`ceph.mgr_cluster_health.fsmap.by_rank.gid`*:: ++ +-- +File system GID + +type: long + +-- + [float] === mgr_osd_disk diff --git a/metricbeat/module/ceph/fields.go b/metricbeat/module/ceph/fields.go index 35bd3a66cf62..1208bf71230f 100644 --- a/metricbeat/module/ceph/fields.go +++ b/metricbeat/module/ceph/fields.go @@ -32,5 +32,5 @@ func init() { // AssetCeph returns asset data. // This is the base64 encoded gzipped contents of ../metricbeat/module/ceph. func AssetCeph() string { - return "eJzEXE+P27oRv+dTDPbUAhujve6hQJpNkUXfJovue6ei0NLiWGJNiQxJreNv/0Dqj2WJlGSbdnx6by3/5seZ0fzTKB9hi/sHSFHmHwAMMxwf4O4zyvzuAwBFnSomDRPlA/zjAwCA/QoKQSuOHwB0LpRJUlFuWPYAG8K1/atCjkTjA2TEXoPGsDLTD/DfO6353T3c5cbIu/99ANgw5FQ/OOSPUJICOy72Y/bSoihRyeYvHkb282Z/9AapKA1hpQaTIxRoFEvtfxMDO1QIOlVEIoWNEgV8/vLyddUA9GkcUeGVNqgSyvS2+9JHa4Ka/QRwjvXUfoZk+oTIO2GcrDmu1nuD+uialhcXZTb4YoKa/XxqUcGhgtg4BTasB5dvhCqIeYAxgZakEYbwqAR/t4hxyFUaaVRuf2ik51MbelqOhJs8hq+NkE73NvGOinCeaENM5dfXFvc7oehpKvte40KNO6W2zqVYgWmO6VavUIo0j2S7ZyLhHZVmopwTq0RV0tU74RVGEt6Bg8NeRiC2KV6PTHBMZOibI9l+3/T72QKPHeFP+abPbOfbwmW1RheTDqHIZsPSlUJCk5OiSDgyzZOr9QNWKJhciSrLZWVAogKNqQj5TsN1p5jBm5N1Us9g6zQrZCKtQ2Aay759FTIh9WnKuxqfWktLCBVMS05SXLnsGplGCw4yg7Iq1oEo3HEQ6/9jamJl0BGLBn4RFUUME14iOiUcabLhgpiAf0tUKZbDb0+lO6bQsqSYKUKRXsVoLfiM0ToO1zFax2KB0Toqv85oHd2w0WS2osSQm0dMmYGVO1G7ymzlGoCbUyPHDUKInPPym5Mzh+YgRMwW/TfnVXWNQYCWK3jQlXSY2D9GujVfMldZY/+6aQqpqEY31LnSX3Ox05CLHRSk3IPMNBCFwMqGlNjMHn3EL3Kx1xhIz9d7QtPVpuL+0L0WgiMZspoR/qQtKIxA+xJLJOoqUi3wjOiqSISmsbLEwBkssvWEuXavZVLJa5OxrilqQp17/vEySYqVv4LU07dJUgoLIiXSRGa3ZvafL8+fXl6+PAb5xezYHdawxuiKwkwl54/sHpnedkNDsal7wrGXdu3tGs3SQYrVld8sQ0YLVPDm0HrDTnvUNkUzzszeyWPasFSvhnnNwxKGsztvqoQpq41oL57YDUWHC5zTCJw00xySCNYLp3FYOhr0ildkF4uCIruLaPjKdVhSss/R8QO3LAK56DQFfHOhwh78++vjZPIZirYdvxSCR+dgO30LvCQT2uviRQ6HNowcjsr5EcNTuMLMTHJE9cVS8AK1UpgPY7kFnICnxyC8L0DDpKr9Ig5q9Fwe0mSfyHbtbjvvNTMHHjFyN5uWJHXV95ZxEYonB/nuimtRmBff9PbzBGaCzgwRmYZ+ND1kOCZbkJ91uoqiqmfykxVV0eu1T9CbNkJFMtmrg1og0z9POk/oIULWJxmB+0q8c56VNTP+65R5Gz0KU4sfzfyLcYTXvTZYwNPjPRCoSvajQmAUS8M2DJV1ztmE8aMSqioSkp03TWh74x8VUdYZM39nXoiyIMOQeH5equF6iYmV9Y3IRAlkLSrTntlKZkaok9NUwcqkEGXS2PWibPLMSlYQ3nKBEGZAtm/WA6emzACH6Sy6QWIqNVFSLsp0YwPaz1sLPmnGQ3RrDdr+bGhQWJgxJSrNtAkF63m1js730iG25KbjoPvZaLx/tvjvDV5QeO8mvI4lLfCkFVunqxd/KlfGl+dbsFpzliaE0nFB3j/SCTp8bgjW0GChUU+nM0XKbZxE3t6RIcCZyvmiAwcxPcqeTt0hBwqKPtZ1oHiY9og+UYvzPnrgehrPEVe/w7eft0Zmv0NCQxh3hQLhfHRGn8svPWj/sPYokxcucwnvmT/VZOdl9DV/LTJfhTZASgpSKGO12oSSRcRKUabL1DRxr3ppfbPI4NvuOWZwjRgVCk6tTN/4dPakR5K+uKGp/yHKZPW65FxRq1g4Sm/U/iDMiRLjs9YRod+INg1SWmcwwwp0Huj9ebfzpJCYC2V/thhhkf2WRmh68ZLp99fHeqATr7kZUpSoNpdStBi2nLA33BWZClHvLc7vq83xbec616BpFOKlBC1GfIJ1cFraY5+883eM39CnZHGrfViEHs9yLl+DbqY/7bj8+bv/4dtIOXDZMubXWhkzUg9H366jn/zf/5wTX293RxNdPyiaF+v2tqNJdePAeaGcaJNUknpSQTANzJ3X5oJdjiXsiAYf9kR7cIFvfSMFzh3Xzd7qqLniIou6Jv+byA7PoIYElizu98kVTKdR2T0zncajp7WJyu719fd45K7+dsalBGdvujOcz1aBQ8jeikJC58ua09fabSYWmtZFWaVJhv1RysJEF5gon64Dt/8jKA4RrxNuOmnB5S6K7yzFJOVk1PvEkGsx7oFxjhnh7v+AlSmvKEJO6T1oTQFNuprIcdZdlyr/gn3BzkXqRcZ3wauAyrp3l27JqnbcCVZnvpMWg9thmjzBT2ZJWRWR7iI93G4ELlIbVuq1KOb27MLGCxWr11p7Hlixweg/1elHwUu7kS7iuZbExrrZ+Hmz0CcUrKt0i+bGQXAgNxgOPdO5uJInAqLQ9B5yoc09KCHMRFzcS0yuaRYL9eTfJExzxqlC/xLwWUpqBLfAwJnVgEZJlLuj13tIRVEQvy5SVek82SHLcv897buZl2YvBw4e8EP2lIHO83Q7OIljwG7++JPpwPbiWbvITNcStWGc1+jWDUph/vL3j3vU9/C3j6X4qz+WK1YQtU/IZsNKZvaxFG+7olrbtoJVSCgrm9cx3MvijdxgcFc44QhnvBqaYwdpCYXExn4b9PBCrgsJzNh8V3EKa4RKWitRsfNvyl+nnLOKqJHBIXfUONsuKeQ2xOSjEfOFfCRRWDbxyjXUddIfhLJRgnUTyUtnvD6QX5hYn2g7UrDEbpBR+1OMoMi6jbzpP5Uw4rKs6bU8476Wd1ikat/HW6atG/yzCBfo6ArDv+36iNefAQAA///q5pTm" + return "eJzEXE2T27gRvftXdM0lSdVYlVznkKqNx7vryo5nsuOtHFIpGiJbFCKQoAFwZP37FMAPUSQAfgiUdbJH1HsPQLO70WjyPRzw9AAxFvt3AIoqhg9w9wGL/d07gARlLGihKM8f4O/vAAD0V5DxpGT4DkDuuVBRzPMdTR9gR5jUfxXIkEh8gJToa1ApmqfyAf5zJyW7u4e7vVLF3X/fAewoskQ+GOT3kJMMWy36o06FRhG8LOq/WBTpz1f9o68Q81wRmktQe4QMlaCx/jdRcESBIGNBCkxgJ3gGHz6+/LqpAboyLqSwUioUUULlof3SJssjTX8cOJfz1Hz6YrqCyBuhjGwZbrYnhfLimkYX43na+8IjTX9+alDBoALfmQmsVfcu33GREfUAQwGNSMUVYUEFftGIYcSVEpOg2v6QmCyX1re0PRKm9iFsbYA039r4GwrCWCQVUaV9vg54OnKRzJuy5woXKlzftLUmRTOM9xgf5AYLHu8Drd0TKeANhaQ8H6MVvMyTzRthJQYib8HBYE8TEHopXi+W4FJI3zYH3HbbtNvZBIsd4Pts07Zsy9fCRLV6LrwGIchuR+ONQJJEs7yI2zONi6vmBzQpqL3gZbovSgUFCpAYc5ft1FqPgiq8uVjDukCtmVleRIU2CIxDrW93Cikv5LzJW01PNUtTBGVUFozEuDHRNbCMBhyKFPIy2zq8cKuBb/+HsQoVQQcqavhJUgRRlFuFyJgwTKId40Q57LtAEWPe/3au3KGERmWCqSAJJqssWgM+smithnUWrVUxYdFaKT9u0Vq57kUr0k1CFLm5xyxS0Lye3LVIN2YDcHNp5HKD4BJnrPzm4tR5c+ASppP+m+sq242BQ5ZJeNCkdBjpPwa6NV9Sk1lj9zq/hJiXgxtqKfvrnh8l7PkRMpKfoEglEIFA81oU340OfaAvcLJXL5Acz/e4TDa7ktld95ZzhqSvaoT8k9SgMADtMuZIxCqsGniEuswiLpNQUaJnDBpZW8LYdq9RUhZri9GmyStBrXn+8eIVRfMfIerTZ68ogRkpCkyiIr21st8/Pv308vLx0akv5I7dYPVzjDYpTEW0vGT3SOWhLRryXbUnHFppu73doppaSNFzZV+WvqIJU/DVoHWKnXqoTYimjKqT4aNS0Vhu+nHNohL6tTtrqATfqg1kT67Y9andCc48AbNqmn0RznxhnoappUErvSDHUBIEOV4lw5auw5SUfUyOHbhR4YhF8ybgs3EVeuDPr4/e4NOn1jv+gnMWXIPe6WvgKZFQXxfOcxi0vucwUpZ7DEviCiM1yYHUFy3BCtSwUBvG9BUwBJ8enfA2Bw3eqbZTnKfRcrlrJrtCDltz21mvGRnwQJG52WRBYpN9HyjjLn9y5jdXrCVhnL7e248LGHE6I0KK2PUjf5HhUmxGvlfhKshUPZHvNCuzzl57xrxJxUWgJXs1UBM47fWkZaRnD1mNZABuS/GWnJXVNf510rydHLipyUczP1OG8HqSCjP49HgPBMqcfisRaIK5ojuKQhvnaMD4VnJRZhFJl1UTmr3xt5IIbYypfWee8TwjfZe4PC5VcJ3ARPPqRqQ8B7LlpWrGrJmp4mJ2mMpoHmU8j+p1vSqaPNGcZoQ1WsCF6eC21Xpgbsh0aPBH0R0SVQpPSjkp0g0XUH++NuDeZTx7t2ZBm5/1FxQmRswChaRSuZz1+LQOxvfSIjbi/H7Q/GxQ3l9M/1zjOck7N+E6K6mBvavYGF3V+FOaND5fvoLlltE4IkkyTMi7Q5oxh0+1wAoaNDRKfzgTJD+ECeTNHekCHMmcrxqwE9My2f7Q7TIgJ/XlXDuSB79FdIVqnLfBges8nQOtdoNvPl9rzu4OCRWhzCQKhLHBGG0mP3Wg3cHqoXgvnGYS1jH/VIkd5+jO/FpifuVSAckTKLhQelZrVzJJWM7zeNo0ee5Vq6zPGhls3T2XCtbwUS7n1HDayqejI71g+miKpvZDFG/2OmVcQbNYuAhvif6BW1NClG21LgT9RqSqkeIqgimaobFA68/bnieBRF3J/UFj+CnPJ10h02mD5g3gpnEh0xmOwZ9f9Ckzy0kDLKyKvfzitv6qIOHaZi+uBzv5zp1dlnafuZQoSIr1aYUeaF31RJK42n26UjqNW6tp0RwK8ylyvKWhJUXxESZbz8w8qi+OzgDor7ezv2se3+96YXmBdT48fY0D8f/bdJPNE1CkMtqeqnP2VTYTXQKvT3r5Rf6pOlVcvJNwtlJcDmdGkH41p5yjWbWtgeKSc+ru4YMV6cLvWo4ERnkcnteOdXEA46zyLeFzobUdYq7Gq3mEj/4+qkHhEK4KtxVcx7ZTzFEQ5qliVa0nc6Outfkb5uZovxq5NrA242QY62uj5Sf3H2sIS/Z6WalchP4v89P+gU6vBKr/s7hTvmYYYnQaG0KmbBWc1z8+vz7Otphb7R1WOKr1ctkbg5bSlcU4o73rZykjzccZPS09S2kbyCH5eRMY0qoNmteod3rDKM2GcbZx06E9zpmWn8/UNqjOKV8wGhtW+2Ba8SAVyZPtKRidB/LWDsKaPi4dlWfjEDmqrtfnsTX2ZGsGyYUyWx1TCV6a0mrICjGylmdG53Ewvu5cWloiYI2aeJfzpnXxLvFoFu/Mr66m9iA35OkKy/tLZ327J+hcJlc/0/z8+lj1D4U7S+9LLFDsrpWoMfRdmse4plLOq8dkxx+PHNPbtBGtIVMJxGsFaozwAqta+NSWjtmPmF7i1/L1ZtOjDazP3Q9bh65/6r5uNmq6M5+e7b3enm3romd/643gCOt56Idt8JH/8x9j9NXLBIJRVyXBcVrzmoBgrKbmOU7KiFRRWSSWkwfnqcPYeGmGcNxjDkciwYbtibpX2NZnkuHYcE2rV+U1N4ynQd/K8BtPzy3PfQFT3hPRFZdRGQdV90RlHE6elCqoutfXL+HErf4ykGsFjt50C4yPSDW41zqFoygZT2vmv0VBR2IukyopKyVJsbtDmRjoHA2M8+fAPG7GE+wjruNuWjbns4QJvtEYo5iRwVF7CF6NcQ+UMUwJM/8DmsesTBD2SXIPUiaAKt54Ypw216mTf8Xjqa2JVM/NvnFWOqasfVXOLVVVhutRtfAVSCG0nZsXPfqKNMoXlrWH1LL/MC0wHmu3Uj2FR81jne7FcyWraz1l31vFGqPbRNz1gtfuRlqPZ7Yk2teN+s+buT4uYFvGB1Q3doI9Xqc7tDSDhWX2OEQuk3vYc6nuQXCuPH7xVOCw6hVwWTSU4xAp3lOWCLQ/c75okmriBhgY1TMgsSCC1IXCmGcZsc9FLEq5j45I0739nrbdzFOjlwEHC/g5ehaOnef8dTCMQ8C2Iv2dSsfDsosefaeyYpSKMlahazPIufrz396fUN7DX9/n/C92Xy5oRsQpIrsdzanql9EXT7zeFVWzXZ3KkITm9ds/zLsJa16ncxfoMYQFbyLbYwtpTukdtKFfPnZ+/5txCVTpeFeyBLYIZaFXKeFH+4sZ1knn9ERUyGCQW2mMHqYkcjui9oOOxiv1FERgXvsrs6Gugn7PlQ0CrKlIXlvjtYH8wMD6KWlKClrYDSJqt4rhpKy2kTd9M+dAy7RNr9YZ9i1Qg86eibN1g7dwXjFHKxT/DtsLXf8PAAD//1D8Md0=" } diff --git a/metricbeat/module/ceph/mgr_cluster_health/_meta/fields.yml b/metricbeat/module/ceph/mgr_cluster_health/_meta/fields.yml index b084b1780e30..e3037e6968fe 100644 --- a/metricbeat/module/ceph/mgr_cluster_health/_meta/fields.yml +++ b/metricbeat/module/ceph/mgr_cluster_health/_meta/fields.yml @@ -78,3 +78,129 @@ - name: created type: date description: Creation time and date + - name: pgmap + type: group + description: > + `pgmap` contains information about placement groups. + fields: + - name: num_pgs + type: long + description: Number of PGs + - name: bytes_avail + type: long + description: Available bytes + - name: read_bytes_sec + type: long + description: Average number of bytes read per second + - name: write_bytes_sec + type: long + description: Average number of bytes written per second + - name: bytes_used + type: long + description: Used bytes + - name: bytes_total + type: long + description: Total bytes + - name: read_op_per_sec + type: long + description: Read operations per second + - name: write_op_per_sec + type: long + description: Write operations per second + - name: pgs_by_state + type: group + description: > + `pgs_by_state` contains information about PGs' states. + fields: + - name: state_name + type: keyword + description: State name + - name: count + type: long + description: Count + - name: num_pools + type: long + description: Number of pools + - name: num_objects + type: long + description: Number of objects + - name: data_bytes + type: long + description: Data bytes + - name: health + type: group + description: > + `health` contains general information about cluster status. + fields: + - name: status + type: keyword + description: Health status + - name: election_epoch + type: long + description: Election epoch + - name: quorum + type: long + description: Quorum ID + - name: quorum_names + type: keyword + description: Quorum names + - name: osdmap + type: group + description: > + `osdmap` contains information about OSDs. + fields: + - name: epoch + type: long + description: Epoch version + - name: num_osds + type: long + description: Number of OSDs + - name: num_up_osds + type: long + description: Number of up OSDs + - name: num_in_osds + type: long + description: Number of in OSDs + - name: num_remapped_pgs + type: long + description: Number of remapped OSDs + - name: fsmap + type: group + description: > + `fsmap` contains information about file systems. + fields: + - name: in + type: long + description: File system in + - name: max + type: long + description: File system max + - name: up:standby + type: long + description: File system up:standby + - name: epoch + type: long + description: Epoch version + - name: up + type: long + description: File system up + - name: by_rank + type: group + description: > + `by_rank` contains information about file systems sorted by rank. + fields: + - name: filesystem_id + type: long + description: File system ID + - name: rank + type: long + description: File system rank + - name: name + type: keyword + description: File system name + - name: status + type: keyword + description: File system status + - name: gid + type: long + description: File system GID From 70210456bbbc737566c9a9ecee0106255465b7d6 Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Wed, 12 Feb 2020 18:59:53 +0100 Subject: [PATCH 27/41] Update fields.go --- metricbeat/docs/fields.asciidoc | 652 ++++++++++++++++++ metricbeat/module/ceph/fields.go | 2 +- .../module/ceph/mgr_osd_disk/_meta/fields.yml | 147 ++++ .../module/ceph/mgr_osd_perf/_meta/fields.yml | 31 + .../ceph/mgr_osd_pool_stats/_meta/fields.yml | 22 + .../module/ceph/mgr_osd_tree/_meta/fields.yml | 38 + 6 files changed, 891 insertions(+), 1 deletion(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 8c71a29bd144..3f0edb8f8d11 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -4624,24 +4624,676 @@ type: long OSD disk metrics of Ceph cluster + +[float] +=== nodes + +`nodes` contains information about active nodes. + + + +*`ceph.mgr_osd_disk.nodes.depth`*:: ++ +-- +Depth + +type: long + +-- + +*`ceph.mgr_osd_disk.nodes.reweight`*:: ++ +-- +Reweight (controls how much data the system tries to allocate to the OSD) + +type: long + +-- + +*`ceph.mgr_osd_disk.nodes.pgs`*:: ++ +-- +Number of PGs + +type: long + +-- + +*`ceph.mgr_osd_disk.nodes.crush_weight`*:: ++ +-- +CRUSH weight + +type: scaled_float + +-- + +*`ceph.mgr_osd_disk.nodes.status`*:: ++ +-- +Node status + +type: keyword + +-- + +*`ceph.mgr_osd_disk.nodes.id`*:: ++ +-- +ID + +type: long + +-- + +*`ceph.mgr_osd_disk.nodes.name`*:: ++ +-- +Name + +type: keyword + +-- + +*`ceph.mgr_osd_disk.nodes.type_id`*:: ++ +-- +Type ID + +type: long + +-- + +*`ceph.mgr_osd_disk.nodes.kb_used_data`*:: ++ +-- +Used data (in KB) + +type: long + +-- + +*`ceph.mgr_osd_disk.nodes.kb_used_omap`*:: ++ +-- +Used omap (in KB) + +type: long + +-- + +*`ceph.mgr_osd_disk.nodes.kb_used_meta`*:: ++ +-- +Used meta (in KB) + +type: long + +-- + +*`ceph.mgr_osd_disk.nodes.kb_avail`*:: ++ +-- +Available space (in KB) + +type: long + +-- + +*`ceph.mgr_osd_disk.nodes.var`*:: ++ +-- +Var + +type: long + +-- + +*`ceph.mgr_osd_disk.nodes.type`*:: ++ +-- +Type + +type: keyword + +-- + +*`ceph.mgr_osd_disk.nodes.kb`*:: ++ +-- +Total space (in KB) + +type: long + +-- + +*`ceph.mgr_osd_disk.nodes.kb_used`*:: ++ +-- +Used space (in KB) + +type: long + +-- + +*`ceph.mgr_osd_disk.nodes.Utilization`*:: ++ +-- +Space utilization + +type: long + +-- + +[float] +=== stray + +`stray` contains information about stray OSD nodes. + + + +*`ceph.mgr_osd_disk.stray.depth`*:: ++ +-- +Depth + +type: long + +-- + +*`ceph.mgr_osd_disk.stray.reweight`*:: ++ +-- +Reweight (controls how much data the system tries to allocate to the OSD) + +type: long + +-- + +*`ceph.mgr_osd_disk.stray.pgs`*:: ++ +-- +Number of PGs + +type: long + +-- + +*`ceph.mgr_osd_disk.stray.crush_weight`*:: ++ +-- +CRUSH weight + +type: scaled_float + +-- + +*`ceph.mgr_osd_disk.stray.status`*:: ++ +-- +Node status + +type: keyword + +-- + +*`ceph.mgr_osd_disk.stray.id`*:: ++ +-- +ID + +type: long + +-- + +*`ceph.mgr_osd_disk.stray.name`*:: ++ +-- +Name + +type: keyword + +-- + +*`ceph.mgr_osd_disk.stray.type_id`*:: ++ +-- +Type ID + +type: long + +-- + +*`ceph.mgr_osd_disk.stray.kb_used_data`*:: ++ +-- +Used space for data (in KB) + +type: long + +-- + +*`ceph.mgr_osd_disk.stray.kb_used_omap`*:: ++ +-- +Used space for omap (in KB) + +type: long + +-- + +*`ceph.mgr_osd_disk.stray.kb_used_meta`*:: ++ +-- +Used space for meta (in KB) + +type: long + +-- + +*`ceph.mgr_osd_disk.stray.kb_avail`*:: ++ +-- +Available space (in KB) + +type: long + +-- + +*`ceph.mgr_osd_disk.stray.var`*:: ++ +-- +Var + +type: long + +-- + +*`ceph.mgr_osd_disk.stray.type`*:: ++ +-- +Type + +type: keyword + +-- + +*`ceph.mgr_osd_disk.stray.kb`*:: ++ +-- +Total space (in KB) + +type: long + +-- + +*`ceph.mgr_osd_disk.stray.kb_used`*:: ++ +-- +Used space (in KB) + +type: long + +-- + +*`ceph.mgr_osd_disk.stray.utilization`*:: ++ +-- +Space utilization + +type: long + +-- + +[float] +=== summary + +`summary` contains general information about OSDs. + + + +*`ceph.mgr_osd_disk.summary.total_kb_used_omap`*:: ++ +-- +Total used space for omap (in KB) + +type: long + +-- + +*`ceph.mgr_osd_disk.summary.average_utilization`*:: ++ +-- +Average space utilization + +type: long + +-- + +*`ceph.mgr_osd_disk.summary.dev`*:: ++ +-- +Deviation + +type: long + +-- + +*`ceph.mgr_osd_disk.summary.total_kb_avail`*:: ++ +-- +Total available space (in KB) + +type: long + +-- + +*`ceph.mgr_osd_disk.summary.min_var`*:: ++ +-- +Minimal variant + +type: long + +-- + +*`ceph.mgr_osd_disk.summary.max_var`*:: ++ +-- +Maximal variant + +type: long + +-- + +*`ceph.mgr_osd_disk.summary.total_kb`*:: ++ +-- +Total space (in KB) + +type: long + +-- + +*`ceph.mgr_osd_disk.summary.total_kb_used`*:: ++ +-- +Total used space (in KB) + +type: long + +-- + +*`ceph.mgr_osd_disk.summary.total_kb_used_data`*:: ++ +-- +Total used space for data (in KB) + +type: long + +-- + +*`ceph.mgr_osd_disk.summary.total_kb_used_meta`*:: ++ +-- +Total used space for meta (in KB) + +type: long + +-- + [float] === mgr_osd_perf OSD performance metrics of Ceph cluster + +*`ceph.mgr_osd_perf.pg_ready`*:: ++ +-- +Placement Group readiness status + +type: boolean + +-- + +[float] +=== osdstats + +`stray` contains OSD statistics. + + + +[float] +=== osd_perf_infos + +OSD performance statistics + + +*`ceph.mgr_osd_perf.osdstats.osd_perf_infos.id`*:: ++ +-- +OSD ID + +type: long + +-- + +[float] +=== perf_stats + +Perforance statistics + + +*`ceph.mgr_osd_perf.osdstats.osd_perf_infos.perf_stats.commit_latency_ms`*:: ++ +-- +Commit latency in ms + +type: long + +-- + +*`ceph.mgr_osd_perf.osdstats.osd_perf_infos.perf_stats.apply_latency_ms`*:: ++ +-- +Apply latency in ms + +type: long + +-- + +*`ceph.mgr_osd_perf.osdstats.osd_perf_infos.perf_stats.commit_latency_ns`*:: ++ +-- +Commit latency in ns + +type: long + +-- + +*`ceph.mgr_osd_perf.osdstats.osd_perf_infos.perf_stats.apply_latency_ns`*:: ++ +-- +Apply latency in ns + +type: long + +-- + [float] === mgr_osd_pool_stats OSD pool stats of Ceph cluster + +*`ceph.mgr_osd_pool_stats.pool_name`*:: ++ +-- +Pool name + +type: keyword + +-- + +*`ceph.mgr_osd_pool_stats.pool_id`*:: ++ +-- +Pool ID + +type: keyword + +-- + +[float] +=== client_io_rate + +Client I/O rates + + +*`ceph.mgr_osd_pool_stats.client_io_rate.read_bytes_sec`*:: ++ +-- +type: long + +-- + +*`ceph.mgr_osd_pool_stats.client_io_rate.write_bytes_sec`*:: ++ +-- +type: long + +-- + +*`ceph.mgr_osd_pool_stats.client_io_rate.read_op_per_sec`*:: ++ +-- +type: long + +-- + +*`ceph.mgr_osd_pool_stats.client_io_rate.write_op_per_sec`*:: ++ +-- +type: long + +-- + [float] === mgr_osd_tree OSD tree stats of Ceph cluster + +[float] +=== nodes + +`nodes` contains disk active OSD nodes. + + + +*`ceph.mgr_osd_tree.nodes.name`*:: ++ +-- +Node name + +type: keyword + +-- + +*`ceph.mgr_osd_tree.nodes.root`*:: ++ +-- +Node type + +type: keyword + +-- + +*`ceph.mgr_osd_tree.nodes.type_id`*:: ++ +-- +Type ID + +type: long + +-- + +*`ceph.mgr_osd_tree.nodes.id`*:: ++ +-- +Node ID + +type: long + +-- + +*`ceph.mgr_osd_tree.nodes.children`*:: ++ +-- +Node children + +type: long + +-- + +*`ceph.mgr_osd_tree.nodes.reweight`*:: ++ +-- +Reweight + +type: long + +-- + +*`ceph.mgr_osd_tree.nodes.depth`*:: ++ +-- +Depth + +type: long + +-- + +*`ceph.mgr_osd_tree.nodes.exists`*:: ++ +-- +Exists + +type: long + +-- + +*`ceph.mgr_osd_tree.nodes.status`*:: ++ +-- +Node status + +type: keyword + +-- + +*`ceph.mgr_osd_tree.nodes.primary_affinity`*:: ++ +-- +Primary affinity + +type: long + +-- + +*`ceph.mgr_osd_tree.nodes.crush_weight`*:: ++ +-- +CRUSH weight + +type: scaled_float + +-- + [float] === monitor_health diff --git a/metricbeat/module/ceph/fields.go b/metricbeat/module/ceph/fields.go index 1208bf71230f..80b084f49e07 100644 --- a/metricbeat/module/ceph/fields.go +++ b/metricbeat/module/ceph/fields.go @@ -32,5 +32,5 @@ func init() { // AssetCeph returns asset data. // This is the base64 encoded gzipped contents of ../metricbeat/module/ceph. func AssetCeph() string { - return "eJzEXE2T27gRvftXdM0lSdVYlVznkKqNx7vryo5nsuOtHFIpGiJbFCKQoAFwZP37FMAPUSQAfgiUdbJH1HsPQLO70WjyPRzw9AAxFvt3AIoqhg9w9wGL/d07gARlLGihKM8f4O/vAAD0V5DxpGT4DkDuuVBRzPMdTR9gR5jUfxXIkEh8gJToa1ApmqfyAf5zJyW7u4e7vVLF3X/fAewoskQ+GOT3kJMMWy36o06FRhG8LOq/WBTpz1f9o68Q81wRmktQe4QMlaCx/jdRcESBIGNBCkxgJ3gGHz6+/LqpAboyLqSwUioUUULlof3SJssjTX8cOJfz1Hz6YrqCyBuhjGwZbrYnhfLimkYX43na+8IjTX9+alDBoALfmQmsVfcu33GREfUAQwGNSMUVYUEFftGIYcSVEpOg2v6QmCyX1re0PRKm9iFsbYA039r4GwrCWCQVUaV9vg54OnKRzJuy5woXKlzftLUmRTOM9xgf5AYLHu8Drd0TKeANhaQ8H6MVvMyTzRthJQYib8HBYE8TEHopXi+W4FJI3zYH3HbbtNvZBIsd4Pts07Zsy9fCRLV6LrwGIchuR+ONQJJEs7yI2zONi6vmBzQpqL3gZbovSgUFCpAYc5ft1FqPgiq8uVjDukCtmVleRIU2CIxDrW93Cikv5LzJW01PNUtTBGVUFozEuDHRNbCMBhyKFPIy2zq8cKuBb/+HsQoVQQcqavhJUgRRlFuFyJgwTKId40Q57LtAEWPe/3au3KGERmWCqSAJJqssWgM+smithnUWrVUxYdFaKT9u0Vq57kUr0k1CFLm5xyxS0Lye3LVIN2YDcHNp5HKD4BJnrPzm4tR5c+ASppP+m+sq242BQ5ZJeNCkdBjpPwa6NV9Sk1lj9zq/hJiXgxtqKfvrnh8l7PkRMpKfoEglEIFA81oU340OfaAvcLJXL5Acz/e4TDa7ktld95ZzhqSvaoT8k9SgMADtMuZIxCqsGniEuswiLpNQUaJnDBpZW8LYdq9RUhZri9GmyStBrXn+8eIVRfMfIerTZ68ogRkpCkyiIr21st8/Pv308vLx0akv5I7dYPVzjDYpTEW0vGT3SOWhLRryXbUnHFppu73doppaSNFzZV+WvqIJU/DVoHWKnXqoTYimjKqT4aNS0Vhu+nHNohL6tTtrqATfqg1kT67Y9andCc48AbNqmn0RznxhnoappUErvSDHUBIEOV4lw5auw5SUfUyOHbhR4YhF8ybgs3EVeuDPr4/e4NOn1jv+gnMWXIPe6WvgKZFQXxfOcxi0vucwUpZ7DEviCiM1yYHUFy3BCtSwUBvG9BUwBJ8enfA2Bw3eqbZTnKfRcrlrJrtCDltz21mvGRnwQJG52WRBYpN9HyjjLn9y5jdXrCVhnL7e248LGHE6I0KK2PUjf5HhUmxGvlfhKshUPZHvNCuzzl57xrxJxUWgJXs1UBM47fWkZaRnD1mNZABuS/GWnJXVNf510rydHLipyUczP1OG8HqSCjP49HgPBMqcfisRaIK5ojuKQhvnaMD4VnJRZhFJl1UTmr3xt5IIbYypfWee8TwjfZe4PC5VcJ3ARPPqRqQ8B7LlpWrGrJmp4mJ2mMpoHmU8j+p1vSqaPNGcZoQ1WsCF6eC21Xpgbsh0aPBH0R0SVQpPSjkp0g0XUH++NuDeZTx7t2ZBm5/1FxQmRswChaRSuZz1+LQOxvfSIjbi/H7Q/GxQ3l9M/1zjOck7N+E6K6mBvavYGF3V+FOaND5fvoLlltE4IkkyTMi7Q5oxh0+1wAoaNDRKfzgTJD+ECeTNHekCHMmcrxqwE9My2f7Q7TIgJ/XlXDuSB79FdIVqnLfBges8nQOtdoNvPl9rzu4OCRWhzCQKhLHBGG0mP3Wg3cHqoXgvnGYS1jH/VIkd5+jO/FpifuVSAckTKLhQelZrVzJJWM7zeNo0ee5Vq6zPGhls3T2XCtbwUS7n1HDayqejI71g+miKpvZDFG/2OmVcQbNYuAhvif6BW1NClG21LgT9RqSqkeIqgimaobFA68/bnieBRF3J/UFj+CnPJ10h02mD5g3gpnEh0xmOwZ9f9Ckzy0kDLKyKvfzitv6qIOHaZi+uBzv5zp1dlnafuZQoSIr1aYUeaF31RJK42n26UjqNW6tp0RwK8ylyvKWhJUXxESZbz8w8qi+OzgDor7ezv2se3+96YXmBdT48fY0D8f/bdJPNE1CkMtqeqnP2VTYTXQKvT3r5Rf6pOlVcvJNwtlJcDmdGkH41p5yjWbWtgeKSc+ru4YMV6cLvWo4ERnkcnteOdXEA46zyLeFzobUdYq7Gq3mEj/4+qkHhEK4KtxVcx7ZTzFEQ5qliVa0nc6Outfkb5uZovxq5NrA242QY62uj5Sf3H2sIS/Z6WalchP4v89P+gU6vBKr/s7hTvmYYYnQaG0KmbBWc1z8+vz7Otphb7R1WOKr1ctkbg5bSlcU4o73rZykjzccZPS09S2kbyCH5eRMY0qoNmteod3rDKM2GcbZx06E9zpmWn8/UNqjOKV8wGhtW+2Ba8SAVyZPtKRidB/LWDsKaPi4dlWfjEDmqrtfnsTX2ZGsGyYUyWx1TCV6a0mrICjGylmdG53Ewvu5cWloiYI2aeJfzpnXxLvFoFu/Mr66m9iA35OkKy/tLZ327J+hcJlc/0/z8+lj1D4U7S+9LLFDsrpWoMfRdmse4plLOq8dkxx+PHNPbtBGtIVMJxGsFaozwAqta+NSWjtmPmF7i1/L1ZtOjDazP3Q9bh65/6r5uNmq6M5+e7b3enm3romd/643gCOt56Idt8JH/8x9j9NXLBIJRVyXBcVrzmoBgrKbmOU7KiFRRWSSWkwfnqcPYeGmGcNxjDkciwYbtibpX2NZnkuHYcE2rV+U1N4ynQd/K8BtPzy3PfQFT3hPRFZdRGQdV90RlHE6elCqoutfXL+HErf4ykGsFjt50C4yPSDW41zqFoygZT2vmv0VBR2IukyopKyVJsbtDmRjoHA2M8+fAPG7GE+wjruNuWjbns4QJvtEYo5iRwVF7CF6NcQ+UMUwJM/8DmsesTBD2SXIPUiaAKt54Ypw216mTf8Xjqa2JVM/NvnFWOqasfVXOLVVVhutRtfAVSCG0nZsXPfqKNMoXlrWH1LL/MC0wHmu3Uj2FR81jne7FcyWraz1l31vFGqPbRNz1gtfuRlqPZ7Yk2teN+s+buT4uYFvGB1Q3doI9Xqc7tDSDhWX2OEQuk3vYc6nuQXCuPH7xVOCw6hVwWTSU4xAp3lOWCLQ/c75okmriBhgY1TMgsSCC1IXCmGcZsc9FLEq5j45I0739nrbdzFOjlwEHC/g5ehaOnef8dTCMQ8C2Iv2dSsfDsosefaeyYpSKMlahazPIufrz396fUN7DX9/n/C92Xy5oRsQpIrsdzanql9EXT7zeFVWzXZ3KkITm9ds/zLsJa16ncxfoMYQFbyLbYwtpTukdtKFfPnZ+/5txCVTpeFeyBLYIZaFXKeFH+4sZ1knn9ERUyGCQW2mMHqYkcjui9oOOxiv1FERgXvsrs6Gugn7PlQ0CrKlIXlvjtYH8wMD6KWlKClrYDSJqt4rhpKy2kTd9M+dAy7RNr9YZ9i1Qg86eibN1g7dwXjFHKxT/DtsLXf8PAAD//1D8Md0=" + return "eJzsXU2T27gRvftXdPmS3arxJLnOIVVej9d27Y5nsrKTQypFQ2SLQgYkaACcsfLrUwA/RJEA+AXJVSnrtOuR3ntoNBuNRpN8BY94uIEYi/0LAEUVwxt4+QaL/csXAAnKWNBCUZ7fwN9eAADoP0HGk5LhCwC550JFMc93NL2BHWFS/6tAhkTiDaREfweVonkqb+BfL6VkL6/g5V6p4uW/XwDsKLJE3hjkV5CTDFst+qMOhUYRvCzqf7Eo0p8v+kdfIOa5IjSXoPYIGSpBY/3fRMEzCgQZC1JgAjvBM3jz9uH9dQ3QlXEihZVSoYgSKh/bP9pkeaTpjwPn1E7Npy+mK4g8EcrIluH19qBQnnyn0cV4nvb+4JGmP68bVDCowHfGgLXq3td3XGRE3cBQQCNScUVYUIGfNGIYcaXEJKi2zxKT5dL6nrZHwtQ+hK8NkOZ7G39CQRiLpCKqtNvrEQ/PXCTzTHZf4UKF6zNb61I0w3iP8aO8xoLH+0Bzd0cKeEIhKc/HaAUv8+T6ibASA5G34GCwpwkIPRWbkyk4FdL3zQG33TftfjbBYwf4Pt+0TdvyuTCrWm0Lr0MIstvR+FogSaJZUcQdmcbFVfYBTQpqL3iZ7otSQYECJMbc5Tu11mdBFV5crGFdoNZYlhdRoR0C41Dz2zUh5YWcZ7yz6amsNEVQRmXBSIzXZnUNLKMBhyKFvMy2jijcauDb/2CsQq2gAxU1/CQpgijKrUJkTBgm0Y5xohz+XaCIMe//da7coYRGZYKpIAkmZ5m0Bnxk0loN55m0VsWESWulfL9Ja+W6J61IrxOiyMUjZpGC5vXkrkV6bTYAF5dGTjcILnHGyy8uTh03By5hOum/uK6y3Rg4ZJmEB01Kh5H+x0CX5kNqMmvsfs8vIebl4IJayr7Z82cJe/4MGckPUKQSiECgeS2K70aHPtAXONmrJ0iO53tcJte7ktlD95ZzhqSvaoT8g9SgMADtMuZIxFlYNfAIdZlFXCahVomeM2hk7Qlj271GSVmcW4x2TV4Jat3z84NXFM2/h6gPH72iBGakKDCJivTSyv54e/f64eHtrVNfyB27wernGG1SmIpoecnulsrHtmjId9WecOil7fZ2i2pqIUXbyj4tfUUTTPDFoHWKnXqozRJNGVUHw0elorG87q9rFpXQr91Zl0rwzdpA9uSKXZ/aneDMEzCrptkX4cwX5mmYWhq00gvyHEqCIM+rZNjSdZiSso/JsQM3Khxr0TwDfDShQg/8fnPrXXz61HrHX3DOgmvQO30NPGUl1N8LFzkMWj9yGCnLI4YlcYWRmuRA6oOWYAVqWKgNY/oMGIIPt054W4AGr6ntFEczWr7usmRXyOPWXHbW74wMeKDIXGyyILHJvh8p4654cuQ33ziXhHH6em8/LmAk6IwIKWLXj/xFhlOxGflWLVdBTHVHvtGszDp77Rl2k4qLQFO2MVATOO31pGWkxwhZjWQAbkvxlpyV1TX+86R5OzkIU5OPZn6lDGFzkAoz+HB7BQTKnH4tEWiCuaI7ikI75+iC8bXkoswiki6rJjR7468lEdoZU/vOPON5Rvohcfm6VMF1FiaaVxci5TmQLS9VM2bNTBUXs5epjOZRxvOontdVq8kdzWlGWKMFXJgOblutB+YumQ4N/lV0h0SVwpNSTlrphhOoP18acO80HqNbM6HNz/oTChNXzAKFpFK5gvW4WQfje2gRG3H+OGh+NijvL6a/r/Gc5J2L8DwzqYG9s9g4XdX4U5o0Pl8+g+WW0TgiSTJMyLtDmmHDu1pgBQ0aGqV/ORMkfwyzkDdXpAtwJHNeNWAnpsXY/qXb5UBO6lNbO5IHv0d0hWqcp8GB6zydA612h28+X2rO7g4JFaHMJAqEscEYbS4/daDdweqheL84zSWsY35diR3n6Fr+XGLec6mA5AkUXCht1TqUTBKW8zyeZibPtWqV9VEjg62751TBOWKUKzg1nLby6ehIT5jemqKp/RDFm71OGVfQLBZOlrdE/8CtKSHKNlsngn4nUtVIcbWCKZqh8UDrz9ueJ4FEreR+ozH8lMeTrpDptEHzLuCmcSHTGY7Bn1/0KTPLSQMsrIo9vHN7f1WQcG2zF9eDnXzHzi5Lu89cShQkxfq0Qg+0rnoiSVztPl0pncats2nRHArzKXK8paElRfERJlvPzDyqT47OAOjPt7O/ax7fH3pieYF1Pjx9jgPx/9N0k80TUKQy2h6qc/azbCa6BN6Y9PBO/qk6VVy8k3C2UpwOZ8YivTGnnKNZta2B4pRz6u7hjRXpJO5ajgRGeRyR1451cgDjrPIt4XOhtR1irsareYS3/j6qQeEQVi23FVzHt1PMURDmqWJVrSdzV11r8zfMzdHeG7k2sDbjZBjr70bLT+7f1hCW7PW0UrkI/e/mp/0DnV4JVP/P4k75mmGI0WlsCJmyVXDe+Hi/uZ3tMZfaO5zhqNbLZW8MWkpXFuOM9q6fpYw0H2f0tPQspW0gh+THTWBIrzZoXqfe6Q2jNBvG2c5Nh/44xyy/HqltUJ1TvmA0Nqz2xrTiRiqSJ9tDMDoP5KUDhDV9XDoqz8YhclRd1+exNfZkbwbJhTJbHVMJXprSasgKMbKWZ0btOBhf15aWlgg4R028y3nRuniXeDSLd+ZXq6k9yA15eobpfdeZ3+4JOpfJ6nua7ze3Vf/Qec7Sc544srclC5FB8x9Mxoo+YUU7dyFKsBhsJ2DehsWKcKxRPCNN97YN5pziRAUCP2krCM7q3t4y3lf3nZim3spvlKAoQXEgjPFY74AVN3+/39z+7CslBMpTfAXBWJRyH40YZE5P4ps/Pm/egwMw7NbrI0/QBRWo083T5Bai7eCjr71AI9lXqRk1u0OBvkHUDXKRdtlVRKYOaRz/J5rDb7+4/bqh5MPUeAGlRplMmWGIUWqUKZQhq+xV89oY6ROxHadN5/uH5fcjZ6qz3P2TDeNosXWebqrT0+zkbgud6QrT6D4ryuh/yeDGsbmUG8NWOtCOsVWQ/g5lzd0Rghy8S735hslffqz2P1Z7G+uP1f7/crWvgt+Oi++x7h/Jv0MGcCT/kQsMwH/kAj461+o9l3JiLlBmGREBs4EKb9Lh3JIDlupOuWCRovKDckG8IFWDRxRqupqGEemdNjjJi55WZkVP1M/QGnt9cKoM3b/XZszIGc2jtWGqadx/IoISz1F/Rr6t5yLfJnE1hr1YDDu5bMJeMbOo1ycW1it2UnpxKmT1Om8VYl3t+5XgAsXuRZ9yZiVYY+hwmsd4poJwkUYCSWJfGiY8FOOhbXl8p4dmGr9ojlL6OiG4TELfu3+6O9W2W37vbTN7kV7MVt4L0p/F1beyBjrP0LpGjqmMCVy38I6bYUD5YMwwbgWYcTtDzLOMqogRhXl8iDIXHkztYe91jWl0qNGB5uAhaFOGomCHcyl6rcFnCurZyHp303JFQxt5COw2CqxoYKPcfsOrucw5ZwMfXxaqmxvVg0doLdFSdpjcdGV/DMAJ/PIbbG2PAGgdj1HMVUR5JIZtsBOD/RuDAR/+fA8apO8pY6F8Xd95Pf4b+OWSHeY91hm95Cu7rlvi79Bx3XJP6bbuX8VKIK69fjXGWa7f8x65V08pqk7ZF9fegxQ1eTJym7Tg3F1snsNiLTtdsn66Et4MwgMf7ylLBK6rNBgSJ9JZTkU8NYzznu3gNypXNtO/tUNc/uiiEDQj4hCR3Y7mVK1rW3yowMAJdtkDoeM9iOYezamPGpn96PNT/Dqq92oS094HMXykzbRWet/bIOqH4DRPDbu7tz+D0HM7xaJn0tc3KIywHoc+qFutH/lvv4zRVy+5CEZdVW/Gac3rK4KxmmOAcVJGpIrKIrHcEatcd8OOjZdmCM97zOGZSLBhe5b8Fb71kWQ4NlzzCKJqr3XNeBr0bSG/8/T4KL6+gCnvL+mKy6iMg6q7ozIOJ09KFVTdZvMpnLizv6RmrcDRi26B8xGpBtdat5aYjNeB57/dQ29QuEyqDUApSYrds7eJC51j3z/fBuYxyDq7oZcINy2bs6qR4BONMYoZGTwCIgSvxrgCyhimhJn/A5rHrEwQ9klyBVImgCq+9qxx2l2nGn/FY9NbF6me5/7EWekwWfsKp0uqqhzXo2rhq7lCaDseY3r0FWmUL7zdckgt+w95h6pzLKmeDk3N48bdk+dKVs/19ofeLNYY3YfbdaPg2iJNG/FMpUbHutH4ebHQxwVsy/gR1YWDYI/XGQ4tFZOwzJ6AyGVyBXsu1ZWpAHniorVyE3BaNJSrVG0vlKwwUk3cAAOj2gISCyJIfQNbzLOM2G3h2Y9XmmwX89TVy4DbmjV99Zql82AYh4De0s3k0+chHZUVo1SUsQpdu0HO1U9/fXVAeQV/eZXzn+2x3F96WWx4vSuq+4bN3cIkoXn9Vhrzzsya1xncHUW6pTNiXkmHR0Eu2tAvxTu+l9CEBKr0eleyBLYIZaFnKeHP9v6186Rz2hAVMhjkVhqjj1MSuR1R+8GTtlbqKYjAvI5XZkNdLfq9UDZYYM0p3tp7D20g33Fh/ZA0JQUt7AIrareK4aSstpEXfWPsQMu0Ta/WGfbtZIMnzky01gXeDrvCRmco/j1uT3T9LwAA//8j4LYZ" } diff --git a/metricbeat/module/ceph/mgr_osd_disk/_meta/fields.yml b/metricbeat/module/ceph/mgr_osd_disk/_meta/fields.yml index 52c0ee98519b..8e8d65a7799c 100644 --- a/metricbeat/module/ceph/mgr_osd_disk/_meta/fields.yml +++ b/metricbeat/module/ceph/mgr_osd_disk/_meta/fields.yml @@ -4,3 +4,150 @@ OSD disk metrics of Ceph cluster release: beta fields: + - name: nodes + type: group + description: > + `nodes` contains information about active nodes. + fields: + - name: depth + type: long + description: Depth + - name: reweight + type: long + description: Reweight (controls how much data the system tries to allocate to the OSD) + - name: pgs + type: long + description: Number of PGs + - name: crush_weight + type: scaled_float + description: CRUSH weight + - name: status + type: keyword + description: Node status + - name: id + type: long + description: ID + - name: name + type: keyword + description: Name + - name: type_id + type: long + description: Type ID + - name: kb_used_data + type: long + description: Used data (in KB) + - name: kb_used_omap + type: long + description: Used omap (in KB) + - name: kb_used_meta + type: long + description: Used meta (in KB) + - name: kb_avail + type: long + description: Available space (in KB) + - name: var + type: long + description: Var + - name: type + type: keyword + description: Type + - name: kb + type: long + description: Total space (in KB) + - name: kb_used + type: long + description: Used space (in KB) + - name: Utilization + type: long + description: Space utilization + - name: stray + type: group + description: > + `stray` contains information about stray OSD nodes. + fields: + - name: depth + type: long + description: Depth + - name: reweight + type: long + description: Reweight (controls how much data the system tries to allocate to the OSD) + - name: pgs + type: long + description: Number of PGs + - name: crush_weight + type: scaled_float + description: CRUSH weight + - name: status + type: keyword + description: Node status + - name: id + type: long + description: ID + - name: name + type: keyword + description: Name + - name: type_id + type: long + description: Type ID + - name: kb_used_data + type: long + description: Used space for data (in KB) + - name: kb_used_omap + type: long + description: Used space for omap (in KB) + - name: kb_used_meta + type: long + description: Used space for meta (in KB) + - name: kb_avail + type: long + description: Available space (in KB) + - name: var + type: long + description: Var + - name: type + type: keyword + description: Type + - name: kb + type: long + description: Total space (in KB) + - name: kb_used + type: long + description: Used space (in KB) + - name: utilization + type: long + description: Space utilization + - name: summary + type: group + description: > + `summary` contains general information about OSDs. + fields: + - name: total_kb_used_omap + type: long + description: Total used space for omap (in KB) + - name: average_utilization + type: long + description: Average space utilization + - name: dev + type: long + description: Deviation + - name: total_kb_avail + type: long + description: Total available space (in KB) + - name: min_var + type: long + description: Minimal variant + - name: max_var + type: long + description: Maximal variant + - name: total_kb + type: long + description: Total space (in KB) + - name: total_kb_used + type: long + description: Total used space (in KB) + - name: total_kb_used_data + type: long + description: Total used space for data (in KB) + - name: total_kb_used_meta + type: long + description: Total used space for meta (in KB) diff --git a/metricbeat/module/ceph/mgr_osd_perf/_meta/fields.yml b/metricbeat/module/ceph/mgr_osd_perf/_meta/fields.yml index c6ceca85837d..8ed76715e564 100644 --- a/metricbeat/module/ceph/mgr_osd_perf/_meta/fields.yml +++ b/metricbeat/module/ceph/mgr_osd_perf/_meta/fields.yml @@ -4,3 +4,34 @@ OSD performance metrics of Ceph cluster release: beta fields: + - name: pg_ready + type: boolean + description: Placement Group readiness status + - name: osdstats + type: group + description: > + `stray` contains OSD statistics. + fields: + - name: osd_perf_infos + type: group + description: OSD performance statistics + fields: + - name: id + type: long + description: OSD ID + - name: perf_stats + type: group + description: Perforance statistics + fields: + - name: commit_latency_ms + type: long + description: Commit latency in ms + - name: apply_latency_ms + type: long + description: Apply latency in ms + - name: commit_latency_ns + type: long + description: Commit latency in ns + - name: apply_latency_ns + type: long + description: Apply latency in ns diff --git a/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/fields.yml b/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/fields.yml index 5d0ea5bf1503..19bd35a0314b 100644 --- a/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/fields.yml +++ b/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/fields.yml @@ -4,3 +4,25 @@ OSD pool stats of Ceph cluster release: beta fields: + - name: pool_name + type: keyword + description: Pool name + - name: pool_id + type: keyword + description: Pool ID + - name: client_io_rate + type: group + description: Client I/O rates + fields: + - name: read_bytes_sec + type: long + keyword: Bytes read per second + - name: write_bytes_sec + type: long + keyword: Bytes written per second + - name: read_op_per_sec + type: long + keyword: Read operations per second + - name: write_op_per_sec + type: long + keyword: Write operations per second diff --git a/metricbeat/module/ceph/mgr_osd_tree/_meta/fields.yml b/metricbeat/module/ceph/mgr_osd_tree/_meta/fields.yml index 76a073160a1e..ae6f1e130aba 100644 --- a/metricbeat/module/ceph/mgr_osd_tree/_meta/fields.yml +++ b/metricbeat/module/ceph/mgr_osd_tree/_meta/fields.yml @@ -4,3 +4,41 @@ OSD tree stats of Ceph cluster release: beta fields: + - name: nodes + type: group + description: > + `nodes` contains disk active OSD nodes. + fields: + - name: name + type: keyword + description: Node name + - name: root + type: keyword + description: Node type + - name: type_id + type: long + description: Type ID + - name: id + type: long + description: Node ID + - name: children + type: long + description: Node children + - name: reweight + type: long + description: Reweight + - name: depth + type: long + description: Depth + - name: exists + type: long + description: Exists + - name: status + type: keyword + description: Node status + - name: primary_affinity + type: long + description: Primary affinity + - name: crush_weight + type: scaled_float + description: CRUSH weight From 252fec24de4e4018bcef74663c05de4f310fdc38 Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Wed, 12 Feb 2020 20:02:07 +0100 Subject: [PATCH 28/41] Update fields.yml --- metricbeat/docs/fields.asciidoc | 781 ++---------------- metricbeat/module/ceph/fields.go | 2 +- .../ceph/mgr_cluster_disk/_meta/fields.yml | 6 +- .../ceph/mgr_cluster_health/_meta/fields.yml | 6 +- .../module/ceph/mgr_osd_disk/_meta/fields.yml | 4 +- .../module/ceph/mgr_osd_perf/_meta/fields.yml | 2 +- .../ceph/mgr_osd_pool_stats/_meta/fields.yml | 2 +- .../module/ceph/mgr_osd_tree/_meta/fields.yml | 40 +- metricbeat/module/ceph/test_ceph_mgr.py | 2 + 9 files changed, 140 insertions(+), 705 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 3f0edb8f8d11..3209bd3198c4 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -3938,160 +3938,23 @@ type: long Disk metrics of Ceph cluster -[float] -=== stats - -`stats` contains disk availability statistics. - - - -*`ceph.mgr_cluster_disk.stats.total_bytes`*:: -+ --- -Total bytes of the cluster - -type: long - --- - -*`ceph.mgr_cluster_disk.stats.total_avail_bytes`*:: -+ --- -Available bytes of the cluster - -type: long - --- - -*`ceph.mgr_cluster_disk.stats.total_used_bytes`*:: -+ --- -Used bytes of the cluster - -type: long - --- - -*`ceph.mgr_cluster_disk.stats.total_used_raw_bytes`*:: -+ --- -Used raw bytes of the cluster - -type: long - --- - -*`ceph.mgr_cluster_disk.stats.total_used_raw_ratio`*:: +*`ceph.mgr_cluster_disk.stats`*:: + -- -Used raw bytes ratio - -type: scaled_float - --- +`stats` contains disk availability statistics. -*`ceph.mgr_cluster_disk.stats.num_osds`*:: -+ --- -Number of OSDs in the cluster -type: long +type: nested -- -*`ceph.mgr_cluster_disk.stats.num_per_pool_osds`*:: +*`ceph.mgr_cluster_disk.pools`*:: + -- -Number of OSDs per pool in the cluster - -type: long - --- - -[float] -=== pools - `pools` contains disk pool statistics. - -*`ceph.mgr_cluster_disk.pools.name`*:: -+ --- -Pool name - -type: keyword - --- - -*`ceph.mgr_cluster_disk.pools.id`*:: -+ --- -Pool ID - -type: long - --- - -[float] -=== stats - -Pool statistics - - -*`ceph.mgr_cluster_disk.pools.stats.kb_used`*:: -+ --- -Used space in kilobytes - -type: long - --- - -*`ceph.mgr_cluster_disk.pools.stats.bytes_used`*:: -+ --- -Used space in bytes - -type: long - --- - -*`ceph.mgr_cluster_disk.pools.stats.percent_used`*:: -+ --- -Used space in pct - -type: scaled_float - -format: percent - --- - -*`ceph.mgr_cluster_disk.pools.stats.max_avail`*:: -+ --- -Maximum available space in bytes - -type: long - --- - -*`ceph.mgr_cluster_disk.pools.stats.stored`*:: -+ --- -Stored bytes - -type: long - --- - -*`ceph.mgr_cluster_disk.pools.stats.objects`*:: -+ --- -Number of stored objects - -type: long +type: nested -- @@ -4209,37 +4072,13 @@ type: keyword Monitor public addresses -[float] -=== addrvec - -`addrvec` contains details of all public addresses. - - - -*`ceph.mgr_cluster_health.monmap.mons.public_addrs.addrvec.type`*:: +*`ceph.mgr_cluster_health.monmap.mons.public_addrs.addrvec`*:: + -- -Address type - -type: keyword - --- - -*`ceph.mgr_cluster_health.monmap.mons.public_addrs.addrvec.addr`*:: -+ --- -Host and port of monitor - -type: keyword - --- +`addrvec` contains details of all public addresses. -*`ceph.mgr_cluster_health.monmap.mons.public_addrs.addrvec.nonce`*:: -+ --- -Nonce value -type: long +type: nested -- @@ -4367,28 +4206,13 @@ type: long -- -[float] -=== pgs_by_state - -`pgs_by_state` contains information about PGs' states. - - - -*`ceph.mgr_cluster_health.pgmap.pgs_by_state.state_name`*:: +*`ceph.mgr_cluster_health.pgmap.pgs_by_state`*:: + -- -State name - -type: keyword - --- +`pgs_by_state` contains information about PGs' states. -*`ceph.mgr_cluster_health.pgmap.pgs_by_state.count`*:: -+ --- -Count -type: long +type: nested -- @@ -4566,636 +4390,207 @@ type: long -- -[float] -=== by_rank - -`by_rank` contains information about file systems sorted by rank. - - - -*`ceph.mgr_cluster_health.fsmap.by_rank.filesystem_id`*:: +*`ceph.mgr_cluster_health.fsmap.by_rank`*:: + -- -File system ID +`by_rank` contains information about file systems sorted by rank. -type: long --- +type: nested -*`ceph.mgr_cluster_health.fsmap.by_rank.rank`*:: -+ -- -File system rank - -type: long --- +[float] +=== mgr_osd_disk -*`ceph.mgr_cluster_health.fsmap.by_rank.name`*:: -+ --- -File system name +OSD disk metrics of Ceph cluster -type: keyword --- -*`ceph.mgr_cluster_health.fsmap.by_rank.status`*:: +*`ceph.mgr_osd_disk.nodes`*:: + -- -File system status +`nodes` contains information about active nodes. -type: keyword --- +type: nested -*`ceph.mgr_cluster_health.fsmap.by_rank.gid`*:: -+ -- -File system GID - -type: long +*`ceph.mgr_osd_disk.stray`*:: ++ -- +`stray` contains information about stray OSD nodes. -[float] -=== mgr_osd_disk - -OSD disk metrics of Ceph cluster +type: nested +-- [float] -=== nodes +=== summary -`nodes` contains information about active nodes. +`summary` contains general information about OSDs. -*`ceph.mgr_osd_disk.nodes.depth`*:: +*`ceph.mgr_osd_disk.summary.total_kb_used_omap`*:: + -- -Depth +Total used space for omap (in KB) type: long -- -*`ceph.mgr_osd_disk.nodes.reweight`*:: +*`ceph.mgr_osd_disk.summary.average_utilization`*:: + -- -Reweight (controls how much data the system tries to allocate to the OSD) +Average space utilization type: long -- -*`ceph.mgr_osd_disk.nodes.pgs`*:: +*`ceph.mgr_osd_disk.summary.dev`*:: + -- -Number of PGs +Deviation type: long -- -*`ceph.mgr_osd_disk.nodes.crush_weight`*:: +*`ceph.mgr_osd_disk.summary.total_kb_avail`*:: + -- -CRUSH weight +Total available space (in KB) -type: scaled_float +type: long -- -*`ceph.mgr_osd_disk.nodes.status`*:: +*`ceph.mgr_osd_disk.summary.min_var`*:: + -- -Node status +Minimal variant -type: keyword +type: long -- -*`ceph.mgr_osd_disk.nodes.id`*:: +*`ceph.mgr_osd_disk.summary.max_var`*:: + -- -ID +Maximal variant type: long -- -*`ceph.mgr_osd_disk.nodes.name`*:: +*`ceph.mgr_osd_disk.summary.total_kb`*:: + -- -Name +Total space (in KB) -type: keyword +type: long -- -*`ceph.mgr_osd_disk.nodes.type_id`*:: +*`ceph.mgr_osd_disk.summary.total_kb_used`*:: + -- -Type ID +Total used space (in KB) type: long -- -*`ceph.mgr_osd_disk.nodes.kb_used_data`*:: +*`ceph.mgr_osd_disk.summary.total_kb_used_data`*:: + -- -Used data (in KB) +Total used space for data (in KB) type: long -- -*`ceph.mgr_osd_disk.nodes.kb_used_omap`*:: +*`ceph.mgr_osd_disk.summary.total_kb_used_meta`*:: + -- -Used omap (in KB) +Total used space for meta (in KB) type: long -- -*`ceph.mgr_osd_disk.nodes.kb_used_meta`*:: -+ --- -Used meta (in KB) +[float] +=== mgr_osd_perf -type: long +OSD performance metrics of Ceph cluster --- -*`ceph.mgr_osd_disk.nodes.kb_avail`*:: + +*`ceph.mgr_osd_perf.pg_ready`*:: + -- -Available space (in KB) +Placement Group readiness status -type: long +type: boolean -- -*`ceph.mgr_osd_disk.nodes.var`*:: -+ --- -Var +[float] +=== osdstats -type: long +`stray` contains OSD statistics. --- -*`ceph.mgr_osd_disk.nodes.type`*:: + +*`ceph.mgr_osd_perf.osdstats.osd_perf_infos`*:: + -- -Type +OSD performance statistics -type: keyword +type: nested -- -*`ceph.mgr_osd_disk.nodes.kb`*:: +*`ceph.mgr_osd_pool_stats`*:: + -- -Total space (in KB) +OSD pool stats of Ceph cluster -type: long --- +type: nested -*`ceph.mgr_osd_disk.nodes.kb_used`*:: -+ -- -Used space (in KB) -type: long +[float] +=== mgr_osd_tree + +OSD tree stats of Ceph cluster --- -*`ceph.mgr_osd_disk.nodes.Utilization`*:: + +*`ceph.mgr_osd_tree.nodes`*:: + -- -Space utilization +`nodes` contains disk active OSD nodes. -type: long + +type: nested -- [float] === stray -`stray` contains information about stray OSD nodes. - - - -*`ceph.mgr_osd_disk.stray.depth`*:: -+ --- -Depth - -type: long - --- - -*`ceph.mgr_osd_disk.stray.reweight`*:: -+ --- -Reweight (controls how much data the system tries to allocate to the OSD) - -type: long - --- - -*`ceph.mgr_osd_disk.stray.pgs`*:: -+ --- -Number of PGs - -type: long - --- - -*`ceph.mgr_osd_disk.stray.crush_weight`*:: -+ --- -CRUSH weight - -type: scaled_float - --- - -*`ceph.mgr_osd_disk.stray.status`*:: -+ --- -Node status - -type: keyword - --- - -*`ceph.mgr_osd_disk.stray.id`*:: -+ --- -ID - -type: long - --- - -*`ceph.mgr_osd_disk.stray.name`*:: -+ --- -Name - -type: keyword - --- - -*`ceph.mgr_osd_disk.stray.type_id`*:: -+ --- -Type ID - -type: long - --- - -*`ceph.mgr_osd_disk.stray.kb_used_data`*:: -+ --- -Used space for data (in KB) - -type: long - --- - -*`ceph.mgr_osd_disk.stray.kb_used_omap`*:: -+ --- -Used space for omap (in KB) - -type: long - --- - -*`ceph.mgr_osd_disk.stray.kb_used_meta`*:: -+ --- -Used space for meta (in KB) - -type: long - --- - -*`ceph.mgr_osd_disk.stray.kb_avail`*:: -+ --- -Available space (in KB) - -type: long - --- - -*`ceph.mgr_osd_disk.stray.var`*:: -+ --- -Var - -type: long - --- - -*`ceph.mgr_osd_disk.stray.type`*:: -+ --- -Type - -type: keyword - --- - -*`ceph.mgr_osd_disk.stray.kb`*:: -+ --- -Total space (in KB) - -type: long - --- - -*`ceph.mgr_osd_disk.stray.kb_used`*:: -+ --- -Used space (in KB) - -type: long - --- - -*`ceph.mgr_osd_disk.stray.utilization`*:: -+ --- -Space utilization - -type: long - --- - -[float] -=== summary - -`summary` contains general information about OSDs. - - - -*`ceph.mgr_osd_disk.summary.total_kb_used_omap`*:: -+ --- -Total used space for omap (in KB) - -type: long - --- - -*`ceph.mgr_osd_disk.summary.average_utilization`*:: -+ --- -Average space utilization - -type: long - --- - -*`ceph.mgr_osd_disk.summary.dev`*:: -+ --- -Deviation - -type: long - --- - -*`ceph.mgr_osd_disk.summary.total_kb_avail`*:: -+ --- -Total available space (in KB) - -type: long - --- - -*`ceph.mgr_osd_disk.summary.min_var`*:: -+ --- -Minimal variant - -type: long - --- - -*`ceph.mgr_osd_disk.summary.max_var`*:: -+ --- -Maximal variant - -type: long - --- - -*`ceph.mgr_osd_disk.summary.total_kb`*:: -+ --- -Total space (in KB) - -type: long - --- - -*`ceph.mgr_osd_disk.summary.total_kb_used`*:: -+ --- -Total used space (in KB) - -type: long - --- - -*`ceph.mgr_osd_disk.summary.total_kb_used_data`*:: -+ --- -Total used space for data (in KB) - -type: long - --- - -*`ceph.mgr_osd_disk.summary.total_kb_used_meta`*:: -+ --- -Total used space for meta (in KB) - -type: long - --- - -[float] -=== mgr_osd_perf - -OSD performance metrics of Ceph cluster - - - -*`ceph.mgr_osd_perf.pg_ready`*:: -+ --- -Placement Group readiness status - -type: boolean - --- - -[float] -=== osdstats - -`stray` contains OSD statistics. - - - -[float] -=== osd_perf_infos - -OSD performance statistics - - -*`ceph.mgr_osd_perf.osdstats.osd_perf_infos.id`*:: -+ --- -OSD ID - -type: long - --- - -[float] -=== perf_stats - -Perforance statistics - - -*`ceph.mgr_osd_perf.osdstats.osd_perf_infos.perf_stats.commit_latency_ms`*:: -+ --- -Commit latency in ms - -type: long - --- - -*`ceph.mgr_osd_perf.osdstats.osd_perf_infos.perf_stats.apply_latency_ms`*:: -+ --- -Apply latency in ms - -type: long - --- - -*`ceph.mgr_osd_perf.osdstats.osd_perf_infos.perf_stats.commit_latency_ns`*:: -+ --- -Commit latency in ns - -type: long - --- - -*`ceph.mgr_osd_perf.osdstats.osd_perf_infos.perf_stats.apply_latency_ns`*:: -+ --- -Apply latency in ns - -type: long - --- - -[float] -=== mgr_osd_pool_stats - -OSD pool stats of Ceph cluster - - - -*`ceph.mgr_osd_pool_stats.pool_name`*:: -+ --- -Pool name - -type: keyword - --- - -*`ceph.mgr_osd_pool_stats.pool_id`*:: -+ --- -Pool ID - -type: keyword - --- - -[float] -=== client_io_rate - -Client I/O rates - - -*`ceph.mgr_osd_pool_stats.client_io_rate.read_bytes_sec`*:: -+ --- -type: long - --- - -*`ceph.mgr_osd_pool_stats.client_io_rate.write_bytes_sec`*:: -+ --- -type: long - --- - -*`ceph.mgr_osd_pool_stats.client_io_rate.read_op_per_sec`*:: -+ --- -type: long - --- - -*`ceph.mgr_osd_pool_stats.client_io_rate.write_op_per_sec`*:: -+ --- -type: long - --- - -[float] -=== mgr_osd_tree - -OSD tree stats of Ceph cluster - - - -[float] -=== nodes - -`nodes` contains disk active OSD nodes. +`stray` contains stray OSD nodes. -*`ceph.mgr_osd_tree.nodes.name`*:: +*`ceph.mgr_osd_tree.stray.name`*:: + -- Node name @@ -5204,7 +4599,7 @@ type: keyword -- -*`ceph.mgr_osd_tree.nodes.root`*:: +*`ceph.mgr_osd_tree.stray.root`*:: + -- Node type @@ -5213,7 +4608,7 @@ type: keyword -- -*`ceph.mgr_osd_tree.nodes.type_id`*:: +*`ceph.mgr_osd_tree.stray.type_id`*:: + -- Type ID @@ -5222,7 +4617,7 @@ type: long -- -*`ceph.mgr_osd_tree.nodes.id`*:: +*`ceph.mgr_osd_tree.stray.id`*:: + -- Node ID @@ -5231,7 +4626,7 @@ type: long -- -*`ceph.mgr_osd_tree.nodes.children`*:: +*`ceph.mgr_osd_tree.stray.children`*:: + -- Node children @@ -5240,7 +4635,7 @@ type: long -- -*`ceph.mgr_osd_tree.nodes.reweight`*:: +*`ceph.mgr_osd_tree.stray.reweight`*:: + -- Reweight @@ -5249,7 +4644,7 @@ type: long -- -*`ceph.mgr_osd_tree.nodes.depth`*:: +*`ceph.mgr_osd_tree.stray.depth`*:: + -- Depth @@ -5258,7 +4653,7 @@ type: long -- -*`ceph.mgr_osd_tree.nodes.exists`*:: +*`ceph.mgr_osd_tree.stray.exists`*:: + -- Exists @@ -5267,7 +4662,7 @@ type: long -- -*`ceph.mgr_osd_tree.nodes.status`*:: +*`ceph.mgr_osd_tree.stray.status`*:: + -- Node status @@ -5276,7 +4671,7 @@ type: keyword -- -*`ceph.mgr_osd_tree.nodes.primary_affinity`*:: +*`ceph.mgr_osd_tree.stray.primary_affinity`*:: + -- Primary affinity @@ -5285,7 +4680,7 @@ type: long -- -*`ceph.mgr_osd_tree.nodes.crush_weight`*:: +*`ceph.mgr_osd_tree.stray.crush_weight`*:: + -- CRUSH weight diff --git a/metricbeat/module/ceph/fields.go b/metricbeat/module/ceph/fields.go index 80b084f49e07..00af6a24fbef 100644 --- a/metricbeat/module/ceph/fields.go +++ b/metricbeat/module/ceph/fields.go @@ -32,5 +32,5 @@ func init() { // AssetCeph returns asset data. // This is the base64 encoded gzipped contents of ../metricbeat/module/ceph. func AssetCeph() string { - return "eJzsXU2T27gRvftXdPmS3arxJLnOIVVej9d27Y5nsrKTQypFQ2SLQgYkaACcsfLrUwA/RJEA+AXJVSnrtOuR3ntoNBuNRpN8BY94uIEYi/0LAEUVwxt4+QaL/csXAAnKWNBCUZ7fwN9eAADoP0HGk5LhCwC550JFMc93NL2BHWFS/6tAhkTiDaREfweVonkqb+BfL6VkL6/g5V6p4uW/XwDsKLJE3hjkV5CTDFst+qMOhUYRvCzqf7Eo0p8v+kdfIOa5IjSXoPYIGSpBY/3fRMEzCgQZC1JgAjvBM3jz9uH9dQ3QlXEihZVSoYgSKh/bP9pkeaTpjwPn1E7Npy+mK4g8EcrIluH19qBQnnyn0cV4nvb+4JGmP68bVDCowHfGgLXq3td3XGRE3cBQQCNScUVYUIGfNGIYcaXEJKi2zxKT5dL6nrZHwtQ+hK8NkOZ7G39CQRiLpCKqtNvrEQ/PXCTzTHZf4UKF6zNb61I0w3iP8aO8xoLH+0Bzd0cKeEIhKc/HaAUv8+T6ibASA5G34GCwpwkIPRWbkyk4FdL3zQG33TftfjbBYwf4Pt+0TdvyuTCrWm0Lr0MIstvR+FogSaJZUcQdmcbFVfYBTQpqL3iZ7otSQYECJMbc5Tu11mdBFV5crGFdoNZYlhdRoR0C41Dz2zUh5YWcZ7yz6amsNEVQRmXBSIzXZnUNLKMBhyKFvMy2jijcauDb/2CsQq2gAxU1/CQpgijKrUJkTBgm0Y5xohz+XaCIMe//da7coYRGZYKpIAkmZ5m0Bnxk0loN55m0VsWESWulfL9Ja+W6J61IrxOiyMUjZpGC5vXkrkV6bTYAF5dGTjcILnHGyy8uTh03By5hOum/uK6y3Rg4ZJmEB01Kh5H+x0CX5kNqMmvsfs8vIebl4IJayr7Z82cJe/4MGckPUKQSiECgeS2K70aHPtAXONmrJ0iO53tcJte7ktlD95ZzhqSvaoT8g9SgMADtMuZIxFlYNfAIdZlFXCahVomeM2hk7Qlj271GSVmcW4x2TV4Jat3z84NXFM2/h6gPH72iBGakKDCJivTSyv54e/f64eHtrVNfyB27wernGG1SmIpoecnulsrHtmjId9WecOil7fZ2i2pqIUXbyj4tfUUTTPDFoHWKnXqozRJNGVUHw0elorG87q9rFpXQr91Zl0rwzdpA9uSKXZ/aneDMEzCrptkX4cwX5mmYWhq00gvyHEqCIM+rZNjSdZiSso/JsQM3Khxr0TwDfDShQg/8fnPrXXz61HrHX3DOgmvQO30NPGUl1N8LFzkMWj9yGCnLI4YlcYWRmuRA6oOWYAVqWKgNY/oMGIIPt054W4AGr6ntFEczWr7usmRXyOPWXHbW74wMeKDIXGyyILHJvh8p4654cuQ33ziXhHH6em8/LmAk6IwIKWLXj/xFhlOxGflWLVdBTHVHvtGszDp77Rl2k4qLQFO2MVATOO31pGWkxwhZjWQAbkvxlpyV1TX+86R5OzkIU5OPZn6lDGFzkAoz+HB7BQTKnH4tEWiCuaI7ikI75+iC8bXkoswiki6rJjR7468lEdoZU/vOPON5Rvohcfm6VMF1FiaaVxci5TmQLS9VM2bNTBUXs5epjOZRxvOontdVq8kdzWlGWKMFXJgOblutB+YumQ4N/lV0h0SVwpNSTlrphhOoP18acO80HqNbM6HNz/oTChNXzAKFpFK5gvW4WQfje2gRG3H+OGh+NijvL6a/r/Gc5J2L8DwzqYG9s9g4XdX4U5o0Pl8+g+WW0TgiSTJMyLtDmmHDu1pgBQ0aGqV/ORMkfwyzkDdXpAtwJHNeNWAnpsXY/qXb5UBO6lNbO5IHv0d0hWqcp8GB6zydA612h28+X2rO7g4JFaHMJAqEscEYbS4/daDdweqheL84zSWsY35diR3n6Fr+XGLec6mA5AkUXCht1TqUTBKW8zyeZibPtWqV9VEjg62751TBOWKUKzg1nLby6ehIT5jemqKp/RDFm71OGVfQLBZOlrdE/8CtKSHKNlsngn4nUtVIcbWCKZqh8UDrz9ueJ4FEreR+ozH8lMeTrpDptEHzLuCmcSHTGY7Bn1/0KTPLSQMsrIo9vHN7f1WQcG2zF9eDnXzHzi5Lu89cShQkxfq0Qg+0rnoiSVztPl0pncats2nRHArzKXK8paElRfERJlvPzDyqT47OAOjPt7O/ax7fH3pieYF1Pjx9jgPx/9N0k80TUKQy2h6qc/azbCa6BN6Y9PBO/qk6VVy8k3C2UpwOZ8YivTGnnKNZta2B4pRz6u7hjRXpJO5ajgRGeRyR1451cgDjrPIt4XOhtR1irsareYS3/j6qQeEQVi23FVzHt1PMURDmqWJVrSdzV11r8zfMzdHeG7k2sDbjZBjr70bLT+7f1hCW7PW0UrkI/e/mp/0DnV4JVP/P4k75mmGI0WlsCJmyVXDe+Hi/uZ3tMZfaO5zhqNbLZW8MWkpXFuOM9q6fpYw0H2f0tPQspW0gh+THTWBIrzZoXqfe6Q2jNBvG2c5Nh/44xyy/HqltUJ1TvmA0Nqz2xrTiRiqSJ9tDMDoP5KUDhDV9XDoqz8YhclRd1+exNfZkbwbJhTJbHVMJXprSasgKMbKWZ0btOBhf15aWlgg4R028y3nRuniXeDSLd+ZXq6k9yA15eobpfdeZ3+4JOpfJ6nua7ze3Vf/Qec7Sc544srclC5FB8x9Mxoo+YUU7dyFKsBhsJ2DehsWKcKxRPCNN97YN5pziRAUCP2krCM7q3t4y3lf3nZim3spvlKAoQXEgjPFY74AVN3+/39z+7CslBMpTfAXBWJRyH40YZE5P4ps/Pm/egwMw7NbrI0/QBRWo083T5Bai7eCjr71AI9lXqRk1u0OBvkHUDXKRdtlVRKYOaRz/J5rDb7+4/bqh5MPUeAGlRplMmWGIUWqUKZQhq+xV89oY6ROxHadN5/uH5fcjZ6qz3P2TDeNosXWebqrT0+zkbgud6QrT6D4ryuh/yeDGsbmUG8NWOtCOsVWQ/g5lzd0Rghy8S735hslffqz2P1Z7G+uP1f7/crWvgt+Oi++x7h/Jv0MGcCT/kQsMwH/kAj461+o9l3JiLlBmGREBs4EKb9Lh3JIDlupOuWCRovKDckG8IFWDRxRqupqGEemdNjjJi55WZkVP1M/QGnt9cKoM3b/XZszIGc2jtWGqadx/IoISz1F/Rr6t5yLfJnE1hr1YDDu5bMJeMbOo1ycW1it2UnpxKmT1Om8VYl3t+5XgAsXuRZ9yZiVYY+hwmsd4poJwkUYCSWJfGiY8FOOhbXl8p4dmGr9ojlL6OiG4TELfu3+6O9W2W37vbTN7kV7MVt4L0p/F1beyBjrP0LpGjqmMCVy38I6bYUD5YMwwbgWYcTtDzLOMqogRhXl8iDIXHkztYe91jWl0qNGB5uAhaFOGomCHcyl6rcFnCurZyHp303JFQxt5COw2CqxoYKPcfsOrucw5ZwMfXxaqmxvVg0doLdFSdpjcdGV/DMAJ/PIbbG2PAGgdj1HMVUR5JIZtsBOD/RuDAR/+fA8apO8pY6F8Xd95Pf4b+OWSHeY91hm95Cu7rlvi79Bx3XJP6bbuX8VKIK69fjXGWa7f8x65V08pqk7ZF9fegxQ1eTJym7Tg3F1snsNiLTtdsn66Et4MwgMf7ylLBK6rNBgSJ9JZTkU8NYzznu3gNypXNtO/tUNc/uiiEDQj4hCR3Y7mVK1rW3yowMAJdtkDoeM9iOYezamPGpn96PNT/Dqq92oS094HMXykzbRWet/bIOqH4DRPDbu7tz+D0HM7xaJn0tc3KIywHoc+qFutH/lvv4zRVy+5CEZdVW/Gac3rK4KxmmOAcVJGpIrKIrHcEatcd8OOjZdmCM97zOGZSLBhe5b8Fb71kWQ4NlzzCKJqr3XNeBr0bSG/8/T4KL6+gCnvL+mKy6iMg6q7ozIOJ09KFVTdZvMpnLizv6RmrcDRi26B8xGpBtdat5aYjNeB57/dQ29QuEyqDUApSYrds7eJC51j3z/fBuYxyDq7oZcINy2bs6qR4BONMYoZGTwCIgSvxrgCyhimhJn/A5rHrEwQ9klyBVImgCq+9qxx2l2nGn/FY9NbF6me5/7EWekwWfsKp0uqqhzXo2rhq7lCaDseY3r0FWmUL7zdckgt+w95h6pzLKmeDk3N48bdk+dKVs/19ofeLNYY3YfbdaPg2iJNG/FMpUbHutH4ebHQxwVsy/gR1YWDYI/XGQ4tFZOwzJ6AyGVyBXsu1ZWpAHniorVyE3BaNJSrVG0vlKwwUk3cAAOj2gISCyJIfQNbzLOM2G3h2Y9XmmwX89TVy4DbmjV99Zql82AYh4De0s3k0+chHZUVo1SUsQpdu0HO1U9/fXVAeQV/eZXzn+2x3F96WWx4vSuq+4bN3cIkoXn9Vhrzzsya1xncHUW6pTNiXkmHR0Eu2tAvxTu+l9CEBKr0eleyBLYIZaFnKeHP9v6186Rz2hAVMhjkVhqjj1MSuR1R+8GTtlbqKYjAvI5XZkNdLfq9UDZYYM0p3tp7D20g33Fh/ZA0JQUt7AIrareK4aSstpEXfWPsQMu0Ta/WGfbtZIMnzky01gXeDrvCRmco/j1uT3T9LwAA//8j4LYZ" + return "eJzsXc9z2ziyvuev6MrlzVQ5fu9dfdiqTJxJUjOOvaNk97C1xUBki8KaJDgAaEf7128B/CGKBECQhJSqLes0E0vf96HRbDQaTfINPOLhBmIs968AJJUZ3sDrd1juX78CSFDEnJaSsuIG/vIKAED9CXKWVBm+AhB7xmUUs2JH0xvYkUyof+WYIRF4AylR30EpaZGKG/jHayGy11fwei9l+fqfrwB2FLNE3GjkN1CQHDst6iMPpULhrCqbfzEoUp9v6kffIGaFJLQQIPcIOUpOY/XfRMIzcgQRc1JiAjvOcnj3/uHjdQPQl3EiJauERB4lVDx2fzTJckhTHwvOqZ3az1BMXxB5IjQj2wyvtweJ4uQ7ra6MFengDw5p6vO2RQWNCmynDdioHnx9x3hO5A2MBbQiJZMkCyrwi0IMI64SmATV9lVgslza0NP2SDK5D+FrI6T53saekJMsi4QksjLb6xEPz4wn80x2X+NCjesyW+dSNMd4j/GjuMaSxftAc3dHSnhCLigrpmg5q4rk+olkFQYi78BBY/sJCD0Vm5MpOBUy9M0Rt9k3zX7m4bEjfJdvmqZt+VzoVa2xhdMhONntaHzNkSTRrChij0zT4mr7gCIFueesSvdlJaFEDgJjZvOdRuszpxIvLlazLlCrLcvKqFQOgXGo+e2bkLJSzDPe2fTUVvIRlFNRZiTGa726BpbRgkOZQlHlW0sU7jSw7b8wlqFW0JGKBt5LCieSMqMQEZMMk2iXMSIt/l0ij7EY/nWu3LGEVmWCKScJJmeZtBZ8YtI6DeeZtE6Fx6R1Un7cpHVy7ZNWptcJkeTiEbNMQfE6ctcyvdYbgItLI6cbBJs47eUXFyePmwObMJX0X1xX1W0MLLJ0woM6pcNI/WOgS/Mh1Zk19r/nlhCzanRBLWXf7NmzgD17hpwUByhTAYQj0KIRxXaTQx/pC5zsNRMkpvM9JpLrXZWZQ/eWsQzJUNUE+SehQGEE2mcskPCzsCrgCeoqj5hIQq0SA2dQyMoTprZ7rZKqPLcY5ZqsFtS559cHpyha/AhRnz47RXHMSVliEpXppZX98f7u7cPD+1urvpA7do01zDG6pDDl0fKS3S0Vj13RkO3qPeHYS7vt7RalbyFF2co8LQUKiTP37N80XK/aqcbartE0o/KgCamQNBbXw4XNIBOGxTvjWgmuaRvJ9i7ZDantGc48AbOKmkMR1oRhngbf2qCRnpPnUBI4eV4lw5Svg0/OPiXHDNyqsCxG8wzwWccKNfD7za1z9RlSqy1/yVgWXIPa6itgn6VQfS9g6NBww9ChtSwPGYbUFSaqkiOpD0qCEahloSYM/ynQBJ9urfCmEH1kqDe3XhxHOxq+bjNlX8njVl94xu9MjHikSF9uoiSxTsAfacZsEeXIr79xLgnT9M32flrARNiZEFIapxMm6wynYnPyvV6wgpjqjnyneZX3ttsz7CYk44GmbKOhPDjNJaVlpMcYWY9kBG7K8pYclzVl/vNkejsxilPepzO/0gxhcxASc/h0ewUEqoL+WSHQBAtJdxS5cs7JJePPivEqj0i6rKDQbo//rAhXzpiaN+c5K3JSGhmG5h9RGBamGq63MtGivhApK4BsWSXbMStmKhmfvU7ltIhyVkTNvK5aTu5oQXOStVrAhmnhNpV7YO6aadHgXkZ3SGTFHUmlafJG1OMJVJ9vLbhzGo/RrZ3Q9mfDCQXPFbNELqiQtmA9bdbR+B46xFacOw7qn40q/Ivp7xs8K3nvIjzPTCpg5yy2Tlf3/lQ6kS+Wz2C1zWgckSQZp+T9Ic2w4V0jsIYGBY3CvZxxUjyGWcjbK9IGOJE6rxqwFdNgbPfSbXMgK/WprS3Jg9sj+kIVztPozHWs07gTsoo1e3z7+daQ9vdIKAnNdKZAsmw0SJPP+460P1o1FucX/XzCOOa3tdhpjr7pzyXmIxMSSJFAybhUVm1iiZewghWxn5kcF6tR1meFDKYOn1MF5whStujUcppKqJMjPWF6rwun5oMUZ/rqM66gaSycrG+J+oFdU0KkabZOBP1OhGyQ4noJkzRH7YHGn3d9TxyJOa74c79TGG7K42lXyHxaozlXcN28kKsUR+PPL/tUueG0ARYWxh4+2L2/rkjY9tmLS8JWvmN3l6HlZy4lcpJic2KhBtoUPpEktpafvpRe89bZtCgOiYWPHGdtaEldfILJ1Dczj+qLpTsAhvNt7fGax/eHmlhWYpMQ+89xIP6/646yeQLKVETbQ33WbiW3Jlk+24k+gzMoPXwQ/1MfLS7eS1j7KU7HM2OV3uijzsm82tRFccrpu394Z0Q6CbyGY4FJHkvoNWOdHMJY63xL+GxoXZuYrftqHuGtu5lqVDqEVettDdfz7RQL5CRz1LHq/pO5y66xAxzmJmkftVwTWJdyZhir70bLj+/fNxCG9PW0VrkI/a/6p8MznUERVP3P4nb5hmGM0etuCJmz1XDO+Hi/uZ3tMZfaPJzhuNbJZe4OWkpXldOM5tafpYy0mGZ09PUspW0hx+THXWBIr9ZoTqfeqR2j0DvG2c5Nx/44xyy/HqlNUL1zvmA0Jqzu7rTyRkhSJNtDMDoH5KUDhLGYuHRUjp1DZKm7BkhkG3BvdwbBuNSbHV0MXprTKsgaMTIWaCYNORpf35iGtgg4R1m8z3nR0nifeDKNtyZYq6kdyC15eobp/dCb3/4hOhPJ6jub7ze3dQ/ReY7TC5ZY0rdF3U8azn04GUv6hDXv3KUowXK0oYB5WxYjwrFM8Yw03Zu2mHPqEzUI/KSswFnWtPhW8b6+/UT39taOIzlFAZIByTIWqz2wZPrv95vbn13VhECZiqsmGPNK7KMJg8zpTHz3x9fNR7AAht18fWYJ2qACtbs5Ot1CtB58drUYKCTzMjWjbHco0TWIpkkuUi67ikiXIrXj/0QL+O0Xu1+3lGycHC+gVCjelDmGGKVC8aEMWWivG9imSJ+I6UTNn+9vht9PHKvOcvcvJoyjxdZ5ui5Q+9nJ3ho60xX86L5KmtF/k9H9Y3MpN5qtsqAdYysnwz3KqpskODk413r9DZ3BvCz3L8u9ifVluf+vXO7r6Ldj/Ecs/EfyH5ACHMlfkoER+Esy4KKzLd9zKT2TgSrPCTenA0uK0A2e1/nckjOW+oa5YJGi9oNqQbwgdZNHFGq62qYR4Zw2OMmLnlZmRU/UzdAZe31wqg09vOFmysg5LaK1Yart3n8inBLHaX9Ovq/nIt+9uFrDXiyGnVw2Ya+YWdTrEwvjFeuVXpwKWb3OG4UYV/thLbhEvns1pJxZC1YYKpwWMZ6pJFymEUeSmJcGj4djPHRtjx/U0HTzFy1QCFczBBOJ/R7+RevRYHeqbLf8Dtx29iK1mNn3K34nX8NpXH1Da6AjDaVr4qRK28B2Jy/Mv6/hQZth2gow46aGmOU5lVFGJBbxIcpteODbyD7oHFPo0KADLcBB0OUMZZkdzqXorQKfKWhgI+M9TssVjW3kIDDbKLCikY0K822v+jpnLBv5uOX6ngrW7f3qwWO00mgoPHh3XpkfB3ACv/w+W9OjADrPyygWMqIs4uNmWM9w/05jwKf/vQcFMnSVqWC+rvu8Gf8N/HLJPvMB64yO8pW91x3xD+i77rh9eq6Hl7HkiGuTLYVxluv3zMfu9fOK6pP2xeX3IHVNlkzcLs0Zs9eb57AYK0+XLKGuhNeDcMDHe5olHNcVGzSJFeksByOOMsZ5j3fwOxUrW+rfmyEuf3pRcpoTfojIbkcLKtc1Lz7UYGAFu/yZ0PQRZYg94MrzyJeA+BIQV1C9BMSXgOgVELtMtr533fcZTLNfC3GK3+S5gzqt37tyxs/68rvDyPWmnObpYO0DFe/uzc9nddxltuh9Hc19WxOsx6GPavnrR/7bL1P09QuAglHXFe1pWv1qn2Cs+mh0mjQjQkZVmRieFFAzG54SMDVemiM877GAZyLAhO1Y8lf41meS49Rw9bPZ6vLTdcbSoG9S+p2lx6eUDgX4vNupLy6nIg6q7o6KOJw8IWRQdZvNl3Dizv4Cr7UCJy+6Bc5HhBxda/3zlWT6bGz+m4+w3OsHs+uKSCVIiv1+BM+FzlIJnW8D/Yh4ld3QS4Sbjs1a503wicYYxRkZPRonBK/CuAKaZZiSTP8f0CLOqgRhnyRXIEQCKONrxxqn3NXX+CteKdG5SP2uiyeWVRaTda+3u6Sq2nEdqha+tjCEtmNrh0NfmUbFwrvQx9Ri+AIMqLtpk/rJ+VS/isE+ebZk9VxvxhnMYoPRf+pnPwquLVt3EU/XrlWsm4yfFwt9jMO2ih9RXjgIDnit4dBQMQnL7AiITCRXsGdCXukKkCMuGis3AadFQdkO78yFkhVGaohbYMiosoDAknDS3NYbszwnZls49uO1JtPF7Lt6aXBXsdJUr1k6D5pxDOgs3Xh35IzpqKgZhaRZVqMrNyiY/On/3xxQXMH/vSnYz+ZY7i69LDa82hU191LohyiQhBbNG7v0+4QbXmtwtxTpls6Ifl0nHgXZaEO/MPT4zlYdEqhU612VJbBFqEo1Swl7Nvf0niedU4aokUEjd9Iy+uiTyO2I3I+eQLhST0k4Fk280hvqetEfhLLRAqv7GtbekW0C+YEL66ekLSkoYRdYUftVDCtlvY286Nu0R1r8Nr1KZ9g3N44exOVprQu8OXuFjc5Q/Hvcnuj6TwAAAP//jdrw9g==" } diff --git a/metricbeat/module/ceph/mgr_cluster_disk/_meta/fields.yml b/metricbeat/module/ceph/mgr_cluster_disk/_meta/fields.yml index de265b599687..a6068b974168 100644 --- a/metricbeat/module/ceph/mgr_cluster_disk/_meta/fields.yml +++ b/metricbeat/module/ceph/mgr_cluster_disk/_meta/fields.yml @@ -4,7 +4,7 @@ release: beta fields: - name: stats - type: group + type: nested description: > `stats` contains disk availability statistics. fields: @@ -30,7 +30,7 @@ type: long description: Number of OSDs per pool in the cluster - name: pools - type: group + type: nested description: > `pools` contains disk pool statistics. fields: @@ -41,7 +41,7 @@ type: long description: Pool ID - name: stats - type: group + type: object description: Pool statistics fields: - name: kb_used diff --git a/metricbeat/module/ceph/mgr_cluster_health/_meta/fields.yml b/metricbeat/module/ceph/mgr_cluster_health/_meta/fields.yml index e3037e6968fe..4bd446c0e93e 100644 --- a/metricbeat/module/ceph/mgr_cluster_health/_meta/fields.yml +++ b/metricbeat/module/ceph/mgr_cluster_health/_meta/fields.yml @@ -50,7 +50,7 @@ description: Monitor public addresses fields: - name: addrvec - type: group + type: nested description: > `addrvec` contains details of all public addresses. fields: @@ -108,7 +108,7 @@ type: long description: Write operations per second - name: pgs_by_state - type: group + type: nested description: > `pgs_by_state` contains information about PGs' states. fields: @@ -185,7 +185,7 @@ type: long description: File system up - name: by_rank - type: group + type: nested description: > `by_rank` contains information about file systems sorted by rank. fields: diff --git a/metricbeat/module/ceph/mgr_osd_disk/_meta/fields.yml b/metricbeat/module/ceph/mgr_osd_disk/_meta/fields.yml index 8e8d65a7799c..73fbe49d8fa3 100644 --- a/metricbeat/module/ceph/mgr_osd_disk/_meta/fields.yml +++ b/metricbeat/module/ceph/mgr_osd_disk/_meta/fields.yml @@ -5,7 +5,7 @@ release: beta fields: - name: nodes - type: group + type: nested description: > `nodes` contains information about active nodes. fields: @@ -61,7 +61,7 @@ type: long description: Space utilization - name: stray - type: group + type: nested description: > `stray` contains information about stray OSD nodes. fields: diff --git a/metricbeat/module/ceph/mgr_osd_perf/_meta/fields.yml b/metricbeat/module/ceph/mgr_osd_perf/_meta/fields.yml index 8ed76715e564..8efa1bbae281 100644 --- a/metricbeat/module/ceph/mgr_osd_perf/_meta/fields.yml +++ b/metricbeat/module/ceph/mgr_osd_perf/_meta/fields.yml @@ -13,7 +13,7 @@ `stray` contains OSD statistics. fields: - name: osd_perf_infos - type: group + type: nested description: OSD performance statistics fields: - name: id diff --git a/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/fields.yml b/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/fields.yml index 19bd35a0314b..351a71a991fe 100644 --- a/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/fields.yml +++ b/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/fields.yml @@ -1,5 +1,5 @@ - name: mgr_osd_pool_stats - type: group + type: nested description: > OSD pool stats of Ceph cluster release: beta diff --git a/metricbeat/module/ceph/mgr_osd_tree/_meta/fields.yml b/metricbeat/module/ceph/mgr_osd_tree/_meta/fields.yml index ae6f1e130aba..81107cc9e5b8 100644 --- a/metricbeat/module/ceph/mgr_osd_tree/_meta/fields.yml +++ b/metricbeat/module/ceph/mgr_osd_tree/_meta/fields.yml @@ -5,7 +5,7 @@ release: beta fields: - name: nodes - type: group + type: nested description: > `nodes` contains disk active OSD nodes. fields: @@ -42,3 +42,41 @@ - name: crush_weight type: scaled_float description: CRUSH weight + - name: stray + type: group + description: > + `stray` contains stray OSD nodes. + fields: + - name: name + type: keyword + description: Node name + - name: root + type: keyword + description: Node type + - name: type_id + type: long + description: Type ID + - name: id + type: long + description: Node ID + - name: children + type: long + description: Node children + - name: reweight + type: long + description: Reweight + - name: depth + type: long + description: Depth + - name: exists + type: long + description: Exists + - name: status + type: keyword + description: Node status + - name: primary_affinity + type: long + description: Primary affinity + - name: crush_weight + type: scaled_float + description: CRUSH weight diff --git a/metricbeat/module/ceph/test_ceph_mgr.py b/metricbeat/module/ceph/test_ceph_mgr.py index 5732473e55e8..189c180bba45 100644 --- a/metricbeat/module/ceph/test_ceph_mgr.py +++ b/metricbeat/module/ceph/test_ceph_mgr.py @@ -57,3 +57,5 @@ def test_ceph_mgr(self, metricset): output = self.read_output_json() for evt in output: assert 'ceph' in evt + # Fails due to nested objects: + # assert self.assert_fields_are_documented(evt) From e8c1712d910bca9df4ccbbc1b2159a04c53448b7 Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Thu, 13 Feb 2020 13:24:24 +0100 Subject: [PATCH 29/41] mgr_cluster_disk as standard metricset --- metricbeat/docs/fields.asciidoc | 31 +++- .../modules/ceph/mgr_cluster_disk.asciidoc | 1 - .../modules/ceph/mgr_cluster_health.asciidoc | 1 - .../docs/modules/ceph/mgr_osd_disk.asciidoc | 1 - .../docs/modules/ceph/mgr_osd_perf.asciidoc | 1 - .../modules/ceph/mgr_osd_pool_stats.asciidoc | 1 - .../docs/modules/ceph/mgr_osd_tree.asciidoc | 1 - metricbeat/include/list_common.go | 1 + .../testdata/mgr_df_sample_response.json | 17 ++ .../_meta/testdata/mgr_failed_response.json | 17 ++ metricbeat/module/ceph/fields.go | 2 +- .../ceph/mgr_cluster_disk/_meta/data.json | 156 ++---------------- .../ceph/mgr_cluster_disk/_meta/fields.yml | 74 ++------- .../module/ceph/mgr_cluster_disk/manifest.yml | 26 --- .../ceph/mgr_cluster_disk/mgr_cluster_disk.go | 90 ++++++++++ .../mgr_cluster_disk_integration_test.go | 63 +++++++ .../mgr_cluster_disk/mgr_cluster_disk_test.go | 97 +++++++++++ .../module/ceph/mgr_cluster_disk/mgr_data.go | 84 ++++++++++ metricbeat/module/ceph/module.yml | 8 - metricbeat/module/ceph/test_ceph_mgr.py | 8 +- 20 files changed, 427 insertions(+), 253 deletions(-) create mode 100644 metricbeat/module/ceph/_meta/testdata/mgr_df_sample_response.json create mode 100644 metricbeat/module/ceph/_meta/testdata/mgr_failed_response.json delete mode 100644 metricbeat/module/ceph/mgr_cluster_disk/manifest.yml create mode 100644 metricbeat/module/ceph/mgr_cluster_disk/mgr_cluster_disk.go create mode 100644 metricbeat/module/ceph/mgr_cluster_disk/mgr_cluster_disk_integration_test.go create mode 100644 metricbeat/module/ceph/mgr_cluster_disk/mgr_cluster_disk_test.go create mode 100644 metricbeat/module/ceph/mgr_cluster_disk/mgr_data.go delete mode 100644 metricbeat/module/ceph/module.yml diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 3209bd3198c4..0aa9071b62ed 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -3935,26 +3935,43 @@ type: long [float] === mgr_cluster_disk -Disk metrics of Ceph cluster +mgr_cluster_disk -*`ceph.mgr_cluster_disk.stats`*:: + +*`ceph.mgr_cluster_disk.available.bytes`*:: + -- -`stats` contains disk availability statistics. +Available bytes of the cluster -type: nested +type: long + +format: bytes -- -*`ceph.mgr_cluster_disk.pools`*:: +*`ceph.mgr_cluster_disk.total.bytes`*:: + -- -`pools` contains disk pool statistics. +Total bytes of the cluster -type: nested +type: long + +format: bytes + +-- + +*`ceph.mgr_cluster_disk.used.bytes`*:: ++ +-- +Used bytes of the cluster + + +type: long + +format: bytes -- diff --git a/metricbeat/docs/modules/ceph/mgr_cluster_disk.asciidoc b/metricbeat/docs/modules/ceph/mgr_cluster_disk.asciidoc index 22f79ae4b3b3..6241a39a8f38 100644 --- a/metricbeat/docs/modules/ceph/mgr_cluster_disk.asciidoc +++ b/metricbeat/docs/modules/ceph/mgr_cluster_disk.asciidoc @@ -9,7 +9,6 @@ beta[] include::../../../module/ceph/mgr_cluster_disk/_meta/docs.asciidoc[] -This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. ==== Fields diff --git a/metricbeat/docs/modules/ceph/mgr_cluster_health.asciidoc b/metricbeat/docs/modules/ceph/mgr_cluster_health.asciidoc index 80580cb8a420..3e2ce6b273be 100644 --- a/metricbeat/docs/modules/ceph/mgr_cluster_health.asciidoc +++ b/metricbeat/docs/modules/ceph/mgr_cluster_health.asciidoc @@ -9,7 +9,6 @@ beta[] include::../../../module/ceph/mgr_cluster_health/_meta/docs.asciidoc[] -This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. ==== Fields diff --git a/metricbeat/docs/modules/ceph/mgr_osd_disk.asciidoc b/metricbeat/docs/modules/ceph/mgr_osd_disk.asciidoc index 5a5ccee54912..22bfe8b3fbc0 100644 --- a/metricbeat/docs/modules/ceph/mgr_osd_disk.asciidoc +++ b/metricbeat/docs/modules/ceph/mgr_osd_disk.asciidoc @@ -9,7 +9,6 @@ beta[] include::../../../module/ceph/mgr_osd_disk/_meta/docs.asciidoc[] -This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. ==== Fields diff --git a/metricbeat/docs/modules/ceph/mgr_osd_perf.asciidoc b/metricbeat/docs/modules/ceph/mgr_osd_perf.asciidoc index e16a0db71167..d3549734d2f1 100644 --- a/metricbeat/docs/modules/ceph/mgr_osd_perf.asciidoc +++ b/metricbeat/docs/modules/ceph/mgr_osd_perf.asciidoc @@ -9,7 +9,6 @@ beta[] include::../../../module/ceph/mgr_osd_perf/_meta/docs.asciidoc[] -This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. ==== Fields diff --git a/metricbeat/docs/modules/ceph/mgr_osd_pool_stats.asciidoc b/metricbeat/docs/modules/ceph/mgr_osd_pool_stats.asciidoc index 2710299b2abc..75c986659ef2 100644 --- a/metricbeat/docs/modules/ceph/mgr_osd_pool_stats.asciidoc +++ b/metricbeat/docs/modules/ceph/mgr_osd_pool_stats.asciidoc @@ -9,7 +9,6 @@ beta[] include::../../../module/ceph/mgr_osd_pool_stats/_meta/docs.asciidoc[] -This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. ==== Fields diff --git a/metricbeat/docs/modules/ceph/mgr_osd_tree.asciidoc b/metricbeat/docs/modules/ceph/mgr_osd_tree.asciidoc index 9c53e3284c5f..3153fd50726b 100644 --- a/metricbeat/docs/modules/ceph/mgr_osd_tree.asciidoc +++ b/metricbeat/docs/modules/ceph/mgr_osd_tree.asciidoc @@ -9,7 +9,6 @@ beta[] include::../../../module/ceph/mgr_osd_tree/_meta/docs.asciidoc[] -This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. ==== Fields diff --git a/metricbeat/include/list_common.go b/metricbeat/include/list_common.go index 17318c683e8c..665fddd77d3b 100644 --- a/metricbeat/include/list_common.go +++ b/metricbeat/include/list_common.go @@ -32,6 +32,7 @@ import ( _ "github.com/elastic/beats/metricbeat/module/ceph/cluster_disk" _ "github.com/elastic/beats/metricbeat/module/ceph/cluster_health" _ "github.com/elastic/beats/metricbeat/module/ceph/cluster_status" + _ "github.com/elastic/beats/metricbeat/module/ceph/mgr_cluster_disk" _ "github.com/elastic/beats/metricbeat/module/ceph/monitor_health" _ "github.com/elastic/beats/metricbeat/module/ceph/osd_df" _ "github.com/elastic/beats/metricbeat/module/ceph/osd_tree" diff --git a/metricbeat/module/ceph/_meta/testdata/mgr_df_sample_response.json b/metricbeat/module/ceph/_meta/testdata/mgr_df_sample_response.json new file mode 100644 index 000000000000..68f483401280 --- /dev/null +++ b/metricbeat/module/ceph/_meta/testdata/mgr_df_sample_response.json @@ -0,0 +1,17 @@ +{ + "failed": [], + "finished": [ + { + "command": "df format=json-pretty", + "outb": "{\n \"stats\": {\n \"total_bytes\": 10737418240,\n \"total_avail_bytes\": 9659875328,\n \"total_used_bytes\": 3801088,\n \"total_used_raw_bytes\": 1077542912,\n \"total_used_raw_ratio\": 0.10035400092601776,\n \"num_osds\": 1,\n \"num_per_pool_osds\": 1\n },\n \"stats_by_class\": {},\n \"pools\": [\n {\n \"name\": \"rbd\",\n \"id\": 1,\n \"stats\": {\n \"stored\": 0,\n \"objects\": 0,\n \"kb_used\": 0,\n \"bytes_used\": 0,\n \"percent_used\": 0,\n \"max_avail\": 9123004416\n }\n },\n {\n \"name\": \"cephfs_data\",\n \"id\": 2,\n \"stats\": {\n \"stored\": 0,\n \"objects\": 0,\n \"kb_used\": 0,\n \"bytes_used\": 0,\n \"percent_used\": 0,\n \"max_avail\": 9123004416\n }\n },\n {\n \"name\": \"cephfs_metadata\",\n \"id\": 3,\n \"stats\": {\n \"stored\": 2286,\n \"objects\": 22,\n \"kb_used\": 512,\n \"bytes_used\": 524288,\n \"percent_used\": 5.7465484132990241e-05,\n \"max_avail\": 9123004416\n }\n },\n {\n \"name\": \".rgw.root\",\n \"id\": 4,\n \"stats\": {\n \"stored\": 2398,\n \"objects\": 6,\n \"kb_used\": 384,\n \"bytes_used\": 393216,\n \"percent_used\": 4.3099731556139886e-05,\n \"max_avail\": 9123004416\n }\n },\n {\n \"name\": \"default.rgw.control\",\n \"id\": 5,\n \"stats\": {\n \"stored\": 0,\n \"objects\": 8,\n \"kb_used\": 0,\n \"bytes_used\": 0,\n \"percent_used\": 0,\n \"max_avail\": 9123004416\n }\n },\n {\n \"name\": \"default.rgw.meta\",\n \"id\": 6,\n \"stats\": {\n \"stored\": 736,\n \"objects\": 5,\n \"kb_used\": 256,\n \"bytes_used\": 262144,\n \"percent_used\": 2.873356788768433e-05,\n \"max_avail\": 9123004416\n }\n },\n {\n \"name\": \"default.rgw.log\",\n \"id\": 7,\n \"stats\": {\n \"stored\": 0,\n \"objects\": 176,\n \"kb_used\": 0,\n \"bytes_used\": 0,\n \"percent_used\": 0,\n \"max_avail\": 9123004416\n }\n },\n {\n \"name\": \"default.rgw.buckets.index\",\n \"id\": 8,\n \"stats\": {\n \"stored\": 0,\n \"objects\": 1,\n \"kb_used\": 0,\n \"bytes_used\": 0,\n \"percent_used\": 0,\n \"max_avail\": 9123004416\n }\n }\n ]\n}\n", + "outs": "" + } + ], + "has_failed": false, + "id": "139687220240336", + "is_finished": true, + "is_waiting": false, + "running": [], + "state": "success", + "waiting": [] +} diff --git a/metricbeat/module/ceph/_meta/testdata/mgr_failed_response.json b/metricbeat/module/ceph/_meta/testdata/mgr_failed_response.json new file mode 100644 index 000000000000..409e19fb2170 --- /dev/null +++ b/metricbeat/module/ceph/_meta/testdata/mgr_failed_response.json @@ -0,0 +1,17 @@ +{ + "failed": [ + { + "command": "dfb format=json-pretty", + "outb": "", + "outs": "command not known" + } + ], + "finished": [], + "has_failed": true, + "id": "139687220237200", + "is_finished": true, + "is_waiting": false, + "running": [], + "state": "failed", + "waiting": [] +} diff --git a/metricbeat/module/ceph/fields.go b/metricbeat/module/ceph/fields.go index 00af6a24fbef..9eb2752d6970 100644 --- a/metricbeat/module/ceph/fields.go +++ b/metricbeat/module/ceph/fields.go @@ -32,5 +32,5 @@ func init() { // AssetCeph returns asset data. // This is the base64 encoded gzipped contents of ../metricbeat/module/ceph. func AssetCeph() string { - return "eJzsXc9z2ziyvuev6MrlzVQ5fu9dfdiqTJxJUjOOvaNk97C1xUBki8KaJDgAaEf7128B/CGKBECQhJSqLes0E0vf96HRbDQaTfINPOLhBmIs968AJJUZ3sDrd1juX78CSFDEnJaSsuIG/vIKAED9CXKWVBm+AhB7xmUUs2JH0xvYkUyof+WYIRF4AylR30EpaZGKG/jHayGy11fwei9l+fqfrwB2FLNE3GjkN1CQHDst6iMPpULhrCqbfzEoUp9v6kffIGaFJLQQIPcIOUpOY/XfRMIzcgQRc1JiAjvOcnj3/uHjdQPQl3EiJauERB4lVDx2fzTJckhTHwvOqZ3az1BMXxB5IjQj2wyvtweJ4uQ7ra6MFengDw5p6vO2RQWNCmynDdioHnx9x3hO5A2MBbQiJZMkCyrwi0IMI64SmATV9lVgslza0NP2SDK5D+FrI6T53saekJMsi4QksjLb6xEPz4wn80x2X+NCjesyW+dSNMd4j/GjuMaSxftAc3dHSnhCLigrpmg5q4rk+olkFQYi78BBY/sJCD0Vm5MpOBUy9M0Rt9k3zX7m4bEjfJdvmqZt+VzoVa2xhdMhONntaHzNkSTRrChij0zT4mr7gCIFueesSvdlJaFEDgJjZvOdRuszpxIvLlazLlCrLcvKqFQOgXGo+e2bkLJSzDPe2fTUVvIRlFNRZiTGa726BpbRgkOZQlHlW0sU7jSw7b8wlqFW0JGKBt5LCieSMqMQEZMMk2iXMSIt/l0ij7EY/nWu3LGEVmWCKScJJmeZtBZ8YtI6DeeZtE6Fx6R1Un7cpHVy7ZNWptcJkeTiEbNMQfE6ctcyvdYbgItLI6cbBJs47eUXFyePmwObMJX0X1xX1W0MLLJ0woM6pcNI/WOgS/Mh1Zk19r/nlhCzanRBLWXf7NmzgD17hpwUByhTAYQj0KIRxXaTQx/pC5zsNRMkpvM9JpLrXZWZQ/eWsQzJUNUE+SehQGEE2mcskPCzsCrgCeoqj5hIQq0SA2dQyMoTprZ7rZKqPLcY5ZqsFtS559cHpyha/AhRnz47RXHMSVliEpXppZX98f7u7cPD+1urvpA7do01zDG6pDDl0fKS3S0Vj13RkO3qPeHYS7vt7RalbyFF2co8LQUKiTP37N80XK/aqcbartE0o/KgCamQNBbXw4XNIBOGxTvjWgmuaRvJ9i7ZDantGc48AbOKmkMR1oRhngbf2qCRnpPnUBI4eV4lw5Svg0/OPiXHDNyqsCxG8wzwWccKNfD7za1z9RlSqy1/yVgWXIPa6itgn6VQfS9g6NBww9ChtSwPGYbUFSaqkiOpD0qCEahloSYM/ynQBJ9urfCmEH1kqDe3XhxHOxq+bjNlX8njVl94xu9MjHikSF9uoiSxTsAfacZsEeXIr79xLgnT9M32flrARNiZEFIapxMm6wynYnPyvV6wgpjqjnyneZX3ttsz7CYk44GmbKOhPDjNJaVlpMcYWY9kBG7K8pYclzVl/vNkejsxilPepzO/0gxhcxASc/h0ewUEqoL+WSHQBAtJdxS5cs7JJePPivEqj0i6rKDQbo//rAhXzpiaN+c5K3JSGhmG5h9RGBamGq63MtGivhApK4BsWSXbMStmKhmfvU7ltIhyVkTNvK5aTu5oQXOStVrAhmnhNpV7YO6aadHgXkZ3SGTFHUmlafJG1OMJVJ9vLbhzGo/RrZ3Q9mfDCQXPFbNELqiQtmA9bdbR+B46xFacOw7qn40q/Ivp7xs8K3nvIjzPTCpg5yy2Tlf3/lQ6kS+Wz2C1zWgckSQZp+T9Ic2w4V0jsIYGBY3CvZxxUjyGWcjbK9IGOJE6rxqwFdNgbPfSbXMgK/WprS3Jg9sj+kIVztPozHWs07gTsoo1e3z7+daQ9vdIKAnNdKZAsmw0SJPP+460P1o1FucX/XzCOOa3tdhpjr7pzyXmIxMSSJFAybhUVm1iiZewghWxn5kcF6tR1meFDKYOn1MF5whStujUcppKqJMjPWF6rwun5oMUZ/rqM66gaSycrG+J+oFdU0KkabZOBP1OhGyQ4noJkzRH7YHGn3d9TxyJOa74c79TGG7K42lXyHxaozlXcN28kKsUR+PPL/tUueG0ARYWxh4+2L2/rkjY9tmLS8JWvmN3l6HlZy4lcpJic2KhBtoUPpEktpafvpRe89bZtCgOiYWPHGdtaEldfILJ1Dczj+qLpTsAhvNt7fGax/eHmlhWYpMQ+89xIP6/646yeQLKVETbQ33WbiW3Jlk+24k+gzMoPXwQ/1MfLS7eS1j7KU7HM2OV3uijzsm82tRFccrpu394Z0Q6CbyGY4FJHkvoNWOdHMJY63xL+GxoXZuYrftqHuGtu5lqVDqEVettDdfz7RQL5CRz1LHq/pO5y66xAxzmJmkftVwTWJdyZhir70bLj+/fNxCG9PW0VrkI/a/6p8MznUERVP3P4nb5hmGM0etuCJmz1XDO+Hi/uZ3tMZfaPJzhuNbJZe4OWkpXldOM5tafpYy0mGZ09PUspW0hx+THXWBIr9ZoTqfeqR2j0DvG2c5Nx/44xyy/HqlNUL1zvmA0Jqzu7rTyRkhSJNtDMDoH5KUDhLGYuHRUjp1DZKm7BkhkG3BvdwbBuNSbHV0MXprTKsgaMTIWaCYNORpf35iGtgg4R1m8z3nR0nifeDKNtyZYq6kdyC15eobp/dCb3/4hOhPJ6jub7ze3dQ/ReY7TC5ZY0rdF3U8azn04GUv6hDXv3KUowXK0oYB5WxYjwrFM8Yw03Zu2mHPqEzUI/KSswFnWtPhW8b6+/UT39taOIzlFAZIByTIWqz2wZPrv95vbn13VhECZiqsmGPNK7KMJg8zpTHz3x9fNR7AAht18fWYJ2qACtbs5Ot1CtB58drUYKCTzMjWjbHco0TWIpkkuUi67ikiXIrXj/0QL+O0Xu1+3lGycHC+gVCjelDmGGKVC8aEMWWivG9imSJ+I6UTNn+9vht9PHKvOcvcvJoyjxdZ5ui5Q+9nJ3ho60xX86L5KmtF/k9H9Y3MpN5qtsqAdYysnwz3KqpskODk413r9DZ3BvCz3L8u9ifVluf+vXO7r6Ldj/Ecs/EfyH5ACHMlfkoER+Esy4KKzLd9zKT2TgSrPCTenA0uK0A2e1/nckjOW+oa5YJGi9oNqQbwgdZNHFGq62qYR4Zw2OMmLnlZmRU/UzdAZe31wqg09vOFmysg5LaK1Yart3n8inBLHaX9Ovq/nIt+9uFrDXiyGnVw2Ya+YWdTrEwvjFeuVXpwKWb3OG4UYV/thLbhEvns1pJxZC1YYKpwWMZ6pJFymEUeSmJcGj4djPHRtjx/U0HTzFy1QCFczBBOJ/R7+RevRYHeqbLf8Dtx29iK1mNn3K34nX8NpXH1Da6AjDaVr4qRK28B2Jy/Mv6/hQZth2gow46aGmOU5lVFGJBbxIcpteODbyD7oHFPo0KADLcBB0OUMZZkdzqXorQKfKWhgI+M9TssVjW3kIDDbKLCikY0K822v+jpnLBv5uOX6ngrW7f3qwWO00mgoPHh3XpkfB3ACv/w+W9OjADrPyygWMqIs4uNmWM9w/05jwKf/vQcFMnSVqWC+rvu8Gf8N/HLJPvMB64yO8pW91x3xD+i77rh9eq6Hl7HkiGuTLYVxluv3zMfu9fOK6pP2xeX3IHVNlkzcLs0Zs9eb57AYK0+XLKGuhNeDcMDHe5olHNcVGzSJFeksByOOMsZ5j3fwOxUrW+rfmyEuf3pRcpoTfojIbkcLKtc1Lz7UYGAFu/yZ0PQRZYg94MrzyJeA+BIQV1C9BMSXgOgVELtMtr533fcZTLNfC3GK3+S5gzqt37tyxs/68rvDyPWmnObpYO0DFe/uzc9nddxltuh9Hc19WxOsx6GPavnrR/7bL1P09QuAglHXFe1pWv1qn2Cs+mh0mjQjQkZVmRieFFAzG54SMDVemiM877GAZyLAhO1Y8lf41meS49Rw9bPZ6vLTdcbSoG9S+p2lx6eUDgX4vNupLy6nIg6q7o6KOJw8IWRQdZvNl3Dizv4Cr7UCJy+6Bc5HhBxda/3zlWT6bGz+m4+w3OsHs+uKSCVIiv1+BM+FzlIJnW8D/Yh4ld3QS4Sbjs1a503wicYYxRkZPRonBK/CuAKaZZiSTP8f0CLOqgRhnyRXIEQCKONrxxqn3NXX+CteKdG5SP2uiyeWVRaTda+3u6Sq2nEdqha+tjCEtmNrh0NfmUbFwrvQx9Ri+AIMqLtpk/rJ+VS/isE+ebZk9VxvxhnMYoPRf+pnPwquLVt3EU/XrlWsm4yfFwt9jMO2ih9RXjgIDnit4dBQMQnL7AiITCRXsGdCXukKkCMuGis3AadFQdkO78yFkhVGaohbYMiosoDAknDS3NYbszwnZls49uO1JtPF7Lt6aXBXsdJUr1k6D5pxDOgs3Xh35IzpqKgZhaRZVqMrNyiY/On/3xxQXMH/vSnYz+ZY7i69LDa82hU191LohyiQhBbNG7v0+4QbXmtwtxTpls6Ifl0nHgXZaEO/MPT4zlYdEqhU612VJbBFqEo1Swl7Nvf0niedU4aokUEjd9Iy+uiTyO2I3I+eQLhST0k4Fk280hvqetEfhLLRAqv7GtbekW0C+YEL66ekLSkoYRdYUftVDCtlvY286Nu0R1r8Nr1KZ9g3N44exOVprQu8OXuFjc5Q/Hvcnuj6TwAAAP//jdrw9g==" + return "eJzsXU1z3DiSvftXZPiy3RGydveqw0a4bbft6Lalbdkzh4kJGkVmVWEEAmwAlFzz6ycAfhSLBMAvVDliRjp1W6r3HhLJRCITRL2CBzzcQIrF/gWApprhDbx8g8X+5QuADFUqaaGp4Dfwfy8AAMyvIBdZyfAFgNoLqZNU8C3d3cCWMGX+VSJDovAGdsT8DWpN+U7dwN9eKsVeXsHLvdbFy7+/ANhSZJm6scivgJMcWy3mRx8KgyJFWdT/4lBkfr6ZD32DVHBNKFeg9wg5aklT899EwxNKBJVKUmAGWylyePPu7sN1DdCVcSKFlUqjTDKqHtpfumQFpJkfD86pnZqfvpiuIPJIKCMbhtebg0Z18jeNLib4rveLgDTz87pBBYsKYmsNWKvu/flWyJzoGxgKaERqoQmLKvCLQYwjrlSYRdX2VWG2XFrf0/ZImN7H8LUB0nxvE48oCWOJ0kSXbns94OFJyGyeyW4rXKhwQ2ZrXYrmmO4xfVDXWIh0H2nuPpECHlEqKvgYrRQlz64fCSsxEnkLDhZ7moDYU3F/MgWnQvq+OeB2+6bbzyZ47AA/5JuuaVs+F3ZVq20RdAhJtluaXkskWTIrivgj07i4yj5gSEHvpSh3+6LUUKAEhanw+U6t9UlSjRcXa1kXqLWWFUVSGIfANNb8dk1IRaHmGe9seiorTRGUU1UwkuK1XV0jy2jAodgBL/ONJwq3GsTmH5jqWCvoQEUNP0mKJJoKpxCVEoZZsmWCaI9/FyhT5P3fzpU7lNCozHAnSYbZWSatAR+ZtFbDeSatVTFh0lopP27SWrn+SSt21xnR5OIRs9iB4Q3krsXu2m4ALi6NnG4QfOKsl19cnD5uDnzCTNJ/cV1luzHwyLIJD9qUDhPzj5Eezbudzayx+3dhCakoBw/UUvb7vXhSsBdPkBN+gGKngEgEymtRYjs69IG+yMlePUFqPN8TKrvelswdujdCMCR9VSPkH5UBhQFol5EjkWdhNcAj1GWeCJXFWiV6zmCQjSeMbfcaJWVxbjHGNUUlqHXPr3dBUZT/CFEfPwdFScxJUWCWFLtLK/vj3afXd3fv3nr1xdyxW6x+jtEmhTuZxCrZBbDaDe4G9XPh7j+8cNf1kyXFu7ro0BSrxbaqRQylLPG6raL9etDkWtGvlCHcH5TGHD6+vQICJad/lgg0Q67plqI0phmN5X+WQpZ5QnbL0ptmsf6zJLLMoQ/TToPgOSmcDH3zDyiGrvGtgus0FCiv3IAKDmQjSt2M2TBTLaS67ruNY15OFFOe5IIn9bwO/i5goMEIPlFOc8IaLeDD9HC7kk8Y8ZWpGsCJ3TooEl3KwSMNwckbUA8n0Px8a8CD03jc2jQT2nysP6EQmNTumAqTTyo93KJONetgfHctYiMuKEDYjw3qDYvpb2s8L3nnITzPTBrg4Cw2Tld1Iku7uefLZ7DcMJomJMv6a8HpkGbY8FMtsIIGA43KZayjCkn4Q5DeExm83F7AhtETBlYN2IvpMLbbHmMO5KU+tbUzyMCIR3SFGpzHQQV4qJOj0uiz0UCs2+Obn281acf5M9SEMpspEMYGg3T5/NSRdkdrxhL8w2k+4Rzz60rsOEfX9OcS80EoDYRnUAipjVXrWDJJGBc8nWamwMPqlPXZIIOr33iq4BxByhedGk7Xhm50pCdM7+w2zl3WCaavU8YVNY2Fk/UtMx/wa8qIds3WiaDfidI1UlotYZrmaD3Q+fG2CyuRuOPKdO43BiNMeay9xcynLVpwBbetlNykOBZ/djrNy9xR+4BZXvnZlhVMDLh77/d+uw9MbN64iu11oKje5Tv2mh0NyLmUKMkO6/qJGWi1/7VtUU8Dsiul00o+mxbDoZFPkVMJKVXgqZii5Ku7UD9kcnXx5lF98fQqoD/f3o7zPL4/zMSKAuuEePocR+L/q+1vzxNQ7FSyOVSVfy+5N8masp3oMgSD0t179V9VoXPxXsLb3Tkdz4xV+t4WXkfzaldP55Rz6v7hjRPpJPAKwWKFXjdWl83dyF7K50Nrm9a+XvA8wrfh1u6gdAir1tsKruPbO+QoCQvUsapu2Nxl13keDeYmaR+sXBdYm3IyTM3fJsubCe9qCEf6elqrXIT+//aj8NHdBqmLoOZ/Fh/eqxmGGJ1eS8ycrYILxsfb+7ezPeZSmwdPJ3Me1TFMmKEGudy9yqV0ZTHO6G5ELmWkfJwx0GVcSttADsmPu8CYXm3Rgk69NTtGZXeMs52bDv1xjll+PVK7oNp9KPkejcaF1bbcihulCc82h2h0AchLBwhnMXHpqAI7h8RTd42QyNbgk90ZlJDabnZsMXhpTmsgK8TEWaAZNeRgfF1j9pbQPnm0sniX86Kl8S7xaBrvTbBWUweQG/LdGab3fWd+u010obLVhzZu79+CwThTO52LzJO+OZ/isaXIwoWbk6mmj1jxzl2KMiwGGwqYt2VxIhzLFE9Id3vXFnNOfaICgZ+MFaRg9YGjMt1Xh2HtSaPKcbSkqEALIIyJ1OyBtbC/v71/+3OomhApUwnVBFNZqn0yYpDA2eYB7Zs/vt5/AA9g3M3XZ5GhD6rNa9ZV2BwhfSSwzhtB6IiBQXIvUzPKdocCQ4N42NgqZGJcdhWRLUVax/+JcvjtF79fN5RimBwvoDQokylzjDFKgzKFMmahXRUkxVHSR+LqqE3n+4vj8yNt1Vnu/sWFcbTYOk+3Beppdqr9Yb0rTKP7qimj/ySD0+xzKe8tW+lBO8ZWSfp7lBVrvYULrvX2L2wG87zcPy/3Ltbn5f7fcrmvot9WyB+x8B/Jf0AKcCR/TgYG4M/JQIjOt3zPpZyYDJR5TqQ7HVhShK7xJvXnlvRYqhcxo0WKyg/KBfGCVIc8kljT1RwaUcFpg5O86HFlVvRIwwytsdcHp8rQZGaIyilP1oap5vT+I5GUBLr9Ofm+not8n8TVGPZiMezksYn7xMyiXp9YOJ/YSenFqZDV67xTiHO179eCC5TbF33KmbVgg2HCKU/xTCXhYpdIJJl7aZjwqu5de+zxvRmaPfxFOSoVOgwhVGZ+6y5FL1qPertTYzvDQJWm6ezVp5m9xCxm/v3KtM5XfxqPshyfnNK8itTSMLpGOlXWBq6JOmWc/F7DnTXDuBVgxksNqchzqhNGNPL0kOQ+PJh6kL13csygQ40OlEOAoM0ZioIdzqXotQGfKahnI+c7TssVDW0UIHDbKLKigY24+7VX+5wLwQY+7nm+x4K1ENWVcNFjtNHoKDxMPnl1Z5QNAE7gl79na8E9x8ZSRpHrhIpEDg/DTgz3bywGfPzvWzAgfVcZC+brTp/X47+BXy55zrzHOuNE+cqz1y3xDzh33XJPOXPdf4y1RFybbBmMszy/Z2672wMDdad9cfk9Sl1TZCOvS0sh/PXmOSzOytMlS6gr4e0gAvDpnrJM4rpigyXxIp2lMRIoY5y3vYPfqVp5pP6dG+Ly3YtC0pzIQ0K2W8qpXnd48a4CAy/Y5XtC4y3KGHvAlf3I54D4HBBXUD0HxOeAOCkgtpls9e761DuYZl9SfYpf57m9Ou20C8CKNNZVi8deZn0nanOR1adb921xgbfMFt0eXr+3NcJ6HPqglr9+5L/9MkZf3WoWjbqqaI/T2vvKorHa1ug4KSNKJ2WROW4KqJgdtwSMjZfmCE975PBEFLiwA0v+Ct/6THIcG67SQmJVfrpmYhf1erjfxe54O1xfwJQL67ricqrSqOo+UZXGk6eUjqru/v5LPHFnv5VwrcDRh26B8xGlB89at7+SjffG5n8PAxZ7e02srYiUiuywex5h4kLnqYTOt4G9sNZkN/QS4aZl89Z5M3ykKSYpI4OrcWLwGowroIzhjjD7f0B5ysoMYZ9lV6BUBqjT68AaZ9x1qvFXXHDdukh18/ajYKXHZO2dnZdUVTluQNXCu1hjaDse7QjoK3YJX/gW+pBa9a/jhuo0bVbd40vtxdD+yfMlq+e6p783izVG99bPbhRcW7ZuI56tXZtYNxo/Lxb6hIRNmT6gvnAQ7PF6w6GjYhKXORAQhcquYC+UvrIVoEBcdFZuIk6LgfI179yFkhVGqokbYGDUWEBhQSSpX+tNRZ4Tty0C+/FKk+thnrp6WfBQsdJVr1k6D5ZxCBgs3Uw+kTOko6piVJoyVqEbN+BC//S/rw6oruB/XnHxszuWh0sviw1vdkX1uxT2EgWSUV5/f4j9dsOa1xvcPUW6pTNivzwMj4J8tLG/vuz4DXI2JFBt1ruSZbBBKAszS5l4cp/pPU86ZwxRIYNFbqUx+jAlkdsSvR/cQLhST0Ek8jpe2Q11tej3QtlggbXnGta+ke0C+YEL68esKSkYYRdYUbtVDC9ltY286FcEDLRM2/QanXG/R2pwEddEa13g6wBW2OgMxb+HzYmufwUAAP//mM8NxQ==" } diff --git a/metricbeat/module/ceph/mgr_cluster_disk/_meta/data.json b/metricbeat/module/ceph/mgr_cluster_disk/_meta/data.json index 44e14bb0a8d9..94c79adb4b49 100644 --- a/metricbeat/module/ceph/mgr_cluster_disk/_meta/data.json +++ b/metricbeat/module/ceph/mgr_cluster_disk/_meta/data.json @@ -1,154 +1,32 @@ { - "@timestamp": "2020-02-12T12:21:29.890Z", - "@metadata": { - "beat": "metricbeat", - "type": "_doc", - "version": "8.0.0" - }, - "ecs": { - "version": "1.4.0" + "@timestamp": "2017-10-12T08:05:34.853Z", + "agent": { + "hostname": "host.example.com", + "name": "host.example.com" }, "ceph": { "mgr_cluster_disk": { - "stats": { - "total_bytes": 10737418240, - "total_avail_bytes": 8342470656, - "total_used_bytes": 1321205760, - "total_used_raw_bytes": 2394947584, - "total_used_raw_ratio": 0.22304686903953552, - "num_osds": 1, - "num_per_pool_osds": 1 + "available": { + "bytes": 0 + }, + "total": { + "bytes": 0 }, - "pools": [ - { - "name": "rbd", - "id": 1, - "stats": { - "bytes_used": 0, - "percent_used": 0, - "max_avail": 7805599744, - "stored": 0, - "objects": 0, - "kb_used": 0 - } - }, - { - "name": "cephfs_data", - "id": 2, - "stats": { - "kb_used": 0, - "bytes_used": 0, - "percent_used": 0, - "max_avail": 7805599744, - "stored": 0, - "objects": 0 - } - }, - { - "name": "cephfs_metadata", - "id": 3, - "stats": { - "max_avail": 7805599744, - "stored": 2286, - "objects": 22, - "kb_used": 512, - "bytes_used": 524288, - "percent_used": 6.716368079651147e-05 - } - }, - { - "name": ".rgw.root", - "id": 4, - "stats": { - "max_avail": 7805599744, - "stored": 2398, - "objects": 6, - "kb_used": 384, - "bytes_used": 393216, - "percent_used": 5.037360460846685e-05 - } - }, - { - "name": "default.rgw.control", - "id": 5, - "stats": { - "kb_used": 0, - "bytes_used": 0, - "percent_used": 0, - "max_avail": 7805599744, - "stored": 0, - "objects": 8 - } - }, - { - "name": "default.rgw.meta", - "id": 6, - "stats": { - "stored": 736, - "objects": 5, - "kb_used": 256, - "bytes_used": 262144, - "percent_used": 3.3582968171685934e-05, - "max_avail": 7805599744 - } - }, - { - "stats": { - "max_avail": 7805599744, - "stored": 0, - "objects": 176, - "kb_used": 0, - "bytes_used": 0, - "percent_used": 0 - }, - "name": "default.rgw.log", - "id": 7 - }, - { - "name": "default.rgw.buckets.index", - "id": 8, - "stats": { - "kb_used": 0, - "bytes_used": 0, - "percent_used": 0, - "max_avail": 7805599744, - "stored": 0, - "objects": 1 - } - }, - { - "name": "scbench", - "id": 9, - "stats": { - "kb_used": 1425408, - "bytes_used": 1459617792, - "percent_used": 0.1575373411178589, - "max_avail": 7805599744, - "stored": 1459617792, - "objects": 339 - } - } - ] + "used": { + "bytes": 0 + } } }, - "agent": { - "hostname": "host", - "id": "993db589-f9d8-40f3-94cb-43b2b69fa9f4", - "version": "8.0.0", - "type": "metricbeat", - "ephemeral_id": "089f698c-74e3-44e1-ba4d-21010fc38433" - }, "event": { - "duration": 49779315, - "dataset": "http.json_namespace", + "dataset": "ceph.mgr_cluster_disk", + "duration": 115000, "module": "ceph" }, "metricset": { - "name": "mgr_cluster_disk", - "period": 60000 + "name": "mgr_cluster_disk" }, "service": { - "type": "ceph", - "address": "localhost:8003" + "address": "localhost:8003", + "type": "ceph" } } diff --git a/metricbeat/module/ceph/mgr_cluster_disk/_meta/fields.yml b/metricbeat/module/ceph/mgr_cluster_disk/_meta/fields.yml index a6068b974168..d24637a1fea0 100644 --- a/metricbeat/module/ceph/mgr_cluster_disk/_meta/fields.yml +++ b/metricbeat/module/ceph/mgr_cluster_disk/_meta/fields.yml @@ -1,65 +1,21 @@ - name: mgr_cluster_disk type: group - description: Disk metrics of Ceph cluster + description: > + mgr_cluster_disk release: beta fields: - - name: stats - type: nested + - name: available.bytes + type: long description: > - `stats` contains disk availability statistics. - fields: - - name: total_bytes - type: long - description: Total bytes of the cluster - - name: total_avail_bytes - type: long - description: Available bytes of the cluster - - name: total_used_bytes - type: long - description: Used bytes of the cluster - - name: total_used_raw_bytes - type: long - description: Used raw bytes of the cluster - - name: total_used_raw_ratio - type: scaled_float - description: Used raw bytes ratio - - name: num_osds - type: long - description: Number of OSDs in the cluster - - name: num_per_pool_osds - type: long - description: Number of OSDs per pool in the cluster - - name: pools - type: nested + Available bytes of the cluster + format: bytes + - name: total.bytes + type: long description: > - `pools` contains disk pool statistics. - fields: - - name: name - type: keyword - description: Pool name - - name: id - type: long - description: Pool ID - - name: stats - type: object - description: Pool statistics - fields: - - name: kb_used - type: long - description: Used space in kilobytes - - name: bytes_used - type: long - description: Used space in bytes - - name: percent_used - type: scaled_float - description: Used space in pct - format: percent - - name: max_avail - type: long - description: Maximum available space in bytes - - name: stored - type: long - description: Stored bytes - - name: objects - type: long - description: Number of stored objects + Total bytes of the cluster + format: bytes + - name: used.bytes + type: long + description: > + Used bytes of the cluster + format: bytes diff --git a/metricbeat/module/ceph/mgr_cluster_disk/manifest.yml b/metricbeat/module/ceph/mgr_cluster_disk/manifest.yml deleted file mode 100644 index 051eee3a2e1f..000000000000 --- a/metricbeat/module/ceph/mgr_cluster_disk/manifest.yml +++ /dev/null @@ -1,26 +0,0 @@ -default: true -input: - module: http - metricset: json - defaults: - method: POST - namespace: "json_namespace" - basepath: "/request" - headers: - Content-Type: application/json - query: - wait: "1" - body: '{"prefix": "df", "format": "json"}' - -processors: - - extract_array: - field: http.json_namespace.finished - mappings: - ceph.request: 0 - - decode_json_fields: - fields: ["ceph.request.outb"] - process_array: true - target: "ceph.mgr_cluster_disk" - - drop_fields: - ignore_missing: true - fields: ["http", "ceph.request", "ceph.mgr_cluster_disk.stats_by_class" ] diff --git a/metricbeat/module/ceph/mgr_cluster_disk/mgr_cluster_disk.go b/metricbeat/module/ceph/mgr_cluster_disk/mgr_cluster_disk.go new file mode 100644 index 000000000000..c5e441413abd --- /dev/null +++ b/metricbeat/module/ceph/mgr_cluster_disk/mgr_cluster_disk.go @@ -0,0 +1,90 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mgr_cluster_disk + +import ( + "fmt" + + "github.com/elastic/beats/metricbeat/helper" + "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/metricbeat/mb/parse" +) + +const ( + defaultScheme = "https" + defaultPath = "/request" + defaultQueryParams = "wait=1" + + cephPrefix = "df" +) + +var ( + hostParser = parse.URLHostParserBuilder{ + DefaultScheme: defaultScheme, + DefaultPath: defaultPath, + QueryParams: defaultQueryParams, + }.Build() +) + +func init() { + mb.Registry.MustAddMetricSet("ceph", "mgr_cluster_disk", New, + mb.WithHostParser(hostParser), + ) +} + +type MetricSet struct { + mb.BaseMetricSet + *helper.HTTP +} + +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + http, err := helper.NewHTTP(base) + if err != nil { + return nil, err + } + http.SetMethod("POST") + http.SetHeader("Content-Type", "application/json") + http.SetHeader("Accept", "application/json") + http.SetBody([]byte(fmt.Sprintf(`{"prefix": "%s", "format": "json"}`, cephPrefix))) + + return &MetricSet{ + base, + http, + }, nil +} + +// Fetch methods implements the data gathering and data conversion to the right +// format. It publishes the event which is then forwarded to the output. In case +// of an error set the Error field of mb.Event or simply call report.Error(). +func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { + content, err := m.HTTP.FetchContent() + if err != nil { + return err + } + + event, err := eventMapping(content) + if err != nil { + return err + } + + if reported := reporter.Event(mb.Event{MetricSetFields: event}); !reported { + m.Logger().Debug("error reporting event") + } + + return nil +} diff --git a/metricbeat/module/ceph/mgr_cluster_disk/mgr_cluster_disk_integration_test.go b/metricbeat/module/ceph/mgr_cluster_disk/mgr_cluster_disk_integration_test.go new file mode 100644 index 000000000000..22b56c5da477 --- /dev/null +++ b/metricbeat/module/ceph/mgr_cluster_disk/mgr_cluster_disk_integration_test.go @@ -0,0 +1,63 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build integration,linux + +package mgr_cluster_disk + +import ( + "encoding/json" + "fmt" + "net/http" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/elastic/beats/libbeat/tests/compose" + mbtest "github.com/elastic/beats/metricbeat/mb/testing" +) + +func TestData(t *testing.T) { + service := compose.EnsureUpWithTimeout(t, 120, "ceph-mgr") + + f := mbtest.NewReportingMetricSetV2Error(t, + getConfig(service.HostForPort(8003), getPassword(t, service.HostForPort(5000)))) + err := mbtest.WriteEventsReporterV2Error(f, t, "") + require.NoError(t, err) +} + +func getPassword(t *testing.T, host string) string { + response, err := http.Get(fmt.Sprintf("http://%s/restful-list-keys.json", host)) + require.NoError(t, err) + + defer response.Body.Close() + + accounts := map[string]string{} + json.NewDecoder(response.Body).Decode(&accounts) + return accounts["demo"] +} + +func getConfig(host, password string) map[string]interface{} { + return map[string]interface{}{ + "module": "ceph", + "metricsets": []string{"mgr_cluster_disk"}, + "hosts": []string{host}, + "username": "demo", + "password": password, + "ssl.verification_mode": "none", + } +} diff --git a/metricbeat/module/ceph/mgr_cluster_disk/mgr_cluster_disk_test.go b/metricbeat/module/ceph/mgr_cluster_disk/mgr_cluster_disk_test.go new file mode 100644 index 000000000000..14097a629f03 --- /dev/null +++ b/metricbeat/module/ceph/mgr_cluster_disk/mgr_cluster_disk_test.go @@ -0,0 +1,97 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mgr_cluster_disk + +import ( + "io/ioutil" + "net/http" + "net/http/httptest" + "path/filepath" + "testing" + + "github.com/elastic/beats/libbeat/common" + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + + "github.com/stretchr/testify/assert" +) + +func TestFetchEventContents(t *testing.T) { + absPath, err := filepath.Abs("../_meta/testdata/") + assert.NoError(t, err) + + response, err := ioutil.ReadFile(absPath + "/mgr_df_sample_response.json") + assert.NoError(t, err) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + w.Header().Set("Content-Type", "application/json;") + w.Write(response) + })) + defer server.Close() + + config := map[string]interface{}{ + "module": "ceph", + "metricsets": []string{"mgr_cluster_disk"}, + "hosts": []string{server.URL}, + } + + f := mbtest.NewReportingMetricSetV2Error(t, config) + events, errs := mbtest.ReportingFetchV2Error(f) + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) + } + assert.NotEmpty(t, events) + event := events[0].MetricSetFields + + t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event.StringToPrint()) + + used := event["used"].(common.MapStr) + assert.EqualValues(t, uint64(3801088), used["bytes"]) + + total := event["total"].(common.MapStr) + assert.EqualValues(t, uint64(10737418240), total["bytes"]) + + available := event["available"].(common.MapStr) + assert.EqualValues(t, uint64(9659875328), available["bytes"]) +} + +func TestFetchEventContents_Failed(t *testing.T) { + absPath, err := filepath.Abs("../_meta/testdata/") + assert.NoError(t, err) + + response, err := ioutil.ReadFile(absPath + "/mgr_failed_response.json") + assert.NoError(t, err) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + w.Header().Set("Content-Type", "application/json;") + w.Write([]byte(response)) + })) + defer server.Close() + + config := map[string]interface{}{ + "module": "ceph", + "metricsets": []string{"mgr_cluster_disk"}, + "hosts": []string{server.URL}, + } + + f := mbtest.NewReportingMetricSetV2Error(t, config) + events, errs := mbtest.ReportingFetchV2Error(f) + assert.Empty(t, events) + assert.NotEmpty(t, errs) +} diff --git a/metricbeat/module/ceph/mgr_cluster_disk/mgr_data.go b/metricbeat/module/ceph/mgr_cluster_disk/mgr_data.go new file mode 100644 index 000000000000..87faae697df0 --- /dev/null +++ b/metricbeat/module/ceph/mgr_cluster_disk/mgr_data.go @@ -0,0 +1,84 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mgr_cluster_disk + +import ( + "encoding/json" + "fmt" + + "github.com/pkg/errors" + + "github.com/elastic/beats/libbeat/common" +) + +type Request struct { + HasFailed bool `json:"has_failed"` + Finished []Result `json:"finished"` + Failed []Result `json:"failed"` +} + +type Result struct { + Command string `json:"command"` + Outb string `json:"outb"` + Outs string `json:"outs"` +} + +type DfResponse struct { + Stats struct { + TotalBytes uint64 `json:"total_bytes"` + TotalAvailableBytes uint64 `json:"total_avail_bytes"` + TotalUsedBytes uint64 `json:"total_used_bytes"` + } `json:"stats"` +} + +func eventMapping(content []byte) (common.MapStr, error) { + var request Request + err := json.Unmarshal(content, &request) + if err != nil { + return nil, errors.Wrap(err, "could not get request data") + } + + if request.HasFailed { + if len(request.Failed) != 1 { + return nil, errors.New("expected single failed command") + } + return nil, fmt.Errorf("%s: %s", request.Failed[0].Outs, request.Failed[0].Command) + } + + if len(request.Finished) != 1 { + return nil, errors.New("expected single finished command") + } + + var response DfResponse + err = json.Unmarshal([]byte(request.Finished[0].Outb), &response) + if err != nil { + return nil, errors.Wrap(err, "could not get response data") + } + + return common.MapStr{ + "used": common.MapStr{ + "bytes": response.Stats.TotalUsedBytes, + }, + "total": common.MapStr{ + "bytes": response.Stats.TotalBytes, + }, + "available": common.MapStr{ + "bytes": response.Stats.TotalAvailableBytes, + }, + }, nil +} diff --git a/metricbeat/module/ceph/module.yml b/metricbeat/module/ceph/module.yml deleted file mode 100644 index 57b6c837edb5..000000000000 --- a/metricbeat/module/ceph/module.yml +++ /dev/null @@ -1,8 +0,0 @@ -name: ceph -metricsets: - - mgr_cluster_health - - mgr_cluster_disk - - mgr_osd_disk - - mgr_osd_pool_stats - - mgr_osd_perf - - mgr_osd_tree diff --git a/metricbeat/module/ceph/test_ceph_mgr.py b/metricbeat/module/ceph/test_ceph_mgr.py index 189c180bba45..c5bebfab691e 100644 --- a/metricbeat/module/ceph/test_ceph_mgr.py +++ b/metricbeat/module/ceph/test_ceph_mgr.py @@ -36,11 +36,6 @@ def get_password(self): @parameterized.expand([ "mgr_cluster_disk", - "mgr_cluster_health", - "mgr_osd_disk", - "mgr_osd_perf", - "mgr_osd_pool_stats", - "mgr_osd_tree", ]) @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") def test_ceph_mgr(self, metricset): @@ -57,5 +52,4 @@ def test_ceph_mgr(self, metricset): output = self.read_output_json() for evt in output: assert 'ceph' in evt - # Fails due to nested objects: - # assert self.assert_fields_are_documented(evt) + self.assert_fields_are_documented(evt) From 4939ca2ede11ee8e899e541c0c40e59a16992331 Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Thu, 13 Feb 2020 14:50:46 +0100 Subject: [PATCH 30/41] mgr_cluster_health as standard metricset --- metricbeat/docs/fields.asciidoc | 413 +----------------- metricbeat/include/list_common.go | 1 + .../testdata/mgr_status_sample_response.json | 17 + .../mgr_time_sync_status_sample_response.json | 17 + metricbeat/module/ceph/fields.go | 2 +- metricbeat/module/ceph/mgr/event_mapping.go | 65 +++ .../module/ceph/mgr_cluster_disk/mgr_data.go | 35 +- .../ceph/mgr_cluster_health/_meta/data.json | 132 +----- .../ceph/mgr_cluster_health/_meta/fields.yml | 208 +-------- .../ceph/mgr_cluster_health/manifest.yml | 43 -- .../mgr_cluster_health/mgr_cluster_health.go | 97 ++++ .../mgr_cluster_health_integration_test.go | 63 +++ .../mgr_cluster_health_test.go | 116 +++++ .../ceph/mgr_cluster_health/mgr_data.go | 64 +++ 14 files changed, 483 insertions(+), 790 deletions(-) create mode 100644 metricbeat/module/ceph/_meta/testdata/mgr_status_sample_response.json create mode 100644 metricbeat/module/ceph/_meta/testdata/mgr_time_sync_status_sample_response.json create mode 100644 metricbeat/module/ceph/mgr/event_mapping.go delete mode 100644 metricbeat/module/ceph/mgr_cluster_health/manifest.yml create mode 100644 metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health.go create mode 100644 metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health_integration_test.go create mode 100644 metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health_test.go create mode 100644 metricbeat/module/ceph/mgr_cluster_health/mgr_data.go diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 0aa9071b62ed..c83b57b74e1f 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -3978,445 +3978,50 @@ format: bytes [float] === mgr_cluster_health -Status metrics of Ceph cluster +mgr_cluster_health -*`ceph.mgr_cluster_health.fsid`*:: -+ --- -File System ID, a unique identifier for the cluster - -type: keyword - --- - -*`ceph.mgr_cluster_health.quorum_age`*:: -+ --- -Cluster quarum age - -type: long - --- - -[float] -=== monmap - -`monmap` contains information about cluster monitors. - - - -*`ceph.mgr_cluster_health.monmap.min_mon_release`*:: -+ --- -Minimal monitor release - -type: long - --- - -*`ceph.mgr_cluster_health.monmap.min_mon_release_name`*:: -+ --- -Minimal monitor release name - -type: keyword - --- - -[float] -=== features - -`features` contains information about available cluster features. - - - -*`ceph.mgr_cluster_health.monmap.features.persistent`*:: -+ --- -Persistent feature - -type: keyword - --- - -*`ceph.mgr_cluster_health.monmap.features.optional`*:: -+ --- -Optional feature - -type: keyword --- - -[float] -=== mons - -`mons` contains information about monitor configurations. - - - -*`ceph.mgr_cluster_health.monmap.mons.public_addr`*:: +*`ceph.mgr_cluster_health.overall_status`*:: + -- -Monitor public address - -type: keyword - --- - -*`ceph.mgr_cluster_health.monmap.mons.rank`*:: -+ --- -Monitor rank - -type: long - --- - -*`ceph.mgr_cluster_health.monmap.mons.name`*:: -+ --- -Monitor name - -type: keyword - --- - -[float] -=== public_addrs - -Monitor public addresses - - -*`ceph.mgr_cluster_health.monmap.mons.public_addrs.addrvec`*:: -+ --- -`addrvec` contains details of all public addresses. - - -type: nested - --- - -*`ceph.mgr_cluster_health.monmap.mons.addr`*:: -+ --- -Monitor address - -type: keyword - --- - -*`ceph.mgr_cluster_health.monmap.epoch`*:: -+ --- -Epoch version - -type: long - --- +Overall status of the cluster -*`ceph.mgr_cluster_health.monmap.fsid`*:: -+ --- -File System ID, a unique identifier for the cluster type: keyword -- -*`ceph.mgr_cluster_health.monmap.modified`*:: +*`ceph.mgr_cluster_health.timechecks.epoch`*:: + -- -Last modification time and date - -type: date - --- - -*`ceph.mgr_cluster_health.monmap.created`*:: -+ --- -Creation time and date - -type: date - --- - -[float] -=== pgmap - -`pgmap` contains information about placement groups. - - - -*`ceph.mgr_cluster_health.pgmap.num_pgs`*:: -+ --- -Number of PGs - -type: long - --- - -*`ceph.mgr_cluster_health.pgmap.bytes_avail`*:: -+ --- -Available bytes - -type: long - --- - -*`ceph.mgr_cluster_health.pgmap.read_bytes_sec`*:: -+ --- -Average number of bytes read per second - -type: long - --- - -*`ceph.mgr_cluster_health.pgmap.write_bytes_sec`*:: -+ --- -Average number of bytes written per second - -type: long - --- - -*`ceph.mgr_cluster_health.pgmap.bytes_used`*:: -+ --- -Used bytes - -type: long - --- - -*`ceph.mgr_cluster_health.pgmap.bytes_total`*:: -+ --- -Total bytes - -type: long - --- - -*`ceph.mgr_cluster_health.pgmap.read_op_per_sec`*:: -+ --- -Read operations per second - -type: long - --- - -*`ceph.mgr_cluster_health.pgmap.write_op_per_sec`*:: -+ --- -Write operations per second - -type: long - --- - -*`ceph.mgr_cluster_health.pgmap.pgs_by_state`*:: -+ --- -`pgs_by_state` contains information about PGs' states. - - -type: nested - --- - -*`ceph.mgr_cluster_health.pgmap.num_pools`*:: -+ --- -Number of pools - -type: long - --- - -*`ceph.mgr_cluster_health.pgmap.num_objects`*:: -+ --- -Number of objects - -type: long - --- +Map version -*`ceph.mgr_cluster_health.pgmap.data_bytes`*:: -+ --- -Data bytes type: long -- -[float] -=== health - -`health` contains general information about cluster status. - - - -*`ceph.mgr_cluster_health.health.status`*:: +*`ceph.mgr_cluster_health.timechecks.round.value`*:: + -- -Health status - -type: keyword - --- +timecheck round -*`ceph.mgr_cluster_health.election_epoch`*:: -+ --- -Election epoch type: long -- -*`ceph.mgr_cluster_health.quorum`*:: +*`ceph.mgr_cluster_health.timechecks.round.status`*:: + -- -Quorum ID - -type: long - --- +Status of the round -*`ceph.mgr_cluster_health.quorum_names`*:: -+ --- -Quorum names type: keyword -- -[float] -=== osdmap - -`osdmap` contains information about OSDs. - - - -*`ceph.mgr_cluster_health.osdmap.epoch`*:: -+ --- -Epoch version - -type: long - --- - -*`ceph.mgr_cluster_health.osdmap.num_osds`*:: -+ --- -Number of OSDs - -type: long - --- - -*`ceph.mgr_cluster_health.osdmap.num_up_osds`*:: -+ --- -Number of up OSDs - -type: long - --- - -*`ceph.mgr_cluster_health.osdmap.num_in_osds`*:: -+ --- -Number of in OSDs - -type: long - --- - -*`ceph.mgr_cluster_health.osdmap.num_remapped_pgs`*:: -+ --- -Number of remapped OSDs - -type: long - --- - -[float] -=== fsmap - -`fsmap` contains information about file systems. - - - -*`ceph.mgr_cluster_health.fsmap.in`*:: -+ --- -File system in - -type: long - --- - -*`ceph.mgr_cluster_health.fsmap.max`*:: -+ --- -File system max - -type: long - --- - -*`ceph.mgr_cluster_health.fsmap.up:standby`*:: -+ --- -File system up:standby - -type: long - --- - -*`ceph.mgr_cluster_health.fsmap.epoch`*:: -+ --- -Epoch version - -type: long - --- - -*`ceph.mgr_cluster_health.fsmap.up`*:: -+ --- -File system up - -type: long - --- - -*`ceph.mgr_cluster_health.fsmap.by_rank`*:: -+ --- -`by_rank` contains information about file systems sorted by rank. - - -type: nested - --- - [float] === mgr_osd_disk diff --git a/metricbeat/include/list_common.go b/metricbeat/include/list_common.go index 665fddd77d3b..9b6383510744 100644 --- a/metricbeat/include/list_common.go +++ b/metricbeat/include/list_common.go @@ -33,6 +33,7 @@ import ( _ "github.com/elastic/beats/metricbeat/module/ceph/cluster_health" _ "github.com/elastic/beats/metricbeat/module/ceph/cluster_status" _ "github.com/elastic/beats/metricbeat/module/ceph/mgr_cluster_disk" + _ "github.com/elastic/beats/metricbeat/module/ceph/mgr_cluster_health" _ "github.com/elastic/beats/metricbeat/module/ceph/monitor_health" _ "github.com/elastic/beats/metricbeat/module/ceph/osd_df" _ "github.com/elastic/beats/metricbeat/module/ceph/osd_tree" diff --git a/metricbeat/module/ceph/_meta/testdata/mgr_status_sample_response.json b/metricbeat/module/ceph/_meta/testdata/mgr_status_sample_response.json new file mode 100644 index 000000000000..258dbe9239a4 --- /dev/null +++ b/metricbeat/module/ceph/_meta/testdata/mgr_status_sample_response.json @@ -0,0 +1,17 @@ +{ + "failed": [], + "finished": [ + { + "command": "status format=json-pretty", + "outb": "{\n \"fsid\": \"aa90baeb-6f42-4c6d-8728-882907da4655\",\n \"health\": {\n \"checks\": {},\n \"status\": \"HEALTH_OK\"\n },\n \"election_epoch\": 3,\n \"quorum\": [\n 0\n ],\n \"quorum_names\": [\n \"42697164830d\"\n ],\n \"quorum_age\": 1200,\n \"monmap\": {\n \"epoch\": 1,\n \"fsid\": \"aa90baeb-6f42-4c6d-8728-882907da4655\",\n \"modified\": \"2020-02-13 12:37:02.294475\",\n \"created\": \"2020-02-13 12:37:02.294475\",\n \"min_mon_release\": 14,\n \"min_mon_release_name\": \"nautilus\",\n \"features\": {\n \"persistent\": [\n \"kraken\",\n \"luminous\",\n \"mimic\",\n \"osdmap-prune\",\n \"nautilus\"\n ],\n \"optional\": []\n },\n \"mons\": [\n {\n \"rank\": 0,\n \"name\": \"42697164830d\",\n \"public_addrs\": {\n \"addrvec\": [\n {\n \"type\": \"v2\",\n \"addr\": \"172.30.0.2:3300\",\n \"nonce\": 0\n }\n ]\n },\n \"addr\": \"172.30.0.2:3300/0\",\n \"public_addr\": \"172.30.0.2:3300/0\"\n }\n ]\n },\n \"osdmap\": {\n \"osdmap\": {\n \"epoch\": 18,\n \"num_osds\": 1,\n \"num_up_osds\": 1,\n \"num_in_osds\": 1,\n \"num_remapped_pgs\": 0\n }\n },\n \"pgmap\": {\n \"pgs_by_state\": [\n {\n \"state_name\": \"active+clean\",\n \"count\": 64\n }\n ],\n \"num_pgs\": 64,\n \"num_pools\": 8,\n \"num_objects\": 218,\n \"data_bytes\": 5420,\n \"bytes_used\": 1077542912,\n \"bytes_avail\": 9659875328,\n \"bytes_total\": 10737418240\n },\n \"fsmap\": {\n \"epoch\": 5,\n \"id\": 1,\n \"up\": 1,\n \"in\": 1,\n \"max\": 1,\n \"by_rank\": [\n {\n \"filesystem_id\": 1,\n \"rank\": 0,\n \"name\": \"demo\",\n \"status\": \"up:active\",\n \"gid\": 4129\n }\n ],\n \"up:standby\": 0\n },\n \"mgrmap\": {\n \"epoch\": 5,\n \"active_gid\": 4100,\n \"active_name\": \"42697164830d\",\n \"active_addrs\": {\n \"addrvec\": [\n {\n \"type\": \"v2\",\n \"addr\": \"172.30.0.2:6800\",\n \"nonce\": 206\n },\n {\n \"type\": \"v1\",\n \"addr\": \"172.30.0.2:6801\",\n \"nonce\": 206\n }\n ]\n },\n \"active_addr\": \"172.30.0.2:6801/206\",\n \"active_change\": \"2020-02-13 12:37:04.535336\",\n \"available\": true,\n \"standbys\": [],\n \"modules\": [\n \"iostat\",\n \"restful\"\n ],\n \"available_modules\": [\n {\n \"name\": \"ansible\",\n \"can_run\": true,\n \"error_string\": \"\",\n \"module_options\": {\n \"password\": {\n \"name\": \"password\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"server_url\": {\n \"name\": \"server_url\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"username\": {\n \"name\": \"username\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"verify_server\": {\n \"name\": \"verify_server\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n }\n }\n },\n {\n \"name\": \"balancer\",\n \"can_run\": true,\n \"error_string\": \"\",\n \"module_options\": {\n \"active\": {\n \"name\": \"active\",\n \"type\": \"bool\",\n \"level\": \"advanced\",\n \"flags\": 1,\n \"default_value\": \"False\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"automatically balance PGs across cluster\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"begin_time\": {\n \"name\": \"begin_time\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 1,\n \"default_value\": \"0000\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"beginning time of day to automatically balance\",\n \"long_desc\": \"This is a time of day in the format HHMM.\",\n \"tags\": [],\n \"see_also\": []\n },\n \"begin_weekday\": {\n \"name\": \"begin_weekday\",\n \"type\": \"uint\",\n \"level\": \"advanced\",\n \"flags\": 1,\n \"default_value\": \"0\",\n \"min\": \"0\",\n \"max\": \"7\",\n \"enum_allowed\": [],\n \"desc\": \"Restrict automatic balancing to this day of the week or later\",\n \"long_desc\": \"0 or 7 = Sunday, 1 = Monday, etc.\",\n \"tags\": [],\n \"see_also\": []\n },\n \"crush_compat_max_iterations\": {\n \"name\": \"crush_compat_max_iterations\",\n \"type\": \"uint\",\n \"level\": \"advanced\",\n \"flags\": 1,\n \"default_value\": \"25\",\n \"min\": \"1\",\n \"max\": \"250\",\n \"enum_allowed\": [],\n \"desc\": \"maximum number of iterations to attempt optimization\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"crush_compat_metrics\": {\n \"name\": \"crush_compat_metrics\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 1,\n \"default_value\": \"pgs,objects,bytes\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"metrics with which to calculate OSD utilization\",\n \"long_desc\": \"Value is a list of one or more of \\\"pgs\\\", \\\"objects\\\", or \\\"bytes\\\", and indicates which metrics to use to balance utilization.\",\n \"tags\": [],\n \"see_also\": []\n },\n \"crush_compat_step\": {\n \"name\": \"crush_compat_step\",\n \"type\": \"float\",\n \"level\": \"advanced\",\n \"flags\": 1,\n \"default_value\": \"0.5\",\n \"min\": \"0.001\",\n \"max\": \"0.999\",\n \"enum_allowed\": [],\n \"desc\": \"aggressiveness of optimization\",\n \"long_desc\": \".99 is very aggressive, .01 is less aggressive\",\n \"tags\": [],\n \"see_also\": []\n },\n \"end_time\": {\n \"name\": \"end_time\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 1,\n \"default_value\": \"2400\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"ending time of day to automatically balance\",\n \"long_desc\": \"This is a time of day in the format HHMM.\",\n \"tags\": [],\n \"see_also\": []\n },\n \"end_weekday\": {\n \"name\": \"end_weekday\",\n \"type\": \"uint\",\n \"level\": \"advanced\",\n \"flags\": 1,\n \"default_value\": \"7\",\n \"min\": \"0\",\n \"max\": \"7\",\n \"enum_allowed\": [],\n \"desc\": \"Restrict automatic balancing to days of the week earlier than this\",\n \"long_desc\": \"0 or 7 = Sunday, 1 = Monday, etc.\",\n \"tags\": [],\n \"see_also\": []\n },\n \"min_score\": {\n \"name\": \"min_score\",\n \"type\": \"float\",\n \"level\": \"advanced\",\n \"flags\": 1,\n \"default_value\": \"0\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"minimum score, below which no optimization is attempted\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"mode\": {\n \"name\": \"mode\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 1,\n \"default_value\": \"none\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [\n \"crush-compat\",\n \"none\",\n \"upmap\"\n ],\n \"desc\": \"Balancer mode\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"pool_ids\": {\n \"name\": \"pool_ids\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 1,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"pools which the automatic balancing will be limited to\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"sleep_interval\": {\n \"name\": \"sleep_interval\",\n \"type\": \"secs\",\n \"level\": \"advanced\",\n \"flags\": 1,\n \"default_value\": \"60\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"how frequently to wake up and attempt optimization\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"upmap_max_deviation\": {\n \"name\": \"upmap_max_deviation\",\n \"type\": \"float\",\n \"level\": \"advanced\",\n \"flags\": 1,\n \"default_value\": \"0.01\",\n \"min\": \"0\",\n \"max\": \"1\",\n \"enum_allowed\": [],\n \"desc\": \"deviation below which no optimization is attempted\",\n \"long_desc\": \"If the ratio between the fullest and least-full OSD is below this value then we stop trying to optimize placement.\",\n \"tags\": [],\n \"see_also\": []\n },\n \"upmap_max_iterations\": {\n \"name\": \"upmap_max_iterations\",\n \"type\": \"uint\",\n \"level\": \"advanced\",\n \"flags\": 1,\n \"default_value\": \"10\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"maximum upmap optimization iterations\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n }\n }\n },\n {\n \"name\": \"crash\",\n \"can_run\": true,\n \"error_string\": \"\",\n \"module_options\": {\n \"retain_interval\": {\n \"name\": \"retain_interval\",\n \"type\": \"secs\",\n \"level\": \"advanced\",\n \"flags\": 1,\n \"default_value\": \"31536000\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"how long to retain crashes before pruning them\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"warn_recent_interval\": {\n \"name\": \"warn_recent_interval\",\n \"type\": \"secs\",\n \"level\": \"advanced\",\n \"flags\": 1,\n \"default_value\": \"1209600\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"time interval in which to warn about recent crashes\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n }\n }\n },\n {\n \"name\": \"dashboard\",\n \"can_run\": true,\n \"error_string\": \"\",\n \"module_options\": {\n \"ALERTMANAGER_API_HOST\": {\n \"name\": \"ALERTMANAGER_API_HOST\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"AUDIT_API_ENABLED\": {\n \"name\": \"AUDIT_API_ENABLED\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"False\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"AUDIT_API_LOG_PAYLOAD\": {\n \"name\": \"AUDIT_API_LOG_PAYLOAD\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"True\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"ENABLE_BROWSABLE_API\": {\n \"name\": \"ENABLE_BROWSABLE_API\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"True\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"FEATURE_TOGGLE_cephfs\": {\n \"name\": \"FEATURE_TOGGLE_cephfs\",\n \"type\": \"bool\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"True\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"FEATURE_TOGGLE_iscsi\": {\n \"name\": \"FEATURE_TOGGLE_iscsi\",\n \"type\": \"bool\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"True\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"FEATURE_TOGGLE_mirroring\": {\n \"name\": \"FEATURE_TOGGLE_mirroring\",\n \"type\": \"bool\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"True\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"FEATURE_TOGGLE_rbd\": {\n \"name\": \"FEATURE_TOGGLE_rbd\",\n \"type\": \"bool\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"True\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"FEATURE_TOGGLE_rgw\": {\n \"name\": \"FEATURE_TOGGLE_rgw\",\n \"type\": \"bool\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"True\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE\": {\n \"name\": \"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"GRAFANA_API_PASSWORD\": {\n \"name\": \"GRAFANA_API_PASSWORD\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"admin\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"GRAFANA_API_URL\": {\n \"name\": \"GRAFANA_API_URL\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"GRAFANA_API_USERNAME\": {\n \"name\": \"GRAFANA_API_USERNAME\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"admin\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"GRAFANA_UPDATE_DASHBOARDS\": {\n \"name\": \"GRAFANA_UPDATE_DASHBOARDS\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"False\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"ISCSI_API_SSL_VERIFICATION\": {\n \"name\": \"ISCSI_API_SSL_VERIFICATION\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"True\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"PROMETHEUS_API_HOST\": {\n \"name\": \"PROMETHEUS_API_HOST\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"REST_REQUESTS_TIMEOUT\": {\n \"name\": \"REST_REQUESTS_TIMEOUT\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"45\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"RGW_API_ACCESS_KEY\": {\n \"name\": \"RGW_API_ACCESS_KEY\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"RGW_API_ADMIN_RESOURCE\": {\n \"name\": \"RGW_API_ADMIN_RESOURCE\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"admin\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"RGW_API_HOST\": {\n \"name\": \"RGW_API_HOST\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"RGW_API_PORT\": {\n \"name\": \"RGW_API_PORT\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"80\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"RGW_API_SCHEME\": {\n \"name\": \"RGW_API_SCHEME\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"http\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"RGW_API_SECRET_KEY\": {\n \"name\": \"RGW_API_SECRET_KEY\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"RGW_API_SSL_VERIFY\": {\n \"name\": \"RGW_API_SSL_VERIFY\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"True\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"RGW_API_USER_ID\": {\n \"name\": \"RGW_API_USER_ID\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"crt_file\": {\n \"name\": \"crt_file\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"jwt_token_ttl\": {\n \"name\": \"jwt_token_ttl\",\n \"type\": \"int\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"28800\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"key_file\": {\n \"name\": \"key_file\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"password\": {\n \"name\": \"password\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"server_addr\": {\n \"name\": \"server_addr\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"0.0.0.0\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"server_port\": {\n \"name\": \"server_port\",\n \"type\": \"int\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"8080\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"ssl\": {\n \"name\": \"ssl\",\n \"type\": \"bool\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"True\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"ssl_server_port\": {\n \"name\": \"ssl_server_port\",\n \"type\": \"int\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"8443\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"standby_behaviour\": {\n \"name\": \"standby_behaviour\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"redirect\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [\n \"error\",\n \"redirect\"\n ],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"standby_error_status_code\": {\n \"name\": \"standby_error_status_code\",\n \"type\": \"int\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"500\",\n \"min\": \"400\",\n \"max\": \"599\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"url_prefix\": {\n \"name\": \"url_prefix\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"username\": {\n \"name\": \"username\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n }\n }\n },\n {\n \"name\": \"deepsea\",\n \"can_run\": true,\n \"error_string\": \"\",\n \"module_options\": {\n \"salt_api_eauth\": {\n \"name\": \"salt_api_eauth\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"sharedsecret\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"salt_api_password\": {\n \"name\": \"salt_api_password\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"salt_api_url\": {\n \"name\": \"salt_api_url\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"salt_api_username\": {\n \"name\": \"salt_api_username\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n }\n }\n },\n {\n \"name\": \"devicehealth\",\n \"can_run\": true,\n \"error_string\": \"\",\n \"module_options\": {\n \"enable_monitoring\": {\n \"name\": \"enable_monitoring\",\n \"type\": \"bool\",\n \"level\": \"advanced\",\n \"flags\": 1,\n \"default_value\": \"False\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"monitor device health metrics\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"mark_out_threshold\": {\n \"name\": \"mark_out_threshold\",\n \"type\": \"secs\",\n \"level\": \"advanced\",\n \"flags\": 1,\n \"default_value\": \"2419200\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"automatically mark OSD if it may fail before this long\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"pool_name\": {\n \"name\": \"pool_name\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 1,\n \"default_value\": \"device_health_metrics\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"name of pool in which to store device health metrics\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"retention_period\": {\n \"name\": \"retention_period\",\n \"type\": \"secs\",\n \"level\": \"advanced\",\n \"flags\": 1,\n \"default_value\": \"15552000\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"how long to retain device health metrics\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"scrape_frequency\": {\n \"name\": \"scrape_frequency\",\n \"type\": \"secs\",\n \"level\": \"advanced\",\n \"flags\": 1,\n \"default_value\": \"86400\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"how frequently to scrape device health metrics\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"self_heal\": {\n \"name\": \"self_heal\",\n \"type\": \"bool\",\n \"level\": \"advanced\",\n \"flags\": 1,\n \"default_value\": \"True\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"preemptively heal cluster around devices that may fail\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"sleep_interval\": {\n \"name\": \"sleep_interval\",\n \"type\": \"secs\",\n \"level\": \"advanced\",\n \"flags\": 1,\n \"default_value\": \"600\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"how frequently to wake up and check device health\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"warn_threshold\": {\n \"name\": \"warn_threshold\",\n \"type\": \"secs\",\n \"level\": \"advanced\",\n \"flags\": 1,\n \"default_value\": \"7257600\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"raise health warning if OSD may fail before this long\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n }\n }\n },\n {\n \"name\": \"diskprediction_local\",\n \"can_run\": true,\n \"error_string\": \"\",\n \"module_options\": {\n \"predict_interval\": {\n \"name\": \"predict_interval\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"86400\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"sleep_interval\": {\n \"name\": \"sleep_interval\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"600\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n }\n }\n },\n {\n \"name\": \"influx\",\n \"can_run\": false,\n \"error_string\": \"influxdb python module not found\",\n \"module_options\": {\n \"batch_size\": {\n \"name\": \"batch_size\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"5000\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"database\": {\n \"name\": \"database\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"ceph\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"hostname\": {\n \"name\": \"hostname\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"None\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"interval\": {\n \"name\": \"interval\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"30\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"password\": {\n \"name\": \"password\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"None\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"port\": {\n \"name\": \"port\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"8086\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"ssl\": {\n \"name\": \"ssl\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"false\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"threads\": {\n \"name\": \"threads\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"5\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"username\": {\n \"name\": \"username\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"None\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"verify_ssl\": {\n \"name\": \"verify_ssl\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"true\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n }\n }\n },\n {\n \"name\": \"insights\",\n \"can_run\": true,\n \"error_string\": \"\",\n \"module_options\": {}\n },\n {\n \"name\": \"iostat\",\n \"can_run\": true,\n \"error_string\": \"\",\n \"module_options\": {}\n },\n {\n \"name\": \"k8sevents\",\n \"can_run\": true,\n \"error_string\": \"\",\n \"module_options\": {\n \"ceph_event_retention_days\": {\n \"name\": \"ceph_event_retention_days\",\n \"type\": \"int\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"7\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"Days to hold ceph event information within local cache\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"config_check_secs\": {\n \"name\": \"config_check_secs\",\n \"type\": \"int\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"10\",\n \"min\": \"10\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"interval (secs) to check for cluster configuration changes\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n }\n }\n },\n {\n \"name\": \"localpool\",\n \"can_run\": true,\n \"error_string\": \"\",\n \"module_options\": {\n \"failure_domain\": {\n \"name\": \"failure_domain\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 1,\n \"default_value\": \"host\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"failure domain for any created local pool\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"min_size\": {\n \"name\": \"min_size\",\n \"type\": \"int\",\n \"level\": \"advanced\",\n \"flags\": 1,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"default min_size for any created local pool\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"num_rep\": {\n \"name\": \"num_rep\",\n \"type\": \"int\",\n \"level\": \"advanced\",\n \"flags\": 1,\n \"default_value\": \"3\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"default replica count for any created local pool\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"pg_num\": {\n \"name\": \"pg_num\",\n \"type\": \"int\",\n \"level\": \"advanced\",\n \"flags\": 1,\n \"default_value\": \"128\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"default pg_num for any created local pool\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"prefix\": {\n \"name\": \"prefix\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 1,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"name prefix for any created local pool\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"subtree\": {\n \"name\": \"subtree\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 1,\n \"default_value\": \"rack\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"CRUSH level for which to create a local pool\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n }\n }\n },\n {\n \"name\": \"orchestrator_cli\",\n \"can_run\": true,\n \"error_string\": \"\",\n \"module_options\": {\n \"orchestrator\": {\n \"name\": \"orchestrator\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n }\n }\n },\n {\n \"name\": \"pg_autoscaler\",\n \"can_run\": true,\n \"error_string\": \"\",\n \"module_options\": {\n \"sleep_interval\": {\n \"name\": \"sleep_interval\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"60\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n }\n }\n },\n {\n \"name\": \"progress\",\n \"can_run\": true,\n \"error_string\": \"\",\n \"module_options\": {\n \"max_completed_events\": {\n \"name\": \"max_completed_events\",\n \"type\": \"int\",\n \"level\": \"advanced\",\n \"flags\": 1,\n \"default_value\": \"50\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"number of past completed events to remember\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"persist_interval\": {\n \"name\": \"persist_interval\",\n \"type\": \"secs\",\n \"level\": \"advanced\",\n \"flags\": 1,\n \"default_value\": \"5\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"how frequently to persist completed events\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n }\n }\n },\n {\n \"name\": \"prometheus\",\n \"can_run\": true,\n \"error_string\": \"\",\n \"module_options\": {\n \"rbd_stats_pools\": {\n \"name\": \"rbd_stats_pools\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"rbd_stats_pools_refresh_interval\": {\n \"name\": \"rbd_stats_pools_refresh_interval\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"scrape_interval\": {\n \"name\": \"scrape_interval\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"server_addr\": {\n \"name\": \"server_addr\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"server_port\": {\n \"name\": \"server_port\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n }\n }\n },\n {\n \"name\": \"rbd_support\",\n \"can_run\": true,\n \"error_string\": \"\",\n \"module_options\": {}\n },\n {\n \"name\": \"restful\",\n \"can_run\": true,\n \"error_string\": \"\",\n \"module_options\": {\n \"key_file\": {\n \"name\": \"key_file\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"server_addr\": {\n \"name\": \"server_addr\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"server_port\": {\n \"name\": \"server_port\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n }\n }\n },\n {\n \"name\": \"rook\",\n \"can_run\": true,\n \"error_string\": \"\",\n \"module_options\": {}\n },\n {\n \"name\": \"selftest\",\n \"can_run\": true,\n \"error_string\": \"\",\n \"module_options\": {\n \"roption1\": {\n \"name\": \"roption1\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"roption2\": {\n \"name\": \"roption2\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"xyz\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"rwoption1\": {\n \"name\": \"rwoption1\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"rwoption2\": {\n \"name\": \"rwoption2\",\n \"type\": \"int\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"rwoption3\": {\n \"name\": \"rwoption3\",\n \"type\": \"float\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"rwoption4\": {\n \"name\": \"rwoption4\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"rwoption5\": {\n \"name\": \"rwoption5\",\n \"type\": \"bool\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"rwoption6\": {\n \"name\": \"rwoption6\",\n \"type\": \"bool\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"True\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"testkey\": {\n \"name\": \"testkey\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"testlkey\": {\n \"name\": \"testlkey\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"testnewline\": {\n \"name\": \"testnewline\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n }\n }\n },\n {\n \"name\": \"status\",\n \"can_run\": true,\n \"error_string\": \"\",\n \"module_options\": {}\n },\n {\n \"name\": \"telegraf\",\n \"can_run\": true,\n \"error_string\": \"\",\n \"module_options\": {\n \"address\": {\n \"name\": \"address\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"unixgram:///tmp/telegraf.sock\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"interval\": {\n \"name\": \"interval\",\n \"type\": \"secs\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"15\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n }\n }\n },\n {\n \"name\": \"telemetry\",\n \"can_run\": true,\n \"error_string\": \"\",\n \"module_options\": {\n \"channel_basic\": {\n \"name\": \"channel_basic\",\n \"type\": \"bool\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"True\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"Share basic cluster information (size, version)\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"channel_crash\": {\n \"name\": \"channel_crash\",\n \"type\": \"bool\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"True\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"channel_device\": {\n \"name\": \"channel_device\",\n \"type\": \"bool\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"True\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"channel_ident\": {\n \"name\": \"channel_ident\",\n \"type\": \"bool\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"False\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"contact\": {\n \"name\": \"contact\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"None\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"description\": {\n \"name\": \"description\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"None\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"device_url\": {\n \"name\": \"device_url\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"https://telemetry.ceph.com/device\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"enabled\": {\n \"name\": \"enabled\",\n \"type\": \"bool\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"False\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"interval\": {\n \"name\": \"interval\",\n \"type\": \"int\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"24\",\n \"min\": \"8\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"last_opt_revision\": {\n \"name\": \"last_opt_revision\",\n \"type\": \"int\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"1\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"leaderboard\": {\n \"name\": \"leaderboard\",\n \"type\": \"bool\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"False\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"organization\": {\n \"name\": \"organization\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"None\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"proxy\": {\n \"name\": \"proxy\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"None\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"url\": {\n \"name\": \"url\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"https://telemetry.ceph.com/report\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n }\n }\n },\n {\n \"name\": \"test_orchestrator\",\n \"can_run\": true,\n \"error_string\": \"\",\n \"module_options\": {}\n },\n {\n \"name\": \"volumes\",\n \"can_run\": true,\n \"error_string\": \"\",\n \"module_options\": {}\n },\n {\n \"name\": \"zabbix\",\n \"can_run\": true,\n \"error_string\": \"\",\n \"module_options\": {\n \"identifier\": {\n \"name\": \"identifier\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"interval\": {\n \"name\": \"interval\",\n \"type\": \"secs\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"60\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"zabbix_host\": {\n \"name\": \"zabbix_host\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"None\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"zabbix_port\": {\n \"name\": \"zabbix_port\",\n \"type\": \"int\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"10051\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n },\n \"zabbix_sender\": {\n \"name\": \"zabbix_sender\",\n \"type\": \"str\",\n \"level\": \"advanced\",\n \"flags\": 0,\n \"default_value\": \"/usr/bin/zabbix_sender\",\n \"min\": \"\",\n \"max\": \"\",\n \"enum_allowed\": [],\n \"desc\": \"\",\n \"long_desc\": \"\",\n \"tags\": [],\n \"see_also\": []\n }\n }\n }\n ],\n \"services\": {\n \"restful\": \"https://42697164830d:8003/\"\n },\n \"always_on_modules\": {\n \"nautilus\": [\n \"balancer\",\n \"crash\",\n \"devicehealth\",\n \"orchestrator_cli\",\n \"progress\",\n \"rbd_support\",\n \"status\",\n \"volumes\"\n ]\n }\n },\n \"servicemap\": {\n \"epoch\": 3,\n \"modified\": \"2020-02-13 12:37:22.548073\",\n \"services\": {\n \"rbd-mirror\": {\n \"daemons\": {\n \"summary\": \"\",\n \"4164\": {\n \"start_epoch\": 3,\n \"start_stamp\": \"2020-02-13 12:37:21.683166\",\n \"gid\": 4164,\n \"addr\": \"172.30.0.2:0/4224599352\",\n \"metadata\": {\n \"arch\": \"x86_64\",\n \"ceph_release\": \"nautilus\",\n \"ceph_version\": \"ceph version 14.2.7 (3d58626ebeec02d8385a4cefb92c6cbc3a45bfe8) nautilus (stable)\",\n \"ceph_version_short\": \"14.2.7\",\n \"cpu\": \"Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz\",\n \"distro\": \"centos\",\n \"distro_description\": \"CentOS Linux 7 (Core)\",\n \"distro_version\": \"7\",\n \"hostname\": \"42697164830d\",\n \"id\": \"admin\",\n \"instance_id\": \"4164\",\n \"kernel_description\": \"#1 SMP Tue Jul 2 22:58:16 UTC 2019\",\n \"kernel_version\": \"4.9.184-linuxkit\",\n \"mem_swap_kb\": \"2097148\",\n \"mem_total_kb\": \"8164244\",\n \"os\": \"Linux\"\n }\n }\n }\n },\n \"rgw\": {\n \"daemons\": {\n \"summary\": \"\",\n \"42697164830d\": {\n \"start_epoch\": 2,\n \"start_stamp\": \"2020-02-13 12:37:19.480410\",\n \"gid\": 4145,\n \"addr\": \"172.30.0.2:0/1483840010\",\n \"metadata\": {\n \"arch\": \"x86_64\",\n \"ceph_release\": \"nautilus\",\n \"ceph_version\": \"ceph version 14.2.7 (3d58626ebeec02d8385a4cefb92c6cbc3a45bfe8) nautilus (stable)\",\n \"ceph_version_short\": \"14.2.7\",\n \"cpu\": \"Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz\",\n \"distro\": \"centos\",\n \"distro_description\": \"CentOS Linux 7 (Core)\",\n \"distro_version\": \"7\",\n \"frontend_config#0\": \"beast endpoint=0.0.0.0:8080\",\n \"frontend_type#0\": \"beast\",\n \"hostname\": \"42697164830d\",\n \"kernel_description\": \"#1 SMP Tue Jul 2 22:58:16 UTC 2019\",\n \"kernel_version\": \"4.9.184-linuxkit\",\n \"mem_swap_kb\": \"2097148\",\n \"mem_total_kb\": \"8164244\",\n \"num_handles\": \"1\",\n \"os\": \"Linux\",\n \"pid\": \"968\",\n \"zone_id\": \"7af69646-d006-4076-bd75-9253e35323b2\",\n \"zone_name\": \"default\",\n \"zonegroup_id\": \"b5760001-d60d-4698-a8a2-2143e35c5543\",\n \"zonegroup_name\": \"default\"\n }\n }\n }\n },\n \"rgw-nfs\": {\n \"daemons\": {\n \"summary\": \"\",\n \"42697164830d\": {\n \"start_epoch\": 3,\n \"start_stamp\": \"2020-02-13 12:37:21.726085\",\n \"gid\": 4168,\n \"addr\": \"172.30.0.2:0/1234614580\",\n \"metadata\": {\n \"arch\": \"x86_64\",\n \"ceph_release\": \"nautilus\",\n \"ceph_version\": \"ceph version 14.2.7 (3d58626ebeec02d8385a4cefb92c6cbc3a45bfe8) nautilus (stable)\",\n \"ceph_version_short\": \"14.2.7\",\n \"cpu\": \"Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz\",\n \"distro\": \"centos\",\n \"distro_description\": \"CentOS Linux 7 (Core)\",\n \"distro_version\": \"7\",\n \"frontend_config#0\": \"rgwlib\",\n \"frontend_type#0\": \"rgw-nfs\",\n \"hostname\": \"42697164830d\",\n \"kernel_description\": \"#1 SMP Tue Jul 2 22:58:16 UTC 2019\",\n \"kernel_version\": \"4.9.184-linuxkit\",\n \"mem_swap_kb\": \"2097148\",\n \"mem_total_kb\": \"8164244\",\n \"num_handles\": \"1\",\n \"os\": \"Linux\",\n \"pid\": \"2004\",\n \"zone_id\": \"7af69646-d006-4076-bd75-9253e35323b2\",\n \"zone_name\": \"default\",\n \"zonegroup_id\": \"b5760001-d60d-4698-a8a2-2143e35c5543\",\n \"zonegroup_name\": \"default\"\n }\n }\n }\n }\n }\n },\n \"progress_events\": {}\n}\n", + "outs": "" + } + ], + "has_failed": false, + "id": "140713918770960", + "is_finished": true, + "is_waiting": false, + "running": [], + "state": "success", + "waiting": [] +} diff --git a/metricbeat/module/ceph/_meta/testdata/mgr_time_sync_status_sample_response.json b/metricbeat/module/ceph/_meta/testdata/mgr_time_sync_status_sample_response.json new file mode 100644 index 000000000000..55dc45fd09b3 --- /dev/null +++ b/metricbeat/module/ceph/_meta/testdata/mgr_time_sync_status_sample_response.json @@ -0,0 +1,17 @@ +{ + "failed": [], + "finished": [ + { + "command": "time-sync-status format=json-pretty", + "outb": "{\n \"timechecks\": {\n \"epoch\": 3,\n \"round\": 0,\n \"round_status\": \"finished\"\n }\n}\n", + "outs": "" + } + ], + "has_failed": false, + "id": "140713918770000", + "is_finished": true, + "is_waiting": false, + "running": [], + "state": "success", + "waiting": [] +} diff --git a/metricbeat/module/ceph/fields.go b/metricbeat/module/ceph/fields.go index 9eb2752d6970..eaf3e951d7ab 100644 --- a/metricbeat/module/ceph/fields.go +++ b/metricbeat/module/ceph/fields.go @@ -32,5 +32,5 @@ func init() { // AssetCeph returns asset data. // This is the base64 encoded gzipped contents of ../metricbeat/module/ceph. func AssetCeph() string { - return "eJzsXU1z3DiSvftXZPiy3RGydveqw0a4bbft6Lalbdkzh4kJGkVmVWEEAmwAlFzz6ycAfhSLBMAvVDliRjp1W6r3HhLJRCITRL2CBzzcQIrF/gWApprhDbx8g8X+5QuADFUqaaGp4Dfwfy8AAMyvIBdZyfAFgNoLqZNU8C3d3cCWMGX+VSJDovAGdsT8DWpN+U7dwN9eKsVeXsHLvdbFy7+/ANhSZJm6scivgJMcWy3mRx8KgyJFWdT/4lBkfr6ZD32DVHBNKFeg9wg5aklT899EwxNKBJVKUmAGWylyePPu7sN1DdCVcSKFlUqjTDKqHtpfumQFpJkfD86pnZqfvpiuIPJIKCMbhtebg0Z18jeNLib4rveLgDTz87pBBYsKYmsNWKvu/flWyJzoGxgKaERqoQmLKvCLQYwjrlSYRdX2VWG2XFrf0/ZImN7H8LUB0nxvE48oCWOJ0kSXbns94OFJyGyeyW4rXKhwQ2ZrXYrmmO4xfVDXWIh0H2nuPpECHlEqKvgYrRQlz64fCSsxEnkLDhZ7moDYU3F/MgWnQvq+OeB2+6bbzyZ47AA/5JuuaVs+F3ZVq20RdAhJtluaXkskWTIrivgj07i4yj5gSEHvpSh3+6LUUKAEhanw+U6t9UlSjRcXa1kXqLWWFUVSGIfANNb8dk1IRaHmGe9seiorTRGUU1UwkuK1XV0jy2jAodgBL/ONJwq3GsTmH5jqWCvoQEUNP0mKJJoKpxCVEoZZsmWCaI9/FyhT5P3fzpU7lNCozHAnSYbZWSatAR+ZtFbDeSatVTFh0lopP27SWrn+SSt21xnR5OIRs9iB4Q3krsXu2m4ALi6NnG4QfOKsl19cnD5uDnzCTNJ/cV1luzHwyLIJD9qUDhPzj5Eezbudzayx+3dhCakoBw/UUvb7vXhSsBdPkBN+gGKngEgEymtRYjs69IG+yMlePUFqPN8TKrvelswdujdCMCR9VSPkH5UBhQFol5EjkWdhNcAj1GWeCJXFWiV6zmCQjSeMbfcaJWVxbjHGNUUlqHXPr3dBUZT/CFEfPwdFScxJUWCWFLtLK/vj3afXd3fv3nr1xdyxW6x+jtEmhTuZxCrZBbDaDe4G9XPh7j+8cNf1kyXFu7ro0BSrxbaqRQylLPG6raL9etDkWtGvlCHcH5TGHD6+vQICJad/lgg0Q67plqI0phmN5X+WQpZ5QnbL0ptmsf6zJLLMoQ/TToPgOSmcDH3zDyiGrvGtgus0FCiv3IAKDmQjSt2M2TBTLaS67ruNY15OFFOe5IIn9bwO/i5goMEIPlFOc8IaLeDD9HC7kk8Y8ZWpGsCJ3TooEl3KwSMNwckbUA8n0Px8a8CD03jc2jQT2nysP6EQmNTumAqTTyo93KJONetgfHctYiMuKEDYjw3qDYvpb2s8L3nnITzPTBrg4Cw2Tld1Iku7uefLZ7DcMJomJMv6a8HpkGbY8FMtsIIGA43KZayjCkn4Q5DeExm83F7AhtETBlYN2IvpMLbbHmMO5KU+tbUzyMCIR3SFGpzHQQV4qJOj0uiz0UCs2+Obn281acf5M9SEMpspEMYGg3T5/NSRdkdrxhL8w2k+4Rzz60rsOEfX9OcS80EoDYRnUAipjVXrWDJJGBc8nWamwMPqlPXZIIOr33iq4BxByhedGk7Xhm50pCdM7+w2zl3WCaavU8YVNY2Fk/UtMx/wa8qIds3WiaDfidI1UlotYZrmaD3Q+fG2CyuRuOPKdO43BiNMeay9xcynLVpwBbetlNykOBZ/djrNy9xR+4BZXvnZlhVMDLh77/d+uw9MbN64iu11oKje5Tv2mh0NyLmUKMkO6/qJGWi1/7VtUU8Dsiul00o+mxbDoZFPkVMJKVXgqZii5Ku7UD9kcnXx5lF98fQqoD/f3o7zPL4/zMSKAuuEePocR+L/q+1vzxNQ7FSyOVSVfy+5N8masp3oMgSD0t179V9VoXPxXsLb3Tkdz4xV+t4WXkfzaldP55Rz6v7hjRPpJPAKwWKFXjdWl83dyF7K50Nrm9a+XvA8wrfh1u6gdAir1tsKruPbO+QoCQvUsapu2Nxl13keDeYmaR+sXBdYm3IyTM3fJsubCe9qCEf6elqrXIT+//aj8NHdBqmLoOZ/Fh/eqxmGGJ1eS8ycrYILxsfb+7ezPeZSmwdPJ3Me1TFMmKEGudy9yqV0ZTHO6G5ELmWkfJwx0GVcSttADsmPu8CYXm3Rgk69NTtGZXeMs52bDv1xjll+PVK7oNp9KPkejcaF1bbcihulCc82h2h0AchLBwhnMXHpqAI7h8RTd42QyNbgk90ZlJDabnZsMXhpTmsgK8TEWaAZNeRgfF1j9pbQPnm0sniX86Kl8S7xaBrvTbBWUweQG/LdGab3fWd+u010obLVhzZu79+CwThTO52LzJO+OZ/isaXIwoWbk6mmj1jxzl2KMiwGGwqYt2VxIhzLFE9Id3vXFnNOfaICgZ+MFaRg9YGjMt1Xh2HtSaPKcbSkqEALIIyJ1OyBtbC/v71/+3OomhApUwnVBFNZqn0yYpDA2eYB7Zs/vt5/AA9g3M3XZ5GhD6rNa9ZV2BwhfSSwzhtB6IiBQXIvUzPKdocCQ4N42NgqZGJcdhWRLUVax/+JcvjtF79fN5RimBwvoDQokylzjDFKgzKFMmahXRUkxVHSR+LqqE3n+4vj8yNt1Vnu/sWFcbTYOk+3Beppdqr9Yb0rTKP7qimj/ySD0+xzKe8tW+lBO8ZWSfp7lBVrvYULrvX2L2wG87zcPy/3Ltbn5f7fcrmvot9WyB+x8B/Jf0AKcCR/TgYG4M/JQIjOt3zPpZyYDJR5TqQ7HVhShK7xJvXnlvRYqhcxo0WKyg/KBfGCVIc8kljT1RwaUcFpg5O86HFlVvRIwwytsdcHp8rQZGaIyilP1oap5vT+I5GUBLr9Ofm+not8n8TVGPZiMezksYn7xMyiXp9YOJ/YSenFqZDV67xTiHO179eCC5TbF33KmbVgg2HCKU/xTCXhYpdIJJl7aZjwqu5de+zxvRmaPfxFOSoVOgwhVGZ+6y5FL1qPertTYzvDQJWm6ezVp5m9xCxm/v3KtM5XfxqPshyfnNK8itTSMLpGOlXWBq6JOmWc/F7DnTXDuBVgxksNqchzqhNGNPL0kOQ+PJh6kL13csygQ40OlEOAoM0ZioIdzqXotQGfKahnI+c7TssVDW0UIHDbKLKigY24+7VX+5wLwQY+7nm+x4K1ENWVcNFjtNHoKDxMPnl1Z5QNAE7gl79na8E9x8ZSRpHrhIpEDg/DTgz3bywGfPzvWzAgfVcZC+brTp/X47+BXy55zrzHOuNE+cqz1y3xDzh33XJPOXPdf4y1RFybbBmMszy/Z2672wMDdad9cfk9Sl1TZCOvS0sh/PXmOSzOytMlS6gr4e0gAvDpnrJM4rpigyXxIp2lMRIoY5y3vYPfqVp5pP6dG+Ly3YtC0pzIQ0K2W8qpXnd48a4CAy/Y5XtC4y3KGHvAlf3I54D4HBBXUD0HxOeAOCkgtpls9e761DuYZl9SfYpf57m9Ou20C8CKNNZVi8deZn0nanOR1adb921xgbfMFt0eXr+3NcJ6HPqglr9+5L/9MkZf3WoWjbqqaI/T2vvKorHa1ug4KSNKJ2WROW4KqJgdtwSMjZfmCE975PBEFLiwA0v+Ct/6THIcG67SQmJVfrpmYhf1erjfxe54O1xfwJQL67ricqrSqOo+UZXGk6eUjqru/v5LPHFnv5VwrcDRh26B8xGlB89at7+SjffG5n8PAxZ7e02srYiUiuywex5h4kLnqYTOt4G9sNZkN/QS4aZl89Z5M3ykKSYpI4OrcWLwGowroIzhjjD7f0B5ysoMYZ9lV6BUBqjT68AaZ9x1qvFXXHDdukh18/ajYKXHZO2dnZdUVTluQNXCu1hjaDse7QjoK3YJX/gW+pBa9a/jhuo0bVbd40vtxdD+yfMlq+e6p783izVG99bPbhRcW7ZuI56tXZtYNxo/Lxb6hIRNmT6gvnAQ7PF6w6GjYhKXORAQhcquYC+UvrIVoEBcdFZuIk6LgfI179yFkhVGqokbYGDUWEBhQSSpX+tNRZ4Tty0C+/FKk+thnrp6WfBQsdJVr1k6D5ZxCBgs3Uw+kTOko6piVJoyVqEbN+BC//S/rw6oruB/XnHxszuWh0sviw1vdkX1uxT2EgWSUV5/f4j9dsOa1xvcPUW6pTNivzwMj4J8tLG/vuz4DXI2JFBt1ruSZbBBKAszS5l4cp/pPU86ZwxRIYNFbqUx+jAlkdsSvR/cQLhST0Ek8jpe2Q11tej3QtlggbXnGta+ke0C+YEL68esKSkYYRdYUbtVDC9ltY286FcEDLRM2/QanXG/R2pwEddEa13g6wBW2OgMxb+HzYmufwUAAP//mM8NxQ==" + return "eJzsXM+P27oRvu9fMdhTHrBx2+seCiTZ4GXxmqyRTdpDUSi0NJbYpUiCpLxx//qClCzbEvXLoh2gtU/vxd7v+zQczgyHpN7CC27vIUaZ3QAYahjew+0HlNntDUCCOlZUGir4Pfz1BgDAfgW5SAqGNwA6E8pEseBrmt7DmjBt/1UhQ6LxHlJif4PGUJ7qe/jnrdbs9g5uM2Pk7b9uANYUWaLvHfJb4CTHWov9mK20KEoUsvoXjyL7+WH/6AfEghtCuQaTIeRoFI3tfxMDr6gQdKyIxATWSuTw4ePy06ICOJRxJIUV2qCKEqpf6i99snqk2U8HzrGddp+mmENBZEMoIyuGi9XWoD76zU4XEzxtfNEjzX7e7VDBoYJYOwNWqhs/XwuVE3MPbQE7kUYYwoIK/GYRw4grNCZBtX3XmJwurelpGRJmshC+1kKa7m1ig4owFmlDTOG31wtuX4VKppnsqcSFErfPbLVL0RzjDOMXvUAp4izQ2H0mEjaoNBV8iFaJgieLDWEFBiKvwcFhjxMQeiiej4bgWEjTN1vcft/0+9kIj23h9/mmb9hOHwuX1Spb9DqEIus1jRcKSRJNiiLdkWlYXGkfsKRgMiWKNJOFAYkKNMaiy3cqra+KGry4WMd6glpnWSEjaR0C41Dje2hCKqSeZryz6SmtNEZQTrVkJMaFy66BZezAQabAi3zVEYVrDWL1b4xNqAzaUlHBj5KiiKHCK0THhGESrZkgpsO/JaoYefPbqXLbEnYqE0wVSTA5y6DtwAcGrdZwnkGrVYwYtFrKrxu0Wm73oMl0kRBDLh4xZQqWt6d2lenCLQAuLo0cLxC6xDkvv7g4s18cdAmzRf/FdRX1wqBDlit40JV0GNl/DDQ1l6mrrPHwd/0SYlG0JtSp7M+ZeNWQiVfICd+CTDUQhUB5JUqsBx+9pS9wsVcNkB6u94ROFuuC+UP3SgiGpKlqgPxRW1BogR4yciTqLKwWeIC6yCOhk1BZouEMFtl6wtByb6ekkOcWY11TlIJq9/y+7BVF+a8Q9filV5TCnEiJSSTTSyv7+vHzu+Xy40OnvpArdofVrDHqojBVUaiWXQ9WvcBdobk27v7PG3eHfjK/edeLdorfXVt41xZe5aVCJ7Oj4tPzA1iMenNFrMveWXvkT/FWLpKO6c1RG5xomB8O7mBXiPJyLlPBgaxEYYDEhm6w5F00571HKBytZqVpejH0OVNL8oMXYYev8BVpmjXr8mkUXysQeGOtoASrMnoRZ+Vq06XyrTaYg1EUNRgBhDER2+xuhPv+6fnht06Z7YJjmsIvLpdbR1r+3kaqe9Cq0Fk0YJCe5kGL9sPX78+foANwx+qdqjAwXdtPKBLsgtoxUR/KeCM+PnQiexaW05/Ah1EHuK3EaKb+b1uJfQ/xsorcGt667Cwil/Sd47+hHP543+3XO0qREzmf0qKMpswxxFNalDGUrjSdRbcvQ7UkMQ6SbkizxprG93fP3x9641x3/+bD2Ftsnqe7inicnSp/mO8K4+i+G8rof0irXTSV8tmxFR1o+9iqyDZcrndwvbne/cJVMNd0f033PtZruv+fTPdl9FsL9SsS/578F5QAe/JrMdACvxYDfXRd6Xsq5chioMhzovzlQLM90aLwVQMl3kE9kCJHRZinLnh6fphcDJQ7ncEiRekHxQnxgmxQkRSjUMP1rsSrZPSh7uuizcyqaEP7GWpjzw9OpaHJxBCVUx7NDVOfKac5YTbcUdLabD3gIj/nc5Gfo7h2hr1YDDuaNmFnzCTq+YWFd8aOKi+OhczO814h3mzf7AVLVOubJuXEXrDFsOGUx3imlrBMI4Uk8aeGEXvhS0ZizJEb+N0+mjsBSDlq7SvRD3Yt7bf+VvRJ+aixOrW2swxUGxpPzj670YtsMuter3jX0S21zWHcy/L8ZZcyGFrbwJBTe3V5lgiHTM4GvoE6ZvQNmJdy6cwwbAUYsMShxljkOTURIwZ5vI3yLjwYY6CW4g8OHSp0oBx6COqaQUq2PZeidxZ8oqCGjfi5bdRD4LdRYEUtG3H/vrKb50Kwlo93zO+hYC1EuWEbPEZbjZ0H2EZsai6tsu5jYBa+FVWmgT/6j6fEjCI3ERWRIsYvfzDcf3AY8PinJ7AgTVcZCub7qwSe8+Uw5F/V89/De3eQwZ1v7zhJfkh6cCcgAKtFM8jHEPcf7x9P/NU+qJDojvHyztPz7WcOwP0Pd2i/n7w5jY1CnFtsWYyzzN8zb7u7AwPVTvvJ7fcgfU2RoB+odk8huvvNU1i8nadLtlBnwruH6IGPM8oShfOaDY6kE+ksGyM9bYzzbu/gT6q9dep4go9+iMvvXkhFc6K2EVmvKaemuTab9lTLEgw6wS6/JzS8RRliDThzP/IaEK8BcQbVNSBeA+KogFhXsoJTI0Yfcp58C/wYv6pzG33acSfsZRzqLtN+L7O6dLg7Zfv5yX8do2UcmHe291NpjAHW/aO3evnzn/yP90P05bWBYNRlR3uY1l0ICMbqtkaHSRnRJipkQtrLopI5afcRhp6X5givGXJ4JRp82D0pf4ZvfSE5Dj2uNkJh2X5aMJEGvX/xN5Hur180BYy5EXIoLqc6DqruM9VxOHlam6Dqnp+/hRN39ms/cwUOTroTnI9o05prh/sryfDe2PQXnaDM3D1M1xEpNEnx8DzCyETX0QmdbgN3I9RWN/QS4aZm6+zzJrihMUYxIzrg9Zia12LcAWUMU8Lc/wHlMSsShCxJ7kDrBNDEi54cZ911rPFn3CCvXaS82r4RrOgwWX0p7pKqSsftUXXiZccQ2vZHO3r0yTTiRR5oFunmfXcoT9Mm5UVZ6m5edw9eV7F6rhdhNEaxwiApeqPg3LZ1HfFc79rGusH4ebHQJxSsivgFzYWDYIO3Mxx6OiZhmXsCotDJHWRCmzvXAeqJi97OTcBhsVBdm3f+RskMI1XEO2Bg1FpAoyTKzejV1u1VE78tetbjpSbfZB6bvRx4X7PS1685dRwcYxuwt3Uz+kROm47qklEbyliJbt2AC/PmL2+3qO/gz2+5+M0fy/tbLycb3q6KqrsUYl2dGqpe0ONeH1rxdgb3jibdqSPirvbiXlAXbejLxfv73S4kUGPzXcESWCEU0o5SIl79Z3rPU85ZQ5TI4JBraYy+jCnk1sRkrSv+M/VIopBX8cotqMuk3whlrQTrzjXMvZHtA/mFifUx2bUUrLALZNTDLkYnZbmMvOg7OFpaxi16rc6wL2rbX7javaFtnLUu8L6NGTY6Q/PvZXWk678BAAD//xvf8Dg=" } diff --git a/metricbeat/module/ceph/mgr/event_mapping.go b/metricbeat/module/ceph/mgr/event_mapping.go new file mode 100644 index 000000000000..ee173cea3d5e --- /dev/null +++ b/metricbeat/module/ceph/mgr/event_mapping.go @@ -0,0 +1,65 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mgr + +import ( + "encoding/json" + "fmt" + + "github.com/pkg/errors" +) + +// Request stores either or finished command result. +type Request struct { + HasFailed bool `json:"has_failed"` + Finished []Result `json:"finished"` + Failed []Result `json:"failed"` +} + +// Result stores ceph command output (and status). +type Result struct { + Command string `json:"command"` + Outb string `json:"outb"` + Outs string `json:"outs"` +} + +// UnmarshalResponse method unmarshals the content to the given response object. +func UnmarshalResponse(content []byte, response interface{}) error { + var request Request + err := json.Unmarshal(content, &request) + if err != nil { + return errors.Wrap(err, "could not get request data") + } + + if request.HasFailed { + if len(request.Failed) != 1 { + return errors.New("expected single failed command") + } + return fmt.Errorf("%s: %s", request.Failed[0].Outs, request.Failed[0].Command) + } + + if len(request.Finished) != 1 { + return errors.New("expected single finished command") + } + + err = json.Unmarshal([]byte(request.Finished[0].Outb), response) + if err != nil { + return errors.Wrap(err, "could not get response data") + } + return nil +} diff --git a/metricbeat/module/ceph/mgr_cluster_disk/mgr_data.go b/metricbeat/module/ceph/mgr_cluster_disk/mgr_data.go index 87faae697df0..cbb69d054162 100644 --- a/metricbeat/module/ceph/mgr_cluster_disk/mgr_data.go +++ b/metricbeat/module/ceph/mgr_cluster_disk/mgr_data.go @@ -18,26 +18,12 @@ package mgr_cluster_disk import ( - "encoding/json" - "fmt" - "github.com/pkg/errors" "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/metricbeat/module/ceph/mgr" ) -type Request struct { - HasFailed bool `json:"has_failed"` - Finished []Result `json:"finished"` - Failed []Result `json:"failed"` -} - -type Result struct { - Command string `json:"command"` - Outb string `json:"outb"` - Outs string `json:"outs"` -} - type DfResponse struct { Stats struct { TotalBytes uint64 `json:"total_bytes"` @@ -47,25 +33,8 @@ type DfResponse struct { } func eventMapping(content []byte) (common.MapStr, error) { - var request Request - err := json.Unmarshal(content, &request) - if err != nil { - return nil, errors.Wrap(err, "could not get request data") - } - - if request.HasFailed { - if len(request.Failed) != 1 { - return nil, errors.New("expected single failed command") - } - return nil, fmt.Errorf("%s: %s", request.Failed[0].Outs, request.Failed[0].Command) - } - - if len(request.Finished) != 1 { - return nil, errors.New("expected single finished command") - } - var response DfResponse - err = json.Unmarshal([]byte(request.Finished[0].Outb), &response) + err := mgr.UnmarshalResponse(content, &response) if err != nil { return nil, errors.Wrap(err, "could not get response data") } diff --git a/metricbeat/module/ceph/mgr_cluster_health/_meta/data.json b/metricbeat/module/ceph/mgr_cluster_health/_meta/data.json index dbef986ed31e..9b22b535b9a3 100644 --- a/metricbeat/module/ceph/mgr_cluster_health/_meta/data.json +++ b/metricbeat/module/ceph/mgr_cluster_health/_meta/data.json @@ -1,125 +1,31 @@ { - "@timestamp": "2020-02-12T12:22:57.142Z", - "@metadata": { - "beat": "metricbeat", - "type": "_doc", - "version": "8.0.0" - }, + "@timestamp": "2017-10-12T08:05:34.853Z", "agent": { - "id": "993db589-f9d8-40f3-94cb-43b2b69fa9f4", - "version": "8.0.0", - "type": "metricbeat", - "ephemeral_id": "59b80233-4d5c-48c9-a3db-a40d6e172aae", - "hostname": "host" + "hostname": "host.example.com", + "name": "host.example.com" + }, + "ceph": { + "cluster_health": { + "overall_status": "HEALTH_OK", + "timechecks": { + "epoch": 3, + "round": { + "status": "finished", + "value": 0 + } + } + } }, "event": { - "duration": 29973749, - "dataset": "http.json_namespace", + "dataset": "ceph.cluster_health", + "duration": 115000, "module": "ceph" }, "metricset": { - "period": 60000, - "name": "mgr_cluster_health" + "name": "cluster_health" }, "service": { - "address": "localhost:8003", + "address": "127.0.0.1:5000", "type": "ceph" - }, - "ecs": { - "version": "1.4.0" - }, - "ceph": { - "mgr_cluster_status": { - "fsid": "ac9834a6-7dc8-434f-b9f8-86fca6849c5b", - "quorum_age": 226, - "monmap": { - "min_mon_release": 14, - "min_mon_release_name": "nautilus", - "features": { - "persistent": [ - "kraken", - "luminous", - "mimic", - "osdmap-prune", - "nautilus" - ], - "optional": [] - }, - "mons": [ - { - "public_addr": "172.30.0.2:3300/0", - "rank": 0, - "name": "c213fde94857", - "public_addrs": { - "addrvec": [ - { - "type": "v2", - "addr": "172.30.0.2:3300", - "nonce": 0 - } - ] - }, - "addr": "172.30.0.2:3300/0" - } - ], - "epoch": 1, - "fsid": "ac9834a6-7dc8-434f-b9f8-86fca6849c5b", - "modified": "2020-02-12 12:19:11.154113", - "created": "2020-02-12 12:19:11.154113" - }, - "pgmap": { - "num_pgs": 164, - "bytes_avail": 6950027264, - "read_bytes_sec": 85, - "write_bytes_sec": 746908239, - "bytes_used": 3787390976, - "bytes_total": 10737418240, - "read_op_per_sec": 0, - "write_op_per_sec": 318, - "pgs_by_state": [ - { - "state_name": "active+clean", - "count": 164 - } - ], - "num_pools": 9, - "num_objects": 950, - "data_bytes": 3070235948 - }, - "health": { - "status": "HEALTH_WARN" - }, - "election_epoch": 3, - "quorum": [ - 0 - ], - "quorum_names": [ - "c213fde94857" - ], - "osdmap": { - "epoch": 21, - "num_osds": 1, - "num_up_osds": 1, - "num_in_osds": 1, - "num_remapped_pgs": 0 - }, - "fsmap": { - "in": 1, - "max": 1, - "by_rank": [ - { - "filesystem_id": 1, - "rank": 0, - "name": "demo", - "status": "up:active", - "gid": 4129 - } - ], - "up:standby": 0, - "epoch": 5, - "id": 1, - "up": 1 - } - } } } diff --git a/metricbeat/module/ceph/mgr_cluster_health/_meta/fields.yml b/metricbeat/module/ceph/mgr_cluster_health/_meta/fields.yml index 4bd446c0e93e..f3030809eb4f 100644 --- a/metricbeat/module/ceph/mgr_cluster_health/_meta/fields.yml +++ b/metricbeat/module/ceph/mgr_cluster_health/_meta/fields.yml @@ -1,206 +1,22 @@ - name: mgr_cluster_health type: group - description: Status metrics of Ceph cluster + description: > + mgr_cluster_health release: beta fields: - - name: fsid + - name: overall_status type: keyword - description: File System ID, a unique identifier for the cluster - - name: quorum_age - type: long - description: Cluster quarum age - - name: monmap - type: group - description: > - `monmap` contains information about cluster monitors. - fields: - - name: min_mon_release - type: long - description: Minimal monitor release - - name: min_mon_release_name - type: keyword - description: Minimal monitor release name - - name: features - type: group - description: > - `features` contains information about available cluster features. - fields: - - name: persistent - type: keyword - description: Persistent feature - - name: optional - type: keyword - description: Optional feature - - name: mons - type: group - description: > - `mons` contains information about monitor configurations. - fields: - - name: public_addr - type: keyword - description: Monitor public address - - name: rank - type: long - description: Monitor rank - - name: name - type: keyword - description: Monitor name - - name: public_addrs - type: group - description: Monitor public addresses - fields: - - name: addrvec - type: nested - description: > - `addrvec` contains details of all public addresses. - fields: - - name: type - type: keyword - description: Address type - - name: addr - type: keyword - description: Host and port of monitor - - name: nonce - type: long - description: Nonce value - - name: addr - type: keyword - description: Monitor address - - name: epoch - type: long - description: Epoch version - - name: fsid - type: keyword - description: File System ID, a unique identifier for the cluster - - name: modified - type: date - description: Last modification time and date - - name: created - type: date - description: Creation time and date - - name: pgmap - type: group - description: > - `pgmap` contains information about placement groups. - fields: - - name: num_pgs - type: long - description: Number of PGs - - name: bytes_avail - type: long - description: Available bytes - - name: read_bytes_sec - type: long - description: Average number of bytes read per second - - name: write_bytes_sec - type: long - description: Average number of bytes written per second - - name: bytes_used - type: long - description: Used bytes - - name: bytes_total - type: long - description: Total bytes - - name: read_op_per_sec - type: long - description: Read operations per second - - name: write_op_per_sec - type: long - description: Write operations per second - - name: pgs_by_state - type: nested - description: > - `pgs_by_state` contains information about PGs' states. - fields: - - name: state_name - type: keyword - description: State name - - name: count - type: long - description: Count - - name: num_pools - type: long - description: Number of pools - - name: num_objects - type: long - description: Number of objects - - name: data_bytes - type: long - description: Data bytes - - name: health - type: group description: > - `health` contains general information about cluster status. - fields: - - name: status - type: keyword - description: Health status - - name: election_epoch + Overall status of the cluster + - name: timechecks.epoch type: long - description: Election epoch - - name: quorum + description: > + Map version + - name: timechecks.round.value type: long - description: Quorum ID - - name: quorum_names - type: keyword - description: Quorum names - - name: osdmap - type: group description: > - `osdmap` contains information about OSDs. - fields: - - name: epoch - type: long - description: Epoch version - - name: num_osds - type: long - description: Number of OSDs - - name: num_up_osds - type: long - description: Number of up OSDs - - name: num_in_osds - type: long - description: Number of in OSDs - - name: num_remapped_pgs - type: long - description: Number of remapped OSDs - - name: fsmap - type: group + timecheck round + - name: timechecks.round.status + type: keyword description: > - `fsmap` contains information about file systems. - fields: - - name: in - type: long - description: File system in - - name: max - type: long - description: File system max - - name: up:standby - type: long - description: File system up:standby - - name: epoch - type: long - description: Epoch version - - name: up - type: long - description: File system up - - name: by_rank - type: nested - description: > - `by_rank` contains information about file systems sorted by rank. - fields: - - name: filesystem_id - type: long - description: File system ID - - name: rank - type: long - description: File system rank - - name: name - type: keyword - description: File system name - - name: status - type: keyword - description: File system status - - name: gid - type: long - description: File system GID + Status of the round diff --git a/metricbeat/module/ceph/mgr_cluster_health/manifest.yml b/metricbeat/module/ceph/mgr_cluster_health/manifest.yml deleted file mode 100644 index 57807639ba95..000000000000 --- a/metricbeat/module/ceph/mgr_cluster_health/manifest.yml +++ /dev/null @@ -1,43 +0,0 @@ -default: true -input: - module: http - metricset: json - defaults: - method: POST - namespace: "json_namespace" - basepath: "/request" - headers: - Content-Type: application/json - query: - wait: "1" - body: '{"prefix": "status", "format": "json"}' - -processors: - - extract_array: - field: http.json_namespace.finished - mappings: - ceph.request: 0 - - decode_json_fields: - fields: ["ceph.request.outb"] - process_array: true - target: "ceph.mgr_cluster_status" - - rename: - ignore_missing: true - fields: - - from: "ceph.mgr_cluster_status.osdmap.osdmap" - - to: "ceph.mgr_cluster_status._tmp_osdmap" - - drop_fields: - ignore_missing: true - fields: - - "http" - - "ceph.request" - - "ceph.mgr_cluster_status.mgrmap" - - "ceph.mgr_cluster_status.servicemap" - - "ceph.mgr_cluster_status.health.checks" - - "ceph.mgr_cluster_status.progress_events" - - "ceph.mgr_cluster_status.osdmap" - - rename: - ignore_missing: true - fields: - - from: "ceph.mgr_cluster_status._tmp_osdmap" - - to: "ceph.mgr_cluster_status.osdmap" diff --git a/metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health.go b/metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health.go new file mode 100644 index 000000000000..fa61a64d1636 --- /dev/null +++ b/metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health.go @@ -0,0 +1,97 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mgr_cluster_health + +import ( + "fmt" + + "github.com/elastic/beats/metricbeat/helper" + "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/metricbeat/mb/parse" +) + +const ( + defaultScheme = "https" + defaultPath = "/request" + defaultQueryParams = "wait=1" + + cephStatusPrefix = "status" + cephTimeSyncStatusPrefix = "time-sync-status" +) + +var ( + hostParser = parse.URLHostParserBuilder{ + DefaultScheme: defaultScheme, + DefaultPath: defaultPath, + QueryParams: defaultQueryParams, + }.Build() +) + +func init() { + mb.Registry.MustAddMetricSet("ceph", "mgr_cluster_health", New, + mb.WithHostParser(hostParser), + ) +} + +type MetricSet struct { + mb.BaseMetricSet + *helper.HTTP +} + +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + http, err := helper.NewHTTP(base) + if err != nil { + return nil, err + } + http.SetMethod("POST") + http.SetHeader("Content-Type", "application/json") + http.SetHeader("Accept", "application/json") + + return &MetricSet{ + base, + http, + }, nil +} + +// Fetch methods implements the data gathering and data conversion to the right +// format. It publishes the event which is then forwarded to the output. In case +// of an error set the Error field of mb.Event or simply call report.Error(). +func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { + m.HTTP.SetBody([]byte(fmt.Sprintf(`{"prefix": "%s", "format": "json"}`, cephStatusPrefix))) + statusContent, err := m.HTTP.FetchContent() + if err != nil { + return err + } + + m.HTTP.SetBody([]byte(fmt.Sprintf(`{"prefix": "%s", "format": "json"}`, cephTimeSyncStatusPrefix))) + timeStatusContent, err := m.HTTP.FetchContent() + if err != nil { + return err + } + + event, err := eventMapping(statusContent, timeStatusContent) + if err != nil { + return err + } + + if reported := reporter.Event(mb.Event{MetricSetFields: event}); !reported { + m.Logger().Debug("error reporting event") + } + + return nil +} diff --git a/metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health_integration_test.go b/metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health_integration_test.go new file mode 100644 index 000000000000..3e53f876565f --- /dev/null +++ b/metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health_integration_test.go @@ -0,0 +1,63 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build integration,linux + +package mgr_cluster_health + +import ( + "encoding/json" + "fmt" + "net/http" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/elastic/beats/libbeat/tests/compose" + mbtest "github.com/elastic/beats/metricbeat/mb/testing" +) + +func TestData(t *testing.T) { + service := compose.EnsureUpWithTimeout(t, 120, "ceph-mgr") + + f := mbtest.NewReportingMetricSetV2Error(t, + getConfig(service.HostForPort(8003), getPassword(t, service.HostForPort(5000)))) + err := mbtest.WriteEventsReporterV2Error(f, t, "") + require.NoError(t, err) +} + +func getPassword(t *testing.T, host string) string { + response, err := http.Get(fmt.Sprintf("http://%s/restful-list-keys.json", host)) + require.NoError(t, err) + + defer response.Body.Close() + + accounts := map[string]string{} + json.NewDecoder(response.Body).Decode(&accounts) + return accounts["demo"] +} + +func getConfig(host, password string) map[string]interface{} { + return map[string]interface{}{ + "module": "ceph", + "metricsets": []string{"mgr_cluster_health"}, + "hosts": []string{host}, + "username": "demo", + "password": password, + "ssl.verification_mode": "none", + } +} diff --git a/metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health_test.go b/metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health_test.go new file mode 100644 index 000000000000..8f08987e5a6a --- /dev/null +++ b/metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health_test.go @@ -0,0 +1,116 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mgr_cluster_health + +import ( + "encoding/json" + "io/ioutil" + "net/http" + "net/http/httptest" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/libbeat/common" + mbtest "github.com/elastic/beats/metricbeat/mb/testing" +) + +type clientRequest struct { + Prefix string `json:"prefix"` +} + +func TestFetchEventContents(t *testing.T) { + absPath, err := filepath.Abs("../_meta/testdata/") + assert.NoError(t, err) + + statusResponse, err := ioutil.ReadFile(absPath + "/mgr_status_sample_response.json") + assert.NoError(t, err) + timeSyncStatusResponse, err := ioutil.ReadFile(absPath + "/mgr_time_sync_status_sample_response.json") + assert.NoError(t, err) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + w.Header().Set("Content-Type", "application/json;") + + defer r.Body.Close() + var request clientRequest + err := json.NewDecoder(r.Body).Decode(&request) + require.NoError(t, err) + + if request.Prefix == "status" { + w.Write(statusResponse) + } else if request.Prefix == "time-sync-status" { + w.Write(timeSyncStatusResponse) + } + })) + defer server.Close() + + config := map[string]interface{}{ + "module": "ceph", + "metricsets": []string{"mgr_cluster_health"}, + "hosts": []string{server.URL}, + } + + f := mbtest.NewReportingMetricSetV2Error(t, config) + events, errs := mbtest.ReportingFetchV2Error(f) + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) + } + assert.NotEmpty(t, events) + event := events[0].MetricSetFields + + t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event.StringToPrint()) + + assert.EqualValues(t, "HEALTH_OK", event["overall_status"]) + + timechecks := event["timechecks"].(common.MapStr) + assert.EqualValues(t, 3, timechecks["epoch"]) + + round := timechecks["round"].(common.MapStr) + assert.EqualValues(t, 0, round["value"]) + assert.EqualValues(t, "finished", round["status"]) +} + +func TestFetchEventContents_Failed(t *testing.T) { + absPath, err := filepath.Abs("../_meta/testdata/") + assert.NoError(t, err) + + response, err := ioutil.ReadFile(absPath + "/mgr_failed_response.json") + assert.NoError(t, err) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + w.Header().Set("Content-Type", "application/json;") + w.Write([]byte(response)) + })) + defer server.Close() + + config := map[string]interface{}{ + "module": "ceph", + "metricsets": []string{"mgr_cluster_health"}, + "hosts": []string{server.URL}, + } + + f := mbtest.NewReportingMetricSetV2Error(t, config) + events, errs := mbtest.ReportingFetchV2Error(f) + assert.Empty(t, events) + assert.NotEmpty(t, errs) +} diff --git a/metricbeat/module/ceph/mgr_cluster_health/mgr_data.go b/metricbeat/module/ceph/mgr_cluster_health/mgr_data.go new file mode 100644 index 000000000000..8dae532bc562 --- /dev/null +++ b/metricbeat/module/ceph/mgr_cluster_health/mgr_data.go @@ -0,0 +1,64 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mgr_cluster_health + +import ( + "github.com/pkg/errors" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/metricbeat/module/ceph/mgr" +) + +type StatusResponse struct { + Health struct { + Status string `json:"status"` + } `json:"health"` +} + +type TimeSyncStatusResponse struct { + Timechecks struct { + RoundStatus string `json:"round_status"` + Epoch int64 `json:"epoch"` + Round int64 `json:"round"` + } `json:"timechecks"` +} + +func eventMapping(statusContent, timeSyncStatusContent []byte) (common.MapStr, error) { + var statusResponse StatusResponse + err := mgr.UnmarshalResponse(statusContent, &statusResponse) + if err != nil { + return nil, errors.Wrap(err, "could not unmarshal response") + } + + var timeSyncStatusResponse TimeSyncStatusResponse + err = mgr.UnmarshalResponse(timeSyncStatusContent, &timeSyncStatusResponse) + if err != nil { + return nil, errors.Wrap(err, "could not unmarshal response") + } + + return common.MapStr{ + "overall_status": statusResponse.Health.Status, + "timechecks": common.MapStr{ + "epoch": timeSyncStatusResponse.Timechecks.Epoch, + "round": common.MapStr{ + "value": timeSyncStatusResponse.Timechecks.Round, + "status": timeSyncStatusResponse.Timechecks.RoundStatus, + }, + }, + }, nil +} From e53af7e1deb52a86cb4945f7723cee6bf5f93e65 Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Thu, 13 Feb 2020 15:37:03 +0100 Subject: [PATCH 31/41] mgr_osd_disk as standard metricset --- metricbeat/docs/fields.asciidoc | 87 ++------- metricbeat/include/list_common.go | 1 + metricbeat/module/ceph/fields.go | 2 +- .../mgr_cluster_disk/{mgr_data.go => data.go} | 0 .../mgr_cluster_disk_integration_test.go | 21 +-- .../ceph/mgr_cluster_health/_meta/data.json | 6 +- .../{mgr_data.go => data.go} | 0 .../mgr_cluster_health_integration_test.go | 23 +-- .../module/ceph/mgr_osd_disk/_meta/data.json | 100 +++-------- .../module/ceph/mgr_osd_disk/_meta/fields.yml | 169 +++--------------- metricbeat/module/ceph/mgr_osd_disk/data.go | 67 +++++++ .../module/ceph/mgr_osd_disk/manifest.yml | 54 ------ .../module/ceph/mgr_osd_disk/mgr_osd_disk.go | 93 ++++++++++ .../mgr_osd_disk_integration_test.go | 52 ++++++ .../ceph/mgr_osd_disk/mgr_osd_disk_test.go | 100 +++++++++++ metricbeat/module/ceph/mgrtest/password.go | 39 ++++ metricbeat/module/ceph/test_ceph_mgr.py | 2 + 17 files changed, 431 insertions(+), 385 deletions(-) rename metricbeat/module/ceph/mgr_cluster_disk/{mgr_data.go => data.go} (100%) rename metricbeat/module/ceph/mgr_cluster_health/{mgr_data.go => data.go} (100%) create mode 100644 metricbeat/module/ceph/mgr_osd_disk/data.go delete mode 100644 metricbeat/module/ceph/mgr_osd_disk/manifest.yml create mode 100644 metricbeat/module/ceph/mgr_osd_disk/mgr_osd_disk.go create mode 100644 metricbeat/module/ceph/mgr_osd_disk/mgr_osd_disk_integration_test.go create mode 100644 metricbeat/module/ceph/mgr_osd_disk/mgr_osd_disk_test.go create mode 100644 metricbeat/module/ceph/mgrtest/password.go diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index c83b57b74e1f..b3d663f289c1 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -4025,122 +4025,69 @@ type: keyword [float] === mgr_osd_disk -OSD disk metrics of Ceph cluster +mgr_osd_disk -*`ceph.mgr_osd_disk.nodes`*:: +*`ceph.mgr_osd_disk.id`*:: + -- -`nodes` contains information about active nodes. +Id of the pool -type: nested +type: long -- -*`ceph.mgr_osd_disk.stray`*:: +*`ceph.mgr_osd_disk.name`*:: + -- -`stray` contains information about stray OSD nodes. +Name of the pool -type: nested +type: keyword -- -[float] -=== summary - -`summary` contains general information about OSDs. - - - -*`ceph.mgr_osd_disk.summary.total_kb_used_omap`*:: +*`ceph.mgr_osd_disk.stats.available.bytes`*:: + -- -Total used space for omap (in KB) - -type: long +Available bytes of the pool --- - -*`ceph.mgr_osd_disk.summary.average_utilization`*:: -+ --- -Average space utilization type: long --- - -*`ceph.mgr_osd_disk.summary.dev`*:: -+ --- -Deviation - -type: long +format: bytes -- -*`ceph.mgr_osd_disk.summary.total_kb_avail`*:: +*`ceph.mgr_osd_disk.stats.objects`*:: + -- -Total available space (in KB) - -type: long - --- +Number of objects of the pool -*`ceph.mgr_osd_disk.summary.min_var`*:: -+ --- -Minimal variant type: long -- -*`ceph.mgr_osd_disk.summary.max_var`*:: +*`ceph.mgr_osd_disk.stats.used.bytes`*:: + -- -Maximal variant - -type: long - --- +Used bytes of the pool -*`ceph.mgr_osd_disk.summary.total_kb`*:: -+ --- -Total space (in KB) type: long --- - -*`ceph.mgr_osd_disk.summary.total_kb_used`*:: -+ --- -Total used space (in KB) - -type: long +format: bytes -- -*`ceph.mgr_osd_disk.summary.total_kb_used_data`*:: +*`ceph.mgr_osd_disk.stats.used.kb`*:: + -- -Total used space for data (in KB) - -type: long - --- +Used kb of the pool -*`ceph.mgr_osd_disk.summary.total_kb_used_meta`*:: -+ --- -Total used space for meta (in KB) type: long diff --git a/metricbeat/include/list_common.go b/metricbeat/include/list_common.go index 9b6383510744..0926771fbd7d 100644 --- a/metricbeat/include/list_common.go +++ b/metricbeat/include/list_common.go @@ -34,6 +34,7 @@ import ( _ "github.com/elastic/beats/metricbeat/module/ceph/cluster_status" _ "github.com/elastic/beats/metricbeat/module/ceph/mgr_cluster_disk" _ "github.com/elastic/beats/metricbeat/module/ceph/mgr_cluster_health" + _ "github.com/elastic/beats/metricbeat/module/ceph/mgr_osd_disk" _ "github.com/elastic/beats/metricbeat/module/ceph/monitor_health" _ "github.com/elastic/beats/metricbeat/module/ceph/osd_df" _ "github.com/elastic/beats/metricbeat/module/ceph/osd_tree" diff --git a/metricbeat/module/ceph/fields.go b/metricbeat/module/ceph/fields.go index eaf3e951d7ab..347d753a4c3f 100644 --- a/metricbeat/module/ceph/fields.go +++ b/metricbeat/module/ceph/fields.go @@ -32,5 +32,5 @@ func init() { // AssetCeph returns asset data. // This is the base64 encoded gzipped contents of ../metricbeat/module/ceph. func AssetCeph() string { - return "eJzsXM+P27oRvu9fMdhTHrBx2+seCiTZ4GXxmqyRTdpDUSi0NJbYpUiCpLxx//qClCzbEvXLoh2gtU/vxd7v+zQczgyHpN7CC27vIUaZ3QAYahjew+0HlNntDUCCOlZUGir4Pfz1BgDAfgW5SAqGNwA6E8pEseBrmt7DmjBt/1UhQ6LxHlJif4PGUJ7qe/jnrdbs9g5uM2Pk7b9uANYUWaLvHfJb4CTHWov9mK20KEoUsvoXjyL7+WH/6AfEghtCuQaTIeRoFI3tfxMDr6gQdKyIxATWSuTw4ePy06ICOJRxJIUV2qCKEqpf6i99snqk2U8HzrGddp+mmENBZEMoIyuGi9XWoD76zU4XEzxtfNEjzX7e7VDBoYJYOwNWqhs/XwuVE3MPbQE7kUYYwoIK/GYRw4grNCZBtX3XmJwurelpGRJmshC+1kKa7m1ig4owFmlDTOG31wtuX4VKppnsqcSFErfPbLVL0RzjDOMXvUAp4izQ2H0mEjaoNBV8iFaJgieLDWEFBiKvwcFhjxMQeiiej4bgWEjTN1vcft/0+9kIj23h9/mmb9hOHwuX1Spb9DqEIus1jRcKSRJNiiLdkWlYXGkfsKRgMiWKNJOFAYkKNMaiy3cqra+KGry4WMd6glpnWSEjaR0C41Dje2hCKqSeZryz6SmtNEZQTrVkJMaFy66BZezAQabAi3zVEYVrDWL1b4xNqAzaUlHBj5KiiKHCK0THhGESrZkgpsO/JaoYefPbqXLbEnYqE0wVSTA5y6DtwAcGrdZwnkGrVYwYtFrKrxu0Wm73oMl0kRBDLh4xZQqWt6d2lenCLQAuLo0cLxC6xDkvv7g4s18cdAmzRf/FdRX1wqBDlit40JV0GNl/DDQ1l6mrrPHwd/0SYlG0JtSp7M+ZeNWQiVfICd+CTDUQhUB5JUqsBx+9pS9wsVcNkB6u94ROFuuC+UP3SgiGpKlqgPxRW1BogR4yciTqLKwWeIC6yCOhk1BZouEMFtl6wtByb6ekkOcWY11TlIJq9/y+7BVF+a8Q9filV5TCnEiJSSTTSyv7+vHzu+Xy40OnvpArdofVrDHqojBVUaiWXQ9WvcBdobk27v7PG3eHfjK/edeLdorfXVt41xZe5aVCJ7Oj4tPzA1iMenNFrMveWXvkT/FWLpKO6c1RG5xomB8O7mBXiPJyLlPBgaxEYYDEhm6w5F00571HKBytZqVpejH0OVNL8oMXYYev8BVpmjXr8mkUXysQeGOtoASrMnoRZ+Vq06XyrTaYg1EUNRgBhDER2+xuhPv+6fnht06Z7YJjmsIvLpdbR1r+3kaqe9Cq0Fk0YJCe5kGL9sPX78+foANwx+qdqjAwXdtPKBLsgtoxUR/KeCM+PnQiexaW05/Ah1EHuK3EaKb+b1uJfQ/xsorcGt667Cwil/Sd47+hHP543+3XO0qREzmf0qKMpswxxFNalDGUrjSdRbcvQ7UkMQ6SbkizxprG93fP3x9641x3/+bD2Ftsnqe7inicnSp/mO8K4+i+G8rof0irXTSV8tmxFR1o+9iqyDZcrndwvbne/cJVMNd0f033PtZruv+fTPdl9FsL9SsS/578F5QAe/JrMdACvxYDfXRd6Xsq5chioMhzovzlQLM90aLwVQMl3kE9kCJHRZinLnh6fphcDJQ7ncEiRekHxQnxgmxQkRSjUMP1rsSrZPSh7uuizcyqaEP7GWpjzw9OpaHJxBCVUx7NDVOfKac5YTbcUdLabD3gIj/nc5Gfo7h2hr1YDDuaNmFnzCTq+YWFd8aOKi+OhczO814h3mzf7AVLVOubJuXEXrDFsOGUx3imlrBMI4Uk8aeGEXvhS0ZizJEb+N0+mjsBSDlq7SvRD3Yt7bf+VvRJ+aixOrW2swxUGxpPzj670YtsMuter3jX0S21zWHcy/L8ZZcyGFrbwJBTe3V5lgiHTM4GvoE6ZvQNmJdy6cwwbAUYsMShxljkOTURIwZ5vI3yLjwYY6CW4g8OHSp0oBx6COqaQUq2PZeidxZ8oqCGjfi5bdRD4LdRYEUtG3H/vrKb50Kwlo93zO+hYC1EuWEbPEZbjZ0H2EZsai6tsu5jYBa+FVWmgT/6j6fEjCI3ERWRIsYvfzDcf3AY8PinJ7AgTVcZCub7qwSe8+Uw5F/V89/De3eQwZ1v7zhJfkh6cCcgAKtFM8jHEPcf7x9P/NU+qJDojvHyztPz7WcOwP0Pd2i/n7w5jY1CnFtsWYyzzN8zb7u7AwPVTvvJ7fcgfU2RoB+odk8huvvNU1i8nadLtlBnwruH6IGPM8oShfOaDY6kE+ksGyM9bYzzbu/gT6q9dep4go9+iMvvXkhFc6K2EVmvKaemuTab9lTLEgw6wS6/JzS8RRliDThzP/IaEK8BcQbVNSBeA+KogFhXsoJTI0Yfcp58C/wYv6pzG33acSfsZRzqLtN+L7O6dLg7Zfv5yX8do2UcmHe291NpjAHW/aO3evnzn/yP90P05bWBYNRlR3uY1l0ICMbqtkaHSRnRJipkQtrLopI5afcRhp6X5givGXJ4JRp82D0pf4ZvfSE5Dj2uNkJh2X5aMJEGvX/xN5Hur180BYy5EXIoLqc6DqruM9VxOHlam6Dqnp+/hRN39ms/cwUOTroTnI9o05prh/sryfDe2PQXnaDM3D1M1xEpNEnx8DzCyETX0QmdbgN3I9RWN/QS4aZm6+zzJrihMUYxIzrg9Zia12LcAWUMU8Lc/wHlMSsShCxJ7kDrBNDEi54cZ911rPFn3CCvXaS82r4RrOgwWX0p7pKqSsftUXXiZccQ2vZHO3r0yTTiRR5oFunmfXcoT9Mm5UVZ6m5edw9eV7F6rhdhNEaxwiApeqPg3LZ1HfFc79rGusH4ebHQJxSsivgFzYWDYIO3Mxx6OiZhmXsCotDJHWRCmzvXAeqJi97OTcBhsVBdm3f+RskMI1XEO2Bg1FpAoyTKzejV1u1VE78tetbjpSbfZB6bvRx4X7PS1685dRwcYxuwt3Uz+kROm47qklEbyliJbt2AC/PmL2+3qO/gz2+5+M0fy/tbLycb3q6KqrsUYl2dGqpe0ONeH1rxdgb3jibdqSPirvbiXlAXbejLxfv73S4kUGPzXcESWCEU0o5SIl79Z3rPU85ZQ5TI4JBraYy+jCnk1sRkrSv+M/VIopBX8cotqMuk3whlrQTrzjXMvZHtA/mFifUx2bUUrLALZNTDLkYnZbmMvOg7OFpaxi16rc6wL2rbX7javaFtnLUu8L6NGTY6Q/PvZXWk678BAAD//xvf8Dg=" + return "eJzsXFFv274Rf8+nOORpA1Jve83DgP7TYg22tkHTYg/DoNLiWeJCkQRJJfO3H0jZsi2RkizTLrC//dRY9u9+vDsej3fnvoMXXN9Djqq8AbDMcryH2wdU5e0NAEWTa6Ysk+Ie/noDAOAeQSVpzfEGwJRS2yyXYsWKe1gRbty7GjkSg/dQEPcZtJaJwtzDv26N4bd3cFtaq27/fQOwYsipuffI70CQClsu7mXXyqFoWavNOwFG7vXTfekn5FJYwoQBWyJUaDXL3b+JhTfUCCbXRCGFlZYVPHx8+rTYAOzTOKDCa2NRZ5SZl/ZhiNYANfeK4BzqafvqktknRF4J42TJcbFcWzQHn9ny4lIUnQcD1Nzr/RYVPCrIlVfghnXn4yupK2LvoU9gS9JKS3hSgt8dYhpytUGalNsPg3Q+ta6nlUi4LVP4Wg/peG+Tr6gJ55mxxNZhfb3g+k1qepzKvja40OAOqa11KVZhXmL+YhaoZF4mst1nouAVtWFSjInVshZ08Up4jYmEt+DgsacRSG2K5wMTHBLp+mZPdtg3w342wWN7+EO+GTLbfFv4U22ji0GH0GS1YvlCI6HZUVEkHpnGyTX6AScUbKllXZSqtqBQg8Fcxnxnw/VNM4sXJ+ulzmDrNStVppxDYJ7KvvsqZFKZ45R3Nj6NlqYQqphRnOS48KdrYhpbcFAFiLpaRqJwy0Eu/4O5TXWC9lhs4CdR0cQyGSRicsKRZisuiY34t0Kdo+g+PZZun8KWJcVCE4r0LEbbgo8YreVwHqO1LCYYraXy64zW0o0bTRULSiy5eMRUBTi5A7mrKhb+AnBxauTwghAj57384uTs7nIQI+aS/ovzqtuLQYSWT3jQp3SYuTcTbc2nwmfWuP+5YQq5rHsbaq7051K+GSjlG1RErEEVBohGYGJDSq5Gl97jlzjZ2xjIjOd70tDFqubh0L2UkiPpshoR/mgcKPRA9yUKJPosUh3wiOi6yqShqU6JjjM4ZOcJY9e9LZNanZuMc03ZEGrd88fTICkmfgWpxy+DpDRWRCmkmSouzezbx8/vn54+fojyS3lj91jdHKNNCgudpSrZDWC1F9wl2mvh7ndeuNv3k9OLd4Noc/zuWsK7lvA2XioNTRIVAzhzPJN1lznXCI90u3YlZTiziSbXsxT+hVQ4KtJZ2SwuGvZ7XKaE1YZn2trAF388O1rbosA0bV0gxJ+gI8/uZZmS2ssyyKu7axXq1Sm79uvzB3AYbqEi33VG5aopfPfD9pwNrYpMI6HrubeWJ05yrFBY+Jtbmq/VMoHGQCBi7uWX3jpBoV0VTTDLT2M1We91kp3unARmLMvNous7AX106HnrZUysZJfkjqhAY7EbhXpsu2bc0Qp8M8YMhkLwIauAU0d5PX4YlOR1EDLUocSQwYIin7waxrUAI5rY55jLqmI248SiyNdZFcODKQrqMX7w6LBBd5fgAQHtrUEpvj4Xo/cO/EhCHR2Jc+toQEBYR4kZ9XQkwjcAv8+l5D0fj+zvsWAtZZNaJ4/RjuMp2dCTYxYv2Dn4SGI3FfwxXEjIOUNhMyYzTWyY/mi4f/AY8Pinr+BAuq4yFsx3Td9AJxDG/Guz/nv4zecjvhMZ6fntC93r3iaQ6tAsiimChxux0wV/cwuVCn3DRUT7nP01J5D9T99eHRbe3cZWI56abDmMs+xfIWkkOQ6mEGPJjofbS3bctQ5Ibtkr+nX458fmPIHwAiNRoMf1i6QYBmrdU8pux2KeFPeNqBT3sB/QYMwBD6R8XysM5UeDWdh0eL+IAfi8ZJxq7KbeM4REkXYh4w1ZUcbtMkXUtxjIrousbLd6dJyID0GELT7+l5lgnjpdwMcwxP59slcBglneG4FqD2TNKqLXGVmtmGC2ezc7blVPDRhEwVqX07UpsxFfGOj09yQ/fPvx/AkCgDuNahK+eKa4A/q/rwHxGhCvAXGmgGtAvEBAbDNZKZiVk9tRR8/rHuJv8lxKZvRCVZ5q6mRXEt+Mh20Lq5+/hhvnPeXAaU2BT40yRqTulp6skLxb+d9/GxPfNHiTiW66u+Niz1A5HxfKibFZrSjpX4saybRfRxhbL6sQ3koU8EYMhLDP33CKLddYqbEpPy24LJK2Uf4hi10XpUtgWgNlR65iJk/K7jMzeTp6xtik7J6fv6cjd/YBjVMJjm66Gc5HjO3ttf3+Ch3vjR3/kxRUpZ+Y8xWR2pACgYlGF4eTBhdpcfvZPZfdsEuEm1ZatM5L8ZXlmOWcmISDDK1ch3EHjHMsCPd/ARM5rylCSekdGEMBbb4YOOOcu05V/gmzvq2LNEPIr5LXEZW1ve1Lsmocd4DVzPmEFNx2M+UD/FSRibpKtItMdzIZuMxdWGlGGpmfkY0bL5asnusnCx0rbjBIgcEoeGrZuo14vnbtYt1o/LxY6JMalnX+gvbCQbAjNxoOAxWTtJIHAqI09A5KaeydrwANxMVg5SahWRxUrHkXLpScoKSN4C0wcOY0YFAR7Xf0cu171SSsi4H7eMMptJmnnl4efKhYGarXzLWDl9gHHCzdnPA7AmYaicYyzht05wZC2j/85d0azR38+Z2QfwzH8uHSy2zFu1tRo22XwTZTQ5ufUvn/6GEjNxrcI0W62TOhJbaQfgQuIjb1GOhuEteHBGbdeVdzCkuEWjkrUfkWnpE9TzrnFNEgg0duqXH2MiWRWxFb9oaxT+SjiEaxiVf+Qt0c+p1Q1jtg/VzDqbOzIZBfeLBex2avY7P/H2Oz/wsAAP//vuBo9A==" } diff --git a/metricbeat/module/ceph/mgr_cluster_disk/mgr_data.go b/metricbeat/module/ceph/mgr_cluster_disk/data.go similarity index 100% rename from metricbeat/module/ceph/mgr_cluster_disk/mgr_data.go rename to metricbeat/module/ceph/mgr_cluster_disk/data.go diff --git a/metricbeat/module/ceph/mgr_cluster_disk/mgr_cluster_disk_integration_test.go b/metricbeat/module/ceph/mgr_cluster_disk/mgr_cluster_disk_integration_test.go index 22b56c5da477..66f9ae41de46 100644 --- a/metricbeat/module/ceph/mgr_cluster_disk/mgr_cluster_disk_integration_test.go +++ b/metricbeat/module/ceph/mgr_cluster_disk/mgr_cluster_disk_integration_test.go @@ -20,43 +20,32 @@ package mgr_cluster_disk import ( - "encoding/json" - "fmt" - "net/http" "testing" "github.com/stretchr/testify/require" "github.com/elastic/beats/libbeat/tests/compose" mbtest "github.com/elastic/beats/metricbeat/mb/testing" + "github.com/elastic/beats/metricbeat/module/ceph/mgrtest" ) +const user = "demo" + func TestData(t *testing.T) { service := compose.EnsureUpWithTimeout(t, 120, "ceph-mgr") f := mbtest.NewReportingMetricSetV2Error(t, - getConfig(service.HostForPort(8003), getPassword(t, service.HostForPort(5000)))) + getConfig(service.HostForPort(8003), mgrtest.GetPassword(t, service.HostForPort(5000), user))) err := mbtest.WriteEventsReporterV2Error(f, t, "") require.NoError(t, err) } -func getPassword(t *testing.T, host string) string { - response, err := http.Get(fmt.Sprintf("http://%s/restful-list-keys.json", host)) - require.NoError(t, err) - - defer response.Body.Close() - - accounts := map[string]string{} - json.NewDecoder(response.Body).Decode(&accounts) - return accounts["demo"] -} - func getConfig(host, password string) map[string]interface{} { return map[string]interface{}{ "module": "ceph", "metricsets": []string{"mgr_cluster_disk"}, "hosts": []string{host}, - "username": "demo", + "username": user, "password": password, "ssl.verification_mode": "none", } diff --git a/metricbeat/module/ceph/mgr_cluster_health/_meta/data.json b/metricbeat/module/ceph/mgr_cluster_health/_meta/data.json index 9b22b535b9a3..14cee62d3281 100644 --- a/metricbeat/module/ceph/mgr_cluster_health/_meta/data.json +++ b/metricbeat/module/ceph/mgr_cluster_health/_meta/data.json @@ -17,15 +17,15 @@ } }, "event": { - "dataset": "ceph.cluster_health", + "dataset": "ceph.mgr_cluster_health", "duration": 115000, "module": "ceph" }, "metricset": { - "name": "cluster_health" + "name": "mgr_cluster_health" }, "service": { - "address": "127.0.0.1:5000", + "address": "127.0.0.1:8003", "type": "ceph" } } diff --git a/metricbeat/module/ceph/mgr_cluster_health/mgr_data.go b/metricbeat/module/ceph/mgr_cluster_health/data.go similarity index 100% rename from metricbeat/module/ceph/mgr_cluster_health/mgr_data.go rename to metricbeat/module/ceph/mgr_cluster_health/data.go diff --git a/metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health_integration_test.go b/metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health_integration_test.go index 3e53f876565f..5361c9ae8812 100644 --- a/metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health_integration_test.go +++ b/metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health_integration_test.go @@ -20,43 +20,32 @@ package mgr_cluster_health import ( - "encoding/json" - "fmt" - "net/http" "testing" + "github.com/elastic/beats/metricbeat/module/ceph/mgrtest" + "github.com/stretchr/testify/require" "github.com/elastic/beats/libbeat/tests/compose" mbtest "github.com/elastic/beats/metricbeat/mb/testing" ) +const user = "demo" + func TestData(t *testing.T) { service := compose.EnsureUpWithTimeout(t, 120, "ceph-mgr") f := mbtest.NewReportingMetricSetV2Error(t, - getConfig(service.HostForPort(8003), getPassword(t, service.HostForPort(5000)))) + getConfig(service.HostForPort(8003), mgrtest.GetPassword(t, service.HostForPort(5000), user))) err := mbtest.WriteEventsReporterV2Error(f, t, "") require.NoError(t, err) } - -func getPassword(t *testing.T, host string) string { - response, err := http.Get(fmt.Sprintf("http://%s/restful-list-keys.json", host)) - require.NoError(t, err) - - defer response.Body.Close() - - accounts := map[string]string{} - json.NewDecoder(response.Body).Decode(&accounts) - return accounts["demo"] -} - func getConfig(host, password string) map[string]interface{} { return map[string]interface{}{ "module": "ceph", "metricsets": []string{"mgr_cluster_health"}, "hosts": []string{host}, - "username": "demo", + "username": user, "password": password, "ssl.verification_mode": "none", } diff --git a/metricbeat/module/ceph/mgr_osd_disk/_meta/data.json b/metricbeat/module/ceph/mgr_osd_disk/_meta/data.json index f3a9f80d0b8f..ce3506e1cf17 100644 --- a/metricbeat/module/ceph/mgr_osd_disk/_meta/data.json +++ b/metricbeat/module/ceph/mgr_osd_disk/_meta/data.json @@ -1,89 +1,31 @@ { - "@timestamp": "2020-02-12T13:08:24.727Z", - "@metadata": { - "beat": "metricbeat", - "type": "_doc", - "version": "8.0.0" - }, - "agent": { - "type": "metricbeat", - "ephemeral_id": "ee373afc-c2ca-4308-91a2-651e4081d823", - "hostname": "host", - "id": "993db589-f9d8-40f3-94cb-43b2b69fa9f4", - "version": "8.0.0" + "@timestamp": "2017-10-12T08:05:34.853Z", + "ceph": { + "pool_disk": { + "id": 0, + "name": "rbd", + "stats": { + "available": { + "bytes": 0 + }, + "objects": 0, + "used": { + "bytes": 0, + "kb": 0 + } + } + } }, "event": { - "dataset": "http.json_namespace", - "module": "ceph", - "duration": 43294329 + "dataset": "ceph.mgr_osd_disk", + "duration": 115000, + "module": "ceph" }, "metricset": { - "name": "mgr_osd_disk", - "period": 60000 + "name": "mgr_osd_disk" }, "service": { - "address": "localhost:8003", + "address": "127.0.0.1:8003", "type": "ceph" - }, - "ecs": { - "version": "1.4.0" - }, - "ceph": { - "mgr_osd_disk": { - "nodes": [ - { - "depth": 2, - "reweight": 1, - "pgs": 164, - "crush_weight": 0.009796142578125, - "status": "up", - "id": 0, - "name": "osd.0", - "type_id": 0, - "kb_used_data": 6950784, - "kb_used_omap": 7, - "kb_used_meta": 1048568, - "kb_avail": 2486336, - "var": 1, - "type": "osd", - "kb": 10485760, - "kb_used": 7999424, - "utilization": 76.2884521484375 - } - ], - "stray": [ - { - "depth": 2, - "reweight": 1, - "pgs": 164, - "crush_weight": 0.009796142578125, - "status": "down", - "id": 1, - "name": "osd.1", - "type_id": 0, - "kb_used_data": 6950784, - "kb_used_omap": 7, - "kb_used_meta": 1048568, - "kb_avail": 2486336, - "var": 1, - "type": "osd", - "kb": 10485760, - "kb_used": 7999424, - "utilization": 76.2884521484375 - } - ], - "summary": { - "total_kb_used_omap": 7, - "average_utilization": 76.2884521484375, - "dev": 0, - "total_kb_avail": 2486336, - "min_var": 1, - "max_var": 1, - "total_kb": 10485760, - "total_kb_used": 7999424, - "total_kb_used_data": 6950784, - "total_kb_used_meta": 1048568 - } - } } } diff --git a/metricbeat/module/ceph/mgr_osd_disk/_meta/fields.yml b/metricbeat/module/ceph/mgr_osd_disk/_meta/fields.yml index 73fbe49d8fa3..d5a02fd9ae6e 100644 --- a/metricbeat/module/ceph/mgr_osd_disk/_meta/fields.yml +++ b/metricbeat/module/ceph/mgr_osd_disk/_meta/fields.yml @@ -1,153 +1,32 @@ - name: mgr_osd_disk type: group description: > - OSD disk metrics of Ceph cluster + mgr_osd_disk release: beta fields: - - name: nodes - type: nested + - name: id + type: long description: > - `nodes` contains information about active nodes. - fields: - - name: depth - type: long - description: Depth - - name: reweight - type: long - description: Reweight (controls how much data the system tries to allocate to the OSD) - - name: pgs - type: long - description: Number of PGs - - name: crush_weight - type: scaled_float - description: CRUSH weight - - name: status - type: keyword - description: Node status - - name: id - type: long - description: ID - - name: name - type: keyword - description: Name - - name: type_id - type: long - description: Type ID - - name: kb_used_data - type: long - description: Used data (in KB) - - name: kb_used_omap - type: long - description: Used omap (in KB) - - name: kb_used_meta - type: long - description: Used meta (in KB) - - name: kb_avail - type: long - description: Available space (in KB) - - name: var - type: long - description: Var - - name: type - type: keyword - description: Type - - name: kb - type: long - description: Total space (in KB) - - name: kb_used - type: long - description: Used space (in KB) - - name: Utilization - type: long - description: Space utilization - - name: stray - type: nested + Id of the pool + - name: name + type: keyword description: > - `stray` contains information about stray OSD nodes. - fields: - - name: depth - type: long - description: Depth - - name: reweight - type: long - description: Reweight (controls how much data the system tries to allocate to the OSD) - - name: pgs - type: long - description: Number of PGs - - name: crush_weight - type: scaled_float - description: CRUSH weight - - name: status - type: keyword - description: Node status - - name: id - type: long - description: ID - - name: name - type: keyword - description: Name - - name: type_id - type: long - description: Type ID - - name: kb_used_data - type: long - description: Used space for data (in KB) - - name: kb_used_omap - type: long - description: Used space for omap (in KB) - - name: kb_used_meta - type: long - description: Used space for meta (in KB) - - name: kb_avail - type: long - description: Available space (in KB) - - name: var - type: long - description: Var - - name: type - type: keyword - description: Type - - name: kb - type: long - description: Total space (in KB) - - name: kb_used - type: long - description: Used space (in KB) - - name: utilization - type: long - description: Space utilization - - name: summary - type: group + Name of the pool + - name: stats.available.bytes + type: long description: > - `summary` contains general information about OSDs. - fields: - - name: total_kb_used_omap - type: long - description: Total used space for omap (in KB) - - name: average_utilization - type: long - description: Average space utilization - - name: dev - type: long - description: Deviation - - name: total_kb_avail - type: long - description: Total available space (in KB) - - name: min_var - type: long - description: Minimal variant - - name: max_var - type: long - description: Maximal variant - - name: total_kb - type: long - description: Total space (in KB) - - name: total_kb_used - type: long - description: Total used space (in KB) - - name: total_kb_used_data - type: long - description: Total used space for data (in KB) - - name: total_kb_used_meta - type: long - description: Total used space for meta (in KB) + Available bytes of the pool + format: bytes + - name: stats.objects + type: long + description: > + Number of objects of the pool + - name: stats.used.bytes + type: long + description: > + Used bytes of the pool + format: bytes + - name: stats.used.kb + type: long + description: > + Used kb of the pool diff --git a/metricbeat/module/ceph/mgr_osd_disk/data.go b/metricbeat/module/ceph/mgr_osd_disk/data.go new file mode 100644 index 000000000000..b0fdb299a639 --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_disk/data.go @@ -0,0 +1,67 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mgr_osd_disk + +import ( + "github.com/pkg/errors" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/metricbeat/module/ceph/mgr" +) + +type DfResponse struct { + Pools []struct { + ID int64 `json:"id"` + Name string `json:"name"` + Stats struct { + BytesUsed uint64 `json:"bytes_used"` + MaxAvail uint64 `json:"max_avail"` + Objects uint64 `json:"objects"` + KbUsed uint64 `json:"kb_used"` + } `json:"stats"` + } `json:"pools"` +} + +func eventsMapping(content []byte) ([]common.MapStr, error) { + var response DfResponse + err := mgr.UnmarshalResponse(content, &response) + if err != nil { + return nil, errors.Wrap(err, "could not get response data") + } + + var events []common.MapStr + for _, Pool := range response.Pools { + event := common.MapStr{ + "name": Pool.Name, + "id": Pool.ID, + "stats": common.MapStr{ + "used": common.MapStr{ + "bytes": Pool.Stats.BytesUsed, + "kb": Pool.Stats.KbUsed, + }, + "available": common.MapStr{ + "bytes": Pool.Stats.MaxAvail, + }, + "objects": Pool.Stats.Objects, + }, + } + + events = append(events, event) + } + return events, nil +} diff --git a/metricbeat/module/ceph/mgr_osd_disk/manifest.yml b/metricbeat/module/ceph/mgr_osd_disk/manifest.yml deleted file mode 100644 index a58b2ba3455d..000000000000 --- a/metricbeat/module/ceph/mgr_osd_disk/manifest.yml +++ /dev/null @@ -1,54 +0,0 @@ -default: true -input: - module: http - metricset: json - defaults: - method: POST - namespace: "json_namespace" - basepath: "/request" - headers: - Content-Type: application/json - query: - wait: "1" - body: '{"prefix": "osd df", "format": "json"}' - -processors: - - extract_array: - field: http.json_namespace.finished - mappings: - ceph.request: 0 - - decode_json_fields: - fields: ["ceph.request.outb"] - process_array: true - target: "ceph.mgr_osd_disk" - - drop_fields: - ignore_missing: true - fields: ["http", "ceph.request", "ceph.mgr_osd_disk.nodes.pool_weight"] - - script: - lang: javascript - source: > - function process(event) { - var nodes = event.Get("ceph.mgr_osd_disk.nodes") - if (nodes != null) { - var filtered = []; - nodes.forEach(function(node, index) { - if (node["pool_weights"] !== undefined) { - delete node["pool_weights"]; - } - filtered.push(node); - }); - event.Put("ceph.mgr_osd_disk.nodes", filtered); - } - - var stray = event.Get("ceph.mgr_osd_disk.stray") - if (stray != null) { - var filtered = []; - stray.forEach(function(node, index) { - if (node["pool_weights"] !== undefined) { - delete node["pool_weights"]; - } - filtered.push(node); - }); - event.Put("ceph.mgr_osd_disk.stray", filtered); - } - } diff --git a/metricbeat/module/ceph/mgr_osd_disk/mgr_osd_disk.go b/metricbeat/module/ceph/mgr_osd_disk/mgr_osd_disk.go new file mode 100644 index 000000000000..b6f004cfba70 --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_disk/mgr_osd_disk.go @@ -0,0 +1,93 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mgr_osd_disk + +import ( + "fmt" + + "github.com/elastic/beats/metricbeat/helper" + "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/metricbeat/mb/parse" +) + +const ( + defaultScheme = "https" + defaultPath = "/request" + defaultQueryParams = "wait=1" + + cephPrefix = "df" +) + +var ( + hostParser = parse.URLHostParserBuilder{ + DefaultScheme: defaultScheme, + DefaultPath: defaultPath, + QueryParams: defaultQueryParams, + }.Build() +) + +func init() { + mb.Registry.MustAddMetricSet("ceph", "mgr_osd_disk", New, + mb.WithHostParser(hostParser), + ) +} + +type MetricSet struct { + mb.BaseMetricSet + *helper.HTTP +} + +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + http, err := helper.NewHTTP(base) + if err != nil { + return nil, err + } + http.SetMethod("POST") + http.SetHeader("Content-Type", "application/json") + http.SetHeader("Accept", "application/json") + http.SetBody([]byte(fmt.Sprintf(`{"prefix": "%s", "format": "json"}`, cephPrefix))) + + return &MetricSet{ + base, + http, + }, nil +} + +// Fetch methods implements the data gathering and data conversion to the right +// format. It publishes the event which is then forwarded to the output. In case +// of an error set the Error field of mb.Event or simply call report.Error(). +func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { + content, err := m.HTTP.FetchContent() + if err != nil { + return err + } + + events, err := eventsMapping(content) + if err != nil { + return err + } + + for _, event := range events { + reported := reporter.Event(mb.Event{MetricSetFields: event}) + if !reported { + m.Logger().Debug("error reporting event") + return nil + } + } + return nil +} diff --git a/metricbeat/module/ceph/mgr_osd_disk/mgr_osd_disk_integration_test.go b/metricbeat/module/ceph/mgr_osd_disk/mgr_osd_disk_integration_test.go new file mode 100644 index 000000000000..42982f131694 --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_disk/mgr_osd_disk_integration_test.go @@ -0,0 +1,52 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build integration,linux + +package mgr_osd_disk + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/elastic/beats/libbeat/tests/compose" + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + "github.com/elastic/beats/metricbeat/module/ceph/mgrtest" +) + +const user = "demo" + +func TestData(t *testing.T) { + service := compose.EnsureUpWithTimeout(t, 120, "ceph-mgr") + + f := mbtest.NewReportingMetricSetV2Error(t, + getConfig(service.HostForPort(8003), mgrtest.GetPassword(t, service.HostForPort(5000), user))) + err := mbtest.WriteEventsReporterV2Error(f, t, "") + require.NoError(t, err) +} + +func getConfig(host, password string) map[string]interface{} { + return map[string]interface{}{ + "module": "ceph", + "metricsets": []string{"mgr_osd_disk"}, + "hosts": []string{host}, + "username": user, + "password": password, + "ssl.verification_mode": "none", + } +} diff --git a/metricbeat/module/ceph/mgr_osd_disk/mgr_osd_disk_test.go b/metricbeat/module/ceph/mgr_osd_disk/mgr_osd_disk_test.go new file mode 100644 index 000000000000..ca1e8ec0ff68 --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_disk/mgr_osd_disk_test.go @@ -0,0 +1,100 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mgr_osd_disk + +import ( + "io/ioutil" + "net/http" + "net/http/httptest" + "path/filepath" + "testing" + + "github.com/elastic/beats/libbeat/common" + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + + "github.com/stretchr/testify/assert" +) + +func TestFetchEventContents(t *testing.T) { + absPath, err := filepath.Abs("../_meta/testdata/") + assert.NoError(t, err) + + response, err := ioutil.ReadFile(absPath + "/mgr_df_sample_response.json") + assert.NoError(t, err) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + w.Header().Set("Content-Type", "application/json;") + w.Write(response) + })) + defer server.Close() + + config := map[string]interface{}{ + "module": "ceph", + "metricsets": []string{"mgr_osd_disk"}, + "hosts": []string{server.URL}, + } + + f := mbtest.NewReportingMetricSetV2Error(t, config) + events, errs := mbtest.ReportingFetchV2Error(f) + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) + } + assert.NotEmpty(t, events) + event := events[0].MetricSetFields + + t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event.StringToPrint()) + + assert.EqualValues(t, "rbd", event["name"]) + assert.EqualValues(t, 1, event["id"]) + + stats := event["stats"].(common.MapStr) + + used := stats["used"].(common.MapStr) + assert.EqualValues(t, 0, used["bytes"]) + assert.EqualValues(t, 0, used["kb"]) + + available := stats["available"].(common.MapStr) + assert.EqualValues(t, uint64(9123004416), available["bytes"]) +} + +func TestFetchEventContents_Failed(t *testing.T) { + absPath, err := filepath.Abs("../_meta/testdata/") + assert.NoError(t, err) + + response, err := ioutil.ReadFile(absPath + "/mgr_failed_response.json") + assert.NoError(t, err) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + w.Header().Set("Content-Type", "application/json;") + w.Write([]byte(response)) + })) + defer server.Close() + + config := map[string]interface{}{ + "module": "ceph", + "metricsets": []string{"mgr_osd_disk"}, + "hosts": []string{server.URL}, + } + + f := mbtest.NewReportingMetricSetV2Error(t, config) + events, errs := mbtest.ReportingFetchV2Error(f) + assert.Empty(t, events) + assert.NotEmpty(t, errs) +} diff --git a/metricbeat/module/ceph/mgrtest/password.go b/metricbeat/module/ceph/mgrtest/password.go new file mode 100644 index 000000000000..c3b646105a26 --- /dev/null +++ b/metricbeat/module/ceph/mgrtest/password.go @@ -0,0 +1,39 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mgrtest + +import ( + "encoding/json" + "fmt" + "net/http" + "testing" + + "github.com/stretchr/testify/require" +) + +// GetPassword method returns the password of the given user. Used to authenticate client with Restful API. +func GetPassword(t *testing.T, host, user string) string { + response, err := http.Get(fmt.Sprintf("http://%s/restful-list-keys.json", host)) + require.NoError(t, err) + + defer response.Body.Close() + + accounts := map[string]string{} + json.NewDecoder(response.Body).Decode(&accounts) + return accounts[user] +} diff --git a/metricbeat/module/ceph/test_ceph_mgr.py b/metricbeat/module/ceph/test_ceph_mgr.py index c5bebfab691e..97640508c303 100644 --- a/metricbeat/module/ceph/test_ceph_mgr.py +++ b/metricbeat/module/ceph/test_ceph_mgr.py @@ -36,6 +36,8 @@ def get_password(self): @parameterized.expand([ "mgr_cluster_disk", + "mgr_cluster_health", + "mgr_osd_disk" ]) @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") def test_ceph_mgr(self, metricset): From 3b6982e3b36efff39f455610888d0dbbe3cae5ba Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Thu, 13 Feb 2020 20:21:10 +0100 Subject: [PATCH 32/41] mgr_osd_perf as standard metricset --- metricbeat/docs/fields.asciidoc | 38 ++++++-- metricbeat/include/list_common.go | 1 + .../mgr_osd_perf_sample_response.json | 17 ++++ metricbeat/module/ceph/fields.go | 2 +- .../ceph/mgr_cluster_health/_meta/data.json | 2 +- .../module/ceph/mgr_osd_disk/_meta/data.json | 2 +- .../module/ceph/mgr_osd_perf/_meta/data.json | 19 ++-- .../module/ceph/mgr_osd_perf/_meta/fields.yml | 46 +++------ metricbeat/module/ceph/mgr_osd_perf/data.go | 62 ++++++++++++ .../module/ceph/mgr_osd_perf/manifest.yml | 26 ----- .../module/ceph/mgr_osd_perf/mgr_osd_perf.go | 93 ++++++++++++++++++ .../mgr_osd_perf_integration_test.go | 52 ++++++++++ .../ceph/mgr_osd_perf/mgr_osd_perf_test.go | 97 +++++++++++++++++++ metricbeat/module/ceph/test_ceph_mgr.py | 3 +- 14 files changed, 377 insertions(+), 83 deletions(-) create mode 100644 metricbeat/module/ceph/_meta/testdata/mgr_osd_perf_sample_response.json create mode 100644 metricbeat/module/ceph/mgr_osd_perf/data.go delete mode 100644 metricbeat/module/ceph/mgr_osd_perf/manifest.yml create mode 100644 metricbeat/module/ceph/mgr_osd_perf/mgr_osd_perf.go create mode 100644 metricbeat/module/ceph/mgr_osd_perf/mgr_osd_perf_integration_test.go create mode 100644 metricbeat/module/ceph/mgr_osd_perf/mgr_osd_perf_test.go diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index b3d663f289c1..4943e6f8606a 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -4100,28 +4100,48 @@ OSD performance metrics of Ceph cluster -*`ceph.mgr_osd_perf.pg_ready`*:: +*`ceph.mgr_osd_perf.id`*:: + -- -Placement Group readiness status +OSD ID -type: boolean +type: long -- -[float] -=== osdstats +*`ceph.mgr_osd_perf.stats.commit_latency_ms`*:: ++ +-- +Commit latency in ms + +type: long + +-- + +*`ceph.mgr_osd_perf.stats.apply_latency_ms`*:: ++ +-- +Apply latency in ms -`stray` contains OSD statistics. +type: long +-- +*`ceph.mgr_osd_perf.stats.commit_latency_ns`*:: ++ +-- +Commit latency in ns -*`ceph.mgr_osd_perf.osdstats.osd_perf_infos`*:: +type: long + +-- + +*`ceph.mgr_osd_perf.stats.apply_latency_ns`*:: + -- -OSD performance statistics +Apply latency in ns -type: nested +type: long -- diff --git a/metricbeat/include/list_common.go b/metricbeat/include/list_common.go index 0926771fbd7d..f8cabf377020 100644 --- a/metricbeat/include/list_common.go +++ b/metricbeat/include/list_common.go @@ -35,6 +35,7 @@ import ( _ "github.com/elastic/beats/metricbeat/module/ceph/mgr_cluster_disk" _ "github.com/elastic/beats/metricbeat/module/ceph/mgr_cluster_health" _ "github.com/elastic/beats/metricbeat/module/ceph/mgr_osd_disk" + _ "github.com/elastic/beats/metricbeat/module/ceph/mgr_osd_perf" _ "github.com/elastic/beats/metricbeat/module/ceph/monitor_health" _ "github.com/elastic/beats/metricbeat/module/ceph/osd_df" _ "github.com/elastic/beats/metricbeat/module/ceph/osd_tree" diff --git a/metricbeat/module/ceph/_meta/testdata/mgr_osd_perf_sample_response.json b/metricbeat/module/ceph/_meta/testdata/mgr_osd_perf_sample_response.json new file mode 100644 index 000000000000..8d2dcf75b580 --- /dev/null +++ b/metricbeat/module/ceph/_meta/testdata/mgr_osd_perf_sample_response.json @@ -0,0 +1,17 @@ +{ + "failed": [], + "finished": [ + { + "command": "osd perf format=json-pretty", + "outb": "{\n \"pg_ready\": true,\n \"osdstats\": {\n \"osd_perf_infos\": [\n {\n \"id\": 1,\n \"perf_stats\": {\n \"commit_latency_ms\": 1,\n \"apply_latency_ms\": 2,\n \"commit_latency_ns\": 3,\n \"apply_latency_ns\": 4\n }\n }\n ]\n }\n\n}\n", + "outs": "" + } + ], + "has_failed": false, + "id": "140301322418768", + "is_finished": true, + "is_waiting": false, + "running": [], + "state": "success", + "waiting": [] +} diff --git a/metricbeat/module/ceph/fields.go b/metricbeat/module/ceph/fields.go index 347d753a4c3f..85000c7b556e 100644 --- a/metricbeat/module/ceph/fields.go +++ b/metricbeat/module/ceph/fields.go @@ -32,5 +32,5 @@ func init() { // AssetCeph returns asset data. // This is the base64 encoded gzipped contents of ../metricbeat/module/ceph. func AssetCeph() string { - return "eJzsXFFv274Rf8+nOORpA1Jve83DgP7TYg22tkHTYg/DoNLiWeJCkQRJJfO3H0jZsi2RkizTLrC//dRY9u9+vDsej3fnvoMXXN9Djqq8AbDMcryH2wdU5e0NAEWTa6Ysk+Ie/noDAOAeQSVpzfEGwJRS2yyXYsWKe1gRbty7GjkSg/dQEPcZtJaJwtzDv26N4bd3cFtaq27/fQOwYsipuffI70CQClsu7mXXyqFoWavNOwFG7vXTfekn5FJYwoQBWyJUaDXL3b+JhTfUCCbXRCGFlZYVPHx8+rTYAOzTOKDCa2NRZ5SZl/ZhiNYANfeK4BzqafvqktknRF4J42TJcbFcWzQHn9ny4lIUnQcD1Nzr/RYVPCrIlVfghnXn4yupK2LvoU9gS9JKS3hSgt8dYhpytUGalNsPg3Q+ta6nlUi4LVP4Wg/peG+Tr6gJ55mxxNZhfb3g+k1qepzKvja40OAOqa11KVZhXmL+YhaoZF4mst1nouAVtWFSjInVshZ08Up4jYmEt+DgsacRSG2K5wMTHBLp+mZPdtg3w342wWN7+EO+GTLbfFv4U22ji0GH0GS1YvlCI6HZUVEkHpnGyTX6AScUbKllXZSqtqBQg8Fcxnxnw/VNM4sXJ+ulzmDrNStVppxDYJ7KvvsqZFKZ45R3Nj6NlqYQqphRnOS48KdrYhpbcFAFiLpaRqJwy0Eu/4O5TXWC9lhs4CdR0cQyGSRicsKRZisuiY34t0Kdo+g+PZZun8KWJcVCE4r0LEbbgo8YreVwHqO1LCYYraXy64zW0o0bTRULSiy5eMRUBTi5A7mrKhb+AnBxauTwghAj57384uTs7nIQI+aS/ovzqtuLQYSWT3jQp3SYuTcTbc2nwmfWuP+5YQq5rHsbaq7051K+GSjlG1RErEEVBohGYGJDSq5Gl97jlzjZ2xjIjOd70tDFqubh0L2UkiPpshoR/mgcKPRA9yUKJPosUh3wiOi6yqShqU6JjjM4ZOcJY9e9LZNanZuMc03ZEGrd88fTICkmfgWpxy+DpDRWRCmkmSouzezbx8/vn54+fojyS3lj91jdHKNNCgudpSrZDWC1F9wl2mvh7ndeuNv3k9OLd4Noc/zuWsK7lvA2XioNTRIVAzhzPJN1lznXCI90u3YlZTiziSbXsxT+hVQ4KtJZ2SwuGvZ7XKaE1YZn2trAF388O1rbosA0bV0gxJ+gI8/uZZmS2ssyyKu7axXq1Sm79uvzB3AYbqEi33VG5aopfPfD9pwNrYpMI6HrubeWJ05yrFBY+Jtbmq/VMoHGQCBi7uWX3jpBoV0VTTDLT2M1We91kp3unARmLMvNous7AX106HnrZUysZJfkjqhAY7EbhXpsu2bc0Qp8M8YMhkLwIauAU0d5PX4YlOR1EDLUocSQwYIin7waxrUAI5rY55jLqmI248SiyNdZFcODKQrqMX7w6LBBd5fgAQHtrUEpvj4Xo/cO/EhCHR2Jc+toQEBYR4kZ9XQkwjcAv8+l5D0fj+zvsWAtZZNaJ4/RjuMp2dCTYxYv2Dn4SGI3FfwxXEjIOUNhMyYzTWyY/mi4f/AY8Pinr+BAuq4yFsx3Td9AJxDG/Guz/nv4zecjvhMZ6fntC93r3iaQ6tAsiimChxux0wV/cwuVCn3DRUT7nP01J5D9T99eHRbe3cZWI56abDmMs+xfIWkkOQ6mEGPJjofbS3bctQ5Ibtkr+nX458fmPIHwAiNRoMf1i6QYBmrdU8pux2KeFPeNqBT3sB/QYMwBD6R8XysM5UeDWdh0eL+IAfi8ZJxq7KbeM4REkXYh4w1ZUcbtMkXUtxjIrousbLd6dJyID0GELT7+l5lgnjpdwMcwxP59slcBglneG4FqD2TNKqLXGVmtmGC2ezc7blVPDRhEwVqX07UpsxFfGOj09yQ/fPvx/AkCgDuNahK+eKa4A/q/rwHxGhCvAXGmgGtAvEBAbDNZKZiVk9tRR8/rHuJv8lxKZvRCVZ5q6mRXEt+Mh20Lq5+/hhvnPeXAaU2BT40yRqTulp6skLxb+d9/GxPfNHiTiW66u+Niz1A5HxfKibFZrSjpX4saybRfRxhbL6sQ3koU8EYMhLDP33CKLddYqbEpPy24LJK2Uf4hi10XpUtgWgNlR65iJk/K7jMzeTp6xtik7J6fv6cjd/YBjVMJjm66Gc5HjO3ttf3+Ch3vjR3/kxRUpZ+Y8xWR2pACgYlGF4eTBhdpcfvZPZfdsEuEm1ZatM5L8ZXlmOWcmISDDK1ch3EHjHMsCPd/ARM5rylCSekdGEMBbb4YOOOcu05V/gmzvq2LNEPIr5LXEZW1ve1Lsmocd4DVzPmEFNx2M+UD/FSRibpKtItMdzIZuMxdWGlGGpmfkY0bL5asnusnCx0rbjBIgcEoeGrZuo14vnbtYt1o/LxY6JMalnX+gvbCQbAjNxoOAxWTtJIHAqI09A5KaeydrwANxMVg5SahWRxUrHkXLpScoKSN4C0wcOY0YFAR7Xf0cu171SSsi4H7eMMptJmnnl4efKhYGarXzLWDl9gHHCzdnPA7AmYaicYyzht05wZC2j/85d0azR38+Z2QfwzH8uHSy2zFu1tRo22XwTZTQ5ufUvn/6GEjNxrcI0W62TOhJbaQfgQuIjb1GOhuEteHBGbdeVdzCkuEWjkrUfkWnpE9TzrnFNEgg0duqXH2MiWRWxFb9oaxT+SjiEaxiVf+Qt0c+p1Q1jtg/VzDqbOzIZBfeLBex2avY7P/H2Oz/wsAAP//vuBo9A==" + return "eJzsW19v27ARf8+nOORpA1Jve/XDgDYt0GBrGzQt9jAMKi2eJS4USZBUPH/7gZQs2xL1xzLtApv1lFjO7368Ox6Pd5d38IrbJaSo8jsAyyzHJdw/osrv7wAomlQzZZkUS/jrHQCAewWFpCXHOwCTS22TVIo1y5awJty4TzVyJAaXkBH3HbSWicws4Z/3xvD7B7jPrVX3/7oDWDPk1Cw98jsQpMCGi3vsVjkULUtVfxJg5J5f7o9+QSqFJUwYsDlCgVaz1P1MLGxQI5hUE4UU1loW8Pjp+fOiBjikcUSFl8aiTigzr83LEK0Bau7pwTnW0+5pkzkkRN4I42TFcbHaWjRH39nx4lJkrRcD1NzzfocKHhXk2iuwZt36+lrqgtgldAnsSFppCY9K8IdDjEOuNEijcvtpkM6n1va0HAm3eQxf6yCd7m3yDTXhPDGW2DKsr1fcbqSmp6nsW4ULFe6Q2hqXYgWmOaavZoFKpnkk230hCt5QGybFmFgtS0EXb4SXGEl4Aw4eexqB2KZ4OTLBMZG2b3Zkh30z7GcTPLaDP+SbIbPNt4U/1WpdDDqEJus1SxcaCU1OiiL9kWmcXKUfcELB5lqWWa5KCwo1GExln+/UXDeaWbw6WS91BluvWakS5RwC01j2PVQhk8qcpryL8am0NIVQwYziJMWFP10j09iBg8pAlMWqJwo3HOTq35jaWCdoh0UNP4mKJpbJIBGTEo40WXNJbI9/K9QpivbbU+l2KexYUsw0oUgvYrQd+IjRGg6XMVrDYoLRGiq/z2gN3X6jqWxBiSVXj5gqAyd3IHdV2cJfAK5OjRxfEPrIeS+/Ojm7vxz0EXNJ/9V5lc3FoIeWT3jQp3SYuA8jbc3nzGfWePi9YQqpLDsbaq70l1xuDORyAwURW1CZAaIRmKhJyfXo0jv8Iid7tYHMeL4nDV2sSx4O3SspOZI2qxHhT8aBQgf0UKJAoi8i1QGPiC6LRBoa65RoOYNDdp4wdt3bMSnVpck415QVocY9fz4PkmLid5B6+jpISmNBlEKaqOzazL5/+vL++fnTx15+MW/sHqudYzRJYaaTWCW7AazmgrtCeyvc/Z8X7g795Pzi3SDaHL+7lfBuJbzaS6WhUaJiAGeOZ7L2Muca4Ynu1q6kDGc2vcn1LIV/JQWOinRWNourhv0OlylhteIZtzbw1R/PjtauKDBNW1cI8WfoyLN7XcWk9roK8mrvWoV6fc6u/fbyERyGW6hI951Rua4K392wfcUN7cg9hZPHSu+pLApmE04sinSbFPOc49GjQI3iLiHFkLGJUnx7rsz3DmSyyNY6Rax1iunrnCmzs04RzpO8L0tZZSPdRpJAY5Ge4NJSVglIbE/2HM85M54ds/6yhoPv2S1TwXt2TMoZCpswmWhiw/TbcaOD/+gx4OlP38CBtF0ipLhDCvvWWKBfAkM+5Z56/Uv44KO279f0dEYOhR70uCJIdWgWxRTBw+2q6YK/u4VKhb4sLXq7Qd01R5D9D9+EGhbe3sZWI557JDmMi+xfIWlPCtEJMSM03fPLwx0MF7nkF0hq2Rv6dfj3ixO3SSC8wEgU6HD9KimGgRr3lLJd150nxf1FrxT3shvQYMwBj6T82CpsRzUYyi5Og/eLGIBPc8apxnZZdYaQXqR9yNggy/J+u0wR9b0PZN9rU7Z9xz5NxMcgwg4f/8NM58JwmoBPYYjDBKVzT4ZZ3tsD1RzImhVEbxOyXjPB7PasVT1XYNAL1ricLk2ejPjCQD+0I/nx+8+XzxAA3GtUkzafiUlBICx6tIOw6H+/BcRbQLwFxJkCbgHxCgGxyWSlYFZOLtqfPNV4jF/nuZTM6BipNFZvfl84rIdoduWnL9/C7cWOcuC80unnShkjUvdLj1Zu26/8bx/GxFdtsGiiqx7YuNgL1BfHhXJibFIqSrrXokoy7dYRxtbLCoRNjgI2xEAI+/Jl+b7lGis1VuWnBZdZ1GLz32W2rzW3CUwrM+/JFcykUdl9YSaNR88YG5Xdy8uPeOQu3sY+l+DoppvhfMTYzl47GMJI6HgH4fTBfVS5nyvyFZHSkAyBiUoXx/3YqzQC/YSTy27YNcJNI623zkvxjaWYpJyYiO3eRq7DeADGOWaE+9+AiZSXFCGn9AGMoYA2XQyccc5dpyr/jInIxkWqUc03ycselTUdwGuyqhx3gNXMLm4MbvvJ2wF+KktEWUTaRaY9vwlcpi6sVINfzE8S9huvL1m91GB3y4o1BskwGAXPLVs3Ec/Xrl2sG42fVwt9UsOqTF/RXjkItuT2hsNAxSSu5IGAKA19gFwa++ArQANxMVi5iWgWB9XXvAsXSs5QUi14BwycOQ0YVET7Hb3aQiqLgoR1MXAfrziFNvPU08uDDxUrQ/WauXbwEruAg6WbM6atmakkGss4r9CdGwhp//CXd1s0D/Dnd0L+MRzLh0svsxXvbkWVtl0Gq5FQJup/OPH/Dl/L7Q3uPUW62ZNzOTaQflCoR2zsYbn9vKIPCcy6867kFFYIpXJWonITniS8TDrnFFEhg0duqHH2OiWRWxObd0ZWz+SjiEZRxyt/oa4O/VYo6xywfq7h3AnDEMhvPFhvw4W34cL/jeHC/wYAAP//JGzILA==" } diff --git a/metricbeat/module/ceph/mgr_cluster_health/_meta/data.json b/metricbeat/module/ceph/mgr_cluster_health/_meta/data.json index 14cee62d3281..65e4061ded4d 100644 --- a/metricbeat/module/ceph/mgr_cluster_health/_meta/data.json +++ b/metricbeat/module/ceph/mgr_cluster_health/_meta/data.json @@ -5,7 +5,7 @@ "name": "host.example.com" }, "ceph": { - "cluster_health": { + "mgr_cluster_health": { "overall_status": "HEALTH_OK", "timechecks": { "epoch": 3, diff --git a/metricbeat/module/ceph/mgr_osd_disk/_meta/data.json b/metricbeat/module/ceph/mgr_osd_disk/_meta/data.json index ce3506e1cf17..f47e93b22d5a 100644 --- a/metricbeat/module/ceph/mgr_osd_disk/_meta/data.json +++ b/metricbeat/module/ceph/mgr_osd_disk/_meta/data.json @@ -1,7 +1,7 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", "ceph": { - "pool_disk": { + "mgr_osd_disk": { "id": 0, "name": "rbd", "stats": { diff --git a/metricbeat/module/ceph/mgr_osd_perf/_meta/data.json b/metricbeat/module/ceph/mgr_osd_perf/_meta/data.json index 5f4b958dcd06..6d3c5f009981 100644 --- a/metricbeat/module/ceph/mgr_osd_perf/_meta/data.json +++ b/metricbeat/module/ceph/mgr_osd_perf/_meta/data.json @@ -30,19 +30,12 @@ }, "ceph": { "mgr_osd_perf": { - "pg_ready": true, - "osdstats": { - "osd_perf_infos": [ - { - "id": 0, - "perf_stats": { - "commit_latency_ms": 23, - "apply_latency_ms": 23, - "commit_latency_ns": 23000000, - "apply_latency_ns": 23000000 - } - } - ] + "id": 0, + "stats": { + "commit_latency_ms": 23, + "apply_latency_ms": 23, + "commit_latency_ns": 23000000, + "apply_latency_ns": 23000000 } } } diff --git a/metricbeat/module/ceph/mgr_osd_perf/_meta/fields.yml b/metricbeat/module/ceph/mgr_osd_perf/_meta/fields.yml index 8efa1bbae281..5adb0f64f1d1 100644 --- a/metricbeat/module/ceph/mgr_osd_perf/_meta/fields.yml +++ b/metricbeat/module/ceph/mgr_osd_perf/_meta/fields.yml @@ -4,34 +4,18 @@ OSD performance metrics of Ceph cluster release: beta fields: - - name: pg_ready - type: boolean - description: Placement Group readiness status - - name: osdstats - type: group - description: > - `stray` contains OSD statistics. - fields: - - name: osd_perf_infos - type: nested - description: OSD performance statistics - fields: - - name: id - type: long - description: OSD ID - - name: perf_stats - type: group - description: Perforance statistics - fields: - - name: commit_latency_ms - type: long - description: Commit latency in ms - - name: apply_latency_ms - type: long - description: Apply latency in ms - - name: commit_latency_ns - type: long - description: Commit latency in ns - - name: apply_latency_ns - type: long - description: Apply latency in ns + - name: id + type: long + description: OSD ID + - name: stats.commit_latency_ms + type: long + description: Commit latency in ms + - name: stats.apply_latency_ms + type: long + description: Apply latency in ms + - name: stats.commit_latency_ns + type: long + description: Commit latency in ns + - name: stats.apply_latency_ns + type: long + description: Apply latency in ns diff --git a/metricbeat/module/ceph/mgr_osd_perf/data.go b/metricbeat/module/ceph/mgr_osd_perf/data.go new file mode 100644 index 000000000000..18b78ac6114b --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_perf/data.go @@ -0,0 +1,62 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mgr_osd_perf + +import ( + "github.com/pkg/errors" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/metricbeat/module/ceph/mgr" +) + +type OsdPerfResponse struct { + OsdStats struct { + OsdPerfInfos []struct { + ID int64 `json:"id"` + PerfStats struct { + CommitLatencyMs uint64 `json:"commit_latency_ms"` + ApplyLatencyMs uint64 `json:"apply_latency_ms"` + CommitLatencyNs uint64 `json:"commit_latency_ns"` + ApplyLatencyNs uint64 `json:"apply_latency_ns"` + } `json:"perf_stats"` + } `json:"osd_perf_infos"` + } `json:"osdstats"` +} + +func eventsMapping(content []byte) ([]common.MapStr, error) { + var response OsdPerfResponse + err := mgr.UnmarshalResponse(content, &response) + if err != nil { + return nil, errors.Wrap(err, "could not get response data") + } + + var events []common.MapStr + for _, OsdPerfInfo := range response.OsdStats.OsdPerfInfos { + event := common.MapStr{ + "id": OsdPerfInfo.ID, + "stats": common.MapStr{ + "commit_latency_ms": OsdPerfInfo.PerfStats.CommitLatencyMs, + "apply_latency_ms": OsdPerfInfo.PerfStats.ApplyLatencyMs, + "commit_latency_ns": OsdPerfInfo.PerfStats.CommitLatencyNs, + "apply_latency_ns": OsdPerfInfo.PerfStats.ApplyLatencyNs, + }, + } + events = append(events, event) + } + return events, nil +} diff --git a/metricbeat/module/ceph/mgr_osd_perf/manifest.yml b/metricbeat/module/ceph/mgr_osd_perf/manifest.yml deleted file mode 100644 index 4886520d7531..000000000000 --- a/metricbeat/module/ceph/mgr_osd_perf/manifest.yml +++ /dev/null @@ -1,26 +0,0 @@ -default: true -input: - module: http - metricset: json - defaults: - method: POST - namespace: "json_namespace" - basepath: "/request" - headers: - Content-Type: application/json - query: - wait: "1" - body: '{"prefix": "osd perf", "format": "json"}' - -processors: - - extract_array: - field: http.json_namespace.finished - mappings: - ceph.request: 0 - - decode_json_fields: - fields: ["ceph.request.outb"] - process_array: true - target: "ceph.mgr_osd_perf" - - drop_fields: - ignore_missing: true - fields: ["http", "ceph.request"] diff --git a/metricbeat/module/ceph/mgr_osd_perf/mgr_osd_perf.go b/metricbeat/module/ceph/mgr_osd_perf/mgr_osd_perf.go new file mode 100644 index 000000000000..d56703395825 --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_perf/mgr_osd_perf.go @@ -0,0 +1,93 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mgr_osd_perf + +import ( + "fmt" + + "github.com/elastic/beats/metricbeat/helper" + "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/metricbeat/mb/parse" +) + +const ( + defaultScheme = "https" + defaultPath = "/request" + defaultQueryParams = "wait=1" + + cephPrefix = "osd perf" +) + +var ( + hostParser = parse.URLHostParserBuilder{ + DefaultScheme: defaultScheme, + DefaultPath: defaultPath, + QueryParams: defaultQueryParams, + }.Build() +) + +func init() { + mb.Registry.MustAddMetricSet("ceph", "mgr_osd_perf", New, + mb.WithHostParser(hostParser), + ) +} + +type MetricSet struct { + mb.BaseMetricSet + *helper.HTTP +} + +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + http, err := helper.NewHTTP(base) + if err != nil { + return nil, err + } + http.SetMethod("POST") + http.SetHeader("Content-Type", "application/json") + http.SetHeader("Accept", "application/json") + http.SetBody([]byte(fmt.Sprintf(`{"prefix": "%s", "format": "json"}`, cephPrefix))) + + return &MetricSet{ + base, + http, + }, nil +} + +// Fetch methods implements the data gathering and data conversion to the right +// format. It publishes the event which is then forwarded to the output. In case +// of an error set the Error field of mb.Event or simply call report.Error(). +func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { + content, err := m.HTTP.FetchContent() + if err != nil { + return err + } + + events, err := eventsMapping(content) + if err != nil { + return err + } + + for _, event := range events { + reported := reporter.Event(mb.Event{MetricSetFields: event}) + if !reported { + m.Logger().Debug("error reporting event") + return nil + } + } + return nil +} diff --git a/metricbeat/module/ceph/mgr_osd_perf/mgr_osd_perf_integration_test.go b/metricbeat/module/ceph/mgr_osd_perf/mgr_osd_perf_integration_test.go new file mode 100644 index 000000000000..4132e99ad7ac --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_perf/mgr_osd_perf_integration_test.go @@ -0,0 +1,52 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build integration,linux + +package mgr_osd_perf + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/elastic/beats/libbeat/tests/compose" + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + "github.com/elastic/beats/metricbeat/module/ceph/mgrtest" +) + +const user = "demo" + +func TestData(t *testing.T) { + service := compose.EnsureUpWithTimeout(t, 120, "ceph-mgr") + + f := mbtest.NewReportingMetricSetV2Error(t, + getConfig(service.HostForPort(8003), mgrtest.GetPassword(t, service.HostForPort(5000), user))) + err := mbtest.WriteEventsReporterV2Error(f, t, "") + require.NoError(t, err) +} + +func getConfig(host, password string) map[string]interface{} { + return map[string]interface{}{ + "module": "ceph", + "metricsets": []string{"mgr_osd_perf"}, + "hosts": []string{host}, + "username": user, + "password": password, + "ssl.verification_mode": "none", + } +} diff --git a/metricbeat/module/ceph/mgr_osd_perf/mgr_osd_perf_test.go b/metricbeat/module/ceph/mgr_osd_perf/mgr_osd_perf_test.go new file mode 100644 index 000000000000..eb03bf9b474d --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_perf/mgr_osd_perf_test.go @@ -0,0 +1,97 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mgr_osd_perf + +import ( + "io/ioutil" + "net/http" + "net/http/httptest" + "path/filepath" + "testing" + + "github.com/elastic/beats/libbeat/common" + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + + "github.com/stretchr/testify/assert" +) + +func TestFetchEventContents(t *testing.T) { + absPath, err := filepath.Abs("../_meta/testdata/") + assert.NoError(t, err) + + response, err := ioutil.ReadFile(absPath + "/mgr_osd_perf_sample_response.json") + assert.NoError(t, err) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + w.Header().Set("Content-Type", "application/json;") + w.Write(response) + })) + defer server.Close() + + config := map[string]interface{}{ + "module": "ceph", + "metricsets": []string{"mgr_osd_perf"}, + "hosts": []string{server.URL}, + } + + f := mbtest.NewReportingMetricSetV2Error(t, config) + events, errs := mbtest.ReportingFetchV2Error(f) + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) + } + assert.NotEmpty(t, events) + event := events[0].MetricSetFields + + t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event.StringToPrint()) + + assert.EqualValues(t, 1, event["id"]) + + stats := event["stats"].(common.MapStr) + + assert.EqualValues(t, 2, stats["apply_latency_ms"]) + assert.EqualValues(t, 4, stats["apply_latency_ns"]) + assert.EqualValues(t, 1, stats["commit_latency_ms"]) + assert.EqualValues(t, 3, stats["commit_latency_ns"]) +} + +func TestFetchEventContents_Failed(t *testing.T) { + absPath, err := filepath.Abs("../_meta/testdata/") + assert.NoError(t, err) + + response, err := ioutil.ReadFile(absPath + "/mgr_failed_response.json") + assert.NoError(t, err) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + w.Header().Set("Content-Type", "application/json;") + w.Write([]byte(response)) + })) + defer server.Close() + + config := map[string]interface{}{ + "module": "ceph", + "metricsets": []string{"mgr_osd_perf"}, + "hosts": []string{server.URL}, + } + + f := mbtest.NewReportingMetricSetV2Error(t, config) + events, errs := mbtest.ReportingFetchV2Error(f) + assert.Empty(t, events) + assert.NotEmpty(t, errs) +} diff --git a/metricbeat/module/ceph/test_ceph_mgr.py b/metricbeat/module/ceph/test_ceph_mgr.py index 97640508c303..bc4a552a9569 100644 --- a/metricbeat/module/ceph/test_ceph_mgr.py +++ b/metricbeat/module/ceph/test_ceph_mgr.py @@ -37,7 +37,8 @@ def get_password(self): @parameterized.expand([ "mgr_cluster_disk", "mgr_cluster_health", - "mgr_osd_disk" + "mgr_osd_disk", + "mgr_osd_perf" ]) @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") def test_ceph_mgr(self, metricset): From e56f1765bf75759a045f65416b04abd5721807a9 Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Thu, 13 Feb 2020 22:59:07 +0100 Subject: [PATCH 33/41] mgr_osd_... as standard metricset --- metricbeat/docs/fields.asciidoc | 116 ++++++++++----- metricbeat/include/list_common.go | 2 + .../mgr_osd_pool_stats_sample_response.json | 17 +++ .../mgr_osd_tree_sample_response.json | 17 +++ metricbeat/module/ceph/fields.go | 2 +- .../module/ceph/mgr_osd_perf/_meta/data.json | 42 ++---- .../ceph/mgr_osd_pool_stats/_meta/data.json | 92 +++--------- .../ceph/mgr_osd_pool_stats/_meta/fields.yml | 6 +- .../module/ceph/mgr_osd_pool_stats/data.go | 60 ++++++++ .../ceph/mgr_osd_pool_stats/manifest.yml | 44 ------ .../mgr_osd_pool_stats/mgr_osd_pool_stats.go | 93 ++++++++++++ .../mgr_osd_pool_stats_integration_test.go | 52 +++++++ .../mgr_osd_pool_stats_test.go | 98 +++++++++++++ .../module/ceph/mgr_osd_tree/_meta/data.json | 86 +++-------- .../module/ceph/mgr_osd_tree/_meta/fields.yml | 126 +++++++--------- metricbeat/module/ceph/mgr_osd_tree/data.go | 106 ++++++++++++++ .../module/ceph/mgr_osd_tree/manifest.yml | 54 ------- .../module/ceph/mgr_osd_tree/mgr_osd_tree.go | 93 ++++++++++++ .../mgr_osd_tree_integration_test.go | 52 +++++++ .../ceph/mgr_osd_tree/mgr_osd_tree_test.go | 134 ++++++++++++++++++ metricbeat/module/ceph/test_ceph_mgr.py | 4 +- 21 files changed, 905 insertions(+), 391 deletions(-) create mode 100644 metricbeat/module/ceph/_meta/testdata/mgr_osd_pool_stats_sample_response.json create mode 100644 metricbeat/module/ceph/_meta/testdata/mgr_osd_tree_sample_response.json create mode 100644 metricbeat/module/ceph/mgr_osd_pool_stats/data.go delete mode 100644 metricbeat/module/ceph/mgr_osd_pool_stats/manifest.yml create mode 100644 metricbeat/module/ceph/mgr_osd_pool_stats/mgr_osd_pool_stats.go create mode 100644 metricbeat/module/ceph/mgr_osd_pool_stats/mgr_osd_pool_stats_integration_test.go create mode 100644 metricbeat/module/ceph/mgr_osd_pool_stats/mgr_osd_pool_stats_test.go create mode 100644 metricbeat/module/ceph/mgr_osd_tree/data.go delete mode 100644 metricbeat/module/ceph/mgr_osd_tree/manifest.yml create mode 100644 metricbeat/module/ceph/mgr_osd_tree/mgr_osd_tree.go create mode 100644 metricbeat/module/ceph/mgr_osd_tree/mgr_osd_tree_integration_test.go create mode 100644 metricbeat/module/ceph/mgr_osd_tree/mgr_osd_tree_test.go diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 4943e6f8606a..ef21b02219d7 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -4145,136 +4145,174 @@ type: long -- -*`ceph.mgr_osd_pool_stats`*:: +[float] +=== mgr_osd_pool_stats + +OSD pool stats of Ceph cluster + + + +*`ceph.mgr_osd_pool_stats.pool_name`*:: + -- -OSD pool stats of Ceph cluster +Pool name +type: keyword + +-- -type: nested +*`ceph.mgr_osd_pool_stats.pool_id`*:: ++ +-- +Pool ID + +type: long + +-- + +*`ceph.mgr_osd_pool_stats.client_io_rate`*:: ++ +-- +Client I/O rates + +type: object -- [float] === mgr_osd_tree -OSD tree stats of Ceph cluster +OSD tree information -*`ceph.mgr_osd_tree.nodes`*:: +*`ceph.mgr_osd_tree.id`*:: + -- -`nodes` contains disk active OSD nodes. +OSD or bucket node ID -type: nested +type: long -- -[float] -=== stray +*`ceph.mgr_osd_tree.name`*:: ++ +-- +OSD or bucket node name -`stray` contains stray OSD nodes. +type: keyword +-- -*`ceph.mgr_osd_tree.stray.name`*:: +*`ceph.mgr_osd_tree.type`*:: + -- -Node name +OSD or bucket node type, illegal type include OSD, host, root etc. + type: keyword -- -*`ceph.mgr_osd_tree.stray.root`*:: +*`ceph.mgr_osd_tree.type_id`*:: + -- -Node type +OSD or bucket node type ID -type: keyword + +type: long -- -*`ceph.mgr_osd_tree.stray.type_id`*:: +*`ceph.mgr_osd_tree.children`*:: + -- -Type ID +Bucket children list, separated by comma. -type: long + +type: keyword -- -*`ceph.mgr_osd_tree.stray.id`*:: +*`ceph.mgr_osd_tree.crush_weight`*:: + -- -Node ID +OSD node CRUSH weight -type: long + +type: float -- -*`ceph.mgr_osd_tree.stray.children`*:: +*`ceph.mgr_osd_tree.depth`*:: + -- -Node children +Node depth + type: long -- -*`ceph.mgr_osd_tree.stray.reweight`*:: +*`ceph.mgr_osd_tree.exists`*:: + -- -Reweight +Node exists (1-yes, 0-no) -type: long + +type: boolean -- -*`ceph.mgr_osd_tree.stray.depth`*:: +*`ceph.mgr_osd_tree.primary_affinity`*:: + -- -Depth +Weight of reading data from primary OSD -type: long + +type: float -- -*`ceph.mgr_osd_tree.stray.exists`*:: +*`ceph.mgr_osd_tree.reweight`*:: + -- -Exists +Reweight of osd + type: long -- -*`ceph.mgr_osd_tree.stray.status`*:: +*`ceph.mgr_osd_tree.status`*:: + -- -Node status +Status of OSD, it should be up or down + type: keyword -- -*`ceph.mgr_osd_tree.stray.primary_affinity`*:: +*`ceph.mgr_osd_tree.device_class`*:: + -- -Primary affinity +Device class of OSD, like HDD, SSD etc. -type: long + +type: keyword -- -*`ceph.mgr_osd_tree.stray.crush_weight`*:: +*`ceph.mgr_osd_tree.father`*:: + -- -CRUSH weight +Parent node of this OSD or bucket node -type: scaled_float + +type: keyword -- diff --git a/metricbeat/include/list_common.go b/metricbeat/include/list_common.go index f8cabf377020..5583bdc4c330 100644 --- a/metricbeat/include/list_common.go +++ b/metricbeat/include/list_common.go @@ -36,6 +36,8 @@ import ( _ "github.com/elastic/beats/metricbeat/module/ceph/mgr_cluster_health" _ "github.com/elastic/beats/metricbeat/module/ceph/mgr_osd_disk" _ "github.com/elastic/beats/metricbeat/module/ceph/mgr_osd_perf" + _ "github.com/elastic/beats/metricbeat/module/ceph/mgr_osd_pool_stats" + _ "github.com/elastic/beats/metricbeat/module/ceph/mgr_osd_tree" _ "github.com/elastic/beats/metricbeat/module/ceph/monitor_health" _ "github.com/elastic/beats/metricbeat/module/ceph/osd_df" _ "github.com/elastic/beats/metricbeat/module/ceph/osd_tree" diff --git a/metricbeat/module/ceph/_meta/testdata/mgr_osd_pool_stats_sample_response.json b/metricbeat/module/ceph/_meta/testdata/mgr_osd_pool_stats_sample_response.json new file mode 100644 index 000000000000..af0a1ef6a303 --- /dev/null +++ b/metricbeat/module/ceph/_meta/testdata/mgr_osd_pool_stats_sample_response.json @@ -0,0 +1,17 @@ +{ + "failed": [], + "finished": [ + { + "command": "osd pool stats format=json-pretty", + "outb": "[\n {\n \"pool_name\": \"rbd\",\n \"pool_id\": 1,\n \"recovery\": {},\n \"recovery_rate\": {},\n \"client_io_rate\": { \"read_bytes_sec\": 1, \"read_op_per_sec\": 2, \"write_bytes_sec\": 3, \"write_op_per_sec\": 4 }\n },\n {\n \"pool_name\": \"cephfs_data\",\n \"pool_id\": 2,\n \"recovery\": {},\n \"recovery_rate\": {},\n \"client_io_rate\": {}\n },\n {\n \"pool_name\": \"cephfs_metadata\",\n \"pool_id\": 3,\n \"recovery\": {},\n \"recovery_rate\": {},\n \"client_io_rate\": {}\n },\n {\n \"pool_name\": \".rgw.root\",\n \"pool_id\": 4,\n \"recovery\": {},\n \"recovery_rate\": {},\n \"client_io_rate\": {}\n },\n {\n \"pool_name\": \"default.rgw.control\",\n \"pool_id\": 5,\n \"recovery\": {},\n \"recovery_rate\": {},\n \"client_io_rate\": {}\n },\n {\n \"pool_name\": \"default.rgw.meta\",\n \"pool_id\": 6,\n \"recovery\": {},\n \"recovery_rate\": {},\n \"client_io_rate\": {}\n },\n {\n \"pool_name\": \"default.rgw.log\",\n \"pool_id\": 7,\n \"recovery\": {},\n \"recovery_rate\": {},\n \"client_io_rate\": {}\n },\n {\n \"pool_name\": \"default.rgw.buckets.index\",\n \"pool_id\": 8,\n \"recovery\": {},\n \"recovery_rate\": {},\n \"client_io_rate\": {}\n }\n]\n", + "outs": "" + } + ], + "has_failed": false, + "id": "140301322349968", + "is_finished": true, + "is_waiting": false, + "running": [], + "state": "success", + "waiting": [] +} diff --git a/metricbeat/module/ceph/_meta/testdata/mgr_osd_tree_sample_response.json b/metricbeat/module/ceph/_meta/testdata/mgr_osd_tree_sample_response.json new file mode 100644 index 000000000000..28f7edd180d9 --- /dev/null +++ b/metricbeat/module/ceph/_meta/testdata/mgr_osd_tree_sample_response.json @@ -0,0 +1,17 @@ +{ + "failed": [], + "finished": [ + { + "command": "osd tree format=json-pretty", + "outb": "{\n \"nodes\": [\n {\n \"children\": [\n -3\n ],\n \"id\": -1,\n \"name\": \"default\",\n \"type\": \"root\",\n \"type_id\": 10\n },\n {\n \"children\": [\n 1,\n 0\n ],\n \"id\": -3,\n \"name\": \"ceph-mon1\",\n \"pool_weights\": {},\n \"type\": \"host\",\n \"type_id\": 1\n },\n {\n \"crush_weight\": 0.048691,\n \"depth\": 2,\n \"device_class\": \"hdd\",\n \"exists\": 1,\n \"id\": 0,\n \"name\": \"osd.0\",\n \"pool_weights\": {},\n \"primary_affinity\": 1.0,\n \"reweight\": 1.0,\n \"status\": \"up\",\n \"type\": \"osd\",\n \"type_id\": 0\n },\n {\n \"crush_weight\": 0.048691,\n \"depth\": 2,\n \"device_class\": \"hdd\",\n \"exists\": 1,\n \"id\": 1,\n \"name\": \"osd.1\",\n \"pool_weights\": {},\n \"primary_affinity\": 1.0,\n \"reweight\": 1.0,\n \"status\": \"up\",\n \"type\": \"osd\",\n \"type_id\": 0\n }\n ],\n \"stray\": []\n }", + "outs": "" + } + ], + "has_failed": false, + "id": "140149339315152", + "is_finished": true, + "is_waiting": false, + "running": [], + "state": "success", + "waiting": [] +} diff --git a/metricbeat/module/ceph/fields.go b/metricbeat/module/ceph/fields.go index 85000c7b556e..194292c33623 100644 --- a/metricbeat/module/ceph/fields.go +++ b/metricbeat/module/ceph/fields.go @@ -32,5 +32,5 @@ func init() { // AssetCeph returns asset data. // This is the base64 encoded gzipped contents of ../metricbeat/module/ceph. func AssetCeph() string { - return "eJzsW19v27ARf8+nOORpA1Jve/XDgDYt0GBrGzQt9jAMKi2eJS4USZBUPH/7gZQs2xL1xzLtApv1lFjO7368Ox6Pd5d38IrbJaSo8jsAyyzHJdw/osrv7wAomlQzZZkUS/jrHQCAewWFpCXHOwCTS22TVIo1y5awJty4TzVyJAaXkBH3HbSWicws4Z/3xvD7B7jPrVX3/7oDWDPk1Cw98jsQpMCGi3vsVjkULUtVfxJg5J5f7o9+QSqFJUwYsDlCgVaz1P1MLGxQI5hUE4UU1loW8Pjp+fOiBjikcUSFl8aiTigzr83LEK0Bau7pwTnW0+5pkzkkRN4I42TFcbHaWjRH39nx4lJkrRcD1NzzfocKHhXk2iuwZt36+lrqgtgldAnsSFppCY9K8IdDjEOuNEijcvtpkM6n1va0HAm3eQxf6yCd7m3yDTXhPDGW2DKsr1fcbqSmp6nsW4ULFe6Q2hqXYgWmOaavZoFKpnkk230hCt5QGybFmFgtS0EXb4SXGEl4Aw4eexqB2KZ4OTLBMZG2b3Zkh30z7GcTPLaDP+SbIbPNt4U/1WpdDDqEJus1SxcaCU1OiiL9kWmcXKUfcELB5lqWWa5KCwo1GExln+/UXDeaWbw6WS91BluvWakS5RwC01j2PVQhk8qcpryL8am0NIVQwYziJMWFP10j09iBg8pAlMWqJwo3HOTq35jaWCdoh0UNP4mKJpbJIBGTEo40WXNJbI9/K9QpivbbU+l2KexYUsw0oUgvYrQd+IjRGg6XMVrDYoLRGiq/z2gN3X6jqWxBiSVXj5gqAyd3IHdV2cJfAK5OjRxfEPrIeS+/Ojm7vxz0EXNJ/9V5lc3FoIeWT3jQp3SYuA8jbc3nzGfWePi9YQqpLDsbaq70l1xuDORyAwURW1CZAaIRmKhJyfXo0jv8Iid7tYHMeL4nDV2sSx4O3SspOZI2qxHhT8aBQgf0UKJAoi8i1QGPiC6LRBoa65RoOYNDdp4wdt3bMSnVpck415QVocY9fz4PkmLid5B6+jpISmNBlEKaqOzazL5/+vL++fnTx15+MW/sHqudYzRJYaaTWCW7AazmgrtCeyvc/Z8X7g795Pzi3SDaHL+7lfBuJbzaS6WhUaJiAGeOZ7L2Muca4Ynu1q6kDGc2vcn1LIV/JQWOinRWNourhv0OlylhteIZtzbw1R/PjtauKDBNW1cI8WfoyLN7XcWk9roK8mrvWoV6fc6u/fbyERyGW6hI951Rua4K392wfcUN7cg9hZPHSu+pLApmE04sinSbFPOc49GjQI3iLiHFkLGJUnx7rsz3DmSyyNY6Rax1iunrnCmzs04RzpO8L0tZZSPdRpJAY5Ge4NJSVglIbE/2HM85M54ds/6yhoPv2S1TwXt2TMoZCpswmWhiw/TbcaOD/+gx4OlP38CBtF0ipLhDCvvWWKBfAkM+5Z56/Uv44KO279f0dEYOhR70uCJIdWgWxRTBw+2q6YK/u4VKhb4sLXq7Qd01R5D9D9+EGhbe3sZWI557JDmMi+xfIWlPCtEJMSM03fPLwx0MF7nkF0hq2Rv6dfj3ixO3SSC8wEgU6HD9KimGgRr3lLJd150nxf1FrxT3shvQYMwBj6T82CpsRzUYyi5Og/eLGIBPc8apxnZZdYaQXqR9yNggy/J+u0wR9b0PZN9rU7Z9xz5NxMcgwg4f/8NM58JwmoBPYYjDBKVzT4ZZ3tsD1RzImhVEbxOyXjPB7PasVT1XYNAL1ricLk2ejPjCQD+0I/nx+8+XzxAA3GtUkzafiUlBICx6tIOw6H+/BcRbQLwFxJkCbgHxCgGxyWSlYFZOLtqfPNV4jF/nuZTM6BipNFZvfl84rIdoduWnL9/C7cWOcuC80unnShkjUvdLj1Zu26/8bx/GxFdtsGiiqx7YuNgL1BfHhXJibFIqSrrXokoy7dYRxtbLCoRNjgI2xEAI+/Jl+b7lGis1VuWnBZdZ1GLz32W2rzW3CUwrM+/JFcykUdl9YSaNR88YG5Xdy8uPeOQu3sY+l+DoppvhfMTYzl47GMJI6HgH4fTBfVS5nyvyFZHSkAyBiUoXx/3YqzQC/YSTy27YNcJNI623zkvxjaWYpJyYiO3eRq7DeADGOWaE+9+AiZSXFCGn9AGMoYA2XQyccc5dpyr/jInIxkWqUc03ycselTUdwGuyqhx3gNXMLm4MbvvJ2wF+KktEWUTaRaY9vwlcpi6sVINfzE8S9huvL1m91GB3y4o1BskwGAXPLVs3Ec/Xrl2sG42fVwt9UsOqTF/RXjkItuT2hsNAxSSu5IGAKA19gFwa++ArQANxMVi5iWgWB9XXvAsXSs5QUi14BwycOQ0YVET7Hb3aQiqLgoR1MXAfrziFNvPU08uDDxUrQ/WauXbwEruAg6WbM6atmakkGss4r9CdGwhp//CXd1s0D/Dnd0L+MRzLh0svsxXvbkWVtl0Gq5FQJup/OPH/Dl/L7Q3uPUW62ZNzOTaQflCoR2zsYbn9vKIPCcy6867kFFYIpXJWonITniS8TDrnFFEhg0duqHH2OiWRWxObd0ZWz+SjiEZRxyt/oa4O/VYo6xywfq7h3AnDEMhvPFhvw4W34cL/jeHC/wYAAP//JGzILA==" + return "eJzsWs9v47oRvuevGOTUAlm3vfpQYHezwAbt7gbJW7xDUWhpcSyxpkiCpOL6vy9IybIjUT9i03LRxqf3Yu83H2eGw5mP/AAb3C0hRZXfAFhmOS7h9jOq/PYGgKJJNVOWSbGEv94AALivoJC05HgDYHKpbZJKsWbZEtaEG/dXjRyJwSVkxP0GrWUiM0v4x60x/PYObnNr1e0/bwDWDDk1S4/8AQQpsOHiPnanHIqWpar/EmDkPr/cP/oFqRSWMGHA5ggFWs1S99/EwhY1gkk1UUhhrWUBn788fl3UAMc0XlHhpbGoE8rMpvkyRGuAmvv04Lz20/7TJnNMiLwQxsmK42K1s2he/WbPi0uRtb4YoOY+H/eo4FFBrr0Da9atn6+lLohdQpfAnqSVlvCoBH9ziHHIlQZpVG4/DdLTqbUzLUfCbR4j1zpIb882+YKacJ4YS2wZ9tcGd1up6dtc9qPChQp3yG1NSrEC0xzTjVmgkmkeKXbfiIIX1IZJMWZWy1LQxQvhJUYy3oCDx55GIHYonl+F4DWRdm52bIdzM5xnEzK2gz+Um6GwnR4Lf6rVvhhMCE3Wa5YuNBKavKmK9FemcXKVf8AZBZtrWWa5Ki0o1GAwlX25U3PdamZxdrLe6glsvWelSpRLCExjxffYhUwq8zbnXYxP5aUphApmFCcpLvzpGpnGHhxUBqIsVj1VuOEgV//C1MY6QTssavhJVDSxTAaJmJRwpMmaS2J78luhTlG0v30r3S6FPUuKmSYU6UWCtgcfCVrD4TJBa1hMCFpD5XpBa+j2B01lC0osmb1iqgyc3YHeVWULPwDMTo28HhD6yPksn52cPQwHfcRc0z87r7IZDHpo+YYHfUuHiftjpK35mPnOGo9/N0whlWVnQ51q/TmXWwO53EJBxA5UZoBoBCZqUnI9uvQOv8jNXh0gM97vSUMX65KHS/dKSo6kzWrE+INxoNABPbYokOiLWHXAI6bLIpGGxjolWsngkF0mjI17eyalujQZl5qyItSk58/HQVJMXIPUw/dBUhoLohTSRGVzM3v68u3j4+OX+15+MSd2j9XuMZqmMNNJLMluAKsZcFdo34W7/3Ph7jhPzhfvBtFOybt3Ce9dwquzVBoapSoGcE7JTNZe5qlBeKD7tSspw51Nb3N9ksO/kwJHTboom8WsZb/DZUpZrXjG1Qa+++PZ0dqLAtO8NUOJP8NHnt1mFZPaZhXk1d61CvX6nF374/keHIZbqEgPN6NyXQnf3bI944Z25B7CzWPl91QWBbMJJxZFukuK05Ljs0eBGsUNIcVQsIlSfHeuzY8OZLLJ1jpFrHWK6es80WZnnSLcJ/lclrLqRsYvksYyWsqq/4idyJ7iOUfGo2PWr2o4+BM3i0fu2S0pZyhswmSiiQ1zr8rxYPp4EHj40w9wKO18CLntmMPhXixwWTK4TPepXbuET75k+8uanmuRY6NHF1wRrDo0i2KK4eG7qumGn9xCpUKvSYveq6DumiPY/t3fQA0bb+9hqxHP3b0OA5ioDt/XzftVu0nHTWpYlekGLQhJsW+/xW0qA3Z7K4gzc1HLDu0OGOeYEe7/D5hIeUnR/foOcmnsHWgpLaBNF70kTy1zk0n21sKccaoxrBif5KVPleU9MHDmXGBQEVcnXZcJ7gAnYWekujR5skWW5WGdPXThNcEl3hGfn34+f4UA+OHeTdlY8/Z3Z7ELuDeF/2amZ444Sbj25ipQ+MNfPuzQ3MGfPwj5x/DRqllB9C4h6zUTzO5iufp371vXZriSz0R9U+ffEdY2XTiCnDQOhP3t/n+q4fx0ZcKSw+UUBr/7mQWTy5JTWCGUyu1LKrdh+YXiC0sxSTkxEQnde1TwqA0tzjYIX+/v7+D5+b6/Lq2JzTsa3xlcHol2HZPfiX6YYyZQrbpnqRTMysma4ZsfVb3Gr/tkl7UTD9SDcqE6HeP5ukV9h7+ffr/9CN9udJwD58Xqa+WMEauHpUeb9g8r/9unMfOVCh/NdCXBj5u9gLwxbpQTY5NSUXeEBi3T7igztl5WIGxzFLAlBkLYl1cF+5ZrrNRYTb8LLrOoWtffZXaQutoEpqlcB3IFM2lUdt+YSePRM8ZGZff8/Fs8che/RTuX4OimOyH5iLGdvXZ0B5zQcQHz7e+GUeX+WQNlZgOlIVnfRDnL5OgfWLgugM1RbhprvVPiZVqvxu7AjJhTegfG0IHhsNkjU51/xoOsJkWql2Ivkpc9LmsuIOZkVSXuAKsTL5FicDs8/Bvgp7JElEWkXWTaz8eAy9RP2P7dCTO9U48PXl+zeql3pa0o1hgk6zb8MYSzpuI16tlo/Zyt9LXUmdmK4HVEs4DlgYIoDb2GaNZDcjbNbPVfp5k1p5cHn0szE/NqZsxUFo1lnFfoLg2EtNdT0NxUtJ2govUV98gqmn8pcwUl7fBcypeE6ytpzhG0paZ5al5NG2/kYqtp/i1EQFHrlrLOAevvVc994BQCueLB+v626f1t0//G26b/BAAA//+/JgAM" } diff --git a/metricbeat/module/ceph/mgr_osd_perf/_meta/data.json b/metricbeat/module/ceph/mgr_osd_perf/_meta/data.json index 6d3c5f009981..ec4fc55b575d 100644 --- a/metricbeat/module/ceph/mgr_osd_perf/_meta/data.json +++ b/metricbeat/module/ceph/mgr_osd_perf/_meta/data.json @@ -1,33 +1,5 @@ { - "@timestamp": "2020-02-12T12:25:29.572Z", - "@metadata": { - "beat": "metricbeat", - "type": "_doc", - "version": "8.0.0" - }, - "agent": { - "type": "metricbeat", - "ephemeral_id": "62fb1ff9-4f2a-4347-bbd1-cee5df88c9da", - "hostname": "host", - "id": "993db589-f9d8-40f3-94cb-43b2b69fa9f4", - "version": "8.0.0" - }, - "event": { - "dataset": "http.json_namespace", - "module": "ceph", - "duration": 27722184 - }, - "metricset": { - "name": "mgr_osd_perf", - "period": 60000 - }, - "service": { - "type": "ceph", - "address": "localhost:8003" - }, - "ecs": { - "version": "1.4.0" - }, + "@timestamp": "2017-10-12T08:05:34.853Z", "ceph": { "mgr_osd_perf": { "id": 0, @@ -38,5 +10,17 @@ "apply_latency_ns": 23000000 } } + }, + "event": { + "dataset": "ceph.mgr_osd_perf", + "duration": 115000, + "module": "ceph" + }, + "metricset": { + "name": "mgr_osd_perf" + }, + "service": { + "address": "127.0.0.1:8003", + "type": "ceph" } } diff --git a/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/data.json b/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/data.json index 1bb99f68b28f..180c8f7a650e 100644 --- a/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/data.json +++ b/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/data.json @@ -1,85 +1,27 @@ { - "@timestamp": "2020-02-12T12:59:59.964Z", - "@metadata": { - "beat": "metricbeat", - "type": "_doc", - "version": "8.0.0" - }, - "service": { - "address": "localhost:8003", - "type": "ceph" - }, - "agent": { - "type": "metricbeat", - "ephemeral_id": "457300cc-fa5e-4818-b95e-54ca42b978cb", - "hostname": "host", - "id": "993db589-f9d8-40f3-94cb-43b2b69fa9f4", - "version": "8.0.0" - }, + "@timestamp": "2017-10-12T08:05:34.853Z", "ceph": { - "mgr_osd_pool_stats": [ - { - "pool_name": "rbd", - "pool_id": "1", - "client_io_rate": {} - }, - { - "pool_id": "2", - "client_io_rate": {}, - "pool_name": "cephfs_data" - }, - { - "client_io_rate": {}, - "pool_name": "cephfs_metadata", - "pool_id": "3" - }, - { - "client_io_rate": {}, - "pool_name": ".rgw.root", - "pool_id": "4" - }, - { - "pool_name": "default.rgw.control", - "pool_id": "5", - "client_io_rate": {} - }, - { - "pool_name": "default.rgw.meta", - "pool_id": "6", - "client_io_rate": {} - }, - { - "client_io_rate": {}, - "pool_name": "default.rgw.log", - "pool_id": "7" - }, - { - "pool_name": "default.rgw.buckets.index", - "pool_id": "8", - "client_io_rate": {} - }, - { - "pool_name": "scbench", - "pool_id": "9", - "client_io_rate": { - "read_bytes_sec": "85", - "write_bytes_sec": "802631707", - "read_op_per_sec": "0", - "write_op_per_sec": "336" - } + "mgr_osd_pool_stats": { + "pool_id": "9", + "pool_name": "scbench", + "client_io_rate": { + "read_bytes_sec": "85", + "write_bytes_sec": "802631707", + "read_op_per_sec": "0", + "write_op_per_sec": "336" } - ] - }, - "ecs": { - "version": "1.4.0" + } }, "event": { - "duration": 9206093, - "dataset": "http.json_namespace", + "dataset": "ceph.mgr_osd_pool_stats", + "duration": 115000, "module": "ceph" }, "metricset": { - "name": "mgr_osd_pool_stats", - "period": 60000 + "name": "mgr_osd_pool_stats" + }, + "service": { + "address": "127.0.0.1:8003", + "type": "ceph" } } diff --git a/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/fields.yml b/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/fields.yml index 351a71a991fe..b6eabf1d6652 100644 --- a/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/fields.yml +++ b/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/fields.yml @@ -1,5 +1,5 @@ - name: mgr_osd_pool_stats - type: nested + type: group description: > OSD pool stats of Ceph cluster release: beta @@ -8,10 +8,10 @@ type: keyword description: Pool name - name: pool_id - type: keyword + type: long description: Pool ID - name: client_io_rate - type: group + type: object description: Client I/O rates fields: - name: read_bytes_sec diff --git a/metricbeat/module/ceph/mgr_osd_pool_stats/data.go b/metricbeat/module/ceph/mgr_osd_pool_stats/data.go new file mode 100644 index 000000000000..4a86c6b9327e --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_pool_stats/data.go @@ -0,0 +1,60 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mgr_osd_pool_stats + +import ( + "github.com/pkg/errors" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/metricbeat/module/ceph/mgr" +) + +type OsdPoolStat struct { + PoolID uint64 `json:"pool_id"` + PoolName string `json:"pool_name"` + ClientIORate struct { + ReadBytesSec uint64 `json:"read_bytes_sec"` + WriteBytesSec uint64 `json:"write_bytes_sec"` + ReadOpPerSec uint64 `json:"read_op_per_sec"` + WriteOpPerSec uint64 `json:"write_op_per_sec"` + } `json:"client_io_rate"` +} + +func eventsMapping(content []byte) ([]common.MapStr, error) { + var response []OsdPoolStat + err := mgr.UnmarshalResponse(content, &response) + if err != nil { + return nil, errors.Wrap(err, "could not get response data") + } + + var events []common.MapStr + for _, stat := range response { + event := common.MapStr{ + "pool_id": stat.PoolID, + "pool_name": stat.PoolName, + "client_io_rate": common.MapStr{ + "read_bytes_sec": stat.ClientIORate.ReadBytesSec, + "write_bytes_sec": stat.ClientIORate.WriteBytesSec, + "read_op_per_sec": stat.ClientIORate.ReadOpPerSec, + "write_op_per_sec": stat.ClientIORate.WriteOpPerSec, + }, + } + events = append(events, event) + } + return events, nil +} diff --git a/metricbeat/module/ceph/mgr_osd_pool_stats/manifest.yml b/metricbeat/module/ceph/mgr_osd_pool_stats/manifest.yml deleted file mode 100644 index 0cbc0ace5ee6..000000000000 --- a/metricbeat/module/ceph/mgr_osd_pool_stats/manifest.yml +++ /dev/null @@ -1,44 +0,0 @@ -default: true -input: - module: http - metricset: json - defaults: - method: POST - namespace: "json_namespace" - basepath: "/request" - headers: - Content-Type: application/json - query: - wait: "1" - body: '{"prefix": "osd pool stats", "format": "json"}' - -processors: - - extract_array: - field: http.json_namespace.finished - mappings: - ceph.request: 0 - - decode_json_fields: - fields: ["ceph.request.outb"] - process_array: true - target: "ceph.mgr_osd_pool_stats" - - drop_fields: - fields: ["http", "ceph.request"] - - script: - lang: javascript - source: > - function process(event) { - var pools = event.Get("ceph.mgr_osd_pool_stats") - if (pools != null) { - var filtered = []; - pools.forEach(function(pool, index) { - if (pool["recovery"] !== undefined) { - delete pool["recovery"]; - } - if (pool["recovery_rate"] !== undefined) { - delete pool["recovery_rate"]; - } - filtered.push(pool); - }); - event.Put("ceph.mgr_osd_pool_stats", filtered); - } - } diff --git a/metricbeat/module/ceph/mgr_osd_pool_stats/mgr_osd_pool_stats.go b/metricbeat/module/ceph/mgr_osd_pool_stats/mgr_osd_pool_stats.go new file mode 100644 index 000000000000..2a31e6b71e80 --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_pool_stats/mgr_osd_pool_stats.go @@ -0,0 +1,93 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mgr_osd_pool_stats + +import ( + "fmt" + + "github.com/elastic/beats/metricbeat/helper" + "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/metricbeat/mb/parse" +) + +const ( + defaultScheme = "https" + defaultPath = "/request" + defaultQueryParams = "wait=1" + + cephPrefix = "osd pool stats" +) + +var ( + hostParser = parse.URLHostParserBuilder{ + DefaultScheme: defaultScheme, + DefaultPath: defaultPath, + QueryParams: defaultQueryParams, + }.Build() +) + +func init() { + mb.Registry.MustAddMetricSet("ceph", "mgr_osd_pool_stats", New, + mb.WithHostParser(hostParser), + ) +} + +type MetricSet struct { + mb.BaseMetricSet + *helper.HTTP +} + +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + http, err := helper.NewHTTP(base) + if err != nil { + return nil, err + } + http.SetMethod("POST") + http.SetHeader("Content-Type", "application/json") + http.SetHeader("Accept", "application/json") + http.SetBody([]byte(fmt.Sprintf(`{"prefix": "%s", "format": "json"}`, cephPrefix))) + + return &MetricSet{ + base, + http, + }, nil +} + +// Fetch methods implements the data gathering and data conversion to the right +// format. It publishes the event which is then forwarded to the output. In case +// of an error set the Error field of mb.Event or simply call report.Error(). +func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { + content, err := m.HTTP.FetchContent() + if err != nil { + return err + } + + events, err := eventsMapping(content) + if err != nil { + return err + } + + for _, event := range events { + reported := reporter.Event(mb.Event{MetricSetFields: event}) + if !reported { + m.Logger().Debug("error reporting event") + return nil + } + } + return nil +} diff --git a/metricbeat/module/ceph/mgr_osd_pool_stats/mgr_osd_pool_stats_integration_test.go b/metricbeat/module/ceph/mgr_osd_pool_stats/mgr_osd_pool_stats_integration_test.go new file mode 100644 index 000000000000..db5e45654a5c --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_pool_stats/mgr_osd_pool_stats_integration_test.go @@ -0,0 +1,52 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build integration,linux + +package mgr_osd_pool_stats + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/elastic/beats/libbeat/tests/compose" + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + "github.com/elastic/beats/metricbeat/module/ceph/mgrtest" +) + +const user = "demo" + +func TestData(t *testing.T) { + service := compose.EnsureUpWithTimeout(t, 120, "ceph-mgr") + + f := mbtest.NewReportingMetricSetV2Error(t, + getConfig(service.HostForPort(8003), mgrtest.GetPassword(t, service.HostForPort(5000), user))) + err := mbtest.WriteEventsReporterV2Error(f, t, "") + require.NoError(t, err) +} + +func getConfig(host, password string) map[string]interface{} { + return map[string]interface{}{ + "module": "ceph", + "metricsets": []string{"mgr_osd_pool_stats"}, + "hosts": []string{host}, + "username": user, + "password": password, + "ssl.verification_mode": "none", + } +} diff --git a/metricbeat/module/ceph/mgr_osd_pool_stats/mgr_osd_pool_stats_test.go b/metricbeat/module/ceph/mgr_osd_pool_stats/mgr_osd_pool_stats_test.go new file mode 100644 index 000000000000..7204ea70e72b --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_pool_stats/mgr_osd_pool_stats_test.go @@ -0,0 +1,98 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mgr_osd_pool_stats + +import ( + "io/ioutil" + "net/http" + "net/http/httptest" + "path/filepath" + "testing" + + "github.com/elastic/beats/libbeat/common" + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + + "github.com/stretchr/testify/assert" +) + +func TestFetchEventContents(t *testing.T) { + absPath, err := filepath.Abs("../_meta/testdata/") + assert.NoError(t, err) + + response, err := ioutil.ReadFile(absPath + "/mgr_osd_pool_stats_sample_response.json") + assert.NoError(t, err) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + w.Header().Set("Content-Type", "application/json;") + w.Write(response) + })) + defer server.Close() + + config := map[string]interface{}{ + "module": "ceph", + "metricsets": []string{"mgr_osd_pool_stats"}, + "hosts": []string{server.URL}, + } + + f := mbtest.NewReportingMetricSetV2Error(t, config) + events, errs := mbtest.ReportingFetchV2Error(f) + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) + } + assert.NotEmpty(t, events) + event := events[0].MetricSetFields + + t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event.StringToPrint()) + + assert.EqualValues(t, 1, event["pool_id"]) + assert.EqualValues(t, "rbd", event["pool_name"]) + + stats := event["client_io_rate"].(common.MapStr) + + assert.EqualValues(t, 1, stats["read_bytes_sec"]) + assert.EqualValues(t, 2, stats["read_op_per_sec"]) + assert.EqualValues(t, 3, stats["write_bytes_sec"]) + assert.EqualValues(t, 4, stats["write_op_per_sec"]) +} + +func TestFetchEventContents_Failed(t *testing.T) { + absPath, err := filepath.Abs("../_meta/testdata/") + assert.NoError(t, err) + + response, err := ioutil.ReadFile(absPath + "/mgr_failed_response.json") + assert.NoError(t, err) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + w.Header().Set("Content-Type", "application/json;") + w.Write([]byte(response)) + })) + defer server.Close() + + config := map[string]interface{}{ + "module": "ceph", + "metricsets": []string{"mgr_osd_pool_stats"}, + "hosts": []string{server.URL}, + } + + f := mbtest.NewReportingMetricSetV2Error(t, config) + events, errs := mbtest.ReportingFetchV2Error(f) + assert.Empty(t, events) + assert.NotEmpty(t, errs) +} diff --git a/metricbeat/module/ceph/mgr_osd_tree/_meta/data.json b/metricbeat/module/ceph/mgr_osd_tree/_meta/data.json index e5b3cf637c85..9239f288557b 100644 --- a/metricbeat/module/ceph/mgr_osd_tree/_meta/data.json +++ b/metricbeat/module/ceph/mgr_osd_tree/_meta/data.json @@ -1,81 +1,27 @@ { - "@timestamp": "2020-02-12T13:05:44.386Z", - "@metadata": { - "beat": "metricbeat", - "type": "_doc", - "version": "8.0.0" - }, - "service": { - "address": "localhost:8003", - "type": "ceph" - }, - "event": { - "dataset": "http.json_namespace", - "module": "ceph", - "duration": 30426747 - }, - "ecs": { - "version": "1.4.0" - }, + "@timestamp": "2017-10-12T08:05:34.853Z", "ceph": { "mgr_osd_tree": { - "nodes": [ - { - "name": "default", - "type": "root", - "type_id": 11, - "children": [ - -2 - ], - "id": -1 - }, - { - "type_id": 1, - "children": [ - 0 - ], - "id": -2, - "name": "c213fde94857", - "type": "host" - }, - { - "reweight": 1, - "id": 0, - "depth": 2, - "exists": 1, - "status": "up", - "primary_affinity": 1, - "name": "osd.0", - "type": "osd", - "type_id": 0, - "crush_weight": 0.009796142578125 - } + "children": [ + "-2" ], - "stray": [ - { - "reweight": 1, - "id": 2, - "depth": 2, - "exists": 1, - "status": "down", - "primary_affinity": 1, - "name": "osd.1", - "type": "osd", - "type_id": 0, - "crush_weight": 0.004796132578111 - } - ] + "father": "", + "id": -1, + "name": "default", + "type": "root", + "type_id": 10 } }, - "agent": { - "ephemeral_id": "fcabd35f-6792-47d9-9cb1-611030bff6e8", - "hostname": "host", - "id": "993db589-f9d8-40f3-94cb-43b2b69fa9f4", - "version": "8.0.0", - "type": "metricbeat" + "event": { + "dataset": "ceph.mgr_osd_tree", + "duration": 115000, + "module": "ceph" }, "metricset": { - "period": 60000, "name": "mgr_osd_tree" + }, + "service": { + "address": "127.0.0.1:8003", + "type": "ceph" } } diff --git a/metricbeat/module/ceph/mgr_osd_tree/_meta/fields.yml b/metricbeat/module/ceph/mgr_osd_tree/_meta/fields.yml index 81107cc9e5b8..93823ed2455f 100644 --- a/metricbeat/module/ceph/mgr_osd_tree/_meta/fields.yml +++ b/metricbeat/module/ceph/mgr_osd_tree/_meta/fields.yml @@ -1,82 +1,58 @@ - name: mgr_osd_tree type: group description: > - OSD tree stats of Ceph cluster + OSD tree information release: beta fields: - - name: nodes - type: nested + - name: id + type: long description: > - `nodes` contains disk active OSD nodes. - fields: - - name: name - type: keyword - description: Node name - - name: root - type: keyword - description: Node type - - name: type_id - type: long - description: Type ID - - name: id - type: long - description: Node ID - - name: children - type: long - description: Node children - - name: reweight - type: long - description: Reweight - - name: depth - type: long - description: Depth - - name: exists - type: long - description: Exists - - name: status - type: keyword - description: Node status - - name: primary_affinity - type: long - description: Primary affinity - - name: crush_weight - type: scaled_float - description: CRUSH weight - - name: stray - type: group + OSD or bucket node ID + - name: name + type: keyword description: > - `stray` contains stray OSD nodes. - fields: - - name: name - type: keyword - description: Node name - - name: root - type: keyword - description: Node type - - name: type_id - type: long - description: Type ID - - name: id - type: long - description: Node ID - - name: children - type: long - description: Node children - - name: reweight - type: long - description: Reweight - - name: depth - type: long - description: Depth - - name: exists - type: long - description: Exists - - name: status - type: keyword - description: Node status - - name: primary_affinity - type: long - description: Primary affinity - - name: crush_weight - type: scaled_float - description: CRUSH weight + OSD or bucket node name + - name: type + type: keyword + description: > + OSD or bucket node type, illegal type include OSD, host, root etc. + - name: type_id + type: long + description: > + OSD or bucket node type ID + - name: children + type: keyword + description: > + Bucket children list, separated by comma. + - name: crush_weight + type: float + description: > + OSD node CRUSH weight + - name: depth + type: long + description: > + Node depth + - name: exists + type: boolean + description: > + Node exists (1-yes, 0-no) + - name: primary_affinity + type: float + description: > + Weight of reading data from primary OSD + - name: reweight + type: long + description: > + Reweight of osd + - name: status + type: keyword + description: > + Status of OSD, it should be up or down + - name: device_class + type: keyword + description: > + Device class of OSD, like HDD, SSD etc. + - name: father + type: keyword + description: > + Parent node of this OSD or bucket node diff --git a/metricbeat/module/ceph/mgr_osd_tree/data.go b/metricbeat/module/ceph/mgr_osd_tree/data.go new file mode 100644 index 000000000000..8694c21f90a3 --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_tree/data.go @@ -0,0 +1,106 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mgr_osd_tree + +import ( + "strconv" + "strings" + + "github.com/pkg/errors" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/metricbeat/module/ceph/mgr" +) + +type OsdTreeResponse struct { + Nodes []struct { + ID int64 `json:"id"` + Name string `json:"name"` + Type string `json:"type"` + TypeID int64 `json:"type_id"` + Children []int64 `json:"children"` + CrushWeight float64 `json:"crush_weight"` + Depth int64 `json:"depth"` + Exist int64 `json:"exists"` + PrimaryAffinity float64 `json:"primary_affinity"` + Reweight float64 `json:"reweight"` + Status string `json:"status"` + DeviceClass string `json:"device_class"` + } `json:"nodes"` +} + +func eventsMapping(content []byte) ([]common.MapStr, error) { + var response OsdTreeResponse + err := mgr.UnmarshalResponse(content, &response) + if err != nil { + return nil, errors.Wrap(err, "could not get response data") + } + + nodeList := response.Nodes + + //generate fatherNode and children map + fatherMap := make(map[string]string) + childrenMap := make(map[string]string) + + for _, node := range nodeList { + if node.ID >= 0 { + continue // it's OSD node + } + var childrenList []string + for _, child := range node.Children { + childIDStr := strconv.FormatInt(child, 10) + childrenList = append(childrenList, childIDStr) + fatherMap[childIDStr] = node.Name + } + // generate bucket node's children list + childrenMap[node.Name] = strings.Join(childrenList, ",") + } + + // OSD node list + var events []common.MapStr + for _, node := range nodeList { + nodeInfo := common.MapStr{} + if node.ID < 0 { + // bucket node + nodeInfo["children"] = strings.Split(childrenMap[node.Name], ",") + } else { + // OSD node + nodeInfo["crush_weight"] = node.CrushWeight + nodeInfo["depth"] = node.Depth + nodeInfo["primary_affinity"] = node.PrimaryAffinity + nodeInfo["reweight"] = node.Reweight + nodeInfo["status"] = node.Status + nodeInfo["device_class"] = node.DeviceClass + if node.Exist > 0 { + nodeInfo["exists"] = true + } else { + nodeInfo["exists"] = false + } + } + nodeInfo["id"] = node.ID + nodeInfo["name"] = node.Name + nodeInfo["type"] = node.Type + nodeInfo["type_id"] = node.TypeID + + idStr := strconv.FormatInt(node.ID, 10) + nodeInfo["father"] = fatherMap[idStr] + + events = append(events, nodeInfo) + } + return events, nil +} diff --git a/metricbeat/module/ceph/mgr_osd_tree/manifest.yml b/metricbeat/module/ceph/mgr_osd_tree/manifest.yml deleted file mode 100644 index 5b8230cb9a10..000000000000 --- a/metricbeat/module/ceph/mgr_osd_tree/manifest.yml +++ /dev/null @@ -1,54 +0,0 @@ -default: true -input: - module: http - metricset: json - defaults: - method: POST - namespace: "json_namespace" - basepath: "/request" - headers: - Content-Type: application/json - query: - wait: "1" - body: '{"prefix": "osd tree", "format": "json"}' - -processors: - - extract_array: - field: http.json_namespace.finished - mappings: - ceph.request: 0 - - decode_json_fields: - fields: ["ceph.request.outb"] - process_array: true - target: "ceph.mgr_osd_tree" - - drop_fields: - ignore_missing: true - fields: ["http", "ceph.request"] - - script: - lang: javascript - source: > - function process(event) { - var nodes = event.Get("ceph.mgr_osd_tree.nodes") - if (nodes != null) { - var filtered = []; - nodes.forEach(function(node, index) { - if (node["pool_weights"] !== undefined) { - delete node["pool_weights"]; - } - filtered.push(node); - }); - event.Put("ceph.mgr_osd_tree.nodes", filtered); - } - - var stray = event.Get("ceph.mgr_osd_tree.stray") - if (stray != null) { - var filtered = []; - stray.forEach(function(node, index) { - if (node["pool_weights"] !== undefined) { - delete node["pool_weights"]; - } - filtered.push(node); - }); - event.Put("ceph.mgr_osd_tree.stray", filtered); - } - } diff --git a/metricbeat/module/ceph/mgr_osd_tree/mgr_osd_tree.go b/metricbeat/module/ceph/mgr_osd_tree/mgr_osd_tree.go new file mode 100644 index 000000000000..bc3d1c1df457 --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_tree/mgr_osd_tree.go @@ -0,0 +1,93 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mgr_osd_tree + +import ( + "fmt" + + "github.com/elastic/beats/metricbeat/helper" + "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/metricbeat/mb/parse" +) + +const ( + defaultScheme = "https" + defaultPath = "/request" + defaultQueryParams = "wait=1" + + cephPrefix = "osd tree" +) + +var ( + hostParser = parse.URLHostParserBuilder{ + DefaultScheme: defaultScheme, + DefaultPath: defaultPath, + QueryParams: defaultQueryParams, + }.Build() +) + +func init() { + mb.Registry.MustAddMetricSet("ceph", "mgr_osd_tree", New, + mb.WithHostParser(hostParser), + ) +} + +type MetricSet struct { + mb.BaseMetricSet + *helper.HTTP +} + +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + http, err := helper.NewHTTP(base) + if err != nil { + return nil, err + } + http.SetMethod("POST") + http.SetHeader("Content-Type", "application/json") + http.SetHeader("Accept", "application/json") + http.SetBody([]byte(fmt.Sprintf(`{"prefix": "%s", "format": "json"}`, cephPrefix))) + + return &MetricSet{ + base, + http, + }, nil +} + +// Fetch methods implements the data gathering and data conversion to the right +// format. It publishes the event which is then forwarded to the output. In case +// of an error set the Error field of mb.Event or simply call report.Error(). +func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { + content, err := m.HTTP.FetchContent() + if err != nil { + return err + } + + events, err := eventsMapping(content) + if err != nil { + return err + } + + for _, event := range events { + reported := reporter.Event(mb.Event{MetricSetFields: event}) + if !reported { + m.Logger().Debug("error reporting event") + return nil + } + } + return nil +} diff --git a/metricbeat/module/ceph/mgr_osd_tree/mgr_osd_tree_integration_test.go b/metricbeat/module/ceph/mgr_osd_tree/mgr_osd_tree_integration_test.go new file mode 100644 index 000000000000..f5dad0f0491d --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_tree/mgr_osd_tree_integration_test.go @@ -0,0 +1,52 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build integration,linux + +package mgr_osd_tree + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/elastic/beats/libbeat/tests/compose" + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + "github.com/elastic/beats/metricbeat/module/ceph/mgrtest" +) + +const user = "demo" + +func TestData(t *testing.T) { + service := compose.EnsureUpWithTimeout(t, 120, "ceph-mgr") + + f := mbtest.NewReportingMetricSetV2Error(t, + getConfig(service.HostForPort(8003), mgrtest.GetPassword(t, service.HostForPort(5000), user))) + err := mbtest.WriteEventsReporterV2Error(f, t, "") + require.NoError(t, err) +} + +func getConfig(host, password string) map[string]interface{} { + return map[string]interface{}{ + "module": "ceph", + "metricsets": []string{"mgr_osd_tree"}, + "hosts": []string{host}, + "username": user, + "password": password, + "ssl.verification_mode": "none", + } +} diff --git a/metricbeat/module/ceph/mgr_osd_tree/mgr_osd_tree_test.go b/metricbeat/module/ceph/mgr_osd_tree/mgr_osd_tree_test.go new file mode 100644 index 000000000000..f52e486c80d7 --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_tree/mgr_osd_tree_test.go @@ -0,0 +1,134 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mgr_osd_tree + +import ( + "io/ioutil" + "net/http" + "net/http/httptest" + "path/filepath" + "testing" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + + "github.com/stretchr/testify/assert" +) + +func TestFetchEventContents(t *testing.T) { + absPath, err := filepath.Abs("../_meta/testdata/") + assert.NoError(t, err) + + response, err := ioutil.ReadFile(absPath + "/mgr_osd_tree_sample_response.json") + assert.NoError(t, err) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + w.Header().Set("Content-Type", "application/json;") + w.Write(response) + })) + defer server.Close() + + config := map[string]interface{}{ + "module": "ceph", + "metricsets": []string{"mgr_osd_tree"}, + "hosts": []string{server.URL}, + } + + f := mbtest.NewReportingMetricSetV2Error(t, config) + events, errs := mbtest.ReportingFetchV2Error(f) + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) + } + + assert.NotEmpty(t, events) + nodeInfo := events[0].MetricSetFields + + t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), nodeInfo.StringToPrint()) + + //check root bucket info + assert.EqualValues(t, "default", nodeInfo["name"]) + assert.EqualValues(t, "root", nodeInfo["type"]) + assert.EqualValues(t, []string{"-3"}, nodeInfo["children"]) + assert.EqualValues(t, -1, nodeInfo["id"]) + assert.EqualValues(t, 10, nodeInfo["type_id"]) + assert.EqualValues(t, "", nodeInfo["father"]) + + //check host bucket info + nodeInfo = events[1].MetricSetFields + assert.EqualValues(t, "ceph-mon1", nodeInfo["name"]) + assert.EqualValues(t, "host", nodeInfo["type"]) + assert.EqualValues(t, []string{"1", "0"}, nodeInfo["children"]) + assert.EqualValues(t, -3, nodeInfo["id"]) + assert.EqualValues(t, 1, nodeInfo["type_id"]) + assert.EqualValues(t, "default", nodeInfo["father"]) + + //check osd bucket info + nodeInfo = events[2].MetricSetFields + assert.EqualValues(t, "up", nodeInfo["status"]) + assert.EqualValues(t, "osd.0", nodeInfo["name"]) + assert.EqualValues(t, "osd", nodeInfo["type"]) + assert.EqualValues(t, 1, nodeInfo["primary_affinity"]) + assert.EqualValues(t, true, nodeInfo["exists"]) + assert.EqualValues(t, 0, nodeInfo["id"]) + assert.EqualValues(t, 0, nodeInfo["type_id"]) + assert.EqualValues(t, 0.048691, nodeInfo["crush_weight"]) + assert.EqualValues(t, "hdd", nodeInfo["device_class"]) + assert.EqualValues(t, 1, nodeInfo["reweight"]) + assert.EqualValues(t, "ceph-mon1", nodeInfo["father"]) + assert.EqualValues(t, 2, nodeInfo["depth"]) + + nodeInfo = events[3].MetricSetFields + assert.EqualValues(t, "up", nodeInfo["status"]) + assert.EqualValues(t, "osd.1", nodeInfo["name"]) + assert.EqualValues(t, "osd", nodeInfo["type"]) + assert.EqualValues(t, 1, nodeInfo["primary_affinity"]) + assert.EqualValues(t, true, nodeInfo["exists"]) + assert.EqualValues(t, 1, nodeInfo["id"]) + assert.EqualValues(t, 0, nodeInfo["type_id"]) + assert.EqualValues(t, 0.048691, nodeInfo["crush_weight"]) + assert.EqualValues(t, "hdd", nodeInfo["device_class"]) + assert.EqualValues(t, 1, nodeInfo["reweight"]) + assert.EqualValues(t, "ceph-mon1", nodeInfo["father"]) + assert.EqualValues(t, 2, nodeInfo["depth"]) +} + +func TestFetchEventContents_Failed(t *testing.T) { + absPath, err := filepath.Abs("../_meta/testdata/") + assert.NoError(t, err) + + response, err := ioutil.ReadFile(absPath + "/mgr_failed_response.json") + assert.NoError(t, err) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + w.Header().Set("Content-Type", "application/json;") + w.Write([]byte(response)) + })) + defer server.Close() + + config := map[string]interface{}{ + "module": "ceph", + "metricsets": []string{"mgr_osd_tree"}, + "hosts": []string{server.URL}, + } + + f := mbtest.NewReportingMetricSetV2Error(t, config) + events, errs := mbtest.ReportingFetchV2Error(f) + assert.Empty(t, events) + assert.NotEmpty(t, errs) +} diff --git a/metricbeat/module/ceph/test_ceph_mgr.py b/metricbeat/module/ceph/test_ceph_mgr.py index bc4a552a9569..f0ddcca27493 100644 --- a/metricbeat/module/ceph/test_ceph_mgr.py +++ b/metricbeat/module/ceph/test_ceph_mgr.py @@ -38,7 +38,9 @@ def get_password(self): "mgr_cluster_disk", "mgr_cluster_health", "mgr_osd_disk", - "mgr_osd_perf" + "mgr_osd_perf", + "mgr_osd_pool_stats", + "mgr_osd_tree" ]) @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") def test_ceph_mgr(self, metricset): From 1521de3fcb9b4a5fd1f401827d572d3d133c47a8 Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Mon, 17 Feb 2020 13:04:54 +0100 Subject: [PATCH 34/41] Update source after code review --- metricbeat/docs/fields.asciidoc | 192 +++++++++--------- metricbeat/docs/modules/ceph.asciidoc | 8 +- ...d_disk.asciidoc => mgr_pool_disk.asciidoc} | 8 +- metricbeat/docs/modules_list.asciidoc | 2 +- metricbeat/include/list_common.go | 2 +- .../_meta/{Dockerfile => Dockerfile.jewel} | 0 metricbeat/module/ceph/_meta/config-mgr.yml | 20 ++ metricbeat/module/ceph/_meta/config.yml | 21 -- .../module/ceph/_meta/supported-versions.yml | 5 + metricbeat/module/ceph/docker-compose.yml | 11 +- metricbeat/module/ceph/fields.go | 2 +- metricbeat/module/ceph/mgr/metricset.go | 52 +++++ .../ceph/mgr_cluster_disk/_meta/data.json | 4 +- .../ceph/mgr_cluster_disk/_meta/fields.yml | 4 +- .../testdata/mgr_df_sample_response.json | 0 .../mgr_df_sample_response.json-expected.json | 1 + .../_meta/testdata/mgr_failed_response.json | 0 .../mgr_failed_response.json-expected.json | 1 + .../ceph/mgr_cluster_disk/mgr_cluster_disk.go | 25 +-- .../mgr_cluster_disk/mgr_cluster_disk_test.go | 73 +------ .../ceph/mgr_cluster_health/_meta/data.json | 4 +- .../ceph/mgr_cluster_health/_meta/fields.yml | 4 +- .../_meta/testdata/mgr_failed_response.json | 17 ++ .../testdata/mgr_status_sample_response.json | 0 .../mgr_time_sync_status_sample_response.json | 0 .../mgr_cluster_health/mgr_cluster_health.go | 21 +- .../mgr_cluster_health_integration_test.go | 3 +- .../mgr_cluster_health_test.go | 4 +- .../ceph/mgr_osd_disk/mgr_osd_disk_test.go | 100 --------- .../_meta/testdata/mgr_failed_response.json | 17 ++ .../mgr_failed_response.json-expected.json | 1 + .../mgr_osd_perf_sample_response.json | 0 ...sd_perf_sample_response.json-expected.json | 1 + .../module/ceph/mgr_osd_perf/mgr_osd_perf.go | 21 +- .../ceph/mgr_osd_perf/mgr_osd_perf_test.go | 73 +------ .../_meta/testdata/mgr_failed_response.json | 17 ++ .../mgr_failed_response.json-expected.json | 1 + .../mgr_osd_pool_stats_sample_response.json | 0 ...l_stats_sample_response.json-expected.json | 1 + .../mgr_osd_pool_stats/mgr_osd_pool_stats.go | 21 +- .../mgr_osd_pool_stats_test.go | 74 +------ .../module/ceph/mgr_osd_tree/_meta/data.json | 4 +- .../module/ceph/mgr_osd_tree/_meta/fields.yml | 2 +- .../_meta/testdata/mgr_failed_response.json | 17 ++ .../mgr_failed_response.json-expected.json | 1 + .../mgr_osd_tree_sample_response.json | 0 ...sd_tree_sample_response.json-expected.json | 1 + .../module/ceph/mgr_osd_tree/mgr_osd_tree.go | 21 +- .../ceph/mgr_osd_tree/mgr_osd_tree_test.go | 110 +--------- .../_meta/data.json | 6 +- .../_meta/docs.asciidoc | 0 .../_meta/fields.yml | 4 +- .../testdata/mgr_df_sample_response.json | 17 ++ .../mgr_df_sample_response.json-expected.json | 1 + .../_meta/testdata/mgr_failed_response.json | 17 ++ .../mgr_failed_response.json-expected.json | 1 + .../{mgr_osd_disk => mgr_pool_disk}/data.go | 2 +- .../mgr_pool_disk.go} | 25 +-- .../mgr_pool_disk_integration_test.go} | 4 +- .../ceph/mgr_pool_disk/mgr_pool_disk_test.go | 28 +++ metricbeat/module/ceph/test_ceph.py | 62 ++++++ metricbeat/module/ceph/test_ceph_mgr.py | 60 ------ metricbeat/modules.d/ceph-mgr.yml.disabled | 23 +++ metricbeat/modules.d/ceph.yml.disabled | 21 -- 64 files changed, 473 insertions(+), 765 deletions(-) rename metricbeat/docs/modules/ceph/{mgr_osd_disk.asciidoc => mgr_pool_disk.asciidoc} (58%) rename metricbeat/module/ceph/_meta/{Dockerfile => Dockerfile.jewel} (100%) create mode 100644 metricbeat/module/ceph/_meta/config-mgr.yml create mode 100644 metricbeat/module/ceph/_meta/supported-versions.yml create mode 100644 metricbeat/module/ceph/mgr/metricset.go rename metricbeat/module/ceph/{ => mgr_cluster_disk}/_meta/testdata/mgr_df_sample_response.json (100%) create mode 100644 metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/mgr_df_sample_response.json-expected.json rename metricbeat/module/ceph/{ => mgr_cluster_disk}/_meta/testdata/mgr_failed_response.json (100%) create mode 100644 metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/mgr_failed_response.json-expected.json create mode 100644 metricbeat/module/ceph/mgr_cluster_health/_meta/testdata/mgr_failed_response.json rename metricbeat/module/ceph/{ => mgr_cluster_health}/_meta/testdata/mgr_status_sample_response.json (100%) rename metricbeat/module/ceph/{ => mgr_cluster_health}/_meta/testdata/mgr_time_sync_status_sample_response.json (100%) delete mode 100644 metricbeat/module/ceph/mgr_osd_disk/mgr_osd_disk_test.go create mode 100644 metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/mgr_failed_response.json create mode 100644 metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/mgr_failed_response.json-expected.json rename metricbeat/module/ceph/{ => mgr_osd_perf}/_meta/testdata/mgr_osd_perf_sample_response.json (100%) create mode 100644 metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/mgr_osd_perf_sample_response.json-expected.json create mode 100644 metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/mgr_failed_response.json create mode 100644 metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/mgr_failed_response.json-expected.json rename metricbeat/module/ceph/{ => mgr_osd_pool_stats}/_meta/testdata/mgr_osd_pool_stats_sample_response.json (100%) create mode 100644 metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/mgr_osd_pool_stats_sample_response.json-expected.json create mode 100644 metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/mgr_failed_response.json create mode 100644 metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/mgr_failed_response.json-expected.json rename metricbeat/module/ceph/{ => mgr_osd_tree}/_meta/testdata/mgr_osd_tree_sample_response.json (100%) create mode 100644 metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/mgr_osd_tree_sample_response.json-expected.json rename metricbeat/module/ceph/{mgr_osd_disk => mgr_pool_disk}/_meta/data.json (85%) rename metricbeat/module/ceph/{mgr_osd_disk => mgr_pool_disk}/_meta/docs.asciidoc (100%) rename metricbeat/module/ceph/{mgr_osd_disk => mgr_pool_disk}/_meta/fields.yml (94%) create mode 100644 metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/mgr_df_sample_response.json create mode 100644 metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/mgr_df_sample_response.json-expected.json create mode 100644 metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/mgr_failed_response.json create mode 100644 metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/mgr_failed_response.json-expected.json rename metricbeat/module/ceph/{mgr_osd_disk => mgr_pool_disk}/data.go (98%) rename metricbeat/module/ceph/{mgr_osd_disk/mgr_osd_disk.go => mgr_pool_disk/mgr_pool_disk.go} (79%) rename metricbeat/module/ceph/{mgr_osd_disk/mgr_osd_disk_integration_test.go => mgr_pool_disk/mgr_pool_disk_integration_test.go} (95%) create mode 100644 metricbeat/module/ceph/mgr_pool_disk/mgr_pool_disk_test.go delete mode 100644 metricbeat/module/ceph/test_ceph_mgr.py create mode 100644 metricbeat/modules.d/ceph-mgr.yml.disabled diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index ef21b02219d7..b408b4e5c92a 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -3933,13 +3933,13 @@ type: long -- [float] -=== mgr_cluster_disk +=== cluster_disk -mgr_cluster_disk +cluster_disk -*`ceph.mgr_cluster_disk.available.bytes`*:: +*`ceph.cluster_disk.available.bytes`*:: + -- Available bytes of the cluster @@ -3951,7 +3951,7 @@ format: bytes -- -*`ceph.mgr_cluster_disk.total.bytes`*:: +*`ceph.cluster_disk.total.bytes`*:: + -- Total bytes of the cluster @@ -3963,7 +3963,7 @@ format: bytes -- -*`ceph.mgr_cluster_disk.used.bytes`*:: +*`ceph.cluster_disk.used.bytes`*:: + -- Used bytes of the cluster @@ -3976,13 +3976,13 @@ format: bytes -- [float] -=== mgr_cluster_health +=== cluster_health -mgr_cluster_health +cluster_health -*`ceph.mgr_cluster_health.overall_status`*:: +*`ceph.cluster_health.overall_status`*:: + -- Overall status of the cluster @@ -3992,7 +3992,7 @@ type: keyword -- -*`ceph.mgr_cluster_health.timechecks.epoch`*:: +*`ceph.cluster_health.timechecks.epoch`*:: + -- Map version @@ -4002,7 +4002,7 @@ type: long -- -*`ceph.mgr_cluster_health.timechecks.round.value`*:: +*`ceph.cluster_health.timechecks.round.value`*:: + -- timecheck round @@ -4012,7 +4012,7 @@ type: long -- -*`ceph.mgr_cluster_health.timechecks.round.status`*:: +*`ceph.cluster_health.timechecks.round.status`*:: + -- Status of the round @@ -4022,77 +4022,6 @@ type: keyword -- -[float] -=== mgr_osd_disk - -mgr_osd_disk - - - -*`ceph.mgr_osd_disk.id`*:: -+ --- -Id of the pool - - -type: long - --- - -*`ceph.mgr_osd_disk.name`*:: -+ --- -Name of the pool - - -type: keyword - --- - -*`ceph.mgr_osd_disk.stats.available.bytes`*:: -+ --- -Available bytes of the pool - - -type: long - -format: bytes - --- - -*`ceph.mgr_osd_disk.stats.objects`*:: -+ --- -Number of objects of the pool - - -type: long - --- - -*`ceph.mgr_osd_disk.stats.used.bytes`*:: -+ --- -Used bytes of the pool - - -type: long - -format: bytes - --- - -*`ceph.mgr_osd_disk.stats.used.kb`*:: -+ --- -Used kb of the pool - - -type: long - --- - [float] === mgr_osd_perf @@ -4180,13 +4109,13 @@ type: object -- [float] -=== mgr_osd_tree +=== osd_tree OSD tree information -*`ceph.mgr_osd_tree.id`*:: +*`ceph.osd_tree.id`*:: + -- OSD or bucket node ID @@ -4196,7 +4125,7 @@ type: long -- -*`ceph.mgr_osd_tree.name`*:: +*`ceph.osd_tree.name`*:: + -- OSD or bucket node name @@ -4206,7 +4135,7 @@ type: keyword -- -*`ceph.mgr_osd_tree.type`*:: +*`ceph.osd_tree.type`*:: + -- OSD or bucket node type, illegal type include OSD, host, root etc. @@ -4216,7 +4145,7 @@ type: keyword -- -*`ceph.mgr_osd_tree.type_id`*:: +*`ceph.osd_tree.type_id`*:: + -- OSD or bucket node type ID @@ -4226,7 +4155,7 @@ type: long -- -*`ceph.mgr_osd_tree.children`*:: +*`ceph.osd_tree.children`*:: + -- Bucket children list, separated by comma. @@ -4236,7 +4165,7 @@ type: keyword -- -*`ceph.mgr_osd_tree.crush_weight`*:: +*`ceph.osd_tree.crush_weight`*:: + -- OSD node CRUSH weight @@ -4246,7 +4175,7 @@ type: float -- -*`ceph.mgr_osd_tree.depth`*:: +*`ceph.osd_tree.depth`*:: + -- Node depth @@ -4256,7 +4185,7 @@ type: long -- -*`ceph.mgr_osd_tree.exists`*:: +*`ceph.osd_tree.exists`*:: + -- Node exists (1-yes, 0-no) @@ -4266,7 +4195,7 @@ type: boolean -- -*`ceph.mgr_osd_tree.primary_affinity`*:: +*`ceph.osd_tree.primary_affinity`*:: + -- Weight of reading data from primary OSD @@ -4276,7 +4205,7 @@ type: float -- -*`ceph.mgr_osd_tree.reweight`*:: +*`ceph.osd_tree.reweight`*:: + -- Reweight of osd @@ -4286,7 +4215,7 @@ type: long -- -*`ceph.mgr_osd_tree.status`*:: +*`ceph.osd_tree.status`*:: + -- Status of OSD, it should be up or down @@ -4296,7 +4225,7 @@ type: keyword -- -*`ceph.mgr_osd_tree.device_class`*:: +*`ceph.osd_tree.device_class`*:: + -- Device class of OSD, like HDD, SSD etc. @@ -4306,7 +4235,7 @@ type: keyword -- -*`ceph.mgr_osd_tree.father`*:: +*`ceph.osd_tree.father`*:: + -- Parent node of this OSD or bucket node @@ -4316,6 +4245,77 @@ type: keyword -- +[float] +=== pool_disk + +pool_disk + + + +*`ceph.pool_disk.id`*:: ++ +-- +Id of the pool + + +type: long + +-- + +*`ceph.pool_disk.name`*:: ++ +-- +Name of the pool + + +type: keyword + +-- + +*`ceph.pool_disk.stats.available.bytes`*:: ++ +-- +Available bytes of the pool + + +type: long + +format: bytes + +-- + +*`ceph.pool_disk.stats.objects`*:: ++ +-- +Number of objects of the pool + + +type: long + +-- + +*`ceph.pool_disk.stats.used.bytes`*:: ++ +-- +Used bytes of the pool + + +type: long + +format: bytes + +-- + +*`ceph.pool_disk.stats.used.kb`*:: ++ +-- +Used kb of the pool + + +type: long + +-- + [float] === monitor_health diff --git a/metricbeat/docs/modules/ceph.asciidoc b/metricbeat/docs/modules/ceph.asciidoc index 558b0753b82c..ffbe40413e2a 100644 --- a/metricbeat/docs/modules/ceph.asciidoc +++ b/metricbeat/docs/modules/ceph.asciidoc @@ -58,14 +58,14 @@ The following metricsets are available: * <> -* <> - * <> * <> * <> +* <> + * <> * <> @@ -84,14 +84,14 @@ include::ceph/mgr_cluster_disk.asciidoc[] include::ceph/mgr_cluster_health.asciidoc[] -include::ceph/mgr_osd_disk.asciidoc[] - include::ceph/mgr_osd_perf.asciidoc[] include::ceph/mgr_osd_pool_stats.asciidoc[] include::ceph/mgr_osd_tree.asciidoc[] +include::ceph/mgr_pool_disk.asciidoc[] + include::ceph/monitor_health.asciidoc[] include::ceph/osd_df.asciidoc[] diff --git a/metricbeat/docs/modules/ceph/mgr_osd_disk.asciidoc b/metricbeat/docs/modules/ceph/mgr_pool_disk.asciidoc similarity index 58% rename from metricbeat/docs/modules/ceph/mgr_osd_disk.asciidoc rename to metricbeat/docs/modules/ceph/mgr_pool_disk.asciidoc index 22bfe8b3fbc0..4e64a31f1246 100644 --- a/metricbeat/docs/modules/ceph/mgr_osd_disk.asciidoc +++ b/metricbeat/docs/modules/ceph/mgr_pool_disk.asciidoc @@ -2,12 +2,12 @@ This file is generated! See scripts/mage/docs_collector.go //// -[[metricbeat-metricset-ceph-mgr_osd_disk]] -=== Ceph mgr_osd_disk metricset +[[metricbeat-metricset-ceph-mgr_pool_disk]] +=== Ceph mgr_pool_disk metricset beta[] -include::../../../module/ceph/mgr_osd_disk/_meta/docs.asciidoc[] +include::../../../module/ceph/mgr_pool_disk/_meta/docs.asciidoc[] ==== Fields @@ -19,5 +19,5 @@ Here is an example document generated by this metricset: [source,json] ---- -include::../../../module/ceph/mgr_osd_disk/_meta/data.json[] +include::../../../module/ceph/mgr_pool_disk/_meta/data.json[] ---- diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index 4f3e8210926d..bf18f416ef09 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -43,10 +43,10 @@ This file is generated! See scripts/mage/docs_collector.go |<> |<> beta[] |<> beta[] -|<> beta[] |<> beta[] |<> beta[] |<> beta[] +|<> beta[] |<> |<> |<> diff --git a/metricbeat/include/list_common.go b/metricbeat/include/list_common.go index 5583bdc4c330..503ce16ef21f 100644 --- a/metricbeat/include/list_common.go +++ b/metricbeat/include/list_common.go @@ -34,10 +34,10 @@ import ( _ "github.com/elastic/beats/metricbeat/module/ceph/cluster_status" _ "github.com/elastic/beats/metricbeat/module/ceph/mgr_cluster_disk" _ "github.com/elastic/beats/metricbeat/module/ceph/mgr_cluster_health" - _ "github.com/elastic/beats/metricbeat/module/ceph/mgr_osd_disk" _ "github.com/elastic/beats/metricbeat/module/ceph/mgr_osd_perf" _ "github.com/elastic/beats/metricbeat/module/ceph/mgr_osd_pool_stats" _ "github.com/elastic/beats/metricbeat/module/ceph/mgr_osd_tree" + _ "github.com/elastic/beats/metricbeat/module/ceph/mgr_pool_disk" _ "github.com/elastic/beats/metricbeat/module/ceph/monitor_health" _ "github.com/elastic/beats/metricbeat/module/ceph/osd_df" _ "github.com/elastic/beats/metricbeat/module/ceph/osd_tree" diff --git a/metricbeat/module/ceph/_meta/Dockerfile b/metricbeat/module/ceph/_meta/Dockerfile.jewel similarity index 100% rename from metricbeat/module/ceph/_meta/Dockerfile rename to metricbeat/module/ceph/_meta/Dockerfile.jewel diff --git a/metricbeat/module/ceph/_meta/config-mgr.yml b/metricbeat/module/ceph/_meta/config-mgr.yml new file mode 100644 index 000000000000..a4bd4ed78d50 --- /dev/null +++ b/metricbeat/module/ceph/_meta/config-mgr.yml @@ -0,0 +1,20 @@ +- module: ceph + metricsets: + - mgr_cluster_health + period: 10s + hosts: [ "https://localhost:8003" ] + #username: "user" + #password: "secret" + +- module: ceph + metricsets: + - mgr_cluster_disk + - mgr_cluster_health + - mgr_osd_disk + - mgr_osd_perf + # - mgr_osd_pool_stats + # - mgr_osd_tree + period: 1m + hosts: [ "https://localhost:8003" ] + #username: "user" + #password: "secret" diff --git a/metricbeat/module/ceph/_meta/config.yml b/metricbeat/module/ceph/_meta/config.yml index 105b67feaf04..6df14df1d80d 100644 --- a/metricbeat/module/ceph/_meta/config.yml +++ b/metricbeat/module/ceph/_meta/config.yml @@ -16,24 +16,3 @@ # - pool_disk period: 1m hosts: ["localhost:5000"] - -- module: ceph - #metricsets: - # - mgr_cluster_health - period: 10s - hosts: [ "https://localhost:8003" ] - #username: "user" - #password: "secret" - -- module: ceph - #metricsets: - # - mgr_cluster_disk - # - mgr_cluster_health - # - mgr_osd_disk - # - mgr_osd_perf - # - mgr_osd_pool_stats - # - mgr_osd_tree - period: 1m - hosts: [ "https://localhost:8003" ] - #username: "user" - #password: "secret" diff --git a/metricbeat/module/ceph/_meta/supported-versions.yml b/metricbeat/module/ceph/_meta/supported-versions.yml new file mode 100644 index 000000000000..e2f6799ecf52 --- /dev/null +++ b/metricbeat/module/ceph/_meta/supported-versions.yml @@ -0,0 +1,5 @@ +variants: + - CEPH_NAME: jewel + CEPH_VERSION: master-6373c6a-jewel-centos-7-x86_64 + - CEPH_NAME: nautilus + CEPH_VERSION: master-97985eb-nautilus-centos-7-x86_64 diff --git a/metricbeat/module/ceph/docker-compose.yml b/metricbeat/module/ceph/docker-compose.yml index 9afdcee7fc36..dccee656f9a4 100644 --- a/metricbeat/module/ceph/docker-compose.yml +++ b/metricbeat/module/ceph/docker-compose.yml @@ -5,19 +5,10 @@ services: image: docker.elastic.co/integrations-ci/beats-ceph:${CEPH_VERSION:-master-6373c6a-jewel-centos-7-x86_64}-1 build: context: ./_meta - dockerfile: Dockerfile + dockerfile: Dockerfile.${CEPH_CODENAME:-jewel} args: CEPH_VERSION: ${CEPH_VERSION:-master-6373c6a-jewel-centos-7-x86_64} ports: - 5000 - ceph-mgr: - image: docker.elastic.co/integrations-ci/beats-ceph:${CEPH_VERSION:-master-97985eb-nautilus-centos-7-x86_64}-1 - build: - context: ./_meta - dockerfile: Dockerfile.nautilus - args: - CEPH_VERSION: ${CEPH_VERSION:-master-97985eb-nautilus-centos-7-x86_64} - ports: - - 5000 - 8003 - 8080 diff --git a/metricbeat/module/ceph/fields.go b/metricbeat/module/ceph/fields.go index 194292c33623..a9f3cc856f8c 100644 --- a/metricbeat/module/ceph/fields.go +++ b/metricbeat/module/ceph/fields.go @@ -32,5 +32,5 @@ func init() { // AssetCeph returns asset data. // This is the base64 encoded gzipped contents of ../metricbeat/module/ceph. func AssetCeph() string { - return "eJzsWs9v47oRvuevGOTUAlm3vfpQYHezwAbt7gbJW7xDUWhpcSyxpkiCpOL6vy9IybIjUT9i03LRxqf3Yu83H2eGw5mP/AAb3C0hRZXfAFhmOS7h9jOq/PYGgKJJNVOWSbGEv94AALivoJC05HgDYHKpbZJKsWbZEtaEG/dXjRyJwSVkxP0GrWUiM0v4x60x/PYObnNr1e0/bwDWDDk1S4/8AQQpsOHiPnanHIqWpar/EmDkPr/cP/oFqRSWMGHA5ggFWs1S99/EwhY1gkk1UUhhrWUBn788fl3UAMc0XlHhpbGoE8rMpvkyRGuAmvv04Lz20/7TJnNMiLwQxsmK42K1s2he/WbPi0uRtb4YoOY+H/eo4FFBrr0Da9atn6+lLohdQpfAnqSVlvCoBH9ziHHIlQZpVG4/DdLTqbUzLUfCbR4j1zpIb882+YKacJ4YS2wZ9tcGd1up6dtc9qPChQp3yG1NSrEC0xzTjVmgkmkeKXbfiIIX1IZJMWZWy1LQxQvhJUYy3oCDx55GIHYonl+F4DWRdm52bIdzM5xnEzK2gz+Um6GwnR4Lf6rVvhhMCE3Wa5YuNBKavKmK9FemcXKVf8AZBZtrWWa5Ki0o1GAwlX25U3PdamZxdrLe6glsvWelSpRLCExjxffYhUwq8zbnXYxP5aUphApmFCcpLvzpGpnGHhxUBqIsVj1VuOEgV//C1MY6QTssavhJVDSxTAaJmJRwpMmaS2J78luhTlG0v30r3S6FPUuKmSYU6UWCtgcfCVrD4TJBa1hMCFpD5XpBa+j2B01lC0osmb1iqgyc3YHeVWULPwDMTo28HhD6yPksn52cPQwHfcRc0z87r7IZDHpo+YYHfUuHiftjpK35mPnOGo9/N0whlWVnQ51q/TmXWwO53EJBxA5UZoBoBCZqUnI9uvQOv8jNXh0gM97vSUMX65KHS/dKSo6kzWrE+INxoNABPbYokOiLWHXAI6bLIpGGxjolWsngkF0mjI17eyalujQZl5qyItSk58/HQVJMXIPUw/dBUhoLohTSRGVzM3v68u3j4+OX+15+MSd2j9XuMZqmMNNJLMluAKsZcFdo34W7/3Ph7jhPzhfvBtFOybt3Ce9dwquzVBoapSoGcE7JTNZe5qlBeKD7tSspw51Nb3N9ksO/kwJHTboom8WsZb/DZUpZrXjG1Qa+++PZ0dqLAtO8NUOJP8NHnt1mFZPaZhXk1d61CvX6nF374/keHIZbqEgPN6NyXQnf3bI944Z25B7CzWPl91QWBbMJJxZFukuK05Ljs0eBGsUNIcVQsIlSfHeuzY8OZLLJ1jpFrHWK6es80WZnnSLcJ/lclrLqRsYvksYyWsqq/4idyJ7iOUfGo2PWr2o4+BM3i0fu2S0pZyhswmSiiQ1zr8rxYPp4EHj40w9wKO18CLntmMPhXixwWTK4TPepXbuET75k+8uanmuRY6NHF1wRrDo0i2KK4eG7qumGn9xCpUKvSYveq6DumiPY/t3fQA0bb+9hqxHP3b0OA5ioDt/XzftVu0nHTWpYlekGLQhJsW+/xW0qA3Z7K4gzc1HLDu0OGOeYEe7/D5hIeUnR/foOcmnsHWgpLaBNF70kTy1zk0n21sKccaoxrBif5KVPleU9MHDmXGBQEVcnXZcJ7gAnYWekujR5skWW5WGdPXThNcEl3hGfn34+f4UA+OHeTdlY8/Z3Z7ELuDeF/2amZ444Sbj25ipQ+MNfPuzQ3MGfPwj5x/DRqllB9C4h6zUTzO5iufp371vXZriSz0R9U+ffEdY2XTiCnDQOhP3t/n+q4fx0ZcKSw+UUBr/7mQWTy5JTWCGUyu1LKrdh+YXiC0sxSTkxEQnde1TwqA0tzjYIX+/v7+D5+b6/Lq2JzTsa3xlcHol2HZPfiX6YYyZQrbpnqRTMysma4ZsfVb3Gr/tkl7UTD9SDcqE6HeP5ukV9h7+ffr/9CN9udJwD58Xqa+WMEauHpUeb9g8r/9unMfOVCh/NdCXBj5u9gLwxbpQTY5NSUXeEBi3T7igztl5WIGxzFLAlBkLYl1cF+5ZrrNRYTb8LLrOoWtffZXaQutoEpqlcB3IFM2lUdt+YSePRM8ZGZff8/Fs8che/RTuX4OimOyH5iLGdvXZ0B5zQcQHz7e+GUeX+WQNlZgOlIVnfRDnL5OgfWLgugM1RbhprvVPiZVqvxu7AjJhTegfG0IHhsNkjU51/xoOsJkWql2Ivkpc9LmsuIOZkVSXuAKsTL5FicDs8/Bvgp7JElEWkXWTaz8eAy9RP2P7dCTO9U48PXl+zeql3pa0o1hgk6zb8MYSzpuI16tlo/Zyt9LXUmdmK4HVEs4DlgYIoDb2GaNZDcjbNbPVfp5k1p5cHn0szE/NqZsxUFo1lnFfoLg2EtNdT0NxUtJ2govUV98gqmn8pcwUl7fBcypeE6ytpzhG0paZ5al5NG2/kYqtp/i1EQFHrlrLOAevvVc994BQCueLB+v626f1t0//G26b/BAAA//+/JgAM" + return "eJzsms+P27oRx+/7Vwz21AIbt736UCDJBsiiTbJYv+AdikKhxZHELkUSJLWu//uClCzb+m2blouH1SmxvTMfDodDzlf8AK+4XUKMKrsDsMxyXML9Z1TZ/R0ARRNrpiyTYgl/vwMAcF9BLmnB8Q7AZFLbKJYiYekSEsKN+1QjR2JwCSlxv0FrmUjNEv51bwy/f4D7zFp1/+87gIQhp2bpLX8AQXKsWdxjt8pZ0bJQ1ScdRO755f7oF8RSWMKEAZsh5Gg1i92/iYUNagQTa6KQQqJlDp+/PH9dVAYOMY5QeGEs6ogy81p/2YU1gOaeHjvHcdo9TZhDIPJGGCdrjov11qI5+s2Oi0uRNr4YQHPPx51V8FZBJj6AFXXj54nUObFLaAPsIK20hAcF/M1ZDANXGKRB2X4apOejNTMtQ8JtFiLXWpZOzzb5hppwHhlLbNEdr1fcbqSmp4XsR2kXSrtDYatTiuUYZxi/mgUqGWeB5u4bUfCG2jApxtxqWQi6eCO8wEDOa+PgbU8DCD0Vq6MpOAZp5mbLd3dudufZhIxt2R/Kza5pO38u/K5WxWIwITRJEhYvNBIanVRF+ivTOFwZH3BOwWZaFmmmCgsKNRiMZV/uVKwbzSzODuu9nkHrIytVpFxCYBxqfg9DyKQypwXvajxllKYA5cwoTmJc+N01MMbOOKgURJGve6pwzSDX/8HYhtpBWxSV+UkomlgmO0FMTDjSKOGS2J78VqhjFM1vT8VtI+woKaaaUKRXmbSd8ZFJqxmuM2k1xYRJq1FuN2k1bv+kqXRBiSWzV0yVgvM7cHZV6cI3ALOjkeMGoQ/OZ/nscHbfHPSBuUP/7FxF3Rj0YPkDD/ojHUbuw0BL8zn1J2s8/N0wQiyL1oI61/sqkxsDmdxATsQWVGqAaAQmKiiZjA69xRf4sFdNkBk/70lDF0nBu0v3WkqOpEk14vzJOKPQMnroUSDRV/HqDI+4LvJIGhpql2gkg7PsMmGs3duRFOraMC41ZQlUp+fP50EoJm4B9fR9EEpjTpRCGql0brKXL98+Pj9/eezlC9mxe1vNM8bcct0a7btg9y7YzSbYnZBv75Ldu2Tn2uNUu00qUqiTS3Lzx+rR9Vt+KYh4/05FJqVk1k6Ac5KWNSMwbX4c3FP3tuNibhaxzHNmI04singb5edVkM/eClRW3PEl765UpVOiFN9e6vOjMzLZZWOcItQ4xfRxnumzNU7RrrR1LktZ1rVxCXoso6UsK1noRPaIvZ3chNX+7Mj6+yFn/szF4i33rJaYMxQ2YjLSxHazl+rOYPp4I/D0lx/grDTzoStshwx7Rb1DZh0cpnuq0C7hk9/XvczbI6geOj2QxgN4ddYsiimOh1Xu6Y5f3EClQq9miV4RuT3mAL5/99r1sPODNiCyGvHSletsABPl0ez4CDDjxtNxPFo9gtSwLuJXtCAkxb61dkl1mOS3t3o4N1f17Kw9AOMcU8L9/4CJmBcU3a8fIJPGPoCW0gLaeNELeW6JmwzZWwczxqnGbp3prCh9Kj3vDANnLgQGFXE10rUh4DZv0h2MWBcmizbI0qxbneuSySeExAfi88vP1VfoML5X65UNdWr/7jy2De5c4X+Z6XkzcZbc5d2VRuFPf/uwRfMAf/0g5J+7t1XNcqK3EUkSJpjdhgr17z627ojhyj0Tlb7vbx9VPt10dDJpHJj20+P/UplzMNJ0Ny7X61P86mcWTCYLTmGNUCi3LqncdDdxFN9YjFHMiQkI9OitgrdaY3H2ivD18fEBVqvH/rqUEJu1FIILWJ6JdqclvxJ9J8dMR7Vq7aP+BHipvNVl5Ka75xPddbMObYZt8zvJcdRl1ebMKeC1WKYIZCVn2Le7373A6qtF9Vp3WrRmEOsuiJGne12HRHtdd3LVvasUzMrJKuHJV6eO7Vc9rdtlJi7hfXarVnd3eW5Xb+p3Efr2o/sdRis4cNnq/loGY8TrfujBMmI/8n98GnNfau7BXJeC+7jbKyyBcaecGBsVirojb6dn2pYdxsbLcoRNhgI2xECX7evvHH3DNVZqLJWqBZdp0Hr4T5nuy2ETYFol3MPlzMRB6b4xE4fDM8YGpVutfgsHd/V3ZpcCji66M5KPGNtaa4cSDx1/2XD67WBUmb+84E6sUBiS9ilAs5xV/TUKd2pnc5Sb2luvqnOdVqn2O6DpZJQ+gDF0QMyp18jU4F9w7apOkfI+2JvkRU/I6kPqnFRl4g5QndlohGDbX+8b4FNpJIo80CoyzUtiwGXsFTF/u4SZXpXCT17fYfVat0cbs1jZIGm7QQ8hdNcVr1a7R+vnbKWvoabOVgRvI3J3eB4oiNLQW4jcPZCzadzr/zuNu969vPG5NG4xr8bNTOnRWMZ5ad2lgZD2doq364o2E1TvvuIeWPX292NuoHzvL0n5knB75dsFgjbUb4/m1e/xg1xo9dvrZR0KeLuUtTbY6yvg82+s7/r3u/79x9C//xcAAP//xbT1Yg==" } diff --git a/metricbeat/module/ceph/mgr/metricset.go b/metricbeat/module/ceph/mgr/metricset.go new file mode 100644 index 000000000000..a982cf706b2c --- /dev/null +++ b/metricbeat/module/ceph/mgr/metricset.go @@ -0,0 +1,52 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mgr + +import ( + "fmt" + + "github.com/elastic/beats/metricbeat/helper" + "github.com/elastic/beats/metricbeat/mb" +) + +type MetricSet struct { + mb.BaseMetricSet + *helper.HTTP +} + +var _ mb.MetricSet = new(MetricSet) + +// NewMetricSet creates an metric set that can be used to build other metricsets that query Ceph mgr API. +func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { + http, err := helper.NewHTTP(base) + if err != nil { + return nil, err + } + http.SetMethod("POST") + http.SetHeader("Content-Type", "application/json") + http.SetHeader("Accept", "application/json") + return &MetricSet{ + base, + http, + }, nil +} + +func (m *MetricSet) WithPrefix(prefix string) *MetricSet { + m.HTTP.SetBody([]byte(fmt.Sprintf(`{"prefix": "%s", "format": "json"}`, prefix))) + return m +} diff --git a/metricbeat/module/ceph/mgr_cluster_disk/_meta/data.json b/metricbeat/module/ceph/mgr_cluster_disk/_meta/data.json index 94c79adb4b49..12aa085f8e15 100644 --- a/metricbeat/module/ceph/mgr_cluster_disk/_meta/data.json +++ b/metricbeat/module/ceph/mgr_cluster_disk/_meta/data.json @@ -5,7 +5,7 @@ "name": "host.example.com" }, "ceph": { - "mgr_cluster_disk": { + "cluster_disk": { "available": { "bytes": 0 }, @@ -18,7 +18,7 @@ } }, "event": { - "dataset": "ceph.mgr_cluster_disk", + "dataset": "ceph.cluster_disk", "duration": 115000, "module": "ceph" }, diff --git a/metricbeat/module/ceph/mgr_cluster_disk/_meta/fields.yml b/metricbeat/module/ceph/mgr_cluster_disk/_meta/fields.yml index d24637a1fea0..fe755709362a 100644 --- a/metricbeat/module/ceph/mgr_cluster_disk/_meta/fields.yml +++ b/metricbeat/module/ceph/mgr_cluster_disk/_meta/fields.yml @@ -1,7 +1,7 @@ -- name: mgr_cluster_disk +- name: cluster_disk type: group description: > - mgr_cluster_disk + cluster_disk release: beta fields: - name: available.bytes diff --git a/metricbeat/module/ceph/_meta/testdata/mgr_df_sample_response.json b/metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/mgr_df_sample_response.json similarity index 100% rename from metricbeat/module/ceph/_meta/testdata/mgr_df_sample_response.json rename to metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/mgr_df_sample_response.json diff --git a/metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/mgr_df_sample_response.json-expected.json b/metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/mgr_df_sample_response.json-expected.json new file mode 100644 index 000000000000..0967ef424bce --- /dev/null +++ b/metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/mgr_df_sample_response.json-expected.json @@ -0,0 +1 @@ +{} diff --git a/metricbeat/module/ceph/_meta/testdata/mgr_failed_response.json b/metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/mgr_failed_response.json similarity index 100% rename from metricbeat/module/ceph/_meta/testdata/mgr_failed_response.json rename to metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/mgr_failed_response.json diff --git a/metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/mgr_failed_response.json-expected.json b/metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/mgr_failed_response.json-expected.json new file mode 100644 index 000000000000..0967ef424bce --- /dev/null +++ b/metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/mgr_failed_response.json-expected.json @@ -0,0 +1 @@ +{} diff --git a/metricbeat/module/ceph/mgr_cluster_disk/mgr_cluster_disk.go b/metricbeat/module/ceph/mgr_cluster_disk/mgr_cluster_disk.go index c5e441413abd..be2e0caa01aa 100644 --- a/metricbeat/module/ceph/mgr_cluster_disk/mgr_cluster_disk.go +++ b/metricbeat/module/ceph/mgr_cluster_disk/mgr_cluster_disk.go @@ -18,11 +18,9 @@ package mgr_cluster_disk import ( - "fmt" - - "github.com/elastic/beats/metricbeat/helper" "github.com/elastic/beats/metricbeat/mb" "github.com/elastic/beats/metricbeat/mb/parse" + "github.com/elastic/beats/metricbeat/module/ceph/mgr" ) const ( @@ -48,24 +46,16 @@ func init() { } type MetricSet struct { - mb.BaseMetricSet - *helper.HTTP + *mgr.MetricSet } func New(base mb.BaseMetricSet) (mb.MetricSet, error) { - http, err := helper.NewHTTP(base) + metricSet, err := mgr.NewMetricSet(base) if err != nil { return nil, err } - http.SetMethod("POST") - http.SetHeader("Content-Type", "application/json") - http.SetHeader("Accept", "application/json") - http.SetBody([]byte(fmt.Sprintf(`{"prefix": "%s", "format": "json"}`, cephPrefix))) - - return &MetricSet{ - base, - http, - }, nil + metricSet = metricSet.WithPrefix(cephPrefix) + return &MetricSet{metricSet}, nil } // Fetch methods implements the data gathering and data conversion to the right @@ -82,9 +72,6 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { return err } - if reported := reporter.Event(mb.Event{MetricSetFields: event}); !reported { - m.Logger().Debug("error reporting event") - } - + reporter.Event(mb.Event{MetricSetFields: event}) return nil } diff --git a/metricbeat/module/ceph/mgr_cluster_disk/mgr_cluster_disk_test.go b/metricbeat/module/ceph/mgr_cluster_disk/mgr_cluster_disk_test.go index 14097a629f03..e57ab6e3eeca 100644 --- a/metricbeat/module/ceph/mgr_cluster_disk/mgr_cluster_disk_test.go +++ b/metricbeat/module/ceph/mgr_cluster_disk/mgr_cluster_disk_test.go @@ -18,80 +18,11 @@ package mgr_cluster_disk import ( - "io/ioutil" - "net/http" - "net/http/httptest" - "path/filepath" "testing" - "github.com/elastic/beats/libbeat/common" mbtest "github.com/elastic/beats/metricbeat/mb/testing" - - "github.com/stretchr/testify/assert" ) -func TestFetchEventContents(t *testing.T) { - absPath, err := filepath.Abs("../_meta/testdata/") - assert.NoError(t, err) - - response, err := ioutil.ReadFile(absPath + "/mgr_df_sample_response.json") - assert.NoError(t, err) - - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(200) - w.Header().Set("Content-Type", "application/json;") - w.Write(response) - })) - defer server.Close() - - config := map[string]interface{}{ - "module": "ceph", - "metricsets": []string{"mgr_cluster_disk"}, - "hosts": []string{server.URL}, - } - - f := mbtest.NewReportingMetricSetV2Error(t, config) - events, errs := mbtest.ReportingFetchV2Error(f) - if len(errs) > 0 { - t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) - } - assert.NotEmpty(t, events) - event := events[0].MetricSetFields - - t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event.StringToPrint()) - - used := event["used"].(common.MapStr) - assert.EqualValues(t, uint64(3801088), used["bytes"]) - - total := event["total"].(common.MapStr) - assert.EqualValues(t, uint64(10737418240), total["bytes"]) - - available := event["available"].(common.MapStr) - assert.EqualValues(t, uint64(9659875328), available["bytes"]) -} - -func TestFetchEventContents_Failed(t *testing.T) { - absPath, err := filepath.Abs("../_meta/testdata/") - assert.NoError(t, err) - - response, err := ioutil.ReadFile(absPath + "/mgr_failed_response.json") - assert.NoError(t, err) - - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(200) - w.Header().Set("Content-Type", "application/json;") - w.Write([]byte(response)) - })) - defer server.Close() - - config := map[string]interface{}{ - "module": "ceph", - "metricsets": []string{"mgr_cluster_disk"}, - "hosts": []string{server.URL}, - } - - f := mbtest.NewReportingMetricSetV2Error(t, config) - events, errs := mbtest.ReportingFetchV2Error(f) - assert.Empty(t, events) - assert.NotEmpty(t, errs) +func TestData(t *testing.T) { + mbtest.TestDataFiles(t, "ceph", "mgr_cluster_disk") } diff --git a/metricbeat/module/ceph/mgr_cluster_health/_meta/data.json b/metricbeat/module/ceph/mgr_cluster_health/_meta/data.json index 65e4061ded4d..66303e4e7207 100644 --- a/metricbeat/module/ceph/mgr_cluster_health/_meta/data.json +++ b/metricbeat/module/ceph/mgr_cluster_health/_meta/data.json @@ -5,7 +5,7 @@ "name": "host.example.com" }, "ceph": { - "mgr_cluster_health": { + "cluster_health": { "overall_status": "HEALTH_OK", "timechecks": { "epoch": 3, @@ -17,7 +17,7 @@ } }, "event": { - "dataset": "ceph.mgr_cluster_health", + "dataset": "ceph.cluster_health", "duration": 115000, "module": "ceph" }, diff --git a/metricbeat/module/ceph/mgr_cluster_health/_meta/fields.yml b/metricbeat/module/ceph/mgr_cluster_health/_meta/fields.yml index f3030809eb4f..606e8031cb3c 100644 --- a/metricbeat/module/ceph/mgr_cluster_health/_meta/fields.yml +++ b/metricbeat/module/ceph/mgr_cluster_health/_meta/fields.yml @@ -1,7 +1,7 @@ -- name: mgr_cluster_health +- name: cluster_health type: group description: > - mgr_cluster_health + cluster_health release: beta fields: - name: overall_status diff --git a/metricbeat/module/ceph/mgr_cluster_health/_meta/testdata/mgr_failed_response.json b/metricbeat/module/ceph/mgr_cluster_health/_meta/testdata/mgr_failed_response.json new file mode 100644 index 000000000000..409e19fb2170 --- /dev/null +++ b/metricbeat/module/ceph/mgr_cluster_health/_meta/testdata/mgr_failed_response.json @@ -0,0 +1,17 @@ +{ + "failed": [ + { + "command": "dfb format=json-pretty", + "outb": "", + "outs": "command not known" + } + ], + "finished": [], + "has_failed": true, + "id": "139687220237200", + "is_finished": true, + "is_waiting": false, + "running": [], + "state": "failed", + "waiting": [] +} diff --git a/metricbeat/module/ceph/_meta/testdata/mgr_status_sample_response.json b/metricbeat/module/ceph/mgr_cluster_health/_meta/testdata/mgr_status_sample_response.json similarity index 100% rename from metricbeat/module/ceph/_meta/testdata/mgr_status_sample_response.json rename to metricbeat/module/ceph/mgr_cluster_health/_meta/testdata/mgr_status_sample_response.json diff --git a/metricbeat/module/ceph/_meta/testdata/mgr_time_sync_status_sample_response.json b/metricbeat/module/ceph/mgr_cluster_health/_meta/testdata/mgr_time_sync_status_sample_response.json similarity index 100% rename from metricbeat/module/ceph/_meta/testdata/mgr_time_sync_status_sample_response.json rename to metricbeat/module/ceph/mgr_cluster_health/_meta/testdata/mgr_time_sync_status_sample_response.json diff --git a/metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health.go b/metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health.go index fa61a64d1636..7dea91f755cd 100644 --- a/metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health.go +++ b/metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health.go @@ -20,9 +20,9 @@ package mgr_cluster_health import ( "fmt" - "github.com/elastic/beats/metricbeat/helper" "github.com/elastic/beats/metricbeat/mb" "github.com/elastic/beats/metricbeat/mb/parse" + "github.com/elastic/beats/metricbeat/module/ceph/mgr" ) const ( @@ -49,23 +49,15 @@ func init() { } type MetricSet struct { - mb.BaseMetricSet - *helper.HTTP + *mgr.MetricSet } func New(base mb.BaseMetricSet) (mb.MetricSet, error) { - http, err := helper.NewHTTP(base) + metricSet, err := mgr.NewMetricSet(base) if err != nil { return nil, err } - http.SetMethod("POST") - http.SetHeader("Content-Type", "application/json") - http.SetHeader("Accept", "application/json") - - return &MetricSet{ - base, - http, - }, nil + return &MetricSet{metricSet}, nil } // Fetch methods implements the data gathering and data conversion to the right @@ -89,9 +81,6 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { return err } - if reported := reporter.Event(mb.Event{MetricSetFields: event}); !reported { - m.Logger().Debug("error reporting event") - } - + reporter.Event(mb.Event{MetricSetFields: event}) return nil } diff --git a/metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health_integration_test.go b/metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health_integration_test.go index 5361c9ae8812..f1299f1ce287 100644 --- a/metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health_integration_test.go +++ b/metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health_integration_test.go @@ -22,12 +22,11 @@ package mgr_cluster_health import ( "testing" - "github.com/elastic/beats/metricbeat/module/ceph/mgrtest" - "github.com/stretchr/testify/require" "github.com/elastic/beats/libbeat/tests/compose" mbtest "github.com/elastic/beats/metricbeat/mb/testing" + "github.com/elastic/beats/metricbeat/module/ceph/mgrtest" ) const user = "demo" diff --git a/metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health_test.go b/metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health_test.go index 8f08987e5a6a..2e8f53536734 100644 --- a/metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health_test.go +++ b/metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health_test.go @@ -38,7 +38,7 @@ type clientRequest struct { } func TestFetchEventContents(t *testing.T) { - absPath, err := filepath.Abs("../_meta/testdata/") + absPath, err := filepath.Abs("./_meta/testdata/") assert.NoError(t, err) statusResponse, err := ioutil.ReadFile(absPath + "/mgr_status_sample_response.json") @@ -90,7 +90,7 @@ func TestFetchEventContents(t *testing.T) { } func TestFetchEventContents_Failed(t *testing.T) { - absPath, err := filepath.Abs("../_meta/testdata/") + absPath, err := filepath.Abs("./_meta/testdata/") assert.NoError(t, err) response, err := ioutil.ReadFile(absPath + "/mgr_failed_response.json") diff --git a/metricbeat/module/ceph/mgr_osd_disk/mgr_osd_disk_test.go b/metricbeat/module/ceph/mgr_osd_disk/mgr_osd_disk_test.go deleted file mode 100644 index ca1e8ec0ff68..000000000000 --- a/metricbeat/module/ceph/mgr_osd_disk/mgr_osd_disk_test.go +++ /dev/null @@ -1,100 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package mgr_osd_disk - -import ( - "io/ioutil" - "net/http" - "net/http/httptest" - "path/filepath" - "testing" - - "github.com/elastic/beats/libbeat/common" - mbtest "github.com/elastic/beats/metricbeat/mb/testing" - - "github.com/stretchr/testify/assert" -) - -func TestFetchEventContents(t *testing.T) { - absPath, err := filepath.Abs("../_meta/testdata/") - assert.NoError(t, err) - - response, err := ioutil.ReadFile(absPath + "/mgr_df_sample_response.json") - assert.NoError(t, err) - - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(200) - w.Header().Set("Content-Type", "application/json;") - w.Write(response) - })) - defer server.Close() - - config := map[string]interface{}{ - "module": "ceph", - "metricsets": []string{"mgr_osd_disk"}, - "hosts": []string{server.URL}, - } - - f := mbtest.NewReportingMetricSetV2Error(t, config) - events, errs := mbtest.ReportingFetchV2Error(f) - if len(errs) > 0 { - t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) - } - assert.NotEmpty(t, events) - event := events[0].MetricSetFields - - t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event.StringToPrint()) - - assert.EqualValues(t, "rbd", event["name"]) - assert.EqualValues(t, 1, event["id"]) - - stats := event["stats"].(common.MapStr) - - used := stats["used"].(common.MapStr) - assert.EqualValues(t, 0, used["bytes"]) - assert.EqualValues(t, 0, used["kb"]) - - available := stats["available"].(common.MapStr) - assert.EqualValues(t, uint64(9123004416), available["bytes"]) -} - -func TestFetchEventContents_Failed(t *testing.T) { - absPath, err := filepath.Abs("../_meta/testdata/") - assert.NoError(t, err) - - response, err := ioutil.ReadFile(absPath + "/mgr_failed_response.json") - assert.NoError(t, err) - - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(200) - w.Header().Set("Content-Type", "application/json;") - w.Write([]byte(response)) - })) - defer server.Close() - - config := map[string]interface{}{ - "module": "ceph", - "metricsets": []string{"mgr_osd_disk"}, - "hosts": []string{server.URL}, - } - - f := mbtest.NewReportingMetricSetV2Error(t, config) - events, errs := mbtest.ReportingFetchV2Error(f) - assert.Empty(t, events) - assert.NotEmpty(t, errs) -} diff --git a/metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/mgr_failed_response.json b/metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/mgr_failed_response.json new file mode 100644 index 000000000000..409e19fb2170 --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/mgr_failed_response.json @@ -0,0 +1,17 @@ +{ + "failed": [ + { + "command": "dfb format=json-pretty", + "outb": "", + "outs": "command not known" + } + ], + "finished": [], + "has_failed": true, + "id": "139687220237200", + "is_finished": true, + "is_waiting": false, + "running": [], + "state": "failed", + "waiting": [] +} diff --git a/metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/mgr_failed_response.json-expected.json b/metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/mgr_failed_response.json-expected.json new file mode 100644 index 000000000000..0967ef424bce --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/mgr_failed_response.json-expected.json @@ -0,0 +1 @@ +{} diff --git a/metricbeat/module/ceph/_meta/testdata/mgr_osd_perf_sample_response.json b/metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/mgr_osd_perf_sample_response.json similarity index 100% rename from metricbeat/module/ceph/_meta/testdata/mgr_osd_perf_sample_response.json rename to metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/mgr_osd_perf_sample_response.json diff --git a/metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/mgr_osd_perf_sample_response.json-expected.json b/metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/mgr_osd_perf_sample_response.json-expected.json new file mode 100644 index 000000000000..0967ef424bce --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/mgr_osd_perf_sample_response.json-expected.json @@ -0,0 +1 @@ +{} diff --git a/metricbeat/module/ceph/mgr_osd_perf/mgr_osd_perf.go b/metricbeat/module/ceph/mgr_osd_perf/mgr_osd_perf.go index d56703395825..bd8fc20efd75 100644 --- a/metricbeat/module/ceph/mgr_osd_perf/mgr_osd_perf.go +++ b/metricbeat/module/ceph/mgr_osd_perf/mgr_osd_perf.go @@ -18,11 +18,9 @@ package mgr_osd_perf import ( - "fmt" - - "github.com/elastic/beats/metricbeat/helper" "github.com/elastic/beats/metricbeat/mb" "github.com/elastic/beats/metricbeat/mb/parse" + "github.com/elastic/beats/metricbeat/module/ceph/mgr" ) const ( @@ -48,24 +46,16 @@ func init() { } type MetricSet struct { - mb.BaseMetricSet - *helper.HTTP + *mgr.MetricSet } func New(base mb.BaseMetricSet) (mb.MetricSet, error) { - http, err := helper.NewHTTP(base) + metricSet, err := mgr.NewMetricSet(base) if err != nil { return nil, err } - http.SetMethod("POST") - http.SetHeader("Content-Type", "application/json") - http.SetHeader("Accept", "application/json") - http.SetBody([]byte(fmt.Sprintf(`{"prefix": "%s", "format": "json"}`, cephPrefix))) - - return &MetricSet{ - base, - http, - }, nil + metricSet = metricSet.WithPrefix(cephPrefix) + return &MetricSet{metricSet}, nil } // Fetch methods implements the data gathering and data conversion to the right @@ -85,7 +75,6 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { for _, event := range events { reported := reporter.Event(mb.Event{MetricSetFields: event}) if !reported { - m.Logger().Debug("error reporting event") return nil } } diff --git a/metricbeat/module/ceph/mgr_osd_perf/mgr_osd_perf_test.go b/metricbeat/module/ceph/mgr_osd_perf/mgr_osd_perf_test.go index eb03bf9b474d..c5528fb700fc 100644 --- a/metricbeat/module/ceph/mgr_osd_perf/mgr_osd_perf_test.go +++ b/metricbeat/module/ceph/mgr_osd_perf/mgr_osd_perf_test.go @@ -18,80 +18,11 @@ package mgr_osd_perf import ( - "io/ioutil" - "net/http" - "net/http/httptest" - "path/filepath" "testing" - "github.com/elastic/beats/libbeat/common" mbtest "github.com/elastic/beats/metricbeat/mb/testing" - - "github.com/stretchr/testify/assert" ) -func TestFetchEventContents(t *testing.T) { - absPath, err := filepath.Abs("../_meta/testdata/") - assert.NoError(t, err) - - response, err := ioutil.ReadFile(absPath + "/mgr_osd_perf_sample_response.json") - assert.NoError(t, err) - - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(200) - w.Header().Set("Content-Type", "application/json;") - w.Write(response) - })) - defer server.Close() - - config := map[string]interface{}{ - "module": "ceph", - "metricsets": []string{"mgr_osd_perf"}, - "hosts": []string{server.URL}, - } - - f := mbtest.NewReportingMetricSetV2Error(t, config) - events, errs := mbtest.ReportingFetchV2Error(f) - if len(errs) > 0 { - t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) - } - assert.NotEmpty(t, events) - event := events[0].MetricSetFields - - t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event.StringToPrint()) - - assert.EqualValues(t, 1, event["id"]) - - stats := event["stats"].(common.MapStr) - - assert.EqualValues(t, 2, stats["apply_latency_ms"]) - assert.EqualValues(t, 4, stats["apply_latency_ns"]) - assert.EqualValues(t, 1, stats["commit_latency_ms"]) - assert.EqualValues(t, 3, stats["commit_latency_ns"]) -} - -func TestFetchEventContents_Failed(t *testing.T) { - absPath, err := filepath.Abs("../_meta/testdata/") - assert.NoError(t, err) - - response, err := ioutil.ReadFile(absPath + "/mgr_failed_response.json") - assert.NoError(t, err) - - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(200) - w.Header().Set("Content-Type", "application/json;") - w.Write([]byte(response)) - })) - defer server.Close() - - config := map[string]interface{}{ - "module": "ceph", - "metricsets": []string{"mgr_osd_perf"}, - "hosts": []string{server.URL}, - } - - f := mbtest.NewReportingMetricSetV2Error(t, config) - events, errs := mbtest.ReportingFetchV2Error(f) - assert.Empty(t, events) - assert.NotEmpty(t, errs) +func TestData(t *testing.T) { + mbtest.TestDataFiles(t, "ceph", "mgr_osd_perf") } diff --git a/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/mgr_failed_response.json b/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/mgr_failed_response.json new file mode 100644 index 000000000000..409e19fb2170 --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/mgr_failed_response.json @@ -0,0 +1,17 @@ +{ + "failed": [ + { + "command": "dfb format=json-pretty", + "outb": "", + "outs": "command not known" + } + ], + "finished": [], + "has_failed": true, + "id": "139687220237200", + "is_finished": true, + "is_waiting": false, + "running": [], + "state": "failed", + "waiting": [] +} diff --git a/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/mgr_failed_response.json-expected.json b/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/mgr_failed_response.json-expected.json new file mode 100644 index 000000000000..0967ef424bce --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/mgr_failed_response.json-expected.json @@ -0,0 +1 @@ +{} diff --git a/metricbeat/module/ceph/_meta/testdata/mgr_osd_pool_stats_sample_response.json b/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/mgr_osd_pool_stats_sample_response.json similarity index 100% rename from metricbeat/module/ceph/_meta/testdata/mgr_osd_pool_stats_sample_response.json rename to metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/mgr_osd_pool_stats_sample_response.json diff --git a/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/mgr_osd_pool_stats_sample_response.json-expected.json b/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/mgr_osd_pool_stats_sample_response.json-expected.json new file mode 100644 index 000000000000..0967ef424bce --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/mgr_osd_pool_stats_sample_response.json-expected.json @@ -0,0 +1 @@ +{} diff --git a/metricbeat/module/ceph/mgr_osd_pool_stats/mgr_osd_pool_stats.go b/metricbeat/module/ceph/mgr_osd_pool_stats/mgr_osd_pool_stats.go index 2a31e6b71e80..8d864a623e1e 100644 --- a/metricbeat/module/ceph/mgr_osd_pool_stats/mgr_osd_pool_stats.go +++ b/metricbeat/module/ceph/mgr_osd_pool_stats/mgr_osd_pool_stats.go @@ -18,11 +18,9 @@ package mgr_osd_pool_stats import ( - "fmt" - - "github.com/elastic/beats/metricbeat/helper" "github.com/elastic/beats/metricbeat/mb" "github.com/elastic/beats/metricbeat/mb/parse" + "github.com/elastic/beats/metricbeat/module/ceph/mgr" ) const ( @@ -48,24 +46,16 @@ func init() { } type MetricSet struct { - mb.BaseMetricSet - *helper.HTTP + *mgr.MetricSet } func New(base mb.BaseMetricSet) (mb.MetricSet, error) { - http, err := helper.NewHTTP(base) + metricSet, err := mgr.NewMetricSet(base) if err != nil { return nil, err } - http.SetMethod("POST") - http.SetHeader("Content-Type", "application/json") - http.SetHeader("Accept", "application/json") - http.SetBody([]byte(fmt.Sprintf(`{"prefix": "%s", "format": "json"}`, cephPrefix))) - - return &MetricSet{ - base, - http, - }, nil + metricSet = metricSet.WithPrefix(cephPrefix) + return &MetricSet{metricSet}, nil } // Fetch methods implements the data gathering and data conversion to the right @@ -85,7 +75,6 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { for _, event := range events { reported := reporter.Event(mb.Event{MetricSetFields: event}) if !reported { - m.Logger().Debug("error reporting event") return nil } } diff --git a/metricbeat/module/ceph/mgr_osd_pool_stats/mgr_osd_pool_stats_test.go b/metricbeat/module/ceph/mgr_osd_pool_stats/mgr_osd_pool_stats_test.go index 7204ea70e72b..c3cd88fb53e4 100644 --- a/metricbeat/module/ceph/mgr_osd_pool_stats/mgr_osd_pool_stats_test.go +++ b/metricbeat/module/ceph/mgr_osd_pool_stats/mgr_osd_pool_stats_test.go @@ -18,81 +18,11 @@ package mgr_osd_pool_stats import ( - "io/ioutil" - "net/http" - "net/http/httptest" - "path/filepath" "testing" - "github.com/elastic/beats/libbeat/common" mbtest "github.com/elastic/beats/metricbeat/mb/testing" - - "github.com/stretchr/testify/assert" ) -func TestFetchEventContents(t *testing.T) { - absPath, err := filepath.Abs("../_meta/testdata/") - assert.NoError(t, err) - - response, err := ioutil.ReadFile(absPath + "/mgr_osd_pool_stats_sample_response.json") - assert.NoError(t, err) - - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(200) - w.Header().Set("Content-Type", "application/json;") - w.Write(response) - })) - defer server.Close() - - config := map[string]interface{}{ - "module": "ceph", - "metricsets": []string{"mgr_osd_pool_stats"}, - "hosts": []string{server.URL}, - } - - f := mbtest.NewReportingMetricSetV2Error(t, config) - events, errs := mbtest.ReportingFetchV2Error(f) - if len(errs) > 0 { - t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) - } - assert.NotEmpty(t, events) - event := events[0].MetricSetFields - - t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event.StringToPrint()) - - assert.EqualValues(t, 1, event["pool_id"]) - assert.EqualValues(t, "rbd", event["pool_name"]) - - stats := event["client_io_rate"].(common.MapStr) - - assert.EqualValues(t, 1, stats["read_bytes_sec"]) - assert.EqualValues(t, 2, stats["read_op_per_sec"]) - assert.EqualValues(t, 3, stats["write_bytes_sec"]) - assert.EqualValues(t, 4, stats["write_op_per_sec"]) -} - -func TestFetchEventContents_Failed(t *testing.T) { - absPath, err := filepath.Abs("../_meta/testdata/") - assert.NoError(t, err) - - response, err := ioutil.ReadFile(absPath + "/mgr_failed_response.json") - assert.NoError(t, err) - - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(200) - w.Header().Set("Content-Type", "application/json;") - w.Write([]byte(response)) - })) - defer server.Close() - - config := map[string]interface{}{ - "module": "ceph", - "metricsets": []string{"mgr_osd_pool_stats"}, - "hosts": []string{server.URL}, - } - - f := mbtest.NewReportingMetricSetV2Error(t, config) - events, errs := mbtest.ReportingFetchV2Error(f) - assert.Empty(t, events) - assert.NotEmpty(t, errs) +func TestData(t *testing.T) { + mbtest.TestDataFiles(t, "ceph", "mgr_osd_pool_stats") } diff --git a/metricbeat/module/ceph/mgr_osd_tree/_meta/data.json b/metricbeat/module/ceph/mgr_osd_tree/_meta/data.json index 9239f288557b..4a2cb3e260df 100644 --- a/metricbeat/module/ceph/mgr_osd_tree/_meta/data.json +++ b/metricbeat/module/ceph/mgr_osd_tree/_meta/data.json @@ -1,7 +1,7 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", "ceph": { - "mgr_osd_tree": { + "osd_tree": { "children": [ "-2" ], @@ -13,7 +13,7 @@ } }, "event": { - "dataset": "ceph.mgr_osd_tree", + "dataset": "ceph.osd_tree", "duration": 115000, "module": "ceph" }, diff --git a/metricbeat/module/ceph/mgr_osd_tree/_meta/fields.yml b/metricbeat/module/ceph/mgr_osd_tree/_meta/fields.yml index 93823ed2455f..d924267ee7f7 100644 --- a/metricbeat/module/ceph/mgr_osd_tree/_meta/fields.yml +++ b/metricbeat/module/ceph/mgr_osd_tree/_meta/fields.yml @@ -1,4 +1,4 @@ -- name: mgr_osd_tree +- name: osd_tree type: group description: > OSD tree information diff --git a/metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/mgr_failed_response.json b/metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/mgr_failed_response.json new file mode 100644 index 000000000000..409e19fb2170 --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/mgr_failed_response.json @@ -0,0 +1,17 @@ +{ + "failed": [ + { + "command": "dfb format=json-pretty", + "outb": "", + "outs": "command not known" + } + ], + "finished": [], + "has_failed": true, + "id": "139687220237200", + "is_finished": true, + "is_waiting": false, + "running": [], + "state": "failed", + "waiting": [] +} diff --git a/metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/mgr_failed_response.json-expected.json b/metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/mgr_failed_response.json-expected.json new file mode 100644 index 000000000000..0967ef424bce --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/mgr_failed_response.json-expected.json @@ -0,0 +1 @@ +{} diff --git a/metricbeat/module/ceph/_meta/testdata/mgr_osd_tree_sample_response.json b/metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/mgr_osd_tree_sample_response.json similarity index 100% rename from metricbeat/module/ceph/_meta/testdata/mgr_osd_tree_sample_response.json rename to metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/mgr_osd_tree_sample_response.json diff --git a/metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/mgr_osd_tree_sample_response.json-expected.json b/metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/mgr_osd_tree_sample_response.json-expected.json new file mode 100644 index 000000000000..0967ef424bce --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/mgr_osd_tree_sample_response.json-expected.json @@ -0,0 +1 @@ +{} diff --git a/metricbeat/module/ceph/mgr_osd_tree/mgr_osd_tree.go b/metricbeat/module/ceph/mgr_osd_tree/mgr_osd_tree.go index bc3d1c1df457..dc1815b09af2 100644 --- a/metricbeat/module/ceph/mgr_osd_tree/mgr_osd_tree.go +++ b/metricbeat/module/ceph/mgr_osd_tree/mgr_osd_tree.go @@ -18,11 +18,9 @@ package mgr_osd_tree import ( - "fmt" - - "github.com/elastic/beats/metricbeat/helper" "github.com/elastic/beats/metricbeat/mb" "github.com/elastic/beats/metricbeat/mb/parse" + "github.com/elastic/beats/metricbeat/module/ceph/mgr" ) const ( @@ -48,24 +46,16 @@ func init() { } type MetricSet struct { - mb.BaseMetricSet - *helper.HTTP + *mgr.MetricSet } func New(base mb.BaseMetricSet) (mb.MetricSet, error) { - http, err := helper.NewHTTP(base) + metricSet, err := mgr.NewMetricSet(base) if err != nil { return nil, err } - http.SetMethod("POST") - http.SetHeader("Content-Type", "application/json") - http.SetHeader("Accept", "application/json") - http.SetBody([]byte(fmt.Sprintf(`{"prefix": "%s", "format": "json"}`, cephPrefix))) - - return &MetricSet{ - base, - http, - }, nil + metricSet = metricSet.WithPrefix(cephPrefix) + return &MetricSet{metricSet}, nil } // Fetch methods implements the data gathering and data conversion to the right @@ -85,7 +75,6 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { for _, event := range events { reported := reporter.Event(mb.Event{MetricSetFields: event}) if !reported { - m.Logger().Debug("error reporting event") return nil } } diff --git a/metricbeat/module/ceph/mgr_osd_tree/mgr_osd_tree_test.go b/metricbeat/module/ceph/mgr_osd_tree/mgr_osd_tree_test.go index f52e486c80d7..15fdccf27e7c 100644 --- a/metricbeat/module/ceph/mgr_osd_tree/mgr_osd_tree_test.go +++ b/metricbeat/module/ceph/mgr_osd_tree/mgr_osd_tree_test.go @@ -18,117 +18,11 @@ package mgr_osd_tree import ( - "io/ioutil" - "net/http" - "net/http/httptest" - "path/filepath" "testing" mbtest "github.com/elastic/beats/metricbeat/mb/testing" - - "github.com/stretchr/testify/assert" ) -func TestFetchEventContents(t *testing.T) { - absPath, err := filepath.Abs("../_meta/testdata/") - assert.NoError(t, err) - - response, err := ioutil.ReadFile(absPath + "/mgr_osd_tree_sample_response.json") - assert.NoError(t, err) - - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(200) - w.Header().Set("Content-Type", "application/json;") - w.Write(response) - })) - defer server.Close() - - config := map[string]interface{}{ - "module": "ceph", - "metricsets": []string{"mgr_osd_tree"}, - "hosts": []string{server.URL}, - } - - f := mbtest.NewReportingMetricSetV2Error(t, config) - events, errs := mbtest.ReportingFetchV2Error(f) - if len(errs) > 0 { - t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) - } - - assert.NotEmpty(t, events) - nodeInfo := events[0].MetricSetFields - - t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), nodeInfo.StringToPrint()) - - //check root bucket info - assert.EqualValues(t, "default", nodeInfo["name"]) - assert.EqualValues(t, "root", nodeInfo["type"]) - assert.EqualValues(t, []string{"-3"}, nodeInfo["children"]) - assert.EqualValues(t, -1, nodeInfo["id"]) - assert.EqualValues(t, 10, nodeInfo["type_id"]) - assert.EqualValues(t, "", nodeInfo["father"]) - - //check host bucket info - nodeInfo = events[1].MetricSetFields - assert.EqualValues(t, "ceph-mon1", nodeInfo["name"]) - assert.EqualValues(t, "host", nodeInfo["type"]) - assert.EqualValues(t, []string{"1", "0"}, nodeInfo["children"]) - assert.EqualValues(t, -3, nodeInfo["id"]) - assert.EqualValues(t, 1, nodeInfo["type_id"]) - assert.EqualValues(t, "default", nodeInfo["father"]) - - //check osd bucket info - nodeInfo = events[2].MetricSetFields - assert.EqualValues(t, "up", nodeInfo["status"]) - assert.EqualValues(t, "osd.0", nodeInfo["name"]) - assert.EqualValues(t, "osd", nodeInfo["type"]) - assert.EqualValues(t, 1, nodeInfo["primary_affinity"]) - assert.EqualValues(t, true, nodeInfo["exists"]) - assert.EqualValues(t, 0, nodeInfo["id"]) - assert.EqualValues(t, 0, nodeInfo["type_id"]) - assert.EqualValues(t, 0.048691, nodeInfo["crush_weight"]) - assert.EqualValues(t, "hdd", nodeInfo["device_class"]) - assert.EqualValues(t, 1, nodeInfo["reweight"]) - assert.EqualValues(t, "ceph-mon1", nodeInfo["father"]) - assert.EqualValues(t, 2, nodeInfo["depth"]) - - nodeInfo = events[3].MetricSetFields - assert.EqualValues(t, "up", nodeInfo["status"]) - assert.EqualValues(t, "osd.1", nodeInfo["name"]) - assert.EqualValues(t, "osd", nodeInfo["type"]) - assert.EqualValues(t, 1, nodeInfo["primary_affinity"]) - assert.EqualValues(t, true, nodeInfo["exists"]) - assert.EqualValues(t, 1, nodeInfo["id"]) - assert.EqualValues(t, 0, nodeInfo["type_id"]) - assert.EqualValues(t, 0.048691, nodeInfo["crush_weight"]) - assert.EqualValues(t, "hdd", nodeInfo["device_class"]) - assert.EqualValues(t, 1, nodeInfo["reweight"]) - assert.EqualValues(t, "ceph-mon1", nodeInfo["father"]) - assert.EqualValues(t, 2, nodeInfo["depth"]) -} - -func TestFetchEventContents_Failed(t *testing.T) { - absPath, err := filepath.Abs("../_meta/testdata/") - assert.NoError(t, err) - - response, err := ioutil.ReadFile(absPath + "/mgr_failed_response.json") - assert.NoError(t, err) - - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(200) - w.Header().Set("Content-Type", "application/json;") - w.Write([]byte(response)) - })) - defer server.Close() - - config := map[string]interface{}{ - "module": "ceph", - "metricsets": []string{"mgr_osd_tree"}, - "hosts": []string{server.URL}, - } - - f := mbtest.NewReportingMetricSetV2Error(t, config) - events, errs := mbtest.ReportingFetchV2Error(f) - assert.Empty(t, events) - assert.NotEmpty(t, errs) +func TestData(t *testing.T) { + mbtest.TestDataFiles(t, "ceph", "mgr_osd_tree") } diff --git a/metricbeat/module/ceph/mgr_osd_disk/_meta/data.json b/metricbeat/module/ceph/mgr_pool_disk/_meta/data.json similarity index 85% rename from metricbeat/module/ceph/mgr_osd_disk/_meta/data.json rename to metricbeat/module/ceph/mgr_pool_disk/_meta/data.json index f47e93b22d5a..730fc6ac899e 100644 --- a/metricbeat/module/ceph/mgr_osd_disk/_meta/data.json +++ b/metricbeat/module/ceph/mgr_pool_disk/_meta/data.json @@ -1,7 +1,7 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", "ceph": { - "mgr_osd_disk": { + "pool_disk": { "id": 0, "name": "rbd", "stats": { @@ -17,12 +17,12 @@ } }, "event": { - "dataset": "ceph.mgr_osd_disk", + "dataset": "ceph.pool_disk", "duration": 115000, "module": "ceph" }, "metricset": { - "name": "mgr_osd_disk" + "name": "mgr_pool_disk" }, "service": { "address": "127.0.0.1:8003", diff --git a/metricbeat/module/ceph/mgr_osd_disk/_meta/docs.asciidoc b/metricbeat/module/ceph/mgr_pool_disk/_meta/docs.asciidoc similarity index 100% rename from metricbeat/module/ceph/mgr_osd_disk/_meta/docs.asciidoc rename to metricbeat/module/ceph/mgr_pool_disk/_meta/docs.asciidoc diff --git a/metricbeat/module/ceph/mgr_osd_disk/_meta/fields.yml b/metricbeat/module/ceph/mgr_pool_disk/_meta/fields.yml similarity index 94% rename from metricbeat/module/ceph/mgr_osd_disk/_meta/fields.yml rename to metricbeat/module/ceph/mgr_pool_disk/_meta/fields.yml index d5a02fd9ae6e..e5215b688a57 100644 --- a/metricbeat/module/ceph/mgr_osd_disk/_meta/fields.yml +++ b/metricbeat/module/ceph/mgr_pool_disk/_meta/fields.yml @@ -1,7 +1,7 @@ -- name: mgr_osd_disk +- name: pool_disk type: group description: > - mgr_osd_disk + pool_disk release: beta fields: - name: id diff --git a/metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/mgr_df_sample_response.json b/metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/mgr_df_sample_response.json new file mode 100644 index 000000000000..68f483401280 --- /dev/null +++ b/metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/mgr_df_sample_response.json @@ -0,0 +1,17 @@ +{ + "failed": [], + "finished": [ + { + "command": "df format=json-pretty", + "outb": "{\n \"stats\": {\n \"total_bytes\": 10737418240,\n \"total_avail_bytes\": 9659875328,\n \"total_used_bytes\": 3801088,\n \"total_used_raw_bytes\": 1077542912,\n \"total_used_raw_ratio\": 0.10035400092601776,\n \"num_osds\": 1,\n \"num_per_pool_osds\": 1\n },\n \"stats_by_class\": {},\n \"pools\": [\n {\n \"name\": \"rbd\",\n \"id\": 1,\n \"stats\": {\n \"stored\": 0,\n \"objects\": 0,\n \"kb_used\": 0,\n \"bytes_used\": 0,\n \"percent_used\": 0,\n \"max_avail\": 9123004416\n }\n },\n {\n \"name\": \"cephfs_data\",\n \"id\": 2,\n \"stats\": {\n \"stored\": 0,\n \"objects\": 0,\n \"kb_used\": 0,\n \"bytes_used\": 0,\n \"percent_used\": 0,\n \"max_avail\": 9123004416\n }\n },\n {\n \"name\": \"cephfs_metadata\",\n \"id\": 3,\n \"stats\": {\n \"stored\": 2286,\n \"objects\": 22,\n \"kb_used\": 512,\n \"bytes_used\": 524288,\n \"percent_used\": 5.7465484132990241e-05,\n \"max_avail\": 9123004416\n }\n },\n {\n \"name\": \".rgw.root\",\n \"id\": 4,\n \"stats\": {\n \"stored\": 2398,\n \"objects\": 6,\n \"kb_used\": 384,\n \"bytes_used\": 393216,\n \"percent_used\": 4.3099731556139886e-05,\n \"max_avail\": 9123004416\n }\n },\n {\n \"name\": \"default.rgw.control\",\n \"id\": 5,\n \"stats\": {\n \"stored\": 0,\n \"objects\": 8,\n \"kb_used\": 0,\n \"bytes_used\": 0,\n \"percent_used\": 0,\n \"max_avail\": 9123004416\n }\n },\n {\n \"name\": \"default.rgw.meta\",\n \"id\": 6,\n \"stats\": {\n \"stored\": 736,\n \"objects\": 5,\n \"kb_used\": 256,\n \"bytes_used\": 262144,\n \"percent_used\": 2.873356788768433e-05,\n \"max_avail\": 9123004416\n }\n },\n {\n \"name\": \"default.rgw.log\",\n \"id\": 7,\n \"stats\": {\n \"stored\": 0,\n \"objects\": 176,\n \"kb_used\": 0,\n \"bytes_used\": 0,\n \"percent_used\": 0,\n \"max_avail\": 9123004416\n }\n },\n {\n \"name\": \"default.rgw.buckets.index\",\n \"id\": 8,\n \"stats\": {\n \"stored\": 0,\n \"objects\": 1,\n \"kb_used\": 0,\n \"bytes_used\": 0,\n \"percent_used\": 0,\n \"max_avail\": 9123004416\n }\n }\n ]\n}\n", + "outs": "" + } + ], + "has_failed": false, + "id": "139687220240336", + "is_finished": true, + "is_waiting": false, + "running": [], + "state": "success", + "waiting": [] +} diff --git a/metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/mgr_df_sample_response.json-expected.json b/metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/mgr_df_sample_response.json-expected.json new file mode 100644 index 000000000000..0967ef424bce --- /dev/null +++ b/metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/mgr_df_sample_response.json-expected.json @@ -0,0 +1 @@ +{} diff --git a/metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/mgr_failed_response.json b/metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/mgr_failed_response.json new file mode 100644 index 000000000000..409e19fb2170 --- /dev/null +++ b/metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/mgr_failed_response.json @@ -0,0 +1,17 @@ +{ + "failed": [ + { + "command": "dfb format=json-pretty", + "outb": "", + "outs": "command not known" + } + ], + "finished": [], + "has_failed": true, + "id": "139687220237200", + "is_finished": true, + "is_waiting": false, + "running": [], + "state": "failed", + "waiting": [] +} diff --git a/metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/mgr_failed_response.json-expected.json b/metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/mgr_failed_response.json-expected.json new file mode 100644 index 000000000000..0967ef424bce --- /dev/null +++ b/metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/mgr_failed_response.json-expected.json @@ -0,0 +1 @@ +{} diff --git a/metricbeat/module/ceph/mgr_osd_disk/data.go b/metricbeat/module/ceph/mgr_pool_disk/data.go similarity index 98% rename from metricbeat/module/ceph/mgr_osd_disk/data.go rename to metricbeat/module/ceph/mgr_pool_disk/data.go index b0fdb299a639..437c164fec90 100644 --- a/metricbeat/module/ceph/mgr_osd_disk/data.go +++ b/metricbeat/module/ceph/mgr_pool_disk/data.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package mgr_osd_disk +package mgr_pool_disk import ( "github.com/pkg/errors" diff --git a/metricbeat/module/ceph/mgr_osd_disk/mgr_osd_disk.go b/metricbeat/module/ceph/mgr_pool_disk/mgr_pool_disk.go similarity index 79% rename from metricbeat/module/ceph/mgr_osd_disk/mgr_osd_disk.go rename to metricbeat/module/ceph/mgr_pool_disk/mgr_pool_disk.go index b6f004cfba70..e0c5cda9a55e 100644 --- a/metricbeat/module/ceph/mgr_osd_disk/mgr_osd_disk.go +++ b/metricbeat/module/ceph/mgr_pool_disk/mgr_pool_disk.go @@ -15,14 +15,12 @@ // specific language governing permissions and limitations // under the License. -package mgr_osd_disk +package mgr_pool_disk import ( - "fmt" - - "github.com/elastic/beats/metricbeat/helper" "github.com/elastic/beats/metricbeat/mb" "github.com/elastic/beats/metricbeat/mb/parse" + "github.com/elastic/beats/metricbeat/module/ceph/mgr" ) const ( @@ -42,30 +40,22 @@ var ( ) func init() { - mb.Registry.MustAddMetricSet("ceph", "mgr_osd_disk", New, + mb.Registry.MustAddMetricSet("ceph", "mgr_pool_disk", New, mb.WithHostParser(hostParser), ) } type MetricSet struct { - mb.BaseMetricSet - *helper.HTTP + *mgr.MetricSet } func New(base mb.BaseMetricSet) (mb.MetricSet, error) { - http, err := helper.NewHTTP(base) + metricSet, err := mgr.NewMetricSet(base) if err != nil { return nil, err } - http.SetMethod("POST") - http.SetHeader("Content-Type", "application/json") - http.SetHeader("Accept", "application/json") - http.SetBody([]byte(fmt.Sprintf(`{"prefix": "%s", "format": "json"}`, cephPrefix))) - - return &MetricSet{ - base, - http, - }, nil + metricSet = metricSet.WithPrefix(cephPrefix) + return &MetricSet{metricSet}, nil } // Fetch methods implements the data gathering and data conversion to the right @@ -85,7 +75,6 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { for _, event := range events { reported := reporter.Event(mb.Event{MetricSetFields: event}) if !reported { - m.Logger().Debug("error reporting event") return nil } } diff --git a/metricbeat/module/ceph/mgr_osd_disk/mgr_osd_disk_integration_test.go b/metricbeat/module/ceph/mgr_pool_disk/mgr_pool_disk_integration_test.go similarity index 95% rename from metricbeat/module/ceph/mgr_osd_disk/mgr_osd_disk_integration_test.go rename to metricbeat/module/ceph/mgr_pool_disk/mgr_pool_disk_integration_test.go index 42982f131694..11e40b6df79a 100644 --- a/metricbeat/module/ceph/mgr_osd_disk/mgr_osd_disk_integration_test.go +++ b/metricbeat/module/ceph/mgr_pool_disk/mgr_pool_disk_integration_test.go @@ -17,7 +17,7 @@ // +build integration,linux -package mgr_osd_disk +package mgr_pool_disk import ( "testing" @@ -43,7 +43,7 @@ func TestData(t *testing.T) { func getConfig(host, password string) map[string]interface{} { return map[string]interface{}{ "module": "ceph", - "metricsets": []string{"mgr_osd_disk"}, + "metricsets": []string{"mgr_pool_disk"}, "hosts": []string{host}, "username": user, "password": password, diff --git a/metricbeat/module/ceph/mgr_pool_disk/mgr_pool_disk_test.go b/metricbeat/module/ceph/mgr_pool_disk/mgr_pool_disk_test.go new file mode 100644 index 000000000000..68b1e8b28a46 --- /dev/null +++ b/metricbeat/module/ceph/mgr_pool_disk/mgr_pool_disk_test.go @@ -0,0 +1,28 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mgr_pool_disk + +import ( + "testing" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" +) + +func TestData(t *testing.T) { + mbtest.TestDataFiles(t, "ceph", "mgr_pool_disk") +} diff --git a/metricbeat/module/ceph/test_ceph.py b/metricbeat/module/ceph/test_ceph.py index 17c34bb5a140..7951353aa9e7 100644 --- a/metricbeat/module/ceph/test_ceph.py +++ b/metricbeat/module/ceph/test_ceph.py @@ -8,6 +8,7 @@ import metricbeat +@metricbeat.parameterized_with_supported_versions class Test(metricbeat.BaseTest): COMPOSE_SERVICES = ['ceph'] @@ -24,4 +25,65 @@ def test_ceph(self, metricset): """ ceph metricsets tests """ + + if not self.old_ceph_version(): + self.skipTest("newer ceph version not supported") + return + self.check_metricset("ceph", metricset, self.get_hosts(), self.FIELDS) + + @parameterized.expand([ + "mgr_cluster_disk", + "mgr_cluster_health", + "mgr_osd_disk", + "mgr_osd_perf", + "mgr_osd_pool_stats", + "mgr_osd_tree" + ]) + @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") + def test_ceph_mgr(self, metricset): + """ + ceph metricsets tests + """ + + if self.old_ceph_version(): + self.skipTest("legacy ceph version not supported") + return + + self.render_config_template(modules=[self.get_ceph_module_config(metricset)]) + proc = self.start_beat(home=self.beat_path) + self.wait_until(lambda: self.output_lines() > 0) + proc.check_kill_and_wait() + self.assert_no_logged_warnings(replace=['SSL/TLS verifications disabled.']) + + output = self.read_output_json() + for evt in output: + assert 'ceph' in evt + self.assert_fields_are_documented(evt) + + def old_ceph_version(self): + if not 'CEPH_NAME' in self.COMPOSE_ENV: + return False + + return self.COMPOSE_ENV['CEPH_NAME'] == 'jewel' + + def get_ceph_module_config(self, metricset): + return { + 'name': 'ceph', + 'metricsets': [metricset], + 'period': '1h', + 'hosts': self.get_ceph_mgr_hosts(), + 'username': 'demo', + 'password': self.get_ceph_mgr_password(), + 'extras': { + 'ssl.verification_mode': 'none' + } + } + + def get_ceph_mgr_hosts(self): + return ['https://' + self.compose_host(port='8003/tcp')] + + def get_ceph_mgr_password(self): + r = requests.get('http://' + self.compose_host(port='5000/tcp') + '/restful-list-keys.json') + keys = r.json() + return keys['demo'] diff --git a/metricbeat/module/ceph/test_ceph_mgr.py b/metricbeat/module/ceph/test_ceph_mgr.py deleted file mode 100644 index f0ddcca27493..000000000000 --- a/metricbeat/module/ceph/test_ceph_mgr.py +++ /dev/null @@ -1,60 +0,0 @@ -import os -import requests -import sys -import time -import unittest -from parameterized import parameterized - -sys.path.append(os.path.join(os.path.dirname(__file__), '../../tests/system')) -import metricbeat - - -class Test(metricbeat.BaseTest): - COMPOSE_SERVICES = ['ceph-mgr'] - FIELDS = ["ceph"] - - def get_ceph_module_config(self, metricset): - return { - 'name': 'ceph', - 'metricsets': [metricset], - 'period': '1h', - 'hosts': self.get_hosts(), - 'username': 'demo', - 'password': self.get_password(), - 'extras': { - 'ssl.verification_mode': 'none' - } - } - - def get_hosts(self): - return ['https://' + self.compose_host(port='8003/tcp')] - - def get_password(self): - r = requests.get('http://' + self.compose_host(port='5000/tcp') + '/restful-list-keys.json') - keys = r.json() - return keys['demo'] - - @parameterized.expand([ - "mgr_cluster_disk", - "mgr_cluster_health", - "mgr_osd_disk", - "mgr_osd_perf", - "mgr_osd_pool_stats", - "mgr_osd_tree" - ]) - @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") - def test_ceph_mgr(self, metricset): - """ - ceph-mgr metricsets tests - """ - - self.render_config_template(modules=[self.get_ceph_module_config(metricset)]) - proc = self.start_beat(home=self.beat_path) - self.wait_until(lambda: self.output_lines() > 0) - proc.check_kill_and_wait() - self.assert_no_logged_warnings(replace=['SSL/TLS verifications disabled.']) - - output = self.read_output_json() - for evt in output: - assert 'ceph' in evt - self.assert_fields_are_documented(evt) diff --git a/metricbeat/modules.d/ceph-mgr.yml.disabled b/metricbeat/modules.d/ceph-mgr.yml.disabled new file mode 100644 index 000000000000..b844c0df2f76 --- /dev/null +++ b/metricbeat/modules.d/ceph-mgr.yml.disabled @@ -0,0 +1,23 @@ +# Module: ceph +# Docs: https://www.elastic.co/guide/en/beats/metricbeat/master/metricbeat-module-ceph.html + +- module: ceph + metricsets: + - mgr_cluster_health + period: 10s + hosts: [ "https://localhost:8003" ] + #username: "user" + #password: "secret" + +- module: ceph + metricsets: + - mgr_cluster_disk + - mgr_cluster_health + - mgr_osd_disk + - mgr_osd_perf + # - mgr_osd_pool_stats + # - mgr_osd_tree + period: 1m + hosts: [ "https://localhost:8003" ] + #username: "user" + #password: "secret" diff --git a/metricbeat/modules.d/ceph.yml.disabled b/metricbeat/modules.d/ceph.yml.disabled index 79d90f7186e7..7e875b274bd9 100644 --- a/metricbeat/modules.d/ceph.yml.disabled +++ b/metricbeat/modules.d/ceph.yml.disabled @@ -19,24 +19,3 @@ # - pool_disk period: 1m hosts: ["localhost:5000"] - -- module: ceph - #metricsets: - # - mgr_cluster_health - period: 10s - hosts: [ "https://localhost:8003" ] - #username: "user" - #password: "secret" - -- module: ceph - #metricsets: - # - mgr_cluster_disk - # - mgr_cluster_health - # - mgr_osd_disk - # - mgr_osd_perf - # - mgr_osd_pool_stats - # - mgr_osd_tree - period: 1m - hosts: [ "https://localhost:8003" ] - #username: "user" - #password: "secret" From cd072920db39c64041757f66dfe7995b68d8679e Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Mon, 17 Feb 2020 15:22:53 +0100 Subject: [PATCH 35/41] Fix: unit tests --- metricbeat/docs/fields.asciidoc | 66 +++++++++---------- metricbeat/module/ceph/fields.go | 2 +- .../ceph/mgr_cluster_disk/_meta/data.json | 4 +- .../ceph/mgr_cluster_disk/_meta/fields.yml | 4 +- .../_meta/testdata/config.yml | 4 ++ .../mgr_df_sample_response.json-expected.json | 4 +- .../mgr_failed_response.json-expected.json | 4 +- .../mgr_cluster_disk/mgr_cluster_disk_test.go | 1 + .../ceph/mgr_cluster_health/_meta/data.json | 4 +- .../ceph/mgr_cluster_health/_meta/fields.yml | 4 +- .../mgr_osd_perf/_meta/testdata/config.yml | 4 ++ .../mgr_failed_response.json-expected.json | 4 +- ...sd_perf_sample_response.json-expected.json | 4 +- .../ceph/mgr_osd_perf/mgr_osd_perf_test.go | 1 + .../_meta/testdata/config.yml | 4 ++ .../mgr_failed_response.json-expected.json | 4 +- ...l_stats_sample_response.json-expected.json | 11 +++- .../mgr_osd_pool_stats_test.go | 1 + .../module/ceph/mgr_osd_tree/_meta/data.json | 4 +- .../module/ceph/mgr_osd_tree/_meta/fields.yml | 2 +- .../mgr_osd_tree/_meta/testdata/config.yml | 4 ++ .../mgr_failed_response.json-expected.json | 4 +- ...sd_tree_sample_response.json-expected.json | 7 +- .../ceph/mgr_osd_tree/mgr_osd_tree_test.go | 1 + .../ceph/mgr_pool_disk/_meta/fields.yml | 4 +- .../mgr_pool_disk/_meta/testdata/config.yml | 4 ++ .../mgr_df_sample_response.json-expected.json | 11 +++- .../mgr_failed_response.json-expected.json | 4 +- .../ceph/mgr_pool_disk/mgr_pool_disk_test.go | 1 + 29 files changed, 119 insertions(+), 57 deletions(-) create mode 100644 metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/config.yml create mode 100644 metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/config.yml create mode 100644 metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/config.yml create mode 100644 metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/config.yml create mode 100644 metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/config.yml diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index b408b4e5c92a..1b91ecb56e47 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -3933,13 +3933,13 @@ type: long -- [float] -=== cluster_disk +=== mgr_cluster_disk -cluster_disk +mgr_cluster_disk -*`ceph.cluster_disk.available.bytes`*:: +*`ceph.mgr_cluster_disk.available.bytes`*:: + -- Available bytes of the cluster @@ -3951,7 +3951,7 @@ format: bytes -- -*`ceph.cluster_disk.total.bytes`*:: +*`ceph.mgr_cluster_disk.total.bytes`*:: + -- Total bytes of the cluster @@ -3963,7 +3963,7 @@ format: bytes -- -*`ceph.cluster_disk.used.bytes`*:: +*`ceph.mgr_cluster_disk.used.bytes`*:: + -- Used bytes of the cluster @@ -3976,13 +3976,13 @@ format: bytes -- [float] -=== cluster_health +=== mgr_cluster_health -cluster_health +mgr_cluster_health -*`ceph.cluster_health.overall_status`*:: +*`ceph.mgr_cluster_health.overall_status`*:: + -- Overall status of the cluster @@ -3992,7 +3992,7 @@ type: keyword -- -*`ceph.cluster_health.timechecks.epoch`*:: +*`ceph.mgr_cluster_health.timechecks.epoch`*:: + -- Map version @@ -4002,7 +4002,7 @@ type: long -- -*`ceph.cluster_health.timechecks.round.value`*:: +*`ceph.mgr_cluster_health.timechecks.round.value`*:: + -- timecheck round @@ -4012,7 +4012,7 @@ type: long -- -*`ceph.cluster_health.timechecks.round.status`*:: +*`ceph.mgr_cluster_health.timechecks.round.status`*:: + -- Status of the round @@ -4109,13 +4109,13 @@ type: object -- [float] -=== osd_tree +=== mgr_osd_tree OSD tree information -*`ceph.osd_tree.id`*:: +*`ceph.mgr_osd_tree.id`*:: + -- OSD or bucket node ID @@ -4125,7 +4125,7 @@ type: long -- -*`ceph.osd_tree.name`*:: +*`ceph.mgr_osd_tree.name`*:: + -- OSD or bucket node name @@ -4135,7 +4135,7 @@ type: keyword -- -*`ceph.osd_tree.type`*:: +*`ceph.mgr_osd_tree.type`*:: + -- OSD or bucket node type, illegal type include OSD, host, root etc. @@ -4145,7 +4145,7 @@ type: keyword -- -*`ceph.osd_tree.type_id`*:: +*`ceph.mgr_osd_tree.type_id`*:: + -- OSD or bucket node type ID @@ -4155,7 +4155,7 @@ type: long -- -*`ceph.osd_tree.children`*:: +*`ceph.mgr_osd_tree.children`*:: + -- Bucket children list, separated by comma. @@ -4165,7 +4165,7 @@ type: keyword -- -*`ceph.osd_tree.crush_weight`*:: +*`ceph.mgr_osd_tree.crush_weight`*:: + -- OSD node CRUSH weight @@ -4175,7 +4175,7 @@ type: float -- -*`ceph.osd_tree.depth`*:: +*`ceph.mgr_osd_tree.depth`*:: + -- Node depth @@ -4185,7 +4185,7 @@ type: long -- -*`ceph.osd_tree.exists`*:: +*`ceph.mgr_osd_tree.exists`*:: + -- Node exists (1-yes, 0-no) @@ -4195,7 +4195,7 @@ type: boolean -- -*`ceph.osd_tree.primary_affinity`*:: +*`ceph.mgr_osd_tree.primary_affinity`*:: + -- Weight of reading data from primary OSD @@ -4205,7 +4205,7 @@ type: float -- -*`ceph.osd_tree.reweight`*:: +*`ceph.mgr_osd_tree.reweight`*:: + -- Reweight of osd @@ -4215,7 +4215,7 @@ type: long -- -*`ceph.osd_tree.status`*:: +*`ceph.mgr_osd_tree.status`*:: + -- Status of OSD, it should be up or down @@ -4225,7 +4225,7 @@ type: keyword -- -*`ceph.osd_tree.device_class`*:: +*`ceph.mgr_osd_tree.device_class`*:: + -- Device class of OSD, like HDD, SSD etc. @@ -4235,7 +4235,7 @@ type: keyword -- -*`ceph.osd_tree.father`*:: +*`ceph.mgr_osd_tree.father`*:: + -- Parent node of this OSD or bucket node @@ -4246,13 +4246,13 @@ type: keyword -- [float] -=== pool_disk +=== mgr_pool_disk -pool_disk +mgr_pool_disk -*`ceph.pool_disk.id`*:: +*`ceph.mgr_pool_disk.id`*:: + -- Id of the pool @@ -4262,7 +4262,7 @@ type: long -- -*`ceph.pool_disk.name`*:: +*`ceph.mgr_pool_disk.name`*:: + -- Name of the pool @@ -4272,7 +4272,7 @@ type: keyword -- -*`ceph.pool_disk.stats.available.bytes`*:: +*`ceph.mgr_pool_disk.stats.available.bytes`*:: + -- Available bytes of the pool @@ -4284,7 +4284,7 @@ format: bytes -- -*`ceph.pool_disk.stats.objects`*:: +*`ceph.mgr_pool_disk.stats.objects`*:: + -- Number of objects of the pool @@ -4294,7 +4294,7 @@ type: long -- -*`ceph.pool_disk.stats.used.bytes`*:: +*`ceph.mgr_pool_disk.stats.used.bytes`*:: + -- Used bytes of the pool @@ -4306,7 +4306,7 @@ format: bytes -- -*`ceph.pool_disk.stats.used.kb`*:: +*`ceph.mgr_pool_disk.stats.used.kb`*:: + -- Used kb of the pool diff --git a/metricbeat/module/ceph/fields.go b/metricbeat/module/ceph/fields.go index a9f3cc856f8c..b280e18c15a4 100644 --- a/metricbeat/module/ceph/fields.go +++ b/metricbeat/module/ceph/fields.go @@ -32,5 +32,5 @@ func init() { // AssetCeph returns asset data. // This is the base64 encoded gzipped contents of ../metricbeat/module/ceph. func AssetCeph() string { - return "eJzsms+P27oRx+/7Vwz21AIbt736UCDJBsiiTbJYv+AdikKhxZHELkUSJLWu//uClCzb+m2blouH1SmxvTMfDodDzlf8AK+4XUKMKrsDsMxyXML9Z1TZ/R0ARRNrpiyTYgl/vwMAcF9BLmnB8Q7AZFLbKJYiYekSEsKN+1QjR2JwCSlxv0FrmUjNEv51bwy/f4D7zFp1/+87gIQhp2bpLX8AQXKsWdxjt8pZ0bJQ1ScdRO755f7oF8RSWMKEAZsh5Gg1i92/iYUNagQTa6KQQqJlDp+/PH9dVAYOMY5QeGEs6ogy81p/2YU1gOaeHjvHcdo9TZhDIPJGGCdrjov11qI5+s2Oi0uRNr4YQHPPx51V8FZBJj6AFXXj54nUObFLaAPsIK20hAcF/M1ZDANXGKRB2X4apOejNTMtQ8JtFiLXWpZOzzb5hppwHhlLbNEdr1fcbqSmp4XsR2kXSrtDYatTiuUYZxi/mgUqGWeB5u4bUfCG2jApxtxqWQi6eCO8wEDOa+PgbU8DCD0Vq6MpOAZp5mbLd3dudufZhIxt2R/Kza5pO38u/K5WxWIwITRJEhYvNBIanVRF+ivTOFwZH3BOwWZaFmmmCgsKNRiMZV/uVKwbzSzODuu9nkHrIytVpFxCYBxqfg9DyKQypwXvajxllKYA5cwoTmJc+N01MMbOOKgURJGve6pwzSDX/8HYhtpBWxSV+UkomlgmO0FMTDjSKOGS2J78VqhjFM1vT8VtI+woKaaaUKRXmbSd8ZFJqxmuM2k1xYRJq1FuN2k1bv+kqXRBiSWzV0yVgvM7cHZV6cI3ALOjkeMGoQ/OZ/nscHbfHPSBuUP/7FxF3Rj0YPkDD/ojHUbuw0BL8zn1J2s8/N0wQiyL1oI61/sqkxsDmdxATsQWVGqAaAQmKiiZjA69xRf4sFdNkBk/70lDF0nBu0v3WkqOpEk14vzJOKPQMnroUSDRV/HqDI+4LvJIGhpql2gkg7PsMmGs3duRFOraMC41ZQlUp+fP50EoJm4B9fR9EEpjTpRCGql0brKXL98+Pj9/eezlC9mxe1vNM8bcct0a7btg9y7YzSbYnZBv75Ldu2Tn2uNUu00qUqiTS3Lzx+rR9Vt+KYh4/05FJqVk1k6Ac5KWNSMwbX4c3FP3tuNibhaxzHNmI04singb5edVkM/eClRW3PEl765UpVOiFN9e6vOjMzLZZWOcItQ4xfRxnumzNU7RrrR1LktZ1rVxCXoso6UsK1noRPaIvZ3chNX+7Mj6+yFn/szF4i33rJaYMxQ2YjLSxHazl+rOYPp4I/D0lx/grDTzoStshwx7Rb1DZh0cpnuq0C7hk9/XvczbI6geOj2QxgN4ddYsiimOh1Xu6Y5f3EClQq9miV4RuT3mAL5/99r1sPODNiCyGvHSletsABPl0ez4CDDjxtNxPFo9gtSwLuJXtCAkxb61dkl1mOS3t3o4N1f17Kw9AOMcU8L9/4CJmBcU3a8fIJPGPoCW0gLaeNELeW6JmwzZWwczxqnGbp3prCh9Kj3vDANnLgQGFXE10rUh4DZv0h2MWBcmizbI0qxbneuSySeExAfi88vP1VfoML5X65UNdWr/7jy2De5c4X+Z6XkzcZbc5d2VRuFPf/uwRfMAf/0g5J+7t1XNcqK3EUkSJpjdhgr17z627ojhyj0Tlb7vbx9VPt10dDJpHJj20+P/UplzMNJ0Ny7X61P86mcWTCYLTmGNUCi3LqncdDdxFN9YjFHMiQkI9OitgrdaY3H2ivD18fEBVqvH/rqUEJu1FIILWJ6JdqclvxJ9J8dMR7Vq7aP+BHipvNVl5Ka75xPddbMObYZt8zvJcdRl1ebMKeC1WKYIZCVn2Le7373A6qtF9Vp3WrRmEOsuiJGne12HRHtdd3LVvasUzMrJKuHJV6eO7Vc9rdtlJi7hfXarVnd3eW5Xb+p3Efr2o/sdRis4cNnq/loGY8TrfujBMmI/8n98GnNfau7BXJeC+7jbKyyBcaecGBsVirojb6dn2pYdxsbLcoRNhgI2xECX7evvHH3DNVZqLJWqBZdp0Hr4T5nuy2ETYFol3MPlzMRB6b4xE4fDM8YGpVutfgsHd/V3ZpcCji66M5KPGNtaa4cSDx1/2XD67WBUmb+84E6sUBiS9ilAs5xV/TUKd2pnc5Sb2luvqnOdVqn2O6DpZJQ+gDF0QMyp18jU4F9w7apOkfI+2JvkRU/I6kPqnFRl4g5QndlohGDbX+8b4FNpJIo80CoyzUtiwGXsFTF/u4SZXpXCT17fYfVat0cbs1jZIGm7QQ8hdNcVr1a7R+vnbKWvoabOVgRvI3J3eB4oiNLQW4jcPZCzadzr/zuNu969vPG5NG4xr8bNTOnRWMZ5ad2lgZD2doq364o2E1TvvuIeWPX292NuoHzvL0n5knB75dsFgjbUb4/m1e/xg1xo9dvrZR0KeLuUtTbY6yvg82+s7/r3u/79x9C//xcAAP//xbT1Yg==" + return "eJzsWk+P274Rve+nGOypBTZue/WhQJINkEWbZLH+Bb9DUSi0OJbYpUiCpNb1ty9IybJXov6sTctFuzoltvfN43BmOPPED/CMuyWkqPIbAMssxyXcfkaV394AUDSpZsoyKZbw1xsAAPcVFJKWHG8ATC61TVIpNixbwoZw4z7VyJEYXEJG3G/QWiYys4R/3BrDb+/gNrdW3f7zBmDDkFOz9MgfQJACGy7usTvlULQsVf1JgJF7frk/+gWpFJYwYcDmCAVazVL3b2JhixrBpJoopLDRsoDPXx6/LmqAYxqvqPDSWNQJZea5+TJEa4Cae3pwXvtp/7TJHBMiL4Rxsua4WO8smle/2fPiUmStLwaouefjHhU8KsiNd2DNuvXzjdQFsUvoEtiTtNISHpXgbw4xDrnSII3K7adBejq1dqTlSLjNY8RaB+nt0SZfUBPOE2OJLcP+esbdVmr6Npf9qHChwh1yWxNSrMA0x/TZLFDJNI+0d9+IghfUhkkxZlbLUtDFC+ElRjLegIPHnkYg9lasXm3BayLt2OzYDsdmOM4mRGwHfyg2Q9t2+l74U632xWBAaLLZsHShkdDkTVWkvzKNk6v8A84o2FzLMstVaUGhBoOp7IudmutWM4uzk/VWT2DrPStVolxAYBprf49dyKQyb3PexfhUXppCqGBGcZLiwp+ukWnswUFlIMpi3VOFGw5y/S9MbawTtMOihp9ERRPLZJCISQlHmmy4JLYnvhXqFEX727fS7VLYs6SYaUKRXmTT9uAjm9ZwuMymNSwmbFpD5Xqb1tDt3zSVLSixZPaKqTJwdgd6V5Ut/AAwOzXyekDoI+ejfHZy9jAc9BFzTf/svMpmMOih5Rse9C0dJu7DSKn5mPnOGo9/N0whlWUnoU61vsrl1kAut1AQsQOVGSAagYmalNyMLr3DL3KzV2+QGe/3pKGLTcnDpXstJUfSZjVi/ME4UOiAHlsUSPRFrDrgEdNlkUhDY50SrWBwyC4Sxsa9PZNSXZqMC01ZEWrC8+fjICkmrkHq4fsgKY0FUQpporK5mT19+fbx8fHLfS+/mBO7x2r3GE1TmOkklmQ3gNUMuGu078Ld/7lwdxwn54t3g2inxN27hPcu4dVRKg1NFOrNOfH5Y3Xv5i+fEiI9vGORm0pC6wbAKUHL2h6Ytj+O3EP4GHI+N4tUFgWzCScWRbpLitMqyWePAjWKa2eKcMWqjBKl+O5cmx8dyGSTrXWKWOsU09d5os3OOkW44vpYlrKqa+OS9FhES1lVstiB7Cn2TnYTsv3RMeufjxz8icnikXuyJeUMhU2YTDSxYe6V2jMYPh4EHv70AxxKOx5CbjvmcFDYA7Lr4DLdU7t2CZ/8+e5l3x6B9djokVQewapDsyimGB5WvacbfnILlQq9uiV6ReXumiPY/t1r2cPG2zlsNeK52eswgImqTXvdBsx4+ARapNU9SA3rMn1GC0JS7Mu3cyrEJLu9FcSZuahlh3YHjHPMCPf/AyZSXlJ0v76DXBp7B1pKC2jTRS/JU8vcZJK9tTBnnGoMa08neelTZXkPDJw5FxhUxNVJN5KAO8BJ2BmpLk2ebJFleVixC0nnE1ziHfH56efqKwTADwq+srE69+/OYhdwbwr/zUzP24qTJDBvrgKFP/zlww7NHfz5g5B/DB+tmhVE7xKy2TDB7C6Wq3/3vnVthiv5TNSav7+RVNt02xHkpHFg29/u/6cazpGRJjy8XG5W8dnPLJhclpzCGqFULi+p3IYHOYovLMUk5cREJHTvUcGjNrQ4e0b4en9/B6vVfX9d2hCbd9SCM7g8Eu06Jp+JfppjJlCtgmep7wRjSF4hoKueog90P9k6ajMcn99JgaMm65FnTlGvw2WKaFbxjPvm97sXX33VqF/5TvPWDALeGT7y7J7XMak9r4O8mryVglk5WTV887Wq1/j1fOtOm4kpfIhu1Zn0zo/t+i3+3kPffoTfb3ScA+dl99fKGSNWD0uPFhGHlf/t05j5SoePZroS4cfNXiAFxo1yYmxSKupa36Bl2pUgxtbLCoRtjgK2xEAI+/InR99yjZUaK9VqwWUWtR7+XWaHctgmMK0SHsgVzKRR2X1jJo1Hzxgbld1q9Vs8chd/j3YuwdGkOyH4iLGdXDt6C5zQ8RcPb785jCr3FxtcxwqlIVmfEjRLr+qvWLjunc1RbhprverOZUamxu6AtpNTegfG0AFRp8mRqc4/40pWEyLVXbEXycselzVN6pysqsAdYHXioBGD2+Hq3wA/lSWiLCJlkWlfIAMuU6+M+ZsnzPSqFX7z+prVS90sbe1ijUGy7qAeQ/BuKl6jeo/Wz9lKX0tVna0IXkfsDlgeKIjS0GuI3T0kZ9O61/91WndzennwubRuMa/WzUxl0VjGeYXuwkBIez3l201F2wnqd19xj6x++7syV1DADxemfEm4vgLuHEFbKrin5lXw8UYutgru9bKAEt4tZZ0DNooKPqiAz3+wvuvf7/r3/4b+/Z8AAAD//5gqAPQ=" } diff --git a/metricbeat/module/ceph/mgr_cluster_disk/_meta/data.json b/metricbeat/module/ceph/mgr_cluster_disk/_meta/data.json index 12aa085f8e15..94c79adb4b49 100644 --- a/metricbeat/module/ceph/mgr_cluster_disk/_meta/data.json +++ b/metricbeat/module/ceph/mgr_cluster_disk/_meta/data.json @@ -5,7 +5,7 @@ "name": "host.example.com" }, "ceph": { - "cluster_disk": { + "mgr_cluster_disk": { "available": { "bytes": 0 }, @@ -18,7 +18,7 @@ } }, "event": { - "dataset": "ceph.cluster_disk", + "dataset": "ceph.mgr_cluster_disk", "duration": 115000, "module": "ceph" }, diff --git a/metricbeat/module/ceph/mgr_cluster_disk/_meta/fields.yml b/metricbeat/module/ceph/mgr_cluster_disk/_meta/fields.yml index fe755709362a..d24637a1fea0 100644 --- a/metricbeat/module/ceph/mgr_cluster_disk/_meta/fields.yml +++ b/metricbeat/module/ceph/mgr_cluster_disk/_meta/fields.yml @@ -1,7 +1,7 @@ -- name: cluster_disk +- name: mgr_cluster_disk type: group description: > - cluster_disk + mgr_cluster_disk release: beta fields: - name: available.bytes diff --git a/metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/config.yml b/metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/config.yml new file mode 100644 index 000000000000..9dad1228403b --- /dev/null +++ b/metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/config.yml @@ -0,0 +1,4 @@ +type: http +url: "/request?wait=1" +suffix: json +remove_fields_from_comparison: diff --git a/metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/mgr_df_sample_response.json-expected.json b/metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/mgr_df_sample_response.json-expected.json index 0967ef424bce..abc17d3539ab 100644 --- a/metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/mgr_df_sample_response.json-expected.json +++ b/metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/mgr_df_sample_response.json-expected.json @@ -1 +1,3 @@ -{} +[ + {"ceph":{"mgr_cluster_disk":{"available":{"bytes":9659875328},"total":{"bytes":10737418240},"used":{"bytes":3801088}}},"event":{"dataset":"ceph.mgr_cluster_disk","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_cluster_disk","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}} +] diff --git a/metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/mgr_failed_response.json-expected.json b/metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/mgr_failed_response.json-expected.json index 0967ef424bce..64265a15562d 100644 --- a/metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/mgr_failed_response.json-expected.json +++ b/metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/mgr_failed_response.json-expected.json @@ -1 +1,3 @@ -{} +[ + {"error":{"message":"could not get response data: command not known: dfb format=json-pretty"},"event":{"dataset":"ceph.mgr_cluster_disk","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_cluster_disk","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}} +] diff --git a/metricbeat/module/ceph/mgr_cluster_disk/mgr_cluster_disk_test.go b/metricbeat/module/ceph/mgr_cluster_disk/mgr_cluster_disk_test.go index e57ab6e3eeca..253665e3ed4c 100644 --- a/metricbeat/module/ceph/mgr_cluster_disk/mgr_cluster_disk_test.go +++ b/metricbeat/module/ceph/mgr_cluster_disk/mgr_cluster_disk_test.go @@ -21,6 +21,7 @@ import ( "testing" mbtest "github.com/elastic/beats/metricbeat/mb/testing" + _ "github.com/elastic/beats/metricbeat/module/ceph" ) func TestData(t *testing.T) { diff --git a/metricbeat/module/ceph/mgr_cluster_health/_meta/data.json b/metricbeat/module/ceph/mgr_cluster_health/_meta/data.json index 66303e4e7207..65e4061ded4d 100644 --- a/metricbeat/module/ceph/mgr_cluster_health/_meta/data.json +++ b/metricbeat/module/ceph/mgr_cluster_health/_meta/data.json @@ -5,7 +5,7 @@ "name": "host.example.com" }, "ceph": { - "cluster_health": { + "mgr_cluster_health": { "overall_status": "HEALTH_OK", "timechecks": { "epoch": 3, @@ -17,7 +17,7 @@ } }, "event": { - "dataset": "ceph.cluster_health", + "dataset": "ceph.mgr_cluster_health", "duration": 115000, "module": "ceph" }, diff --git a/metricbeat/module/ceph/mgr_cluster_health/_meta/fields.yml b/metricbeat/module/ceph/mgr_cluster_health/_meta/fields.yml index 606e8031cb3c..f3030809eb4f 100644 --- a/metricbeat/module/ceph/mgr_cluster_health/_meta/fields.yml +++ b/metricbeat/module/ceph/mgr_cluster_health/_meta/fields.yml @@ -1,7 +1,7 @@ -- name: cluster_health +- name: mgr_cluster_health type: group description: > - cluster_health + mgr_cluster_health release: beta fields: - name: overall_status diff --git a/metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/config.yml b/metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/config.yml new file mode 100644 index 000000000000..9dad1228403b --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/config.yml @@ -0,0 +1,4 @@ +type: http +url: "/request?wait=1" +suffix: json +remove_fields_from_comparison: diff --git a/metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/mgr_failed_response.json-expected.json b/metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/mgr_failed_response.json-expected.json index 0967ef424bce..d61d21f62448 100644 --- a/metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/mgr_failed_response.json-expected.json +++ b/metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/mgr_failed_response.json-expected.json @@ -1 +1,3 @@ -{} +[ + {"error":{"message":"could not get response data: command not known: dfb format=json-pretty"},"event":{"dataset":"ceph.mgr_osd_perf","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_osd_perf","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}} +] diff --git a/metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/mgr_osd_perf_sample_response.json-expected.json b/metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/mgr_osd_perf_sample_response.json-expected.json index 0967ef424bce..f36c0d5ef61f 100644 --- a/metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/mgr_osd_perf_sample_response.json-expected.json +++ b/metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/mgr_osd_perf_sample_response.json-expected.json @@ -1 +1,3 @@ -{} +[ + {"ceph":{"mgr_osd_perf":{"id":1,"stats":{"apply_latency_ms":2,"apply_latency_ns":4,"commit_latency_ms":1,"commit_latency_ns":3}}},"event":{"dataset":"ceph.mgr_osd_perf","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_osd_perf","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}} +] diff --git a/metricbeat/module/ceph/mgr_osd_perf/mgr_osd_perf_test.go b/metricbeat/module/ceph/mgr_osd_perf/mgr_osd_perf_test.go index c5528fb700fc..d24b579fb29c 100644 --- a/metricbeat/module/ceph/mgr_osd_perf/mgr_osd_perf_test.go +++ b/metricbeat/module/ceph/mgr_osd_perf/mgr_osd_perf_test.go @@ -21,6 +21,7 @@ import ( "testing" mbtest "github.com/elastic/beats/metricbeat/mb/testing" + _ "github.com/elastic/beats/metricbeat/module/ceph" ) func TestData(t *testing.T) { diff --git a/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/config.yml b/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/config.yml new file mode 100644 index 000000000000..9dad1228403b --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/config.yml @@ -0,0 +1,4 @@ +type: http +url: "/request?wait=1" +suffix: json +remove_fields_from_comparison: diff --git a/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/mgr_failed_response.json-expected.json b/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/mgr_failed_response.json-expected.json index 0967ef424bce..7ec863fab229 100644 --- a/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/mgr_failed_response.json-expected.json +++ b/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/mgr_failed_response.json-expected.json @@ -1 +1,3 @@ -{} +[ + {"error":{"message":"could not get response data: command not known: dfb format=json-pretty"},"event":{"dataset":"ceph.mgr_osd_pool_stats","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_osd_pool_stats","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}} +] diff --git a/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/mgr_osd_pool_stats_sample_response.json-expected.json b/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/mgr_osd_pool_stats_sample_response.json-expected.json index 0967ef424bce..50146e84c548 100644 --- a/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/mgr_osd_pool_stats_sample_response.json-expected.json +++ b/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/mgr_osd_pool_stats_sample_response.json-expected.json @@ -1 +1,10 @@ -{} +[ + {"ceph":{"mgr_osd_pool_stats":{"client_io_rate":{"read_bytes_sec":0,"read_op_per_sec":0,"write_bytes_sec":0,"write_op_per_sec":0},"pool_id":5,"pool_name":"default.rgw.control"}},"event":{"dataset":"ceph.mgr_osd_pool_stats","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_osd_pool_stats","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}}, + {"ceph":{"mgr_osd_pool_stats":{"client_io_rate":{"read_bytes_sec":0,"read_op_per_sec":0,"write_bytes_sec":0,"write_op_per_sec":0},"pool_id":7,"pool_name":"default.rgw.log"}},"event":{"dataset":"ceph.mgr_osd_pool_stats","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_osd_pool_stats","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}}, + {"ceph":{"mgr_osd_pool_stats":{"client_io_rate":{"read_bytes_sec":0,"read_op_per_sec":0,"write_bytes_sec":0,"write_op_per_sec":0},"pool_id":4,"pool_name":".rgw.root"}},"event":{"dataset":"ceph.mgr_osd_pool_stats","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_osd_pool_stats","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}}, + {"ceph":{"mgr_osd_pool_stats":{"client_io_rate":{"read_bytes_sec":0,"read_op_per_sec":0,"write_bytes_sec":0,"write_op_per_sec":0},"pool_id":3,"pool_name":"cephfs_metadata"}},"event":{"dataset":"ceph.mgr_osd_pool_stats","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_osd_pool_stats","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}}, + {"ceph":{"mgr_osd_pool_stats":{"client_io_rate":{"read_bytes_sec":0,"read_op_per_sec":0,"write_bytes_sec":0,"write_op_per_sec":0},"pool_id":2,"pool_name":"cephfs_data"}},"event":{"dataset":"ceph.mgr_osd_pool_stats","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_osd_pool_stats","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}}, + {"ceph":{"mgr_osd_pool_stats":{"client_io_rate":{"read_bytes_sec":1,"read_op_per_sec":2,"write_bytes_sec":3,"write_op_per_sec":4},"pool_id":1,"pool_name":"rbd"}},"event":{"dataset":"ceph.mgr_osd_pool_stats","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_osd_pool_stats","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}}, + {"ceph":{"mgr_osd_pool_stats":{"client_io_rate":{"read_bytes_sec":0,"read_op_per_sec":0,"write_bytes_sec":0,"write_op_per_sec":0},"pool_id":6,"pool_name":"default.rgw.meta"}},"event":{"dataset":"ceph.mgr_osd_pool_stats","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_osd_pool_stats","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}}, + {"ceph":{"mgr_osd_pool_stats":{"client_io_rate":{"read_bytes_sec":0,"read_op_per_sec":0,"write_bytes_sec":0,"write_op_per_sec":0},"pool_id":8,"pool_name":"default.rgw.buckets.index"}},"event":{"dataset":"ceph.mgr_osd_pool_stats","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_osd_pool_stats","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}} +] diff --git a/metricbeat/module/ceph/mgr_osd_pool_stats/mgr_osd_pool_stats_test.go b/metricbeat/module/ceph/mgr_osd_pool_stats/mgr_osd_pool_stats_test.go index c3cd88fb53e4..ca2eb6414bd0 100644 --- a/metricbeat/module/ceph/mgr_osd_pool_stats/mgr_osd_pool_stats_test.go +++ b/metricbeat/module/ceph/mgr_osd_pool_stats/mgr_osd_pool_stats_test.go @@ -21,6 +21,7 @@ import ( "testing" mbtest "github.com/elastic/beats/metricbeat/mb/testing" + _ "github.com/elastic/beats/metricbeat/module/ceph" ) func TestData(t *testing.T) { diff --git a/metricbeat/module/ceph/mgr_osd_tree/_meta/data.json b/metricbeat/module/ceph/mgr_osd_tree/_meta/data.json index 4a2cb3e260df..9239f288557b 100644 --- a/metricbeat/module/ceph/mgr_osd_tree/_meta/data.json +++ b/metricbeat/module/ceph/mgr_osd_tree/_meta/data.json @@ -1,7 +1,7 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", "ceph": { - "osd_tree": { + "mgr_osd_tree": { "children": [ "-2" ], @@ -13,7 +13,7 @@ } }, "event": { - "dataset": "ceph.osd_tree", + "dataset": "ceph.mgr_osd_tree", "duration": 115000, "module": "ceph" }, diff --git a/metricbeat/module/ceph/mgr_osd_tree/_meta/fields.yml b/metricbeat/module/ceph/mgr_osd_tree/_meta/fields.yml index d924267ee7f7..93823ed2455f 100644 --- a/metricbeat/module/ceph/mgr_osd_tree/_meta/fields.yml +++ b/metricbeat/module/ceph/mgr_osd_tree/_meta/fields.yml @@ -1,4 +1,4 @@ -- name: osd_tree +- name: mgr_osd_tree type: group description: > OSD tree information diff --git a/metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/config.yml b/metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/config.yml new file mode 100644 index 000000000000..9dad1228403b --- /dev/null +++ b/metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/config.yml @@ -0,0 +1,4 @@ +type: http +url: "/request?wait=1" +suffix: json +remove_fields_from_comparison: diff --git a/metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/mgr_failed_response.json-expected.json b/metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/mgr_failed_response.json-expected.json index 0967ef424bce..4deb6443f439 100644 --- a/metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/mgr_failed_response.json-expected.json +++ b/metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/mgr_failed_response.json-expected.json @@ -1 +1,3 @@ -{} +[ + {"error":{"message":"could not get response data: command not known: dfb format=json-pretty"},"event":{"dataset":"ceph.mgr_osd_tree","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_osd_tree","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}} +] diff --git a/metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/mgr_osd_tree_sample_response.json-expected.json b/metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/mgr_osd_tree_sample_response.json-expected.json index 0967ef424bce..eeb38333ca43 100644 --- a/metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/mgr_osd_tree_sample_response.json-expected.json +++ b/metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/mgr_osd_tree_sample_response.json-expected.json @@ -1 +1,6 @@ -{} +[ + {"ceph":{"mgr_osd_tree":{"children":["1","0"],"father":"default","id":-3,"name":"ceph-mon1","type":"host","type_id":1}},"event":{"dataset":"ceph.mgr_osd_tree","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_osd_tree","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}}, + {"ceph":{"mgr_osd_tree":{"children":["-3"],"father":"","id":-1,"name":"default","type":"root","type_id":10}},"event":{"dataset":"ceph.mgr_osd_tree","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_osd_tree","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}}, + {"ceph":{"mgr_osd_tree":{"crush_weight":0.048691,"depth":2,"device_class":"hdd","exists":true,"father":"ceph-mon1","id":1,"name":"osd.1","primary_affinity":1,"reweight":1,"status":"up","type":"osd","type_id":0}},"event":{"dataset":"ceph.mgr_osd_tree","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_osd_tree","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}}, + {"ceph":{"mgr_osd_tree":{"crush_weight":0.048691,"depth":2,"device_class":"hdd","exists":true,"father":"ceph-mon1","id":0,"name":"osd.0","primary_affinity":1,"reweight":1,"status":"up","type":"osd","type_id":0}},"event":{"dataset":"ceph.mgr_osd_tree","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_osd_tree","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}} +] diff --git a/metricbeat/module/ceph/mgr_osd_tree/mgr_osd_tree_test.go b/metricbeat/module/ceph/mgr_osd_tree/mgr_osd_tree_test.go index 15fdccf27e7c..cc6ce604ed6d 100644 --- a/metricbeat/module/ceph/mgr_osd_tree/mgr_osd_tree_test.go +++ b/metricbeat/module/ceph/mgr_osd_tree/mgr_osd_tree_test.go @@ -21,6 +21,7 @@ import ( "testing" mbtest "github.com/elastic/beats/metricbeat/mb/testing" + _ "github.com/elastic/beats/metricbeat/module/ceph" ) func TestData(t *testing.T) { diff --git a/metricbeat/module/ceph/mgr_pool_disk/_meta/fields.yml b/metricbeat/module/ceph/mgr_pool_disk/_meta/fields.yml index e5215b688a57..f3eeea23c227 100644 --- a/metricbeat/module/ceph/mgr_pool_disk/_meta/fields.yml +++ b/metricbeat/module/ceph/mgr_pool_disk/_meta/fields.yml @@ -1,7 +1,7 @@ -- name: pool_disk +- name: mgr_pool_disk type: group description: > - pool_disk + mgr_pool_disk release: beta fields: - name: id diff --git a/metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/config.yml b/metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/config.yml new file mode 100644 index 000000000000..9dad1228403b --- /dev/null +++ b/metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/config.yml @@ -0,0 +1,4 @@ +type: http +url: "/request?wait=1" +suffix: json +remove_fields_from_comparison: diff --git a/metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/mgr_df_sample_response.json-expected.json b/metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/mgr_df_sample_response.json-expected.json index 0967ef424bce..92e2e2b79f65 100644 --- a/metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/mgr_df_sample_response.json-expected.json +++ b/metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/mgr_df_sample_response.json-expected.json @@ -1 +1,10 @@ -{} +[ + {"ceph":{"mgr_pool_disk":{"id":7,"name":"default.rgw.log","stats":{"available":{"bytes":9123004416},"objects":176,"used":{"bytes":0,"kb":0}}}},"event":{"dataset":"ceph.mgr_pool_disk","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_pool_disk","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}}, + {"ceph":{"mgr_pool_disk":{"id":4,"name":".rgw.root","stats":{"available":{"bytes":9123004416},"objects":6,"used":{"bytes":393216,"kb":384}}}},"event":{"dataset":"ceph.mgr_pool_disk","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_pool_disk","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}}, + {"ceph":{"mgr_pool_disk":{"id":1,"name":"rbd","stats":{"available":{"bytes":9123004416},"objects":0,"used":{"bytes":0,"kb":0}}}},"event":{"dataset":"ceph.mgr_pool_disk","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_pool_disk","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}}, + {"ceph":{"mgr_pool_disk":{"id":2,"name":"cephfs_data","stats":{"available":{"bytes":9123004416},"objects":0,"used":{"bytes":0,"kb":0}}}},"event":{"dataset":"ceph.mgr_pool_disk","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_pool_disk","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}}, + {"ceph":{"mgr_pool_disk":{"id":3,"name":"cephfs_metadata","stats":{"available":{"bytes":9123004416},"objects":22,"used":{"bytes":524288,"kb":512}}}},"event":{"dataset":"ceph.mgr_pool_disk","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_pool_disk","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}}, + {"ceph":{"mgr_pool_disk":{"id":6,"name":"default.rgw.meta","stats":{"available":{"bytes":9123004416},"objects":5,"used":{"bytes":262144,"kb":256}}}},"event":{"dataset":"ceph.mgr_pool_disk","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_pool_disk","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}}, + {"ceph":{"mgr_pool_disk":{"id":5,"name":"default.rgw.control","stats":{"available":{"bytes":9123004416},"objects":8,"used":{"bytes":0,"kb":0}}}},"event":{"dataset":"ceph.mgr_pool_disk","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_pool_disk","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}}, + {"ceph":{"mgr_pool_disk":{"id":8,"name":"default.rgw.buckets.index","stats":{"available":{"bytes":9123004416},"objects":1,"used":{"bytes":0,"kb":0}}}},"event":{"dataset":"ceph.mgr_pool_disk","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_pool_disk","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}} +] diff --git a/metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/mgr_failed_response.json-expected.json b/metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/mgr_failed_response.json-expected.json index 0967ef424bce..2398162a66a1 100644 --- a/metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/mgr_failed_response.json-expected.json +++ b/metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/mgr_failed_response.json-expected.json @@ -1 +1,3 @@ -{} +[ + {"error":{"message":"could not get response data: command not known: dfb format=json-pretty"},"event":{"dataset":"ceph.mgr_pool_disk","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_pool_disk","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}} +] diff --git a/metricbeat/module/ceph/mgr_pool_disk/mgr_pool_disk_test.go b/metricbeat/module/ceph/mgr_pool_disk/mgr_pool_disk_test.go index 68b1e8b28a46..f22650a189ba 100644 --- a/metricbeat/module/ceph/mgr_pool_disk/mgr_pool_disk_test.go +++ b/metricbeat/module/ceph/mgr_pool_disk/mgr_pool_disk_test.go @@ -21,6 +21,7 @@ import ( "testing" mbtest "github.com/elastic/beats/metricbeat/mb/testing" + _ "github.com/elastic/beats/metricbeat/module/ceph" ) func TestData(t *testing.T) { From 4e6adb71ce733f6a40609ac92abfb18fac4a7e96 Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Mon, 17 Feb 2020 16:21:05 +0100 Subject: [PATCH 36/41] Fix: integration-tests --- libbeat/tests/compose/project.go | 7 +++++++ libbeat/tests/compose/wrapper.go | 6 ++++++ metricbeat/module/ceph/_meta/supported-versions.yml | 4 ++-- .../cluster_health/cluster_health_integration_test.go | 2 +- .../mgr_cluster_disk_integration_test.go | 9 +++++++-- .../ceph/mgr_cluster_disk/mgr_cluster_disk_test.go | 2 +- .../mgr_cluster_health_integration_test.go | 7 ++++++- .../ceph/mgr_osd_perf/mgr_osd_perf_integration_test.go | 7 ++++++- metricbeat/module/ceph/mgr_osd_perf/mgr_osd_perf_test.go | 2 +- .../mgr_osd_pool_stats_integration_test.go | 7 ++++++- .../ceph/mgr_osd_pool_stats/mgr_osd_pool_stats_test.go | 2 +- .../ceph/mgr_osd_tree/mgr_osd_tree_integration_test.go | 7 ++++++- metricbeat/module/ceph/mgr_osd_tree/mgr_osd_tree_test.go | 2 +- .../ceph/mgr_pool_disk/mgr_pool_disk_integration_test.go | 7 ++++++- .../module/ceph/mgr_pool_disk/mgr_pool_disk_test.go | 2 +- metricbeat/module/ceph/test_ceph.py | 4 ++-- 16 files changed, 60 insertions(+), 17 deletions(-) diff --git a/libbeat/tests/compose/project.go b/libbeat/tests/compose/project.go index f4ebfb904e58..ba7413bcbcce 100644 --- a/libbeat/tests/compose/project.go +++ b/libbeat/tests/compose/project.go @@ -45,6 +45,8 @@ type UpOptions struct { // use to advertise itself to clients SetupAdvertisedHostEnvFile bool SetupAdvertisedHostEnvFilePort int + + Environment []string } // UpOption is a modifier for UpOptions @@ -71,6 +73,11 @@ func UpWithAdvertisedHostEnvFileForPort(port int) UpOption { } } +// UpWithEnvironmentVariable add environment variable. +func UpWithEnvironmentVariable(keyValue string) UpOption { + return func(options *UpOptions) { options.Environment = append(options.Environment, keyValue) } +} + // Filter options for services type Filter struct { State State diff --git a/libbeat/tests/compose/wrapper.go b/libbeat/tests/compose/wrapper.go index 35e37098a38c..1a663a5cd71d 100644 --- a/libbeat/tests/compose/wrapper.go +++ b/libbeat/tests/compose/wrapper.go @@ -181,6 +181,12 @@ func (d *wrapperDriver) Up(ctx context.Context, opts UpOptions, service string) args = append(args, "--force-recreate") } + if len(opts.Environment) > 0 { + for _, keyValue := range opts.Environment { + d.Environment = append(d.Environment, keyValue) + } + } + if service != "" { args = append(args, service) } diff --git a/metricbeat/module/ceph/_meta/supported-versions.yml b/metricbeat/module/ceph/_meta/supported-versions.yml index e2f6799ecf52..b8fc1d529c5b 100644 --- a/metricbeat/module/ceph/_meta/supported-versions.yml +++ b/metricbeat/module/ceph/_meta/supported-versions.yml @@ -1,5 +1,5 @@ variants: - - CEPH_NAME: jewel + - CEPH_CODENAME: jewel CEPH_VERSION: master-6373c6a-jewel-centos-7-x86_64 - - CEPH_NAME: nautilus + - CEPH_CODENAME: nautilus CEPH_VERSION: master-97985eb-nautilus-centos-7-x86_64 diff --git a/metricbeat/module/ceph/cluster_health/cluster_health_integration_test.go b/metricbeat/module/ceph/cluster_health/cluster_health_integration_test.go index a02999f92ee1..63ee0e3bbaaf 100644 --- a/metricbeat/module/ceph/cluster_health/cluster_health_integration_test.go +++ b/metricbeat/module/ceph/cluster_health/cluster_health_integration_test.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -// +build integration,linux +// build integration,linux package cluster_health diff --git a/metricbeat/module/ceph/mgr_cluster_disk/mgr_cluster_disk_integration_test.go b/metricbeat/module/ceph/mgr_cluster_disk/mgr_cluster_disk_integration_test.go index 66f9ae41de46..8bdf4daf4b8f 100644 --- a/metricbeat/module/ceph/mgr_cluster_disk/mgr_cluster_disk_integration_test.go +++ b/metricbeat/module/ceph/mgr_cluster_disk/mgr_cluster_disk_integration_test.go @@ -15,12 +15,13 @@ // specific language governing permissions and limitations // under the License. -// +build integration,linux +// build integration,linux package mgr_cluster_disk import ( "testing" + "time" "github.com/stretchr/testify/require" @@ -32,7 +33,11 @@ import ( const user = "demo" func TestData(t *testing.T) { - service := compose.EnsureUpWithTimeout(t, 120, "ceph-mgr") + service := compose.EnsureUp(t, "ceph", + compose.UpWithTimeout(120*time.Second), + compose.UpWithEnvironmentVariable("CEPH_CODENAME=nautilus"), + compose.UpWithEnvironmentVariable("CEPH_VERSION=master-97985eb-nautilus-centos-7-x86_64"), + ) f := mbtest.NewReportingMetricSetV2Error(t, getConfig(service.HostForPort(8003), mgrtest.GetPassword(t, service.HostForPort(5000), user))) diff --git a/metricbeat/module/ceph/mgr_cluster_disk/mgr_cluster_disk_test.go b/metricbeat/module/ceph/mgr_cluster_disk/mgr_cluster_disk_test.go index 253665e3ed4c..b227f4c3f021 100644 --- a/metricbeat/module/ceph/mgr_cluster_disk/mgr_cluster_disk_test.go +++ b/metricbeat/module/ceph/mgr_cluster_disk/mgr_cluster_disk_test.go @@ -24,6 +24,6 @@ import ( _ "github.com/elastic/beats/metricbeat/module/ceph" ) -func TestData(t *testing.T) { +func TestDataFiles(t *testing.T) { mbtest.TestDataFiles(t, "ceph", "mgr_cluster_disk") } diff --git a/metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health_integration_test.go b/metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health_integration_test.go index f1299f1ce287..d07a180aa7a1 100644 --- a/metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health_integration_test.go +++ b/metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health_integration_test.go @@ -21,6 +21,7 @@ package mgr_cluster_health import ( "testing" + "time" "github.com/stretchr/testify/require" @@ -32,7 +33,11 @@ import ( const user = "demo" func TestData(t *testing.T) { - service := compose.EnsureUpWithTimeout(t, 120, "ceph-mgr") + service := compose.EnsureUp(t, "ceph", + compose.UpWithTimeout(120*time.Second), + compose.UpWithEnvironmentVariable("CEPH_CODENAME=nautilus"), + compose.UpWithEnvironmentVariable("CEPH_VERSION=master-97985eb-nautilus-centos-7-x86_64"), + ) f := mbtest.NewReportingMetricSetV2Error(t, getConfig(service.HostForPort(8003), mgrtest.GetPassword(t, service.HostForPort(5000), user))) diff --git a/metricbeat/module/ceph/mgr_osd_perf/mgr_osd_perf_integration_test.go b/metricbeat/module/ceph/mgr_osd_perf/mgr_osd_perf_integration_test.go index 4132e99ad7ac..1866de6ced5d 100644 --- a/metricbeat/module/ceph/mgr_osd_perf/mgr_osd_perf_integration_test.go +++ b/metricbeat/module/ceph/mgr_osd_perf/mgr_osd_perf_integration_test.go @@ -21,6 +21,7 @@ package mgr_osd_perf import ( "testing" + "time" "github.com/stretchr/testify/require" @@ -32,7 +33,11 @@ import ( const user = "demo" func TestData(t *testing.T) { - service := compose.EnsureUpWithTimeout(t, 120, "ceph-mgr") + service := compose.EnsureUp(t, "ceph", + compose.UpWithTimeout(120*time.Second), + compose.UpWithEnvironmentVariable("CEPH_CODENAME=nautilus"), + compose.UpWithEnvironmentVariable("CEPH_VERSION=master-97985eb-nautilus-centos-7-x86_64"), + ) f := mbtest.NewReportingMetricSetV2Error(t, getConfig(service.HostForPort(8003), mgrtest.GetPassword(t, service.HostForPort(5000), user))) diff --git a/metricbeat/module/ceph/mgr_osd_perf/mgr_osd_perf_test.go b/metricbeat/module/ceph/mgr_osd_perf/mgr_osd_perf_test.go index d24b579fb29c..9cd253b9d07b 100644 --- a/metricbeat/module/ceph/mgr_osd_perf/mgr_osd_perf_test.go +++ b/metricbeat/module/ceph/mgr_osd_perf/mgr_osd_perf_test.go @@ -24,6 +24,6 @@ import ( _ "github.com/elastic/beats/metricbeat/module/ceph" ) -func TestData(t *testing.T) { +func TestDataFiles(t *testing.T) { mbtest.TestDataFiles(t, "ceph", "mgr_osd_perf") } diff --git a/metricbeat/module/ceph/mgr_osd_pool_stats/mgr_osd_pool_stats_integration_test.go b/metricbeat/module/ceph/mgr_osd_pool_stats/mgr_osd_pool_stats_integration_test.go index db5e45654a5c..d93576154d67 100644 --- a/metricbeat/module/ceph/mgr_osd_pool_stats/mgr_osd_pool_stats_integration_test.go +++ b/metricbeat/module/ceph/mgr_osd_pool_stats/mgr_osd_pool_stats_integration_test.go @@ -21,6 +21,7 @@ package mgr_osd_pool_stats import ( "testing" + "time" "github.com/stretchr/testify/require" @@ -32,7 +33,11 @@ import ( const user = "demo" func TestData(t *testing.T) { - service := compose.EnsureUpWithTimeout(t, 120, "ceph-mgr") + service := compose.EnsureUp(t, "ceph", + compose.UpWithTimeout(120*time.Second), + compose.UpWithEnvironmentVariable("CEPH_CODENAME=nautilus"), + compose.UpWithEnvironmentVariable("CEPH_VERSION=master-97985eb-nautilus-centos-7-x86_64"), + ) f := mbtest.NewReportingMetricSetV2Error(t, getConfig(service.HostForPort(8003), mgrtest.GetPassword(t, service.HostForPort(5000), user))) diff --git a/metricbeat/module/ceph/mgr_osd_pool_stats/mgr_osd_pool_stats_test.go b/metricbeat/module/ceph/mgr_osd_pool_stats/mgr_osd_pool_stats_test.go index ca2eb6414bd0..9e3e27729112 100644 --- a/metricbeat/module/ceph/mgr_osd_pool_stats/mgr_osd_pool_stats_test.go +++ b/metricbeat/module/ceph/mgr_osd_pool_stats/mgr_osd_pool_stats_test.go @@ -24,6 +24,6 @@ import ( _ "github.com/elastic/beats/metricbeat/module/ceph" ) -func TestData(t *testing.T) { +func TestDataFiles(t *testing.T) { mbtest.TestDataFiles(t, "ceph", "mgr_osd_pool_stats") } diff --git a/metricbeat/module/ceph/mgr_osd_tree/mgr_osd_tree_integration_test.go b/metricbeat/module/ceph/mgr_osd_tree/mgr_osd_tree_integration_test.go index f5dad0f0491d..e0d57380189c 100644 --- a/metricbeat/module/ceph/mgr_osd_tree/mgr_osd_tree_integration_test.go +++ b/metricbeat/module/ceph/mgr_osd_tree/mgr_osd_tree_integration_test.go @@ -21,6 +21,7 @@ package mgr_osd_tree import ( "testing" + "time" "github.com/stretchr/testify/require" @@ -32,7 +33,11 @@ import ( const user = "demo" func TestData(t *testing.T) { - service := compose.EnsureUpWithTimeout(t, 120, "ceph-mgr") + service := compose.EnsureUp(t, "ceph", + compose.UpWithTimeout(120*time.Second), + compose.UpWithEnvironmentVariable("CEPH_CODENAME=nautilus"), + compose.UpWithEnvironmentVariable("CEPH_VERSION=master-97985eb-nautilus-centos-7-x86_64"), + ) f := mbtest.NewReportingMetricSetV2Error(t, getConfig(service.HostForPort(8003), mgrtest.GetPassword(t, service.HostForPort(5000), user))) diff --git a/metricbeat/module/ceph/mgr_osd_tree/mgr_osd_tree_test.go b/metricbeat/module/ceph/mgr_osd_tree/mgr_osd_tree_test.go index cc6ce604ed6d..d59ac2803e0b 100644 --- a/metricbeat/module/ceph/mgr_osd_tree/mgr_osd_tree_test.go +++ b/metricbeat/module/ceph/mgr_osd_tree/mgr_osd_tree_test.go @@ -24,6 +24,6 @@ import ( _ "github.com/elastic/beats/metricbeat/module/ceph" ) -func TestData(t *testing.T) { +func TestDataFiles(t *testing.T) { mbtest.TestDataFiles(t, "ceph", "mgr_osd_tree") } diff --git a/metricbeat/module/ceph/mgr_pool_disk/mgr_pool_disk_integration_test.go b/metricbeat/module/ceph/mgr_pool_disk/mgr_pool_disk_integration_test.go index 11e40b6df79a..421f20ff07d3 100644 --- a/metricbeat/module/ceph/mgr_pool_disk/mgr_pool_disk_integration_test.go +++ b/metricbeat/module/ceph/mgr_pool_disk/mgr_pool_disk_integration_test.go @@ -21,6 +21,7 @@ package mgr_pool_disk import ( "testing" + "time" "github.com/stretchr/testify/require" @@ -32,7 +33,11 @@ import ( const user = "demo" func TestData(t *testing.T) { - service := compose.EnsureUpWithTimeout(t, 120, "ceph-mgr") + service := compose.EnsureUp(t, "ceph", + compose.UpWithTimeout(120*time.Second), + compose.UpWithEnvironmentVariable("CEPH_CODENAME=nautilus"), + compose.UpWithEnvironmentVariable("CEPH_VERSION=master-97985eb-nautilus-centos-7-x86_64"), + ) f := mbtest.NewReportingMetricSetV2Error(t, getConfig(service.HostForPort(8003), mgrtest.GetPassword(t, service.HostForPort(5000), user))) diff --git a/metricbeat/module/ceph/mgr_pool_disk/mgr_pool_disk_test.go b/metricbeat/module/ceph/mgr_pool_disk/mgr_pool_disk_test.go index f22650a189ba..ab0c3f03d690 100644 --- a/metricbeat/module/ceph/mgr_pool_disk/mgr_pool_disk_test.go +++ b/metricbeat/module/ceph/mgr_pool_disk/mgr_pool_disk_test.go @@ -24,6 +24,6 @@ import ( _ "github.com/elastic/beats/metricbeat/module/ceph" ) -func TestData(t *testing.T) { +func TestDataFiles(t *testing.T) { mbtest.TestDataFiles(t, "ceph", "mgr_pool_disk") } diff --git a/metricbeat/module/ceph/test_ceph.py b/metricbeat/module/ceph/test_ceph.py index 7951353aa9e7..546b63967918 100644 --- a/metricbeat/module/ceph/test_ceph.py +++ b/metricbeat/module/ceph/test_ceph.py @@ -62,10 +62,10 @@ def test_ceph_mgr(self, metricset): self.assert_fields_are_documented(evt) def old_ceph_version(self): - if not 'CEPH_NAME' in self.COMPOSE_ENV: + if not 'CEPH_CODENAME' in self.COMPOSE_ENV: return False - return self.COMPOSE_ENV['CEPH_NAME'] == 'jewel' + return self.COMPOSE_ENV['CEPH_CODENAME'] == 'jewel' def get_ceph_module_config(self, metricset): return { From a672525a84d93ab2ec4f96d28c9ba1e5b10fcfea Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Tue, 18 Feb 2020 09:44:16 +0100 Subject: [PATCH 37/41] Adjust source after code review --- .../cluster_disk_integration_test.go | 2 +- .../cluster_health_integration_test.go | 4 +- .../cluster_status_integration_test.go | 2 +- metricbeat/module/ceph/docker-compose.yml | 15 +++++-- .../mgr_cluster_disk_integration_test.go | 10 +---- .../mgr_cluster_health_integration_test.go | 8 +--- .../mgr_osd_perf_integration_test.go | 8 +--- .../mgr_osd_pool_stats_integration_test.go | 8 +--- .../mgr_osd_tree_integration_test.go | 8 +--- .../module/ceph/mgr_pool_disk/_meta/data.json | 4 +- .../mgr_pool_disk_integration_test.go | 8 +--- metricbeat/module/ceph/mgrtest/compose.go | 41 +++++++++++++++++++ .../ceph/osd_df/osd_df_integration_test.go | 2 +- .../osd_tree/osd_tree_integration_test.go | 2 +- .../pool_disk/pool_disk_integration_test.go | 2 +- metricbeat/module/ceph/test_ceph.py | 17 ++++---- 16 files changed, 76 insertions(+), 65 deletions(-) create mode 100644 metricbeat/module/ceph/mgrtest/compose.go diff --git a/metricbeat/module/ceph/cluster_disk/cluster_disk_integration_test.go b/metricbeat/module/ceph/cluster_disk/cluster_disk_integration_test.go index 7f3fa8116083..b91147bfecc3 100644 --- a/metricbeat/module/ceph/cluster_disk/cluster_disk_integration_test.go +++ b/metricbeat/module/ceph/cluster_disk/cluster_disk_integration_test.go @@ -27,7 +27,7 @@ import ( ) func TestData(t *testing.T) { - service := compose.EnsureUpWithTimeout(t, 120, "ceph") + service := compose.EnsureUpWithTimeout(t, 120, "ceph-api") f := mbtest.NewReportingMetricSetV2Error(t, getConfig(service.Host())) diff --git a/metricbeat/module/ceph/cluster_health/cluster_health_integration_test.go b/metricbeat/module/ceph/cluster_health/cluster_health_integration_test.go index 63ee0e3bbaaf..ff7c7d8e68cd 100644 --- a/metricbeat/module/ceph/cluster_health/cluster_health_integration_test.go +++ b/metricbeat/module/ceph/cluster_health/cluster_health_integration_test.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -// build integration,linux +// +build integration,linux package cluster_health @@ -27,7 +27,7 @@ import ( ) func TestData(t *testing.T) { - service := compose.EnsureUpWithTimeout(t, 120, "ceph") + service := compose.EnsureUpWithTimeout(t, 120, "ceph-api") f := mbtest.NewReportingMetricSetV2Error(t, getConfig(service.Host())) diff --git a/metricbeat/module/ceph/cluster_status/cluster_status_integration_test.go b/metricbeat/module/ceph/cluster_status/cluster_status_integration_test.go index 5948dc4366e2..1741918bf668 100644 --- a/metricbeat/module/ceph/cluster_status/cluster_status_integration_test.go +++ b/metricbeat/module/ceph/cluster_status/cluster_status_integration_test.go @@ -27,7 +27,7 @@ import ( ) func TestData(t *testing.T) { - service := compose.EnsureUpWithTimeout(t, 120, "ceph") + service := compose.EnsureUpWithTimeout(t, 120, "ceph-api") f := mbtest.NewReportingMetricSetV2Error(t, getConfig(service.Host())) diff --git a/metricbeat/module/ceph/docker-compose.yml b/metricbeat/module/ceph/docker-compose.yml index dccee656f9a4..181dfd6ffbe4 100644 --- a/metricbeat/module/ceph/docker-compose.yml +++ b/metricbeat/module/ceph/docker-compose.yml @@ -2,13 +2,22 @@ version: '3.7' services: ceph: - image: docker.elastic.co/integrations-ci/beats-ceph:${CEPH_VERSION:-master-6373c6a-jewel-centos-7-x86_64}-1 + image: docker.elastic.co/integrations-ci/beats-ceph:${CEPH_VERSION:-master-97985eb-nautilus-centos-7-x86_64}-1 build: context: ./_meta - dockerfile: Dockerfile.${CEPH_CODENAME:-jewel} + dockerfile: Dockerfile.${CEPH_CODENAME:-nautilus} args: - CEPH_VERSION: ${CEPH_VERSION:-master-6373c6a-jewel-centos-7-x86_64} + CEPH_VERSION: ${CEPH_VERSION:-master-97985eb-nautilus-centos-7-x86_64} ports: - 5000 - 8003 - 8080 + ceph-api: + image: docker.elastic.co/integrations-ci/beats-ceph:master-6373c6a-jewel-centos-7-x86_64-1 + build: + context: ./_meta + dockerfile: Dockerfile.jewel + args: + CEPH_VERSION: master-6373c6a-jewel-centos-7-x86_64 + ports: + - 5000 diff --git a/metricbeat/module/ceph/mgr_cluster_disk/mgr_cluster_disk_integration_test.go b/metricbeat/module/ceph/mgr_cluster_disk/mgr_cluster_disk_integration_test.go index 8bdf4daf4b8f..14677668defb 100644 --- a/metricbeat/module/ceph/mgr_cluster_disk/mgr_cluster_disk_integration_test.go +++ b/metricbeat/module/ceph/mgr_cluster_disk/mgr_cluster_disk_integration_test.go @@ -15,17 +15,15 @@ // specific language governing permissions and limitations // under the License. -// build integration,linux +// +build integration,linux package mgr_cluster_disk import ( "testing" - "time" "github.com/stretchr/testify/require" - "github.com/elastic/beats/libbeat/tests/compose" mbtest "github.com/elastic/beats/metricbeat/mb/testing" "github.com/elastic/beats/metricbeat/module/ceph/mgrtest" ) @@ -33,11 +31,7 @@ import ( const user = "demo" func TestData(t *testing.T) { - service := compose.EnsureUp(t, "ceph", - compose.UpWithTimeout(120*time.Second), - compose.UpWithEnvironmentVariable("CEPH_CODENAME=nautilus"), - compose.UpWithEnvironmentVariable("CEPH_VERSION=master-97985eb-nautilus-centos-7-x86_64"), - ) + service := mgrtest.EnsureUp(t) f := mbtest.NewReportingMetricSetV2Error(t, getConfig(service.HostForPort(8003), mgrtest.GetPassword(t, service.HostForPort(5000), user))) diff --git a/metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health_integration_test.go b/metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health_integration_test.go index d07a180aa7a1..a73fec46e86d 100644 --- a/metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health_integration_test.go +++ b/metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health_integration_test.go @@ -21,11 +21,9 @@ package mgr_cluster_health import ( "testing" - "time" "github.com/stretchr/testify/require" - "github.com/elastic/beats/libbeat/tests/compose" mbtest "github.com/elastic/beats/metricbeat/mb/testing" "github.com/elastic/beats/metricbeat/module/ceph/mgrtest" ) @@ -33,11 +31,7 @@ import ( const user = "demo" func TestData(t *testing.T) { - service := compose.EnsureUp(t, "ceph", - compose.UpWithTimeout(120*time.Second), - compose.UpWithEnvironmentVariable("CEPH_CODENAME=nautilus"), - compose.UpWithEnvironmentVariable("CEPH_VERSION=master-97985eb-nautilus-centos-7-x86_64"), - ) + service := mgrtest.EnsureUp(t) f := mbtest.NewReportingMetricSetV2Error(t, getConfig(service.HostForPort(8003), mgrtest.GetPassword(t, service.HostForPort(5000), user))) diff --git a/metricbeat/module/ceph/mgr_osd_perf/mgr_osd_perf_integration_test.go b/metricbeat/module/ceph/mgr_osd_perf/mgr_osd_perf_integration_test.go index 1866de6ced5d..9da9f2dc6313 100644 --- a/metricbeat/module/ceph/mgr_osd_perf/mgr_osd_perf_integration_test.go +++ b/metricbeat/module/ceph/mgr_osd_perf/mgr_osd_perf_integration_test.go @@ -21,11 +21,9 @@ package mgr_osd_perf import ( "testing" - "time" "github.com/stretchr/testify/require" - "github.com/elastic/beats/libbeat/tests/compose" mbtest "github.com/elastic/beats/metricbeat/mb/testing" "github.com/elastic/beats/metricbeat/module/ceph/mgrtest" ) @@ -33,11 +31,7 @@ import ( const user = "demo" func TestData(t *testing.T) { - service := compose.EnsureUp(t, "ceph", - compose.UpWithTimeout(120*time.Second), - compose.UpWithEnvironmentVariable("CEPH_CODENAME=nautilus"), - compose.UpWithEnvironmentVariable("CEPH_VERSION=master-97985eb-nautilus-centos-7-x86_64"), - ) + service := mgrtest.EnsureUp(t) f := mbtest.NewReportingMetricSetV2Error(t, getConfig(service.HostForPort(8003), mgrtest.GetPassword(t, service.HostForPort(5000), user))) diff --git a/metricbeat/module/ceph/mgr_osd_pool_stats/mgr_osd_pool_stats_integration_test.go b/metricbeat/module/ceph/mgr_osd_pool_stats/mgr_osd_pool_stats_integration_test.go index d93576154d67..3bb8ff693e2a 100644 --- a/metricbeat/module/ceph/mgr_osd_pool_stats/mgr_osd_pool_stats_integration_test.go +++ b/metricbeat/module/ceph/mgr_osd_pool_stats/mgr_osd_pool_stats_integration_test.go @@ -21,11 +21,9 @@ package mgr_osd_pool_stats import ( "testing" - "time" "github.com/stretchr/testify/require" - "github.com/elastic/beats/libbeat/tests/compose" mbtest "github.com/elastic/beats/metricbeat/mb/testing" "github.com/elastic/beats/metricbeat/module/ceph/mgrtest" ) @@ -33,11 +31,7 @@ import ( const user = "demo" func TestData(t *testing.T) { - service := compose.EnsureUp(t, "ceph", - compose.UpWithTimeout(120*time.Second), - compose.UpWithEnvironmentVariable("CEPH_CODENAME=nautilus"), - compose.UpWithEnvironmentVariable("CEPH_VERSION=master-97985eb-nautilus-centos-7-x86_64"), - ) + service := mgrtest.EnsureUp(t) f := mbtest.NewReportingMetricSetV2Error(t, getConfig(service.HostForPort(8003), mgrtest.GetPassword(t, service.HostForPort(5000), user))) diff --git a/metricbeat/module/ceph/mgr_osd_tree/mgr_osd_tree_integration_test.go b/metricbeat/module/ceph/mgr_osd_tree/mgr_osd_tree_integration_test.go index e0d57380189c..44a357dc6ffa 100644 --- a/metricbeat/module/ceph/mgr_osd_tree/mgr_osd_tree_integration_test.go +++ b/metricbeat/module/ceph/mgr_osd_tree/mgr_osd_tree_integration_test.go @@ -21,11 +21,9 @@ package mgr_osd_tree import ( "testing" - "time" "github.com/stretchr/testify/require" - "github.com/elastic/beats/libbeat/tests/compose" mbtest "github.com/elastic/beats/metricbeat/mb/testing" "github.com/elastic/beats/metricbeat/module/ceph/mgrtest" ) @@ -33,11 +31,7 @@ import ( const user = "demo" func TestData(t *testing.T) { - service := compose.EnsureUp(t, "ceph", - compose.UpWithTimeout(120*time.Second), - compose.UpWithEnvironmentVariable("CEPH_CODENAME=nautilus"), - compose.UpWithEnvironmentVariable("CEPH_VERSION=master-97985eb-nautilus-centos-7-x86_64"), - ) + service := mgrtest.EnsureUp(t) f := mbtest.NewReportingMetricSetV2Error(t, getConfig(service.HostForPort(8003), mgrtest.GetPassword(t, service.HostForPort(5000), user))) diff --git a/metricbeat/module/ceph/mgr_pool_disk/_meta/data.json b/metricbeat/module/ceph/mgr_pool_disk/_meta/data.json index 730fc6ac899e..53f69b41bf27 100644 --- a/metricbeat/module/ceph/mgr_pool_disk/_meta/data.json +++ b/metricbeat/module/ceph/mgr_pool_disk/_meta/data.json @@ -1,7 +1,7 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", "ceph": { - "pool_disk": { + "mgr_pool_disk": { "id": 0, "name": "rbd", "stats": { @@ -17,7 +17,7 @@ } }, "event": { - "dataset": "ceph.pool_disk", + "dataset": "ceph.mgr_pool_disk", "duration": 115000, "module": "ceph" }, diff --git a/metricbeat/module/ceph/mgr_pool_disk/mgr_pool_disk_integration_test.go b/metricbeat/module/ceph/mgr_pool_disk/mgr_pool_disk_integration_test.go index 421f20ff07d3..1e9394d8cdba 100644 --- a/metricbeat/module/ceph/mgr_pool_disk/mgr_pool_disk_integration_test.go +++ b/metricbeat/module/ceph/mgr_pool_disk/mgr_pool_disk_integration_test.go @@ -21,11 +21,9 @@ package mgr_pool_disk import ( "testing" - "time" "github.com/stretchr/testify/require" - "github.com/elastic/beats/libbeat/tests/compose" mbtest "github.com/elastic/beats/metricbeat/mb/testing" "github.com/elastic/beats/metricbeat/module/ceph/mgrtest" ) @@ -33,11 +31,7 @@ import ( const user = "demo" func TestData(t *testing.T) { - service := compose.EnsureUp(t, "ceph", - compose.UpWithTimeout(120*time.Second), - compose.UpWithEnvironmentVariable("CEPH_CODENAME=nautilus"), - compose.UpWithEnvironmentVariable("CEPH_VERSION=master-97985eb-nautilus-centos-7-x86_64"), - ) + service := mgrtest.EnsureUp(t) f := mbtest.NewReportingMetricSetV2Error(t, getConfig(service.HostForPort(8003), mgrtest.GetPassword(t, service.HostForPort(5000), user))) diff --git a/metricbeat/module/ceph/mgrtest/compose.go b/metricbeat/module/ceph/mgrtest/compose.go new file mode 100644 index 000000000000..02c2b55bbf75 --- /dev/null +++ b/metricbeat/module/ceph/mgrtest/compose.go @@ -0,0 +1,41 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mgrtest + +import ( + "fmt" + "testing" + "time" + + "github.com/elastic/beats/libbeat/tests/compose" +) + +const ( + upTimeout = 120 * time.Second + + defaultCodename = "nautilus" + defaultVersion = "master-97985eb-nautilus-centos-7-x86_64" +) + +func EnsureUp(t *testing.T) compose.HostInfo { + return compose.EnsureUp(t, "ceph", + compose.UpWithTimeout(upTimeout), + compose.UpWithEnvironmentVariable(fmt.Sprintf("CEPH_CODENAME=%s", defaultCodename)), + compose.UpWithEnvironmentVariable(fmt.Sprintf("CEPH_VERSION=%s", defaultVersion)), + ) +} diff --git a/metricbeat/module/ceph/osd_df/osd_df_integration_test.go b/metricbeat/module/ceph/osd_df/osd_df_integration_test.go index 7c828d0bd0c4..8d6ba84664fa 100644 --- a/metricbeat/module/ceph/osd_df/osd_df_integration_test.go +++ b/metricbeat/module/ceph/osd_df/osd_df_integration_test.go @@ -27,7 +27,7 @@ import ( ) func TestData(t *testing.T) { - service := compose.EnsureUpWithTimeout(t, 120, "ceph") + service := compose.EnsureUpWithTimeout(t, 120, "ceph-api") f := mbtest.NewReportingMetricSetV2Error(t, getConfig(service.Host())) diff --git a/metricbeat/module/ceph/osd_tree/osd_tree_integration_test.go b/metricbeat/module/ceph/osd_tree/osd_tree_integration_test.go index 07bb5ab4fbdf..27bd36120a1f 100644 --- a/metricbeat/module/ceph/osd_tree/osd_tree_integration_test.go +++ b/metricbeat/module/ceph/osd_tree/osd_tree_integration_test.go @@ -27,7 +27,7 @@ import ( ) func TestData(t *testing.T) { - service := compose.EnsureUpWithTimeout(t, 120, "ceph") + service := compose.EnsureUpWithTimeout(t, 120, "ceph-api") f := mbtest.NewReportingMetricSetV2Error(t, getConfig(service.Host())) diff --git a/metricbeat/module/ceph/pool_disk/pool_disk_integration_test.go b/metricbeat/module/ceph/pool_disk/pool_disk_integration_test.go index b6a9d9ed3f86..cc1f68cdd81b 100644 --- a/metricbeat/module/ceph/pool_disk/pool_disk_integration_test.go +++ b/metricbeat/module/ceph/pool_disk/pool_disk_integration_test.go @@ -27,7 +27,7 @@ import ( ) func TestData(t *testing.T) { - service := compose.EnsureUpWithTimeout(t, 120, "ceph") + service := compose.EnsureUpWithTimeout(t, 120, "ceph-api") f := mbtest.NewReportingMetricSetV2Error(t, getConfig(service.Host())) diff --git a/metricbeat/module/ceph/test_ceph.py b/metricbeat/module/ceph/test_ceph.py index 546b63967918..9fe207e6d657 100644 --- a/metricbeat/module/ceph/test_ceph.py +++ b/metricbeat/module/ceph/test_ceph.py @@ -1,4 +1,5 @@ import os +import requests import sys import time import unittest @@ -29,16 +30,15 @@ def test_ceph(self, metricset): if not self.old_ceph_version(): self.skipTest("newer ceph version not supported") return - - self.check_metricset("ceph", metricset, self.get_hosts(), self.FIELDS) + self.check_metricset("ceph", metricset, ['http://' + self.compose_host(port='5000/tcp')], self.FIELDS) @parameterized.expand([ "mgr_cluster_disk", "mgr_cluster_health", - "mgr_osd_disk", "mgr_osd_perf", "mgr_osd_pool_stats", - "mgr_osd_tree" + "mgr_osd_tree", + "mgr_pool_disk" ]) @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") def test_ceph_mgr(self, metricset): @@ -50,7 +50,7 @@ def test_ceph_mgr(self, metricset): self.skipTest("legacy ceph version not supported") return - self.render_config_template(modules=[self.get_ceph_module_config(metricset)]) + self.render_config_template(modules=[self.get_ceph_mgr_module_config(metricset)]) proc = self.start_beat(home=self.beat_path) self.wait_until(lambda: self.output_lines() > 0) proc.check_kill_and_wait() @@ -67,12 +67,12 @@ def old_ceph_version(self): return self.COMPOSE_ENV['CEPH_CODENAME'] == 'jewel' - def get_ceph_module_config(self, metricset): + def get_ceph_mgr_module_config(self, metricset): return { 'name': 'ceph', 'metricsets': [metricset], 'period': '1h', - 'hosts': self.get_ceph_mgr_hosts(), + 'hosts': ['https://' + self.compose_host(port='8003/tcp')], 'username': 'demo', 'password': self.get_ceph_mgr_password(), 'extras': { @@ -80,9 +80,6 @@ def get_ceph_module_config(self, metricset): } } - def get_ceph_mgr_hosts(self): - return ['https://' + self.compose_host(port='8003/tcp')] - def get_ceph_mgr_password(self): r = requests.get('http://' + self.compose_host(port='5000/tcp') + '/restful-list-keys.json') keys = r.json() From f9e222d8aeec1359e56c736b71e502d8de6c3b6f Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Tue, 18 Feb 2020 11:31:46 +0100 Subject: [PATCH 38/41] Fix windows: too long path --- .../module/ceph/mgr_cluster_disk/_meta/testdata/config.yml | 1 - .../testdata/{mgr_failed_response.json => failed.json} | 0 ...esponse.json-expected.json => failed.json-expected.json} | 0 .../testdata/{mgr_df_sample_response.json => sample.json} | 0 ...esponse.json-expected.json => sample.json-expected.json} | 0 .../testdata/{mgr_failed_response.json => failed.json} | 0 .../{mgr_status_sample_response.json => status.json} | 0 ...nc_status_sample_response.json => time_sync_status.json} | 0 .../ceph/mgr_cluster_health/mgr_cluster_health_test.go | 6 +++--- .../module/ceph/mgr_osd_perf/_meta/testdata/config.yml | 1 - .../testdata/{mgr_failed_response.json => failed.json} | 0 ...esponse.json-expected.json => failed.json-expected.json} | 0 .../{mgr_osd_perf_sample_response.json => sample.json} | 0 ...esponse.json-expected.json => sample.json-expected.json} | 0 .../ceph/mgr_osd_pool_stats/_meta/testdata/config.yml | 1 - .../testdata/{mgr_failed_response.json => failed.json} | 0 ...esponse.json-expected.json => failed.json-expected.json} | 0 ...{mgr_osd_pool_stats_sample_response.json => sample.json} | 0 ...esponse.json-expected.json => sample.json-expected.json} | 0 .../module/ceph/mgr_osd_tree/_meta/testdata/config.yml | 1 - .../testdata/{mgr_failed_response.json => failed.json} | 0 ...esponse.json-expected.json => failed.json-expected.json} | 0 .../{mgr_osd_tree_sample_response.json => sample.json} | 0 ...esponse.json-expected.json => sample.json-expected.json} | 0 .../module/ceph/mgr_pool_disk/_meta/testdata/config.yml | 1 - .../testdata/{mgr_failed_response.json => failed.json} | 0 ...esponse.json-expected.json => failed.json-expected.json} | 0 .../testdata/{mgr_df_sample_response.json => sample.json} | 0 ...esponse.json-expected.json => sample.json-expected.json} | 0 29 files changed, 3 insertions(+), 8 deletions(-) rename metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/{mgr_failed_response.json => failed.json} (100%) rename metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/{mgr_failed_response.json-expected.json => failed.json-expected.json} (100%) rename metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/{mgr_df_sample_response.json => sample.json} (100%) rename metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/{mgr_df_sample_response.json-expected.json => sample.json-expected.json} (100%) rename metricbeat/module/ceph/mgr_cluster_health/_meta/testdata/{mgr_failed_response.json => failed.json} (100%) rename metricbeat/module/ceph/mgr_cluster_health/_meta/testdata/{mgr_status_sample_response.json => status.json} (100%) rename metricbeat/module/ceph/mgr_cluster_health/_meta/testdata/{mgr_time_sync_status_sample_response.json => time_sync_status.json} (100%) rename metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/{mgr_failed_response.json => failed.json} (100%) rename metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/{mgr_failed_response.json-expected.json => failed.json-expected.json} (100%) rename metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/{mgr_osd_perf_sample_response.json => sample.json} (100%) rename metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/{mgr_osd_perf_sample_response.json-expected.json => sample.json-expected.json} (100%) rename metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/{mgr_failed_response.json => failed.json} (100%) rename metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/{mgr_failed_response.json-expected.json => failed.json-expected.json} (100%) rename metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/{mgr_osd_pool_stats_sample_response.json => sample.json} (100%) rename metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/{mgr_osd_pool_stats_sample_response.json-expected.json => sample.json-expected.json} (100%) rename metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/{mgr_failed_response.json => failed.json} (100%) rename metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/{mgr_failed_response.json-expected.json => failed.json-expected.json} (100%) rename metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/{mgr_osd_tree_sample_response.json => sample.json} (100%) rename metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/{mgr_osd_tree_sample_response.json-expected.json => sample.json-expected.json} (100%) rename metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/{mgr_failed_response.json => failed.json} (100%) rename metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/{mgr_failed_response.json-expected.json => failed.json-expected.json} (100%) rename metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/{mgr_df_sample_response.json => sample.json} (100%) rename metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/{mgr_df_sample_response.json-expected.json => sample.json-expected.json} (100%) diff --git a/metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/config.yml b/metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/config.yml index 9dad1228403b..fdc68400c578 100644 --- a/metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/config.yml +++ b/metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/config.yml @@ -1,4 +1,3 @@ type: http url: "/request?wait=1" suffix: json -remove_fields_from_comparison: diff --git a/metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/mgr_failed_response.json b/metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/failed.json similarity index 100% rename from metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/mgr_failed_response.json rename to metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/failed.json diff --git a/metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/mgr_failed_response.json-expected.json b/metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/failed.json-expected.json similarity index 100% rename from metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/mgr_failed_response.json-expected.json rename to metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/failed.json-expected.json diff --git a/metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/mgr_df_sample_response.json b/metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/sample.json similarity index 100% rename from metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/mgr_df_sample_response.json rename to metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/sample.json diff --git a/metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/mgr_df_sample_response.json-expected.json b/metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/sample.json-expected.json similarity index 100% rename from metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/mgr_df_sample_response.json-expected.json rename to metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/sample.json-expected.json diff --git a/metricbeat/module/ceph/mgr_cluster_health/_meta/testdata/mgr_failed_response.json b/metricbeat/module/ceph/mgr_cluster_health/_meta/testdata/failed.json similarity index 100% rename from metricbeat/module/ceph/mgr_cluster_health/_meta/testdata/mgr_failed_response.json rename to metricbeat/module/ceph/mgr_cluster_health/_meta/testdata/failed.json diff --git a/metricbeat/module/ceph/mgr_cluster_health/_meta/testdata/mgr_status_sample_response.json b/metricbeat/module/ceph/mgr_cluster_health/_meta/testdata/status.json similarity index 100% rename from metricbeat/module/ceph/mgr_cluster_health/_meta/testdata/mgr_status_sample_response.json rename to metricbeat/module/ceph/mgr_cluster_health/_meta/testdata/status.json diff --git a/metricbeat/module/ceph/mgr_cluster_health/_meta/testdata/mgr_time_sync_status_sample_response.json b/metricbeat/module/ceph/mgr_cluster_health/_meta/testdata/time_sync_status.json similarity index 100% rename from metricbeat/module/ceph/mgr_cluster_health/_meta/testdata/mgr_time_sync_status_sample_response.json rename to metricbeat/module/ceph/mgr_cluster_health/_meta/testdata/time_sync_status.json diff --git a/metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health_test.go b/metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health_test.go index 2e8f53536734..d280566f0459 100644 --- a/metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health_test.go +++ b/metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health_test.go @@ -41,9 +41,9 @@ func TestFetchEventContents(t *testing.T) { absPath, err := filepath.Abs("./_meta/testdata/") assert.NoError(t, err) - statusResponse, err := ioutil.ReadFile(absPath + "/mgr_status_sample_response.json") + statusResponse, err := ioutil.ReadFile(absPath + "/status.json") assert.NoError(t, err) - timeSyncStatusResponse, err := ioutil.ReadFile(absPath + "/mgr_time_sync_status_sample_response.json") + timeSyncStatusResponse, err := ioutil.ReadFile(absPath + "/time_sync_status.json") assert.NoError(t, err) server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -93,7 +93,7 @@ func TestFetchEventContents_Failed(t *testing.T) { absPath, err := filepath.Abs("./_meta/testdata/") assert.NoError(t, err) - response, err := ioutil.ReadFile(absPath + "/mgr_failed_response.json") + response, err := ioutil.ReadFile(absPath + "/failed.json") assert.NoError(t, err) server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/config.yml b/metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/config.yml index 9dad1228403b..fdc68400c578 100644 --- a/metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/config.yml +++ b/metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/config.yml @@ -1,4 +1,3 @@ type: http url: "/request?wait=1" suffix: json -remove_fields_from_comparison: diff --git a/metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/mgr_failed_response.json b/metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/failed.json similarity index 100% rename from metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/mgr_failed_response.json rename to metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/failed.json diff --git a/metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/mgr_failed_response.json-expected.json b/metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/failed.json-expected.json similarity index 100% rename from metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/mgr_failed_response.json-expected.json rename to metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/failed.json-expected.json diff --git a/metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/mgr_osd_perf_sample_response.json b/metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/sample.json similarity index 100% rename from metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/mgr_osd_perf_sample_response.json rename to metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/sample.json diff --git a/metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/mgr_osd_perf_sample_response.json-expected.json b/metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/sample.json-expected.json similarity index 100% rename from metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/mgr_osd_perf_sample_response.json-expected.json rename to metricbeat/module/ceph/mgr_osd_perf/_meta/testdata/sample.json-expected.json diff --git a/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/config.yml b/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/config.yml index 9dad1228403b..fdc68400c578 100644 --- a/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/config.yml +++ b/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/config.yml @@ -1,4 +1,3 @@ type: http url: "/request?wait=1" suffix: json -remove_fields_from_comparison: diff --git a/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/mgr_failed_response.json b/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/failed.json similarity index 100% rename from metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/mgr_failed_response.json rename to metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/failed.json diff --git a/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/mgr_failed_response.json-expected.json b/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/failed.json-expected.json similarity index 100% rename from metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/mgr_failed_response.json-expected.json rename to metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/failed.json-expected.json diff --git a/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/mgr_osd_pool_stats_sample_response.json b/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/sample.json similarity index 100% rename from metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/mgr_osd_pool_stats_sample_response.json rename to metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/sample.json diff --git a/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/mgr_osd_pool_stats_sample_response.json-expected.json b/metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/sample.json-expected.json similarity index 100% rename from metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/mgr_osd_pool_stats_sample_response.json-expected.json rename to metricbeat/module/ceph/mgr_osd_pool_stats/_meta/testdata/sample.json-expected.json diff --git a/metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/config.yml b/metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/config.yml index 9dad1228403b..fdc68400c578 100644 --- a/metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/config.yml +++ b/metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/config.yml @@ -1,4 +1,3 @@ type: http url: "/request?wait=1" suffix: json -remove_fields_from_comparison: diff --git a/metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/mgr_failed_response.json b/metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/failed.json similarity index 100% rename from metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/mgr_failed_response.json rename to metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/failed.json diff --git a/metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/mgr_failed_response.json-expected.json b/metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/failed.json-expected.json similarity index 100% rename from metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/mgr_failed_response.json-expected.json rename to metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/failed.json-expected.json diff --git a/metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/mgr_osd_tree_sample_response.json b/metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/sample.json similarity index 100% rename from metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/mgr_osd_tree_sample_response.json rename to metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/sample.json diff --git a/metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/mgr_osd_tree_sample_response.json-expected.json b/metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/sample.json-expected.json similarity index 100% rename from metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/mgr_osd_tree_sample_response.json-expected.json rename to metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/sample.json-expected.json diff --git a/metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/config.yml b/metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/config.yml index 9dad1228403b..fdc68400c578 100644 --- a/metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/config.yml +++ b/metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/config.yml @@ -1,4 +1,3 @@ type: http url: "/request?wait=1" suffix: json -remove_fields_from_comparison: diff --git a/metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/mgr_failed_response.json b/metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/failed.json similarity index 100% rename from metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/mgr_failed_response.json rename to metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/failed.json diff --git a/metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/mgr_failed_response.json-expected.json b/metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/failed.json-expected.json similarity index 100% rename from metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/mgr_failed_response.json-expected.json rename to metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/failed.json-expected.json diff --git a/metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/mgr_df_sample_response.json b/metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/sample.json similarity index 100% rename from metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/mgr_df_sample_response.json rename to metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/sample.json diff --git a/metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/mgr_df_sample_response.json-expected.json b/metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/sample.json-expected.json similarity index 100% rename from metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/mgr_df_sample_response.json-expected.json rename to metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/sample.json-expected.json From 481ff0a4a88071a817257a4507198a1b545657c5 Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Tue, 18 Feb 2020 11:39:26 +0100 Subject: [PATCH 39/41] Fix --- libbeat/tests/compose/project.go | 2 +- metricbeat/module/ceph/docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libbeat/tests/compose/project.go b/libbeat/tests/compose/project.go index ba7413bcbcce..5ef0422e1160 100644 --- a/libbeat/tests/compose/project.go +++ b/libbeat/tests/compose/project.go @@ -73,7 +73,7 @@ func UpWithAdvertisedHostEnvFileForPort(port int) UpOption { } } -// UpWithEnvironmentVariable add environment variable. +// UpWithEnvironmentVariable adds environment variable. func UpWithEnvironmentVariable(keyValue string) UpOption { return func(options *UpOptions) { options.Environment = append(options.Environment, keyValue) } } diff --git a/metricbeat/module/ceph/docker-compose.yml b/metricbeat/module/ceph/docker-compose.yml index 181dfd6ffbe4..1fd436942602 100644 --- a/metricbeat/module/ceph/docker-compose.yml +++ b/metricbeat/module/ceph/docker-compose.yml @@ -1,4 +1,4 @@ -version: '3.7' +version: '2.3' services: ceph: From 0672054a723203e09fdc3cc890e121f87ca794b9 Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Tue, 18 Feb 2020 20:11:47 +0100 Subject: [PATCH 40/41] Another review round --- libbeat/tests/compose/project.go | 7 -- libbeat/tests/compose/wrapper.go | 6 -- metricbeat/docs/fields.asciidoc | 66 +++++++++---------- metricbeat/module/ceph/docker-compose.yml | 2 +- metricbeat/module/ceph/fields.go | 2 +- .../ceph/mgr_cluster_disk/_meta/data.json | 2 +- .../ceph/mgr_cluster_disk/_meta/fields.yml | 4 +- .../_meta/testdata/sample.json-expected.json | 2 +- .../ceph/mgr_cluster_disk/mgr_cluster_disk.go | 6 +- .../mgr_cluster_disk_integration_test.go | 3 +- .../ceph/mgr_cluster_health/_meta/data.json | 2 +- .../ceph/mgr_cluster_health/_meta/fields.yml | 4 +- .../mgr_cluster_health/mgr_cluster_health.go | 6 +- .../mgr_cluster_health_integration_test.go | 3 +- .../mgr_cluster_health_test.go | 2 +- .../mgr_osd_perf_integration_test.go | 3 +- .../mgr_osd_pool_stats_integration_test.go | 3 +- .../module/ceph/mgr_osd_tree/_meta/data.json | 2 +- .../module/ceph/mgr_osd_tree/_meta/fields.yml | 2 +- .../_meta/testdata/sample.json-expected.json | 8 +-- .../module/ceph/mgr_osd_tree/mgr_osd_tree.go | 6 +- .../mgr_osd_tree_integration_test.go | 3 +- .../module/ceph/mgr_pool_disk/_meta/data.json | 2 +- .../ceph/mgr_pool_disk/_meta/fields.yml | 4 +- .../_meta/testdata/sample.json-expected.json | 16 ++--- .../ceph/mgr_pool_disk/mgr_pool_disk.go | 6 +- .../mgr_pool_disk_integration_test.go | 3 +- metricbeat/module/ceph/mgrtest/compose.go | 41 ------------ 28 files changed, 92 insertions(+), 124 deletions(-) delete mode 100644 metricbeat/module/ceph/mgrtest/compose.go diff --git a/libbeat/tests/compose/project.go b/libbeat/tests/compose/project.go index 5ef0422e1160..f4ebfb904e58 100644 --- a/libbeat/tests/compose/project.go +++ b/libbeat/tests/compose/project.go @@ -45,8 +45,6 @@ type UpOptions struct { // use to advertise itself to clients SetupAdvertisedHostEnvFile bool SetupAdvertisedHostEnvFilePort int - - Environment []string } // UpOption is a modifier for UpOptions @@ -73,11 +71,6 @@ func UpWithAdvertisedHostEnvFileForPort(port int) UpOption { } } -// UpWithEnvironmentVariable adds environment variable. -func UpWithEnvironmentVariable(keyValue string) UpOption { - return func(options *UpOptions) { options.Environment = append(options.Environment, keyValue) } -} - // Filter options for services type Filter struct { State State diff --git a/libbeat/tests/compose/wrapper.go b/libbeat/tests/compose/wrapper.go index 1a663a5cd71d..35e37098a38c 100644 --- a/libbeat/tests/compose/wrapper.go +++ b/libbeat/tests/compose/wrapper.go @@ -181,12 +181,6 @@ func (d *wrapperDriver) Up(ctx context.Context, opts UpOptions, service string) args = append(args, "--force-recreate") } - if len(opts.Environment) > 0 { - for _, keyValue := range opts.Environment { - d.Environment = append(d.Environment, keyValue) - } - } - if service != "" { args = append(args, service) } diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 1b91ecb56e47..b408b4e5c92a 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -3933,13 +3933,13 @@ type: long -- [float] -=== mgr_cluster_disk +=== cluster_disk -mgr_cluster_disk +cluster_disk -*`ceph.mgr_cluster_disk.available.bytes`*:: +*`ceph.cluster_disk.available.bytes`*:: + -- Available bytes of the cluster @@ -3951,7 +3951,7 @@ format: bytes -- -*`ceph.mgr_cluster_disk.total.bytes`*:: +*`ceph.cluster_disk.total.bytes`*:: + -- Total bytes of the cluster @@ -3963,7 +3963,7 @@ format: bytes -- -*`ceph.mgr_cluster_disk.used.bytes`*:: +*`ceph.cluster_disk.used.bytes`*:: + -- Used bytes of the cluster @@ -3976,13 +3976,13 @@ format: bytes -- [float] -=== mgr_cluster_health +=== cluster_health -mgr_cluster_health +cluster_health -*`ceph.mgr_cluster_health.overall_status`*:: +*`ceph.cluster_health.overall_status`*:: + -- Overall status of the cluster @@ -3992,7 +3992,7 @@ type: keyword -- -*`ceph.mgr_cluster_health.timechecks.epoch`*:: +*`ceph.cluster_health.timechecks.epoch`*:: + -- Map version @@ -4002,7 +4002,7 @@ type: long -- -*`ceph.mgr_cluster_health.timechecks.round.value`*:: +*`ceph.cluster_health.timechecks.round.value`*:: + -- timecheck round @@ -4012,7 +4012,7 @@ type: long -- -*`ceph.mgr_cluster_health.timechecks.round.status`*:: +*`ceph.cluster_health.timechecks.round.status`*:: + -- Status of the round @@ -4109,13 +4109,13 @@ type: object -- [float] -=== mgr_osd_tree +=== osd_tree OSD tree information -*`ceph.mgr_osd_tree.id`*:: +*`ceph.osd_tree.id`*:: + -- OSD or bucket node ID @@ -4125,7 +4125,7 @@ type: long -- -*`ceph.mgr_osd_tree.name`*:: +*`ceph.osd_tree.name`*:: + -- OSD or bucket node name @@ -4135,7 +4135,7 @@ type: keyword -- -*`ceph.mgr_osd_tree.type`*:: +*`ceph.osd_tree.type`*:: + -- OSD or bucket node type, illegal type include OSD, host, root etc. @@ -4145,7 +4145,7 @@ type: keyword -- -*`ceph.mgr_osd_tree.type_id`*:: +*`ceph.osd_tree.type_id`*:: + -- OSD or bucket node type ID @@ -4155,7 +4155,7 @@ type: long -- -*`ceph.mgr_osd_tree.children`*:: +*`ceph.osd_tree.children`*:: + -- Bucket children list, separated by comma. @@ -4165,7 +4165,7 @@ type: keyword -- -*`ceph.mgr_osd_tree.crush_weight`*:: +*`ceph.osd_tree.crush_weight`*:: + -- OSD node CRUSH weight @@ -4175,7 +4175,7 @@ type: float -- -*`ceph.mgr_osd_tree.depth`*:: +*`ceph.osd_tree.depth`*:: + -- Node depth @@ -4185,7 +4185,7 @@ type: long -- -*`ceph.mgr_osd_tree.exists`*:: +*`ceph.osd_tree.exists`*:: + -- Node exists (1-yes, 0-no) @@ -4195,7 +4195,7 @@ type: boolean -- -*`ceph.mgr_osd_tree.primary_affinity`*:: +*`ceph.osd_tree.primary_affinity`*:: + -- Weight of reading data from primary OSD @@ -4205,7 +4205,7 @@ type: float -- -*`ceph.mgr_osd_tree.reweight`*:: +*`ceph.osd_tree.reweight`*:: + -- Reweight of osd @@ -4215,7 +4215,7 @@ type: long -- -*`ceph.mgr_osd_tree.status`*:: +*`ceph.osd_tree.status`*:: + -- Status of OSD, it should be up or down @@ -4225,7 +4225,7 @@ type: keyword -- -*`ceph.mgr_osd_tree.device_class`*:: +*`ceph.osd_tree.device_class`*:: + -- Device class of OSD, like HDD, SSD etc. @@ -4235,7 +4235,7 @@ type: keyword -- -*`ceph.mgr_osd_tree.father`*:: +*`ceph.osd_tree.father`*:: + -- Parent node of this OSD or bucket node @@ -4246,13 +4246,13 @@ type: keyword -- [float] -=== mgr_pool_disk +=== pool_disk -mgr_pool_disk +pool_disk -*`ceph.mgr_pool_disk.id`*:: +*`ceph.pool_disk.id`*:: + -- Id of the pool @@ -4262,7 +4262,7 @@ type: long -- -*`ceph.mgr_pool_disk.name`*:: +*`ceph.pool_disk.name`*:: + -- Name of the pool @@ -4272,7 +4272,7 @@ type: keyword -- -*`ceph.mgr_pool_disk.stats.available.bytes`*:: +*`ceph.pool_disk.stats.available.bytes`*:: + -- Available bytes of the pool @@ -4284,7 +4284,7 @@ format: bytes -- -*`ceph.mgr_pool_disk.stats.objects`*:: +*`ceph.pool_disk.stats.objects`*:: + -- Number of objects of the pool @@ -4294,7 +4294,7 @@ type: long -- -*`ceph.mgr_pool_disk.stats.used.bytes`*:: +*`ceph.pool_disk.stats.used.bytes`*:: + -- Used bytes of the pool @@ -4306,7 +4306,7 @@ format: bytes -- -*`ceph.mgr_pool_disk.stats.used.kb`*:: +*`ceph.pool_disk.stats.used.kb`*:: + -- Used kb of the pool diff --git a/metricbeat/module/ceph/docker-compose.yml b/metricbeat/module/ceph/docker-compose.yml index 1fd436942602..e381bed4080c 100644 --- a/metricbeat/module/ceph/docker-compose.yml +++ b/metricbeat/module/ceph/docker-compose.yml @@ -2,7 +2,7 @@ version: '2.3' services: ceph: - image: docker.elastic.co/integrations-ci/beats-ceph:${CEPH_VERSION:-master-97985eb-nautilus-centos-7-x86_64}-1 + image: docker.elastic.co/integrations-ci/beats-ceph:${CEPH_VERSION:-master-97985eb-nautilus-centos-7-x86_64}-2 build: context: ./_meta dockerfile: Dockerfile.${CEPH_CODENAME:-nautilus} diff --git a/metricbeat/module/ceph/fields.go b/metricbeat/module/ceph/fields.go index b280e18c15a4..a9f3cc856f8c 100644 --- a/metricbeat/module/ceph/fields.go +++ b/metricbeat/module/ceph/fields.go @@ -32,5 +32,5 @@ func init() { // AssetCeph returns asset data. // This is the base64 encoded gzipped contents of ../metricbeat/module/ceph. func AssetCeph() string { - return "eJzsWk+P274Rve+nGOypBTZue/WhQJINkEWbZLH+Bb9DUSi0OJbYpUiCpNb1ty9IybJXov6sTctFuzoltvfN43BmOPPED/CMuyWkqPIbAMssxyXcfkaV394AUDSpZsoyKZbw1xsAAPcVFJKWHG8ATC61TVIpNixbwoZw4z7VyJEYXEJG3G/QWiYys4R/3BrDb+/gNrdW3f7zBmDDkFOz9MgfQJACGy7usTvlULQsVf1JgJF7frk/+gWpFJYwYcDmCAVazVL3b2JhixrBpJoopLDRsoDPXx6/LmqAYxqvqPDSWNQJZea5+TJEa4Cae3pwXvtp/7TJHBMiL4Rxsua4WO8smle/2fPiUmStLwaouefjHhU8KsiNd2DNuvXzjdQFsUvoEtiTtNISHpXgbw4xDrnSII3K7adBejq1dqTlSLjNY8RaB+nt0SZfUBPOE2OJLcP+esbdVmr6Npf9qHChwh1yWxNSrMA0x/TZLFDJNI+0d9+IghfUhkkxZlbLUtDFC+ElRjLegIPHnkYg9lasXm3BayLt2OzYDsdmOM4mRGwHfyg2Q9t2+l74U632xWBAaLLZsHShkdDkTVWkvzKNk6v8A84o2FzLMstVaUGhBoOp7IudmutWM4uzk/VWT2DrPStVolxAYBprf49dyKQyb3PexfhUXppCqGBGcZLiwp+ukWnswUFlIMpi3VOFGw5y/S9MbawTtMOihp9ERRPLZJCISQlHmmy4JLYnvhXqFEX727fS7VLYs6SYaUKRXmTT9uAjm9ZwuMymNSwmbFpD5Xqb1tDt3zSVLSixZPaKqTJwdgd6V5Ut/AAwOzXyekDoI+ejfHZy9jAc9BFzTf/svMpmMOih5Rse9C0dJu7DSKn5mPnOGo9/N0whlWUnoU61vsrl1kAut1AQsQOVGSAagYmalNyMLr3DL3KzV2+QGe/3pKGLTcnDpXstJUfSZjVi/ME4UOiAHlsUSPRFrDrgEdNlkUhDY50SrWBwyC4Sxsa9PZNSXZqMC01ZEWrC8+fjICkmrkHq4fsgKY0FUQpporK5mT19+fbx8fHLfS+/mBO7x2r3GE1TmOkklmQ3gNUMuGu078Ld/7lwdxwn54t3g2inxN27hPcu4dVRKg1NFOrNOfH5Y3Xv5i+fEiI9vGORm0pC6wbAKUHL2h6Ytj+O3EP4GHI+N4tUFgWzCScWRbpLitMqyWePAjWKa2eKcMWqjBKl+O5cmx8dyGSTrXWKWOsU09d5os3OOkW44vpYlrKqa+OS9FhES1lVstiB7Cn2TnYTsv3RMeufjxz8icnikXuyJeUMhU2YTDSxYe6V2jMYPh4EHv70AxxKOx5CbjvmcFDYA7Lr4DLdU7t2CZ/8+e5l3x6B9djokVQewapDsyimGB5WvacbfnILlQq9uiV6ReXumiPY/t1r2cPG2zlsNeK52eswgImqTXvdBsx4+ARapNU9SA3rMn1GC0JS7Mu3cyrEJLu9FcSZuahlh3YHjHPMCPf/AyZSXlJ0v76DXBp7B1pKC2jTRS/JU8vcZJK9tTBnnGoMa08neelTZXkPDJw5FxhUxNVJN5KAO8BJ2BmpLk2ebJFleVixC0nnE1ziHfH56efqKwTADwq+srE69+/OYhdwbwr/zUzP24qTJDBvrgKFP/zlww7NHfz5g5B/DB+tmhVE7xKy2TDB7C6Wq3/3vnVthiv5TNSav7+RVNt02xHkpHFg29/u/6cazpGRJjy8XG5W8dnPLJhclpzCGqFULi+p3IYHOYovLMUk5cREJHTvUcGjNrQ4e0b4en9/B6vVfX9d2hCbd9SCM7g8Eu06Jp+JfppjJlCtgmep7wRjSF4hoKueog90P9k6ajMcn99JgaMm65FnTlGvw2WKaFbxjPvm97sXX33VqF/5TvPWDALeGT7y7J7XMak9r4O8mryVglk5WTV887Wq1/j1fOtOm4kpfIhu1Zn0zo/t+i3+3kPffoTfb3ScA+dl99fKGSNWD0uPFhGHlf/t05j5SoePZroS4cfNXiAFxo1yYmxSKupa36Bl2pUgxtbLCoRtjgK2xEAI+/InR99yjZUaK9VqwWUWtR7+XWaHctgmMK0SHsgVzKRR2X1jJo1Hzxgbld1q9Vs8chd/j3YuwdGkOyH4iLGdXDt6C5zQ8RcPb785jCr3FxtcxwqlIVmfEjRLr+qvWLjunc1RbhprverOZUamxu6AtpNTegfG0AFRp8mRqc4/40pWEyLVXbEXycselzVN6pysqsAdYHXioBGD2+Hq3wA/lSWiLCJlkWlfIAMuU6+M+ZsnzPSqFX7z+prVS90sbe1ijUGy7qAeQ/BuKl6jeo/Wz9lKX0tVna0IXkfsDlgeKIjS0GuI3T0kZ9O61/91WndzennwubRuMa/WzUxl0VjGeYXuwkBIez3l201F2wnqd19xj6x++7syV1DADxemfEm4vgLuHEFbKrin5lXw8UYutgru9bKAEt4tZZ0DNooKPqiAz3+wvuvf7/r3/4b+/Z8AAAD//5gqAPQ=" + return "eJzsms+P27oRx+/7Vwz21AIbt736UCDJBsiiTbJYv+AdikKhxZHELkUSJLWu//uClCzb+m2blouH1SmxvTMfDodDzlf8AK+4XUKMKrsDsMxyXML9Z1TZ/R0ARRNrpiyTYgl/vwMAcF9BLmnB8Q7AZFLbKJYiYekSEsKN+1QjR2JwCSlxv0FrmUjNEv51bwy/f4D7zFp1/+87gIQhp2bpLX8AQXKsWdxjt8pZ0bJQ1ScdRO755f7oF8RSWMKEAZsh5Gg1i92/iYUNagQTa6KQQqJlDp+/PH9dVAYOMY5QeGEs6ogy81p/2YU1gOaeHjvHcdo9TZhDIPJGGCdrjov11qI5+s2Oi0uRNr4YQHPPx51V8FZBJj6AFXXj54nUObFLaAPsIK20hAcF/M1ZDANXGKRB2X4apOejNTMtQ8JtFiLXWpZOzzb5hppwHhlLbNEdr1fcbqSmp4XsR2kXSrtDYatTiuUYZxi/mgUqGWeB5u4bUfCG2jApxtxqWQi6eCO8wEDOa+PgbU8DCD0Vq6MpOAZp5mbLd3dudufZhIxt2R/Kza5pO38u/K5WxWIwITRJEhYvNBIanVRF+ivTOFwZH3BOwWZaFmmmCgsKNRiMZV/uVKwbzSzODuu9nkHrIytVpFxCYBxqfg9DyKQypwXvajxllKYA5cwoTmJc+N01MMbOOKgURJGve6pwzSDX/8HYhtpBWxSV+UkomlgmO0FMTDjSKOGS2J78VqhjFM1vT8VtI+woKaaaUKRXmbSd8ZFJqxmuM2k1xYRJq1FuN2k1bv+kqXRBiSWzV0yVgvM7cHZV6cI3ALOjkeMGoQ/OZ/nscHbfHPSBuUP/7FxF3Rj0YPkDD/ojHUbuw0BL8zn1J2s8/N0wQiyL1oI61/sqkxsDmdxATsQWVGqAaAQmKiiZjA69xRf4sFdNkBk/70lDF0nBu0v3WkqOpEk14vzJOKPQMnroUSDRV/HqDI+4LvJIGhpql2gkg7PsMmGs3duRFOraMC41ZQlUp+fP50EoJm4B9fR9EEpjTpRCGql0brKXL98+Pj9/eezlC9mxe1vNM8bcct0a7btg9y7YzSbYnZBv75Ldu2Tn2uNUu00qUqiTS3Lzx+rR9Vt+KYh4/05FJqVk1k6Ac5KWNSMwbX4c3FP3tuNibhaxzHNmI04singb5edVkM/eClRW3PEl765UpVOiFN9e6vOjMzLZZWOcItQ4xfRxnumzNU7RrrR1LktZ1rVxCXoso6UsK1noRPaIvZ3chNX+7Mj6+yFn/szF4i33rJaYMxQ2YjLSxHazl+rOYPp4I/D0lx/grDTzoStshwx7Rb1DZh0cpnuq0C7hk9/XvczbI6geOj2QxgN4ddYsiimOh1Xu6Y5f3EClQq9miV4RuT3mAL5/99r1sPODNiCyGvHSletsABPl0ez4CDDjxtNxPFo9gtSwLuJXtCAkxb61dkl1mOS3t3o4N1f17Kw9AOMcU8L9/4CJmBcU3a8fIJPGPoCW0gLaeNELeW6JmwzZWwczxqnGbp3prCh9Kj3vDANnLgQGFXE10rUh4DZv0h2MWBcmizbI0qxbneuSySeExAfi88vP1VfoML5X65UNdWr/7jy2De5c4X+Z6XkzcZbc5d2VRuFPf/uwRfMAf/0g5J+7t1XNcqK3EUkSJpjdhgr17z627ojhyj0Tlb7vbx9VPt10dDJpHJj20+P/UplzMNJ0Ny7X61P86mcWTCYLTmGNUCi3LqncdDdxFN9YjFHMiQkI9OitgrdaY3H2ivD18fEBVqvH/rqUEJu1FIILWJ6JdqclvxJ9J8dMR7Vq7aP+BHipvNVl5Ka75xPddbMObYZt8zvJcdRl1ebMKeC1WKYIZCVn2Le7373A6qtF9Vp3WrRmEOsuiJGne12HRHtdd3LVvasUzMrJKuHJV6eO7Vc9rdtlJi7hfXarVnd3eW5Xb+p3Efr2o/sdRis4cNnq/loGY8TrfujBMmI/8n98GnNfau7BXJeC+7jbKyyBcaecGBsVirojb6dn2pYdxsbLcoRNhgI2xECX7evvHH3DNVZqLJWqBZdp0Hr4T5nuy2ETYFol3MPlzMRB6b4xE4fDM8YGpVutfgsHd/V3ZpcCji66M5KPGNtaa4cSDx1/2XD67WBUmb+84E6sUBiS9ilAs5xV/TUKd2pnc5Sb2luvqnOdVqn2O6DpZJQ+gDF0QMyp18jU4F9w7apOkfI+2JvkRU/I6kPqnFRl4g5QndlohGDbX+8b4FNpJIo80CoyzUtiwGXsFTF/u4SZXpXCT17fYfVat0cbs1jZIGm7QQ8hdNcVr1a7R+vnbKWvoabOVgRvI3J3eB4oiNLQW4jcPZCzadzr/zuNu969vPG5NG4xr8bNTOnRWMZ5ad2lgZD2doq364o2E1TvvuIeWPX292NuoHzvL0n5knB75dsFgjbUb4/m1e/xg1xo9dvrZR0KeLuUtTbY6yvg82+s7/r3u/79x9C//xcAAP//xbT1Yg==" } diff --git a/metricbeat/module/ceph/mgr_cluster_disk/_meta/data.json b/metricbeat/module/ceph/mgr_cluster_disk/_meta/data.json index 94c79adb4b49..1ca708c45eaf 100644 --- a/metricbeat/module/ceph/mgr_cluster_disk/_meta/data.json +++ b/metricbeat/module/ceph/mgr_cluster_disk/_meta/data.json @@ -5,7 +5,7 @@ "name": "host.example.com" }, "ceph": { - "mgr_cluster_disk": { + "cluster_disk": { "available": { "bytes": 0 }, diff --git a/metricbeat/module/ceph/mgr_cluster_disk/_meta/fields.yml b/metricbeat/module/ceph/mgr_cluster_disk/_meta/fields.yml index d24637a1fea0..fe755709362a 100644 --- a/metricbeat/module/ceph/mgr_cluster_disk/_meta/fields.yml +++ b/metricbeat/module/ceph/mgr_cluster_disk/_meta/fields.yml @@ -1,7 +1,7 @@ -- name: mgr_cluster_disk +- name: cluster_disk type: group description: > - mgr_cluster_disk + cluster_disk release: beta fields: - name: available.bytes diff --git a/metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/sample.json-expected.json b/metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/sample.json-expected.json index abc17d3539ab..bc10caba3430 100644 --- a/metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/sample.json-expected.json +++ b/metricbeat/module/ceph/mgr_cluster_disk/_meta/testdata/sample.json-expected.json @@ -1,3 +1,3 @@ [ - {"ceph":{"mgr_cluster_disk":{"available":{"bytes":9659875328},"total":{"bytes":10737418240},"used":{"bytes":3801088}}},"event":{"dataset":"ceph.mgr_cluster_disk","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_cluster_disk","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}} + {"ceph":{"cluster_disk":{"available":{"bytes":9659875328},"total":{"bytes":10737418240},"used":{"bytes":3801088}}},"event":{"dataset":"ceph.mgr_cluster_disk","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_cluster_disk","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}} ] diff --git a/metricbeat/module/ceph/mgr_cluster_disk/mgr_cluster_disk.go b/metricbeat/module/ceph/mgr_cluster_disk/mgr_cluster_disk.go index be2e0caa01aa..0cb42828faf7 100644 --- a/metricbeat/module/ceph/mgr_cluster_disk/mgr_cluster_disk.go +++ b/metricbeat/module/ceph/mgr_cluster_disk/mgr_cluster_disk.go @@ -18,6 +18,7 @@ package mgr_cluster_disk import ( + "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/metricbeat/mb" "github.com/elastic/beats/metricbeat/mb/parse" "github.com/elastic/beats/metricbeat/module/ceph/mgr" @@ -72,6 +73,9 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { return err } - reporter.Event(mb.Event{MetricSetFields: event}) + reporter.Event(mb.Event{ + ModuleFields: common.MapStr{ + "cluster_disk": event, + }}) return nil } diff --git a/metricbeat/module/ceph/mgr_cluster_disk/mgr_cluster_disk_integration_test.go b/metricbeat/module/ceph/mgr_cluster_disk/mgr_cluster_disk_integration_test.go index 14677668defb..47f0fa15e4df 100644 --- a/metricbeat/module/ceph/mgr_cluster_disk/mgr_cluster_disk_integration_test.go +++ b/metricbeat/module/ceph/mgr_cluster_disk/mgr_cluster_disk_integration_test.go @@ -24,6 +24,7 @@ import ( "github.com/stretchr/testify/require" + "github.com/elastic/beats/libbeat/tests/compose" mbtest "github.com/elastic/beats/metricbeat/mb/testing" "github.com/elastic/beats/metricbeat/module/ceph/mgrtest" ) @@ -31,7 +32,7 @@ import ( const user = "demo" func TestData(t *testing.T) { - service := mgrtest.EnsureUp(t) + service := compose.EnsureUpWithTimeout(t, 120, "ceph") f := mbtest.NewReportingMetricSetV2Error(t, getConfig(service.HostForPort(8003), mgrtest.GetPassword(t, service.HostForPort(5000), user))) diff --git a/metricbeat/module/ceph/mgr_cluster_health/_meta/data.json b/metricbeat/module/ceph/mgr_cluster_health/_meta/data.json index 65e4061ded4d..14cee62d3281 100644 --- a/metricbeat/module/ceph/mgr_cluster_health/_meta/data.json +++ b/metricbeat/module/ceph/mgr_cluster_health/_meta/data.json @@ -5,7 +5,7 @@ "name": "host.example.com" }, "ceph": { - "mgr_cluster_health": { + "cluster_health": { "overall_status": "HEALTH_OK", "timechecks": { "epoch": 3, diff --git a/metricbeat/module/ceph/mgr_cluster_health/_meta/fields.yml b/metricbeat/module/ceph/mgr_cluster_health/_meta/fields.yml index f3030809eb4f..606e8031cb3c 100644 --- a/metricbeat/module/ceph/mgr_cluster_health/_meta/fields.yml +++ b/metricbeat/module/ceph/mgr_cluster_health/_meta/fields.yml @@ -1,7 +1,7 @@ -- name: mgr_cluster_health +- name: cluster_health type: group description: > - mgr_cluster_health + cluster_health release: beta fields: - name: overall_status diff --git a/metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health.go b/metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health.go index 7dea91f755cd..b8dc57d2931c 100644 --- a/metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health.go +++ b/metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health.go @@ -20,6 +20,7 @@ package mgr_cluster_health import ( "fmt" + "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/metricbeat/mb" "github.com/elastic/beats/metricbeat/mb/parse" "github.com/elastic/beats/metricbeat/module/ceph/mgr" @@ -81,6 +82,9 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { return err } - reporter.Event(mb.Event{MetricSetFields: event}) + reporter.Event(mb.Event{ + ModuleFields: common.MapStr{ + "cluster_health": event, + }}) return nil } diff --git a/metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health_integration_test.go b/metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health_integration_test.go index a73fec46e86d..9113992a86be 100644 --- a/metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health_integration_test.go +++ b/metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health_integration_test.go @@ -24,6 +24,7 @@ import ( "github.com/stretchr/testify/require" + "github.com/elastic/beats/libbeat/tests/compose" mbtest "github.com/elastic/beats/metricbeat/mb/testing" "github.com/elastic/beats/metricbeat/module/ceph/mgrtest" ) @@ -31,7 +32,7 @@ import ( const user = "demo" func TestData(t *testing.T) { - service := mgrtest.EnsureUp(t) + service := compose.EnsureUpWithTimeout(t, 120, "ceph") f := mbtest.NewReportingMetricSetV2Error(t, getConfig(service.HostForPort(8003), mgrtest.GetPassword(t, service.HostForPort(5000), user))) diff --git a/metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health_test.go b/metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health_test.go index d280566f0459..255d96cfd7fd 100644 --- a/metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health_test.go +++ b/metricbeat/module/ceph/mgr_cluster_health/mgr_cluster_health_test.go @@ -75,7 +75,7 @@ func TestFetchEventContents(t *testing.T) { t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) } assert.NotEmpty(t, events) - event := events[0].MetricSetFields + event := events[0].ModuleFields["cluster_health"].(common.MapStr) t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event.StringToPrint()) diff --git a/metricbeat/module/ceph/mgr_osd_perf/mgr_osd_perf_integration_test.go b/metricbeat/module/ceph/mgr_osd_perf/mgr_osd_perf_integration_test.go index 9da9f2dc6313..c592f764ed35 100644 --- a/metricbeat/module/ceph/mgr_osd_perf/mgr_osd_perf_integration_test.go +++ b/metricbeat/module/ceph/mgr_osd_perf/mgr_osd_perf_integration_test.go @@ -24,6 +24,7 @@ import ( "github.com/stretchr/testify/require" + "github.com/elastic/beats/libbeat/tests/compose" mbtest "github.com/elastic/beats/metricbeat/mb/testing" "github.com/elastic/beats/metricbeat/module/ceph/mgrtest" ) @@ -31,7 +32,7 @@ import ( const user = "demo" func TestData(t *testing.T) { - service := mgrtest.EnsureUp(t) + service := compose.EnsureUpWithTimeout(t, 120, "ceph") f := mbtest.NewReportingMetricSetV2Error(t, getConfig(service.HostForPort(8003), mgrtest.GetPassword(t, service.HostForPort(5000), user))) diff --git a/metricbeat/module/ceph/mgr_osd_pool_stats/mgr_osd_pool_stats_integration_test.go b/metricbeat/module/ceph/mgr_osd_pool_stats/mgr_osd_pool_stats_integration_test.go index 3bb8ff693e2a..af2d8898fc22 100644 --- a/metricbeat/module/ceph/mgr_osd_pool_stats/mgr_osd_pool_stats_integration_test.go +++ b/metricbeat/module/ceph/mgr_osd_pool_stats/mgr_osd_pool_stats_integration_test.go @@ -24,6 +24,7 @@ import ( "github.com/stretchr/testify/require" + "github.com/elastic/beats/libbeat/tests/compose" mbtest "github.com/elastic/beats/metricbeat/mb/testing" "github.com/elastic/beats/metricbeat/module/ceph/mgrtest" ) @@ -31,7 +32,7 @@ import ( const user = "demo" func TestData(t *testing.T) { - service := mgrtest.EnsureUp(t) + service := compose.EnsureUpWithTimeout(t, 120, "ceph") f := mbtest.NewReportingMetricSetV2Error(t, getConfig(service.HostForPort(8003), mgrtest.GetPassword(t, service.HostForPort(5000), user))) diff --git a/metricbeat/module/ceph/mgr_osd_tree/_meta/data.json b/metricbeat/module/ceph/mgr_osd_tree/_meta/data.json index 9239f288557b..9816f8f0527e 100644 --- a/metricbeat/module/ceph/mgr_osd_tree/_meta/data.json +++ b/metricbeat/module/ceph/mgr_osd_tree/_meta/data.json @@ -1,7 +1,7 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", "ceph": { - "mgr_osd_tree": { + "osd_tree": { "children": [ "-2" ], diff --git a/metricbeat/module/ceph/mgr_osd_tree/_meta/fields.yml b/metricbeat/module/ceph/mgr_osd_tree/_meta/fields.yml index 93823ed2455f..d924267ee7f7 100644 --- a/metricbeat/module/ceph/mgr_osd_tree/_meta/fields.yml +++ b/metricbeat/module/ceph/mgr_osd_tree/_meta/fields.yml @@ -1,4 +1,4 @@ -- name: mgr_osd_tree +- name: osd_tree type: group description: > OSD tree information diff --git a/metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/sample.json-expected.json b/metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/sample.json-expected.json index eeb38333ca43..c2118414dcb2 100644 --- a/metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/sample.json-expected.json +++ b/metricbeat/module/ceph/mgr_osd_tree/_meta/testdata/sample.json-expected.json @@ -1,6 +1,6 @@ [ - {"ceph":{"mgr_osd_tree":{"children":["1","0"],"father":"default","id":-3,"name":"ceph-mon1","type":"host","type_id":1}},"event":{"dataset":"ceph.mgr_osd_tree","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_osd_tree","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}}, - {"ceph":{"mgr_osd_tree":{"children":["-3"],"father":"","id":-1,"name":"default","type":"root","type_id":10}},"event":{"dataset":"ceph.mgr_osd_tree","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_osd_tree","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}}, - {"ceph":{"mgr_osd_tree":{"crush_weight":0.048691,"depth":2,"device_class":"hdd","exists":true,"father":"ceph-mon1","id":1,"name":"osd.1","primary_affinity":1,"reweight":1,"status":"up","type":"osd","type_id":0}},"event":{"dataset":"ceph.mgr_osd_tree","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_osd_tree","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}}, - {"ceph":{"mgr_osd_tree":{"crush_weight":0.048691,"depth":2,"device_class":"hdd","exists":true,"father":"ceph-mon1","id":0,"name":"osd.0","primary_affinity":1,"reweight":1,"status":"up","type":"osd","type_id":0}},"event":{"dataset":"ceph.mgr_osd_tree","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_osd_tree","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}} + {"ceph":{"osd_tree":{"children":["1","0"],"father":"default","id":-3,"name":"ceph-mon1","type":"host","type_id":1}},"event":{"dataset":"ceph.mgr_osd_tree","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_osd_tree","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}}, + {"ceph":{"osd_tree":{"children":["-3"],"father":"","id":-1,"name":"default","type":"root","type_id":10}},"event":{"dataset":"ceph.mgr_osd_tree","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_osd_tree","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}}, + {"ceph":{"osd_tree":{"crush_weight":0.048691,"depth":2,"device_class":"hdd","exists":true,"father":"ceph-mon1","id":1,"name":"osd.1","primary_affinity":1,"reweight":1,"status":"up","type":"osd","type_id":0}},"event":{"dataset":"ceph.mgr_osd_tree","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_osd_tree","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}}, + {"ceph":{"osd_tree":{"crush_weight":0.048691,"depth":2,"device_class":"hdd","exists":true,"father":"ceph-mon1","id":0,"name":"osd.0","primary_affinity":1,"reweight":1,"status":"up","type":"osd","type_id":0}},"event":{"dataset":"ceph.mgr_osd_tree","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_osd_tree","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}} ] diff --git a/metricbeat/module/ceph/mgr_osd_tree/mgr_osd_tree.go b/metricbeat/module/ceph/mgr_osd_tree/mgr_osd_tree.go index dc1815b09af2..49b197f2682f 100644 --- a/metricbeat/module/ceph/mgr_osd_tree/mgr_osd_tree.go +++ b/metricbeat/module/ceph/mgr_osd_tree/mgr_osd_tree.go @@ -18,6 +18,7 @@ package mgr_osd_tree import ( + "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/metricbeat/mb" "github.com/elastic/beats/metricbeat/mb/parse" "github.com/elastic/beats/metricbeat/module/ceph/mgr" @@ -73,7 +74,10 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { } for _, event := range events { - reported := reporter.Event(mb.Event{MetricSetFields: event}) + reported := reporter.Event(mb.Event{ + ModuleFields: common.MapStr{ + "osd_tree": event, + }}) if !reported { return nil } diff --git a/metricbeat/module/ceph/mgr_osd_tree/mgr_osd_tree_integration_test.go b/metricbeat/module/ceph/mgr_osd_tree/mgr_osd_tree_integration_test.go index 44a357dc6ffa..cedc692e0791 100644 --- a/metricbeat/module/ceph/mgr_osd_tree/mgr_osd_tree_integration_test.go +++ b/metricbeat/module/ceph/mgr_osd_tree/mgr_osd_tree_integration_test.go @@ -24,6 +24,7 @@ import ( "github.com/stretchr/testify/require" + "github.com/elastic/beats/libbeat/tests/compose" mbtest "github.com/elastic/beats/metricbeat/mb/testing" "github.com/elastic/beats/metricbeat/module/ceph/mgrtest" ) @@ -31,7 +32,7 @@ import ( const user = "demo" func TestData(t *testing.T) { - service := mgrtest.EnsureUp(t) + service := compose.EnsureUpWithTimeout(t, 120, "ceph") f := mbtest.NewReportingMetricSetV2Error(t, getConfig(service.HostForPort(8003), mgrtest.GetPassword(t, service.HostForPort(5000), user))) diff --git a/metricbeat/module/ceph/mgr_pool_disk/_meta/data.json b/metricbeat/module/ceph/mgr_pool_disk/_meta/data.json index 53f69b41bf27..87d03b1e1445 100644 --- a/metricbeat/module/ceph/mgr_pool_disk/_meta/data.json +++ b/metricbeat/module/ceph/mgr_pool_disk/_meta/data.json @@ -1,7 +1,7 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", "ceph": { - "mgr_pool_disk": { + "pool_disk": { "id": 0, "name": "rbd", "stats": { diff --git a/metricbeat/module/ceph/mgr_pool_disk/_meta/fields.yml b/metricbeat/module/ceph/mgr_pool_disk/_meta/fields.yml index f3eeea23c227..e5215b688a57 100644 --- a/metricbeat/module/ceph/mgr_pool_disk/_meta/fields.yml +++ b/metricbeat/module/ceph/mgr_pool_disk/_meta/fields.yml @@ -1,7 +1,7 @@ -- name: mgr_pool_disk +- name: pool_disk type: group description: > - mgr_pool_disk + pool_disk release: beta fields: - name: id diff --git a/metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/sample.json-expected.json b/metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/sample.json-expected.json index 92e2e2b79f65..9026c18a1681 100644 --- a/metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/sample.json-expected.json +++ b/metricbeat/module/ceph/mgr_pool_disk/_meta/testdata/sample.json-expected.json @@ -1,10 +1,10 @@ [ - {"ceph":{"mgr_pool_disk":{"id":7,"name":"default.rgw.log","stats":{"available":{"bytes":9123004416},"objects":176,"used":{"bytes":0,"kb":0}}}},"event":{"dataset":"ceph.mgr_pool_disk","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_pool_disk","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}}, - {"ceph":{"mgr_pool_disk":{"id":4,"name":".rgw.root","stats":{"available":{"bytes":9123004416},"objects":6,"used":{"bytes":393216,"kb":384}}}},"event":{"dataset":"ceph.mgr_pool_disk","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_pool_disk","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}}, - {"ceph":{"mgr_pool_disk":{"id":1,"name":"rbd","stats":{"available":{"bytes":9123004416},"objects":0,"used":{"bytes":0,"kb":0}}}},"event":{"dataset":"ceph.mgr_pool_disk","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_pool_disk","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}}, - {"ceph":{"mgr_pool_disk":{"id":2,"name":"cephfs_data","stats":{"available":{"bytes":9123004416},"objects":0,"used":{"bytes":0,"kb":0}}}},"event":{"dataset":"ceph.mgr_pool_disk","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_pool_disk","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}}, - {"ceph":{"mgr_pool_disk":{"id":3,"name":"cephfs_metadata","stats":{"available":{"bytes":9123004416},"objects":22,"used":{"bytes":524288,"kb":512}}}},"event":{"dataset":"ceph.mgr_pool_disk","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_pool_disk","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}}, - {"ceph":{"mgr_pool_disk":{"id":6,"name":"default.rgw.meta","stats":{"available":{"bytes":9123004416},"objects":5,"used":{"bytes":262144,"kb":256}}}},"event":{"dataset":"ceph.mgr_pool_disk","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_pool_disk","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}}, - {"ceph":{"mgr_pool_disk":{"id":5,"name":"default.rgw.control","stats":{"available":{"bytes":9123004416},"objects":8,"used":{"bytes":0,"kb":0}}}},"event":{"dataset":"ceph.mgr_pool_disk","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_pool_disk","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}}, - {"ceph":{"mgr_pool_disk":{"id":8,"name":"default.rgw.buckets.index","stats":{"available":{"bytes":9123004416},"objects":1,"used":{"bytes":0,"kb":0}}}},"event":{"dataset":"ceph.mgr_pool_disk","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_pool_disk","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}} + {"ceph":{"pool_disk":{"id":7,"name":"default.rgw.log","stats":{"available":{"bytes":9123004416},"objects":176,"used":{"bytes":0,"kb":0}}}},"event":{"dataset":"ceph.mgr_pool_disk","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_pool_disk","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}}, + {"ceph":{"pool_disk":{"id":4,"name":".rgw.root","stats":{"available":{"bytes":9123004416},"objects":6,"used":{"bytes":393216,"kb":384}}}},"event":{"dataset":"ceph.mgr_pool_disk","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_pool_disk","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}}, + {"ceph":{"pool_disk":{"id":1,"name":"rbd","stats":{"available":{"bytes":9123004416},"objects":0,"used":{"bytes":0,"kb":0}}}},"event":{"dataset":"ceph.mgr_pool_disk","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_pool_disk","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}}, + {"ceph":{"pool_disk":{"id":2,"name":"cephfs_data","stats":{"available":{"bytes":9123004416},"objects":0,"used":{"bytes":0,"kb":0}}}},"event":{"dataset":"ceph.mgr_pool_disk","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_pool_disk","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}}, + {"ceph":{"pool_disk":{"id":3,"name":"cephfs_metadata","stats":{"available":{"bytes":9123004416},"objects":22,"used":{"bytes":524288,"kb":512}}}},"event":{"dataset":"ceph.mgr_pool_disk","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_pool_disk","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}}, + {"ceph":{"pool_disk":{"id":6,"name":"default.rgw.meta","stats":{"available":{"bytes":9123004416},"objects":5,"used":{"bytes":262144,"kb":256}}}},"event":{"dataset":"ceph.mgr_pool_disk","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_pool_disk","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}}, + {"ceph":{"pool_disk":{"id":5,"name":"default.rgw.control","stats":{"available":{"bytes":9123004416},"objects":8,"used":{"bytes":0,"kb":0}}}},"event":{"dataset":"ceph.mgr_pool_disk","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_pool_disk","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}}, + {"ceph":{"pool_disk":{"id":8,"name":"default.rgw.buckets.index","stats":{"available":{"bytes":9123004416},"objects":1,"used":{"bytes":0,"kb":0}}}},"event":{"dataset":"ceph.mgr_pool_disk","duration":115000,"module":"ceph"},"metricset":{"name":"mgr_pool_disk","period":10000},"service":{"address":"127.0.0.1:55555","type":"ceph"}} ] diff --git a/metricbeat/module/ceph/mgr_pool_disk/mgr_pool_disk.go b/metricbeat/module/ceph/mgr_pool_disk/mgr_pool_disk.go index e0c5cda9a55e..41b025abf24d 100644 --- a/metricbeat/module/ceph/mgr_pool_disk/mgr_pool_disk.go +++ b/metricbeat/module/ceph/mgr_pool_disk/mgr_pool_disk.go @@ -18,6 +18,7 @@ package mgr_pool_disk import ( + "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/metricbeat/mb" "github.com/elastic/beats/metricbeat/mb/parse" "github.com/elastic/beats/metricbeat/module/ceph/mgr" @@ -73,7 +74,10 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { } for _, event := range events { - reported := reporter.Event(mb.Event{MetricSetFields: event}) + reported := reporter.Event(mb.Event{ + ModuleFields: common.MapStr{ + "pool_disk": event, + }}) if !reported { return nil } diff --git a/metricbeat/module/ceph/mgr_pool_disk/mgr_pool_disk_integration_test.go b/metricbeat/module/ceph/mgr_pool_disk/mgr_pool_disk_integration_test.go index 1e9394d8cdba..f3b7748263dd 100644 --- a/metricbeat/module/ceph/mgr_pool_disk/mgr_pool_disk_integration_test.go +++ b/metricbeat/module/ceph/mgr_pool_disk/mgr_pool_disk_integration_test.go @@ -24,6 +24,7 @@ import ( "github.com/stretchr/testify/require" + "github.com/elastic/beats/libbeat/tests/compose" mbtest "github.com/elastic/beats/metricbeat/mb/testing" "github.com/elastic/beats/metricbeat/module/ceph/mgrtest" ) @@ -31,7 +32,7 @@ import ( const user = "demo" func TestData(t *testing.T) { - service := mgrtest.EnsureUp(t) + service := compose.EnsureUpWithTimeout(t, 120, "ceph") f := mbtest.NewReportingMetricSetV2Error(t, getConfig(service.HostForPort(8003), mgrtest.GetPassword(t, service.HostForPort(5000), user))) diff --git a/metricbeat/module/ceph/mgrtest/compose.go b/metricbeat/module/ceph/mgrtest/compose.go deleted file mode 100644 index 02c2b55bbf75..000000000000 --- a/metricbeat/module/ceph/mgrtest/compose.go +++ /dev/null @@ -1,41 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package mgrtest - -import ( - "fmt" - "testing" - "time" - - "github.com/elastic/beats/libbeat/tests/compose" -) - -const ( - upTimeout = 120 * time.Second - - defaultCodename = "nautilus" - defaultVersion = "master-97985eb-nautilus-centos-7-x86_64" -) - -func EnsureUp(t *testing.T) compose.HostInfo { - return compose.EnsureUp(t, "ceph", - compose.UpWithTimeout(upTimeout), - compose.UpWithEnvironmentVariable(fmt.Sprintf("CEPH_CODENAME=%s", defaultCodename)), - compose.UpWithEnvironmentVariable(fmt.Sprintf("CEPH_VERSION=%s", defaultVersion)), - ) -} From b1550b77513a412b88e99764fa65255cc339d75e Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Tue, 18 Feb 2020 20:32:06 +0100 Subject: [PATCH 41/41] Define fields only in original metricset --- metricbeat/docs/fields.asciidoc | 290 +----------------- metricbeat/module/ceph/fields.go | 2 +- .../ceph/mgr_cluster_disk/_meta/fields.yml | 19 +- .../ceph/mgr_cluster_health/_meta/fields.yml | 20 +- .../module/ceph/mgr_osd_tree/_meta/fields.yml | 56 +--- .../ceph/mgr_pool_disk/_meta/fields.yml | 30 +- 6 files changed, 17 insertions(+), 400 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index b408b4e5c92a..6394e568e15d 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -3933,94 +3933,16 @@ type: long -- [float] -=== cluster_disk - -cluster_disk - - - -*`ceph.cluster_disk.available.bytes`*:: -+ --- -Available bytes of the cluster - - -type: long - -format: bytes - --- +=== mgr_cluster_disk -*`ceph.cluster_disk.total.bytes`*:: -+ --- -Total bytes of the cluster +see: cluster_disk -type: long - -format: bytes - --- - -*`ceph.cluster_disk.used.bytes`*:: -+ --- -Used bytes of the cluster - - -type: long - -format: bytes - --- - [float] -=== cluster_health - -cluster_health - - - -*`ceph.cluster_health.overall_status`*:: -+ --- -Overall status of the cluster - - -type: keyword - --- - -*`ceph.cluster_health.timechecks.epoch`*:: -+ --- -Map version - - -type: long - --- - -*`ceph.cluster_health.timechecks.round.value`*:: -+ --- -timecheck round +=== mgr_cluster_health +see: cluster_health -type: long - --- - -*`ceph.cluster_health.timechecks.round.status`*:: -+ --- -Status of the round - - -type: keyword - --- [float] === mgr_osd_perf @@ -4109,212 +4031,16 @@ type: object -- [float] -=== osd_tree - -OSD tree information - - - -*`ceph.osd_tree.id`*:: -+ --- -OSD or bucket node ID - - -type: long - --- - -*`ceph.osd_tree.name`*:: -+ --- -OSD or bucket node name - - -type: keyword - --- - -*`ceph.osd_tree.type`*:: -+ --- -OSD or bucket node type, illegal type include OSD, host, root etc. - - -type: keyword - --- - -*`ceph.osd_tree.type_id`*:: -+ --- -OSD or bucket node type ID - - -type: long - --- - -*`ceph.osd_tree.children`*:: -+ --- -Bucket children list, separated by comma. - - -type: keyword - --- - -*`ceph.osd_tree.crush_weight`*:: -+ --- -OSD node CRUSH weight - - -type: float - --- - -*`ceph.osd_tree.depth`*:: -+ --- -Node depth - - -type: long - --- - -*`ceph.osd_tree.exists`*:: -+ --- -Node exists (1-yes, 0-no) +=== mgr_osd_tree +see: osd_tree -type: boolean - --- - -*`ceph.osd_tree.primary_affinity`*:: -+ --- -Weight of reading data from primary OSD - - -type: float - --- - -*`ceph.osd_tree.reweight`*:: -+ --- -Reweight of osd - - -type: long - --- - -*`ceph.osd_tree.status`*:: -+ --- -Status of OSD, it should be up or down - - -type: keyword - --- - -*`ceph.osd_tree.device_class`*:: -+ --- -Device class of OSD, like HDD, SSD etc. - - -type: keyword - --- - -*`ceph.osd_tree.father`*:: -+ --- -Parent node of this OSD or bucket node - - -type: keyword - --- [float] -=== pool_disk - -pool_disk - - - -*`ceph.pool_disk.id`*:: -+ --- -Id of the pool - - -type: long - --- - -*`ceph.pool_disk.name`*:: -+ --- -Name of the pool +=== mgr_pool_disk +see: pool_disk -type: keyword - --- - -*`ceph.pool_disk.stats.available.bytes`*:: -+ --- -Available bytes of the pool - - -type: long - -format: bytes - --- - -*`ceph.pool_disk.stats.objects`*:: -+ --- -Number of objects of the pool - - -type: long - --- - -*`ceph.pool_disk.stats.used.bytes`*:: -+ --- -Used bytes of the pool - - -type: long - -format: bytes - --- - -*`ceph.pool_disk.stats.used.kb`*:: -+ --- -Used kb of the pool - - -type: long - --- [float] === monitor_health diff --git a/metricbeat/module/ceph/fields.go b/metricbeat/module/ceph/fields.go index a9f3cc856f8c..c0b558982fc4 100644 --- a/metricbeat/module/ceph/fields.go +++ b/metricbeat/module/ceph/fields.go @@ -32,5 +32,5 @@ func init() { // AssetCeph returns asset data. // This is the base64 encoded gzipped contents of ../metricbeat/module/ceph. func AssetCeph() string { - return "eJzsms+P27oRx+/7Vwz21AIbt736UCDJBsiiTbJYv+AdikKhxZHELkUSJLWu//uClCzb+m2blouH1SmxvTMfDodDzlf8AK+4XUKMKrsDsMxyXML9Z1TZ/R0ARRNrpiyTYgl/vwMAcF9BLmnB8Q7AZFLbKJYiYekSEsKN+1QjR2JwCSlxv0FrmUjNEv51bwy/f4D7zFp1/+87gIQhp2bpLX8AQXKsWdxjt8pZ0bJQ1ScdRO755f7oF8RSWMKEAZsh5Gg1i92/iYUNagQTa6KQQqJlDp+/PH9dVAYOMY5QeGEs6ogy81p/2YU1gOaeHjvHcdo9TZhDIPJGGCdrjov11qI5+s2Oi0uRNr4YQHPPx51V8FZBJj6AFXXj54nUObFLaAPsIK20hAcF/M1ZDANXGKRB2X4apOejNTMtQ8JtFiLXWpZOzzb5hppwHhlLbNEdr1fcbqSmp4XsR2kXSrtDYatTiuUYZxi/mgUqGWeB5u4bUfCG2jApxtxqWQi6eCO8wEDOa+PgbU8DCD0Vq6MpOAZp5mbLd3dudufZhIxt2R/Kza5pO38u/K5WxWIwITRJEhYvNBIanVRF+ivTOFwZH3BOwWZaFmmmCgsKNRiMZV/uVKwbzSzODuu9nkHrIytVpFxCYBxqfg9DyKQypwXvajxllKYA5cwoTmJc+N01MMbOOKgURJGve6pwzSDX/8HYhtpBWxSV+UkomlgmO0FMTDjSKOGS2J78VqhjFM1vT8VtI+woKaaaUKRXmbSd8ZFJqxmuM2k1xYRJq1FuN2k1bv+kqXRBiSWzV0yVgvM7cHZV6cI3ALOjkeMGoQ/OZ/nscHbfHPSBuUP/7FxF3Rj0YPkDD/ojHUbuw0BL8zn1J2s8/N0wQiyL1oI61/sqkxsDmdxATsQWVGqAaAQmKiiZjA69xRf4sFdNkBk/70lDF0nBu0v3WkqOpEk14vzJOKPQMnroUSDRV/HqDI+4LvJIGhpql2gkg7PsMmGs3duRFOraMC41ZQlUp+fP50EoJm4B9fR9EEpjTpRCGql0brKXL98+Pj9/eezlC9mxe1vNM8bcct0a7btg9y7YzSbYnZBv75Ldu2Tn2uNUu00qUqiTS3Lzx+rR9Vt+KYh4/05FJqVk1k6Ac5KWNSMwbX4c3FP3tuNibhaxzHNmI04singb5edVkM/eClRW3PEl765UpVOiFN9e6vOjMzLZZWOcItQ4xfRxnumzNU7RrrR1LktZ1rVxCXoso6UsK1noRPaIvZ3chNX+7Mj6+yFn/szF4i33rJaYMxQ2YjLSxHazl+rOYPp4I/D0lx/grDTzoStshwx7Rb1DZh0cpnuq0C7hk9/XvczbI6geOj2QxgN4ddYsiimOh1Xu6Y5f3EClQq9miV4RuT3mAL5/99r1sPODNiCyGvHSletsABPl0ez4CDDjxtNxPFo9gtSwLuJXtCAkxb61dkl1mOS3t3o4N1f17Kw9AOMcU8L9/4CJmBcU3a8fIJPGPoCW0gLaeNELeW6JmwzZWwczxqnGbp3prCh9Kj3vDANnLgQGFXE10rUh4DZv0h2MWBcmizbI0qxbneuSySeExAfi88vP1VfoML5X65UNdWr/7jy2De5c4X+Z6XkzcZbc5d2VRuFPf/uwRfMAf/0g5J+7t1XNcqK3EUkSJpjdhgr17z627ojhyj0Tlb7vbx9VPt10dDJpHJj20+P/UplzMNJ0Ny7X61P86mcWTCYLTmGNUCi3LqncdDdxFN9YjFHMiQkI9OitgrdaY3H2ivD18fEBVqvH/rqUEJu1FIILWJ6JdqclvxJ9J8dMR7Vq7aP+BHipvNVl5Ka75xPddbMObYZt8zvJcdRl1ebMKeC1WKYIZCVn2Le7373A6qtF9Vp3WrRmEOsuiJGne12HRHtdd3LVvasUzMrJKuHJV6eO7Vc9rdtlJi7hfXarVnd3eW5Xb+p3Efr2o/sdRis4cNnq/loGY8TrfujBMmI/8n98GnNfau7BXJeC+7jbKyyBcaecGBsVirojb6dn2pYdxsbLcoRNhgI2xECX7evvHH3DNVZqLJWqBZdp0Hr4T5nuy2ETYFol3MPlzMRB6b4xE4fDM8YGpVutfgsHd/V3ZpcCji66M5KPGNtaa4cSDx1/2XD67WBUmb+84E6sUBiS9ilAs5xV/TUKd2pnc5Sb2luvqnOdVqn2O6DpZJQ+gDF0QMyp18jU4F9w7apOkfI+2JvkRU/I6kPqnFRl4g5QndlohGDbX+8b4FNpJIo80CoyzUtiwGXsFTF/u4SZXpXCT17fYfVat0cbs1jZIGm7QQ8hdNcVr1a7R+vnbKWvoabOVgRvI3J3eB4oiNLQW4jcPZCzadzr/zuNu969vPG5NG4xr8bNTOnRWMZ5ad2lgZD2doq364o2E1TvvuIeWPX292NuoHzvL0n5knB75dsFgjbUb4/m1e/xg1xo9dvrZR0KeLuUtTbY6yvg82+s7/r3u/79x9C//xcAAP//xbT1Yg==" + return "eJzEm89v47oRx+/5KwY5tUDitlcfCuwvYIM2m2Czix6KQkuTI4k1RRIkFT//9w+kZNnWb9u0V6f3bO93PhoOh8Mh8whr3C6Bos7vABx3Apdw/wl1fn8HwNBSw7XjSi7hn3cAAP4rKBQrBd4B2FwZl1AlU54tISXC+k8NCiQWl5AR/xt0jsvMLuG/99aK+we4z53T9/+7A0g5CmaXQfkRJCmwYfGP22qvYlSp6096iPzzy/+jX0CVdIRLCy5HKNAZTv1/EwcbNAiWGqKRQWpUAZ++vH5d1AKHGEcoorQOTcK4XTdf9mGNoPlnQOfYT7unDXMIRN4JF2QlcLHaOrRHv9lxCSWz1hcjaP75sFOFoAoqDQ6sqVs/T5UpiFtCF2AH6ZQjIirgD68YB660yKKy/bTIzkdrR1qORLg8Rqx1lE6PNvWOhgiRWEdc2e+vNW43yrDTXPZS6UKlO+a2JqR4gTRHurYL1IrmkcbumWh4R2O5klNmjSolW7wTUWIk4404BO15ALGH4u1oCI5B2rHZsd0fm/1xNiNiO/pjsdk3bOePRVjVal+MBoQhacrpwiBhyUlZZDgzTcNV/gFvFFxuVJnlunSg0YBFqoZip2bdGO7w5rDB6hm0wbNKJ9oHBNJY43voQq60Pc15V+OpvDQHqOBWC0JxEVbXyBg7cdAZyLJYDWThhkGt/o/UxVpBOxS1/CwUQxxXvSCWEoEsSYUibiC+NRqKsv3tqbhdhB0lw8wQhuwqg7YTnxi0huE6g9ZQzBi0BuX3DVqDOzxoOlsw4sjNM6bOwNsdqV11tggbgJujkeMNwhBciPKbw7n95mAIzBf9N+cqm43BAFYoeDCUdJj4DyNNzdcsVNZ4+LtxBKrKzoQ61/pbrjYWcrWBgsgt6MwCMQhc1lAqnXz1Dl/kYq8eIDtd7ynLFmkp+lP3SimBpE01YfzJelHoiB5alEjMVax64QnTZZEoy2KtEq1g8Mo+Eqa2ezuSUl8bxoemqoCa8Pz5OgrF5e+Aevo2CmWwIFojS3R2a7LvX54/vL5++TzIF3PHHrTaNUZTFGYmidWyszjc/2t2uCt0Y72UPq7LGzxHZMNdnhPYlGV+f5NeQvXy9tlXZmHdlHTffVVptbnuTve5pIe0vN3emBdHHu6pP0B9GNsFVUXBXSKIQ0m3SXHeHPoUVKBW8Ymu6F/7K6NEa7G91OYHLzLbZOs9Zaz3lPPf80ybnfeU3SZqE8tKVU3L6WbVVEQrVbUpYwdyQBys+Wa08l492XDl5OXPnCxBeWC2UMFRuoSrxBDXz17tA0fDJ4jA099e/C6sU173ue2QYd9762nIjL6mf2rXLuFjaNmHhtBA6+XQ6EETLYJVr+ZQzjE83g+bb/i7f1GlMex75WC7qfvOEWz/J3S5xo2357AziBevkj1C56yPYTZFKSj6lE4mUpI7NbuSOLkdf6xfZz9GZqe2/Rml7uSBy08o6+7P7sTi+aW/Lu44By47J/laOWPC6v7V16vob/6vj1Pmq4PXaKarU9dps+FINZrVcJ46bVQQ65JSM+Kwf5lj3QVq6n15gbDJUcKGWOjT3hm/ZOHu2v1GCpx6XeuUwaqmWQiVRT3B/rfK9gfYbYA5Z+qHcAW3NCrdM7c0Hp61Lird29uPeHBXvzhxKeDkpDsj+Ih1nbl20D1I2PS29PQTZ1/IK8vAL8VQWpIhcFn54rgVeIXNaNcHoTWnGLYVr5NuGmuDuweG75xiQgWxEa8aNHa9xgNwITAjIvwfcElFyRByxh7AWgbo6GJkjfPhOtf5F7TymxCpzhjelSgHXNZcK7olVRW4I1RnXheLwbY/Mhrh01kiyyLSLLLtgwcQivq0UnUseWiBDw/eULF6rRPJ1ijWGiTD3ix46XaoyXheKOS6yfx5s9SnDKxKukZ34yTYsjuYDr2Zq1oeSYjKsgfIlXUPYJRyI3lxq/Hcjs9syKGuUM4FM9h/PneWk2rDO2EQ3HvAoiYmzOjVFqgqCtLvC2pKmycb5FneP6f7JvPc1SuIQ4/4fvXUAzvP08chWOwK7kzhH9wO3Og465iQ28qidVyISt2HgVTuL/943KJ9gL8/SvXX/lxueEHMNiFpyiV321iO97uiytu+gjVIGJf1TYlwj7u2O5jcDY4Ewhm3NnNsJD3QkNnYFzX3d2VDSuDOr3elYLBCKLUfJaY2/YfY1ynnvCMqZQjKDZrg6zmFXEpc3rkkfSGPJgZlna/Chrpa9FuprLPARunvjbb2br+wPrFdS8GD3WBFPexiDJqsD4Ru+VcMHZZ5m17PGffG3LdwaB2mSX1Vbp63bvAXCxf46ArNv/XqiOvPAAAA//+0W4ni" } diff --git a/metricbeat/module/ceph/mgr_cluster_disk/_meta/fields.yml b/metricbeat/module/ceph/mgr_cluster_disk/_meta/fields.yml index fe755709362a..f6ae58cefaa9 100644 --- a/metricbeat/module/ceph/mgr_cluster_disk/_meta/fields.yml +++ b/metricbeat/module/ceph/mgr_cluster_disk/_meta/fields.yml @@ -1,21 +1,6 @@ -- name: cluster_disk +- name: mgr_cluster_disk type: group description: > - cluster_disk + see: cluster_disk release: beta fields: - - name: available.bytes - type: long - description: > - Available bytes of the cluster - format: bytes - - name: total.bytes - type: long - description: > - Total bytes of the cluster - format: bytes - - name: used.bytes - type: long - description: > - Used bytes of the cluster - format: bytes diff --git a/metricbeat/module/ceph/mgr_cluster_health/_meta/fields.yml b/metricbeat/module/ceph/mgr_cluster_health/_meta/fields.yml index 606e8031cb3c..6d627dcb88d2 100644 --- a/metricbeat/module/ceph/mgr_cluster_health/_meta/fields.yml +++ b/metricbeat/module/ceph/mgr_cluster_health/_meta/fields.yml @@ -1,22 +1,6 @@ -- name: cluster_health +- name: mgr_cluster_health type: group description: > - cluster_health + see: cluster_health release: beta fields: - - name: overall_status - type: keyword - description: > - Overall status of the cluster - - name: timechecks.epoch - type: long - description: > - Map version - - name: timechecks.round.value - type: long - description: > - timecheck round - - name: timechecks.round.status - type: keyword - description: > - Status of the round diff --git a/metricbeat/module/ceph/mgr_osd_tree/_meta/fields.yml b/metricbeat/module/ceph/mgr_osd_tree/_meta/fields.yml index d924267ee7f7..2be502ea171f 100644 --- a/metricbeat/module/ceph/mgr_osd_tree/_meta/fields.yml +++ b/metricbeat/module/ceph/mgr_osd_tree/_meta/fields.yml @@ -1,58 +1,6 @@ -- name: osd_tree +- name: mgr_osd_tree type: group description: > - OSD tree information + see: osd_tree release: beta fields: - - name: id - type: long - description: > - OSD or bucket node ID - - name: name - type: keyword - description: > - OSD or bucket node name - - name: type - type: keyword - description: > - OSD or bucket node type, illegal type include OSD, host, root etc. - - name: type_id - type: long - description: > - OSD or bucket node type ID - - name: children - type: keyword - description: > - Bucket children list, separated by comma. - - name: crush_weight - type: float - description: > - OSD node CRUSH weight - - name: depth - type: long - description: > - Node depth - - name: exists - type: boolean - description: > - Node exists (1-yes, 0-no) - - name: primary_affinity - type: float - description: > - Weight of reading data from primary OSD - - name: reweight - type: long - description: > - Reweight of osd - - name: status - type: keyword - description: > - Status of OSD, it should be up or down - - name: device_class - type: keyword - description: > - Device class of OSD, like HDD, SSD etc. - - name: father - type: keyword - description: > - Parent node of this OSD or bucket node diff --git a/metricbeat/module/ceph/mgr_pool_disk/_meta/fields.yml b/metricbeat/module/ceph/mgr_pool_disk/_meta/fields.yml index e5215b688a57..2ecc0bdf78de 100644 --- a/metricbeat/module/ceph/mgr_pool_disk/_meta/fields.yml +++ b/metricbeat/module/ceph/mgr_pool_disk/_meta/fields.yml @@ -1,32 +1,6 @@ -- name: pool_disk +- name: mgr_pool_disk type: group description: > - pool_disk + see: pool_disk release: beta fields: - - name: id - type: long - description: > - Id of the pool - - name: name - type: keyword - description: > - Name of the pool - - name: stats.available.bytes - type: long - description: > - Available bytes of the pool - format: bytes - - name: stats.objects - type: long - description: > - Number of objects of the pool - - name: stats.used.bytes - type: long - description: > - Used bytes of the pool - format: bytes - - name: stats.used.kb - type: long - description: > - Used kb of the pool