From db217fa319c71ad8f30913b5a96fab847951ebeb Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Thu, 22 Feb 2018 20:55:55 +0100 Subject: [PATCH] Metricbeat: Support multiple apache versions Apache before version 2.4.16 had less information in its status page. Add support to both, old and new status pages. --- CHANGELOG.asciidoc | 1 + metricbeat/docker-compose.yml | 5 ++ metricbeat/docs/modules/apache.asciidoc | 4 +- .../module/apache/_meta/Dockerfile.2.4.12 | 4 ++ metricbeat/module/apache/_meta/docs.asciidoc | 4 +- metricbeat/module/apache/_meta/env | 1 + metricbeat/module/apache/status/data.go | 35 +++++++++++- .../module/apache/status/status_test.go | 2 +- metricbeat/tests/system/test_apache.py | 56 ++++++++++++++----- 9 files changed, 92 insertions(+), 20 deletions(-) create mode 100644 metricbeat/module/apache/_meta/Dockerfile.2.4.12 diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 102b633a8d87..cea835602cad 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -66,6 +66,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di - Add filtering option by exact device names in system.diskio. `diskio.include_devices`. {pull}6085[6085] - Fix dealing with new process status codes in Linux kernel 4.14+. {pull}6306[6306] - Add config option for windows/perfmon metricset to ignore non existent counters. {pull}6432[6432] +- Support apache status pages for versions older than 2.4.16. {pull}6450[6450] *Packetbeat* diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml index 79b46569746f..7902b862eaed 100644 --- a/metricbeat/docker-compose.yml +++ b/metricbeat/docker-compose.yml @@ -45,6 +45,11 @@ services: apache: build: ./module/apache/_meta + apache_2_4_12: + build: + context: ./module/apache/_meta + dockerfile: Dockerfile.2.4.12 + ceph: build: ./module/ceph/_meta diff --git a/metricbeat/docs/modules/apache.asciidoc b/metricbeat/docs/modules/apache.asciidoc index e2d21540ab70..291a73ccbc4b 100644 --- a/metricbeat/docs/modules/apache.asciidoc +++ b/metricbeat/docs/modules/apache.asciidoc @@ -11,8 +11,8 @@ HTTPD] servers. [float] === Compatibility -The Apache metricsets were tested with Apache 2.4.20 and are expected to work with all versions ->= 2.2.31 and >= 2.4.16. +The Apache metricsets were tested with Apache 2.4.12 and 2.4.20 and are expected to work with +all versions >= 2.2.31 and >= 2.4.16. [float] diff --git a/metricbeat/module/apache/_meta/Dockerfile.2.4.12 b/metricbeat/module/apache/_meta/Dockerfile.2.4.12 new file mode 100644 index 000000000000..f35ea2d95c03 --- /dev/null +++ b/metricbeat/module/apache/_meta/Dockerfile.2.4.12 @@ -0,0 +1,4 @@ +FROM httpd:2.4.12 +RUN apt-get update && apt-get install -y curl +HEALTHCHECK --interval=1s --retries=90 CMD curl -f http://localhost +COPY ./httpd.conf /usr/local/apache2/conf/httpd.conf diff --git a/metricbeat/module/apache/_meta/docs.asciidoc b/metricbeat/module/apache/_meta/docs.asciidoc index 7e86891a24c5..b83474b20700 100644 --- a/metricbeat/module/apache/_meta/docs.asciidoc +++ b/metricbeat/module/apache/_meta/docs.asciidoc @@ -4,8 +4,8 @@ HTTPD] servers. [float] === Compatibility -The Apache metricsets were tested with Apache 2.4.20 and are expected to work with all versions ->= 2.2.31 and >= 2.4.16. +The Apache metricsets were tested with Apache 2.4.12 and 2.4.20 and are expected to work with +all versions >= 2.2.31 and >= 2.4.16. [float] diff --git a/metricbeat/module/apache/_meta/env b/metricbeat/module/apache/_meta/env index ffc83c4482e3..5339c3407ed8 100644 --- a/metricbeat/module/apache/_meta/env +++ b/metricbeat/module/apache/_meta/env @@ -1,2 +1,3 @@ APACHE_HOST=apache +APACHE_OLD_HOST=apache_2_4_12 APACHE_PORT=80 diff --git a/metricbeat/module/apache/status/data.go b/metricbeat/module/apache/status/data.go index baa0d715da22..aa124aaaa7cb 100644 --- a/metricbeat/module/apache/status/data.go +++ b/metricbeat/module/apache/status/data.go @@ -51,8 +51,40 @@ var ( "15": c.Float("Load15", s.Optional), }, } + + // Schema used till apache 2.4.12 + schemaOld = s.Schema{ + "total_accesses": c.Int("Total Accesses"), + "total_kbytes": c.Int("Total kBytes"), + "requests_per_sec": c.Float("ReqPerSec", s.Optional), + "bytes_per_sec": c.Float("BytesPerSec", s.Optional), + "workers": s.Object{ + "busy": c.Int("BusyWorkers"), + "idle": c.Int("IdleWorkers"), + }, + "uptime": s.Object{ + "uptime": c.Int("Uptime"), + }, + "connections": s.Object{ + "total": c.Int("ConnsTotal", s.Optional), + "async": s.Object{ + "writing": c.Int("ConnsAsyncWriting", s.Optional), + "keep_alive": c.Int("ConnsAsyncKeepAlive", s.Optional), + "closing": c.Int("ConnsAsyncClosing", s.Optional), + }, + }, + } ) +func applySchema(event common.MapStr, fullEvent map[string]interface{}) *s.Errors { + applicableSchema := schema + if _, found := fullEvent["ServerUptimeSeconds"]; !found { + applicableSchema = schemaOld + } + _, err := applicableSchema.ApplyTo(event, fullEvent) + return err +} + // Map body to MapStr func eventMapping(scanner *bufio.Scanner, hostname string) (common.MapStr, *s.Errors) { var ( @@ -140,9 +172,8 @@ func eventMapping(scanner *bufio.Scanner, hostname string) (common.MapStr, *s.Er "total": totalAll, }, } - _, err := schema.ApplyTo(event, fullEvent) - return event, err + return event, applySchema(event, fullEvent) } /* diff --git a/metricbeat/module/apache/status/status_test.go b/metricbeat/module/apache/status/status_test.go index e419a5d2d548..77a628232c01 100644 --- a/metricbeat/module/apache/status/status_test.go +++ b/metricbeat/module/apache/status/status_test.go @@ -123,8 +123,8 @@ func TestFetchEventContents(t *testing.T) { assert.EqualValues(t, 63, event["total_kbytes"]) uptime := event["uptime"].(common.MapStr) - assert.EqualValues(t, 1026782, uptime["server_uptime"]) assert.EqualValues(t, 1026782, uptime["uptime"]) + assert.EqualValues(t, 1026782, uptime["server_uptime"]) } // TestFetchTimeout verifies that the HTTP request times out and an error is diff --git a/metricbeat/tests/system/test_apache.py b/metricbeat/tests/system/test_apache.py index 21f32c1516a0..41f73c03bd1b 100644 --- a/metricbeat/tests/system/test_apache.py +++ b/metricbeat/tests/system/test_apache.py @@ -7,13 +7,24 @@ APACHE_FIELDS = metricbeat.COMMON_FIELDS + ["apache"] -APACHE_STATUS_FIELDS = ["hostname", "total_accesses", "total_kbytes", - "requests_per_sec", "bytes_per_sec", "bytes_per_request", - "workers.busy", "workers.idle", "uptime", "cpu", - "connections", "load", "scoreboard"] +APACHE_STATUS_FIELDS = [ + "hostname", "total_accesses", "total_kbytes", + "requests_per_sec", "bytes_per_sec", "bytes_per_request", + "workers.busy", "workers.idle", "uptime", "cpu", + "connections", "load", "scoreboard" +] -CPU_FIELDS = ["load", "user", "system", "children_user", - "children_system"] +APACHE_OLD_STATUS_FIELDS = [ + "hostname", "total_accesses", "total_kbytes", + "requests_per_sec", "bytes_per_sec", + "workers.busy", "workers.idle", "uptime", + "connections", "scoreboard" +] + + +CPU_FIELDS = [ + "load", "user", "system", "children_user", "children_system" +] class ApacheStatusTest(metricbeat.BaseTest): @@ -37,7 +48,7 @@ def test_output(self): found = False # Waits until CPULoad is part of the status - while found == False: + while not found: res = urllib2.urlopen(hosts[0] + "/server-status?auto").read() if "CPULoad" in res: found = True @@ -52,16 +63,35 @@ def test_output(self): self.assertEqual(len(output), 1) evt = output[0] - # Verify the required fields are present. - self.assertItemsEqual(self.de_dot(APACHE_FIELDS), evt.keys()) - apache_status = evt["apache"]["status"] - self.assertItemsEqual(self.de_dot(APACHE_STATUS_FIELDS), apache_status.keys()) - self.assertItemsEqual(self.de_dot(CPU_FIELDS), apache_status["cpu"].keys()) - # There are more fields that could be checked. + self.verify_fields(evt) # Verify all fields present are documented. self.assert_fields_are_documented(evt) + def verify_fields(self, evt): + self.assertItemsEqual(self.de_dot(APACHE_FIELDS), evt.keys()) + apache_status = evt["apache"]["status"] + self.assertItemsEqual( + self.de_dot(APACHE_STATUS_FIELDS), apache_status.keys()) + self.assertItemsEqual( + self.de_dot(CPU_FIELDS), apache_status["cpu"].keys()) + # There are more fields that could be checked. + def get_hosts(self): return ['http://' + os.getenv('APACHE_HOST', 'localhost') + ':' + os.getenv('APACHE_PORT', '80')] + + +class ApacheOldStatusTest(ApacheStatusTest): + + COMPOSE_SERVICES = ['apache_2_4_12'] + + def verify_fields(self, evt): + self.assertItemsEqual(self.de_dot(APACHE_FIELDS), evt.keys()) + apache_status = evt["apache"]["status"] + self.assertItemsEqual( + self.de_dot(APACHE_OLD_STATUS_FIELDS), apache_status.keys()) + + def get_hosts(self): + return ['http://' + os.getenv('APACHE_OLD_HOST', 'localhost') + ':' + + os.getenv('APACHE_PORT', '80')]