diff --git a/metricbeat/_meta/beat.full.yml b/metricbeat/_meta/beat.full.yml index f4065b153c0d..e93bebd8bdf4 100644 --- a/metricbeat/_meta/beat.full.yml +++ b/metricbeat/_meta/beat.full.yml @@ -200,6 +200,14 @@ metricbeat.modules: # Path to server status. Default server-status #server_status_path: "server-status" +#------------------------------- php_fpm Module ------------------------------ +#- module: php_fpm + #metricsets: ["pool"] + #enabled: true + #period: 10s + #status_path: "/status" + #hosts: ["localhost:8080"] + #----------------------------- PostgreSQL Module ----------------------------- #- module: postgresql #metricsets: diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 1afc271c34d4..72222092b270 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -23,6 +23,7 @@ grouped in the following categories: * <> * <> * <> +* <> * <> * <> * <> @@ -3802,6 +3803,90 @@ type: long The current number of idle client connections waiting for a request. +[[exported-fields-php_fpm]] +== php_fpm Fields + +PHP-FPM server status metrics collected from PHP-FPM. +experimental[] + + + +[float] +== php_fpm Fields + +`php_fpm` contains the metrics that were obtained from PHP-FPM status page call. + + + +[float] +== pool Fields + +`pool` contains the metrics that were obtained from the PHP-FPM process pool. + + + +[float] +=== php_fpm.pool.pool + +type: keyword + +The name of the pool. + + +[float] +== connections Fields + +Connection state specific statistics. + + + +[float] +=== php_fpm.pool.connections.accepted + +type: long + +The number of incoming requests that the PHP-FPM server has accepted; when a connection is accepted it is removed from the listen queue. + + +[float] +=== php_fpm.pool.connections.queued + +type: long + +The current number of connections that have been initiated, but not yet accepted. If this value is non-zero it typically means that all the available server processes are currently busy, and there are no processes available to serve the next request. Raising `pm.max_children` (provided the server can handle it) should help keep this number low. This property follows from the fact that PHP-FPM listens via a socket (TCP or file based), and thus inherits some of the characteristics of sockets. + + +[float] +== processes Fields + +Process state specific statistics. + + + +[float] +=== php_fpm.pool.processes.idle + +type: long + +The number of servers in the `waiting to process` state (i.e. not currently serving a page). This value should fall between the `pm.min_spare_servers` and `pm.max_spare_servers` values when the process manager is `dynamic`. + + +[float] +=== php_fpm.pool.processes.active + +type: long + +The number of servers current processing a page - the minimum is `1` (so even on a fully idle server, the result will be not read `0`). + + +[float] +=== php_fpm.pool.slow_requests + +type: long + +The number of times a request execution time has exceeded `request_slowlog_timeout`. + + [[exported-fields-postgresql]] == PostgreSQL Fields diff --git a/metricbeat/docs/modules/php_fpm.asciidoc b/metricbeat/docs/modules/php_fpm.asciidoc new file mode 100644 index 000000000000..2440b1935f09 --- /dev/null +++ b/metricbeat/docs/modules/php_fpm.asciidoc @@ -0,0 +1,61 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[metricbeat-module-php_fpm]] +== PHP-FPM Module + +This module periodically fetches metrics from https://php-fpm.org[PHP-FPM] +servers. + +[float] +=== Module-Specific Configuration Notes + +You need to enable the PHP-FPM status page by properly configuring +`pm.status_path`. + +Here is a sample nginx configuration to forward requests to the PHP-FPM status +page (assuming `pm.status_path` is configured with default value `/status`): +```nginx +location ~ /status { + allow 127.0.0.1; + deny all; + include fastcgi_params; + fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name; + fastcgi_pass 127.0.0.1:9000; +} +``` + +[float] +=== Compatibility + +The PHP-FPM metricsets were tested with PHP 5.6.29 and are expected to +work with all versions >= 5. + + +[float] +=== Example Configuration + +The php_fpm module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +#- module: php_fpm + #metricsets: ["pool"] + #enabled: true + #period: 10s + #status_path: "/status" + #hosts: ["localhost:8080"] +---- + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +include::php_fpm/pool.asciidoc[] + diff --git a/metricbeat/docs/modules/php_fpm/pool.asciidoc b/metricbeat/docs/modules/php_fpm/pool.asciidoc new file mode 100644 index 000000000000..2c147e2e7e5a --- /dev/null +++ b/metricbeat/docs/modules/php_fpm/pool.asciidoc @@ -0,0 +1,19 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[metricbeat-metricset-php_fpm-pool]] +include::../../../module/php_fpm/pool/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/php_fpm/pool/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index 74b85359d7c5..6d2b1b45333f 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -10,6 +10,7 @@ This file is generated! See scripts/docs_collector.py * <> * <> * <> + * <> * <> * <> * <> @@ -27,6 +28,7 @@ include::modules/kafka.asciidoc[] include::modules/mongodb.asciidoc[] include::modules/mysql.asciidoc[] include::modules/nginx.asciidoc[] +include::modules/php_fpm.asciidoc[] include::modules/postgresql.asciidoc[] include::modules/prometheus.asciidoc[] include::modules/redis.asciidoc[] diff --git a/metricbeat/include/list.go b/metricbeat/include/list.go index 3d88f2937d8d..aec198929878 100644 --- a/metricbeat/include/list.go +++ b/metricbeat/include/list.go @@ -34,6 +34,8 @@ import ( _ "github.com/elastic/beats/metricbeat/module/mysql/status" _ "github.com/elastic/beats/metricbeat/module/nginx" _ "github.com/elastic/beats/metricbeat/module/nginx/stubstatus" + _ "github.com/elastic/beats/metricbeat/module/php_fpm" + _ "github.com/elastic/beats/metricbeat/module/php_fpm/pool" _ "github.com/elastic/beats/metricbeat/module/postgresql" _ "github.com/elastic/beats/metricbeat/module/postgresql/activity" _ "github.com/elastic/beats/metricbeat/module/postgresql/bgwriter" diff --git a/metricbeat/metricbeat.full.yml b/metricbeat/metricbeat.full.yml index 683008110ede..ede25a1c8ed1 100644 --- a/metricbeat/metricbeat.full.yml +++ b/metricbeat/metricbeat.full.yml @@ -200,6 +200,14 @@ metricbeat.modules: # Path to server status. Default server-status #server_status_path: "server-status" +#------------------------------- php_fpm Module ------------------------------ +#- module: php_fpm + #metricsets: ["pool"] + #enabled: true + #period: 10s + #status_path: "/status" + #hosts: ["localhost:8080"] + #----------------------------- PostgreSQL Module ----------------------------- #- module: postgresql #metricsets: diff --git a/metricbeat/metricbeat.template-es2x.json b/metricbeat/metricbeat.template-es2x.json index f3d85fbb414d..67d90128a557 100644 --- a/metricbeat/metricbeat.template-es2x.json +++ b/metricbeat/metricbeat.template-es2x.json @@ -2168,6 +2168,42 @@ } } }, + "php_fpm": { + "properties": { + "pool": { + "properties": { + "connections": { + "properties": { + "accepted": { + "type": "long" + }, + "queued": { + "type": "long" + } + } + }, + "pool": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "processes": { + "properties": { + "active": { + "type": "long" + }, + "idle": { + "type": "long" + } + } + }, + "slow_requests": { + "type": "long" + } + } + } + } + }, "postgresql": { "properties": { "activity": { diff --git a/metricbeat/metricbeat.template.json b/metricbeat/metricbeat.template.json index d42d1cd726dd..5d4b27fb5153 100644 --- a/metricbeat/metricbeat.template.json +++ b/metricbeat/metricbeat.template.json @@ -2147,6 +2147,41 @@ } } }, + "php_fpm": { + "properties": { + "pool": { + "properties": { + "connections": { + "properties": { + "accepted": { + "type": "long" + }, + "queued": { + "type": "long" + } + } + }, + "pool": { + "ignore_above": 1024, + "type": "keyword" + }, + "processes": { + "properties": { + "active": { + "type": "long" + }, + "idle": { + "type": "long" + } + } + }, + "slow_requests": { + "type": "long" + } + } + } + } + }, "postgresql": { "properties": { "activity": { diff --git a/metricbeat/module/php_fpm/_meta/Dockerfile b/metricbeat/module/php_fpm/_meta/Dockerfile new file mode 100644 index 000000000000..7e70cdb2b8ad --- /dev/null +++ b/metricbeat/module/php_fpm/_meta/Dockerfile @@ -0,0 +1,4 @@ +FROM richarvey/nginx-php-fpm + +RUN echo "pm.status_path = /status" >> /etc/php7/php-fpm.d/www.conf +ADD ./php-fpm.conf /etc/nginx/sites-enabled diff --git a/metricbeat/module/php_fpm/_meta/config.yml b/metricbeat/module/php_fpm/_meta/config.yml new file mode 100644 index 000000000000..89527a864c42 --- /dev/null +++ b/metricbeat/module/php_fpm/_meta/config.yml @@ -0,0 +1,6 @@ +#- module: php_fpm + #metricsets: ["pool"] + #enabled: true + #period: 10s + #status_path: "/status" + #hosts: ["localhost:8080"] diff --git a/metricbeat/module/php_fpm/_meta/docs.asciidoc b/metricbeat/module/php_fpm/_meta/docs.asciidoc new file mode 100644 index 000000000000..4cd1272731a8 --- /dev/null +++ b/metricbeat/module/php_fpm/_meta/docs.asciidoc @@ -0,0 +1,28 @@ +== PHP-FPM Module + +This module periodically fetches metrics from https://php-fpm.org[PHP-FPM] +servers. + +[float] +=== Module-Specific Configuration Notes + +You need to enable the PHP-FPM status page by properly configuring +`pm.status_path`. + +Here is a sample nginx configuration to forward requests to the PHP-FPM status +page (assuming `pm.status_path` is configured with default value `/status`): +```nginx +location ~ /status { + allow 127.0.0.1; + deny all; + include fastcgi_params; + fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name; + fastcgi_pass 127.0.0.1:9000; +} +``` + +[float] +=== Compatibility + +The PHP-FPM metricsets were tested with PHP 5.6.29 and are expected to +work with all versions >= 5. diff --git a/metricbeat/module/php_fpm/_meta/fields.yml b/metricbeat/module/php_fpm/_meta/fields.yml new file mode 100644 index 000000000000..a8a8ac8ea361 --- /dev/null +++ b/metricbeat/module/php_fpm/_meta/fields.yml @@ -0,0 +1,14 @@ +- key: php_fpm + title: "php_fpm" + description: > + PHP-FPM server status metrics collected from PHP-FPM. + + experimental[] + short_config: false + fields: + - name: php_fpm + type: group + description: > + `php_fpm` contains the metrics that were obtained from PHP-FPM status + page call. + fields: diff --git a/metricbeat/module/php_fpm/_meta/php-fpm.conf b/metricbeat/module/php_fpm/_meta/php-fpm.conf new file mode 100644 index 000000000000..470698d5abd2 --- /dev/null +++ b/metricbeat/module/php_fpm/_meta/php-fpm.conf @@ -0,0 +1,16 @@ +server { + listen 81; ## listen for ipv4; this line is default and implied + listen [::]:81 default ipv6only=on; ## listen for ipv6 + + # Make site accessible from http://localhost/ + server_name _; + + error_log /dev/stdout info; + access_log /dev/stdout; + + location ~ /status { + fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name; + fastcgi_pass unix:/var/run/php-fpm.sock; + include fastcgi_params; + } +} diff --git a/metricbeat/module/php_fpm/doc.go b/metricbeat/module/php_fpm/doc.go new file mode 100644 index 000000000000..3fed1f04cf7d --- /dev/null +++ b/metricbeat/module/php_fpm/doc.go @@ -0,0 +1,4 @@ +/* +Package php_fpm is a Metricbeat module that contains MetricSets. +*/ +package php_fpm diff --git a/metricbeat/module/php_fpm/php_fpm.go b/metricbeat/module/php_fpm/php_fpm.go new file mode 100644 index 000000000000..e20d352e3dba --- /dev/null +++ b/metricbeat/module/php_fpm/php_fpm.go @@ -0,0 +1,59 @@ +package php_fpm + +import ( + "fmt" + "io" + "net/http" + + "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/metricbeat/mb/parse" +) + +const ( + defaultScheme = "http" + defaultPath = "/status" +) + +// HostParser is used for parsing the configured php-fpm hosts. +var HostParser = parse.URLHostParserBuilder{ + DefaultScheme: defaultScheme, + DefaultPath: defaultPath, + QueryParams: "json", + PathConfigKey: "status_path", +}.Build() + +// StatsClient provides access to php-fpm stats api +type StatsClient struct { + address string + user string + password string + http *http.Client +} + +// NewStatsClient creates a new StatsClient +func NewStatsClient(m mb.BaseMetricSet) *StatsClient { + return &StatsClient{ + address: m.HostData().SanitizedURI, + user: m.HostData().User, + password: m.HostData().Password, + http: &http.Client{Timeout: m.Module().Config().Timeout}, + } +} + +// Fetch php-fpm stats +func (c *StatsClient) Fetch() (io.ReadCloser, error) { + req, err := http.NewRequest("GET", c.address, nil) + if c.user != "" || c.password != "" { + req.SetBasicAuth(c.user, c.password) + } + resp, err := c.http.Do(req) + if err != nil { + return nil, fmt.Errorf("error making http request: %v", err) + } + + if resp.StatusCode != 200 { + return nil, fmt.Errorf("HTTP error %d: %s", resp.StatusCode, resp.Status) + } + + return resp.Body, nil +} diff --git a/metricbeat/module/php_fpm/php_fpm_test.go b/metricbeat/module/php_fpm/php_fpm_test.go new file mode 100644 index 000000000000..e5e6fe1342d3 --- /dev/null +++ b/metricbeat/module/php_fpm/php_fpm_test.go @@ -0,0 +1,30 @@ +package php_fpm + +import ( + "testing" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + + "github.com/stretchr/testify/assert" +) + +func TestHostParser(t *testing.T) { + tests := []struct { + host, expected string + }{ + {"localhost", "http://localhost/status?json="}, + {"localhost:123", "http://localhost:123/status?json="}, + {"http://localhost:123", "http://localhost:123/status?json="}, + } + + m := mbtest.NewTestModule(t, map[string]interface{}{}) + + for _, test := range tests { + hi, err := HostParser(m, test.host) + if err != nil { + t.Error("failed on", test.host, err) + continue + } + assert.Equal(t, test.expected, hi.URI) + } +} diff --git a/metricbeat/module/php_fpm/pool/_meta/data.json b/metricbeat/module/php_fpm/pool/_meta/data.json new file mode 100644 index 000000000000..06cb28526330 --- /dev/null +++ b/metricbeat/module/php_fpm/pool/_meta/data.json @@ -0,0 +1,25 @@ +{ + "@timestamp": "2017-01-18T23:57:23.960Z", + "beat": { + "hostname": "host.example.com", + "name": "host.example.com" + }, + "metricset": { + "host": "localhost:8081", + "module": "php_fpm", + "name": "pool", + "rtt": 1237 + }, + "php_fpm": { + "pool": { + "connections.accepted": 803, + "connections.queued": 0, + "hostname": "localhost:8081", + "pool": "www", + "processes.active": 1, + "processes.idle": 2, + "requests.slow": 0 + } + }, + "type": "metricsets" +} diff --git a/metricbeat/module/php_fpm/pool/_meta/docs.asciidoc b/metricbeat/module/php_fpm/pool/_meta/docs.asciidoc new file mode 100644 index 000000000000..0e158655e080 --- /dev/null +++ b/metricbeat/module/php_fpm/pool/_meta/docs.asciidoc @@ -0,0 +1,3 @@ +=== php_fpm pool MetricSet + +This is the pool metricset of the module php_fpm. diff --git a/metricbeat/module/php_fpm/pool/_meta/fields.yml b/metricbeat/module/php_fpm/pool/_meta/fields.yml new file mode 100644 index 000000000000..ba22858f91ed --- /dev/null +++ b/metricbeat/module/php_fpm/pool/_meta/fields.yml @@ -0,0 +1,53 @@ +- name: pool + type: group + description: > + `pool` contains the metrics that were obtained from the PHP-FPM process + pool. + fields: + - name: pool + type: keyword + description: > + The name of the pool. + - name: connections + type: group + description: > + Connection state specific statistics. + fields: + - name: accepted + type: long + description: > + The number of incoming requests that the PHP-FPM server has accepted; + when a connection is accepted it is removed from the listen queue. + - name: queued + type: long + description: > + The current number of connections that have been initiated, but not + yet accepted. If this value is non-zero it typically means that all + the available server processes are currently busy, and there are no + processes available to serve the next request. Raising + `pm.max_children` (provided the server can handle it) should help + keep this number low. This property follows from the fact that + PHP-FPM listens via a socket (TCP or file based), and thus inherits + some of the characteristics of sockets. + - name: processes + type: group + description: > + Process state specific statistics. + fields: + - name: idle + type: long + description: > + The number of servers in the `waiting to process` state (i.e. not + currently serving a page). This value should fall between the + `pm.min_spare_servers` and `pm.max_spare_servers` values when the + process manager is `dynamic`. + - name: active + type: long + description: > + The number of servers current processing a page - the minimum is `1` + (so even on a fully idle server, the result will be not read `0`). + - name: slow_requests + type: long + description: > + The number of times a request execution time has exceeded + `request_slowlog_timeout`. diff --git a/metricbeat/module/php_fpm/pool/data.go b/metricbeat/module/php_fpm/pool/data.go new file mode 100644 index 000000000000..48a797f36cf1 --- /dev/null +++ b/metricbeat/module/php_fpm/pool/data.go @@ -0,0 +1,18 @@ +package pool + +type poolStats struct { + Pool string `json:"pool"` + ProcessManager string `json:"process manager"` + StartTime int `json:"start time"` + StartSince int `json:"start since"` + AcceptedConn int `json:"accepted conn"` + ListenQueue int `json:"listen queue"` + MaxListenQueue int `json:"max listen queue"` + ListenQueueLen int `json:"listen queue len"` + IdleProcesses int `json:"idle processes"` + ActiveProcesses int `json:"active processes"` + TotalProcesses int `json:"total processes"` + MaxActiveProcesses int `json:"max active processes"` + MaxChildrenReached int `json:"max children reached"` + SlowRequests int `json:"slow requests"` +} diff --git a/metricbeat/module/php_fpm/pool/pool.go b/metricbeat/module/php_fpm/pool/pool.go new file mode 100644 index 000000000000..982fe1e3899b --- /dev/null +++ b/metricbeat/module/php_fpm/pool/pool.go @@ -0,0 +1,74 @@ +package pool + +import ( + "encoding/json" + "fmt" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/beats/metricbeat/mb" + + "github.com/elastic/beats/metricbeat/module/php_fpm" +) + +// init registers the MetricSet with the central registry. +// The New method will be called after the setup of the module and before starting to fetch data +func init() { + if err := mb.Registry.AddMetricSet("php_fpm", "pool", New, php_fpm.HostParser); err != nil { + panic(err) + } +} + +// MetricSet type defines all fields of the MetricSet +// As a minimum it must inherit the mb.BaseMetricSet fields, but can be extended with +// additional entries. These variables can be used to persist data or configuration between +// multiple fetch calls. +type MetricSet struct { + mb.BaseMetricSet + client *php_fpm.StatsClient // StatsClient that is reused across requests. +} + +// New create a new instance of the MetricSet +// Part of new is also setting up the configuration by processing additional +// configuration entries if needed. +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + logp.Warn("EXPERIMENTAL: The php-fpm pool metricset is experimental") + return &MetricSet{ + BaseMetricSet: base, + client: php_fpm.NewStatsClient(base), + }, nil +} + +// Fetch methods implements the data gathering and data conversion to the right format +// It returns the event which is then forward to the output. In case of an error, a +// descriptive error must be returned. +func (m *MetricSet) Fetch() (common.MapStr, error) { + body, err := m.client.Fetch() + + if err != nil { + return nil, err + } + + defer body.Close() + + stats := &poolStats{} + err = json.NewDecoder(body).Decode(stats) + if err != nil { + return nil, fmt.Errorf("error parsing json: %v", err) + } + + return common.MapStr{ + "hostname": m.Host(), + + "pool": stats.Pool, + "connections": common.MapStr{ + "queue": stats.ListenQueue, + "accepted": stats.AcceptedConn, + }, + "processes": common.MapStr{ + "idle": stats.IdleProcesses, + "active": stats.ActiveProcesses, + }, + "slow_requests": stats.SlowRequests, + }, nil +} diff --git a/metricbeat/module/php_fpm/pool/pool_integration_test.go b/metricbeat/module/php_fpm/pool/pool_integration_test.go new file mode 100644 index 000000000000..b5026cfa7d20 --- /dev/null +++ b/metricbeat/module/php_fpm/pool/pool_integration_test.go @@ -0,0 +1,25 @@ +// +build integration + +package pool + +import ( + "testing" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" +) + +func TestData(t *testing.T) { + f := mbtest.NewEventFetcher(t, getConfig()) + err := mbtest.WriteEvent(f, t) + if err != nil { + t.Fatal("write", err) + } +} + +func getConfig() map[string]interface{} { + return map[string]interface{}{ + "module": "php_fpm", + "metricsets": []string{"pool"}, + "hosts": []string{"127.0.0.1:81"}, + } +}