diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 2d11a7c6202b..d968af509b01 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -207,6 +207,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Release http.server metricset as GA. {pull}10240[10240] - Making RabbitMQ Metricbeat module GA. {pull}10165[10165] - Release use of xpack.enabled: true flag in Elasticsearch and Kibana modules as GA. {pull}10222[10222] +- Rename 'db' Metricset to 'transaction_log' in MSSQL Metricbeat module {pull}10109[10109] *Packetbeat* diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 3601ca7020e8..cdc921bdb890 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -16613,84 +16613,27 @@ MS SQL module -[float] -== db fields - -db metricset will fetch information about each db from a sql server - - [float] == database fields -Database where the metrics have been fetched from - - -*`mssql.db.database.id`*:: -+ --- -type: keyword - -The database ID - --- - -[float] -== log_space_usage fields - -Space usage information for the transaction log - - -[float] -== total fields - -Total space usage +The database that the metrics is being referred to -*`mssql.db.log_space_usage.total.bytes`*:: +*`mssql.database.id`*:: + -- type: long -The size of the log +Unique ID of the database inside MSSQL -- -[float] -== used fields - -The occupied size of the log - - -*`mssql.db.log_space_usage.used.bytes`*:: +*`mssql.database.name`*:: + -- -type: long - -The occupied size of the log in bytes - --- - -*`mssql.db.log_space_usage.used.pct`*:: -+ --- -type: float - -A percentaje of the occupied size of the log as a percent of the total log size - --- - -[float] -== since_last_backup fields - -The amount of space used since the last log backup - - -*`mssql.db.log_space_usage.since_last_backup.bytes`*:: -+ --- -type: long +type: keyword -The amount of space used since the last log backup in bytes +Name of the database -- @@ -16857,6 +16800,147 @@ Ideal number of pages in the buffer pool. -- +[float] +== transaction_log fields + +transaction_log metricset will fetch information about the operation and transaction log of each database from a MSSQL instance + + +[float] +== space_usage fields + +Space usage information for the transaction log + + +[float] +== since_last_backup fields + +The amount of space used since the last log backup + + +*`mssql.transaction_log.space_usage.since_last_backup.bytes`*:: ++ +-- +type: long + +The amount of space used since the last log backup in bytes + +-- + +[float] +== total fields + +The size of the log + + +*`mssql.transaction_log.space_usage.total.bytes`*:: ++ +-- +type: long + +The size of the log in bytes + +-- + +[float] +== used fields + +The occupied size of the log + + +*`mssql.transaction_log.space_usage.used.bytes`*:: ++ +-- +type: long + +The occupied size of the log in bytes + +-- + +*`mssql.transaction_log.space_usage.used.pct`*:: ++ +-- +type: float + +A percentage of the occupied size of the log as a percent of the total log size + +-- + +[float] +== stats fields + +Returns summary level attributes and information on transaction log files of databases. Use this information for monitoring and diagnostics of transaction log health. + + +[float] +== active_size fields + +Total active transaction log size. + + +*`mssql.transaction_log.stats.active_size.bytes`*:: ++ +-- +type: long + +Total active transaction log size in bytes + +-- + +*`mssql.transaction_log.stats.backup_time`*:: ++ +-- +type: date + +Last transaction log backup time. + +-- + +[float] +== recovery_size fields + +Log size since log recovery log sequence number (LSN). + + +*`mssql.transaction_log.stats.recovery_size.bytes`*:: ++ +-- +type: long + +Log size in bytes since log recovery log sequence number (LSN). + +-- + +[float] +== since_last_checkpoint fields + +Log size since last checkpoint log sequence number (LSN). + + +*`mssql.transaction_log.stats.since_last_checkpoint.bytes`*:: ++ +-- +type: long + +Log size in bytes since last checkpoint log sequence number (LSN). + +-- + +[float] +== total_size fields + +Total transaction log size. + + +*`mssql.transaction_log.stats.total_size.bytes`*:: ++ +-- +type: long + +Total transaction log size in bytes. + +-- + [[exported-fields-munin]] == Munin fields diff --git a/x-pack/metricbeat/include/list.go b/x-pack/metricbeat/include/list.go index d6fb7ee60c27..4f662b04b824 100644 --- a/x-pack/metricbeat/include/list.go +++ b/x-pack/metricbeat/include/list.go @@ -11,6 +11,6 @@ import ( _ "github.com/elastic/beats/x-pack/metricbeat/module/aws" _ "github.com/elastic/beats/x-pack/metricbeat/module/aws/ec2" _ "github.com/elastic/beats/x-pack/metricbeat/module/mssql" - _ "github.com/elastic/beats/x-pack/metricbeat/module/mssql/db" _ "github.com/elastic/beats/x-pack/metricbeat/module/mssql/performance" + _ "github.com/elastic/beats/x-pack/metricbeat/module/mssql/transaction_log" ) diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index dc0698dac5ac..39ada9fd1734 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -507,7 +507,7 @@ metricbeat.modules: #-------------------------------- Mssql Module -------------------------------- - module: mssql metricsets: - - "db" + - "transaction_log" - "performance" hosts: ["sqlserver://sa@localhost"] period: 10s diff --git a/x-pack/metricbeat/module/mssql/_meta/config.yml b/x-pack/metricbeat/module/mssql/_meta/config.yml index 038955a319cc..c79d01d06214 100644 --- a/x-pack/metricbeat/module/mssql/_meta/config.yml +++ b/x-pack/metricbeat/module/mssql/_meta/config.yml @@ -1,6 +1,6 @@ - module: mssql metricsets: - - "db" + - "transaction_log" - "performance" hosts: ["sqlserver://sa@localhost"] period: 10s diff --git a/x-pack/metricbeat/module/mssql/_meta/docs.asciidoc b/x-pack/metricbeat/module/mssql/_meta/docs.asciidoc index 54b9b3d8aab1..17a88ce93ed6 100644 --- a/x-pack/metricbeat/module/mssql/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/mssql/_meta/docs.asciidoc @@ -8,8 +8,12 @@ The module is being tested with https://hub.docker.com/r/microsoft/mssql-server- [float] == Metricsets -The following Metricset is already included: +The following Metricsets are already included: -=== `db` +=== `transaction_log` -`db` Metricset fetches a set of values from https://docs.microsoft.com/en-us/sql/relational-databases/system-dynamic-management-views/sys-dm-db-log-space-usage-transact-sql?view=sql-server-2017[log space usage] which returns space usage information for the transaction log for each different database. +`transaction_log` Metricset fetches information about the operation and transaction log of each MSSQL database in the monitored instance. All data is extracted from the https://docs.microsoft.com/en-us/sql/relational-databases/system-dynamic-management-views/database-related-dynamic-management-views-transact-sql?view=sql-server-2017[Database Dynamic Management Views] + +=== `performance` + +`performance` Metricset fetches information from what's commonly known as https://docs.microsoft.com/en-us/sql/relational-databases/system-dynamic-management-views/sys-dm-os-performance-counters-transact-sql?view=sql-server-2017[Performance Counters] in MSSQL. diff --git a/x-pack/metricbeat/module/mssql/_meta/fields.yml b/x-pack/metricbeat/module/mssql/_meta/fields.yml index 9701a0776860..96c871f9ed97 100644 --- a/x-pack/metricbeat/module/mssql/_meta/fields.yml +++ b/x-pack/metricbeat/module/mssql/_meta/fields.yml @@ -7,3 +7,13 @@ type: group description: > fields: + - name: database + type: group + description: The database that the metrics is being referred to + fields: + - name: id + type: long + description: Unique ID of the database inside MSSQL + - name: name + type: keyword + description: Name of the database diff --git a/x-pack/metricbeat/module/mssql/connection.go b/x-pack/metricbeat/module/mssql/connection.go index 3528e2371beb..fa4c0fa36aeb 100644 --- a/x-pack/metricbeat/module/mssql/connection.go +++ b/x-pack/metricbeat/module/mssql/connection.go @@ -7,6 +7,9 @@ package mssql import ( "database/sql" + // Register driver. + _ "github.com/denisenkom/go-mssqldb" + "github.com/pkg/errors" ) diff --git a/x-pack/metricbeat/module/mssql/db/_meta/data.json b/x-pack/metricbeat/module/mssql/db/_meta/data.json deleted file mode 100644 index c1be9ff8d414..000000000000 --- a/x-pack/metricbeat/module/mssql/db/_meta/data.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "@timestamp": "2017-10-12T08:05:34.853Z", - "agent": { - "hostname": "host.example.com", - "name": "host.example.com" - }, - "event": { - "dataset": "mssql.db", - "duration": 115000, - "module": "mssql" - }, - "metricset": { - "name": "db" - }, - "mssql": { - "db": { - "database": { - "id": "1" - }, - "log_space_usage": { - "since_last_backup": { - "bytes": 462848 - }, - "total": { - "bytes": 2088960 - }, - "used": { - "bytes": 1028096, - "pct": 49.2156867980957 - } - } - } - }, - "service": { - "address": "172.26.0.2", - "type": "mssql" - } -} \ No newline at end of file diff --git a/x-pack/metricbeat/module/mssql/db/_meta/docs.asciidoc b/x-pack/metricbeat/module/mssql/db/_meta/docs.asciidoc deleted file mode 100644 index d2a92f914eb5..000000000000 --- a/x-pack/metricbeat/module/mssql/db/_meta/docs.asciidoc +++ /dev/null @@ -1 +0,0 @@ -`db` Metricset fetches a set of values from https://docs.microsoft.com/en-us/sql/relational-databases/system-dynamic-management-views/sys-dm-db-log-space-usage-transact-sql?view=sql-server-2017[log space usage] which returns space usage information for the transaction log for each different database. diff --git a/x-pack/metricbeat/module/mssql/db/_meta/fields.yml b/x-pack/metricbeat/module/mssql/db/_meta/fields.yml deleted file mode 100644 index 4cceaa4fa6da..000000000000 --- a/x-pack/metricbeat/module/mssql/db/_meta/fields.yml +++ /dev/null @@ -1,39 +0,0 @@ -- name: db - type: group - description: db metricset will fetch information about each db from a sql server - fields: - - name: database - type: group - description: Database where the metrics have been fetched from - fields: - - name: id - description: The database ID - type: keyword - - name: log_space_usage - type: group - description: Space usage information for the transaction log - fields: - - name: total - type: group - description: Total space usage - fields: - - name: bytes - description: The size of the log - type: long - - name: used - type: group - description: The occupied size of the log - fields: - - name: bytes - description: The occupied size of the log in bytes - type: long - - name: pct - description: A percentaje of the occupied size of the log as a percent of the total log size - type: float - - name: since_last_backup - description: The amount of space used since the last log backup - type: group - fields: - - name: bytes - description: The amount of space used since the last log backup in bytes - type: long diff --git a/x-pack/metricbeat/module/mssql/db/data.go b/x-pack/metricbeat/module/mssql/db/data.go deleted file mode 100644 index 76fdea7257a9..000000000000 --- a/x-pack/metricbeat/module/mssql/db/data.go +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package db - -import ( - s "github.com/elastic/beats/libbeat/common/schema" - c "github.com/elastic/beats/libbeat/common/schema/mapstrstr" -) - -var ( - schema = s.Schema{ - "database": s.Object{ - "id": c.Str("database_id"), - }, - // Returns space usage information for the transaction log. - "log_space_usage": s.Object{ - "total": s.Object{ - "bytes": c.Int("total_log_size_in_bytes"), - }, - "used": s.Object{ - "bytes": c.Int("used_log_space_in_bytes"), - "pct": c.Float("used_log_space_in_percent"), - }, - "since_last_backup": s.Object{ - "bytes": c.Int("log_space_in_bytes_since_last_backup"), - }, - }, - } -) diff --git a/x-pack/metricbeat/module/mssql/db/db.go b/x-pack/metricbeat/module/mssql/db/db.go deleted file mode 100644 index d46a22885fec..000000000000 --- a/x-pack/metricbeat/module/mssql/db/db.go +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package db - -import ( - "github.com/pkg/errors" - - "github.com/elastic/beats/libbeat/common/cfgwarn" - "github.com/elastic/beats/libbeat/logp" - "github.com/elastic/beats/metricbeat/mb" - "github.com/elastic/beats/x-pack/metricbeat/module/mssql" -) - -func init() { - mb.Registry.MustAddMetricSet("mssql", "db", New, - mb.DefaultMetricSet(), - mb.WithHostParser(mssql.HostParser)) -} - -// MetricSet type defines all fields of the MetricSet -type MetricSet struct { - mb.BaseMetricSet - log *logp.Logger - fetcher *mssql.Fetcher -} - -// New create a new instance of the MetricSet -func New(base mb.BaseMetricSet) (mb.MetricSet, error) { - cfgwarn.Beta("The mssql db metricset is beta.") - - logger := logp.NewLogger("mssql.db").With("host", base.HostData().SanitizedURI) - - fetcher, err := mssql.NewFetcher(base.HostData().URI, - []string{"SELECT * FROM sys.dm_db_log_space_usage;"}, &schema, logger) - if err != nil { - return nil, errors.Wrap(err, "error creating fetcher") - } - - return &MetricSet{ - BaseMetricSet: base, - log: logger, - fetcher: fetcher, - }, nil -} - -// Fetch methods implements the data gathering and data conversion to the right format -// It returns the event which is then forward to the output. In case of an error, a -// descriptive error must be returned. -func (m *MetricSet) Fetch(reporter mb.ReporterV2) { - m.fetcher.Report(reporter) -} - -// Close the connection to the server at the engine level -func (m *MetricSet) Close() error { - return m.fetcher.Close() -} diff --git a/x-pack/metricbeat/module/mssql/fetcher.go b/x-pack/metricbeat/module/mssql/fetcher.go deleted file mode 100644 index 692d901ca9c6..000000000000 --- a/x-pack/metricbeat/module/mssql/fetcher.go +++ /dev/null @@ -1,155 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package mssql - -import ( - "database/sql" - - "github.com/pkg/errors" - - s "github.com/elastic/beats/libbeat/common/schema" - "github.com/elastic/beats/libbeat/logp" - "github.com/elastic/beats/metricbeat/mb" - - // Register driver. - _ "github.com/denisenkom/go-mssqldb" -) - -// NewFetcher is called from every metricset to initialize their fetching routines. It opens a different connection to -// the database for each fetch operation -func NewFetcher(uri string, qs []string, schema *s.Schema, log *logp.Logger) (*Fetcher, error) { - db, err := sql.Open("sqlserver", uri) - if err != nil { - return nil, errors.Wrap(err, "could not create db instance") - } - - // Check the connection before executing all queries to reduce the number - // of connection errors that we might encounter. - if err = db.Ping(); err != nil { - return nil, err - } - - f := &Fetcher{ - queries: qs, - schema: schema, - db: db, - log: log, - } - - return f, nil -} - -// Fetcher will make queries sequentially to the database to fetch results. It -// must be created by each metricset and fed with the queries that must be -// executed, a SQL implementor and a pointer to the schema to "apply". A -// metricset could, potentially, need more than one query to fill all its -// results. -type Fetcher struct { - schema *s.Schema // Schema is the metricset schema to apply to the fetched data to turn it into a result mapstr. - db *sql.DB // Database to execute queries against. - log *logp.Logger - - queries []string // List of queries to execute concurrently. -} - -// Close closes db connection to the server -func (f *Fetcher) Close() error { - return f.db.Close() -} - -// Report receives a mb.ReporterV2 to send the data to the outputs. Because the operations are common between metricsets -// it can be extracted to a common function -func (f *Fetcher) Report(reporter mb.ReporterV2) { - var err error - for _, q := range f.queries { - if err = f.getEventsWithQuery(f.db, q, reporter); err != nil { - logp.Error(err) - } - } -} - -type rowsResultHandler struct { - dest []interface{} - rows *sql.Rows - reporter mb.ReporterV2 - columnNames []string -} - -func newRowsResult(reporter mb.ReporterV2, columnNames []string, rows *sql.Rows) *rowsResultHandler { - return &rowsResultHandler{ - dest: make([]interface{}, len(columnNames)), - columnNames: columnNames, - rows: rows, - reporter: reporter, - } -} - -// getEventsWithQuery performs the query on the database and creates a rowsResultHandler to handle them -func (f *Fetcher) getEventsWithQuery(db *sql.DB, query string, reporter mb.ReporterV2) (err error) { - var rows *sql.Rows - rows, err = db.Query(query) - if err != nil { - return errors.Wrapf(err, "error performing db query='%v'", query) - } - defer func() { - if err2 := rows.Close(); err != nil { - if err != nil { - err = errors.Wrap(err, err2.Error()) - } else { - err = err2 - } - } - }() - - columnNames, err := rows.Columns() - if err != nil { - return errors.Wrap(err, "error getting column names") - } - - rowsResult := newRowsResult(reporter, columnNames, rows) - - if err = rowsResult.handle(f.schema); err != nil { - return errors.Wrap(err, "could not convert rows result to beats mapstr") - } - - return nil -} - -// handle is called with the result of each query. -func (rr *rowsResultHandler) handle(s *s.Schema) (err error) { - for rr.rows.Next() { - // We assign pointers to the destination to pass it to Scan. - rawResult := make([]*string, len(rr.columnNames)) - for i := range rawResult { - rr.dest[i] = &rawResult[i] - } - - if err = rr.rows.Scan(rr.dest...); err != nil { - return errors.Wrap(err, "error scanning row of result") - } - - // Now we need to get the values of the pointers back into a normal - // map[string]interface{} to use it with the schema. - mapOfResults := make(map[string]interface{}) - - for i, res := range rawResult { - if res != nil { - mapOfResults[rr.columnNames[i]] = *res - } - } - - result, err := s.Apply(mapOfResults) - if err != nil { - err = errors.Wrap(err, "error trying to apply schema") - logp.Error(err) - rr.reporter.Error(err) - continue - } - - rr.reporter.Event(mb.Event{MetricSetFields: result}) - } - - return nil -} diff --git a/x-pack/metricbeat/module/mssql/fields.go b/x-pack/metricbeat/module/mssql/fields.go index 724eac799a59..b95e5414d5c3 100644 --- a/x-pack/metricbeat/module/mssql/fields.go +++ b/x-pack/metricbeat/module/mssql/fields.go @@ -18,5 +18,5 @@ func init() { // Asset returns asset data func Asset() string { - return "eJy8WE9v27gTvedTDHpqgSS95/AD+tsCuwX2L9K7MKJGEjcUqXJGdr2ffjGUZEu25NhusLkElsjhm8fHN0M9wAvtnqBh/ubuAMSKoyd4l36/uwMoiE20rdjgn+C3Z3j+61doQtE5ugOI5AiZniAnwTuA0pIr+OkOAOABPDZ0iKx/smvpCaoYunZ4Mgv/v+HhNMw0VJHvHy3FOolX5NCQRGuYBLbWOShJTA3WlyE2qIMA89AJEJpah5cxNIDA3xwwxQ3FSehjVDNkKJgj0+zlGsYTnJ+H2bCtKRJITSNuqHFDkBP5HjoVCeJRsCVkU3S2OHl1hOBrTfsc4MvnheF9Ki+024ZYLHLgQpVxi4ayjrG6kYpnDQApwGybyhATLxLRM5r0zIXqSiIkCLrV5JbAnTKlIYAPMBcmrMGYQsl3Qrw4YmFr2P5DEMrEwGnW8yxc8KcDxlU7piUtXJN/TRCM6VpLxQW43pyKtbXB+rOBzpAzxdIauQTJJ2gpGvKCf+9BrCJDBhzHj8+TEtNbHX0Wc+kCnoIaAbP1hjKHLFmO5uWyDcQmdD2WUcgJtje9+Wi0hG014nnBvPmmX4f3Binst59i8hxv6LpSM5k4qTm9Z/NCwVHYf07m/KQJUuQL602LFWXcOiuctRQzJrPotwt6n8H+vWtyikqsRoQ+oiYDTCb4AqRGScqOKmNFHYk7l/YibCiWLmyt19NX0PcUhB/3C66UCfOSbfHtkGtAiPStI17Ari9spCJhN+gcRZAACuAVoB1TzEzwnlK5OVbTZTD7cuH3YDUoLAVdhDCpdsfLn13lzLwF2ItL6+wNZUJNmwnm7uQ0reyGTggR4w76WR/TP9hgtOm3Hs6O6fFGWBPqskhM8kMiOubNhcp6BhaMMrRavW72q0IbgntV4RplBdg16x+0/Ahfa8tQBGLwQcB647qCEhoqpqzcSqwLVehWz+RrqNXSQksxWdxiAjfCimRC01rXB34jx2BBoUar8RieeEZ2cuPe7iaHyjbEy3NRu/Zoq4oiFY/wM3mK6NzuHnahgy16GaxzP0MC5NodbF/R0gXZrySpF7Xp7FmCX3xhDQot59hLPiEFE1RmKDVYBtIaRWl+0h/DPiGDqSjnuwNDD4425I62UM+/QntO16tH+FQKxcmT3iGT+1jZ6aKcHOQeRE/ABl2nRGKqqwgshMWw5q0ay1FMnY0V5DqWvw5O+zDQ3aAv+oDEmjnZzdJBVryWxRpNEMuSjPTkoXN6mlkiWpXge+5MraX3y8c/7o/qCN+DUR5SC3mftsLRd2UtlPuCeA+KiAME/+ERfrFV3cM7VMyG0EMVgpbIGLqqbju5mcquLGeXZjjbLL52X0vpZbVd6smvuLPM1T527hWNfQ9DGTpfqDR1QJ/EwO3WSq3uVuNG+xwJKr6hNBSWX46ZOpcWXH/R0O43uaoK5XBxOAihR1lr21bYjS16Ga2PdCG8dC2n5u3QPJe0Bc2TVSypFURjiJl4PKGYdl/Js6EYreK+t7WErwkbYthQ3IGzIo4e4f+UXCExpuQd6mmCYhkaVbcjZqDvLXm2G+3o0c+nKM8zK7U8LNrbaG2r+myvX4Qud7QsXThuq50tKVM4RtCb3VtK77AZvRuki6GynT5OseDuSINa3/cSjFRSJG/opMhfKrpTW/sPQcN768cQH5YygAuMZpqNqcm8tMF6ydIpXnHui9MbrMB1XOt9ISTdTS8U6s+TVSFECFLr5LH1md05kpkXNspuCN2rdVhgiYFrsh8/1/W5/1jOK/t3+CRoghfyJ2UBzurummwEY0VyYS4FzYxtLYeb0f4bAAD//5U2pe8=" + return "eJzMWM+P27gOvs9fQfTUAjPpfQ4P6HsP2C0w7W53pmdDkWlbGFnyiHTS7F+/oOQkjmPnd4vtoYPYFvnxI/WR0gO84uoRaqI3ewfAhi0+wrv4+90dQI6kg2nYePcIX57h+dsT1D5vLd4BBLSoCB9hjqzuAAqDNqfHOwCAB3Cqxq1l+cerBh+hDL5tuic75v/TPeyb6ZvKFau5Ity8GLO4Z/Wlws1K4EoxcIVQIwejCQzBHI0rIWCBIWAO7HuWhlD6cEy+83gNxnpXDl7s4PnuzFuL8Pn/4IsIZQPOODI5wpfn529Pox7l/1Gfr7ha+jDEs+P2q6px6HGP4QZD4UOtnD6T5N7CNbXIUCDrCgmMiy/lU1Bz36YU/Nlb8z/fOsZAJ1LfqBIzaqxhyhoMGaG+KBdf23qOQWgRi5AsSjBAqL3LU714rdsAiiLqgNRaliV+gaGwfinVY1yOP6IRmm0cjkK3Xr9mS3U75GIQAr61SCPY5YWJZV0haGUtBmAPAuAI0JYwZNo7h1pc0kUwXzwrC24DVozCmNFRCByUIzXq/qCXA+tGYI+6ltULzBjrJmM1t3gQwDYbssAHFVaQVn2Mf2Chgom/wTghYXYhrB51WUBCvqqIhrxZXxpHQKwCYw5F8HWqm41XaLy3RytcrEwAO8f/tpZn8FIZgtwjgfMMxmnb5hjRYN5n5VJirS99O7knj6EWSfMNhihxowFcCCug9nVjbDJ8I8UgVow1Ot6YR9ohO6pxkrvepjI10vhaFRA4mLLEgPkMfkOHQVm7uoeVb2GpHHfSuVnBHuYI1i+P1NIJ0U8EKYNKf/VOgJ9dbrRiHI8xlXxECtpLmSmuZFJA6VEY18f6I9gEpFVLmMN8tWXoweIC7SCFsv8F2jOGBYYZfCoYQ+9JUsioPoZX4pSigtwDyw5YKNsKkSr2VQXEqPLO56U1Nlesq2zdQc5j+aVT2oeO7lq5PBlEksjRLMY2suA1xEZLgKooUHMiT1kru5k4KCMl+J5aXUnr/fzxj/tBH6F70MIDkPkb72MqLP4Q1nyxaYj3IIjIg3cfZvC7KasEb9sxa1QOSu+lRQbfllXT8sVUtkWBYXTxcHyCiRFnp/olvKwyvPfFIat76dqt9gaDRscy73RzD0HhW5dLacoHKYiO26XhStStUguZc9hL8XWtITf0OmTqUFj90Bo9FtQedBneo6pKoQg4HohvQlnJ2JabhclTGU1/ab1/bRuKw1v8zCqSQXUJEidJscRRUGmNREjrHapi9oU84/O1VNwnWYv4ar9AggWGFVjDbHEG/8WoCpExIW/bTyMUQ1BLdVskAvzRoCOziEcUt7tEeN6RUkOd0ySjlSmrCTJTleS+nVscL10YjtXWFJgJHM3K6dUtS2+bjKQGomCR7aWxVkRhNahB6e+bEownNHQa95r8qUW3L2u/EDS8N25t4sNYBHCC0PSj0RXq18Ybx1ncxRPKfXJ4nRTYlqp4DI511z9QiD73vIIP4LmSxevRZ+fMEcU8N4FXnelUrZ2DMQbOiX59hk2xXxfzRP62J3PtHaPbawtwsO7OiYZVKJFPjCXHHWGbiuFqtCPHsMz68rybgcHi3u1A3D/ximDigmBbViLLPUMyckvkMgZtkxS1UqXrEzCOeHCNcehCgRqlMWtJleMXLGNitxPmsxiAaGAnmsKnNjNAf+YgQMZpzKRVZXOlX4/rrvRNVcsYH4WrA4d5srRtfELkpMXDOn+K4M5XPFrPN8ArBX/I/OShqFfXMiFc199euvFzfbO2n9qfQtXA5Q24ELKvp8Jr3TYmpu2XczLl+wpy4PyB9dNgvuZDyFScJdL36+dpbJW38vVBzIX1isfljBWP33wdFbK/kNvgCKitayXDbDzEKuZg5q00UBVPCluFk64/kOYiHol9sdFmmsH3ePtuaE8da+8M+yDTrpjOjSqdl7MhDa7youkKleVq2NqO6Wd3mzfB6MjtTlqw510MTA8uv0Ypj+G7gRgkhc3kiDMZbK74KJdPItlDjJ18i/F9Kvv3XnKWmkrZGbr0tOYltRKBsLaeOMO3Vgb09UT1/un56+h8ftM0Pg2zdTU8OC23vVFiO8rflmHJeu+Y8C8k+RqEJ/IcdfwG5Zv2+6lC9BOk5qDGnEDQPwEAAP//0zqtWg==" } diff --git a/x-pack/metricbeat/module/mssql/performance/performance.go b/x-pack/metricbeat/module/mssql/performance/performance.go index f7efc1dc4bb3..0399e2dc4e8c 100644 --- a/x-pack/metricbeat/module/mssql/performance/performance.go +++ b/x-pack/metricbeat/module/mssql/performance/performance.go @@ -42,9 +42,8 @@ func init() { // interface methods except for Fetch. type MetricSet struct { mb.BaseMetricSet - log *logp.Logger - fetcher *mssql.Fetcher - db *sql.DB + log *logp.Logger + db *sql.DB } // New creates a new instance of the MetricSet. New is responsible for unpacking diff --git a/x-pack/metricbeat/module/mssql/transaction_log/_meta/data.json b/x-pack/metricbeat/module/mssql/transaction_log/_meta/data.json new file mode 100644 index 000000000000..ce46579d561a --- /dev/null +++ b/x-pack/metricbeat/module/mssql/transaction_log/_meta/data.json @@ -0,0 +1,54 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "agent": { + "hostname": "host.example.com", + "name": "host.example.com" + }, + "event": { + "dataset": "mssql.transaction_log", + "duration": 115000, + "module": "mssql" + }, + "metricset": { + "name": "transaction_log" + }, + "mssql": { + "database": { + "id": 1, + "name": "master" + }, + "transaction_log": { + "space_usage": { + "since_last_backup": { + "bytes": 135168 + }, + "total": { + "bytes": 2088960 + }, + "used": { + "bytes": 622592, + "pct": 29.80392074584961 + } + }, + "stats": { + "active_size": { + "bytes": 135167.737856 + }, + "backup_time": "1900-01-01T00:00:00Z", + "recovery_size": { + "bytes": 0.128906 + }, + "since_last_checkpoint": { + "bytes": 135167.737856 + }, + "total_size": { + "bytes": 2088959.475712 + } + } + } + }, + "service": { + "address": "172.26.0.2", + "type": "mssql" + } +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/mssql/transaction_log/_meta/docs.asciidoc b/x-pack/metricbeat/module/mssql/transaction_log/_meta/docs.asciidoc new file mode 100644 index 000000000000..72cb0e52fd68 --- /dev/null +++ b/x-pack/metricbeat/module/mssql/transaction_log/_meta/docs.asciidoc @@ -0,0 +1,11 @@ +`transaction_log` Metricset fetches information about the operation and transaction log of each MSSQL database in the monitored instance. All data is extracted from the https://docs.microsoft.com/en-us/sql/relational-databases/system-dynamic-management-views/database-related-dynamic-management-views-transact-sql?view=sql-server-2017[Database Dynamic Management Views] + +* *space_usage.since_last_backup.bytes*: The amount of space used since the last log backup in bytes +* *space_usage.total.bytes*: The size of the log in bytes +* *space_usage.used.bytes*: The occupied size of the log in bytes +* *space_usage.used.pct*: A percentage of the occupied size of the log as a percent of the total log size +* *stats.active_size.bytes*: Total active transaction log size in bytes +* *stats.backup_time*: Last transaction log backup time. +* *stats.recovery_size.bytes*: Log size in bytes since log recovery log sequence number (LSN). +* *stats.since_last_checkpoint.bytes*: Log size in bytes since last checkpoint log sequence number (LSN). +* *stats.total_size.bytes*: Total transaction log size in bytes. diff --git a/x-pack/metricbeat/module/mssql/transaction_log/_meta/fields.yml b/x-pack/metricbeat/module/mssql/transaction_log/_meta/fields.yml new file mode 100644 index 000000000000..e27d2f3c315a --- /dev/null +++ b/x-pack/metricbeat/module/mssql/transaction_log/_meta/fields.yml @@ -0,0 +1,67 @@ +- name: transaction_log + type: group + description: transaction_log metricset will fetch information about the operation and transaction log of each database from a MSSQL instance + fields: + - name: space_usage + type: group + description: Space usage information for the transaction log + fields: + - name: since_last_backup + description: The amount of space used since the last log backup + type: group + fields: + - name: bytes + description: The amount of space used since the last log backup in bytes + type: long + - name: total + type: group + description: The size of the log + fields: + - name: bytes + description: The size of the log in bytes + type: long + - name: used + type: group + description: The occupied size of the log + fields: + - name: bytes + description: The occupied size of the log in bytes + type: long + - name: pct + description: A percentage of the occupied size of the log as a percent of the total log size + type: float + - name: stats + type: group + description: Returns summary level attributes and information on transaction log files of databases. Use this information for monitoring and diagnostics of transaction log health. + fields: + - name: active_size + description: Total active transaction log size. + type: group + fields: + - name: bytes + description: Total active transaction log size in bytes + type: long + - name: backup_time + type: date + description: Last transaction log backup time. + - name: recovery_size + type: group + description: Log size since log recovery log sequence number (LSN). + fields: + - name: bytes + description: Log size in bytes since log recovery log sequence number (LSN). + type: long + - name: since_last_checkpoint + type: group + description: Log size since last checkpoint log sequence number (LSN). + fields: + - name: bytes + description: Log size in bytes since last checkpoint log sequence number (LSN). + type: long + - name: total_size + type: group + description: Total transaction log size. + fields: + - name: bytes + description: Total transaction log size in bytes. + type: long diff --git a/x-pack/metricbeat/module/mssql/db/data_integration_test.go b/x-pack/metricbeat/module/mssql/transaction_log/data_integration_test.go similarity index 84% rename from x-pack/metricbeat/module/mssql/db/data_integration_test.go rename to x-pack/metricbeat/module/mssql/transaction_log/data_integration_test.go index 7a8fc8bafc5d..8616e427c0bf 100644 --- a/x-pack/metricbeat/module/mssql/db/data_integration_test.go +++ b/x-pack/metricbeat/module/mssql/transaction_log/data_integration_test.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package db +package transaction_log import ( "testing" @@ -14,7 +14,7 @@ import ( func TestData(t *testing.T) { t.Skip("Skipping `data.json` generation test") - f := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig("db")) + f := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig("transaction_log")) err := mbtest.WriteEventsReporterV2(f, t, "") if err != nil { diff --git a/x-pack/metricbeat/module/mssql/transaction_log/transaction_log.go b/x-pack/metricbeat/module/mssql/transaction_log/transaction_log.go new file mode 100644 index 000000000000..e2e56fa8ec12 --- /dev/null +++ b/x-pack/metricbeat/module/mssql/transaction_log/transaction_log.go @@ -0,0 +1,215 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package transaction_log + +import ( + "database/sql" + "fmt" + + "github.com/pkg/errors" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/common/cfgwarn" + "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/x-pack/metricbeat/module/mssql" +) + +type dbInfo struct { + id int + name string +} + +type logSpace struct { + id int + totalLogSizeInBytes int + usedLogSpaceInBytes int + usedLogSpaceInPercent float64 + logSpaceInBytesSinceLastBackup int +} + +type logStats struct { + databaseID int + sizeMB float64 + activeSizeMB float64 + backupTime string + sinceLastBackupMB *float64 + sinceLastCheckpointMB float64 + recoverySizeMB float64 +} + +func init() { + mb.Registry.MustAddMetricSet("mssql", "transaction_log", New, + mb.DefaultMetricSet(), + mb.WithHostParser(mssql.HostParser)) +} + +// MetricSet type defines all fields of the MetricSet +type MetricSet struct { + mb.BaseMetricSet + log *logp.Logger + db *sql.DB +} + +// New create a new instance of the MetricSet +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + cfgwarn.Beta("The mssql transaction_log metricset is beta.") + + logger := logp.NewLogger("mssql.transaction_log").With("host", base.HostData().SanitizedURI) + + db, err := mssql.NewConnection(base.HostData().URI) + if err != nil { + return nil, errors.Wrap(err, "could not create connection to db") + } + + return &MetricSet{ + BaseMetricSet: base, + log: logger, + db: db, + }, nil +} + +// Fetch methods implements the data gathering and data conversion to the right format +// It returns the event which is then forward to the output. In case of an error, a +// descriptive error must be returned. +func (m *MetricSet) Fetch(reporter mb.ReporterV2) { + dbs, err := m.getDbsNames() + if err != nil { + reporter.Error(err) + return + } + + for _, db := range dbs { + moduleFields := common.MapStr{ + "database": common.MapStr{ + "id": db.id, + "name": db.name, + }, + } + metricsetFields := common.MapStr{} + + spaceUsage, err := m.getLogSpaceUsageForDb(db.name) + if err != nil { + reporter.Error(err) + } else { + metricsetFields["space_usage"] = spaceUsage + } + + stats, err := m.getLogStats(db) + if err != nil { + reporter.Error(err) + } else { + metricsetFields["stats"] = stats + } + + if len(metricsetFields) == 0 { + m.log.Debug("no data to report") + continue + } + + // Both log space and log size are available, so report both + if isReported := reporter.Event(mb.Event{ + ModuleFields: moduleFields, + MetricSetFields: metricsetFields, + }); !isReported { + m.log.Debug("event not reported") + } + + } +} + +// Close the connection to the server at the engine level +func (m *MetricSet) Close() error { + return m.db.Close() +} + +func (m *MetricSet) getLogSpaceUsageForDb(dbName string) (common.MapStr, error) { + // According to MS docs a single result is always returned for this query + row := m.db.QueryRow(fmt.Sprintf(`USE %s; SELECT * FROM sys.dm_db_log_space_usage;`, dbName)) + + var res logSpace + if err := row.Scan(&res.id, &res.totalLogSizeInBytes, &res.usedLogSpaceInBytes, &res.usedLogSpaceInPercent, + &res.logSpaceInBytesSinceLastBackup); err != nil { + // Because this query only returns a single result an error in the first scan is + // probably a "data returned but not properly scanned" + err = errors.Wrap(err, "error scanning single result") + return nil, err + } + + return common.MapStr{ + "total": common.MapStr{ + "bytes": res.totalLogSizeInBytes, + }, + "used": common.MapStr{ + "bytes": res.usedLogSpaceInBytes, + "pct": res.usedLogSpaceInPercent, + }, + "since_last_backup": common.MapStr{ + "bytes": res.logSpaceInBytesSinceLastBackup, + }, + }, nil +} +func (m *MetricSet) getLogStats(db dbInfo) (common.MapStr, error) { + // According to MS docs a single result is always returned for this query + row := m.db.QueryRow(fmt.Sprintf(`USE %s; SELECT database_id,total_log_size_mb,active_log_size_mb,log_backup_time,log_since_last_log_backup_mb,log_since_last_checkpoint_mb,log_recovery_size_mb FROM sys.dm_db_log_stats(%d);`, db.name, db.id)) + + var res logStats + if err := row.Scan(&res.databaseID, &res.sizeMB, &res.activeSizeMB, &res.backupTime, &res.sinceLastBackupMB, &res.sinceLastCheckpointMB, &res.recoverySizeMB); err != nil { + // Because this query only returns a single result an error in the first scan is + // probably a "data returned but not properly scanned" + err = errors.Wrap(err, "error scanning single result") + return nil, err + } + + result := common.MapStr{ + "total_size": common.MapStr{ + "bytes": res.sizeMB * 1048576, + }, + "active_size": common.MapStr{ + "bytes": res.activeSizeMB * 1048576, + }, + "backup_time": res.backupTime, + "since_last_checkpoint": common.MapStr{ + "bytes": res.sinceLastCheckpointMB * 1048576, + }, + "recovery_size": common.MapStr{ + "bytes": res.recoverySizeMB, + }, + } + + if res.sinceLastBackupMB != nil { + result.Put("since_last_backup.bytes", *res.sinceLastBackupMB*1048576) + } + + return result, nil +} + +func (m *MetricSet) getDbsNames() ([]dbInfo, error) { + res := make([]dbInfo, 0) + + var rows *sql.Rows + rows, err := m.db.Query("SELECT name, database_id FROM sys.databases") + if err != nil { + return nil, errors.Wrap(err, "error doing query 'SELECT name, database_id FROM sys.databases'") + } + defer closeRows(rows) + + for rows.Next() { + var row dbInfo + if err = rows.Scan(&row.name, &row.id); err != nil { + return nil, errors.Wrap(err, "error scanning row results") + } + + res = append(res, row) + } + + return res, nil +} + +func closeRows(rows *sql.Rows) { + if err := rows.Close(); err != nil { + logp.Err("error closing rows: %s", err) + } +} diff --git a/x-pack/metricbeat/module/mssql/db/db_integration_test.go b/x-pack/metricbeat/module/mssql/transaction_log/transaction_log_integration_test.go similarity index 88% rename from x-pack/metricbeat/module/mssql/db/db_integration_test.go rename to x-pack/metricbeat/module/mssql/transaction_log/transaction_log_integration_test.go index 4fc9f594b7da..f899ec261414 100644 --- a/x-pack/metricbeat/module/mssql/db/db_integration_test.go +++ b/x-pack/metricbeat/module/mssql/transaction_log/transaction_log_integration_test.go @@ -4,7 +4,7 @@ // +build integration -package db +package transaction_log import ( "testing" @@ -21,7 +21,7 @@ func TestFetch(t *testing.T) { logp.TestingSetup() compose.EnsureUp(t, "mssql") - f := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig("db")) + f := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig("transaction_log")) events, errs := mbtest.ReportingFetchV2(f) if len(errs) > 0 { t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) @@ -29,7 +29,7 @@ func TestFetch(t *testing.T) { assert.NotEmpty(t, events) for _, event := range events { - const key = "log_space_usage.used.pct" + const key = "space_usage.used.pct" usedPercent, err := event.MetricSetFields.GetValue(key) if err != nil { diff --git a/x-pack/metricbeat/modules.d/mssql.yml.disabled b/x-pack/metricbeat/modules.d/mssql.yml.disabled index 038955a319cc..c79d01d06214 100644 --- a/x-pack/metricbeat/modules.d/mssql.yml.disabled +++ b/x-pack/metricbeat/modules.d/mssql.yml.disabled @@ -1,6 +1,6 @@ - module: mssql metricsets: - - "db" + - "transaction_log" - "performance" hosts: ["sqlserver://sa@localhost"] period: 10s diff --git a/x-pack/metricbeat/tests/system/test_mssql.py b/x-pack/metricbeat/tests/system/test_mssql.py index 252c8a1cb066..963177b718c1 100644 --- a/x-pack/metricbeat/tests/system/test_mssql.py +++ b/x-pack/metricbeat/tests/system/test_mssql.py @@ -24,7 +24,7 @@ def test_status(self): """ self.render_config_template(modules=[{ "name": "mssql", - "metricsets": ["db"], + "metricsets": ["transaction_log"], "hosts": self.get_hosts(), "username": self.get_username(), "password": self.get_password(), @@ -36,12 +36,12 @@ def test_status(self): self.assert_no_logged_warnings() output = self.read_output_json() - self.assertEqual(len(output), 1) + self.assertEqual(len(output), 4) evt = output[0] self.assertItemsEqual(self.de_dot(MSSQL_FIELDS), evt.keys()) - self.assertTrue(evt["mssql"]["db"]["log_space_usage"]["used"]["pct"] > 0) - self.assertTrue(evt["mssql"]["db"]["log_space_usage"]["used"]["bytes"] > 0) + self.assertTrue(evt["mssql"]["transaction_log"]["space_usage"]["used"]["pct"] > 0) + self.assertTrue(evt["mssql"]["transaction_log"]["stats"]["active_size"]["bytes"] > 0) self.assert_fields_are_documented(evt)