diff --git a/packages/microsoft_sqlserver/_dev/build/docs/README.md b/packages/microsoft_sqlserver/_dev/build/docs/README.md index 4c997427a0a..2decfd86e9d 100644 --- a/packages/microsoft_sqlserver/_dev/build/docs/README.md +++ b/packages/microsoft_sqlserver/_dev/build/docs/README.md @@ -76,6 +76,7 @@ See: [Instructions about each performance counter metrics](https://docs.microsof ### transaction_log metrics Collects system level `transaction_log` metrics information for SQL Server instance. +Metrics for user level databases can be collected by providing list of user dbs for which metrics is to be collected. See: [Instructions and the operations supported by transaction log](https://docs.microsoft.com/en-us/sql/relational-databases/logs/the-transaction-log-sql-server?view=sql-server-ver15) diff --git a/packages/microsoft_sqlserver/changelog.yml b/packages/microsoft_sqlserver/changelog.yml index d0043ba8056..f489356765f 100644 --- a/packages/microsoft_sqlserver/changelog.yml +++ b/packages/microsoft_sqlserver/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.11.0" + changes: + - description: Support user databases for transaction log. + type: enhancement + link: https://github.com/elastic/integrations/pull/4869 - version: "1.10.0" changes: - description: Ingest instance name and server names. diff --git a/packages/microsoft_sqlserver/data_stream/transaction_log/agent/stream/stream.yml.hbs b/packages/microsoft_sqlserver/data_stream/transaction_log/agent/stream/stream.yml.hbs index ef2cdc5a142..99982ef7b00 100644 --- a/packages/microsoft_sqlserver/data_stream/transaction_log/agent/stream/stream.yml.hbs +++ b/packages/microsoft_sqlserver/data_stream/transaction_log/agent/stream/stream.yml.hbs @@ -9,11 +9,11 @@ raw_data.enabled: true sql_queries: {{#if databases}} {{#each databases as |database_name i|}} - - query: "SELECT @@servername AS server_name, @@servicename AS instance_name, name As 'database_name', database_id FROM sys.databases WHERE name='{{database_name}}';" - response_format: table - - query: "SELECT @@servername AS server_name, @@servicename AS instance_name, name As 'database_name', l.database_id, l.total_log_size_mb, l.active_log_size_mb,l.log_backup_time,l.log_since_last_log_backup_mb,l.log_since_last_checkpoint_mb,l.log_recovery_size_mb from sys.dm_db_log_stats(DB_ID('{{database_name}}')) l INNER JOIN sys.databases s ON l.database_id = s.database_id WHERE s.database_id = DB_ID('{{database_name}}') ;" - response_format: table - - query: "SELECT @@servername AS server_name, @@servicename AS instance_name, name As 'database_name', l.database_id, l.total_log_size_in_bytes As total_log_size_bytes, l.used_log_space_in_bytes As used_log_space_bytes, l.used_log_space_in_percent As used_log_space_pct, l.log_space_in_bytes_since_last_backup from sys.dm_db_log_space_usage l INNER JOIN sys.databases s ON l.database_id = s.database_id WHERE s.database_id = DB_ID('{{database_name}}') ;" - response_format: table + - query: "SELECT @@servername AS server_name, @@servicename AS instance_name, name As 'database_name', database_id FROM sys.databases WHERE name='{{database_name}}';" + response_format: table + - query: "SELECT @@servername AS server_name, @@servicename AS instance_name, name As 'database_name', l.database_id, l.total_log_size_mb, l.active_log_size_mb,l.log_backup_time,l.log_since_last_log_backup_mb,l.log_since_last_checkpoint_mb,l.log_recovery_size_mb from sys.dm_db_log_stats(DB_ID('{{database_name}}')) l INNER JOIN sys.databases s ON l.database_id = s.database_id WHERE s.database_id = DB_ID('{{database_name}}') ;" + response_format: table + - query: "USE {{database_name}} ; SELECT @@servername AS server_name, @@servicename AS instance_name, name As 'database_name', l.database_id, l.total_log_size_in_bytes As total_log_size_bytes, l.used_log_space_in_bytes As used_log_space_bytes, l.used_log_space_in_percent As used_log_space_pct, l.log_space_in_bytes_since_last_backup from sys.dm_db_log_space_usage l INNER JOIN sys.databases s ON l.database_id = s.database_id WHERE s.database_id = DB_ID('{{database_name}}') ;" + response_format: table {{/each}} -{{/if}} +{{/if}} \ No newline at end of file diff --git a/packages/microsoft_sqlserver/data_stream/transaction_log/manifest.yml b/packages/microsoft_sqlserver/data_stream/transaction_log/manifest.yml index a4705806094..1a13dca435e 100644 --- a/packages/microsoft_sqlserver/data_stream/transaction_log/manifest.yml +++ b/packages/microsoft_sqlserver/data_stream/transaction_log/manifest.yml @@ -17,8 +17,8 @@ streams: title: Databases multi: true required: true - show_user: false - description: Default system databases are preloaded. For custom database please add additional rows and enter the database name. + show_user: true + description: Default system databases are preloaded. For user defined databases please add additional rows and enter the database name. default: ["master", "model", "tempdb", "msdb"] title: Microsoft SQL Server transaction_log metrics description: Collect Microsoft SQL Server transaction_log metrics diff --git a/packages/microsoft_sqlserver/data_stream/transaction_log/sample_event.json b/packages/microsoft_sqlserver/data_stream/transaction_log/sample_event.json index 6c91b5516e2..b8aa2143cc4 100644 --- a/packages/microsoft_sqlserver/data_stream/transaction_log/sample_event.json +++ b/packages/microsoft_sqlserver/data_stream/transaction_log/sample_event.json @@ -1,67 +1,52 @@ { - "@timestamp": "2022-11-23T10:53:05.080Z", + "@timestamp": "2022-12-20T07:34:29.687Z", "agent": { - "ephemeral_id": "70f5c0c1-37b1-486b-9806-8105b2cdcd20", - "id": "6d444a4a-2158-445e-8953-dc6eef720a34", - "name": "docker-fleet-agent", + "ephemeral_id": "8d528ff8-5e90-4572-89f6-61fb3a6c96f1", + "id": "d44a1c4a-95bf-47e9-afb0-453a2ef43c00", + "name": "192.168.1.2", "type": "metricbeat", - "version": "8.5.0" - }, - "cloud": { - "account": {}, - "instance": { - "id": "b30e45e6-7900-4900-8d67-e37cb13374bc", - "name": "obs-int-windows-dev" - }, - "machine": { - "type": "Standard_D16ds_v5" - }, - "provider": "azure", - "region": "CentralIndia", - "service": { - "name": "Virtual Machines" - } + "version": "8.5.3" }, "data_stream": { "dataset": "microsoft_sqlserver.transaction_log", - "namespace": "ep", + "namespace": "default", "type": "metrics" }, "ecs": { "version": "8.0.0" }, "elastic_agent": { - "id": "6d444a4a-2158-445e-8953-dc6eef720a34", + "id": "d44a1c4a-95bf-47e9-afb0-453a2ef43c00", "snapshot": false, - "version": "8.5.0" + "version": "8.5.3" }, "event": { "agent_id_status": "verified", "dataset": "microsoft_sqlserver.transaction_log", - "duration": 1449739900, - "ingested": "2022-11-23T10:53:07Z", + "duration": 2147044750, + "ingested": "2022-12-20T07:34:32Z", "module": "sql" }, "host": { "architecture": "x86_64", "containerized": false, - "hostname": "docker-fleet-agent", - "id": "66392b0697b84641af8006d87aeb89f1", + "hostname": "192.168.1.2", + "id": "627E8AE5-E918-5073-A58E-8A2D9ED96875", "ip": [ - "172.18.0.5" + "192.168.1.2" ], "mac": [ - "02-42-AC-12-00-05" + "36-F7-DC-28-23-80" ], - "name": "docker-fleet-agent", + "name": "192.168.1.2", "os": { - "codename": "focal", - "family": "debian", - "kernel": "5.10.104-linuxkit", - "name": "Ubuntu", - "platform": "ubuntu", - "type": "linux", - "version": "20.04.5 LTS (Focal Fossa)" + "build": "21D62", + "family": "darwin", + "kernel": "21.3.0", + "name": "macOS", + "platform": "darwin", + "type": "macos", + "version": "12.2.1" } }, "metricset": { @@ -70,18 +55,18 @@ }, "mssql": { "metrics": { - "database_id": 1, - "database_name": "master", + "server_name": "obs-int-mssql20", "instance_name": "MSSQLSERVER", - "log_space_in_bytes_since_last_backup": 139264, - "server_name": "obs-service-int", + "database_name": "master", + "database_id": 1, + "log_space_in_bytes_since_last_backup": 602112, "total_log_size_bytes": 2088960, - "used_log_space_bytes": 626688, - "used_log_space_pct": 30 + "used_log_space_pct": 49.01960754394531, + "used_log_space_bytes": 1024000 } }, "service": { - "address": "20.121.218.233", + "address": "20.228.135.242", "type": "sql" } } \ No newline at end of file diff --git a/packages/microsoft_sqlserver/docs/README.md b/packages/microsoft_sqlserver/docs/README.md index 8fbc61282e3..e9956a82648 100644 --- a/packages/microsoft_sqlserver/docs/README.md +++ b/packages/microsoft_sqlserver/docs/README.md @@ -76,6 +76,7 @@ See: [Instructions about each performance counter metrics](https://docs.microsof ### transaction_log metrics Collects system level `transaction_log` metrics information for SQL Server instance. +Metrics for user level databases can be collected by providing list of user dbs for which metrics is to be collected. See: [Instructions and the operations supported by transaction log](https://docs.microsoft.com/en-us/sql/relational-databases/logs/the-transaction-log-sql-server?view=sql-server-ver15) @@ -494,69 +495,54 @@ An example event for `transaction_log` looks as following: ```json { - "@timestamp": "2022-11-23T10:53:05.080Z", + "@timestamp": "2022-12-20T07:34:29.687Z", "agent": { - "ephemeral_id": "70f5c0c1-37b1-486b-9806-8105b2cdcd20", - "id": "6d444a4a-2158-445e-8953-dc6eef720a34", - "name": "docker-fleet-agent", + "ephemeral_id": "8d528ff8-5e90-4572-89f6-61fb3a6c96f1", + "id": "d44a1c4a-95bf-47e9-afb0-453a2ef43c00", + "name": "192.168.1.2", "type": "metricbeat", - "version": "8.5.0" - }, - "cloud": { - "account": {}, - "instance": { - "id": "b30e45e6-7900-4900-8d67-e37cb13374bc", - "name": "obs-int-windows-dev" - }, - "machine": { - "type": "Standard_D16ds_v5" - }, - "provider": "azure", - "region": "CentralIndia", - "service": { - "name": "Virtual Machines" - } + "version": "8.5.3" }, "data_stream": { "dataset": "microsoft_sqlserver.transaction_log", - "namespace": "ep", + "namespace": "default", "type": "metrics" }, "ecs": { "version": "8.0.0" }, "elastic_agent": { - "id": "6d444a4a-2158-445e-8953-dc6eef720a34", + "id": "d44a1c4a-95bf-47e9-afb0-453a2ef43c00", "snapshot": false, - "version": "8.5.0" + "version": "8.5.3" }, "event": { "agent_id_status": "verified", "dataset": "microsoft_sqlserver.transaction_log", - "duration": 1449739900, - "ingested": "2022-11-23T10:53:07Z", + "duration": 2147044750, + "ingested": "2022-12-20T07:34:32Z", "module": "sql" }, "host": { "architecture": "x86_64", "containerized": false, - "hostname": "docker-fleet-agent", - "id": "66392b0697b84641af8006d87aeb89f1", + "hostname": "192.168.1.2", + "id": "627E8AE5-E918-5073-A58E-8A2D9ED96875", "ip": [ - "172.18.0.5" + "192.168.1.2" ], "mac": [ - "02-42-AC-12-00-05" + "36-F7-DC-28-23-80" ], - "name": "docker-fleet-agent", + "name": "192.168.1.2", "os": { - "codename": "focal", - "family": "debian", - "kernel": "5.10.104-linuxkit", - "name": "Ubuntu", - "platform": "ubuntu", - "type": "linux", - "version": "20.04.5 LTS (Focal Fossa)" + "build": "21D62", + "family": "darwin", + "kernel": "21.3.0", + "name": "macOS", + "platform": "darwin", + "type": "macos", + "version": "12.2.1" } }, "metricset": { @@ -565,18 +551,18 @@ An example event for `transaction_log` looks as following: }, "mssql": { "metrics": { - "database_id": 1, - "database_name": "master", + "server_name": "obs-int-mssql20", "instance_name": "MSSQLSERVER", - "log_space_in_bytes_since_last_backup": 139264, - "server_name": "obs-service-int", + "database_name": "master", + "database_id": 1, + "log_space_in_bytes_since_last_backup": 602112, "total_log_size_bytes": 2088960, - "used_log_space_bytes": 626688, - "used_log_space_pct": 30 + "used_log_space_pct": 49.01960754394531, + "used_log_space_bytes": 1024000 } }, "service": { - "address": "20.121.218.233", + "address": "20.228.135.242", "type": "sql" } } diff --git a/packages/microsoft_sqlserver/manifest.yml b/packages/microsoft_sqlserver/manifest.yml index 08a1242b70f..05d253f3eda 100644 --- a/packages/microsoft_sqlserver/manifest.yml +++ b/packages/microsoft_sqlserver/manifest.yml @@ -1,7 +1,7 @@ format_version: 1.0.0 name: microsoft_sqlserver title: "Microsoft SQL Server" -version: "1.10.0" +version: "1.11.0" license: basic description: Collect events from Microsoft SQL Server with Elastic Agent type: integration