From 1e024a53e26f80d8f1c2b7419a4fb2fe55b86225 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Tue, 4 Feb 2020 08:55:42 -0700 Subject: [PATCH] [Metricbeat] Add collecting AuroraDB metrics in rds metricset (#16004) * Add collecting AuroraDB metrics in rds metricset * update changelog * Rename to use iops as unit * run mage fmt update --- CHANGELOG.next.asciidoc | 1 + metricbeat/docs/fields.asciidoc | 230 +++++++++++++++++- x-pack/metricbeat/module/aws/fields.go | 2 +- .../metricbeat/module/aws/rds/_meta/data.json | 64 +++-- .../module/aws/rds/_meta/docs.asciidoc | 9 +- .../module/aws/rds/_meta/fields.yml | 95 +++++++- x-pack/metricbeat/module/aws/rds/data.go | 34 ++- 7 files changed, 401 insertions(+), 34 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 687f5344334c..192551724a31 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -132,6 +132,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add `up` metric to prometheus metrics collected from host {pull}15948[15948] - Add citadel metricset for Istio Metricbeat module {pull}15990[15990] - Add support for processors in light modules. {issue}14740[14740] {pull}15923[15923] +- Add collecting AuroraDB metrics in rds metricset. {issue}14142[14142] {pull}16004[16004] - Reuse connections in SQL module. {pull}16001[16001] *Packetbeat* diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index d8c82bd80e54..a02062e33f9f 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -2121,7 +2121,7 @@ type: long -- -*`aws.rds.read_io.ops_per_sec`*:: +*`aws.rds.read.iops`*:: + -- The average number of disk read I/O operations per second. @@ -2165,7 +2165,7 @@ type: long -- -*`aws.rds.write_io.ops_per_sec`*:: +*`aws.rds.write.iops`*:: + -- The average number of disk write I/O operations per second. @@ -2201,6 +2201,30 @@ type: long The amount of storage used by your Aurora DB instance, in bytes. +type: long + +format: bytes + +-- + +*`aws.rds.volume.read.iops`*:: ++ +-- +The number of billed read I/O operations from a cluster volume, reported at 5-minute intervals. + + +type: long + +format: bytes + +-- + +*`aws.rds.volume.write.iops`*:: ++ +-- +The number of write disk I/O operations to the cluster volume, reported at 5-minute intervals. + + type: long format: bytes @@ -2439,6 +2463,18 @@ format: duration The average amount of time taken per disk I/O operation. +type: float + +format: duration + +-- + +*`aws.rds.latency.delete`*:: ++ +-- +The amount of latency for delete queries, in milliseconds. + + type: float format: duration @@ -2527,6 +2563,196 @@ type: keyword -- +*`aws.rds.aurora_bin_log_replica_lag`*:: ++ +-- +The amount of time a replica DB cluster running on Aurora with MySQL compatibility lags behind the source DB cluster. + + +type: long + +-- + +*`aws.rds.aurora_global_db.replicated_write_io.bytes`*:: ++ +-- +In an Aurora Global Database, the number of write I/O operations replicated from the primary AWS Region to the cluster volume in a secondary AWS Region. + + +type: long + +-- + +*`aws.rds.aurora_global_db.data_transfer.bytes`*:: ++ +-- +In an Aurora Global Database, the amount of redo log data transferred from the master AWS Region to a secondary AWS Region. + + +type: long + +-- + +*`aws.rds.aurora_global_db.replication_lag.ms`*:: ++ +-- +For an Aurora Global Database, the amount of lag when replicating updates from the primary AWS Region, in milliseconds. + + +type: long + +-- + +*`aws.rds.aurora_replica.lag.ms`*:: ++ +-- +For an Aurora Replica, the amount of lag when replicating updates from the primary instance, in milliseconds. + + +type: long + +-- + +*`aws.rds.aurora_replica.lag_max.ms`*:: ++ +-- +The maximum amount of lag between the primary instance and each Aurora DB instance in the DB cluster, in milliseconds. + + +type: long + +-- + +*`aws.rds.aurora_replica.lag_min.ms`*:: ++ +-- +The minimum amount of lag between the primary instance and each Aurora DB instance in the DB cluster, in milliseconds. + + +type: long + +-- + +*`aws.rds.backtrack_change_records.creation_rate`*:: ++ +-- +The number of backtrack change records created over five minutes for your DB cluster. + + +type: long + +-- + +*`aws.rds.backtrack_change_records.stored`*:: ++ +-- +The actual number of backtrack change records used by your DB cluster. + + +type: long + +-- + +*`aws.rds.backtrack_window.actual`*:: ++ +-- +The difference between the target backtrack window and the actual backtrack window. + + +type: long + +-- + +*`aws.rds.backtrack_window.alert`*:: ++ +-- +The number of times that the actual backtrack window is smaller than the target backtrack window for a given period of time. + + +type: long + +-- + +*`aws.rds.storage_used.backup_retention_period.bytes`*:: ++ +-- +The total amount of backup storage in bytes used to support the point-in-time restore feature within the Aurora DB cluster's backup retention window. + + +type: long + +-- + +*`aws.rds.storage_used.snapshot.bytes`*:: ++ +-- +The total amount of backup storage in bytes consumed by all Aurora snapshots for an Aurora DB cluster outside its backup retention window. + + +type: long + +-- + +*`aws.rds.cache_hit_ratio.buffer`*:: ++ +-- +The percentage of requests that are served by the buffer cache. + + +type: long + +-- + +*`aws.rds.cache_hit_ratio.result_set`*:: ++ +-- +The percentage of requests that are served by the Resultset cache. + + +type: long + +-- + +*`aws.rds.engine_uptime.sec`*:: ++ +-- +The amount of time that the instance has been running, in seconds. + + +type: long + +-- + +*`aws.rds.rds_to_aurora_postgresql_replica_lag.sec`*:: ++ +-- +The amount of lag in seconds when replicating updates from the primary RDS PostgreSQL instance to other nodes in the cluster. + + +type: long + +-- + +*`aws.rds.backup_storage_billed_total.bytes`*:: ++ +-- +The total amount of backup storage in bytes for which you are billed for a given Aurora DB cluster. + + +type: long + +-- + +*`aws.rds.aurora_volume_left_total.bytes`*:: ++ +-- +The remaining available space for the cluster volume, measured in bytes. + + +type: long + +-- + [float] === s3_daily_storage diff --git a/x-pack/metricbeat/module/aws/fields.go b/x-pack/metricbeat/module/aws/fields.go index f87d93694180..fe593513892b 100644 --- a/x-pack/metricbeat/module/aws/fields.go +++ b/x-pack/metricbeat/module/aws/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAws returns asset data. // This is the base64 encoded gzipped contents of module/aws. func AssetAws() string { - return "eJzsXV9z27ayf8+n2OnLSTKOzj1tz33Iw51xLM89nnFbx3KnjywErCgcgwCNP5aV6Ye/gz+kKImyRImUc2ZuHtqMKAG/3+5isdhdMJ/gEZefgSzMOwDLrcDP8ANZmB/eATA0VPPSciU/w/+8AwD4kyzMn1Ao5gQCVUIgtQYu/5hAoSS3SnOZQ4FWc2pgplURnl0J5diCWDofvQPQKJAY/Aw5eQcw4yiY+RxG/wSSFPgZqP/+iFCqnLQj/1l4DGCXJX72iBdKs/RZC0r/52GOcRxI44SxQWkgghMDziADq4AzlJbPlsD4bIYapQX/geVogEsgUDhh+SeLkoRHz1wrWaC0ozXIUYArjLlWrnwNYZN3cyBLcjP6WH9cjaem/0ZqGx/HD7I2iWw8zgpSllzm6bs/fPyh8b0d0gsSJLkfGJ6JcAgl4TqplCwMaDTKaYpmtMXA/DSaOvqIa5rbpb09GH4NOpsBgclPkEbdmpDxAqXhSn4ngvsl2H8T1hbkv30cpVUy+jj6+LeOqJlyU4FDgDZg58SCRuu0RBb1vVq+cHl3A08O9XKb0pQLwWW+RaW5EvZg+DON8SdQJS3h0sNBQGN5QSwyoHOiczQwUxqWyungXar1zeWGo6n+1A5nipY0Pt+1BGk9yklkVsNs8Cmaol6gRjBUk7ISd+0x/wgiX8w5na8GaPGzxjutadODeR6mJGvLc9Px7pJCUxL1OGtPd6/kPSKB5JfrYcGUSPmMI4PFHGU0rYb8gZS8Zb0vJSkUm56knWqQM+nG/3Acphx/OcU2cWpOoo1Tc0bG118m3Q2wpkp/PI0q/fGcVK9+PG2t0dKNrLJEjMo1z78ibygRyLKZUGTzCwcsuhI1RWlJHjdUIRQNPvX66kegqiidRXCS2yQeohGo096diKX3rc4gKBnkyKWxRFIc7SRCNTJuM2dI3u47hFrbKg7kIF0xRe3xX939DnES451I1EMTW9gj/Lec5YJ/I37YvXinRPjfDoIYSdhRm8CjoOUK85wYv51phwwM959wCwtiQBAn6RyZj1+NJdoi203GOF0KZ7IzkEpTrTOak2eEKaJcaYZIcFLwgnuLq+kGn+9/dnX3+1UY4UvEmmJObuAbanUoU5PF+GBzR+qJauDSStivFamsj5EZMLWQnvK2vi+ASJbcip07f76gTnvZEMa4R0FECnHaKUu0C6UfR1yOSuJjYTMI0zQ2aKTIn73RSe8vqumBS4t65qOLzUV3KOysRJ0ZpIPCL1GDQaoki45aOdsXE+XsWTQwIO7/dBVwOZouLR4s/5nSBbGfoe1HnbiFAYZYG2HgQdUSoTeU0jcLb19vqZUh1ssbqKUvGoybR65GGgk7m1q+pOVBUlDt8dcbvrFKIzwr4Qo0QJ4JF2QqEKw6hk3PSmkgb+hiGBILzS2eWSd+TovS4xySzyBaqbA3FDMEjWBbquxrU79SRSnQh7zBqlSJOpxDzPFWFXPSq7RJiZor5r2I5cVh5HpW0G6SfayiE/hGmxxCm2HkJtMjbbEPcoNpc4vk6WvvGL7GEuvMiM6RPmYzwkVvx7t7LJW2xp9C7Rz1OlJ/Ei+JMchgqux8/WHEBAFTONP5p2ZpLBbrz3jMlwhiLBRcOns4ySyOd2auQxCp5nkDKu0aO5RMvWVQpf1/nGzPy/mwLEd9cjpL6VTa2L9f1U95QXIc8fY1cXSC/mYcFqWH4cevi6UxDdUF3yprOvI66LGQcCMZpyQEB8kSGNpgcc1ULTeA0vuiHfmyGmip+TOxOGLSZBt1yx4EmkaH8a+TVIeO4t2K7A9Eyct2S9z8uAO0m7vnn4EwptEYIMYoykN+eMGT++uM1U0Fp0MJNAy+Jc8DrTJB61GKleASjmvvXDiFm7v6yXsv4A8wVS5uoMeINCyhEVWsXZpHO6Iw7qYML4AYIPCP//405RacNDyXIXsbJjkIaf96b0UK70uUzC/3v0A7KePfzNxZy2X+KWRk/wKLuuAy2PRfPmIJZfLqr8g+7GFk5z6+jfGWd9VDbQVpnhBuVdvCdg0UxWnlTxTnrHxe37YXPQ+qAwpSTBk5iW0c4oyEb8OEpxR6NTut0KvZOQu99+MjCr3QV/WzSo2kEucRu8labbRr1fD/q5znrnIyYsmUGMyokhJpOJ8OQqeaCBoTpWL4DmTT+rwzIlr2t/ldFuSbknCf+u5CZ9z7y/tfPwQTQELn3mXsB0UFMe2yOgrWVdPDNCOxqqXAH48LLJReAiUlodwuIWCovjj+si8310CfujX51hbbBwXi1ao/GVeWgiNbKX816wge5tw0PvAHDM/CSf7kMPRLBnuvv+GH7UQxHlX7ozdJ6ZaIM7V0NOMobmqmu1NO2ZNDhxnD0s5bsR3ZlbJaaspZL4EQxt38ZuC9D4P+HrNQGp8cGms+wIJwH9OFDBSlPq72rDzCduxVMuVJZAb1M+qM5Cht9m81HcZjxAlh8vUWJmFCuPQTgp8QmAs76EHZh5lG9AfXLK6es9bVSBE6KtWskcvTRDJVVFJPoHYiz4xVmuTnq3Hsgp1wQOg3bMdbkBdeuCJzBllmNZGGBE+fcdanjaRpoDED3IyrlhkTO2Y8hhFcBg8U8sp3ythc4+TrbTt4JRgam2ksBadh/8+MUDYTJB8V0x7hC5Ln3ngN/1Y7+TRr/SyEmcqEXlx/3ApO/o/L2+Bg6mpzJ37eC2Rc7U11H+l/yDMG82hs+dw8xlLGzd9/a89/70IahBEk3yEhX9k8c3GiU8ze8gKBwL1Hf59009h8vJ68nc15lbOOsURzf2rq5pfl5OvtBfxCNCfjL7F9aaWvtWl2RB5mQcoYH7+RI/AA4tqPSczUwbjGOGzp8VTjt3OpbMN/+Ohq5czbWTZ9hlC5yXKU2KrNUxZgMMwGFX8UaLgSP3GnlRW21vMvrbijd1xbTw41P9x8jkKX5gB8Qeossr2gGBImFH0cFlY9S1W3qMPSffhiPS5sa2+1+tLmW9lrvLPhtNJrfunCUwuTvRJQCEWJeOOwouKz7h4sFqXSRC/B+s9M8JN+Oe7jJVTOZSiTOT2wcaewNMwIxHrIdv+ys3OtXD4vnR1RVRS8PTPTm3+Ic3TxCw2ADAXuqC/158DCHLWn6IKOiWGhjce39TGpE7BiYGBcGtTWXIArGbGYuqCjJDshjQOdA+wxCk6lnF7h1X6nqhOt5otdCXX/Zuyv8VGdjwgKbm0s2FLBUVoT+9HpfK0Tw+8lyReHQM97ZCqcsahXjusIEWQJVZ+i4JKqwp8w3t/HwT+sZKLJbMZpS2TnWVDhQkYhiIs6Y1WBerWFVj/2oqsybONJ/XHYt7yLb6S/SeibrU9bB0ul0kyfYlHO5iqI5SGN/p8jF3/EGmIxb5yGLHlMfYMh+lyLO/diNChwRzmiN5cT5zjG5USHOiy6OMcx6EKcPyy46WZfa1DxPoyCWJR02TWi6fOcniCEJbQV9ATnW3AheGSxK3hMNLpEFkNxCOkdhjMueTyLEpk7r6v34/Hthzou6cqsQ2gyFLNXo5eOfDoGMMNSqpZ0Rw6dvHYPDPpy6hX+jh59KB2sO/2OOujo94fisL41dOTQbXf4jgwpFMNSRpOHdN8b5SIa6UBFqSt5TLFMuSR6GdIPVehXEB/Tb2d2Y5Zbv5rAbdDdLDH0W15oyW42JgQ/Icy4wG45zgb8zSTt4PBPSs42fmxG/v87zlN95YequnBz3pQJ9cG9kkBkdVpc1cWr0+TesLDJZioUfez1kvY2nTUam3nT+s52QrI/0dsoz7Nplg7J2RDNCEe2FyRI1d0KSoSITjod3lY51/TN/US1Ej32bo6/gB/QgOCPCH/c3zxc34PScH99Ob6+v+gTOMqcS+y55/ia0PlaKU07mWQf57uIzDZLZo1ymQ8c0dK2F2ZljHCxrJLb7zYRd+n72xxsowkwPKvz2EO2BE5+Oq0jML3ty/Bvb5XtD6eaOhEW7jixeIVp801kTdzRCWVqlsW3cvWZ1w8dks2emThDC7awhogQtapDz1X769qydD471e7SMA2LS59813ZWHU5j++mAyvrXw8PdKkVfEBbux/ltNbq6+g13F6AxJ5qJ6kLBstzRuVJjz7H9THNqN1PA/L/XDxu4vXFVtsdlG4c9eEs3IN6733vH+0pxqRfI4+vb64frvlHPdx3We8H8r+vL8UH2vM8WlBnSGH6bbFrDUShfSRycinOFZHJ9e331AL8FpYfGVO/oeraKyCQzlEh55s6AzdRttckmLPGu2cHiOIV99YbL74J+/brNM/AXfMjVVmMLO72fKzWDB+gmvsr1NZxMLaRQhL2NZqJaVhjCYjtsy17MfVgTmyJNqWQ4HlPhWDiiTRXb0SzryremWyGIOkthF5A6ePPYL7p7TtRaaTP6+eVlOHP7+eUllbjjdPU9b8XwIL3FFUfS++bUDJCHq8T/5c+m/3iV2D+HJPbPlxeI/epnJFblZmdcG5t54xgVnS3y+BRtifpTZXMhORvOBtVdktB4U5sk+tNA3f0QXzC4JQKr4hsG1xZluFcUUnBTrB3v6/IIgXx1ujmrSFCQ0sQE1Q7RBF2FhbwSR7pcHG4YhCfhuLRv7dbnQXnaxUMjz3nxcPJr+8XDA29ZmqcTyT6dlezXE29ZpssCBRpDcsxIjmfuVC9LrV7Cy7whXbb0MouwQCr5KR60GCSIVe423D7acXsjfjOc0ciyv7zyuleuZlkDtMokp7lDjm+7f1wjCc02vCiQcWJR7AgGai5S2eyZGz7dkYM9dZOp6dQMuISZ4Pl8x25eIzsLqk3xWc3xmYiV2zvQHrwpDYu0stdOyCpPPSy0+lQxXQIlQphqY0gtdL+kJRbLJHsgm+171X3rnLG4d5HXZIhFaZdVh+EwN/g2xHN5d1OJz68VxuMKj9IFUhHYcf8F5crdnj2VXV3V6ijj+Kjf+snk6yT5zLVx61OQObXuEUbYsRdv/isWfh1oQh/jtKEJTxUYttiWf9dkf1TxfwEAAP//ZwWk3g==" + return "eJzsXUtz27iy3udXdM1mkpStc8887iKLW+XEvue4yjOT2JmaJQcEWhSOQYDBw7JS8+Nv4UGKop6USDlTdbOYcZES8H2NRqPR3YAu4REX74DMzSsAy63Ad/AdmZvvXgEwNFTzynIl38H/vAIA+JPMzZ9QKuYEAlVCILUGrv54gFJJbpXmsoASrebUwFSrMrz7IJRjc2LpbPIKQKNAYvAdFOQVwJSjYOZdaP0SJCnxHVD/+QmhVDlpJ/5ZeA1gFxW+84jnSrP0bANK/+/zDGM7kNoJbYPSQAQnBpxBBlYBZygtny6A8ekUNUoL/oHlaIBLIFA6YfmlRUnCqyeulSxR2skK5CjAJcZCK1ftQtjm3W7IksJM3jaP6/ZU/h+ktvU4Psg2SaTzOitJVXFZpM9+9/a71ue2SC9IkBS+YXgiwiFUhOs0pGRuQKNRTlM0kzUG5sdJ7ugjrozcttHbg+HXMGZTIPDwI6RW1zpkvERpuJLfiOB+CfrfhrUG+fu3kzRLJm8nb7/viZoplwscA7QBOyMWNFqnJbI43svpC1cfb+GLQ71Yp5RzIbgs1qi0Z8IeDH+mNv4EqqQlXHo4CGgsL4lFBnRGdIEGpkrDQjkdrEs9v7nsGJr6X2NwcrSk9XzbFKRNKyeRWTbT4VO2RT1HjWCoJlUt7sZi/hFEPp9xOls2sMHOGm+08rYF8zxMRVamZ9fwbpNCWxJNOytvt8/kPSKBZJebZsFUSPmUI4P5DGVUrZb8gVR8w3xfSFIqlp80OnUjZxob/8Xr0OX1+1N0E3NzEm3MzRkZ37x/6K+ADVX6w2lU6Q/npPrhh9PmGq3cxCpLxKRasfxL8oYSgSybCkW6Hzhg0lWoKUpLirigCqFosKk3H34AqsrKWQQnuU3iIRqBOu3NiVh42+oMgpJBjlwaSyTFyVYiVCPjNnOGFJtth1ArS8WBHKQrc9Qe/4ePv0PsxHgjEsehjS2sEf5TznLBvxLf7F68ORH+u6MgRhJW1DbwKGi5xDwjxi9n2iEDw/0TbmFODAjiJJ0h8/6rsURbZNvJGKcr4Ux2BlKpq1VGM/KEkCPK5cgQCU4KXnKvcQ3dYPP91z58/P1DaOF9xJp8Tm7gK2p1KFOTRf+guyINRDVw2UjYzxWprPeRGTA1l57y+nhfAJEsmRU7c35/QZ32siGMcY+CiOTibKYs0c6VfpxwOamI94XNKExT26CRIn/ySie9vai7By4t6qn3LrqT7lDYWYU6M0hHhV+hBoNUSRYNtXJ2KCbK2bOMwIi4/+5DwOUkX1g8WP5TpUti38GmL/XiFhoYY26Ehkcdlgi9NShDs/D69ZKjMsZ8eYFhGYoG4+aRq4lGws42LO/T9CDJqfb4mwXfWKURnpRwJRogT4QLkgsEq45hM/CgtJC3xmIcEnPNLZ55THyfFqXHOSafUUalxt4amDFoBN1S1VCL+gdVVgK9yxu0SlWowz7EHK9VMSa9DJtUqLli3opYXh5GbuAB2k5yiFl0At+ok2OMZmi5zfRIXRyC3GijuUby9Ll3DF9jiXVmQmdIH7Mp4WKw7d09Vkpb43ehdoZ6FanfiVfEGGSQKztbfRkxQcAU9nT+rVkYi+XqOx7jJYIYCyWXzh5OMovtnZnrGETqfl6AyuYRO5RMs2RQpf1/nNwcl/NuWYH65HCW0im1sX+9at7ykhQ44ZvnxNEB+tvrMCk9DN9+kyyNYag++JZR04kfgwETCbeScUqCc5A0gaENGtcO1XIDKL0t2hIva4BWmj8RixMmTdbJWw4g0NQ6XP/6kPLQUbxrnv2BKHm1WRO7j3tAu/349BMQxjQaA8QYRXmID895Mn+9sbpccDqWQEPja/I8UCsTtAGlWAsu4bjxxoVTuP3YvHntBfwGcuXiAnqMSMMUmlDFNkvzaEMU2u3K8AKIAQL//O/LnFtw0vBChuht6OQgpMOP+0ak8LpCyfx0/wu0kzL+ZWbOWi6LyxCR/Qss6pLLoNN/eY8lpMnrP5G92cPIzrx/G/0tb6rHWgpSP8HdqpeF9RwoitPSnyjOmfm8uduc9DwoDyhImTNyEtvYxBkJ34UOT0n0anZaolezcyZ676+PSPTCUNnPOjSSUpxHrCYrudG+WcP/z3KeO8vJiCU5MZhRJSXSsD8dhU7dEbQ6SsnwLcjyZr8zIVoOt/hdleSrknCf6u5CZdzrq/tf3wQVQEJn3mTsB0UFMZtldRSsD20L0/bE6pICvz0usVR6AZRUhHK7gICh/uD1+32xuRb6VK3J15bYISgQP6z60riqEhzZcvCXvU7g84yb1gO/wfAsnORfHIZ6yaDvzSd8s70oxq3qcPQeUrgl4kwlHW0/ipuG6faQU/bFocOMYWVnG7EdWZWynGrKWS+B4Mbd/mbgtXeD/hGjUBq/ODTWvIE54d6nCxEoSr1f7Vl5hJux18GULyIzqJ9QZ6RAabP/qHwcixE7hIdPd/AQOoQr3yH4DoG5sIIeFH2YakS/cc3i7DlrXo2UoaJSTVuxPE0kU2Ut9QRqK/LMWKVJcb4cxzbYCQeEesPNeEvyzEtXZs4gy6wm0pBg6TPOhtSR1A20eoDb67pkxsSKGY9hAlfBAoW48kdlbKHx4dPdZvBKMDQ201gJTsP6nxmhbCZIMSnzAeELUhReeQ3/2hj51GvzLriZyoRaXL/dCkb+j6u7YGCabHMvfiFrwbdF8Y+0OuQJg1K0FnpuHmMC4/Yfv22Oem/DF0QQ5N0jDF9rOnOxo1OU3fISgcC9R3+fRqS15PjR8do143WkOnoQ7VWpPSK/LB4+3V3AL0Rzcv0+Fi0tR2mlmy3+hpmTKnrFLzT9PYA442PoMtUtrjAOC3ncy/hFXCrbshrep1qa8M0s25ZCqMJkBUrcOJqnTLugmC0qfgPQMiC+417zKSaOzjSh4urdc0Z9caj54UpzFLrUB+AzUmeR7QXFkDCh6OO4sJpe6hxF44Luwxdzb2EJe6k5lxbaWkvj+QynlV6xRheeWuhsF5HJbrM/aiURF6JOZnc0N2axgQpnLOoE9cIvAcrvVYFY+Pky+nQxpPtExG6ae2bjmDzj3AzTtEMzBaeHoBlcQaEoES/sENbauWriLZaV0kQvwPpnJqx13qTu01KhCi5DgtPpkU1V2lCEHoFYD9nuN6J2ppUrZpWzE6rKkm+OqQ1m7WMffax8CyBDgVsyg8MtR6GPxu73QcfEuNCur++aDW4vYOXIwLg0qK25AFcxYjHVr0dJ9kIaGzoH2GMGOCXhBoXX2J06w7fsL9aTNJW3cU3xnrn36kpubUy1U8FRWhNPEtDZSg2Nt85pZQ3Oul9fk7VeGq4jRJAlVEOKgkuqSr83fH0fG3+zlIkm0ymnG7xzz4IKF2JBQVzUGatK1EuHqP6yF10dG71+aB4HL8Sb+FbigoSK52affLBU6pEZUizK2UIFsXxOrf995OJdozEmc2dHa8ljqvhcd1L2YjQocEsiaTCTE/s4xuREgzouutjHMeiCZzguuLxbkRyGeB9GQSxKuujr0QwZa0kQwhRac3qC8S25EDyy2OY8Jhp9PIuxOITAHMMplzzGE4gsnB+r19fXd28av6Qvsx6uyVjMdnovPfn0dGDGpVRP6Z4celntARgMZdRr/D0t+lhjsGr0e45BT7s/FofVpaEnh36rwzeoSD23m6NZ3pUd6YGDEFKxKbLOQ9j5heIprbC0otRVPAb9ci6JXoQQSu2+lsTvS9YzDDHCpncmElp0uwmuYZNbG6LsrQ7BdwhTLrBfrL0Fv5ssGB3+SUmC1pfNxP9/y55wqBhXXZXQ7jfF5v0GRUkgst7xLqsy6h3xXte2zSYXij4OekXAOp0VGt1IfnNjQEKyP/XQKg5heZY2+tkYpTBHFrfUkeJ0socSIaKNSxvQZRYgfXI/Ua3EgJXD1+/BN2hA8EeEP+5vP9/cg9Jwf3N1fXN/MSRwlAWXOHDF+w2hs5WUrnYyyT72dxGZdVO3rbStd37R0s0ESOCZpSUla+W0h5wn3YS1Xuaqaw2qq7eXsg/HD+KCQVVZEctzLrhd7Mhq7xyrRLUQKiciY3mzsCDLgmuTcdVvTd17TqVlvP4VuoXrZAwuYtlcJyfTSccsAcZgoY2HNkq/0IbaWyy8kd+YtUkn7oJ1Wf38gdLxZisGwKaozyyXpcJoZMqvYnG7WsPRbYlEN6MjkJOotz2OUE0zFPP/Vfpw6oIU8Z6cBo4s6i3tLn040KFMrFPjkxF5ppKR0/itZJGPYZeV5Hk4hu2yrlVKOdo5otwIPtpib9LX0+O1u9CJ6B9HlcuBqXL5LVDNCX20mtDHjM6ILDDTSJVmZkI1xumqt+2yT76qo+4aYteQuobQNTJQT6hhyp8w1Xa27qrctzJtpRVOVw/qsVLriDiE1koxx+EE5lwyNZ/Efgbd58QbLimuaJ0lukDbYhH7b45mJ77d94eyENtif6dqk/eD0vmIHTC9F25KIkQ47Ex2U/baRqDgTzE4csDx+lAXkQqHCH10VabRev9eySy2MOiy7yUQDvq0rEjst6nRaDKY9bFj46pK6SikSnFpL7m8DE6kxnj1wBSJdRqDt7iaIF0q7fem7qghuFMRVkRjJKnMTPW83mhAWVAljSvjbCRC1PRqXNHOkA1bllBYzxmGi/Z6CYASOsNsxm0WXNFJ7vzsG5D76rGrpgai2SGHin1Wn3mK3UdUhwHWaJywmcEhp28/0PcBgkG7C3faM7oqzNNhr5nqhk1rY7NyGiuUnqe9V1iEd66/mpnMqix5HFXcY5ovIjuyFrpnCLVoAezhOt5fP7T3ww1/q0CF6wOkYtiEa/YudK6qK9qyWDGYxeOLL2Uf/PSPZzYXysX4UixkbK8IB8Yz0simmlKBUzsSOY0l4WHD3zqwEcKY9TUY3SLEEolxOhxD79bnLe+fzxjhYlGPz6su1j7HaLuNdc7UhnfNYIx5wvbhx9MO2KbL8w3/+lIlmGHv3uhrdGpjfKJ7sX8bd/SWMjXN4iX3w8+t1hG02MMGbHEWCdEMdTjCuEX70ppwqt6lZloal55803pWL4jxNPeIg/Xvz58/LpffkrBgyr0HFGO3zQ9GXIDGgmgm6vs5FtWWdbjBXgzqMXQw/+vmcwe3V65a97jcxGEP3sqNiPfj74Pj3ZGCHQTy9c3dzeeboVHPtlVQDIL53zdX1wfp8z5dUGZMZfjtoasNR6HcUc1xKs4lkoebu5sPn+G3MOjhnLc3dANrRWSSGUqkPPPhm249Xb3IJiwxd3KwOE5hX/9gzDdBv/n1mjPwF3zM2ba6u/R9pbsVAnQTfxlpF06m5lIowl5mZOKwLDGEyXbYkj2febcmnjE2lZIh30+FYyHnnCu25ey5q16abo0gjllyu0K2MzpvHvtFf8uJWittJj89P4+nbj89P6dzB7G75tpExfCgcYszjqSfb1BTQB621v8FSsM/dxL7eUxiPz8/x7iMPiOxut5syrWxmVeOHtmY06vOKtSXtc6F0E8TEaHpjtelSqLfDTRHUuLvdayJwKoYbVmZlOGanlBTlGNjeHfLIzjy9e7mrCJBQSoTK262iCaMVZjIS3GkxHq4sCO8CdulfXO32Q/K0+7xMvKc93g9/Lr5Hq8DLy0zX04k++WsZD+deGlZunujRGNIgRkpegVvBygsrSqtnsNv40GKR3uZRVgglbyMGy0GCWId3QyX+Wy5DCV+MuzRyGK4tOOqVa57WQG0jKGnvkMCb/1iBo0knIDiZYmME4tiizPQcJHKZk/c8HxLUdmpi0xDp2HAJUwFL2ZbVvMG2VlQdcVnNccnIpZm70B98Ko0LtJaX3shqy31uNCaXUW+AEqEaM7Jp3ONv6QpFus+90A269cUDj3mjMW1i+ySIZaVXdTHPse5EKsjnquPt7X4/FxhPM7wKF0gNYEtKVmUS3N79lB2ffNRTxnHV8MWhD58ekg2c6XdZhdkTs17hBa2rMXdH4X18yDUIIQvhZORqsSwxG74meD9XsX/BQAA///Tsm18" } diff --git a/x-pack/metricbeat/module/aws/rds/_meta/data.json b/x-pack/metricbeat/module/aws/rds/_meta/data.json index 42fc3be4033d..70fa4d531b0e 100644 --- a/x-pack/metricbeat/module/aws/rds/_meta/data.json +++ b/x-pack/metricbeat/module/aws/rds/_meta/data.json @@ -2,48 +2,62 @@ "@timestamp": "2017-10-12T08:05:34.853Z", "aws": { "rds": { + "aurora_bin_log_replica_lag": 0, + "aurora_replica.lag.ms": 19.683, + "aurora_replica.lag_max.ms": 19.651500701904297, + "aurora_replica.lag_min.ms": 19.651500701904297, + "cache_hit_ratio.buffer": 100, + "cache_hit_ratio.result_set": 0, "cpu": { "total": { - "pct": 0.999999999997575 + "pct": 0.035 } }, "database_connections": 0, - "db_instance": { - "arn": "arn:aws:rds:ap-southeast-1:627959692251:db:database-1", - "class": "db.t2.micro", - "identifier": "database-1", - "status": "available" - }, - "disk_queue_depth": 0.00006666888896296543, + "db_instance.class": "db.r5.large", + "deadlocks": 0, "disk_usage": { - "bin_log.bytes": 2752 + "bin_log.bytes": 0 }, - "free_storage.bytes": 20402475008, - "freeable_memory.bytes": 441335808, + "engine_uptime.sec": 278598.25, + "free_local_storage.bytes": 33112842240, + "freeable_memory.bytes": 4879514624, "latency": { - "read": 0, - "write": 0.0002909090909090909 + "commit": 2.9322999999999997, + "ddl": 0, + "delete": 0, + "dml": 0.08325833333333334, + "insert": 0.08325833333333334, + "select": 0.20890321021571023, + "update": 0 }, - "read_io.ops_per_sec": 0, - "swap_usage.bytes": 1167360, + "login_failures": 0, + "queries": 7.807070323085375, "throughput": { - "network_receive": 570.7618714269047, - "network_transmit": 3228.0614030701536, - "read": 0, - "write": 3754.7918263942133 + "commit": 0.2500020925349523, + "ddl": 0, + "delete": 0, + "dml": 0.2500020925349523, + "insert": 0.2500020925349523, + "network": 1.7498192019524124, + "network_receive": 0.8749096009762062, + "network_transmit": 0.8749096009762062, + "select": 2.8751975752392616, + "update": 0 }, - "transactions": {}, - "write_io.ops_per_sec": 0.40002000100005003 + "transactions": { + "active": 0, + "blocked": 0 + } } }, "cloud": { "account": { - "id": "627959692251", - "name": "elastic-test" + "id": "428152502467", + "name": "elastic-beats" }, - "availability_zone": "ap-southeast-1a", "provider": "aws", - "region": "ap-southeast-1" + "region": "eu-west-1" }, "event": { "dataset": "aws.rds", diff --git a/x-pack/metricbeat/module/aws/rds/_meta/docs.asciidoc b/x-pack/metricbeat/module/aws/rds/_meta/docs.asciidoc index 1aa2090ae915..9081d2799691 100644 --- a/x-pack/metricbeat/module/aws/rds/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/aws/rds/_meta/docs.asciidoc @@ -1,6 +1,11 @@ -The rds metricset of aws module allows you to monitor your AWS RDS service. `rds` metricset fetches a set of values from -https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/MonitoringOverview.html[Amazon RDS Metrics]. +The rds metricset of aws module allows you to monitor your AWS RDS service. `rds` metricset fetches a set of metrics from +https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/MonitoringOverview.html[Amazon RDS] and +https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/Aurora.Monitoring.html[Amazon Aurora DB]. +with Amazon RDS, users can monitor network throughput, I/O for read, write, and/or +metadata operations, client connections, and burst credit balances for their DB instances. Amazon RDS sends metrics and dimensions to Amazon CloudWatch every minute. +Amazon Aurora provides a variety of Amazon CloudWatch metrics that users can +use to monitor health and performance of their Aurora DB cluster. [float] === AWS Permissions diff --git a/x-pack/metricbeat/module/aws/rds/_meta/fields.yml b/x-pack/metricbeat/module/aws/rds/_meta/fields.yml index fbb3e02811aa..4242b6845aa5 100644 --- a/x-pack/metricbeat/module/aws/rds/_meta/fields.yml +++ b/x-pack/metricbeat/module/aws/rds/_meta/fields.yml @@ -63,7 +63,7 @@ type: long description: > The lagging size of the replica lagging the most in terms of WAL data received. Applies to PostgreSQL. - - name: read_io.ops_per_sec + - name: read.iops type: float description: > The average number of disk read I/O operations per second. @@ -81,7 +81,7 @@ type: long description: > The disk space used by transaction logs. Applies to PostgreSQL. - - name: write_io.ops_per_sec + - name: write.iops type: float description: > The average number of disk write I/O operations per second. @@ -98,6 +98,16 @@ format: bytes description: > The amount of storage used by your Aurora DB instance, in bytes. + - name: volume.read.iops + type: long + format: bytes + description: > + The number of billed read I/O operations from a cluster volume, reported at 5-minute intervals. + - name: volume.write.iops + type: long + format: bytes + description: > + The number of write disk I/O operations to the cluster volume, reported at 5-minute intervals. - name: free_local_storage.bytes type: long format: bytes @@ -195,6 +205,11 @@ format: duration description: > The average amount of time taken per disk I/O operation. + - name: latency.delete + type: float + format: duration + description: > + The amount of latency for delete queries, in milliseconds. - name: disk_usage.bin_log.bytes type: long format: bytes @@ -228,3 +243,79 @@ type: keyword description: > Each DB instance runs a DB engine, like MySQL, MariaDB, PostgreSQL and etc. + - name: aurora_bin_log_replica_lag + type: long + description: > + The amount of time a replica DB cluster running on Aurora with MySQL compatibility lags behind the source DB cluster. + - name: aurora_global_db.replicated_write_io.bytes + type: long + description: > + In an Aurora Global Database, the number of write I/O operations replicated from the primary AWS Region to the cluster volume in a secondary AWS Region. + - name: aurora_global_db.data_transfer.bytes + type: long + description: > + In an Aurora Global Database, the amount of redo log data transferred from the master AWS Region to a secondary AWS Region. + - name: aurora_global_db.replication_lag.ms + type: long + description: > + For an Aurora Global Database, the amount of lag when replicating updates from the primary AWS Region, in milliseconds. + - name: aurora_replica.lag.ms + type: long + description: > + For an Aurora Replica, the amount of lag when replicating updates from the primary instance, in milliseconds. + - name: aurora_replica.lag_max.ms + type: long + description: > + The maximum amount of lag between the primary instance and each Aurora DB instance in the DB cluster, in milliseconds. + - name: aurora_replica.lag_min.ms + type: long + description: > + The minimum amount of lag between the primary instance and each Aurora DB instance in the DB cluster, in milliseconds. + - name: backtrack_change_records.creation_rate + type: long + description: > + The number of backtrack change records created over five minutes for your DB cluster. + - name: backtrack_change_records.stored + type: long + description: > + The actual number of backtrack change records used by your DB cluster. + - name: backtrack_window.actual + type: long + description: > + The difference between the target backtrack window and the actual backtrack window. + - name: backtrack_window.alert + type: long + description: > + The number of times that the actual backtrack window is smaller than the target backtrack window for a given period of time. + - name: storage_used.backup_retention_period.bytes + type: long + description: > + The total amount of backup storage in bytes used to support the point-in-time restore feature within the Aurora DB cluster's backup retention window. + - name: storage_used.snapshot.bytes + type: long + description: > + The total amount of backup storage in bytes consumed by all Aurora snapshots for an Aurora DB cluster outside its backup retention window. + - name: cache_hit_ratio.buffer + type: long + description: > + The percentage of requests that are served by the buffer cache. + - name: cache_hit_ratio.result_set + type: long + description: > + The percentage of requests that are served by the Resultset cache. + - name: engine_uptime.sec + type: long + description: > + The amount of time that the instance has been running, in seconds. + - name: rds_to_aurora_postgresql_replica_lag.sec + type: long + description: > + The amount of lag in seconds when replicating updates from the primary RDS PostgreSQL instance to other nodes in the cluster. + - name: backup_storage_billed_total.bytes + type: long + description: > + The total amount of backup storage in bytes for which you are billed for a given Aurora DB cluster. + - name: aurora_volume_left_total.bytes + type: long + description: > + The remaining available space for the cluster volume, measured in bytes. diff --git a/x-pack/metricbeat/module/aws/rds/data.go b/x-pack/metricbeat/module/aws/rds/data.go index ababd9af1b4c..2bb4da5d93c6 100644 --- a/x-pack/metricbeat/module/aws/rds/data.go +++ b/x-pack/metricbeat/module/aws/rds/data.go @@ -26,7 +26,7 @@ var ( "free_storage.bytes": c.Float("FreeStorageSpace"), "maximum_used_transaction_ids": c.Float("MaximumUsedTransactionIDs"), "oldest_replication_slot_lag.mb": c.Float("OldestReplicationSlotLag"), - "read_io.ops_per_sec": c.Float("ReadIOPS"), + "read.iops": c.Float("ReadIOPS"), "throughput": s.Object{ "commit": c.Float("CommitThroughput"), "delete": c.Float("DeleteThroughput"), @@ -50,6 +50,7 @@ var ( "select": c.Float("SelectLatency"), "update": c.Float("UpdateLatency"), "write": c.Float("WriteLatency"), + "delete": c.Float("DeleteLatency"), }, "replica_lag.sec": c.Float("ReplicaLag"), "disk_usage": s.Object{ @@ -59,7 +60,7 @@ var ( }, "swap_usage.bytes": c.Float("SwapUsage"), "transaction_logs_generation": c.Float("TransactionLogsGeneration"), - "write_io.ops_per_sec": c.Float("WriteIOPS"), + "write.iops": c.Float("WriteIOPS"), "queries": c.Float("Queries"), "deadlocks": c.Float("Deadlocks"), "volume_used.bytes": c.Float("VolumeBytesUsed"), @@ -75,5 +76,34 @@ var ( "db_instance.class": c.Str("DatabaseClass"), "db_instance.role": c.Str("Role"), "db_instance.engine_name": c.Str("EngineName"), + + "aurora_bin_log_replica_lag": c.Float("AuroraBinlogReplicaLag"), + + "aurora_global_db.replicated_write_io.bytes": c.Float("AuroraGlobalDBReplicatedWriteIO"), + "aurora_global_db.data_transfer.bytes": c.Float("AuroraGlobalDBDataTransferBytes"), + "aurora_global_db.replication_lag.ms": c.Float("AuroraGlobalDBReplicationLag"), + + "aurora_replica.lag.ms": c.Float("AuroraReplicaLag"), + "aurora_replica.lag_max.ms": c.Float("AuroraReplicaLagMaximum"), + "aurora_replica.lag_min.ms": c.Float("AuroraReplicaLagMinimum"), + + "backtrack_change_records.creation_rate": c.Float("BacktrackChangeRecordsCreationRate"), + "backtrack_change_records.stored": c.Float("BacktrackChangeRecordsStored"), + + "backtrack_window.actual": c.Float("BacktrackWindowActual"), + "backtrack_window.alert": c.Float("BacktrackWindowAlert"), + + "storage_used.backup_retention_period.bytes": c.Float("BackupRetentionPeriodStorageUsed"), + "storage_used.snapshot.bytes": c.Float("SnapshotStorageUsed"), + + "cache_hit_ratio.buffer": c.Float("BufferCacheHitRatio"), + "cache_hit_ratio.result_set": c.Float("ResultSetCacheHitRatio"), + + "engine_uptime.sec": c.Float("EngineUptime"), + "volume.read.iops": c.Float("VolumeReadIOPs"), + "volume.write.iops": c.Float("VolumeWriteIOPs"), + "rds_to_aurora_postgresql_replica_lag.sec": c.Float("RDSToAuroraPostgreSQLReplicaLag"), + "backup_storage_billed_total.bytes": c.Float("TotalBackupStorageBilled"), + "aurora_volume_left_total.bytes": c.Float("AuroraVolumeBytesLeftTotal"), } )