diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index d468c751626e..51d8d577eaaa 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -350,6 +350,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Release Oracle module as GA. {issue}14279[14279] {pull}16833[16833] - Add Storage metricsets to GCP module {pull}15598[15598] - Release vsphere module as GA. {issue}15798[15798] {pull}17119[17119] +- Add PubSub metricset to Google Cloud Platform module {pull}15536[15536] *Packetbeat* diff --git a/metricbeat/docs/modules/googlecloud.asciidoc b/metricbeat/docs/modules/googlecloud.asciidoc index 6daf064b7720..18fc1b9c165c 100644 --- a/metricbeat/docs/modules/googlecloud.asciidoc +++ b/metricbeat/docs/modules/googlecloud.asciidoc @@ -44,7 +44,7 @@ Authentication and authorization in Google Cloud Platform can be achieved in man } ---- -Generally, you have to create a Service Account and assign it the following roles or the permissions described on each role: +Generally, you have to create a Service Account and assign it the following roles or the permissions described on each role (applies to all metricsets): * `Compute Viewer` (1 permission in use of 138): - `monitoring.metricDescriptors.list` @@ -104,6 +104,7 @@ metricbeat.modules: - pubsub - loadbalancing zone: "us-central1-a" + region: "us-central1" project_id: "your project id" credentials_file_path: "your JSON credentials file path" exclude_labels: false @@ -128,11 +129,15 @@ The following metricsets are available: * <> +* <> + * <> include::googlecloud/compute.asciidoc[] include::googlecloud/loadbalancing.asciidoc[] +include::googlecloud/pubsub.asciidoc[] + include::googlecloud/storage.asciidoc[] diff --git a/metricbeat/docs/modules/googlecloud/pubsub.asciidoc b/metricbeat/docs/modules/googlecloud/pubsub.asciidoc new file mode 100644 index 000000000000..66798829a79c --- /dev/null +++ b/metricbeat/docs/modules/googlecloud/pubsub.asciidoc @@ -0,0 +1,18 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-googlecloud-pubsub]] +=== Google Cloud Platform pubsub metricset + +beta[] + +include::../../../../x-pack/metricbeat/module/googlecloud/pubsub/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index 9d85ff6d66fa..5de200451d53 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -109,8 +109,9 @@ This file is generated! See scripts/mage/docs_collector.go .2+| .2+| |<> |<> |<> beta[] |image:./images/icon-yes.png[Prebuilt dashboards are available] | -.3+| .3+| |<> beta[] +.4+| .4+| |<> beta[] |<> beta[] +|<> beta[] |<> beta[] |<> |image:./images/icon-no.png[No prebuilt dashboards] | .1+| .1+| |<> diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 77bd945ca46d..d01c084f872e 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -487,6 +487,7 @@ metricbeat.modules: - pubsub - loadbalancing zone: "us-central1-a" + region: "us-central1" project_id: "your project id" credentials_file_path: "your JSON credentials file path" exclude_labels: false diff --git a/x-pack/metricbeat/module/googlecloud/_meta/config.yml b/x-pack/metricbeat/module/googlecloud/_meta/config.yml index 5df057bba18f..71a7885defcd 100644 --- a/x-pack/metricbeat/module/googlecloud/_meta/config.yml +++ b/x-pack/metricbeat/module/googlecloud/_meta/config.yml @@ -4,6 +4,7 @@ - pubsub - loadbalancing zone: "us-central1-a" + region: "us-central1" project_id: "your project id" credentials_file_path: "your JSON credentials file path" exclude_labels: false diff --git a/x-pack/metricbeat/module/googlecloud/_meta/docs.asciidoc b/x-pack/metricbeat/module/googlecloud/_meta/docs.asciidoc index 1dd26b9fcbb9..ff4b03cb0238 100644 --- a/x-pack/metricbeat/module/googlecloud/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/googlecloud/_meta/docs.asciidoc @@ -34,7 +34,7 @@ Authentication and authorization in Google Cloud Platform can be achieved in man } ---- -Generally, you have to create a Service Account and assign it the following roles or the permissions described on each role: +Generally, you have to create a Service Account and assign it the following roles or the permissions described on each role (applies to all metricsets): * `Compute Viewer` (1 permission in use of 138): - `monitoring.metricDescriptors.list` diff --git a/x-pack/metricbeat/module/googlecloud/fields.go b/x-pack/metricbeat/module/googlecloud/fields.go index 827247a0b365..3f8bc6b15c19 100644 --- a/x-pack/metricbeat/module/googlecloud/fields.go +++ b/x-pack/metricbeat/module/googlecloud/fields.go @@ -19,5 +19,5 @@ func init() { // AssetGooglecloud returns asset data. // This is the base64 encoded gzipped contents of module/googlecloud. func AssetGooglecloud() string { - return "eJzsW02T27gRvc+v6JvtlD17yG0OqbKV1K4r4+xUPOurCgRbEjIgwADN0cq/PgWApEhJJEGJ1HpTus2IZPfrBvBe4+sDvODuAdZaryVyqYv0DoAESXyANz/7X2HhfoYnyWilTfbmDsCgRGbxARIkdgeQouVG5CS0eoC/3QEA/Lx4gkynhcQ7gJVAmdoH/+ADKJZhy+W9ZAlK6x8D0C7HB9DJf5BT+VPz+6aNwqK5/0v988lv929nSCxlxEZ9YQS38R/YnSXM4t/nOssLwsbrh6ltm1kbXeSNX1uJbzXXIliuQmh8c5jMJh6hLDHFsfWwy3mXsabBlTC4ZVIevdBntM9w03hqdJ5jukx2hHbJdaHo5PuVL6nVuuOFViI/K64zodbgDVduINkBbbAvpGNoOeMvSLOAK01Hw6v7XF7M0hoGLZpXTJdcG7S9saa6SORhJzsZ7b+KLEEDegXeau0EtPLRbrQl99T93dF52ygLElJ8Z876RBCfXdIN4+6/CgqTUnNGmMLi6TegDSMQFnhhDCqSOxDKcVcVRBxwy9a4JJF1gRqL+zdnEFbaOLRleoUCi1yr9Lj96p4t7MtM/YdNP5QXzpJrlTCUnQ9YGZ11hXEAR+ezgHHOA5bPv4LO0fj+eHrMVHi2RhDOnR/nhFAB6eEEBUBzZsh7GEhRPTzyjqHRC6Dl/Be99e/5MfntC2yYhQRRgSmUEmr9PmZ4KKStNnONEI7idQbBOxolwU8YKS4b3VGdQDe95tX4Ks07D6FFRXPnzvkA/YpmHKor5KwfWYVGapYmTDLFRcvPVHXpo2YpfKocjCxPN0T54dDrHlYXgOgC0gSTuLyqdCkZoeLiZMnTgvDRsRoZkRTNUiF8vgPOJC+krxl8p95uMFQHBv9boCXYsrINy0IvN/r3nWNq908JBgpFQjYeHw+U6s3g25Lvtw6MQZtrZfF+Fu66oGPvZ2Osv2YbtJAUbiiEQdZfocaa0nm3cg+lDQZS13T3a6+fOF8x/po+//F7rhUqEkx+8tH2uY+HEAujCWVt9JY2yxXjpM3gV1G18LETy1nE2xE8fGxaFdlyJZSv4KJSOeDnkINKjhgQt/hSyM1pVD31aigbsxUf2UApvzw/P/301csGBN1wAqMrZPaYTbqwT426xlnOGJNdDco9PgU8BmwgyStlOjgrU12jf6sNcMY3+M5lelwgK6MVuUiI50tDp8BHiNa/n58dHdvCOGnRBpDxDXCtFIZ5cIK0daUzl8IHo9IgSDdxuYnLTVwGjf8Q4iIUoVFsyrXbMbXz/0/1PNx5+7tsDM1F9JwIqhthJYbuxpvrp7w4Fhge//HEF887caxzNv2NIcB4CjyfBEfS4HgiHEWFF5LhgK/9CldMlT0I/MxKO5RSNr7kOwY+B+QTZfZ4eFFF9ZSJbRXWXXOYMuN90EkTk1eWsVqVWlJ2qF1rTfBx8c9m3wGtgmhV8fu03BTrInM3xboYSDwYuClWhK9xejXHqtBorYpVqgkXg8YC++MWfkbrU7w6/Qm16bZc1GHqtlw0GkIsDLgtF530U+8a//W+Y7XoytuzFSBcG7S2l6x7UhdH1J69Pj9+2i8KVZQMb1fawPPiCVZSby0IemMD6+xPFWoFLM+l4P5gC1gyyDLQSu7eHbLeQVD9ZwXOC6t1SKAnsFITXFhdKIW6Vu5LTKRbYOfJfRXVdZJ/MrIuaIaoR+MHFL61k+NPh7jU7XdyrN/ecSh8Oo8FuY9ohxeCz5zrD8pw1NfDEjzGTJ/8TjObHhLeaaeaIyV3nlnnOLEdIbXjhDZaZi8Q2QGJJZ4vrZVLX0H/sSLbGL5SW3/iu6aLi+ZEzdPeewLyx6e3aBAITSaUn4JUVPXT16+PXVvLI4qB81AeCtK3Lw2pLKw/pd8PbZrteJtpTRtMPZu/FQoy+27P6s2p2Bvr+d0S4y/vw659JlRB2BJEyXZoyuhyZstpaL1lFgK8zc1uc7Pb3GzQ+A+ylX8tCtxXrt++xFGgwu3VBIQbHK0eOkc1GcBFuALUKP51QZaYSl2m2qCNLtYbT90dSD90nsr2lzo9yadGvGJzKMx3kbAeH6QNW89xqfFrsDzy1DjLRXS91L+q0L86HTv5+ztKYkHoXAf4+PQZOJPSvg+I9ortnmRIG516//4UXb08ynV6sDxax1vQ5vuFETescblMmKvywh3WJePcUcnlJHKQh3qJ3g9Xg7aQBEIBU+XtWUjQDZK1YcqN4YADrJYod5AW6EinfPPj4rFnWdyFtL9SNEEk4Sqda8vFY+OyEiRGv6CCVG+Vb9NdfmJFu+aZkF6bIxcrwZcOZVaQNzV9tvmGqTVayFjaTFzl/UQG+284nd/BIm81nR3pwZ2mwxswl446GHfB6MIwTl7imYo4jon7spYte/QEe2WamGxqpjdsIXezFF8ntVJQxgFcMmvv4XkjLLwyWSAI21jsUhy9hZTt3vsUudTV7xnMQ63AKMyTLMty6QjIr3S/MllJtC7If5qynioibMW5Rlx2XSH8c/TxfSCXRRDa1IrvnjiZlHWrirCt+IM17P8CAAD//5+zFC8=" + return "eJzsXE1z2zgSvedX9C3JlqMc9ubamipHszuTWmfHtXZyVYFgS8QaBLj4sEb59VMASIqURBKUKE08pZstkujXDeC9RgPkB3jGzS2spFxxpFza9A2AYYbjLbz9xf8Kc/czPHBillLlb98AKORINN5Cgoa8AUhRU8UKw6S4hZ/eAAD8Mn+AXKaW4xuAJUOe6lt/4QMIkmPL5IyTBLn2lwHMpsBbkMn/kJryp+bzzTasRjX7W/3zwWe3d+doSEoMGfWEYlTHP6A32mAefz+VeWENNm7fDW27mZWStmj82gp8q7vmoeXKhcYzu8Fs4mFCGyIoti52Ge9qrNngkilcE873buhrtK/hZuOpkkWB6SLZGNQLKq0wB++vbHEpVh03tAL5WVCZM7EC33BlBpINmAz7XNqHVhD6jOYs4Mqmo+HVY66wZ+kNhRrVC6YLKhXqXl9TaRO+O8gOevsfmyeoQC7Bt1obASm8t5nUxl11f3cM3jZKaxhn34lrfSKITy7oilD3XwWFcC4pMZjC/OErmIwYYBqoVQqF4RtgwnFX5UQccE1WuDAs7wI1FvdX1yAspXJoy/AyARqpFOl+/9Ujm+nnM40fMv1UnruWXK+EqexswFLJvMuNHTiyOAsYZzxg+fwbyAKVH4+H50yFZ62YwXPHxxkxKMDI4QAFQOeMkLcwEKJ6ehQdU6MXQMv4r3Lt7/Nz8tsXyIiGBFGAskIwsbqJmR4CzVqqc80QiuzlDIK3N0uCnTBTXDS6vTqAbnrNq/FVmnccQo3CnDt2zgbIF1TjUF0gZv3IKjRckjQhnAjKWnamykvvJUnhU2VgZHqaGVPsTr3uaXUCiC4gTTCJi6tIF5wYFJQdTHlaEO4cqxnFEttMFcLjG6CEU8t9zuAH9TrDkB0o/L9FbWBNyj4sE71Cyd83jqndPyUYsMIw3ri8P1GqO4Ntbfy4dWAU6kIKjbOzcNcJA3u7GiP9OdtgC4l1UyFMsv4MNbYpWXQr91DYYCB0TXO/9dqJsxVjr2nzn78XUqAwjPBP3ts+8/EQYmE0oayUXJtssSTUSDX4VFQuvG9EUxJxdwQP7zctbL5YMuEzuKhQDtjZ5aCSIwbELT4VcmsaUS+9GspGdMVHOlDKr09PDx8fvWxA0A0nMLJCpvfZpAv71KhrnOWKMdnUoNzlQ8BjwAaSvFCkg7Ey1DX6d1IBJTTD9y7S4xxZKimM88TQYqHMIfARovXfpydHx9oqJy1SARKaAZVCYFgHJ2jWLnWmnHlnRBoE6SouV3G5istg4z+EuDBhUAkyZe12TO7818mehwdv/5CNobmIkRNBdSNaiaG78c31U14cCwzP/3jii+edONY5mv7GEGA8BR5PgiNpcDwRjqLCE8lwwNa2whWTZQ8CPzLTDqmUjk/59oGfA/KBNHs8vKikesrAthLrrjVMGfE+6EYawi8sY7UqtaRsV7tW0sDd/N/NsQNSBNGq/PdhuSrWSc1dFetkIPFg4KpYEbbG6dU5qkKjtSpWqSYsBo0F9ucVfkbrU7w6vUJtupaLOpq6lotGQ4iFAddy0UE79a7x32cd1aILb89WgHClUOtesu4JXRxRe/b6fP9pWxSqKBneLaWCp/kDLLlca2DmrQ6ssz1VKAWQouCM+oMtoI1CkoMUfPN+l/V2nOo/K3CcW61DAj2OlZrg3OpCycSlYl9iMrIF9jyxr7y6TPAPetYFTRnTo/EDCt/ayfGnQ1zotjs52m/vOBQ+nPuC3Ee0w4XgI9f6gzIc9fSwBI9ppk9+p1lNDwnvtEvNkZJ7nlXnOLEdIbXjhDZaZk8Q2QGJNbRYaM0XPoP+c0W2MX251P7Ed00XJ62Jmqe9twTkj0+vUSEYVDkTfglSUdXHx8f7rq3lEcnAcSh3Benbl4ZUWu1P6fdDm2Y7XudSmgxTz+bvmIBcv9+yenMp9lZ7fteG0OebsGufM2ENtgSRkw2q0ruC6HIZWm+ZBQeva7Pr2uy6Nhts/AfZyr8UBW4z129f4ihQ4PpiAkIVjlYPWaCYDOA8vALUSP6lNdoQkbpItUEraVeZp+4OpBXCwibaJg3TUx3WfrDJo01GHtLWNqnbOy5PebT1P84V32PH5SeEPi9y1P7VqdNH/dzmlhPDXjDIlOu+snXtTAm55piuguTebf+vq883wfdwQ4qcvaDaeAD9Jw+5XIWpe1rFWRrCw4EVzb5jlTxY0YJeO/SOzJ5nZFYBqC+8ByaAtLq5Z3LbfNEY4VVnTDXLa7BlMNHXKtrg3moorM4ARVpIJswNJNaAkAY2aFrd1u+GFbWRqd04axdInqI2C29jC31BVqe9LXW3Qp9qli9Eva+GUzDX4VKXR6McKizni+bMrl8PO9Mcb3hSO7J9Ja01qRVqy80M/iUVEEjRi3mZqR96VGODaz9uybYVio+5TL3DKZKUM4FdnkfEbLoNtqFIHSS9Mj79OC/Vr87Yq+7Q83amD8+Rvaizs4PTGRBjMC8Og4OvgrNn9F7om7ACds/4DWAFLC845ihMWPGmErXXhIQYmvkPS9RsO4NHGZbK1U6nFHzjkjVDmNAgBbYemPnybtOYckMg1J1RKancOHEytWIvKFrPAiW+No1EQW65YQVHMCzHnl3eVrT7NnvjI/7zTpHBO1N5X5sIpQZGlaxEYNQo8S8+Xig7q47+tqemkdW0dEJcbqcfk6SFHQSX4FxcnB4r0w9uuv4FpOpALM9LJbXBNuPBmpkMhBQfHMdsWqFl6ThC3PHpUmNjx7NLDYh/UJniT0cNi9gI1sdxLjQkyvM4g52+PXlTMHrc6vfJPXriuveVRAvaapBeMPNLONPZq6XIad+b7ItQVP73obPi5L8u5ncbUuWE9IRiVPwXrbazQKr2MneqwthjaHlkZYwULJoS+o+39Pd97CmEn5Ebsu3yu4fPQInPlht97RJndyVHk8nU2/evc9bn9BzZHyZBYk32/USPm7U8vkiIxnQRPqa2IJSinqKavROHOgHwdeMw1n19QpSfcYMEHdmtFBGOogMO0JIj30Bq0eWV5Z138/uezN25tOWPCTwJ33RyfTm/bzATJEo+o1vlrIXv095ktgyvLpCyJaMLhzK3Zio+3ok2zYhwGXpO0mbgKusHItj/qZ3jB1jk53WO9nTn4zq7n2I5ddbBuC/dnOjGwa/JTEUc+8R9Ws+WI3qCQ9u+hN7YvPENayhQlTu9rRCUfgDlROsZPGVMwwvhFoHpxqkrQdG3kJLNjQ+RC119n8IibFoRE6oQmuQFdwTkj1y+EF7tFUlr/KMp6dnOCmfCXScuur5l9TrG+NaRKbZFqh0Rwnndqyycb//BOvaPAAAA//+LJduy" } diff --git a/x-pack/metricbeat/module/googlecloud/module.yml b/x-pack/metricbeat/module/googlecloud/module.yml index c2915aa8a2b6..15246c09f3c5 100644 --- a/x-pack/metricbeat/module/googlecloud/module.yml +++ b/x-pack/metricbeat/module/googlecloud/module.yml @@ -1,5 +1,6 @@ name: googlecloud metricsets: - compute +- pubsub - loadbalancing - storage diff --git a/x-pack/metricbeat/module/googlecloud/pubsub/_meta/data_subscription.json b/x-pack/metricbeat/module/googlecloud/pubsub/_meta/data_subscription.json new file mode 100644 index 000000000000..1503d6aa80fa --- /dev/null +++ b/x-pack/metricbeat/module/googlecloud/pubsub/_meta/data_subscription.json @@ -0,0 +1,53 @@ +{ + "@timestamp": "2020-01-14T11:46:00.000Z", + "@metadata": { + "beat": "metricbeat", + "type": "_doc", + "version": "8.0.0" + }, + "googlecloud": { + "labels": { + "resource": { + "subscription_id": "test-subscription" + } + }, + "pubsub": { + "subscription": { + "num_undelivered_messages": 0, + "oldest_unacked_message_age": 0, + "backlog_bytes": 0 + } + } + }, + "service": { + "type": "googlecloud" + }, + "cloud": { + "account": { + "id": "elastic-metricbeat" + }, + "provider": "googlecloud" + }, + "host": { + "name": "mcastro" + }, + "agent": { + "version": "8.0.0", + "type": "metricbeat", + "ephemeral_id": "b3b8d9da-0094-4d54-b1dc-4d5e103d38aa", + "hostname": "mcastro", + "id": "7e36a073-1a32-4a94-b65b-4c7f971fb228" + }, + "ecs": { + "version": "1.4.0" + }, + "event": { + "dataset": "googlecloud.pubsub", + "module": "googlecloud", + "duration": 933264720 + }, + "metricset": { + "name": "pubsub", + "period": 300000 + } +} diff --git a/x-pack/metricbeat/module/googlecloud/pubsub/_meta/data_topic.json b/x-pack/metricbeat/module/googlecloud/pubsub/_meta/data_topic.json new file mode 100644 index 000000000000..f85d81487820 --- /dev/null +++ b/x-pack/metricbeat/module/googlecloud/pubsub/_meta/data_topic.json @@ -0,0 +1,61 @@ +{ + "@timestamp": "2020-01-14T11:42:00.000Z", + "@metadata": { + "beat": "metricbeat", + "type": "_doc", + "version": "8.0.0" + }, + "ecs": { + "version": "1.4.0" + }, + "cloud": { + "provider": "googlecloud", + "account": { + "id": "elastic-metricbeat" + } + }, + "metricset": { + "name": "pubsub", + "period": 300000 + }, + "event": { + "duration": 932060852, + "dataset": "googlecloud.pubsub", + "module": "googlecloud" + }, + "googlecloud": { + "labels": { + "resource": { + "subscription_id": "test-subscription" + } + }, + "pubsub": { + "topic": { + "message_sizes": { + "bucket_options": { + "Options": { + "ExponentialBuckets": { + "growth_factor": 4, + "scale": 1, + "num_finite_buckets": 16 + } + } + } + } + } + } + }, + "service": { + "type": "googlecloud" + }, + "host": { + "name": "mcastro" + }, + "agent": { + "ephemeral_id": "b3b8d9da-0094-4d54-b1dc-4d5e103d38aa", + "hostname": "mcastro", + "id": "7e36a073-1a32-4a94-b65b-4c7f971fb228", + "version": "8.0.0", + "type": "metricbeat" + } +} diff --git a/x-pack/metricbeat/module/googlecloud/pubsub/_meta/docs.asciidoc b/x-pack/metricbeat/module/googlecloud/pubsub/_meta/docs.asciidoc new file mode 100644 index 000000000000..9fa3ea4f50c7 --- /dev/null +++ b/x-pack/metricbeat/module/googlecloud/pubsub/_meta/docs.asciidoc @@ -0,0 +1,25 @@ +PubSub Metricset to fetch metrics from https://cloud.google.com/pubsub/[Pub/Sub] topics and subscriptions in Google Cloud Platform. + +The `pubsub` Metricset contains all metrics exported from the https://cloud.google.com/monitoring/api/metrics_gcp#gcp-pubsub[Stackdriver API]. The field names have been left untouched for people already familiar with them. + +No special permissions are needed apart from the ones detailed in the module section of the docs. + +- `pubsub.subscription.ack_message_count`: Cumulative count of messages acknowledged by Acknowledge requests, grouped by delivery type. +- `pubsub.subscription.backlog_bytes`: Total byte size of the unacknowledged messages (a.k.a. backlog messages) in a subscription. +- `pubsub.subscription.num_outstanding_messages`: Number of messages delivered to a subscription's push endpoint, but not yet acknowledged. +- `pubsub.subscription.num_undelivered_messages`: Number of unacknowledged messages (a.k.a. backlog messages) in a subscription. +- `pubsub.subscription.oldest_unacked_message_age`: Age (in seconds) of the oldest unacknowledged message (a.k.a. backlog message) in a subscription. +- `pubsub.subscription.pull_ack_message_operation_count`: Cumulative count of acknowledge message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count. +- `pubsub.subscription.pull_ack_request_count`: Cumulative count of acknowledge requests, grouped by result. +- `pubsub.subscription.pull_message_operation_count`: Cumulative count of pull message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count. +- `pubsub.subscription.pull_request_count`: Cumulative count of pull requests, grouped by result. +- `pubsub.subscription.push_request_count`: Cumulative count of push attempts, grouped by result. Unlike pulls, the push server implementation does not batch user messages. So each request only contains one user message. The push server retries on errors, so a given user message can appear multiple times. +- `pubsub.subscription.push_request_latencies`: Distribution of push request latencies (in microseconds), grouped by result. +- `pubsub.subscription.sent_message_count`: Cumulative count of messages sent by Cloud Pub/Sub to subscriber clients, grouped by delivery type. +- `pubsub.subscription.streaming_pull_ack_message_operation_count`: Cumulative count of StreamingPull acknowledge message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count. +- `pubsub.subscription.streaming_pull_ack_request_count`: Cumulative count of streaming pull requests with non-empty acknowledge ids, grouped by result. +- `pubsub.subscription.streaming_pull_message_operation_count`: Cumulative count of streaming pull message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count +- `pubsub.subscription.streaming_pull_response_count`: Cumulative count of streaming pull responses, grouped by result. +- `pubsub.topic.streaming_pull_response_count`: Cumulative count of streaming pull responses, grouped by result. +- `pubsub.topic.send_message_operation_count`: Cumulative count of publish message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count. +- `pubsub.topic.send_request_count`: Cumulative count of publish requests, grouped by result. diff --git a/x-pack/metricbeat/module/googlecloud/pubsub/_meta/fields.yml b/x-pack/metricbeat/module/googlecloud/pubsub/_meta/fields.yml new file mode 100644 index 000000000000..64063c444973 --- /dev/null +++ b/x-pack/metricbeat/module/googlecloud/pubsub/_meta/fields.yml @@ -0,0 +1,70 @@ +- name: pubsub + release: beta + type: group + description: Google Cloud PubSub metrics + fields: + - name: subscription + type: group + description: Suscription related metrics + fields: + - name: ack_message_count + type: long + description: Cumulative count of messages acknowledged by Acknowledge requests, grouped by delivery type. + - name: backlog_bytes + type: long + description: Total byte size of the unacknowledged messages (a.k.a. backlog messages) in a subscription. + - name: num_outstanding_messages + type: long + description: Number of messages delivered to a subscription's push endpoint, but not yet acknowledged. + - name: num_undelivered_messages + type: long + description: Number of unacknowledged messages (a.k.a. backlog messages) in a subscription. + - name: oldest_unacked_message_age + type: long + description: Age (in seconds) of the oldest unacknowledged message (a.k.a. backlog message) in a subscription. + - name: pull_ack_message_operation_count + type: long + description: Cumulative count of acknowledge message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count. + - name: pull_ack_request_count + type: long + description: Cumulative count of acknowledge requests, grouped by result. + - name: pull_message_operation_count + type: long + description: Cumulative count of pull message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count. + - name: pull_request_count + type: long + description: Cumulative count of pull requests, grouped by result. + - name: push_request_count + type: long + description: Cumulative count of push attempts, grouped by result. Unlike pulls, the push server implementation does not batch user messages. So each request only contains one user message. The push server retries on errors, so a given user message can appear multiple times. + - name: push_request_latencies + type: long + description: Distribution of push request latencies (in microseconds), grouped by result. + - name: sent_message_count + type: long + description: Cumulative count of messages sent by Cloud Pub/Sub to subscriber clients, grouped by delivery type. + - name: streaming_pull_ack_message_operation_count + type: long + description: Cumulative count of StreamingPull acknowledge message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count. + - name: streaming_pull_ack_request_count + type: long + description: Cumulative count of streaming pull requests with non-empty acknowledge ids, grouped by result. + - name: streaming_pull_message_operation_count + type: long + description: Cumulative count of streaming pull message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count + - name: streaming_pull_response_count + type: long + description: Cumulative count of streaming pull responses, grouped by result. + - name: topic + type: group + description: Topic related metrics + fields: + - name: streaming_pull_response_count + type: long + description: Cumulative count of streaming pull responses, grouped by result. + - name: send_message_operation_count + type: long + description: Cumulative count of publish message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count. + - name: send_request_count + type: long + description: Cumulative count of publish requests, grouped by result. diff --git a/x-pack/metricbeat/module/googlecloud/pubsub/manifest.yml b/x-pack/metricbeat/module/googlecloud/pubsub/manifest.yml new file mode 100644 index 000000000000..e7e3e032b714 --- /dev/null +++ b/x-pack/metricbeat/module/googlecloud/pubsub/manifest.yml @@ -0,0 +1,27 @@ +default: true +input: + module: googlecloud + metricset: stackdriver + defaults: + stackdriver: + service: pubsub + metrics: + - "pubsub.googleapis.com/subscription/ack_message_count" + - "pubsub.googleapis.com/subscription/backlog_bytes" + - "pubsub.googleapis.com/subscription/num_outstanding_messages" + - "pubsub.googleapis.com/subscription/num_undelivered_messages" + - "pubsub.googleapis.com/subscription/oldest_unacked_message_age" + - "pubsub.googleapis.com/subscription/pull_ack_message_operation_count" + - "pubsub.googleapis.com/subscription/pull_ack_request_count" + - "pubsub.googleapis.com/subscription/pull_message_operation_count" + - "pubsub.googleapis.com/subscription/pull_request_count" + - "pubsub.googleapis.com/subscription/push_request_count" + - "pubsub.googleapis.com/subscription/push_request_latencies" + - "pubsub.googleapis.com/subscription/sent_message_count" + - "pubsub.googleapis.com/subscription/streaming_pull_ack_message_operation_count" + - "pubsub.googleapis.com/subscription/streaming_pull_ack_request_count" + - "pubsub.googleapis.com/subscription/streaming_pull_message_operation_count" + - "pubsub.googleapis.com/subscription/streaming_pull_response_count" + - "pubsub.googleapis.com/topic/message_sizes" + - "pubsub.googleapis.com/topic/send_message_operation_count" + - "pubsub.googleapis.com/topic/send_request_count" diff --git a/x-pack/metricbeat/modules.d/googlecloud.yml.disabled b/x-pack/metricbeat/modules.d/googlecloud.yml.disabled index 01898897cd6e..802c08c08aed 100644 --- a/x-pack/metricbeat/modules.d/googlecloud.yml.disabled +++ b/x-pack/metricbeat/modules.d/googlecloud.yml.disabled @@ -7,6 +7,7 @@ - pubsub - loadbalancing zone: "us-central1-a" + region: "us-central1" project_id: "your project id" credentials_file_path: "your JSON credentials file path" exclude_labels: false