enhancement(source metrics): Adding a histogram for event byte size (#…

…19686) * Adding a histogram for event byte size. Useful when debugging low-level sources like UDP, TCP sockets * Adding documentation and fixing CI * fixing things up * Making sure only low-level sources have event size histograms (especially those based on UDP). * Add docs fixes * cargo fmt * fixup
vectordotdev · Apr 9, 2024 · 2f9ded0 · 2f9ded0
1 parent e4e7321
commit 2f9ded0
Show file tree

Hide file tree

Showing 6 changed files with 21 additions and 1 deletion.
diff --git a/changelog.d/19686_element_size_histogram.enhancement.md b/changelog.d/19686_element_size_histogram.enhancement.md
@@ -0,0 +1,7 @@
+Added a new histogram metric, `component_received_bytes`, that measures the byte-size of individual events received by the following sources:
+
+- `socket`
+- `statsd`
+- `syslog`
+
+authors: pabloem
diff --git a/src/internal_events/socket.rs b/src/internal_events/socket.rs
@@ -1,4 +1,4 @@
-use metrics::counter;
+use metrics::{counter, histogram};
 use vector_lib::internal_event::{ComponentEventsDropped, InternalEvent, UNINTENTIONAL};
 use vector_lib::{
     internal_event::{error_stage, error_type},
@@ -40,6 +40,7 @@ impl InternalEvent for SocketBytesReceived {
             "component_received_bytes_total", self.byte_size as u64,
             "protocol" => protocol,
         );
+        histogram!("component_received_bytes", self.byte_size as f64);
     }
 }
 
@@ -61,6 +62,7 @@ impl InternalEvent for SocketEventsReceived {
         );
         counter!("component_received_events_total", self.count as u64, "mode" => mode);
         counter!("component_received_event_bytes_total", self.byte_size.get() as u64, "mode" => mode);
+        histogram!("component_received_bytes", self.byte_size.get() as f64, "mode" => mode);
     }
 }
 

diff --git a/website/cue/reference/components/sources/internal_metrics.cue b/website/cue/reference/components/sources/internal_metrics.cue
@@ -370,6 +370,12 @@ components: sources: internal_metrics: {
 			default_namespace: "vector"
 			tags:              component_received_events_total.tags
 		}
+		component_received_bytes: {
+			description:       string | *"The size in bytes of each event received by the source."
+			type:              "histogram"
+			default_namespace: "vector"
+			tags:              component_received_events_total.tags
+		}
 		component_received_events_total: {
 			description: """
 				The number of events accepted by this component either from tagged

diff --git a/website/cue/reference/components/sources/socket.cue b/website/cue/reference/components/sources/socket.cue
@@ -114,5 +114,6 @@ components: sources: socket: {
 		connection_established_total: components.sources.internal_metrics.output.metrics.connection_established_total
 		connection_send_errors_total: components.sources.internal_metrics.output.metrics.connection_send_errors_total
 		connection_shutdown_total:    components.sources.internal_metrics.output.metrics.connection_shutdown_total
+		component_received_bytes:     components.sources.internal_metrics.output.metrics.component_received_bytes
 	}
 }
diff --git a/website/cue/reference/components/sources/statsd.cue b/website/cue/reference/components/sources/statsd.cue
@@ -84,4 +84,7 @@ components: sources: statsd: {
 				"""
 		}
 	}
+	telemetry: metrics: {
+		component_received_bytes: components.sources.internal_metrics.output.metrics.component_received_bytes
+	}
 }
diff --git a/website/cue/reference/components/sources/syslog.cue b/website/cue/reference/components/sources/syslog.cue
@@ -213,5 +213,6 @@ components: sources: syslog: {
 	telemetry: metrics: {
 		connection_read_errors_total: components.sources.internal_metrics.output.metrics.connection_read_errors_total
 		utf8_convert_errors_total:    components.sources.internal_metrics.output.metrics.utf8_convert_errors_total
+		component_received_bytes:     components.sources.internal_metrics.output.metrics.component_received_bytes
 	}
 }
-Original file line number
+Diff line change
@@ Expand Up / @@ -84,4 +84,7 @@ components: sources: statsd: { @@
     				"""
     		}
     	}
+    	telemetry: metrics: {
+    		component_received_bytes: components.sources.internal_metrics.output.metrics.component_received_bytes
+    	}
     }