From 30e388b6c9ce895689549024474e722fe6c9f9f4 Mon Sep 17 00:00:00 2001 From: Adam Boguszewski <108867528+aboguszewski-sumo@users.noreply.github.com> Date: Mon, 31 Oct 2022 14:07:54 +0100 Subject: [PATCH] [receiver/elasticsearch]: add segment metrics with total aggregation (#14786) * feat: add segment metrics with total aggregation --- ...elasticsearch-receiver-segments-total.yaml | 5 + .../elasticsearchreceiver/documentation.md | 4 + .../internal/metadata/generated_metrics.go | 230 +++++++++++++++ .../internal/model/nodestats.go | 10 + receiver/elasticsearchreceiver/metadata.yaml | 35 +++ receiver/elasticsearchreceiver/scraper.go | 35 ++- .../elasticsearchreceiver/scraper_test.go | 3 + .../testdata/expected_metrics/full.json | 270 ++++++++++++++++++ .../testdata/sample_payloads/indices.json | 24 +- 9 files changed, 603 insertions(+), 13 deletions(-) create mode 100644 .chloggen/elasticsearch-receiver-segments-total.yaml diff --git a/.chloggen/elasticsearch-receiver-segments-total.yaml b/.chloggen/elasticsearch-receiver-segments-total.yaml new file mode 100644 index 000000000000..2e9d13341bfd --- /dev/null +++ b/.chloggen/elasticsearch-receiver-segments-total.yaml @@ -0,0 +1,5 @@ +change_type: enhancement +component: elastisearchreceiver +note: Add new metrics related to segments, aggregated by all shards +issues: [14635] + diff --git a/receiver/elasticsearchreceiver/documentation.md b/receiver/elasticsearchreceiver/documentation.md index 193b8fb23d89..7bfdf7d89e90 100644 --- a/receiver/elasticsearchreceiver/documentation.md +++ b/receiver/elasticsearchreceiver/documentation.md @@ -26,6 +26,9 @@ These are the metrics available for this scraper. | elasticsearch.index.operations.merge.docs_count | The total number of documents in merge operations for an index. | {documents} | Sum(Int) | | | elasticsearch.index.operations.merge.size | The total size of merged segments for an index. | By | Sum(Int) | | | **elasticsearch.index.operations.time** | Time spent on operations for an index. | ms | Sum(Int) | | +| elasticsearch.index.segments.count | Number of segments of an index. | {segments} | Sum(Int) | | +| elasticsearch.index.segments.memory | Size of memory for segment object of an index. | By | Sum(Int) | | +| elasticsearch.index.segments.size | Size of segments of an index. | By | Sum(Int) | | | **elasticsearch.index.shards.size** | The size of the shards assigned to this index. | By | Sum(Int) | | | **elasticsearch.indexing_pressure.memory.limit** | Configured memory limit, in bytes, for the indexing requests. | By | Gauge(Int) | | | **elasticsearch.indexing_pressure.memory.total.primary_rejections** | Cumulative number of indexing requests rejected in the primary stage. | 1 | Sum(Int) | | @@ -121,6 +124,7 @@ metrics: | memory_state (state) | State of the memory | free, used | | operation (operation) | The type of operation. | index, delete, get, query, fetch, scroll, suggest, merge, refresh, flush, warmer | | query_cache_count_type (type) | Type of query cache count | hit, miss | +| segments_memory_object_type (object) | Type of object in segment | term, doc_value, index_writer, fixed_bit_set | | shard_state (state) | The state of the shard. | active, relocating, initializing, unassigned | | task_state (state) | The state of the task. | rejected, completed | | thread_pool_name | The name of the thread pool. | | diff --git a/receiver/elasticsearchreceiver/internal/metadata/generated_metrics.go b/receiver/elasticsearchreceiver/internal/metadata/generated_metrics.go index 51fd29418fca..b7785f85fcea 100644 --- a/receiver/elasticsearchreceiver/internal/metadata/generated_metrics.go +++ b/receiver/elasticsearchreceiver/internal/metadata/generated_metrics.go @@ -55,6 +55,9 @@ type MetricsSettings struct { ElasticsearchIndexOperationsMergeDocsCount MetricSettings `mapstructure:"elasticsearch.index.operations.merge.docs_count"` ElasticsearchIndexOperationsMergeSize MetricSettings `mapstructure:"elasticsearch.index.operations.merge.size"` ElasticsearchIndexOperationsTime MetricSettings `mapstructure:"elasticsearch.index.operations.time"` + ElasticsearchIndexSegmentsCount MetricSettings `mapstructure:"elasticsearch.index.segments.count"` + ElasticsearchIndexSegmentsMemory MetricSettings `mapstructure:"elasticsearch.index.segments.memory"` + ElasticsearchIndexSegmentsSize MetricSettings `mapstructure:"elasticsearch.index.segments.size"` ElasticsearchIndexShardsSize MetricSettings `mapstructure:"elasticsearch.index.shards.size"` ElasticsearchIndexingPressureMemoryLimit MetricSettings `mapstructure:"elasticsearch.indexing_pressure.memory.limit"` ElasticsearchIndexingPressureMemoryTotalPrimaryRejections MetricSettings `mapstructure:"elasticsearch.indexing_pressure.memory.total.primary_rejections"` @@ -167,6 +170,15 @@ func DefaultMetricsSettings() MetricsSettings { ElasticsearchIndexOperationsTime: MetricSettings{ Enabled: true, }, + ElasticsearchIndexSegmentsCount: MetricSettings{ + Enabled: false, + }, + ElasticsearchIndexSegmentsMemory: MetricSettings{ + Enabled: false, + }, + ElasticsearchIndexSegmentsSize: MetricSettings{ + Enabled: false, + }, ElasticsearchIndexShardsSize: MetricSettings{ Enabled: true, }, @@ -756,6 +768,40 @@ var MapAttributeQueryCacheCountType = map[string]AttributeQueryCacheCountType{ "miss": AttributeQueryCacheCountTypeMiss, } +// AttributeSegmentsMemoryObjectType specifies the a value segments_memory_object_type attribute. +type AttributeSegmentsMemoryObjectType int + +const ( + _ AttributeSegmentsMemoryObjectType = iota + AttributeSegmentsMemoryObjectTypeTerm + AttributeSegmentsMemoryObjectTypeDocValue + AttributeSegmentsMemoryObjectTypeIndexWriter + AttributeSegmentsMemoryObjectTypeFixedBitSet +) + +// String returns the string representation of the AttributeSegmentsMemoryObjectType. +func (av AttributeSegmentsMemoryObjectType) String() string { + switch av { + case AttributeSegmentsMemoryObjectTypeTerm: + return "term" + case AttributeSegmentsMemoryObjectTypeDocValue: + return "doc_value" + case AttributeSegmentsMemoryObjectTypeIndexWriter: + return "index_writer" + case AttributeSegmentsMemoryObjectTypeFixedBitSet: + return "fixed_bit_set" + } + return "" +} + +// MapAttributeSegmentsMemoryObjectType is a helper map of string to AttributeSegmentsMemoryObjectType attribute value. +var MapAttributeSegmentsMemoryObjectType = map[string]AttributeSegmentsMemoryObjectType{ + "term": AttributeSegmentsMemoryObjectTypeTerm, + "doc_value": AttributeSegmentsMemoryObjectTypeDocValue, + "index_writer": AttributeSegmentsMemoryObjectTypeIndexWriter, + "fixed_bit_set": AttributeSegmentsMemoryObjectTypeFixedBitSet, +} + // AttributeShardState specifies the a value shard_state attribute. type AttributeShardState int @@ -1787,6 +1833,166 @@ func newMetricElasticsearchIndexOperationsTime(settings MetricSettings) metricEl return m } +type metricElasticsearchIndexSegmentsCount struct { + data pmetric.Metric // data buffer for generated metric. + settings MetricSettings // metric settings provided by user. + capacity int // max observed number of data points added to the metric. +} + +// init fills elasticsearch.index.segments.count metric with initial data. +func (m *metricElasticsearchIndexSegmentsCount) init() { + m.data.SetName("elasticsearch.index.segments.count") + m.data.SetDescription("Number of segments of an index.") + m.data.SetUnit("{segments}") + m.data.SetEmptySum() + m.data.Sum().SetIsMonotonic(false) + m.data.Sum().SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) + m.data.Sum().DataPoints().EnsureCapacity(m.capacity) +} + +func (m *metricElasticsearchIndexSegmentsCount) recordDataPoint(start pcommon.Timestamp, ts pcommon.Timestamp, val int64, indexAggregationTypeAttributeValue string) { + if !m.settings.Enabled { + return + } + dp := m.data.Sum().DataPoints().AppendEmpty() + dp.SetStartTimestamp(start) + dp.SetTimestamp(ts) + dp.SetIntValue(val) + dp.Attributes().PutStr("aggregation", indexAggregationTypeAttributeValue) +} + +// updateCapacity saves max length of data point slices that will be used for the slice capacity. +func (m *metricElasticsearchIndexSegmentsCount) updateCapacity() { + if m.data.Sum().DataPoints().Len() > m.capacity { + m.capacity = m.data.Sum().DataPoints().Len() + } +} + +// emit appends recorded metric data to a metrics slice and prepares it for recording another set of data points. +func (m *metricElasticsearchIndexSegmentsCount) emit(metrics pmetric.MetricSlice) { + if m.settings.Enabled && m.data.Sum().DataPoints().Len() > 0 { + m.updateCapacity() + m.data.MoveTo(metrics.AppendEmpty()) + m.init() + } +} + +func newMetricElasticsearchIndexSegmentsCount(settings MetricSettings) metricElasticsearchIndexSegmentsCount { + m := metricElasticsearchIndexSegmentsCount{settings: settings} + if settings.Enabled { + m.data = pmetric.NewMetric() + m.init() + } + return m +} + +type metricElasticsearchIndexSegmentsMemory struct { + data pmetric.Metric // data buffer for generated metric. + settings MetricSettings // metric settings provided by user. + capacity int // max observed number of data points added to the metric. +} + +// init fills elasticsearch.index.segments.memory metric with initial data. +func (m *metricElasticsearchIndexSegmentsMemory) init() { + m.data.SetName("elasticsearch.index.segments.memory") + m.data.SetDescription("Size of memory for segment object of an index.") + m.data.SetUnit("By") + m.data.SetEmptySum() + m.data.Sum().SetIsMonotonic(false) + m.data.Sum().SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) + m.data.Sum().DataPoints().EnsureCapacity(m.capacity) +} + +func (m *metricElasticsearchIndexSegmentsMemory) recordDataPoint(start pcommon.Timestamp, ts pcommon.Timestamp, val int64, indexAggregationTypeAttributeValue string, segmentsMemoryObjectTypeAttributeValue string) { + if !m.settings.Enabled { + return + } + dp := m.data.Sum().DataPoints().AppendEmpty() + dp.SetStartTimestamp(start) + dp.SetTimestamp(ts) + dp.SetIntValue(val) + dp.Attributes().PutStr("aggregation", indexAggregationTypeAttributeValue) + dp.Attributes().PutStr("object", segmentsMemoryObjectTypeAttributeValue) +} + +// updateCapacity saves max length of data point slices that will be used for the slice capacity. +func (m *metricElasticsearchIndexSegmentsMemory) updateCapacity() { + if m.data.Sum().DataPoints().Len() > m.capacity { + m.capacity = m.data.Sum().DataPoints().Len() + } +} + +// emit appends recorded metric data to a metrics slice and prepares it for recording another set of data points. +func (m *metricElasticsearchIndexSegmentsMemory) emit(metrics pmetric.MetricSlice) { + if m.settings.Enabled && m.data.Sum().DataPoints().Len() > 0 { + m.updateCapacity() + m.data.MoveTo(metrics.AppendEmpty()) + m.init() + } +} + +func newMetricElasticsearchIndexSegmentsMemory(settings MetricSettings) metricElasticsearchIndexSegmentsMemory { + m := metricElasticsearchIndexSegmentsMemory{settings: settings} + if settings.Enabled { + m.data = pmetric.NewMetric() + m.init() + } + return m +} + +type metricElasticsearchIndexSegmentsSize struct { + data pmetric.Metric // data buffer for generated metric. + settings MetricSettings // metric settings provided by user. + capacity int // max observed number of data points added to the metric. +} + +// init fills elasticsearch.index.segments.size metric with initial data. +func (m *metricElasticsearchIndexSegmentsSize) init() { + m.data.SetName("elasticsearch.index.segments.size") + m.data.SetDescription("Size of segments of an index.") + m.data.SetUnit("By") + m.data.SetEmptySum() + m.data.Sum().SetIsMonotonic(false) + m.data.Sum().SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) + m.data.Sum().DataPoints().EnsureCapacity(m.capacity) +} + +func (m *metricElasticsearchIndexSegmentsSize) recordDataPoint(start pcommon.Timestamp, ts pcommon.Timestamp, val int64, indexAggregationTypeAttributeValue string) { + if !m.settings.Enabled { + return + } + dp := m.data.Sum().DataPoints().AppendEmpty() + dp.SetStartTimestamp(start) + dp.SetTimestamp(ts) + dp.SetIntValue(val) + dp.Attributes().PutStr("aggregation", indexAggregationTypeAttributeValue) +} + +// updateCapacity saves max length of data point slices that will be used for the slice capacity. +func (m *metricElasticsearchIndexSegmentsSize) updateCapacity() { + if m.data.Sum().DataPoints().Len() > m.capacity { + m.capacity = m.data.Sum().DataPoints().Len() + } +} + +// emit appends recorded metric data to a metrics slice and prepares it for recording another set of data points. +func (m *metricElasticsearchIndexSegmentsSize) emit(metrics pmetric.MetricSlice) { + if m.settings.Enabled && m.data.Sum().DataPoints().Len() > 0 { + m.updateCapacity() + m.data.MoveTo(metrics.AppendEmpty()) + m.init() + } +} + +func newMetricElasticsearchIndexSegmentsSize(settings MetricSettings) metricElasticsearchIndexSegmentsSize { + m := metricElasticsearchIndexSegmentsSize{settings: settings} + if settings.Enabled { + m.data = pmetric.NewMetric() + m.init() + } + return m +} + type metricElasticsearchIndexShardsSize struct { data pmetric.Metric // data buffer for generated metric. settings MetricSettings // metric settings provided by user. @@ -4579,6 +4785,9 @@ type MetricsBuilder struct { metricElasticsearchIndexOperationsMergeDocsCount metricElasticsearchIndexOperationsMergeDocsCount metricElasticsearchIndexOperationsMergeSize metricElasticsearchIndexOperationsMergeSize metricElasticsearchIndexOperationsTime metricElasticsearchIndexOperationsTime + metricElasticsearchIndexSegmentsCount metricElasticsearchIndexSegmentsCount + metricElasticsearchIndexSegmentsMemory metricElasticsearchIndexSegmentsMemory + metricElasticsearchIndexSegmentsSize metricElasticsearchIndexSegmentsSize metricElasticsearchIndexShardsSize metricElasticsearchIndexShardsSize metricElasticsearchIndexingPressureMemoryLimit metricElasticsearchIndexingPressureMemoryLimit metricElasticsearchIndexingPressureMemoryTotalPrimaryRejections metricElasticsearchIndexingPressureMemoryTotalPrimaryRejections @@ -4668,6 +4877,9 @@ func NewMetricsBuilder(settings MetricsSettings, buildInfo component.BuildInfo, metricElasticsearchIndexOperationsMergeDocsCount: newMetricElasticsearchIndexOperationsMergeDocsCount(settings.ElasticsearchIndexOperationsMergeDocsCount), metricElasticsearchIndexOperationsMergeSize: newMetricElasticsearchIndexOperationsMergeSize(settings.ElasticsearchIndexOperationsMergeSize), metricElasticsearchIndexOperationsTime: newMetricElasticsearchIndexOperationsTime(settings.ElasticsearchIndexOperationsTime), + metricElasticsearchIndexSegmentsCount: newMetricElasticsearchIndexSegmentsCount(settings.ElasticsearchIndexSegmentsCount), + metricElasticsearchIndexSegmentsMemory: newMetricElasticsearchIndexSegmentsMemory(settings.ElasticsearchIndexSegmentsMemory), + metricElasticsearchIndexSegmentsSize: newMetricElasticsearchIndexSegmentsSize(settings.ElasticsearchIndexSegmentsSize), metricElasticsearchIndexShardsSize: newMetricElasticsearchIndexShardsSize(settings.ElasticsearchIndexShardsSize), metricElasticsearchIndexingPressureMemoryLimit: newMetricElasticsearchIndexingPressureMemoryLimit(settings.ElasticsearchIndexingPressureMemoryLimit), metricElasticsearchIndexingPressureMemoryTotalPrimaryRejections: newMetricElasticsearchIndexingPressureMemoryTotalPrimaryRejections(settings.ElasticsearchIndexingPressureMemoryTotalPrimaryRejections), @@ -4813,6 +5025,9 @@ func (mb *MetricsBuilder) EmitForResource(rmo ...ResourceMetricsOption) { mb.metricElasticsearchIndexOperationsMergeDocsCount.emit(ils.Metrics()) mb.metricElasticsearchIndexOperationsMergeSize.emit(ils.Metrics()) mb.metricElasticsearchIndexOperationsTime.emit(ils.Metrics()) + mb.metricElasticsearchIndexSegmentsCount.emit(ils.Metrics()) + mb.metricElasticsearchIndexSegmentsMemory.emit(ils.Metrics()) + mb.metricElasticsearchIndexSegmentsSize.emit(ils.Metrics()) mb.metricElasticsearchIndexShardsSize.emit(ils.Metrics()) mb.metricElasticsearchIndexingPressureMemoryLimit.emit(ils.Metrics()) mb.metricElasticsearchIndexingPressureMemoryTotalPrimaryRejections.emit(ils.Metrics()) @@ -4976,6 +5191,21 @@ func (mb *MetricsBuilder) RecordElasticsearchIndexOperationsTimeDataPoint(ts pco mb.metricElasticsearchIndexOperationsTime.recordDataPoint(mb.startTime, ts, val, operationAttributeValue.String(), indexAggregationTypeAttributeValue.String()) } +// RecordElasticsearchIndexSegmentsCountDataPoint adds a data point to elasticsearch.index.segments.count metric. +func (mb *MetricsBuilder) RecordElasticsearchIndexSegmentsCountDataPoint(ts pcommon.Timestamp, val int64, indexAggregationTypeAttributeValue AttributeIndexAggregationType) { + mb.metricElasticsearchIndexSegmentsCount.recordDataPoint(mb.startTime, ts, val, indexAggregationTypeAttributeValue.String()) +} + +// RecordElasticsearchIndexSegmentsMemoryDataPoint adds a data point to elasticsearch.index.segments.memory metric. +func (mb *MetricsBuilder) RecordElasticsearchIndexSegmentsMemoryDataPoint(ts pcommon.Timestamp, val int64, indexAggregationTypeAttributeValue AttributeIndexAggregationType, segmentsMemoryObjectTypeAttributeValue AttributeSegmentsMemoryObjectType) { + mb.metricElasticsearchIndexSegmentsMemory.recordDataPoint(mb.startTime, ts, val, indexAggregationTypeAttributeValue.String(), segmentsMemoryObjectTypeAttributeValue.String()) +} + +// RecordElasticsearchIndexSegmentsSizeDataPoint adds a data point to elasticsearch.index.segments.size metric. +func (mb *MetricsBuilder) RecordElasticsearchIndexSegmentsSizeDataPoint(ts pcommon.Timestamp, val int64, indexAggregationTypeAttributeValue AttributeIndexAggregationType) { + mb.metricElasticsearchIndexSegmentsSize.recordDataPoint(mb.startTime, ts, val, indexAggregationTypeAttributeValue.String()) +} + // RecordElasticsearchIndexShardsSizeDataPoint adds a data point to elasticsearch.index.shards.size metric. func (mb *MetricsBuilder) RecordElasticsearchIndexShardsSizeDataPoint(ts pcommon.Timestamp, val int64, indexAggregationTypeAttributeValue AttributeIndexAggregationType) { mb.metricElasticsearchIndexShardsSize.recordDataPoint(mb.startTime, ts, val, indexAggregationTypeAttributeValue.String()) diff --git a/receiver/elasticsearchreceiver/internal/model/nodestats.go b/receiver/elasticsearchreceiver/internal/model/nodestats.go index 59502425a220..31ad85d2f92a 100644 --- a/receiver/elasticsearchreceiver/internal/model/nodestats.go +++ b/receiver/elasticsearchreceiver/internal/model/nodestats.go @@ -176,6 +176,16 @@ type NodeStatsNodesInfoIndices struct { QueryCache BasicCacheInfo `json:"query_cache"` FieldDataCache BasicCacheInfo `json:"fielddata"` TranslogStats TranslogStats `json:"translog"` + SegmentsStats SegmentsStats `json:"segments"` +} + +type SegmentsStats struct { + Count int64 `json:"count"` + DocumentValuesMemoryInBy int64 `json:"doc_values_memory_in_bytes"` + IndexWriterMemoryInBy int64 `json:"index_writer_memory_in_bytes"` + MemoryInBy int64 `json:"memory_in_bytes"` + TermsMemoryInBy int64 `json:"terms_memory_in_bytes"` + FixedBitSetMemoryInBy int64 `json:"fixed_bit_set_memory_in_bytes"` } type TranslogStats struct { diff --git a/receiver/elasticsearchreceiver/metadata.yaml b/receiver/elasticsearchreceiver/metadata.yaml index d842183caee6..d0014205d1bd 100644 --- a/receiver/elasticsearchreceiver/metadata.yaml +++ b/receiver/elasticsearchreceiver/metadata.yaml @@ -146,6 +146,14 @@ attributes: enum: - primary_shards - total + segments_memory_object_type: + value: object + description: Type of object in segment + enum: + - term + - doc_value + - index_writer + - fixed_bit_set metrics: # these metrics are from /_nodes/stats, and are node level metrics @@ -785,3 +793,30 @@ metrics: value_type: int attributes: [index_aggregation_type] enabled: false + elasticsearch.index.segments.count: + description: Number of segments of an index. + unit: "{segments}" + sum: + monotonic: false + aggregation: cumulative + value_type: int + attributes: [index_aggregation_type] + enabled: false + elasticsearch.index.segments.size: + description: Size of segments of an index. + unit: By + sum: + monotonic: false + aggregation: cumulative + value_type: int + attributes: [index_aggregation_type] + enabled: false + elasticsearch.index.segments.memory: + description: Size of memory for segment object of an index. + unit: By + sum: + monotonic: false + aggregation: cumulative + value_type: int + attributes: [index_aggregation_type, segments_memory_object_type] + enabled: false diff --git a/receiver/elasticsearchreceiver/scraper.go b/receiver/elasticsearchreceiver/scraper.go index 71c46ca2f824..052849995187 100644 --- a/receiver/elasticsearchreceiver/scraper.go +++ b/receiver/elasticsearchreceiver/scraper.go @@ -333,7 +333,7 @@ func (r *elasticsearchScraper) scrapeIndicesMetrics(ctx context.Context, now pco indexStats, err := r.client.IndexStats(ctx, r.cfg.Indices) if err != nil { - errs.AddPartial(8, err) + errs.AddPartial(14, err) return } @@ -374,5 +374,38 @@ func (r *elasticsearchScraper) scrapeOneIndexMetrics(now pcommon.Timestamp, name now, stats.Total.StoreInfo.SizeInBy, metadata.AttributeIndexAggregationTypeTotal, ) + r.mb.RecordElasticsearchIndexSegmentsCountDataPoint( + now, stats.Total.SegmentsStats.Count, metadata.AttributeIndexAggregationTypeTotal, + ) + + r.mb.RecordElasticsearchIndexSegmentsSizeDataPoint( + now, stats.Total.SegmentsStats.MemoryInBy, metadata.AttributeIndexAggregationTypeTotal, + ) + + r.mb.RecordElasticsearchIndexSegmentsMemoryDataPoint( + now, + stats.Total.SegmentsStats.DocumentValuesMemoryInBy, + metadata.AttributeIndexAggregationTypeTotal, + metadata.AttributeSegmentsMemoryObjectTypeDocValue, + ) + r.mb.RecordElasticsearchIndexSegmentsMemoryDataPoint( + now, + stats.Total.SegmentsStats.FixedBitSetMemoryInBy, + metadata.AttributeIndexAggregationTypeTotal, + metadata.AttributeSegmentsMemoryObjectTypeFixedBitSet, + ) + r.mb.RecordElasticsearchIndexSegmentsMemoryDataPoint( + now, + stats.Total.SegmentsStats.IndexWriterMemoryInBy, + metadata.AttributeIndexAggregationTypeTotal, + metadata.AttributeSegmentsMemoryObjectTypeIndexWriter, + ) + r.mb.RecordElasticsearchIndexSegmentsMemoryDataPoint( + now, + stats.Total.SegmentsStats.TermsMemoryInBy, + metadata.AttributeIndexAggregationTypeTotal, + metadata.AttributeSegmentsMemoryObjectTypeTerm, + ) + r.mb.EmitForResource(metadata.WithElasticsearchIndexName(name), metadata.WithElasticsearchClusterName(r.clusterName)) } diff --git a/receiver/elasticsearchreceiver/scraper_test.go b/receiver/elasticsearchreceiver/scraper_test.go index e749847a7ade..9326eaf1be19 100644 --- a/receiver/elasticsearchreceiver/scraper_test.go +++ b/receiver/elasticsearchreceiver/scraper_test.go @@ -44,6 +44,9 @@ func TestScraper(t *testing.T) { config := createDefaultConfig().(*Config) config.Metrics.ElasticsearchIndexOperationsMergeSize.Enabled = true config.Metrics.ElasticsearchIndexOperationsMergeDocsCount.Enabled = true + config.Metrics.ElasticsearchIndexSegmentsCount.Enabled = true + config.Metrics.ElasticsearchIndexSegmentsSize.Enabled = true + config.Metrics.ElasticsearchIndexSegmentsMemory.Enabled = true sc := newElasticSearchScraper(componenttest.NewNopReceiverCreateSettings(), config) diff --git a/receiver/elasticsearchreceiver/testdata/expected_metrics/full.json b/receiver/elasticsearchreceiver/testdata/expected_metrics/full.json index 777be54ae69c..6e56a1fbe89b 100644 --- a/receiver/elasticsearchreceiver/testdata/expected_metrics/full.json +++ b/receiver/elasticsearchreceiver/testdata/expected_metrics/full.json @@ -2585,6 +2585,141 @@ ] }, "unit": "{documents}" + }, + { + "description": "Number of segments of an index.", + "name": "elasticsearch.index.segments.count", + "sum": { + "aggregationTemporality": "AGGREGATION_TEMPORALITY_CUMULATIVE", + "isMonotonic": false, + "dataPoints": [ + { + "asInt": "5", + "attributes": [ + { + "key": "aggregation", + "value": { + "stringValue": "total" + } + } + ], + "startTimeUnixNano": "1661811689941624000", + "timeUnixNano": "1661811689943245000" + } + ] + }, + "unit": "{segments}" + }, + { + "description": "Size of segments of an index.", + "name": "elasticsearch.index.segments.size", + "sum": { + "aggregationTemporality": "AGGREGATION_TEMPORALITY_CUMULATIVE", + "isMonotonic": false, + "dataPoints": [ + { + "asInt": "5460", + "attributes": [ + { + "key": "aggregation", + "value": { + "stringValue": "total" + } + } + ], + "startTimeUnixNano": "1661811689941624000", + "timeUnixNano": "1661811689943245000" + } + ] + }, + "unit": "By" + }, + { + "description": "Size of memory for segment object of an index.", + "name": "elasticsearch.index.segments.memory", + "sum": { + "aggregationTemporality": "AGGREGATION_TEMPORALITY_CUMULATIVE", + "isMonotonic": false, + "dataPoints": [ + { + "asInt": "2560", + "attributes": [ + { + "key": "aggregation", + "value": { + "stringValue": "total" + } + }, + { + "key": "object", + "value": { + "stringValue": "term" + } + } + ], + "startTimeUnixNano": "1661811689941624000", + "timeUnixNano": "1661811689943245000" + }, + { + "asInt": "380", + "attributes": [ + { + "key": "aggregation", + "value": { + "stringValue": "total" + } + }, + { + "key": "object", + "value": { + "stringValue": "doc_value" + } + } + ], + "startTimeUnixNano": "1661811689941624000", + "timeUnixNano": "1661811689943245000" + }, + { + "asInt": "37", + "attributes": [ + { + "key": "aggregation", + "value": { + "stringValue": "total" + } + }, + { + "key": "object", + "value": { + "stringValue": "index_writer" + } + } + ], + "startTimeUnixNano": "1661811689941624000", + "timeUnixNano": "1661811689943245000" + }, + { + "asInt": "21", + "attributes": [ + { + "key": "aggregation", + "value": { + "stringValue": "total" + } + }, + { + "key": "object", + "value": { + "stringValue": "fixed_bit_set" + } + } + ], + "startTimeUnixNano": "1661811689941624000", + "timeUnixNano": "1661811689943245000" + } + ] + }, + "unit": "By" } ], "scope": { @@ -2802,6 +2937,141 @@ ] }, "unit": "{documents}" + }, + { + "description": "Number of segments of an index.", + "name": "elasticsearch.index.segments.count", + "sum": { + "aggregationTemporality": "AGGREGATION_TEMPORALITY_CUMULATIVE", + "isMonotonic": false, + "dataPoints": [ + { + "asInt": "5", + "attributes": [ + { + "key": "aggregation", + "value": { + "stringValue": "total" + } + } + ], + "startTimeUnixNano": "1661811689941624000", + "timeUnixNano": "1661811689943245000" + } + ] + }, + "unit": "{segments}" + }, + { + "description": "Size of segments of an index.", + "name": "elasticsearch.index.segments.size", + "sum": { + "aggregationTemporality": "AGGREGATION_TEMPORALITY_CUMULATIVE", + "isMonotonic": false, + "dataPoints": [ + { + "asInt": "5460", + "attributes": [ + { + "key": "aggregation", + "value": { + "stringValue": "total" + } + } + ], + "startTimeUnixNano": "1661811689941624000", + "timeUnixNano": "1661811689943245000" + } + ] + }, + "unit": "By" + }, + { + "description": "Size of memory for segment object of an index.", + "name": "elasticsearch.index.segments.memory", + "sum": { + "aggregationTemporality": "AGGREGATION_TEMPORALITY_CUMULATIVE", + "isMonotonic": false, + "dataPoints": [ + { + "asInt": "2560", + "attributes": [ + { + "key": "aggregation", + "value": { + "stringValue": "total" + } + }, + { + "key": "object", + "value": { + "stringValue": "term" + } + } + ], + "startTimeUnixNano": "1661811689941624000", + "timeUnixNano": "1661811689943245000" + }, + { + "asInt": "380", + "attributes": [ + { + "key": "aggregation", + "value": { + "stringValue": "total" + } + }, + { + "key": "object", + "value": { + "stringValue": "doc_value" + } + } + ], + "startTimeUnixNano": "1661811689941624000", + "timeUnixNano": "1661811689943245000" + }, + { + "asInt": "37", + "attributes": [ + { + "key": "aggregation", + "value": { + "stringValue": "total" + } + }, + { + "key": "object", + "value": { + "stringValue": "index_writer" + } + } + ], + "startTimeUnixNano": "1661811689941624000", + "timeUnixNano": "1661811689943245000" + }, + { + "asInt": "21", + "attributes": [ + { + "key": "aggregation", + "value": { + "stringValue": "total" + } + }, + { + "key": "object", + "value": { + "stringValue": "fixed_bit_set" + } + } + ], + "startTimeUnixNano": "1661811689941624000", + "timeUnixNano": "1661811689943245000" + } + ] + }, + "unit": "By" } ], "scope": { diff --git a/receiver/elasticsearchreceiver/testdata/sample_payloads/indices.json b/receiver/elasticsearchreceiver/testdata/sample_payloads/indices.json index 196c40585c0d..b49890d36a93 100644 --- a/receiver/elasticsearchreceiver/testdata/sample_payloads/indices.json +++ b/receiver/elasticsearchreceiver/testdata/sample_payloads/indices.json @@ -104,13 +104,13 @@ "memory_in_bytes" : 5460, "terms_memory_in_bytes" : 2560, "stored_fields_memory_in_bytes" : 2520, - "term_vectors_memory_in_bytes" : 0, + "term_vectors_memory_in_bytes" : 30, "norms_memory_in_bytes" : 0, "points_memory_in_bytes" : 0, "doc_values_memory_in_bytes" : 380, - "index_writer_memory_in_bytes" : 0, + "index_writer_memory_in_bytes" : 37, "version_map_memory_in_bytes" : 0, - "fixed_bit_set_memory_in_bytes" : 0, + "fixed_bit_set_memory_in_bytes" : 21, "max_unsafe_auto_id_timestamp" : -1, "file_sizes" : { } }, @@ -232,13 +232,13 @@ "memory_in_bytes" : 5460, "terms_memory_in_bytes" : 2560, "stored_fields_memory_in_bytes" : 2520, - "term_vectors_memory_in_bytes" : 0, + "term_vectors_memory_in_bytes" : 30, "norms_memory_in_bytes" : 0, "points_memory_in_bytes" : 0, "doc_values_memory_in_bytes" : 380, - "index_writer_memory_in_bytes" : 0, + "index_writer_memory_in_bytes" : 37, "version_map_memory_in_bytes" : 0, - "fixed_bit_set_memory_in_bytes" : 0, + "fixed_bit_set_memory_in_bytes" : 21, "max_unsafe_auto_id_timestamp" : -1, "file_sizes" : { } }, @@ -364,13 +364,13 @@ "memory_in_bytes" : 5460, "terms_memory_in_bytes" : 2560, "stored_fields_memory_in_bytes" : 2520, - "term_vectors_memory_in_bytes" : 0, + "term_vectors_memory_in_bytes" : 30, "norms_memory_in_bytes" : 0, "points_memory_in_bytes" : 0, "doc_values_memory_in_bytes" : 380, - "index_writer_memory_in_bytes" : 0, + "index_writer_memory_in_bytes" : 37, "version_map_memory_in_bytes" : 0, - "fixed_bit_set_memory_in_bytes" : 0, + "fixed_bit_set_memory_in_bytes" : 21, "max_unsafe_auto_id_timestamp" : -1, "file_sizes" : { } }, @@ -492,13 +492,13 @@ "memory_in_bytes" : 5460, "terms_memory_in_bytes" : 2560, "stored_fields_memory_in_bytes" : 2520, - "term_vectors_memory_in_bytes" : 0, + "term_vectors_memory_in_bytes" : 30, "norms_memory_in_bytes" : 0, "points_memory_in_bytes" : 0, "doc_values_memory_in_bytes" : 380, - "index_writer_memory_in_bytes" : 0, + "index_writer_memory_in_bytes" : 37, "version_map_memory_in_bytes" : 0, - "fixed_bit_set_memory_in_bytes" : 0, + "fixed_bit_set_memory_in_bytes" : 21, "max_unsafe_auto_id_timestamp" : -1, "file_sizes" : { } },