Revert "perf: add some unnecessary indexes"

This reverts commit 6fcd8b4.
go-faster · Dec 4, 2024 · 8505a42 · 8505a42
1 parent 6fcd8b4
commit 8505a42
Show file tree

Hide file tree

Showing 8 changed files with 18 additions and 62 deletions.
diff --git a/internal/chstorage/_golden/log_columns_ddl.sql b/internal/chstorage/_golden/log_columns_ddl.sql
@@ -16,15 +16,10 @@ CREATE TABLE IF NOT EXISTS `table`
 	`scope_version`       LowCardinality(String),
 	`scope`               LowCardinality(String),
 
-	INDEX `idx_trace_id`           trace_id TYPE bloom_filter(0.001) GRANULARITY 1,
-	INDEX `idx_body`               body TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 1,
-	INDEX `idx_ts`                 timestamp TYPE minmax GRANULARITY 8192,
-	INDEX `idx_arr_join_attribute` arrayJoin(JSONExtractKeys(attribute)) TYPE set(100),
-	INDEX `idx_keys_attribute`     JSONExtractKeys(attribute) TYPE set(100),
-	INDEX `idx_arr_join_resource`  arrayJoin(JSONExtractKeys(resource)) TYPE set(100),
-	INDEX `idx_keys_resource`      JSONExtractKeys(resource) TYPE set(100),
-	INDEX `idx_arr_join_scope`     arrayJoin(JSONExtractKeys(scope)) TYPE set(100),
-	INDEX `idx_keys_scope`         JSONExtractKeys(scope) TYPE set(100)
+	INDEX `idx_trace_id`   trace_id TYPE bloom_filter(0.001) GRANULARITY 1,
+	INDEX `idx_body`       body TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 1,
+	INDEX `idx_ts`         timestamp TYPE minmax GRANULARITY 8192,
+	INDEX `attribute_keys` arrayConcat(JSONExtractKeys(attribute), JSONExtractKeys(scope), JSONExtractKeys(resource)) TYPE set(100)
 )
 ENGINE = MergeTree
 PARTITION BY toYYYYMMDD(timestamp)

diff --git a/internal/chstorage/_golden/schema.logs.sql b/internal/chstorage/_golden/schema.logs.sql
@@ -16,15 +16,10 @@ CREATE TABLE IF NOT EXISTS `logs`
 	`scope_version`       LowCardinality(String),
 	`scope`               LowCardinality(String),
 
-	INDEX `idx_trace_id`           trace_id TYPE bloom_filter(0.001) GRANULARITY 1,
-	INDEX `idx_body`               body TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 1,
-	INDEX `idx_ts`                 timestamp TYPE minmax GRANULARITY 8192,
-	INDEX `idx_arr_join_attribute` arrayJoin(JSONExtractKeys(attribute)) TYPE set(100),
-	INDEX `idx_keys_attribute`     JSONExtractKeys(attribute) TYPE set(100),
-	INDEX `idx_arr_join_resource`  arrayJoin(JSONExtractKeys(resource)) TYPE set(100),
-	INDEX `idx_keys_resource`      JSONExtractKeys(resource) TYPE set(100),
-	INDEX `idx_arr_join_scope`     arrayJoin(JSONExtractKeys(scope)) TYPE set(100),
-	INDEX `idx_keys_scope`         JSONExtractKeys(scope) TYPE set(100)
+	INDEX `idx_trace_id`   trace_id TYPE bloom_filter(0.001) GRANULARITY 1,
+	INDEX `idx_body`       body TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 1,
+	INDEX `idx_ts`         timestamp TYPE minmax GRANULARITY 8192,
+	INDEX `attribute_keys` arrayConcat(JSONExtractKeys(attribute), JSONExtractKeys(scope), JSONExtractKeys(resource)) TYPE set(100)
 )
 ENGINE = MergeTree
 PARTITION BY toYYYYMMDD(timestamp)

diff --git a/internal/chstorage/_golden/schema.metrics_exemplars.sql b/internal/chstorage/_golden/schema.metrics_exemplars.sql
@@ -10,14 +10,7 @@ CREATE TABLE IF NOT EXISTS `metrics_exemplars`
 	`span_id`             FixedString(8),
 	`attribute`           LowCardinality(String),
 	`resource`            LowCardinality(String),
-	`scope`               LowCardinality(String),
-
-	INDEX `idx_arr_join_attribute` arrayJoin(JSONExtractKeys(attribute)) TYPE set(100),
-	INDEX `idx_keys_attribute`     JSONExtractKeys(attribute) TYPE set(100),
-	INDEX `idx_arr_join_resource`  arrayJoin(JSONExtractKeys(resource)) TYPE set(100),
-	INDEX `idx_keys_resource`      JSONExtractKeys(resource) TYPE set(100),
-	INDEX `idx_arr_join_scope`     arrayJoin(JSONExtractKeys(scope)) TYPE set(100),
-	INDEX `idx_keys_scope`         JSONExtractKeys(scope) TYPE set(100)
+	`scope`               LowCardinality(String)
 )
 ENGINE = MergeTree
 PARTITION BY toYYYYMMDD(timestamp)

diff --git a/internal/chstorage/_golden/schema.metrics_exp_histograms.sql b/internal/chstorage/_golden/schema.metrics_exp_histograms.sql
@@ -16,14 +16,7 @@ CREATE TABLE IF NOT EXISTS `metrics_exp_histograms`
 	`flags`                                UInt8                  CODEC(T64, ZSTD(1)),
 	`attribute`                            LowCardinality(String),
 	`resource`                             LowCardinality(String),
-	`scope`                                LowCardinality(String),
-
-	INDEX `idx_arr_join_attribute` arrayJoin(JSONExtractKeys(attribute)) TYPE set(100),
-	INDEX `idx_keys_attribute`     JSONExtractKeys(attribute) TYPE set(100),
-	INDEX `idx_arr_join_resource`  arrayJoin(JSONExtractKeys(resource)) TYPE set(100),
-	INDEX `idx_keys_resource`      JSONExtractKeys(resource) TYPE set(100),
-	INDEX `idx_arr_join_scope`     arrayJoin(JSONExtractKeys(scope)) TYPE set(100),
-	INDEX `idx_keys_scope`         JSONExtractKeys(scope) TYPE set(100)
+	`scope`                                LowCardinality(String)
 )
 ENGINE = MergeTree
 PARTITION BY toYYYYMMDD(timestamp)

diff --git a/internal/chstorage/_golden/schema.metrics_points.sql b/internal/chstorage/_golden/schema.metrics_points.sql
@@ -20,13 +20,7 @@ CREATE TABLE IF NOT EXISTS `metrics_points`
 	`resource`        LowCardinality(String),
 	`scope`           LowCardinality(String),
 
-	INDEX `idx_ts`                 timestamp TYPE minmax GRANULARITY 8192,
-	INDEX `idx_arr_join_attribute` arrayJoin(JSONExtractKeys(attribute)) TYPE set(100),
-	INDEX `idx_keys_attribute`     JSONExtractKeys(attribute) TYPE set(100),
-	INDEX `idx_arr_join_resource`  arrayJoin(JSONExtractKeys(resource)) TYPE set(100),
-	INDEX `idx_keys_resource`      JSONExtractKeys(resource) TYPE set(100),
-	INDEX `idx_arr_join_scope`     arrayJoin(JSONExtractKeys(scope)) TYPE set(100),
-	INDEX `idx_keys_scope`         JSONExtractKeys(scope) TYPE set(100)
+	INDEX `idx_ts` timestamp TYPE minmax GRANULARITY 8192
 )
 ENGINE = MergeTree
 PARTITION BY toYYYYMMDD(timestamp)

diff --git a/internal/chstorage/_golden/schema.traces_spans.sql b/internal/chstorage/_golden/schema.traces_spans.sql
@@ -28,13 +28,7 @@ CREATE TABLE IF NOT EXISTS `traces_spans`
 	`links_tracestates`   Array(String),
 	`links_attributes`    Array(String),
 
-	INDEX `idx_trace_id`           trace_id TYPE bloom_filter(0.001) GRANULARITY 1,
-	INDEX `idx_arr_join_attribute` arrayJoin(JSONExtractKeys(attribute)) TYPE set(100),
-	INDEX `idx_keys_attribute`     JSONExtractKeys(attribute) TYPE set(100),
-	INDEX `idx_arr_join_resource`  arrayJoin(JSONExtractKeys(resource)) TYPE set(100),
-	INDEX `idx_keys_resource`      JSONExtractKeys(resource) TYPE set(100),
-	INDEX `idx_arr_join_scope`     arrayJoin(JSONExtractKeys(scope)) TYPE set(100),
-	INDEX `idx_keys_scope`         JSONExtractKeys(scope) TYPE set(100)
+	INDEX `idx_trace_id` trace_id TYPE bloom_filter(0.001) GRANULARITY 1
 )
 ENGINE = MergeTree
 ORDER BY (`service_namespace`, `service_name`, `resource`, `start`)

diff --git a/internal/chstorage/attributes.go b/internal/chstorage/attributes.go
@@ -282,20 +282,6 @@ func (a *Attributes) DDL(table *ddl.Table) {
 			Type: a.Value.Type(),
 		},
 	)
-	table.Indexes = append(table.Indexes,
-		ddl.Index{
-			Name:   "idx_arr_join_" + a.Name,
-			Target: "arrayJoin(JSONExtractKeys(" + a.Name + "))",
-			Type:   "set",
-			Params: []string{"100"},
-		},
-		ddl.Index{
-			Name:   "idx_keys_" + a.Name,
-			Target: "JSONExtractKeys(" + a.Name + ")",
-			Type:   "set",
-			Params: []string{"100"},
-		},
-	)
 }
 
 func attrsToLabels(m otelstorage.Attrs, to map[string]string) {

diff --git a/internal/chstorage/columns_logs.go b/internal/chstorage/columns_logs.go
@@ -48,6 +48,12 @@ func (c *logColumns) DDL() ddl.Table {
 				Type:        "minmax",
 				Granularity: 8192,
 			},
+			{
+				Name:   "attribute_keys",
+				Target: "arrayConcat(JSONExtractKeys(attribute), JSONExtractKeys(scope), JSONExtractKeys(resource))",
+				Type:   "set",
+				Params: []string{"100"},
+			},
 		},
 		Columns: []ddl.Column{
 			{