Skip to content

Commit

Permalink
feat(chstorage): write explicit 128-bit attr hash
Browse files Browse the repository at this point in the history
  • Loading branch information
@ernado
ernado committed Jan 10, 2024
1 parent 661af3e commit ac22714
Show file tree
Hide file tree
Showing 4 changed files with 22 additions and 7 deletions.
3 changes: 3 additions & 0 deletions internal/chstorage/attributes.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ type Attributes struct {
Name string
Data *proto.ColMap[string, string]
Types *proto.ColMap[string, uint8]
Hash proto.ColRawOf[otelstorage.Hash]
}

type (
Expand All @@ -84,6 +85,7 @@ func (a *Attributes) Columns() Columns {
return Columns{
{Name: a.Name, Data: a.Data},
{Name: a.Name + "_types", Data: a.Types},
{Name: a.Name + "_hash", Data: &a.Hash},
}
}

Expand All @@ -92,6 +94,7 @@ func (a *Attributes) Append(kv otelstorage.Attrs) {
va, vt := encodeAttributesRow(kv.AsMap())
a.Data.AppendKV(va)
a.Types.AppendKV(vt)
a.Hash.Append(kv.Hash())
}

// Row returns a new map of attributes for a given row.
Expand Down
7 changes: 5 additions & 2 deletions internal/chstorage/schema_logs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,22 +26,25 @@ const (
attributes Map(LowCardinality(String), String) CODEC(ZSTD(1)), -- string[str | json]
attributes_types Map(LowCardinality(String), UInt8) CODEC(ZSTD(5)), -- string[type]
attributes_hash FixedString(16),
resource Map(LowCardinality(String), String) CODEC(ZSTD(1)), -- string[str | json]
resource_types Map(LowCardinality(String), UInt8) CODEC(ZSTD(5)), -- string[type]
resource_hash FixedString(16),
scope_name LowCardinality(String),
scope_version LowCardinality(String),
scope_attributes Map(LowCardinality(String), String) CODEC(ZSTD(1)), -- string[str | json]
scope_attributes_types Map(LowCardinality(String), UInt8) CODEC(ZSTD(5)), -- string[type]
scope_attributes_hash FixedString(16),
INDEX idx_trace_id trace_id TYPE bloom_filter(0.001) GRANULARITY 1,
INDEX idx_body body TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 1,
INDEX idx_ts timestamp TYPE minmax GRANULARITY 8192,
)
ENGINE = MergeTree
PARTITION BY toYYYYMMDD(timestamp)
PRIMARY KEY (severity_number, service_namespace, service_name, cityHash64(resource))
ORDER BY (severity_number, service_namespace, service_name, cityHash64(resource), timestamp)
PRIMARY KEY (severity_number, service_namespace, service_name, resource_hash)
ORDER BY (severity_number, service_namespace, service_name, resource_hash, timestamp)
`

logAttrsSchema = `
Expand Down
12 changes: 9 additions & 3 deletions internal/chstorage/schema_metrics.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,11 @@ const (
flags UInt8 CODEC(T64),
attributes Map(LowCardinality(String), String) CODEC(ZSTD(1)), -- string[str | json]
attributes_hash FixedString(16),
attributes_types Map(LowCardinality(String), UInt8) CODEC(ZSTD(5)), -- string[type]
resource Map(LowCardinality(String), String) CODEC(ZSTD(1)), -- string[str | json]
resource_types Map(LowCardinality(String), UInt8) CODEC(ZSTD(5)), -- string[type]
resource_hash FixedString(16),
INDEX idx_ts timestamp TYPE minmax GRANULARITY 8192,
INDEX idx_res_attr_key mapKeys(attributes) TYPE bloom_filter(0.01) GRANULARITY 1,
Expand All @@ -46,8 +48,8 @@ const (
)
ENGINE = MergeTree()
PARTITION BY toYYYYMMDD(timestamp)
PRIMARY KEY (name_normalized, mapping, cityHash64(resource), cityHash64(attributes))
ORDER BY (name_normalized, mapping, cityHash64(resource), cityHash64(attributes), timestamp)`
PRIMARY KEY (name_normalized, mapping, resource_hash, attributes_hash)
ORDER BY (name_normalized, mapping, resource_hash, attributes_hash, timestamp)`
metricMappingDDL = `
'NO_MAPPING' = 0,
'HISTOGRAM_COUNT' = 1,
Expand Down Expand Up @@ -78,9 +80,11 @@ const (
flags UInt32,
attributes Map(LowCardinality(String), String) CODEC(ZSTD(1)), -- string[str | json]
attributes_hash FixedString(16),
attributes_types Map(LowCardinality(String), UInt8) CODEC(ZSTD(5)), -- string[type]
resource Map(LowCardinality(String), String) CODEC(ZSTD(1)), -- string[str | json]
resource_types Map(LowCardinality(String), UInt8) CODEC(ZSTD(5)), -- string[type]
resource_hash FixedString(16),
)
ENGINE = MergeTree()
ORDER BY timestamp`
Expand All @@ -97,12 +101,14 @@ const (
trace_id FixedString(16),
attributes Map(LowCardinality(String), String) CODEC(ZSTD(1)), -- string[str | json]
attributes_hash FixedString(16),
attributes_types Map(LowCardinality(String), UInt8) CODEC(ZSTD(5)), -- string[type]
resource Map(LowCardinality(String), String) CODEC(ZSTD(1)), -- string[str | json]
resource_types Map(LowCardinality(String), UInt8) CODEC(ZSTD(5)), -- string[type]
resource_hash FixedString(16),
)
ENGINE = MergeTree()
ORDER BY (name_normalized, cityHash64(resource), cityHash64(attributes), timestamp)`
ORDER BY (name_normalized, resource_hash, attributes_hash, timestamp)`

labelsSchema = `
(
Expand Down
7 changes: 5 additions & 2 deletions internal/chstorage/schema_traces.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,13 +28,16 @@ const (
batch_id UUID,
attributes Map(LowCardinality(String), String) CODEC(ZSTD(1)), -- string[str | json]
attributes_types Map(LowCardinality(String), UInt8) CODEC(ZSTD(5)), -- string[type]
attributes_hash FixedString(16),
resource Map(LowCardinality(String), String) CODEC(ZSTD(1)), -- string[str | json]
resource_types Map(LowCardinality(String), UInt8) CODEC(ZSTD(5)), -- string[type]
resource_hash FixedString(16),
scope_name LowCardinality(String),
scope_version LowCardinality(String),
scope_attributes Map(LowCardinality(String), String) CODEC(ZSTD(1)), -- string[str | json]
scope_attributes_types Map(LowCardinality(String), UInt8) CODEC(ZSTD(5)), -- string[type]
scope_attributes_hash FixedString(16),
events_timestamps Array(DateTime64(9)),
events_names Array(String),
Expand All @@ -49,8 +52,8 @@ const (
)
ENGINE = MergeTree()
PARTITION BY toYYYYMMDD(start)
PRIMARY KEY (service_namespace, service_name, cityHash64(resource))
ORDER BY (service_namespace, service_name, cityHash64(resource), start)
PRIMARY KEY (service_namespace, service_name, resource_hash)
ORDER BY (service_namespace, service_name, resource_hash, start)
`
kindDDL = `'KIND_UNSPECIFIED' = 0,'KIND_INTERNAL' = 1,'KIND_SERVER' = 2,'KIND_CLIENT' = 3,'KIND_PRODUCER' = 4,'KIND_CONSUMER' = 5`
tagsSchema = `
Expand Down

0 comments on commit ac22714

Please sign in to comment.