Skip to content

Commit

Permalink
perf(chstorage): use hasToken to skip blocks
Browse files Browse the repository at this point in the history
```
                            │     base     │                 new                  │
                            │    sec/op    │    sec/op     vs base                │
LogQL/Failed_POST_requests   74.24m ± 11%   45.39m ± 10%  -38.86% (p=0.000 n=15)

                            │     base     │                 new                  │
                            │  ch-sec/op   │  ch-sec/op    vs base                │
LogQL/Failed_POST_requests   64.22m ± 12%   35.14m ± 12%  -45.28% (p=0.000 n=15)

                            │      base       │                   new                   │
                            │ ch-mem-bytes/op │ ch-mem-bytes/op  vs base                │
LogQL/Failed_POST_requests      61.83Mi ± 2%     41.81Mi ± 10%  -32.38% (p=0.000 n=15)

                            │       base       │                   new                    │
                            │ ch-read-bytes/op │ ch-read-bytes/op  vs base                │
LogQL/Failed_POST_requests      304.29Mi ± 0%       73.85Mi ± 0%  -75.73% (p=0.000 n=15)

                            │      base       │                   new                   │
                            │ ch-read-rows/op │ ch-read-rows/op  vs base                │
LogQL/Failed_POST_requests      1000.1k ± 0%       227.3k ± 0%  -77.27% (p=0.000 n=15)
```
  • Loading branch information
tdakkota committed Jun 21, 2024
1 parent 1550ef9 commit 7e7fe5a
Showing 1 changed file with 14 additions and 2 deletions.
16 changes: 14 additions & 2 deletions internal/chstorage/querier_logs_query.go
Original file line number Diff line number Diff line change
Expand Up @@ -343,13 +343,25 @@ func (q *Querier) lineFilter(m logql.LineFilter) (e chsql.Expr, rerr error) {
switch m.Op {
case logql.OpEq, logql.OpNotEq:
expr := chsql.Contains("body", m.By.Value)

// Clickhouse does not use tokenbf_v1 index to skip blocks
// with position* functions for some reason.
//
// Force to skip using hasToken function.
//
// Note that such optimization is applied only if operation is not negated to
// avoid false-negative skipping.
if val := m.By.Value; m.Op != logql.OpNotEq && chsql.IsSingleToken(val) {
expr = chsql.And(expr,
chsql.HasToken(chsql.Ident("body"), val),
)
}

{
// HACK: check for special case of hex-encoded trace_id and span_id.
// Like `{http_method=~".+"} |= "af36000000000000c517000000000003"`.
// TODO(ernado): also handle regex?

v, _ := hex.DecodeString(m.By.Value)

switch len(v) {
case len(otelstorage.TraceID{}):
expr = chsql.Or(expr, chsql.Eq(
Expand Down

0 comments on commit 7e7fe5a

Please sign in to comment.